From a92e6d350549cfa7a76a4e0a4891816c8a240e35 Mon Sep 17 00:00:00 2001 From: Mark Eschbach Date: Mon, 22 May 2017 08:20:39 -0700 Subject: [PATCH 1/6] Adds framing strategy for octect-counting frames --- syslogssl.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/syslogssl.py b/syslogssl.py index c1de46a..a65c1f3 100755 --- a/syslogssl.py +++ b/syslogssl.py @@ -8,6 +8,20 @@ import socket +class NewLineFraming: + + def frame( self, message ): + return message + '\n' + + +class OctectCountingFraming: + + def frame( self, message ): + length = len( message ) + frame = str( length ) + " " + message + return frame + + class SSLSysLogHandler(logging.handlers.SysLogHandler): # We need to paste all this in because __init__ bitches otherwise @@ -97,6 +111,8 @@ class SSLSysLogHandler(logging.handlers.SysLogHandler): "CRITICAL" : "critical" } + framing_strategy = NewLineFraming() + def __init__(self, address, certs=None, facility=LOG_USER): @@ -123,7 +139,7 @@ def close(self): def emit(self, record): - msg = self.format(record) + '\n' + msg = self.format(record) prio = '<%d>' % self.encodePriority(self.facility, self.mapPriority(record.levelname)) if type(msg) is unicode: @@ -131,8 +147,9 @@ def emit(self, record): if codecs: msg = codecs.BOM_UTF8 + msg msg = prio + msg + framed_message = self.framing_strategy.frame(msg) try: - self.socket.write(msg) + self.socket.write( framed_message ) except(KeyboardInterrupt, SystemExit): raise except: @@ -142,8 +159,9 @@ def emit(self, record): ### Example Usage ### if __name__ == '__main__': - host = 'logs.papertrailapp.com' - port = 514 # default, you'll want to change this + import os + host = os.getenv( 'SYSLOG_HOST', 'logs.papertrailapp.com' ) + port = int( os.getenv( 'SYSLOG_PORT', '514' ) ) # default, you'll want to change this address = (host, port) # We don't want this to hang From 583d964703ff7b76efc5e33c70070fd43991fb62 Mon Sep 17 00:00:00 2001 From: Mark Eschbach Date: Mon, 22 May 2017 08:25:37 -0700 Subject: [PATCH 2/6] Fixes mispelling of Octet --- syslogssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syslogssl.py b/syslogssl.py index a65c1f3..56e4f08 100755 --- a/syslogssl.py +++ b/syslogssl.py @@ -14,7 +14,7 @@ def frame( self, message ): return message + '\n' -class OctectCountingFraming: +class OctetCountingFraming: def frame( self, message ): length = len( message ) From ca2ffb43ba455f0c4f7012735c398010902da781 Mon Sep 17 00:00:00 2001 From: Mark Eschbach Date: Mon, 22 May 2017 09:57:40 -0700 Subject: [PATCH 3/6] Adds RFC 5424 metadata --- syslogssl.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/syslogssl.py b/syslogssl.py index 56e4f08..3808e57 100755 --- a/syslogssl.py +++ b/syslogssl.py @@ -2,10 +2,13 @@ import codecs +from datetime import datetime import logging import logging.handlers +import pytz import ssl import socket +import tzlocal class NewLineFraming: @@ -113,6 +116,12 @@ class SSLSysLogHandler(logging.handlers.SysLogHandler): framing_strategy = NewLineFraming() + # Host name to attach to the records + hostname = socket.gethostname() + + # Overrides the process name from the record + process_name = None + def __init__(self, address, certs=None, facility=LOG_USER): @@ -138,6 +147,21 @@ def close(self): logging.Handler.close(self) + def format_header(self, record, priority, message ): + created_at_local_notz = datetime.fromtimestamp( record.created ) + local_tz = tzlocal.get_localzone() + created_at_local = local_tz.localize( created_at_local_notz ) + created_at_utc = created_at_local.astimezone( pytz.utc ) + when = created_at_utc.isoformat()[ 0:-6 ] + "Z" + + if self.process_name is None: + name = record.processName + else: + name = self.process_name + + return priority + "1 " + when + " " + self.hostname + " " + name + " " + str( record.process ) + " - - " + message + + def emit(self, record): msg = self.format(record) prio = '<%d>' % self.encodePriority(self.facility, @@ -146,8 +170,9 @@ def emit(self, record): msg = msg.encode('utf-8') if codecs: msg = codecs.BOM_UTF8 + msg - msg = prio + msg - framed_message = self.framing_strategy.frame(msg) + + full_message = self.format_header( record, prio, msg ) + framed_message = self.framing_strategy.frame( full_message ) try: self.socket.write( framed_message ) except(KeyboardInterrupt, SystemExit): @@ -170,6 +195,8 @@ def emit(self, record): logger = logging.getLogger() logger.setLevel(logging.INFO) syslog = SSLSysLogHandler(address=address, certs='syslog.papertrail.crt') + syslog.framing_strategy = OctetCountingFraming() logger.addHandler(syslog) logger.info('testing SSLSysLogHandler') + From c634b9d73d108203e46d79aeb34889bb44953153 Mon Sep 17 00:00:00 2001 From: Mark Eschbach Date: Mon, 19 Jun 2017 16:52:01 -0700 Subject: [PATCH 4/6] Retry connecting when we encounter errors --- syslogssl.py | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/syslogssl.py b/syslogssl.py index 3808e57..6e3d81e 100755 --- a/syslogssl.py +++ b/syslogssl.py @@ -122,25 +122,48 @@ class SSLSysLogHandler(logging.handlers.SysLogHandler): # Overrides the process name from the record process_name = None + # Allow retrying + allows_retries = False + def __init__(self, address, certs=None, facility=LOG_USER): logging.Handler.__init__(self) self.address = address + self.certs = certs self.facility = facility self.unixsocket = 0 - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - if certs: - self.socket = ssl.wrap_socket(s, - ca_certs=certs, - cert_reqs=ssl.CERT_REQUIRED) - else: - self.socket = ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) - self.socket.connect(address) + def _connect(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + if self.certs: + self.socket = ssl.wrap_socket(s, + ca_certs=self.certs, + cert_reqs=ssl.CERT_REQUIRED) + else: + self.socket = ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.socket.connect(address) + + def _retry(self, record): + if self.is_retrying and self.allows_retries: + return True + + self.is_retrying = True + if self.socket is not None: + try: + self.socket.close() + except: + pass # Sometimes sockets are already closed + finally: + self.socket = None + + self._connect() + self.emit(self, record) + self.is_retrying = False + return False def close(self): self.socket.close() @@ -163,6 +186,9 @@ def format_header(self, record, priority, message ): def emit(self, record): + if self.socket is not None: + self._connect() + msg = self.format(record) prio = '<%d>' % self.encodePriority(self.facility, self.mapPriority(record.levelname)) @@ -177,6 +203,9 @@ def emit(self, record): self.socket.write( framed_message ) except(KeyboardInterrupt, SystemExit): raise + except ssl.SSLError as problem: + if self._rety(): + raise except: self.handleError(record) From 0584be7c3d10fda5f755179cdd6651703ce8ee46 Mon Sep 17 00:00:00 2001 From: Mark Eschbach Date: Mon, 19 Jun 2017 17:36:20 -0700 Subject: [PATCH 5/6] Defaults restored to original; reconnection logic added --- syslogssl.py | 65 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/syslogssl.py b/syslogssl.py index 6e3d81e..200f2a7 100755 --- a/syslogssl.py +++ b/syslogssl.py @@ -25,6 +25,30 @@ def frame( self, message ): return frame +class RFC5424Header: + + def format_header(self, syslog, record, priority, message): + created_at_local_notz = datetime.fromtimestamp(record.created) + local_tz = tzlocal.get_localzone() + created_at_local = local_tz.localize(created_at_local_notz) + created_at_utc = created_at_local.astimezone(pytz.utc) + when = created_at_utc.isoformat()[0:-6] + "Z" + + if syslog.process_name is None: + name = record.processName + else: + name = syslog.process_name + + return priority + "1 " + when + " " + syslog.hostname + " " + name + " " + str( + record.process) + " - - " + message + + +class TraditionalHeader: + + def format_header(self, syslog, record, priority, message): + return priority + message + + class SSLSysLogHandler(logging.handlers.SysLogHandler): # We need to paste all this in because __init__ bitches otherwise @@ -115,6 +139,7 @@ class SSLSysLogHandler(logging.handlers.SysLogHandler): } framing_strategy = NewLineFraming() + header_format = TraditionalHeader() # Host name to attach to the records hostname = socket.gethostname() @@ -135,6 +160,7 @@ def __init__(self, address, certs=None, self.facility = facility self.unixsocket = 0 + self.socket = None def _connect(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -170,23 +196,8 @@ def close(self): logging.Handler.close(self) - def format_header(self, record, priority, message ): - created_at_local_notz = datetime.fromtimestamp( record.created ) - local_tz = tzlocal.get_localzone() - created_at_local = local_tz.localize( created_at_local_notz ) - created_at_utc = created_at_local.astimezone( pytz.utc ) - when = created_at_utc.isoformat()[ 0:-6 ] + "Z" - - if self.process_name is None: - name = record.processName - else: - name = self.process_name - - return priority + "1 " + when + " " + self.hostname + " " + name + " " + str( record.process ) + " - - " + message - - def emit(self, record): - if self.socket is not None: + if self.socket is None: self._connect() msg = self.format(record) @@ -197,7 +208,7 @@ def emit(self, record): if codecs: msg = codecs.BOM_UTF8 + msg - full_message = self.format_header( record, prio, msg ) + full_message = self.header_format.format_header( self, record, prio, msg ) framed_message = self.framing_strategy.frame( full_message ) try: self.socket.write( framed_message ) @@ -213,19 +224,29 @@ def emit(self, record): ### Example Usage ### if __name__ == '__main__': + def test_handler( handler, message ): + logger.addHandler( handler ) + logger.info( message ) + logger.removeHandler( handler ) + import os host = os.getenv( 'SYSLOG_HOST', 'logs.papertrailapp.com' ) port = int( os.getenv( 'SYSLOG_PORT', '514' ) ) # default, you'll want to change this address = (host, port) # We don't want this to hang - socket.setdefaulttimeout(5.0) + socket.setdefaulttimeout(.5) logger = logging.getLogger() logger.setLevel(logging.INFO) - syslog = SSLSysLogHandler(address=address, certs='syslog.papertrail.crt') - syslog.framing_strategy = OctetCountingFraming() - logger.addHandler(syslog) - logger.info('testing SSLSysLogHandler') + # Test original format + original_wire = SSLSysLogHandler(address=address, certs='syslog.papertrail.crt') + test_handler( original_wire, "Default usage" ) + + # Test RFC5424 wire format + rfc5424 = SSLSysLogHandler( address=address, certs='syslog.papertrail.crt' ) + rfc5424.framing_strategy = OctetCountingFraming() + rfc5424.header_format = RFC5424Header() + test_handler( rfc5424, "RFC5424 frame" ) From 30afaac9aa3a1897034024aabae8148199132c55 Mon Sep 17 00:00:00 2001 From: Mark Eschbach Date: Mon, 19 Jun 2017 21:45:41 -0700 Subject: [PATCH 6/6] Fixes error with address references --- syslogssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syslogssl.py b/syslogssl.py index 200f2a7..bd873aa 100755 --- a/syslogssl.py +++ b/syslogssl.py @@ -171,7 +171,7 @@ def _connect(self): cert_reqs=ssl.CERT_REQUIRED) else: self.socket = ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) - self.socket.connect(address) + self.socket.connect( self.address ) def _retry(self, record): if self.is_retrying and self.allows_retries: