diff --git a/lib/syslog_protocol.rb b/lib/syslog_protocol.rb index 965c40e..d59d8bd 100644 --- a/lib/syslog_protocol.rb +++ b/lib/syslog_protocol.rb @@ -2,6 +2,8 @@ require 'syslog_protocol/packet' require 'syslog_protocol/logger' require 'syslog_protocol/parser' +require 'syslog_protocol/syslogrfc5424packet' +require 'syslog_protocol/syslogrfc5424parser' module SyslogProtocol VERSION = '0.9.2' diff --git a/lib/syslog_protocol/common.rb b/lib/syslog_protocol/common.rb index 045d365..2d68d6d 100644 --- a/lib/syslog_protocol/common.rb +++ b/lib/syslog_protocol/common.rb @@ -54,18 +54,22 @@ module SyslogProtocol 22 => 'local6', 23 => 'local7' } - + SEVERITIES = { 'emerg' => 0, + 'emergency' => 0, 'alert' => 1, 'crit' => 2, + 'critical' => 2, 'err' => 3, + 'error' => 3, 'warn' => 4, + 'warning' => 4, 'notice' => 5, 'info' => 6, - 'debug' => 7 + 'debug' => 7 } - + SEVERITY_INDEX = { 0 => 'emerg', 1 => 'alert', @@ -76,4 +80,6 @@ module SyslogProtocol 6 => 'info', 7 => 'debug' } -end \ No newline at end of file + +end + diff --git a/lib/syslog_protocol/packet.rb b/lib/syslog_protocol/packet.rb index 20dae92..17de609 100644 --- a/lib/syslog_protocol/packet.rb +++ b/lib/syslog_protocol/packet.rb @@ -3,6 +3,12 @@ class Packet attr_reader :facility, :severity, :hostname, :tag attr_accessor :time, :content + def initialize() + super() + @hostname = nil + @time = nil + end + def to_s assemble end @@ -11,7 +17,9 @@ def assemble(max_size = 1024) unless @hostname and @facility and @severity and @tag raise "Could not assemble packet without hostname, tag, facility, and severity" end - data = "<#{pri}>#{generate_timestamp} #{@hostname} #{@tag}: #{@content}" + + fmt = "<%s>%s %s %s: %s" + data = fmt % [pri, generate_timestamp, @hostname, @tag, @content] if string_bytesize(data) > max_size data = data.slice(0, max_size) @@ -98,7 +106,7 @@ def severity_name end def pri - (@facility * 8) + @severity + (@facility << 3) | @severity end def pri=(p) @@ -118,6 +126,15 @@ def generate_timestamp time.strftime("%b #{day} %H:%M:%S") end + private + def format_field(text, max_length) + if text + text[0, max_length].gsub(/\s+/, '') + else + '-' + end + end + if "".respond_to?(:bytesize) def string_bytesize(string) string.bytesize @@ -132,4 +149,4 @@ def string_bytesize(string) define_method("#{k}?") { SEVERITIES[k] == @severity } end end -end \ No newline at end of file +end diff --git a/lib/syslog_protocol/syslogrfc5424packet.rb b/lib/syslog_protocol/syslogrfc5424packet.rb new file mode 100644 index 0000000..011c628 --- /dev/null +++ b/lib/syslog_protocol/syslogrfc5424packet.rb @@ -0,0 +1,109 @@ +module SyslogProtocol + class SyslogRfc5424Packet < Packet + attr_reader :appname, :procid, :msgid, :structured_data + + def initialize(appname = nil, procid = nil, msgid = nil, facility = nil) + super() + @msgid = format_field(msgid, 32) + @procid = format_field(procid, 128) + @appname = format_field(appname, 48) + @structured_data = {} + end + + def assemble(max_size = 1024) + unless @hostname and @facility and @severity and @appname + raise "Could not assemble packet without hostname, tag, facility, and severity" + end + sd = '-' + unless @structured_data.empty? + sd = format_sdata(@structured_data) + end + fmt = "<%s>1 %s %s %s %s %s %s %s" + data = fmt % [pri, @time, @hostname, + @appname, format_field(@procid, 128),@msgid, sd, @content] + + if string_bytesize(data) > max_size + data = data.slice(0, max_size) + while string_bytesize(data) > max_size + data = data.slice(0, data.length - 1) + end + end + + data + end + + def generate_timestamp + @time || Time.now.to_datetime.rfc3339(6) + end + + def appname=(a) + unless a && a.is_a?(String) && a.length > 0 + raise ArgumentError, "Appname must not be omitted" + end + if a.length > 48 + raise ArgumentError, "Appname must not be longer than 48 characters" + end + if a =~ /\s/ + raise ArgumentError, "Appname may not contain spaces" + end + if a =~ /[^\x21-\x7E]/ + raise ArgumentError, "Appname may only contain ASCII characters 33-126" + end + + @appname = a + end + + def procid=(p) + if p.length > 128 + raise ArgumentError.new("Procid can't be bigger than 128") + end + @procid = format_field(p, 128) + end + + def msgid=(m) + if m.is_a? Integer + @msgid = format_field(m.to_s, 32) + elsif m.is_a? String + if m.length > 32 + raise ArgumentError, "msgid must not be longer than 32 characters" + else + @msgid = format_field(m, 32) + end + else + raise ArgumentError.new "msgid must be a number or string" + end + end + + def structured_data=(s) + if s.is_a? Hash + @structured_data = s + else + raise ArgumentError.new "structured_data must be a dict" + end + end + + def format_sdata(sdata) + if sdata.empty? + '-' + end + r = [] + sdata.each { |sid, hash| + s = [] + s.push(sid.to_s.gsub(/[^-@\w]/, "")) + hash.each { |n, v| + # RFC-5424 requires SD-NAME to be 32 length + paramname = format_field(n.to_s.gsub(/[^-@\w]/, ""), 32) + paramvalue = v.to_s.gsub(/[\]"=]/, "") + s.push("#{paramname}=\"#{paramvalue}\"") + } + r.push("["+s.join(" ")+"]") + } + rx = [] + r.each { |x| + rx.push("[#{x}]") + } + r.join("") + end + + end +end diff --git a/lib/syslog_protocol/syslogrfc5424parser.rb b/lib/syslog_protocol/syslogrfc5424parser.rb new file mode 100644 index 0000000..deab101 --- /dev/null +++ b/lib/syslog_protocol/syslogrfc5424parser.rb @@ -0,0 +1,128 @@ +require 'time' + +module SyslogProtocol + + def self.syslog5424_parse(msg, origin=nil) + packet = SyslogRfc5424Packet.new + original_msg = msg.dup + pri = syslog5424_parse_pri(msg) + if pri and (pri = pri.to_i).is_a? Integer and (0..191).include?(pri) + packet.pri = pri + else + # If there isn't a valid PRI, treat the entire message as content + packet.pri = 13 + packet.time = Time.now + packet.hostname = origin || 'unknown' + packet.content = original_msg + + return packet + end + time = syslog5424_parse_time(msg) + if time + packet.time = Time.parse(time) + else + packet.time = Time.now + end + hostname = syslog5424_parse_hostname(msg) + packet.hostname = hostname || origin + appname = syslog5424_parse_appname(msg) + packet.appname = appname + procid = syslog5424_parse_procid(msg) + packet.procid = procid + msgid = syslog5424_parse_msgid(msg) + packet.msgid = msgid + structured_data = syslog5424_parse_structured_data(msg) + packet.structured_data = structured_data + content = syslog5424_parse_content(msg) + packet.content = content + + packet + end + + private + + def self.syslog5424_parse_pri(msg) + pri = msg.slice!(/<(\d\d?\d?)>1/) + pri = pri.slice(/\d\d?\d?/) if pri + if !pri or (pri =~ /^0/ and pri !~ /^0$/) + return nil + else + return pri + end + end + + def self.syslog5424_parse_time(msg) + msg.slice!(/(\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d)/) + end + + def self.syslog5424_parse_hostname(msg) + m = msg.split(" ") + if m.nil? or m.empty? + raise ArgumentError, "Message format is not correct" + end + return m[0] + + end + + def self.syslog5424_parse_appname(msg) + m = msg.split(" ") + if m.nil? or m.empty? + raise ArgumentError, "Message format is not correct" + end + return m[1] + end + def self.syslog5424_parse_procid(msg) + m = msg.split(" ") + + if m.nil? or m.empty? + raise ArgumentError, "Message format is not correct" + end + return m[2] + end + def self.syslog5424_parse_msgid(msg) + m = msg.split(" ") + if m.nil? or m.empty? + raie ArgumentError, "Message format is not correct" + end + return m[3] + end + + def self.syslog5424_parse_content(msg) + m = msg.match(/(.*)\s(.*)\s(.*)\s(.*)\s(-)\s(.*)/) + if m.nil? + s = msg.match(/(.*)\s(\[.*\])\s(.*)/) + if s.nil? + raise ArgumentError, "Message format is not correct" + else + return s[3] + end + else + return m[6] + end + end + + def self.syslog5424_parse_structured_data(msg) + s_data = {} + m = msg.match(/(.*)\s(\[.*\])\s/) + s_data = parse_structured_data(m[2]) unless m.nil? + return s_data + end + + private + def self.parse_structured_data(sdata) + structured_data = {} + sdata_key = '' + sdata_value = {} + arr = sdata.sub(/^\[/, "").sub(/\]/,"").split(" ") + arr.each { |item| + if item.include?("@") + sdata_key = item + else + key, value = item.split("=") + sdata_value[key] = value + end + } + structured_data[sdata_key] = sdata_value + return structured_data + end +end \ No newline at end of file diff --git a/syslog_protocol.gemspec b/syslog_protocol.gemspec index 90a3e8b..877e81a 100644 --- a/syslog_protocol.gemspec +++ b/syslog_protocol.gemspec @@ -67,11 +67,15 @@ Gem::Specification.new do |s| lib/syslog_protocol/logger.rb lib/syslog_protocol/packet.rb lib/syslog_protocol/parser.rb + lib/syslogrfc5424packet.rb + lib/syslogrfc5424parser.rb syslog_protocol.gemspec test/helper.rb test/test_logger.rb test/test_packet.rb test/test_parser.rb + test/test_syslogrfc5424packet.rb + test/test_syslogrfc5424parser.rb ] # = MANIFEST = diff --git a/test/test_packet.rb b/test/test_packet.rb index d511755..f6566a2 100644 --- a/test/test_packet.rb +++ b/test/test_packet.rb @@ -44,12 +44,6 @@ @p.severity.should.equal 6 end - it "severity can be checked using 'some_severity?' methods" do - @p.info?.should.equal true - @p.alert?.should.equal false - @p.emerg?.should.equal false - end - it "PRI is calculated from the facility and severity" do @p.pri.should.equal 134 end @@ -93,4 +87,4 @@ end end -end \ No newline at end of file +end diff --git a/test/test_syslogrfc5424packet.rb b/test/test_syslogrfc5424packet.rb new file mode 100644 index 0000000..f94a3ff --- /dev/null +++ b/test/test_syslogrfc5424packet.rb @@ -0,0 +1,99 @@ +require File.expand_path('../helper', __FILE__) + +describe "a syslog 5424 format message packet" do + + @p = SyslogProtocol::SyslogRfc5424Packet.new + + it "should embarrass a person who does not set the fields" do + lambda { @p.to_s }.should.raise RuntimeError + end + + it "hostname may not be omitted" do + lambda {@p.hostname = ""}.should.raise ArgumentError + end + + it "hostname may only contain ASCII characters 33-126 (no spaces!)" do + lambda {@p.hostname = "linux box"}.should.raise ArgumentError + lambda {@p.hostname = "\000" + "linuxbox"}.should.raise ArgumentError + lambda {@p.hostname = "space_station"}.should.not.raise + end + + it 'tag may only contain ASCII characters 33-126 (no spaces!)' do + lambda {@p.tag = "linux box"}.should.raise ArgumentError + lambda {@p.tag = "\000" + "linuxbox"}.should.raise ArgumentError + lambda {@p.tag = "test"}.should.not.raise + end + + it "facility may only be set within 0-23 or with a proper string name" do + lambda {@p.facility = 666}.should.raise ArgumentError + lambda {@p.facility = "mir space station"}.should.raise ArgumentError + + lambda {@p.facility = 16}.should.not.raise + @p.facility.should.equal 16 + lambda {@p.facility = 'local0'}.should.not.raise + @p.facility.should.equal 16 + end + + it "severity may only be set within 0-7 or with a proper string name" do + lambda {@p.severity = 9876}.should.raise ArgumentError + lambda {@p.severity = "omgbroken"}.should.raise ArgumentError + + lambda {@p.severity = 6}.should.not.raise + @p.severity.should.equal 6 + lambda {@p.severity = 'info'}.should.not.raise + @p.severity.should.equal 6 + end + + it "PRI is calculated from the facility and severity" do + @p.pri.should.equal 134 + end + + it "PRI may only be within 0-191" do + lambda {@p.pri = 22331}.should.raise ArgumentError + lambda {@p.pri = "foo"}.should.raise ArgumentError + end + + it "facility and severity are deduced and set from setting a valid PRI" do + @p.pri = 165 + @p.severity.should.equal 5 + @p.facility.should.equal 20 + end + + it "return the proper names for facility and severity" do + @p.severity_name.should.equal 'notice' + @p.facility_name.should.equal 'local4' + end + + it "set a message, which apparently can be anything" do + @p.content = "exploring ze black hole" + @p.content.should.equal "exploring ze black hole" + end + + it "packets larger than 1024 will be truncated" do + @p.content = "space warp" * 1000 + if "".respond_to?(:bytesize) + @p.to_s.bytesize.should.equal 1024 + else + @p.to_s.size.should.equal 1024 + end + end + + it "use the current time and assemble the packet" do + @p.hostname = "127.0.0.1" + @p.msgid = "1234567" + @p.procid = "erlang" + @p.appname = "fluentd" + @p.content = "message is sent" + @p.time = @p.generate_timestamp + @p.structured_data = {"test@xxxxx" => { "kube-namespace" => "test", "pod_name" => "test-0", "container_name" => "test"}} + expected_string = "<#{@p.pri}>1 #{@p.time} #{@p.hostname} #{@p.appname} #{@p.procid} #{@p.msgid} [test@xxxxx kube-namespace=\"test\" pod_name=\"test-0\" container_name=\"test\"] #{@p.content}" + @p.to_s.should.equal expected_string + end + + it "truncate sd-param to 32 bytes per RFC-5424 says" do + @p.structured_data = {"test@xxxxx" => { "statefulset-kubernetes-iopod-name" => "test", "pod_name" => "test-0"}} + expected_string = "<#{@p.pri}>1 #{@p.time} #{@p.hostname} #{@p.appname} #{@p.procid} #{@p.msgid} [test@xxxxx statefulset-kubernetes-iopod-nam=\"test\" pod_name=\"test-0\"] #{@p.content}" + @p.to_s.should.equal expected_string + end + +end diff --git a/test/test_syslogrfc5424parser.rb b/test/test_syslogrfc5424parser.rb new file mode 100644 index 0000000..6cc41c7 --- /dev/null +++ b/test/test_syslogrfc5424parser.rb @@ -0,0 +1,71 @@ +require File.expand_path('../helper', __FILE__) + +describe "a syslog 5424 format message packet parser" do + + it "parse some valid packets" do + p = SyslogProtocol.syslog5424_parse("<34>1 2018-11-14T12:41:48.686781+08:00 mymachine fluentd erlang 1234567 [test@xxxxx kube-namespace=\"test\" pod_name=\"test-0\" container_name=\"test\"] message is sent") + p.facility.should.equal 4 + p.severity.should.equal 2 + p.pri.should.equal 34 + p.hostname.should.equal "mymachine" + p.appname.should.equal 'fluentd' + p.msgid = "1234567" + p.procid = "erlang" + p.structured_data = {"test@xxxxx" => { "kube-namespace" => "test", "pod_name" => "test-0", "container_name" => "test"}} + p.content.should.equal "message is sent" + p.time.should.equal Time.parse("2018-11-14T12:41:48.686781+08:00") + + p = SyslogProtocol.syslog5424_parse("<13>1 2018-10-01T06:11:48.686781+08:00 10.0.0.99 fluentd erlang 1234567 [test@xxxxx kube-namespace=\"test\" pod_name=\"test-0\" container_name=\"test\"] Use the BFG!") + p.facility.should.equal 1 + p.severity.should.equal 5 + p.pri.should.equal 13 + p.hostname.should.equal "10.0.0.99" + p.appname.should.equal 'fluentd' + p.msgid = "1234567" + p.procid = "erlang" + p.structured_data = {"test@xxxxx" => { "kube-namespace" => "test", "pod_name" => "test-0", "container_name" => "test"}} + p.content.should.equal "Use the BFG!" + p.time.should.equal Time.parse("2018-10-01T06:11:48.686781+08:00") + end + + it "treat a packet with no valid PRI as all content, setting defaults" do + p = SyslogProtocol.syslog5424_parse("nomnom") + p.facility.should.equal 1 + p.severity.should.equal 5 + p.pri.should.equal 13 + p.hostname.should.equal 'unknown' + p.content.should.equal "nomnom" + end + + it "PRI with preceding 0's shall be considered invalid" do + p = SyslogProtocol.syslog5424_parse("<045>1 Oct 11 22:14:15 space_station my PRI is not valid") + p.facility.should.equal 1 + p.severity.should.equal 5 + p.pri.should.equal 13 + p.hostname.should.equal 'unknown' + p.content.should.equal "<045>1 Oct 11 22:14:15 space_station my PRI is not valid" + end + + it "allow the user to pass an origin to be used as the hostname if packet is invalid" do + p = SyslogProtocol.syslog5424_parse("<045>1 Oct 11 22:14:15 space_station my PRI is not valid", '127.0.0.1') + p.facility.should.equal 1 + p.severity.should.equal 5 + p.pri.should.equal 13 + p.hostname.should.equal '127.0.0.1' + p.content.should.equal "<045>1 Oct 11 22:14:15 space_station my PRI is not valid" + end + + it "parse a packet with structured_data default value of '-'" do + p = SyslogProtocol.syslog5424_parse("<13>1 2018-10-01T06:11:48.686781+08:00 10.0.0.99 fluentd erlang 1234567 - Use the BFG!") + p.facility.should.equal 1 + p.severity.should.equal 5 + p.pri.should.equal 13 + p.hostname.should.equal "10.0.0.99" + p.appname.should.equal 'fluentd' + p.msgid = "1234567" + p.procid = "erlang" + p.structured_data = {} + p.content.should.equal "Use the BFG!" + p.time.should.equal Time.parse("2018-10-01T06:11:48.686781+08:00") + end +end