fix stream consumption and add 3164 test for consuming

Author: ottobackwards

simple_syslog/parser.py

@@ -194,8 +194,7 @@ def consume_stream(self, stream: TextIOBase, consumer: SyslogConsumer[T]) -> Non
             ValueError: if stream or consumer are None
 
         """
-        line = stream.readline()
-        while line:
+        for line in stream:
             self.consume(line, consumer)
 
     def consume_stream_with_errors(
@@ -211,8 +210,7 @@ def consume_stream_with_errors(
             consumer: Called with T
             error_consumer: Called with any ParseError or DeviationError
         """
-        line = stream.readline()
-        while line:
+        for line in stream:
             try:
                 self.consume(line, consumer)
             except (DeviationError, ParseError) as e:

tests/test_3164_parser.py

@@ -22,7 +22,11 @@
     SyslogFieldKey,
     SyslogFieldKeyDefaults,
 )
-from simple_syslog.parser import AbstractSyslogParser, Rfc3164SyslogParser
+from simple_syslog.parser import (
+    AbstractSyslogParser,
+    Rfc3164SyslogParser,
+    SyslogConsumer,
+)
 from simple_syslog.specification import SyslogSpecification
 
 expectedMessageOne = (
@@ -118,6 +122,56 @@ def test_parse_line(file_of_3164_single_ise_txt) -> None:
     )
 
 
+def test_parse_line_consumer(file_of_3164_single_ise_txt) -> None:
+    """Test parsing with consumer callback."""
+    builder = DefaultBuilder(
+        specification=SyslogSpecification.RFC_3164,
+        key_provider=DefaultKeyProvider(),
+        nil_policy=None,
+        allowed_deviations=None,
+    )
+    parser = Rfc3164SyslogParser(builder)
+
+    def fun(data_set: SyslogDataSet):
+        assert data_set
+        assert (
+            expectedMessageOne
+            == data_set.data[SyslogFieldKeyDefaults[SyslogFieldKey.MESSAGE]]
+        )
+        assert (
+            expectedHostNameOne
+            == data_set.data[SyslogFieldKeyDefaults[SyslogFieldKey.HEADER_HOSTNAME]]
+        )
+        assert (
+            expectedPriOne
+            == data_set.data[SyslogFieldKeyDefaults[SyslogFieldKey.HEADER_PRI]]
+        )
+        assert (
+            expectedSeverityOne
+            == data_set.data[SyslogFieldKeyDefaults[SyslogFieldKey.HEADER_PRI_SEVERITY]]
+        )
+        assert (
+            expectedFacilityOne
+            == data_set.data[SyslogFieldKeyDefaults[SyslogFieldKey.HEADER_PRI_FACILITY]]
+        )
+        assert (
+            expectedTimestampOne
+            == data_set.data[SyslogFieldKeyDefaults[SyslogFieldKey.HEADER_TIMESTAMP]]
+        )
+
+    with file_of_3164_single_ise_txt.open("r") as f:
+        consume_from_file(f, parser, fun)
+
+
+def consume_from_file(
+    f: TextIOBase,
+    parser: AbstractSyslogParser[SyslogDataSet],
+    consumer: SyslogConsumer[SyslogDataSet],
+) -> None:
+    """Open parse with callback."""
+    return parser.consume_stream(f, consumer)
+
+
 def generate_from_file(
     f: TextIOBase, parser: AbstractSyslogParser[SyslogDataSet]
 ) -> Generator[SyslogDataSet, None, None]:

tests/tmp.txt

@@ -0,0 +1,95 @@
+
+#  @Test
+#  public void testParseLineConsumer() throws Exception {
+#    Default3164MessageHandler builder = new Default3164MessageHandler();
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164)
+#        .withSyslogBuilder(builder).build();
+#    handleLine(readFileToString("src/test/resources/logs/3164/single_ise.txt"), parser, (map) -> {
+#      Assert.assertEquals(expectedMessageOne, map.get(SyslogFieldKeys.MESSAGE.getField()));
+#      Assert.assertEquals(expectedHostNameOne, map.get(SyslogFieldKeys.HEADER_HOSTNAME.getField()));
+#      Assert.assertEquals(expectedPriOne, map.get(SyslogFieldKeys.HEADER_PRI.getField()));
+#      Assert.assertEquals(expectedSeverityOne, map.get(SyslogFieldKeys.HEADER_PRI_SEVERITY.getField()));
+#      Assert.assertEquals(expectedFacilityOne, map.get(SyslogFieldKeys.HEADER_PRI_FACILITY.getField()));
+#      Assert.assertEquals(expectedTimestampOne, map.get(SyslogFieldKeys.HEADER_TIMESTAMP.getField()));
+#    });
+#  }
+#
+#  @Test
+#  public void testParseLinesConsumerAndErrorConsumer() throws Exception {
+#    final AtomicInteger mapCount = new AtomicInteger();
+#    final AtomicInteger errorCount = new AtomicInteger();
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String,String>>().forSpecification(SyslogSpecification.RFC_3164).withSyslogBuilder(new Default3164MessageHandler()).build();
+#    handleFile("src/test/resources/logs/3164/many_with_errors.txt", parser, (map) -> mapCount.incrementAndGet(),
+#        (line, throwable) -> errorCount.incrementAndGet());
+#    Assert.assertEquals(3, mapCount.get());
+#    Assert.assertEquals(1, errorCount.get());
+#  }
+#
+#  @Test
+#  public void testParseLines() throws Exception {
+#    Default3164MessageHandler builder = new Default3164MessageHandler();
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164).withSyslogBuilder(builder).build();
+#    List<Map<String, String>> mapList = handleFile("src/test/resources/logs/3164/many_ise.txt", parser);
+#    Assert.assertEquals(308, mapList.size());
+#  }
+#
+#  @Test
+#  public void testParseLinesMixDates() throws Exception {
+#    Default3164MessageHandler builder = new Default3164MessageHandler();
+#    SyslogParser<Map<String,String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164).withSyslogBuilder(builder).build();
+#    List<Map<String, String>> mapList = handleFile("src/test/resources/logs/3164/two_ise_mix_date.txt", parser);
+#    Assert.assertEquals(2, mapList.size());
+#  }
+#
+#  @Test
+#  public void testParseLinesDeviations() throws Exception {
+#    Default3164MessageHandler
+#        builder = new Default3164MessageHandler(new DefaultKeyProvider(), EnumSet.of(AllowableDeviations.PRIORITY));
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164)
+#        .withSyslogBuilder(builder).build();
+#    List<Map<String, String>> mapList = handleFile("src/test/resources/logs/3164/many_ise_deviations.txt", parser);
+#    Assert.assertEquals(308, mapList.size());
+#  }
+#
+#  @Test
+#  public void testParseLinesConsumer() throws Exception {
+#    Default3164MessageHandler builder = new Default3164MessageHandler();
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164).withSyslogBuilder(builder).build();
+#    final AtomicInteger count = new AtomicInteger();
+#    handleFile("src/test/resources/logs/3164/many_ise.txt", parser, (map) -> {
+#      count.incrementAndGet();
+#    });
+#
+#    Assert.assertEquals(count.get(), 308);
+#  }
+#
+#  @Test
+#  public void testParseLinesConsumerMixDates() throws Exception {
+#    Default3164MessageHandler builder = new Default3164MessageHandler();
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164).withSyslogBuilder(builder).build();
+#    final AtomicInteger count = new AtomicInteger();
+#    handleFile("src/test/resources/logs/3164/two_ise_mix_date.txt", parser, (map) -> {
+#      count.incrementAndGet();
+#    });
+#
+#    Assert.assertEquals(count.get(), 2);
+#  }
+#
+#  @Test(expected = ParseException.class)
+#  @SuppressWarnings("unchecked")
+#  public void testInvalidLine() throws Exception {
+#    Default3164MessageHandler builder = new Default3164MessageHandler();
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164).withSyslogBuilder(builder).build();
+#    Map<String, String> map = handleLine("localhost some body of the message", parser);
+#  }
+#
+#  @Test(expected = ParseException.class)
+#  @SuppressWarnings("unchecked")
+#  public void testInvalidLineConsumer() throws Exception {
+#    Default3164MessageHandler builder = new Default3164MessageHandler();
+#    SyslogParser<Map<String, String>> parser = new SyslogParserBuilder<Map<String, String>>().forSpecification(SyslogSpecification.RFC_3164).withSyslogBuilder(builder).build();
+#    handleLine("localhost some body of the message", parser, (map) -> {
+#      Assert.fail();
+#    });
+#  }
+# }