From cb1b47c84409936db101e98931bd96de17058470 Mon Sep 17 00:00:00 2001 From: Ben Bernard Date: Tue, 24 Feb 2026 15:51:01 -0800 Subject: [PATCH 1/4] feat: add snippet method/SDK tests and make Perl $r a RecsSDK object Add comprehensive tests for method calls on {{}} expanded values and record SDK methods across all three languages (JS, Python, Perl). Change Perl RecsSDK to bless the data hash directly (like the original App::RecordStream::Record) so $r is now a RecsSDK object with both direct hash access ($r->{field}) and SDK methods ($r->get("a/b"), $r->set(), $r->has()). Co-Authored-By: Claude Opus 4.6 --- src/snippets/perl/RecsSDK.pm | 46 +- src/snippets/perl/runner.pl | 23 +- tests/snippets/TemplateExpansion.test.ts | 635 +++++++++++++++++++++++ 3 files changed, 677 insertions(+), 27 deletions(-) diff --git a/src/snippets/perl/RecsSDK.pm b/src/snippets/perl/RecsSDK.pm index 45e3227..f1839a6 100644 --- a/src/snippets/perl/RecsSDK.pm +++ b/src/snippets/perl/RecsSDK.pm @@ -1,41 +1,53 @@ package RecsSDK; -# RecordStream Perl SDK — optional utility module for snippet authors. +# RecordStream Perl SDK for snippet execution. # -# In the standard snippet environment $r is a plain hashref, exactly -# like the original App::RecordStream. This module is available for -# users who want KeySpec-style access (nested paths, array indices, -# fuzzy matching) from Perl code. +# The object IS the data hashref (blessed directly), so $r->{field} +# works for direct access and $r->get("a/b") works for KeySpec access. +# This matches the original App::RecordStream::Record design. use strict; use warnings; +use Scalar::Util qw(reftype); sub new { my ($class, $data) = @_; $data = {} unless defined $data; - return bless { data => $data }, $class; + return bless $data, $class; } # get($keyspec) — resolve a /-separated key path. -# foo/bar => $data->{foo}{bar} -# foo/#0 => $data->{foo}[0] -# foo/#0/bar => $data->{foo}[0]{bar} +# foo/bar => $self->{foo}{bar} +# foo/#0 => $self->{foo}[0] +# foo/#0/bar => $self->{foo}[0]{bar} sub get { my ($self, $keyspec) = @_; - return _resolve($self->{data}, $keyspec); + return _resolve($self, $keyspec); } # set($keyspec, $value) — set a value at a /-separated key path, # auto-vivifying intermediate hashes or arrays as needed. sub set { my ($self, $keyspec, $value) = @_; - _set_path($self->{data}, $keyspec, $value); + _set_path($self, $keyspec, $value); return $value; } +# has($keyspec) — check whether a key path exists. +sub has { + my ($self, $keyspec) = @_; + return defined _resolve($self, $keyspec); +} + sub to_hash { my ($self) = @_; - return $self->{data}; + return { %$self }; +} + +# TO_JSON — called by JSON::PP with convert_blessed to serialize. +sub TO_JSON { + my ($self) = @_; + return { %$self }; } # --- internal helpers --- @@ -47,12 +59,13 @@ sub _resolve { my @parts = _split_keyspec($keyspec); for my $part (@parts) { return undef unless defined $node; + my $rt = reftype($node) // ''; if ($part =~ /^#(\d+)$/) { my $idx = $1; - return undef unless ref($node) eq 'ARRAY'; + return undef unless $rt eq 'ARRAY'; $node = $node->[$idx]; } else { - return undef unless ref($node) eq 'HASH'; + return undef unless $rt eq 'HASH'; $node = $node->{$part}; } } @@ -68,15 +81,16 @@ sub _set_path { my $part = $parts[$i]; my $next_part = $parts[$i + 1]; my $next_is_array = ($next_part =~ /^#\d+$/); + my $rt = reftype($node) // ''; if ($part =~ /^#(\d+)$/) { my $idx = $1; - if (!defined $node->[$idx] || ref($node->[$idx]) eq '') { + if (!defined $node->[$idx] || !ref($node->[$idx])) { $node->[$idx] = $next_is_array ? [] : {}; } $node = $node->[$idx]; } else { - if (!defined $node->{$part} || ref($node->{$part}) eq '') { + if (!defined $node->{$part} || !ref($node->{$part})) { $node->{$part} = $next_is_array ? [] : {}; } $node = $node->{$part}; diff --git a/src/snippets/perl/runner.pl b/src/snippets/perl/runner.pl index 85cfd59..8b2dee3 100644 --- a/src/snippets/perl/runner.pl +++ b/src/snippets/perl/runner.pl @@ -14,11 +14,12 @@ use warnings; use JSON::PP; +use Scalar::Util qw(blessed reftype); use FindBin; use lib $FindBin::Bin; use RecsSDK; -my $json = JSON::PP->new->utf8->canonical; +my $json = JSON::PP->new->utf8->canonical->allow_blessed->convert_blessed; # ---------------------------------------------------------------- # I/O helpers @@ -46,6 +47,9 @@ sub write_message { sub send_done { write_message({ type => "record_done" }) } sub send_error { write_message({ type => "error", message => $_[0] }) } +# Check if a value is a hashref (blessed or plain) +sub _is_hash { return ref($_[0]) && (reftype($_[0]) // '') eq 'HASH' } + # ---------------------------------------------------------------- # Read init message # ---------------------------------------------------------------- @@ -90,10 +94,8 @@ sub __set { # Users call push_record($hashref) to emit a record. sub push_record { for my $rec (@_) { - if (ref($rec) eq 'HASH') { + if (_is_hash($rec)) { push @_emitted, $rec; - } elsif (ref($rec) && $rec->isa('RecsSDK')) { - push @_emitted, $rec->to_hash(); } } } @@ -150,15 +152,14 @@ sub push_record { next unless $msg->{type} eq 'record'; $line_num++; - my $r = $msg->{data}; # plain hashref, just like original recs + my $r = RecsSDK->new($msg->{data}); eval { if ($mode eq 'eval') { my $result = $compiled->($r, $line_num, $msg->{filename} // 'NONE'); # $result should be the (possibly modified) $r $result = $r unless defined $result; - $result = $result if ref($result) eq 'HASH'; - send_result(ref($result) eq 'HASH' ? $result : $r); + send_result(_is_hash($result) ? $result : $r); } elsif ($mode eq 'grep') { my $passed = $compiled->($r, $line_num, $msg->{filename} // 'NONE'); @@ -177,9 +178,9 @@ sub push_record { # Use return value: could be arrayref of hashrefs, or a single hashref if (ref($result) eq 'ARRAY') { for my $item (@$result) { - send_emit(ref($item) eq 'HASH' ? $item : $r); + send_emit(_is_hash($item) ? $item : $r); } - } elsif (ref($result) eq 'HASH') { + } elsif (_is_hash($result)) { send_emit($result); } elsif (defined $result) { send_emit($r); @@ -198,9 +199,9 @@ sub push_record { } else { if (ref($result) eq 'ARRAY') { for my $item (@$result) { - send_emit($item) if ref($item) eq 'HASH'; + send_emit($item) if _is_hash($item); } - } elsif (ref($result) eq 'HASH' && $result != $r) { + } elsif (_is_hash($result) && $result != $r) { # Only emit if it's a different hashref than $r send_emit($result); } diff --git a/tests/snippets/TemplateExpansion.test.ts b/tests/snippets/TemplateExpansion.test.ts index 82785a0..61b2445 100644 --- a/tests/snippets/TemplateExpansion.test.ts +++ b/tests/snippets/TemplateExpansion.test.ts @@ -155,3 +155,638 @@ for (const { name, create } of runners) { }); }); } + +// ── Language-specific: method calls on {{}} values ────────────── + +describe("{{}} method calls on expanded values [js]", () => { + test("{{field}}.toFixed(2) formats number", () => { + const runner = new JsSnippetRunner(); + void runner.init("{{formatted}} = {{price}}.toFixed(2)", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ price: 3.14159 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["formatted"]).toBe("3.14"); + }); + + test("{{field}}.toUpperCase() transforms string", () => { + const runner = new JsSnippetRunner(); + void runner.init("{{upper}} = {{name}}.toUpperCase()", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ name: "alice" })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["upper"]).toBe("ALICE"); + }); + + test("{{field}}.length reads string length in grep", () => { + const runner = new JsSnippetRunner(); + void runner.init("{{name}}.length > 3", { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ name: "ab" }), + new Record({ name: "alice" }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(false); + expect(results[1]!.passed).toBe(true); + }); + + test("{{field}}.includes() checks substring", () => { + const runner = new JsSnippetRunner(); + void runner.init('{{name}}.includes("li")', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ name: "alice" }), + new Record({ name: "bob" }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("{{field}}.toString() on number", () => { + const runner = new JsSnippetRunner(); + void runner.init("{{str}} = {{x}}.toString()", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 42 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["str"]).toBe("42"); + }); + + test("Math.round({{field}}) wraps expanded value in function", () => { + const runner = new JsSnippetRunner(); + void runner.init("{{rounded}} = Math.round({{x}})", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 3.7 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["rounded"]).toBe(4); + }); +}); + +describe("{{}} method calls on expanded values [python]", () => { + test("str({{field}}) converts to string", () => { + const runner = new PythonSnippetRunner(); + void runner.init("{{s}} = str({{x}})", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 42 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["s"]).toBe("42"); + }); + + test("round({{field}}, N) rounds number", () => { + const runner = new PythonSnippetRunner(); + // Use r["key"] = ... to avoid comma in {{}} = value (regex stops at commas) + void runner.init('r["rounded"] = round({{price}}, 2)', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ price: 3.14159 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["rounded"]).toBe(3.14); + }); + + test("{{field}}.upper() transforms string", () => { + const runner = new PythonSnippetRunner(); + void runner.init("{{upper}} = {{name}}.upper()", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ name: "alice" })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["upper"]).toBe("ALICE"); + }); + + test("len({{field}}) in grep", () => { + const runner = new PythonSnippetRunner(); + void runner.init("len({{name}}) > 3", { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ name: "ab" }), + new Record({ name: "alice" }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(false); + expect(results[1]!.passed).toBe(true); + }); + + test("'substring' in {{field}} checks containment", () => { + const runner = new PythonSnippetRunner(); + void runner.init('"li" in {{name}}', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ name: "alice" }), + new Record({ name: "bob" }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("int({{field}}) converts float to int", () => { + const runner = new PythonSnippetRunner(); + void runner.init("{{truncated}} = int({{x}})", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 3.9 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["truncated"]).toBe(3); + }); +}); + +describe("{{}} method calls on expanded values [perl]", () => { + test("sprintf formats {{field}}", () => { + const runner = new PerlSnippetRunner(); + // Use $r->{key} = ... to avoid comma in {{}} = value (regex stops at commas) + void runner.init('$r->{formatted} = sprintf("%.2f", {{price}})', { + mode: "eval", + }); + + const results = runner.executeBatch([new Record({ price: 3.14159 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["formatted"]).toBe("3.14"); + }); + + test("uc({{field}}) uppercases string", () => { + const runner = new PerlSnippetRunner(); + void runner.init("{{upper}} = uc({{name}})", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ name: "alice" })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["upper"]).toBe("ALICE"); + }); + + test("length({{field}}) in grep", () => { + const runner = new PerlSnippetRunner(); + void runner.init("length({{name}}) > 3", { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ name: "ab" }), + new Record({ name: "alice" }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(false); + expect(results[1]!.passed).toBe(true); + }); + + test("index({{field}}, substr) checks containment", () => { + const runner = new PerlSnippetRunner(); + void runner.init('index({{name}}, "li") >= 0', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ name: "alice" }), + new Record({ name: "bob" }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("int({{field}}) truncates float", () => { + const runner = new PerlSnippetRunner(); + void runner.init("{{truncated}} = int({{x}})", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 3.9 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["truncated"]).toBe(3); + }); + + test("lc({{field}}) lowercases string", () => { + const runner = new PerlSnippetRunner(); + void runner.init("{{lower}} = lc({{name}})", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ name: "ALICE" })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["lower"]).toBe("alice"); + }); +}); + +// ── Language-specific: record method access ───────────────────── + +describe("record method access [js]", () => { + test("r.keys() returns field names", () => { + const runner = new JsSnippetRunner(); + void runner.init("{{count}} = r.keys().length", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ a: 1, b: 2, c: 3 })]); + expect(results).toHaveLength(1); + // count itself is added, so 3 original + 1 new = 4 + expect(results[0]!.record!["count"]).toBe(3); + }); + + test("r.has() checks field existence in grep", () => { + const runner = new JsSnippetRunner(); + void runner.init('r.has("x")', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ x: 1 }), + new Record({ y: 2 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("r.get() reads a field", () => { + const runner = new JsSnippetRunner(); + void runner.init('r.get("x") > 5', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ x: 10 }), + new Record({ x: 3 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("r.set() writes a field", () => { + const runner = new JsSnippetRunner(); + void runner.init('r.set("y", {{x}} * 2)', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 5 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["y"]).toBe(10); + }); + + test("r.toJSON() returns plain object", () => { + const runner = new JsSnippetRunner(); + void runner.init( + "{{isObj}} = typeof r.toJSON() === 'object'", + { mode: "eval" }, + ); + + const results = runner.executeBatch([new Record({ b: 2, a: 1 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["isObj"]).toBe(true); + }); + + test("r.remove() deletes a field", () => { + const runner = new JsSnippetRunner(); + void runner.init('r.remove("y")', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 1, y: 2 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["x"]).toBe(1); + expect(results[0]!.record!["y"]).toBeUndefined(); + }); + + test("r.rename() renames a field", () => { + const runner = new JsSnippetRunner(); + void runner.init('r.rename("x", "newX")', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 42, y: 1 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["newX"]).toBe(42); + expect(results[0]!.record!["x"]).toBeUndefined(); + expect(results[0]!.record!["y"]).toBe(1); + }); + + test("r.pruneTo() keeps only specified fields", () => { + const runner = new JsSnippetRunner(); + void runner.init('r.pruneTo("a", "c")', { mode: "eval" }); + + const results = runner.executeBatch([ + new Record({ a: 1, b: 2, c: 3, d: 4 }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["a"]).toBe(1); + expect(results[0]!.record!["c"]).toBe(3); + expect(results[0]!.record!["b"]).toBeUndefined(); + expect(results[0]!.record!["d"]).toBeUndefined(); + }); + + test("r.clone() creates independent copy", () => { + const runner = new JsSnippetRunner(); + // Clone r, modify the clone's field, assign its value to original — + // confirms clone() returns a real Record with separate data + void runner.init( + 'var c = r.clone(); c.set("x", 999); {{cloneVal}} = c.get("x")', + { mode: "eval" }, + ); + + const results = runner.executeBatch([new Record({ x: 1 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["cloneVal"]).toBe(999); + // original x is unchanged + expect(results[0]!.record!["x"]).toBe(1); + }); +}); + +describe("record method access [python]", () => { + test("r.keys() returns field names", () => { + const runner = new PythonSnippetRunner(); + void runner.init("{{count}} = len(r.keys())", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ a: 1, b: 2, c: 3 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["count"]).toBe(3); + }); + + test("r.has() checks field existence in grep", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.has("@x")', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ x: 1 }), + new Record({ y: 2 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("r['field'] dict-style access", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r["x"] > 5', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ x: 10 }), + new Record({ x: 3 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("r['field'] = value assigns via dict interface", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r["y"] = {{x}} * 2', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 5 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["y"]).toBe(10); + }); + + test("'key' in r checks membership", () => { + const runner = new PythonSnippetRunner(); + void runner.init('"x" in r', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ x: 1 }), + new Record({ y: 2 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("len(r) returns field count", () => { + const runner = new PythonSnippetRunner(); + void runner.init("len(r) == 3", { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ a: 1, b: 2, c: 3 }), + new Record({ a: 1 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("r.to_dict() returns plain dict", () => { + const runner = new PythonSnippetRunner(); + void runner.init("{{is_dict}} = type(r.to_dict()).__name__", { + mode: "eval", + }); + + const results = runner.executeBatch([new Record({ a: 1 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["is_dict"]).toBe("dict"); + }); + + // ── Python Record SDK keyspec methods ─────────────────────── + + test("r.get() with nested keyspec", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.get("a/b") > 0', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ a: { b: 5 } }), + new Record({ a: { b: -1 } }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("r.set() with nested keyspec auto-vivifies", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.set("a/b/c", 42)', { mode: "eval" }); + + const results = runner.executeBatch([new Record({})]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["a"]).toEqual({ b: { c: 42 } }); + }); + + test("r.has() with nested keyspec", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.has("a/b")', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ a: { b: 1 } }), + new Record({ a: { c: 1 } }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("r.get() with array index keyspec", () => { + const runner = new PythonSnippetRunner(); + void runner.init('{{val}} = r.get("items/#1")', { mode: "eval" }); + + const results = runner.executeBatch([ + new Record({ items: [10, 20, 30] }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["val"]).toBe(20); + }); + + test("r.set() with array index keyspec", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.set("items/#0", 99)', { mode: "eval" }); + + const results = runner.executeBatch([ + new Record({ items: [10, 20, 30] }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["items"]).toEqual([99, 20, 30]); + }); + + test("r.data_ref() returns mutable dict reference", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.data_ref()["injected"] = 1', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 1 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["injected"]).toBe(1); + expect(results[0]!.record!["x"]).toBe(1); + }); + + test("del r['field'] removes a field", () => { + const runner = new PythonSnippetRunner(); + void runner.init('del r["y"]', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 1, y: 2 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["x"]).toBe(1); + expect(results[0]!.record!["y"]).toBeUndefined(); + }); + + test("Record() constructor creates new record", () => { + const runner = new PythonSnippetRunner(); + void runner.init('{{val}} = Record({"a": 1}).get("a")', { mode: "eval" }); + + const results = runner.executeBatch([new Record({})]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["val"]).toBe(1); + }); +}); + +describe("record method access [perl]", () => { + test("$r->{field} direct hash access in grep", () => { + const runner = new PerlSnippetRunner(); + void runner.init("$r->{x} > 5", { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ x: 10 }), + new Record({ x: 3 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("$r->{field} = value assigns via hash", () => { + const runner = new PerlSnippetRunner(); + void runner.init("$r->{y} = {{x}} * 2", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 5 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["y"]).toBe(10); + }); + + test("exists $r->{field} checks existence", () => { + const runner = new PerlSnippetRunner(); + void runner.init("exists $r->{x}", { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ x: 1 }), + new Record({ y: 2 }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + test("keys %$r returns field names", () => { + const runner = new PerlSnippetRunner(); + void runner.init("{{count}} = scalar(keys %$r)", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ a: 1, b: 2, c: 3 })]); + expect(results).toHaveLength(1); + // count itself is added via __set before keys is evaluated, so 3 original + 1 = 4 + // Actually: transformCode expands to __set($r, "count", scalar(keys %$r)) + // __set modifies the hash first (sets "count" to undef), then keys runs + // Let's just check it's a number >= 3 + expect(results[0]!.record!["count"]).toBeGreaterThanOrEqual(3); + }); + + test("delete $r->{field} removes a field", () => { + const runner = new PerlSnippetRunner(); + void runner.init("delete $r->{y}", { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 1, y: 2 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["x"]).toBe(1); + expect(results[0]!.record!["y"]).toBeUndefined(); + }); + + test("$r->{nested}{key} accesses nested hash directly", () => { + const runner = new PerlSnippetRunner(); + void runner.init("$r->{a}{b} > 0", { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ a: { b: 5 } }), + new Record({ a: { b: -1 } }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); + + // ── Perl RecsSDK keyspec methods ($r is a RecsSDK object) ─── + + test("$r->get() reads nested keyspec", () => { + const runner = new PerlSnippetRunner(); + void runner.init( + '{{val}} = $r->get("a/b")', + { mode: "eval" }, + ); + + const results = runner.executeBatch([new Record({ a: { b: 42 } })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["val"]).toBe(42); + }); + + test("$r->get() with array index", () => { + const runner = new PerlSnippetRunner(); + void runner.init( + '{{val}} = $r->get("items/#1")', + { mode: "eval" }, + ); + + const results = runner.executeBatch([ + new Record({ items: [10, 20, 30] }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["val"]).toBe(20); + }); + + test("$r->set() writes nested keyspec with auto-vivification", () => { + const runner = new PerlSnippetRunner(); + void runner.init( + '$r->set("a/b/c", 99)', + { mode: "eval" }, + ); + + const results = runner.executeBatch([new Record({})]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["a"]).toEqual({ b: { c: 99 } }); + }); + + test("$r->get() with mixed nested path (hash/array/hash)", () => { + const runner = new PerlSnippetRunner(); + void runner.init( + '{{val}} = $r->get("data/#0/name")', + { mode: "eval" }, + ); + + const results = runner.executeBatch([ + new Record({ data: [{ name: "alice" }, { name: "bob" }] }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["val"]).toBe("alice"); + }); + + test("$r->set() with array index", () => { + const runner = new PerlSnippetRunner(); + void runner.init( + '$r->set("items/#0", 99)', + { mode: "eval" }, + ); + + const results = runner.executeBatch([ + new Record({ items: [10, 20, 30] }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["items"]).toEqual([99, 20, 30]); + }); + + test("$r->has() checks nested keyspec existence", () => { + const runner = new PerlSnippetRunner(); + void runner.init('$r->has("a/b")', { mode: "grep" }); + + const results = runner.executeBatch([ + new Record({ a: { b: 1 } }), + new Record({ a: { c: 1 } }), + ]); + expect(results).toHaveLength(2); + expect(results[0]!.passed).toBe(true); + expect(results[1]!.passed).toBe(false); + }); +}); From e4a00b3c4e87aff58c5555cf2346c14401bf51bc Mon Sep 17 00:00:00 2001 From: Ben Bernard Date: Tue, 24 Feb 2026 16:02:45 -0800 Subject: [PATCH 2/4] feat: add remove, rename, prune_to methods to Python and Perl SDKs Add missing core record manipulation methods to bring Python and Perl snippet SDKs to parity with the JS Record API. Also adds keys() to the Perl SDK. Includes tests for all new methods in both languages. Co-Authored-By: Claude Opus 4.6 --- src/snippets/perl/RecsSDK.pm | 28 +++++++ src/snippets/python/recs_sdk.py | 20 +++++ tests/snippets/TemplateExpansion.test.ts | 93 ++++++++++++++++++++++++ 3 files changed, 141 insertions(+) diff --git a/src/snippets/perl/RecsSDK.pm b/src/snippets/perl/RecsSDK.pm index f1839a6..a1a3248 100644 --- a/src/snippets/perl/RecsSDK.pm +++ b/src/snippets/perl/RecsSDK.pm @@ -39,6 +39,34 @@ sub has { return defined _resolve($self, $keyspec); } +sub keys { + my ($self) = @_; + return CORE::keys(%$self); +} + +sub remove { + my ($self, @fields) = @_; + my @old; + for my $field (@fields) { + push @old, delete $self->{$field}; + } + return @old; +} + +sub rename { + my ($self, $old_key, $new_key) = @_; + $self->{$new_key} = $self->{$old_key}; + delete $self->{$old_key}; +} + +sub prune_to { + my ($self, @ok) = @_; + my %ok = map { ($_ => 1) } @ok; + for my $field (CORE::keys(%$self)) { + delete $self->{$field} unless exists $ok{$field}; + } +} + sub to_hash { my ($self) = @_; return { %$self }; diff --git a/src/snippets/python/recs_sdk.py b/src/snippets/python/recs_sdk.py index b4b3130..27b7838 100644 --- a/src/snippets/python/recs_sdk.py +++ b/src/snippets/python/recs_sdk.py @@ -71,6 +71,26 @@ def has(self, keyspec: str) -> bool: except NoSuchKeyError: return False + def remove(self, *keys: str) -> list[Any]: + """Remove one or more fields. Returns list of old values.""" + old = [] + for key in keys: + old.append(self._data.pop(key, None)) + return old + + def rename(self, old_key: str, new_key: str) -> None: + """Rename a field. If old field doesn't exist, creates new field with None.""" + value = self._data.get(old_key) + self._data[new_key] = value + self._data.pop(old_key, None) + + def prune_to(self, *keys: str) -> None: + """Remove all fields except those specified.""" + keep = set(keys) + for key in list(self._data.keys()): + if key not in keep: + del self._data[key] + # --- dict-like interface --- def __getitem__(self, key: str) -> Any: diff --git a/tests/snippets/TemplateExpansion.test.ts b/tests/snippets/TemplateExpansion.test.ts index 61b2445..2fb37bd 100644 --- a/tests/snippets/TemplateExpansion.test.ts +++ b/tests/snippets/TemplateExpansion.test.ts @@ -635,6 +635,54 @@ describe("record method access [python]", () => { expect(results).toHaveLength(1); expect(results[0]!.record!["val"]).toBe(1); }); + + test("r.remove() deletes a field and returns old value", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.remove("y")', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 1, y: 2 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["x"]).toBe(1); + expect(results[0]!.record!["y"]).toBeUndefined(); + }); + + test("r.remove() multi-key removal", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.remove("a", "b")', { mode: "eval" }); + + const results = runner.executeBatch([ + new Record({ a: 1, b: 2, c: 3 }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["a"]).toBeUndefined(); + expect(results[0]!.record!["b"]).toBeUndefined(); + expect(results[0]!.record!["c"]).toBe(3); + }); + + test("r.rename() renames a field", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.rename("x", "newX")', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 42, y: 1 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["newX"]).toBe(42); + expect(results[0]!.record!["x"]).toBeUndefined(); + expect(results[0]!.record!["y"]).toBe(1); + }); + + test("r.prune_to() keeps only specified fields", () => { + const runner = new PythonSnippetRunner(); + void runner.init('r.prune_to("a", "c")', { mode: "eval" }); + + const results = runner.executeBatch([ + new Record({ a: 1, b: 2, c: 3, d: 4 }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["a"]).toBe(1); + expect(results[0]!.record!["c"]).toBe(3); + expect(results[0]!.record!["b"]).toBeUndefined(); + expect(results[0]!.record!["d"]).toBeUndefined(); + }); }); describe("record method access [perl]", () => { @@ -789,4 +837,49 @@ describe("record method access [perl]", () => { expect(results[0]!.passed).toBe(true); expect(results[1]!.passed).toBe(false); }); + + test("$r->keys() returns field names", () => { + const runner = new PerlSnippetRunner(); + void runner.init('{{count}} = scalar($r->keys())', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ a: 1, b: 2, c: 3 })]); + expect(results).toHaveLength(1); + // count field is set via __set which modifies the hash before keys() runs + expect(results[0]!.record!["count"]).toBeGreaterThanOrEqual(3); + }); + + test("$r->remove() deletes a field and returns old value", () => { + const runner = new PerlSnippetRunner(); + void runner.init('$r->remove("y")', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 1, y: 2 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["x"]).toBe(1); + expect(results[0]!.record!["y"]).toBeUndefined(); + }); + + test("$r->rename() renames a field", () => { + const runner = new PerlSnippetRunner(); + void runner.init('$r->rename("x", "newX")', { mode: "eval" }); + + const results = runner.executeBatch([new Record({ x: 42, y: 1 })]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["newX"]).toBe(42); + expect(results[0]!.record!["x"]).toBeUndefined(); + expect(results[0]!.record!["y"]).toBe(1); + }); + + test("$r->prune_to() keeps only specified fields", () => { + const runner = new PerlSnippetRunner(); + void runner.init('$r->prune_to("a", "c")', { mode: "eval" }); + + const results = runner.executeBatch([ + new Record({ a: 1, b: 2, c: 3, d: 4 }), + ]); + expect(results).toHaveLength(1); + expect(results[0]!.record!["a"]).toBe(1); + expect(results[0]!.record!["c"]).toBe(3); + expect(results[0]!.record!["b"]).toBeUndefined(); + expect(results[0]!.record!["d"]).toBeUndefined(); + }); }); From 6f84dda6b43dfaac4d803467746f2c9b2a499453 Mon Sep 17 00:00:00 2001 From: Ben Bernard Date: Tue, 24 Feb 2026 16:13:27 -0800 Subject: [PATCH 3/4] feat: remove checked-in man pages; add `recs install-manpages` Man pages were committed to man/man1/ and regenerated every test run, causing constant git status noise from date-stamp changes. Now they are gitignored and generated on demand. - Extract generateManPages(outDir) from scripts/generate-manpages.ts - Add `recs install-manpages` subcommand that installs to ~/.local/share/man/man1/ - Update manpages.test.ts to use a temp directory instead of repo man/ - Add man/ to .gitignore and untrack existing files Co-Authored-By: Claude Opus 4.6 --- .gitignore | 3 + bin/recs.ts | 17 ++++ man/man1/recs-annotate.1 | 43 --------- man/man1/recs-assert.1 | 46 ---------- man/man1/recs-chain.1 | 43 --------- man/man1/recs-collate.1 | 110 ----------------------- man/man1/recs-decollate.1 | 46 ---------- man/man1/recs-delta.1 | 32 ------- man/man1/recs-eval.1 | 51 ----------- man/man1/recs-expandjson.1 | 65 -------------- man/man1/recs-flatten.1 | 77 ----------------- man/man1/recs-fromapache.1 | 46 ---------- man/man1/recs-fromatomfeed.1 | 54 ------------ man/man1/recs-fromcsv.1 | 66 -------------- man/man1/recs-fromdb.1 | 46 ---------- man/man1/recs-fromjsonarray.1 | 37 -------- man/man1/recs-fromkv.1 | 43 --------- man/man1/recs-frommongo.1 | 44 ---------- man/man1/recs-frommultire.1 | 58 ------------- man/man1/recs-fromps.1 | 40 --------- man/man1/recs-fromre.1 | 51 ----------- man/man1/recs-fromsplit.1 | 52 ----------- man/man1/recs-fromtcpdump.1 | 29 ------- man/man1/recs-fromxferlog.1 | 24 ------ man/man1/recs-fromxls.1 | 60 ------------- man/man1/recs-fromxml.1 | 43 --------- man/man1/recs-generate.1 | 35 -------- man/man1/recs-grep.1 | 60 ------------- man/man1/recs-join.1 | 55 ------------ man/man1/recs-multiplex.1 | 81 ----------------- man/man1/recs-normalizetime.1 | 57 ------------ man/man1/recs-parsedate.1 | 66 -------------- man/man1/recs-sort.1 | 51 ----------- man/man1/recs-stream2table.1 | 58 ------------- man/man1/recs-substream.1 | 43 --------- man/man1/recs-tochart.1 | 96 --------------------- man/man1/recs-tocsv.1 | 46 ---------- man/man1/recs-todb.1 | 58 ------------- man/man1/recs-togdgraph.1 | 75 ---------------- man/man1/recs-tognuplot.1 | 86 ------------------ man/man1/recs-tohtml.1 | 49 ----------- man/man1/recs-tojsonarray.1 | 27 ------ man/man1/recs-topn.1 | 54 ------------ man/man1/recs-toprettyprint.1 | 52 ----------- man/man1/recs-toptable.1 | 69 --------------- man/man1/recs-totable.1 | 60 ------------- man/man1/recs-xform.1 | 68 --------------- man/man1/recs.1 | 158 ---------------------------------- scripts/generate-manpages.ts | 26 ++++-- tests/cli/manpages.test.ts | 21 ++--- 50 files changed, 49 insertions(+), 2628 deletions(-) delete mode 100644 man/man1/recs-annotate.1 delete mode 100644 man/man1/recs-assert.1 delete mode 100644 man/man1/recs-chain.1 delete mode 100644 man/man1/recs-collate.1 delete mode 100644 man/man1/recs-decollate.1 delete mode 100644 man/man1/recs-delta.1 delete mode 100644 man/man1/recs-eval.1 delete mode 100644 man/man1/recs-expandjson.1 delete mode 100644 man/man1/recs-flatten.1 delete mode 100644 man/man1/recs-fromapache.1 delete mode 100644 man/man1/recs-fromatomfeed.1 delete mode 100644 man/man1/recs-fromcsv.1 delete mode 100644 man/man1/recs-fromdb.1 delete mode 100644 man/man1/recs-fromjsonarray.1 delete mode 100644 man/man1/recs-fromkv.1 delete mode 100644 man/man1/recs-frommongo.1 delete mode 100644 man/man1/recs-frommultire.1 delete mode 100644 man/man1/recs-fromps.1 delete mode 100644 man/man1/recs-fromre.1 delete mode 100644 man/man1/recs-fromsplit.1 delete mode 100644 man/man1/recs-fromtcpdump.1 delete mode 100644 man/man1/recs-fromxferlog.1 delete mode 100644 man/man1/recs-fromxls.1 delete mode 100644 man/man1/recs-fromxml.1 delete mode 100644 man/man1/recs-generate.1 delete mode 100644 man/man1/recs-grep.1 delete mode 100644 man/man1/recs-join.1 delete mode 100644 man/man1/recs-multiplex.1 delete mode 100644 man/man1/recs-normalizetime.1 delete mode 100644 man/man1/recs-parsedate.1 delete mode 100644 man/man1/recs-sort.1 delete mode 100644 man/man1/recs-stream2table.1 delete mode 100644 man/man1/recs-substream.1 delete mode 100644 man/man1/recs-tochart.1 delete mode 100644 man/man1/recs-tocsv.1 delete mode 100644 man/man1/recs-todb.1 delete mode 100644 man/man1/recs-togdgraph.1 delete mode 100644 man/man1/recs-tognuplot.1 delete mode 100644 man/man1/recs-tohtml.1 delete mode 100644 man/man1/recs-tojsonarray.1 delete mode 100644 man/man1/recs-topn.1 delete mode 100644 man/man1/recs-toprettyprint.1 delete mode 100644 man/man1/recs-toptable.1 delete mode 100644 man/man1/recs-totable.1 delete mode 100644 man/man1/recs-xform.1 delete mode 100644 man/man1/recs.1 diff --git a/.gitignore b/.gitignore index 475cdc4..aba5393 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ node_modules # compiled binary bin/recs +# generated man pages (use `recs install-manpages` to install) +man/ + # output out dist diff --git a/bin/recs.ts b/bin/recs.ts index 8bc0ac1..72cd3f0 100644 --- a/bin/recs.ts +++ b/bin/recs.ts @@ -63,6 +63,23 @@ if (command === "--version" || command === "-V") { process.exit(0); } +if (command === "install-manpages") { + const { generateManPages } = await import("../scripts/generate-manpages.ts"); + const { join } = await import("node:path"); + const { homedir } = await import("node:os"); + const manDir = join(homedir(), ".local", "share", "man", "man1"); + const count = await generateManPages(manDir); + console.log(`Installed ${count} man pages to ${manDir}`); + + const manpath = process.env["MANPATH"] ?? ""; + const manParent = join(homedir(), ".local", "share", "man"); + if (!manpath.includes(manParent)) { + console.log(`\nHint: add ${manParent} to your MANPATH:`); + console.log(` export MANPATH="${manParent}:\$MANPATH"`); + } + process.exit(0); +} + if (command === "--list" || command === "-l" || command === "list") { const docs = loadAllDocs(); for (const doc of docs.sort((a, b) => a.name.localeCompare(b.name))) { diff --git a/man/man1/recs-annotate.1 b/man/man1/recs-annotate.1 deleted file mode 100644 index dd36394..0000000 --- a/man/man1/recs-annotate.1 +++ /dev/null @@ -1,43 +0,0 @@ -.TH RECS\-ANNOTATE 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-annotate \- Evaluate an expression on each record and cache the resulting changes by key grouping - -.SH SYNOPSIS -.B recs annotate [options] [files...] - -.SH DESCRIPTION -Evaluate an expression on each record and cache the resulting changes by key grouping. When a record with the same key values is seen again, the cached annotation is applied instead of re\-evaluating the expression. Only use this if you have \-\-keys fields that are repeated; otherwise recs xform will be faster. -.PP - -.SH OPTIONS -.TP -\fB--keys\fR, \fB-k\fR \fI\fR -Keys to match records by. May be specified multiple times. May be a keygroup or keyspec. -.TP -\fB--expr\fR, \fB-e\fR \fI\fR -Inline expression to evaluate (alternative to positional argument). - -.SH EXAMPLES -Annotate records with IPs with hostnames, only doing lookup once -.PP -.RS 4 -.nf -\fBrecs annotate --key ip \'r.hostname = lookupHost(r.ip)\'\fR -.fi -.RE - -Record md5sums of files -.PP -.RS 4 -.nf -\fBrecs annotate --key filename \'r.md5 = computeMd5(r.filename)\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-xform\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-assert.1 b/man/man1/recs-assert.1 deleted file mode 100644 index 505e97e..0000000 --- a/man/man1/recs-assert.1 +++ /dev/null @@ -1,46 +0,0 @@ -.TH RECS\-ASSERT 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-assert \- Asserts that every record in the stream must pass the given expression - -.SH SYNOPSIS -.B recs assert [options] [files...] - -.SH DESCRIPTION -Asserts that every record in the stream must pass the given expression. The expression is evaluated on each record with r set to the current Record object and line set to the current line number (starting at 1). If the expression does not evaluate to true, processing is immediately aborted and an error message is printed. -.PP - -.SH OPTIONS -.TP -\fB--diagnostic\fR, \fB-d\fR \fI\fR -Include this diagnostic string in any failed assertion errors. -.TP -\fB--verbose\fR, \fB-v\fR -Verbose output for failed assertions; dumps the current record. -.TP -\fB--expr\fR, \fB-e\fR \fI\fR -Inline expression to evaluate (alternative to positional argument). - -.SH EXAMPLES -Require each record to have a date field -.PP -.RS 4 -.nf -\fBrecs assert \'r.date\'\fR -.fi -.RE - -Assert all values are positive with a diagnostic -.PP -.RS 4 -.nf -\fBrecs assert -d \'values must be positive\' \'r.value > 0\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-grep\fR(1), \fBrecs\-xform\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-chain.1 b/man/man1/recs-chain.1 deleted file mode 100644 index 34a75de..0000000 --- a/man/man1/recs-chain.1 +++ /dev/null @@ -1,43 +0,0 @@ -.TH RECS\-CHAIN 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-chain \- Creates an in\-memory chain of recs operations - -.SH SYNOPSIS -.B recs chain | | ... - -.SH DESCRIPTION -Creates an in\-memory chain of recs operations. This avoids serialization and deserialization of records at each step in a complex recs pipeline. Arguments are specified on the command line separated by pipes. For most shells, you will need to escape the pipe character to avoid having the shell interpret it as a shell pipe. -.PP - -.SH OPTIONS -.TP -\fB--show-chain\fR -Before running the commands, print out what will happen in the chain. -.TP -\fB--dry-run\fR, \fB-n\fR -Do not run commands. Implies \-\-show\-chain. - -.SH EXAMPLES -Parse some fields, sort and collate, all in memory -.PP -.RS 4 -.nf -\fBrecs chain frommultire \'data,time=(\\S+) (\\S+)\' \\| sort --key time=n \\| collate --a perc,90,data\fR -.fi -.RE - -Use shell commands in your recs stream -.PP -.RS 4 -.nf -\fBrecs chain frommultire \'data,time=(\\S+) (\\S+)\' \\| sort --key time=n \\| grep foo \\| collate --a perc,90,data\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-collate\fR(1), \fBrecs\-sort\fR(1), \fBrecs\-xform\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-collate.1 b/man/man1/recs-collate.1 deleted file mode 100644 index bccd598..0000000 --- a/man/man1/recs-collate.1 +++ /dev/null @@ -1,110 +0,0 @@ -.TH RECS\-COLLATE 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-collate \- Take records, grouped together by \-\-keys, and compute statistics (like average, count, sum, concat, etc - -.SH SYNOPSIS -.B recs collate [options] [files...] - -.SH DESCRIPTION -Take records, grouped together by \-\-keys, and compute statistics (like average, count, sum, concat, etc.) within those groups. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma\-separated list of key fields for grouping. May be a key spec or key group. -.TP -\fB--aggregator\fR, \fB-a\fR \fI\fR -Colon\-separated aggregator specification in the form [=][,]. -.TP -\fB--dlaggregator\fR, \fB-A\fR \fI=\fR -Domain language aggregator in the form name=expression. The expression is evaluated as JavaScript to produce an aggregator. -.TP -\fB--mr-agg\fR \fI \fR -MapReduce aggregator: takes 4 arguments: name, map snippet, reduce snippet, squish snippet. -.TP -\fB--ii-agg\fR \fI \fR -InjectInto aggregator: takes 4 arguments: name, initial snippet, combine snippet, squish snippet. -.TP -\fB--dlkey\fR, \fB-K\fR \fI=\fR -Domain language key: name=expression where the expression evaluates as a valuation. -.TP -\fB--incremental\fR, \fB-i\fR -Output a record every time an input record is added to a clump (instead of every time a clump is flushed). -.TP -\fB--bucket\fR -Output one record per clump (default). -.TP -\fB--no-bucket\fR -Output one record for each record that went into the clump. -.TP -\fB--adjacent\fR, \fB-1\fR -Only group together adjacent records. Avoids spooling records into memory. -.TP -\fB--size\fR, \fB--sz\fR, \fB-n\fR \fI\fR -Number of running clumps to keep. -.TP -\fB--cube\fR -Enable cube mode: output all key combinations with ALL placeholders. -.TP -\fB--clumper\fR, \fB-c\fR \fI\fR -Clumper specification (e.g. keylru,field,size or keyperfect,field or window,size). -.TP -\fB--dlclumper\fR, \fB-C\fR \fI\fR -Domain language clumper specification. -.TP -\fB--perfect\fR -Group records regardless of order (perfect hashing). -.TP -\fB--list-aggregators\fR -List available aggregators and exit. -.TP -\fB--show-aggregator\fR \fI\fR -Show details of a specific aggregator and exit. -.TP -\fB--list-clumpers\fR -List available clumpers and exit. -.TP -\fB--show-clumper\fR \fI\fR -Show details of a specific clumper and exit. - -.SH EXAMPLES -Count number of each x field value in the entire file -.PP -.RS 4 -.nf -\fBrecs collate --key x --aggregator count\fR -.fi -.RE - -Find the maximum latency for each date, hour pair -.PP -.RS 4 -.nf -\fBrecs collate --key date,hour --aggregator worst_latency=max,latency\fR -.fi -.RE - -Produce a cumulative sum of profit up to each date -.PP -.RS 4 -.nf -\fBrecs collate --key date --adjacent --incremental --aggregator profit_to_date=sum,profit\fR -.fi -.RE - -Count clumps of adjacent lines with matching x fields -.PP -.RS 4 -.nf -\fBrecs collate --adjacent --key x --aggregator count\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-decollate\fR(1), \fBrecs\-sort\fR(1), \fBrecs\-multiplex\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-decollate.1 b/man/man1/recs-decollate.1 deleted file mode 100644 index e1f2ecf..0000000 --- a/man/man1/recs-decollate.1 +++ /dev/null @@ -1,46 +0,0 @@ -.TH RECS\-DECOLLATE 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-decollate \- Reverse of collate: takes a single record and produces multiple records using deaggregators - -.SH SYNOPSIS -.B recs decollate [options] [files...] - -.SH DESCRIPTION -Reverse of collate: takes a single record and produces multiple records using deaggregators. Decollate records of input into output records. -.PP - -.SH OPTIONS -.TP -\fB--deaggregator\fR, \fB-d\fR \fI\fR -Deaggregator specification (colon\-separated). -.TP -\fB--only\fR, \fB-o\fR -Only output deaggregated fields, excluding original record fields. Useful when you only want the expanded data, not the source record. -.TP -\fB--list-deaggregators\fR -List available deaggregators and exit. - -.SH EXAMPLES -Split the \'hosts\' field into individual \'host\' fields -.PP -.RS 4 -.nf -\fBrecs decollate --deaggregator \'split,hosts,/\\s*,\\s*/,host\'\fR -.fi -.RE - -Decollate and only keep deaggregated fields -.PP -.RS 4 -.nf -\fBrecs decollate --only -d \'unarray,items,,item\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-collate\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-delta.1 b/man/man1/recs-delta.1 deleted file mode 100644 index 610aac1..0000000 --- a/man/man1/recs-delta.1 +++ /dev/null @@ -1,32 +0,0 @@ -.TH RECS\-DELTA 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-delta \- Transforms absolute values into deltas between adjacent records - -.SH SYNOPSIS -.B recs delta [options] [files...] - -.SH DESCRIPTION -Transforms absolute values into deltas between adjacent records. Fields specified by \-\-key are replaced with the difference between the current and previous record values. Fields not in this list are passed through unchanged, using the first record of each delta pair. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma\-separated list of the fields that should be transformed. May be a keyspec or a keygroup. - -.SH EXAMPLES -Transform a cumulative counter of errors into a count of errors per record -.PP -.RS 4 -.nf -\fBrecs delta --key errors\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-xform\fR(1), \fBrecs\-collate\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-eval.1 b/man/man1/recs-eval.1 deleted file mode 100644 index fef27ee..0000000 --- a/man/man1/recs-eval.1 +++ /dev/null @@ -1,51 +0,0 @@ -.TH RECS\-EVAL 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-eval \- Evaluate an expression on each record and print the result as a line of text - -.SH SYNOPSIS -.B recs eval [options] [files...] - -.SH DESCRIPTION -Evaluate an expression on each record and print the result as a line of text. This is NOT a record stream output \-\- it prints raw text lines. The expression is evaluated with r set to the current Record object and line set to the current line number (starting at 1). When \-\-lang is used with a non\-JS language, the record is modified by the snippet and output as a JSON line. -.PP - -.SH OPTIONS -.TP -\fB--chomp\fR -Chomp eval results (remove trailing newlines to avoid duplicate newlines when already newline\-terminated). -.TP -\fB--lang\fR, \fB-l\fR \fI\fR -Snippet language: js (default), python/py, perl/pl. - -.SH EXAMPLES -Print the host field from each record -.PP -.RS 4 -.nf -\fBrecs eval \'r.host\'\fR -.fi -.RE - -Prepare to gnuplot field y against field x -.PP -.RS 4 -.nf -\fBrecs eval \'r.x + " " + r.y\'\fR -.fi -.RE - -Add a field using Python -.PP -.RS 4 -.nf -\fBrecs eval --lang python \'r["b"] = r["a"] + 1\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-xform\fR(1), \fBrecs\-grep\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-expandjson.1 b/man/man1/recs-expandjson.1 deleted file mode 100644 index bca6264..0000000 --- a/man/man1/recs-expandjson.1 +++ /dev/null @@ -1,65 +0,0 @@ -.TH RECS\-EXPANDJSON 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-expandjson \- Expand JSON strings embedded in record fields into actual JSON values - -.SH SYNOPSIS -.B recs expandjson [options] [files...] - -.SH DESCRIPTION -Expand JSON strings embedded in record fields into actual JSON values. When a field contains a string that is valid JSON (object, array, etc.), this operation parses it and replaces the string with the parsed structure. With no \-\-key options, all top\-level string fields that look like JSON are expanded. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Key containing a JSON string to expand. May be a keyspec. May be specified multiple times for multiple keys. -.TP -\fB--recursive\fR, \fB-r\fR -Recursively expand JSON strings found in nested values after initial expansion. - -.SH EXAMPLES -Expand a metadata field containing a JSON string -.PP -.RS 4 -.nf -\fBrecs expandjson --key metadata\fR -.fi -.RE -.PP -Input: -.RS 4 -.nf -{"name":"alice","metadata":"{\\"role\\":\\"admin\\",\\"level\\":3}"} -.fi -.RE -.PP -Output: -.RS 4 -.nf -{"name":"alice","metadata":{"role":"admin","level":3}} -.fi -.RE - -Recursively expand nested JSON strings -.PP -.RS 4 -.nf -\fBrecs expandjson -r --key payload\fR -.fi -.RE - -Expand all JSON\-like string fields automatically -.PP -.RS 4 -.nf -\fBrecs expandjson\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-flatten\fR(1), \fBrecs\-eval\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-flatten.1 b/man/man1/recs-flatten.1 deleted file mode 100644 index 347d222..0000000 --- a/man/man1/recs-flatten.1 +++ /dev/null @@ -1,77 +0,0 @@ -.TH RECS\-FLATTEN 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-flatten \- Flatten nested hash/array structures in records into top\-level fields - -.SH SYNOPSIS -.B recs flatten [options] [files...] - -.SH DESCRIPTION -Flatten nested hash/array structures in records into top\-level fields. Note: this implements a strategy for dealing with nested structures that is almost always better handled by using keyspecs or keygroups. -.PP - -.SH OPTIONS -.TP -\fB--depth\fR \fI\fR -Change the default flatten depth. Negative values mean arbitrary depth. Default is 1. -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma\-separated list of fields to flatten at the default depth. -.TP -\fB--deep\fR \fI\fR -Comma\-separated list of fields to flatten to arbitrary depth. -.TP -\fB--separator\fR \fI\fR -String used to separate joined field names. Default is \'\-\'. - -.SH EXAMPLES -Flatten a nested field one level deep -.PP -.RS 4 -.nf -\fBrecs flatten -k field\fR -.fi -.RE -.PP -Input: -.RS 4 -.nf -{"field":{"subfield":"value"}} -.fi -.RE -.PP -Output: -.RS 4 -.nf -{"field-subfield":"value"} -.fi -.RE - -Flatten a deeply nested structure to arbitrary depth -.PP -.RS 4 -.nf -\fBrecs flatten --deep x\fR -.fi -.RE -.PP -Input: -.RS 4 -.nf -{"x":{"y":[{"z":"v"}]}} -.fi -.RE -.PP -Output: -.RS 4 -.nf -{"x-y-0-z":"v"} -.fi -.RE - -.SH SEE ALSO -\fBrecs\-xform\fR(1), \fBrecs\-stream2table\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromapache.1 b/man/man1/recs-fromapache.1 deleted file mode 100644 index 5a75ac2..0000000 --- a/man/man1/recs-fromapache.1 +++ /dev/null @@ -1,46 +0,0 @@ -.TH RECS\-FROMAPACHE 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromapache \- Each line of input (or lines of ) is parsed to produce an output record from Apache access logs - -.SH SYNOPSIS -.B recs fromapache [options] [files...] - -.SH DESCRIPTION -Each line of input (or lines of ) is parsed to produce an output record from Apache access logs. Supports combined, common, and vhost_common log formats. -.PP - -.SH OPTIONS -.TP -\fB--fast\fR -Use the fast parser which works relatively fast. It can process only \'common\', \'combined\' and custom styles with compatibility with \'common\', and cannot work with backslash\-quoted double\-quotes in fields. This is the default. -.TP -\fB--strict\fR -Use the strict parser which works relatively slow. It can process any style format logs, with specification about separator, and checker for perfection. It can also process backslash\-quoted double\-quotes properly. -.TP -\fB--verbose\fR -Verbose output. - -.SH EXAMPLES -Get records from typical apache log -.PP -.RS 4 -.nf -\fBrecs fromapache < /var/log/httpd-access.log\fR -.fi -.RE - -Use strict parser with specific formats -.PP -.RS 4 -.nf -\fBrecs fromapache --strict \'["combined","common","vhost_common"]\' < /var/log/httpd-access.log\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromre\fR(1), \fBrecs\-frommultire\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromatomfeed.1 b/man/man1/recs-fromatomfeed.1 deleted file mode 100644 index b9a11ba..0000000 --- a/man/man1/recs-fromatomfeed.1 +++ /dev/null @@ -1,54 +0,0 @@ -.TH RECS\-FROMATOMFEED 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromatomfeed \- Produce records from atom feed entries - -.SH SYNOPSIS -.B recs fromatomfeed [options] [] - -.SH DESCRIPTION -Produce records from atom feed entries. Recs fromatomfeed will get entries from paginated atom feeds and create a record stream from the results. The keys of the record will be the fields in the atom feed entry. By default, it follows \'next\' links in a feed to retrieve all entries. -.PP - -.SH OPTIONS -.TP -\fB--follow\fR -Follow atom feed next links (default on). -.TP -\fB--nofollow\fR -Do not follow next links. -.TP -\fB--max\fR \fI\fR -Print at most entries and then exit. - -.SH EXAMPLES -Dump an entire feed -.PP -.RS 4 -.nf -\fBrecs fromatomfeed "http://my.xml.com"\fR -.fi -.RE - -Dump just the first page of entries -.PP -.RS 4 -.nf -\fBrecs fromatomfeed --nofollow "http://my.xml.com"\fR -.fi -.RE - -Dump just the first 10 entries -.PP -.RS 4 -.nf -\fBrecs fromatomfeed --max 10 "http://my.xml.com"\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromxml\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromcsv.1 b/man/man1/recs-fromcsv.1 deleted file mode 100644 index 3c845ee..0000000 --- a/man/man1/recs-fromcsv.1 +++ /dev/null @@ -1,66 +0,0 @@ -.TH RECS\-FROMCSV 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromcsv \- Each line of input (or lines of ) is split on commas to produce an output record - -.SH SYNOPSIS -.B recs fromcsv [options] [] - -.SH DESCRIPTION -Each line of input (or lines of ) is split on commas to produce an output record. Fields are named numerically (0, 1, etc.), or as given by \-\-key, or as read by \-\-header. Lines may be split on delimiters other than commas by providing \-\-delim. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma separated list of field names. May be specified multiple times, may be key specs. -.TP -\fB--field\fR, \fB-f\fR \fI\fR -Comma separated list of field names. May be specified multiple times, may be key specs. -.TP -\fB--header\fR -Take field names from the first line of input. -.TP -\fB--strict\fR -Do not trim whitespace, allow loose quoting (quotes inside quotes), or allow the use of escape characters when not strictly needed. -.TP -\fB--delim\fR, \fB-d\fR \fI\fR -Field delimiter to use when reading input lines (default \',\'). -.TP -\fB--escape\fR \fI\fR -Escape character used in quoted fields (default \'"\'). -.TP -\fB--quote\fR \fI\fR -Quote character used in quoted fields (default \'"\'). Use the empty string to indicate no quoted fields. - -.SH EXAMPLES -Parse csv separated fields x and y -.PP -.RS 4 -.nf -\fBrecs fromcsv --field x,y\fR -.fi -.RE - -Parse data with a header line specifying fields -.PP -.RS 4 -.nf -\fBrecs fromcsv --header\fR -.fi -.RE - -Parse tsv data (using bash syntax for a literal tab) -.PP -.RS 4 -.nf -\fBrecs fromcsv --delim $\'\\t\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromsplit\fR(1), \fBrecs\-fromre\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromdb.1 b/man/man1/recs-fromdb.1 deleted file mode 100644 index 08acce5..0000000 --- a/man/man1/recs-fromdb.1 +++ /dev/null @@ -1,46 +0,0 @@ -.TH RECS\-FROMDB 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromdb \- Execute a select statement on a database and create a record stream from the results - -.SH SYNOPSIS -.B recs fromdb [options] - -.SH DESCRIPTION -Execute a select statement on a database and create a record stream from the results. The keys of the record will be the column names and the values the row values. -.PP - -.SH OPTIONS -.TP -\fB--table\fR \fI\fR -Table name (shortcut for SELECT * FROM table). -.TP -\fB--sql\fR \fI\fR -SQL select statement to run. -.TP -\fB--dbfile\fR \fI\fR -Path to the database file. -.TP -\fB--type\fR \fI\fR -Database type (default: sqlite). - -.SH EXAMPLES -Dump a table -.PP -.RS 4 -.nf -\fBrecs fromdb --type sqlite --dbfile testDb --table recs\fR -.fi -.RE - -Run a select statement -.PP -.RS 4 -.nf -\fBrecs fromdb --dbfile testDb --sql \'SELECT * FROM recs WHERE id > 9\'\fR -.fi -.RE - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromjsonarray.1 b/man/man1/recs-fromjsonarray.1 deleted file mode 100644 index 4b5c700..0000000 --- a/man/man1/recs-fromjsonarray.1 +++ /dev/null @@ -1,37 +0,0 @@ -.TH RECS\-FROMJSONARRAY 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromjsonarray \- Import JSON objects from within a JSON array - -.SH SYNOPSIS -.B recs fromjsonarray [options] [] - -.SH DESCRIPTION -Import JSON objects from within a JSON array. Each object in the array becomes an output record. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Optional comma separated list of field names to extract. If none specified, use all keys. May be specified multiple times, may be key specs. - -.SH EXAMPLES -Parse a JSON array file into records -.PP -.RS 4 -.nf -\fBrecs fromjsonarray data.json\fR -.fi -.RE - -Extract only specific keys from a JSON array -.PP -.RS 4 -.nf -\fBrecs fromjsonarray --key name,age data.json\fR -.fi -.RE - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromkv.1 b/man/man1/recs-fromkv.1 deleted file mode 100644 index 92c7888..0000000 --- a/man/man1/recs-fromkv.1 +++ /dev/null @@ -1,43 +0,0 @@ -.TH RECS\-FROMKV 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromkv \- Records are generated from character input with the form " - -.SH SYNOPSIS -.B recs fromkv [options] [] - -.SH DESCRIPTION -Records are generated from character input with the form "...". Records have the form "...". Entries are pairs of the form "". -.PP - -.SH OPTIONS -.TP -\fB--kv-delim\fR, \fB-f\fR \fI\fR -Delimiter for separating key/value pairs within an entry (default \' \'). -.TP -\fB--entry-delim\fR, \fB-e\fR \fI\fR -Delimiter for separating entries within records (default \'\\n\'). -.TP -\fB--record-delim\fR, \fB-r\fR \fI\fR -Delimiter for separating records (default \'END\\n\'). - -.SH EXAMPLES -Parse memcached stat metrics into records -.PP -.RS 4 -.nf -\fBecho -ne \'stats\\r\\n\' | nc -i1 localhost 11211 | tr -d \'\\r\' | awk \'{if (! /END/) {print $2" "$3} else {print $0}}\' | recs fromkv\fR -.fi -.RE - -Parse records separated by \'E\\n\' with entries separated by \'|\' and pairs separated by \'=\' -.PP -.RS 4 -.nf -\fBrecs fromkv --kv-delim \'=\' --entry-delim \'|\' --record-delim $(echo -ne \'E\\n\')\fR -.fi -.RE - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-frommongo.1 b/man/man1/recs-frommongo.1 deleted file mode 100644 index 67987ed..0000000 --- a/man/man1/recs-frommongo.1 +++ /dev/null @@ -1,44 +0,0 @@ -.TH RECS\-FROMMONGO 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-frommongo \- Generate records from a MongoDB query - -.SH SYNOPSIS -.B recs frommongo --host --name --collection --query - -.SH DESCRIPTION -Generate records from a MongoDB query. Connects to a MongoDB instance, runs the specified query against the given collection, and outputs each matching document as a record. -.PP - -.SH OPTIONS -.TP -\fB--host\fR \fI\fR -URI for your mongo instance, may include user:pass@URI. -.TP -\fB--user\fR \fI\fR -User to authenticate as. -.TP -\fB--password\fR, \fB--pass\fR \fI\fR -Password for \-\-user. -.TP -\fB--name\fR, \fB--dbname\fR \fI\fR -Name of database to connect to. -.TP -\fB--collection\fR \fI\fR -Name of collection to query against. -.TP -\fB--query\fR \fI\fR -JSON query string to run against the collection. - -.SH EXAMPLES -Make a query against a MongoDB instance -.PP -.RS 4 -.nf -\fBrecs frommongo --host mongodb://user:pass@dharma.mongohq.com:10069 --name my_app --collection my_collection --query \'{doc_key: {$not: {$size: 0}}}\'\fR -.fi -.RE - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-frommultire.1 b/man/man1/recs-frommultire.1 deleted file mode 100644 index 40f1bad..0000000 --- a/man/man1/recs-frommultire.1 +++ /dev/null @@ -1,58 +0,0 @@ -.TH RECS\-FROMMULTIRE 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-frommultire \- Match multiple regexes against each line of input (or lines of ) - -.SH SYNOPSIS -.B recs frommultire [options] [] - -.SH DESCRIPTION -Match multiple regexes against each line of input (or lines of ). Various parameters control when the accumulated fields are flushed to output as a record and which, if any, fields are cleared when the record is flushed. By default regexes do not necessarily flush on either side, would\-be field collisions cause a flush, EOF causes a flush if any fields are set, and all fields are cleared on a flush. Regex syntax is: \',=REGEX\'. KEY field names are optional. If a field matches $NUM, then that match number in the regex will be used as the field name. -.PP - -.SH OPTIONS -.TP -\fB--no-flush-regex\fR, \fB--regex\fR, \fB--re\fR \fI\fR -Add a normal regex (no flushing). -.TP -\fB--pre-flush-regex\fR, \fB--pre\fR \fI\fR -Add a regex that flushes before interpreting fields when matched. -.TP -\fB--post-flush-regex\fR, \fB--post\fR \fI\fR -Add a regex that flushes after interpreting fields when matched. -.TP -\fB--double-flush-regex\fR, \fB--double\fR \fI\fR -Add a regex that flushes both before and after interpreting fields when matched. -.TP -\fB--clobber\fR -Do not flush records when a field from a match would clobber an already existing field and do not flush at EOF. -.TP -\fB--keep-all\fR -Do not clear any fields on a flush. -.TP -\fB--keep\fR \fI\fR -Do not clear this comma separated list of fields on a flush. - -.SH EXAMPLES -Parse several fields on separate lines -.PP -.RS 4 -.nf -\fBrecs frommultire --re \'fname,lname=^Name: (.*) (.*)$\' --re \'addr=^Address: (.*)$\'\fR -.fi -.RE - -Some fields apply to multiple records (department here) -.PP -.RS 4 -.nf -\fBrecs frommultire --post \'fname,lname=^Name: (.*) (.*)$\' --re \'department=^Department: (.*)$\' --clobber --keep team\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromre\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromps.1 b/man/man1/recs-fromps.1 deleted file mode 100644 index f1650d3..0000000 --- a/man/man1/recs-fromps.1 +++ /dev/null @@ -1,40 +0,0 @@ -.TH RECS\-FROMPS 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromps \- Generate records from the process table - -.SH SYNOPSIS -.B recs fromps [options] - -.SH DESCRIPTION -Generate records from the process table. Prints out JSON records converted from the process table. Fields default to all available fields from ps. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Fields to output. May be specified multiple times, may be comma separated. Defaults to all fields. -.TP -\fB--field\fR, \fB-f\fR \fI\fR -Fields to output. May be specified multiple times, may be comma separated. Defaults to all fields. - -.SH EXAMPLES -Get records for the process table -.PP -.RS 4 -.nf -\fBrecs fromps\fR -.fi -.RE - -Only get uid and pid -.PP -.RS 4 -.nf -\fBrecs fromps --key uid,pid\fR -.fi -.RE - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromre.1 b/man/man1/recs-fromre.1 deleted file mode 100644 index 1bd026b..0000000 --- a/man/man1/recs-fromre.1 +++ /dev/null @@ -1,51 +0,0 @@ -.TH RECS\-FROMRE 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromre \- The regex is matched against each line of input (or lines of ) - -.SH SYNOPSIS -.B recs fromre [options] [] - -.SH DESCRIPTION -The regex is matched against each line of input (or lines of ). Each successful match results in one output record whose field values are the capture groups from the match. Lines that do not match are ignored. Keys are named numerically (0, 1, etc.) or as given by \-\-key. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma separated list of key names. May be specified multiple times, may be key specs. -.TP -\fB--field\fR, \fB-f\fR \fI\fR -Comma separated list of key names. May be specified multiple times, may be key specs. - -.SH EXAMPLES -Parse greetings -.PP -.RS 4 -.nf -\fBrecs fromre --key name,age \'^Hello, my name is (.*) and I am (\\d*) years? old$\'\fR -.fi -.RE - -Parse a single key named time from a group of digits at the beginning of the line -.PP -.RS 4 -.nf -\fBrecs fromre --key time \'^(\\d+)\'\fR -.fi -.RE - -Map three sets of <>s to a record with keys named 0, 1, and 2 -.PP -.RS 4 -.nf -\fBrecs fromre \'<(.*)>\\s*<(.*)>\\s*<(.*)>\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromsplit\fR(1), \fBrecs\-frommultire\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromsplit.1 b/man/man1/recs-fromsplit.1 deleted file mode 100644 index 4ed3ac8..0000000 --- a/man/man1/recs-fromsplit.1 +++ /dev/null @@ -1,52 +0,0 @@ -.TH RECS\-FROMSPLIT 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromsplit \- Each line of input (or lines of ) is split on the provided delimiter to produce an output record - -.SH SYNOPSIS -.B recs fromsplit [options] [] - -.SH DESCRIPTION -Each line of input (or lines of ) is split on the provided delimiter to produce an output record. Keys are named numerically (0, 1, etc.) or as given by \-\-key. -.PP - -.SH OPTIONS -.TP -\fB--delim\fR, \fB-d\fR \fI\fR -Delimiter to use for splitting input lines (default \',\'). -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma separated list of key names. May be specified multiple times, may be key specs. -.TP -\fB--field\fR, \fB-f\fR \fI\fR -Comma separated list of key names. May be specified multiple times, may be key specs. -.TP -\fB--header\fR -Take key names from the first line of input. -.TP -\fB--strict\fR -Delimiter is not treated as a regex. - -.SH EXAMPLES -Parse space separated keys x and y -.PP -.RS 4 -.nf -\fBrecs fromsplit --key x,y --delim \' \'\fR -.fi -.RE - -Parse comma separated keys a, b, and c -.PP -.RS 4 -.nf -\fBrecs fromsplit --key a,b,c\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromcsv\fR(1), \fBrecs\-fromre\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromtcpdump.1 b/man/man1/recs-fromtcpdump.1 deleted file mode 100644 index fa6c0fa..0000000 --- a/man/man1/recs-fromtcpdump.1 +++ /dev/null @@ -1,29 +0,0 @@ -.TH RECS\-FROMTCPDUMP 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromtcpdump \- Runs tcpdump and puts out records, one for each packet - -.SH SYNOPSIS -.B recs fromtcpdump [options] [ ...] - -.SH DESCRIPTION -Runs tcpdump and puts out records, one for each packet. Expects pcap files. Will put the name of the originating capture file in the \'file\' field. Will parse packet types: ethernet, ip, udp, arp, tcp. The type key will indicate the highest level parsed. By default, data output is suppressed due to poor interaction with terminal programs. -.PP - -.SH OPTIONS -.TP -\fB--data\fR -Include raw data bytes of deepest packet level. - -.SH EXAMPLES -Get records for all packets -.PP -.RS 4 -.nf -\fBrecs fromtcpdump capture.pcap\fR -.fi -.RE - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromxferlog.1 b/man/man1/recs-fromxferlog.1 deleted file mode 100644 index 001f7f1..0000000 --- a/man/man1/recs-fromxferlog.1 +++ /dev/null @@ -1,24 +0,0 @@ -.TH RECS\-FROMXFERLOG 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromxferlog \- Each line of input (or lines of ) is parsed as an FTP transfer log (xferlog format) to produce an output record - -.SH SYNOPSIS -.B recs fromxferlog [files...] - -.SH DESCRIPTION -Each line of input (or lines of ) is parsed as an FTP transfer log (xferlog format) to produce an output record. Fields include day_name, month, day, current_time, year, transfer_time, remote_host, file_size, filename, transfer_type, special_action_flag, direction, access_mode, username, service_name, authentication_method, authenticated_user_id, and completion_status. -.PP - -.SH EXAMPLES -Get records from typical xferlog -.PP -.RS 4 -.nf -\fBrecs fromxferlog < /var/log/xferlog\fR -.fi -.RE - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromxls.1 b/man/man1/recs-fromxls.1 deleted file mode 100644 index c865970..0000000 --- a/man/man1/recs-fromxls.1 +++ /dev/null @@ -1,60 +0,0 @@ -.TH RECS\-FROMXLS 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromxls \- Parse Excel files (xls, xlsx, xlsb, xlsm) into records - -.SH SYNOPSIS -.B recs fromxls [options] - -.SH DESCRIPTION -Parse Excel files (xls, xlsx, xlsb, xlsm) into records. By default, reads the first sheet and uses the first row as header names. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma separated list of field names. Overrides header detection. -.TP -\fB--field\fR, \fB-f\fR \fI\fR -Comma separated list of field names. Overrides header detection. -.TP -\fB--no-header\fR, \fB-n\fR -Do not treat the first row as a header. Fields will be named numerically (0, 1, 2, ...). -.TP -\fB--sheet\fR, \fB-s\fR \fI\fR -Specify a sheet name to read. Defaults to the first sheet. -.TP -\fB--all-sheets\fR -Read all sheets in the workbook, adding a \'sheet\' field to each record. - -.SH EXAMPLES -Read an Excel file using headers from the first row -.PP -.RS 4 -.nf -\fBrecs fromxls data.xlsx\fR -.fi -.RE - -Read a specific sheet without headers -.PP -.RS 4 -.nf -\fBrecs fromxls --sheet \'Sheet2\' --no-header -k name,value data.xlsx\fR -.fi -.RE - -Read all sheets -.PP -.RS 4 -.nf -\fBrecs fromxls --all-sheets data.xlsx\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromcsv\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-fromxml.1 b/man/man1/recs-fromxml.1 deleted file mode 100644 index 0792c9f..0000000 --- a/man/man1/recs-fromxml.1 +++ /dev/null @@ -1,43 +0,0 @@ -.TH RECS\-FROMXML 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-fromxml \- Reads either from STDIN or from the specified URIs - -.SH SYNOPSIS -.B recs fromxml [options] [] - -.SH DESCRIPTION -Reads either from STDIN or from the specified URIs. Parses the XML documents and creates records for the specified elements. If multiple element types are specified, will add an \'element\' field to the output record. -.PP - -.SH OPTIONS -.TP -\fB--element\fR \fI\fR -May be comma separated, may be specified multiple times. Sets the elements/attributes to print records for. -.TP -\fB--nested\fR -Search for elements at all levels of the XML document. - -.SH EXAMPLES -Create records for the bar element at the top level of myXMLDoc -.PP -.RS 4 -.nf -\fBrecs fromxml --element bar file:myXMLDoc\fR -.fi -.RE - -Create records for all foo and bar elements from a URL -.PP -.RS 4 -.nf -\fBrecs fromxml --element foo,bar --nested http://google.com\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-fromatomfeed\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-generate.1 b/man/man1/recs-generate.1 deleted file mode 100644 index 2931123..0000000 --- a/man/man1/recs-generate.1 +++ /dev/null @@ -1,35 +0,0 @@ -.TH RECS\-GENERATE 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-generate \- Execute an expression for each record to generate new records - -.SH SYNOPSIS -.B recs generate [options] [files...] - -.SH DESCRIPTION -Execute an expression for each record to generate new records. The expression should return an array of new record objects (or a single record). Each generated record gets a chain link back to the original input record under the \'_chain\' key (configurable via \-\-keychain). -.PP - -.SH OPTIONS -.TP -\fB--keychain\fR \fI\fR -Key name for the chain link back to the original record. Default is \'_chain\'. May be a key spec. -.TP -\fB--passthrough\fR -Emit the input record in addition to the generated records. - -.SH EXAMPLES -Generate sub\-records from a feed and chain back to the original -.PP -.RS 4 -.nf -\fBrecs generate \'fetchFeed(r.url).map(item => ({ title: item.title }))\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-xform\fR(1), \fBrecs\-chain\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-grep.1 b/man/man1/recs-grep.1 deleted file mode 100644 index 19e1dee..0000000 --- a/man/man1/recs-grep.1 +++ /dev/null @@ -1,60 +0,0 @@ -.TH RECS\-GREP 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-grep \- Filter records where an expression evaluates to true - -.SH SYNOPSIS -.B recs grep [options] [files...] - -.SH DESCRIPTION -Filter records where an expression evaluates to true. The expression is evaluated on each record with r set to the current Record object and line set to the current line number (starting at 1). Records for which the expression is truthy are passed through. -.PP - -.SH OPTIONS -.TP -\fB--invert-match\fR, \fB-v\fR -Anti\-match: records NOT matching the expression will be returned. -.TP -\fB--context\fR, \fB-C\fR \fI\fR -Provide NUM records of context around matches (equivalent to \-A NUM and \-B NUM). -.TP -\fB--after-context\fR, \fB-A\fR \fI\fR -Print out NUM following records after a match. -.TP -\fB--before-context\fR, \fB-B\fR \fI\fR -Print out the previous NUM records on a match. -.TP -\fB--lang\fR, \fB-l\fR \fI\fR -Snippet language: js (default), python/py, perl/pl. - -.SH EXAMPLES -Filter to records with field \'name\' equal to \'John\' -.PP -.RS 4 -.nf -\fBrecs grep \'r.name === "John"\'\fR -.fi -.RE - -Find records without ppid equal to 3456 -.PP -.RS 4 -.nf -\fBrecs grep -v \'r.ppid === 3456\'\fR -.fi -.RE - -Filter using Python -.PP -.RS 4 -.nf -\fBrecs grep --lang python \'r["age"] > 30\'\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-xform\fR(1), \fBrecs\-assert\fR(1), \fBrecs\-substream\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-join.1 b/man/man1/recs-join.1 deleted file mode 100644 index 75965a3..0000000 --- a/man/man1/recs-join.1 +++ /dev/null @@ -1,55 +0,0 @@ -.TH RECS\-JOIN 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-join \- Join two record streams on a key - -.SH SYNOPSIS -.B recs join [options] [files...] - -.SH DESCRIPTION -Join two record streams on a key. Records of input are joined against records in dbfile, using field inputkey from input and field dbkey from dbfile. Each pair of matches will be combined to form a larger record, with fields from the dbfile overwriting fields from the input stream. -.PP - -.SH OPTIONS -.TP -\fB--left\fR -Do a left join (include unmatched db records). -.TP -\fB--right\fR -Do a right join (include unmatched input records). -.TP -\fB--inner\fR -Do an inner join (default). Only matched pairs are output. -.TP -\fB--outer\fR -Do an outer join (include all unmatched records from both sides). -.TP -\fB--operation\fR \fI\fR -A JS expression for merging two records together, in place of the default behavior of db fields overwriting input fields. Variables d and i are the db record and input record respectively. -.TP -\fB--accumulate-right\fR -Accumulate all input records with the same key onto each db record matching that key. - -.SH EXAMPLES -Join type from input and typeName from dbfile -.PP -.RS 4 -.nf -\fBcat recs | recs join type typeName dbfile\fR -.fi -.RE - -Join host name from a mapping file to machines -.PP -.RS 4 -.nf -\fBrecs join host host hostIpMapping machines\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-collate\fR(1), \fBrecs\-xform\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-multiplex.1 b/man/man1/recs-multiplex.1 deleted file mode 100644 index 50af699..0000000 --- a/man/man1/recs-multiplex.1 +++ /dev/null @@ -1,81 +0,0 @@ -.TH RECS\-MULTIPLEX 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-multiplex \- Take records, grouped together by \-\-keys, and run a separate operation instance for each group - -.SH SYNOPSIS -.B recs multiplex [options] -- - -.SH DESCRIPTION -Take records, grouped together by \-\-keys, and run a separate operation instance for each group. Each group gets its own operation instance. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Comma\-separated list of key fields for grouping. May be a key spec or key group. -.TP -\fB--dlkey\fR, \fB-K\fR \fI=\fR -Domain language key: name=expression where the expression evaluates as a valuation. -.TP -\fB--line-key\fR, \fB-L\fR \fI\fR -Use the value of this key as line input for the nested operation (rather than the entire record). Use with recs from* operations generally. -.TP -\fB--output-file-key\fR, \fB-o\fR \fI\fR -Write each group\'s output to a separate file, using the value of the given key as the filename. -.TP -\fB--output-file-eval\fR, \fB-O\fR \fI\fR -Write each group\'s output to a separate file, with filename determined by the given expression. Supports {{key}} interpolation with group key values. -.TP -\fB--adjacent\fR, \fB-1\fR -Only group together adjacent records. Avoids spooling records into memory. -.TP -\fB--size\fR, \fB--sz\fR, \fB-n\fR \fI\fR -Number of running clumps to keep. -.TP -\fB--cube\fR -Enable cube mode. -.TP -\fB--clumper\fR, \fB-c\fR \fI\fR -Clumper specification (e.g. keylru,field,size or keyperfect,field). -.TP -\fB--dlclumper\fR \fI\fR -Domain language clumper specification. -.TP -\fB--list-clumpers\fR -List available clumpers and exit. -.TP -\fB--show-clumper\fR \fI\fR -Show details of a specific clumper and exit. - -.SH EXAMPLES -Tag lines with counts by thread -.PP -.RS 4 -.nf -\fBrecs multiplex -k thread -- recs eval \'r.nbr = ++nbr\'\fR -.fi -.RE - -Separate out a stream of text by PID into separate invocations of an operation -.PP -.RS 4 -.nf -\fBrecs fromre \'^(.*PID=([0-9]*).*)$\' -f line,pid | recs multiplex -L line -k pid -- recs frommultire ...\fR -.fi -.RE - -Write each group\'s CSV output to separate files by department -.PP -.RS 4 -.nf -\fBrecs multiplex -k department -O \'output-{{department}}.csv\' -- recs tocsv\fR -.fi -.RE - -.SH SEE ALSO -\fBrecs\-collate\fR(1), \fBrecs\-chain\fR(1) - -.SH AUTHOR -Benjamin Bernard - diff --git a/man/man1/recs-normalizetime.1 b/man/man1/recs-normalizetime.1 deleted file mode 100644 index 70a2524..0000000 --- a/man/man1/recs-normalizetime.1 +++ /dev/null @@ -1,57 +0,0 @@ -.TH RECS\-NORMALIZETIME 1 "2026-02-24" "recs 0.1.0" "RecordStream Manual" - -.SH NAME -recs\-normalizetime \- Given a single key field containing a date/time value, construct a normalized version of the value and place it into a field named \'n_\' - -.SH SYNOPSIS -.B recs normalizetime [options] [files...] - -.SH DESCRIPTION -Given a single key field containing a date/time value, construct a normalized version of the value and place it into a field named \'n_\'. Used in conjunction with recs collate to aggregate information over normalized time buckets. -.PP - -.SH OPTIONS -.TP -\fB--key\fR, \fB-k\fR \fI\fR -Key field containing the date/time value. May be a key spec. -.TP -\fB--threshold\fR, \fB-n\fR \fI