Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ jobs:

- name: Run benchmarks
id: bench
continue-on-error: true
run: |
BENCH_ARGS="--ci --markdown-file benchmark-report.md --json-file benchmark-results.json"
if [ -f perf-baseline.json ]; then
Expand All @@ -77,11 +78,9 @@ jobs:
console.log('No benchmark report found, skipping comment.');
return;
}
let body = fs.readFileSync(reportPath, 'utf-8').trim();
const body = fs.readFileSync(reportPath, 'utf-8').trim();
if (!body) return;

body += '\n\n---\n*Compared against master baseline. Threshold: 25% regression.*';

const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
Expand Down
28 changes: 28 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions man/man1/recs-decollate.1
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ Reverse of collate: takes a single record and produces multiple records using de
\fB--deaggregator\fR, \fB-d\fR \fI<deaggregators>\fR
Deaggregator specification (colon\-separated).
.TP
\fB--only\fR, \fB-o\fR
Only output deaggregated fields, excluding original record fields. Useful when you only want the expanded data, not the source record.
.TP
\fB--list-deaggregators\fR
List available deaggregators and exit.

Expand All @@ -27,6 +30,14 @@ Split the \'hosts\' field into individual \'host\' fields
.fi
.RE

Decollate and only keep deaggregated fields
.PP
.RS 4
.nf
\fBrecs decollate --only -d \'unarray,items,,item\'\fR
.fi
.RE

.SH SEE ALSO
\fBrecs\-collate\fR(1)

Expand Down
60 changes: 60 additions & 0 deletions man/man1/recs-fromxls.1
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
.TH RECS\-FROMXLS 1 "2026-02-22" "recs 0.1.0" "RecordStream Manual"

.SH NAME
recs\-fromxls \- Parse Excel files (xls, xlsx, xlsb, xlsm) into records

.SH SYNOPSIS
.B recs fromxls [options] <files...>

.SH DESCRIPTION
Parse Excel files (xls, xlsx, xlsb, xlsm) into records. By default, reads the first sheet and uses the first row as header names.
.PP

.SH OPTIONS
.TP
\fB--key\fR, \fB-k\fR \fI<keys>\fR
Comma separated list of field names. Overrides header detection.
.TP
\fB--field\fR, \fB-f\fR \fI<keys>\fR
Comma separated list of field names. Overrides header detection.
.TP
\fB--no-header\fR, \fB-n\fR
Do not treat the first row as a header. Fields will be named numerically (0, 1, 2, ...).
.TP
\fB--sheet\fR, \fB-s\fR \fI<name>\fR
Specify a sheet name to read. Defaults to the first sheet.
.TP
\fB--all-sheets\fR
Read all sheets in the workbook, adding a \'sheet\' field to each record.

.SH EXAMPLES
Read an Excel file using headers from the first row
.PP
.RS 4
.nf
\fBrecs fromxls data.xlsx\fR
.fi
.RE

Read a specific sheet without headers
.PP
.RS 4
.nf
\fBrecs fromxls --sheet \'Sheet2\' --no-header -k name,value data.xlsx\fR
.fi
.RE

Read all sheets
.PP
.RS 4
.nf
\fBrecs fromxls --all-sheets data.xlsx\fR
.fi
.RE

.SH SEE ALSO
\fBrecs\-fromcsv\fR(1)

.SH AUTHOR
Benjamin Bernard <perlhacker@benjaminbernard.com>

14 changes: 14 additions & 0 deletions man/man1/recs-multiplex.1
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ Domain language key: name=expression where the expression evaluates as a valuati
\fB--line-key\fR, \fB-L\fR \fI<key>\fR
Use the value of this key as line input for the nested operation (rather than the entire record). Use with recs from* operations generally.
.TP
\fB--output-file-key\fR, \fB-o\fR \fI<key>\fR
Write each group\'s output to a separate file, using the value of the given key as the filename.
.TP
\fB--output-file-eval\fR, \fB-O\fR \fI<expression>\fR
Write each group\'s output to a separate file, with filename determined by the given expression. Supports {{key}} interpolation with group key values.
.TP
\fB--adjacent\fR, \fB-1\fR
Only group together adjacent records. Avoids spooling records into memory.
.TP
Expand Down Expand Up @@ -59,6 +65,14 @@ Separate out a stream of text by PID into separate invocations of an operation
.fi
.RE

Write each group\'s CSV output to separate files by department
.PP
.RS 4
.nf
\fBrecs multiplex -k department -O \'output-{{department}}.csv\' -- recs tocsv\fR
.fi
.RE

.SH SEE ALSO
\fBrecs\-collate\fR(1), \fBrecs\-chain\fR(1)

Expand Down
66 changes: 66 additions & 0 deletions man/man1/recs-parsedate.1
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
.TH RECS\-PARSEDATE 1 "2026-02-22" "recs 0.1.0" "RecordStream Manual"

.SH NAME
recs\-parsedate \- Parse date/time strings from a field and output them in a normalized format

.SH SYNOPSIS
.B recs parsedate [options] [files...]

.SH DESCRIPTION
Parse date/time strings from a field and output them in a normalized format. Supports epoch seconds, ISO 8601, and custom strftime\-style format strings for both input and output.
.PP

.SH OPTIONS
.TP
\fB--key\fR, \fB-k\fR \fI<key>\fR
Key field containing the date/time string to parse.
.TP
\fB--output\fR, \fB-o\fR \fI<key>\fR
Output key for the parsed date (defaults to \'parsed_<key>\').
.TP
\fB--format\fR, \fB-f\fR \fI<format>\fR
Input format for parsing (strftime\-style). Common directives: %Y (4\-digit year), %m (month), %d (day), %H (hour 24h), %M (minute), %S (second), %b (abbreviated month name), %p (AM/PM).
.TP
\fB--output-format\fR, \fB-F\fR \fI<format>\fR
Output format (strftime\-style). Defaults to ISO 8601 if not specified.
.TP
\fB--epoch\fR, \fB-e\fR
Input date is in epoch seconds.
.TP
\fB--output-epoch\fR, \fB-E\fR
Output as epoch seconds instead of a formatted string.
.TP
\fB--timezone\fR, \fB-z\fR \fI<timezone>\fR
Timezone for output formatting (IANA name like \'America/New_York\').

.SH EXAMPLES
Parse dates and output as epoch seconds
.PP
.RS 4
.nf
\fBrecs parsedate -k date -E\fR
.fi
.RE

Parse a custom date format and reformat it
.PP
.RS 4
.nf
\fBrecs parsedate -k timestamp -f \'%d/%b/%Y:%H:%M:%S\' -F \'%Y-%m-%d %H:%M:%S\'\fR
.fi
.RE

Convert epoch seconds to ISO 8601
.PP
.RS 4
.nf
\fBrecs parsedate -k time -e -o iso_time\fR
.fi
.RE

.SH SEE ALSO
\fBrecs\-normalizetime\fR(1)

.SH AUTHOR
Benjamin Bernard <perlhacker@benjaminbernard.com>

6 changes: 6 additions & 0 deletions man/man1/recs.1
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ Runs tcpdump and puts out records, one for each packet
\fBrecs fromxferlog\fR
Each line of input (or lines of <files>) is parsed as an FTP transfer log (xferlog format) to produce an output record
.TP
\fBrecs fromxls\fR
Parse Excel files (xls, xlsx, xlsb, xlsm) into records
.TP
\fBrecs fromxml\fR
Reads either from STDIN or from the specified URIs

Expand Down Expand Up @@ -100,6 +103,9 @@ Take records, grouped together by \-\-keys, and run a separate operation instanc
\fBrecs normalizetime\fR
Given a single key field containing a date/time value, construct a normalized version of the value and place it into a field named \'n_<key>\'
.TP
\fBrecs parsedate\fR
Parse date/time strings from a field and output them in a normalized format
.TP
\fBrecs sort\fR
Sort records from input or from files
.TP
Expand Down
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
"fast-xml-parser": "^5.3.7",
"mongodb": "^7.1.0",
"openai": "^6.22.0",
"papaparse": "^5.5.3"
"papaparse": "^5.5.3",
"string-width": "^8.2.0",
"xlsx": "^0.18.5"
}
}
6 changes: 6 additions & 0 deletions src/Operation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,18 @@ export class PrinterReceiver implements RecordReceiver {
*/
export class CollectorReceiver implements RecordReceiver {
readonly records: Record[] = [];
readonly lines: string[] = [];

acceptRecord(record: Record): boolean {
this.records.push(record);
return true;
}

acceptLine(line: string): boolean {
this.lines.push(line);
return true;
}

finish(): void {
// nothing to do
}
Expand Down
4 changes: 4 additions & 0 deletions src/cli/dispatcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { FromRe } from "../operations/input/fromre.ts";
import { FromSplit } from "../operations/input/fromsplit.ts";
import { FromTcpdump } from "../operations/input/fromtcpdump.ts";
import { FromXferlog } from "../operations/input/fromxferlog.ts";
import { FromXls } from "../operations/input/fromxls.ts";
import { FromXml } from "../operations/input/fromxml.ts";

// -- Transform operations --
Expand All @@ -40,6 +41,7 @@ import { JoinOperation } from "../operations/transform/join.ts";
import { CollateOperation } from "../operations/transform/collate.ts";
import { DecollateOperation } from "../operations/transform/decollate.ts";
import { ExpandJsonOperation } from "../operations/transform/expandjson.ts";
import { ParseDateOperation } from "../operations/transform/parsedate.ts";
import { ChainOperation, registerOperationFactory } from "../operations/transform/chain.ts";
import { MultiplexOperation } from "../operations/transform/multiplex.ts";

Expand Down Expand Up @@ -75,6 +77,7 @@ const operationRegistry = new Map<string, OpConstructor>([
["fromsplit", FromSplit],
["fromtcpdump", FromTcpdump],
["fromxferlog", FromXferlog],
["fromxls", FromXls],
["fromxml", FromXml],
// Transform
["grep", GrepOperation],
Expand All @@ -94,6 +97,7 @@ const operationRegistry = new Map<string, OpConstructor>([
["join", JoinOperation],
["collate", CollateOperation],
["decollate", DecollateOperation],
["parsedate", ParseDateOperation],
["chain", ChainOperation],
["multiplex", MultiplexOperation],
// Output
Expand Down
Loading