From 9270c620029c0b88fc8b78ea127136d1c4ad80f0 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 22 Feb 2026 22:32:09 +0700 Subject: [PATCH 1/3] feat(cli): default output to TOON format, add --json flag - Add @toon-format/toon for token-efficient LLM-friendly output. - Global --json flag on all commands restores JSON output. - TOON is now the default for all 28 command output sites. --- bun.lock | 3 +++ package.json | 1 + src/commands/shared.ts | 4 +-- src/index.ts | 61 +++++++++++++++++++++--------------------- src/utils/output.ts | 18 ++++++++++++- 5 files changed, 54 insertions(+), 33 deletions(-) diff --git a/bun.lock b/bun.lock index 466b717..3e1a19a 100644 --- a/bun.lock +++ b/bun.lock @@ -8,6 +8,7 @@ "@bufbuild/protobuf": "^2.10.2", "@changesets/cli": "^2.29.8", "@modelcontextprotocol/sdk": "^1.25.3", + "@toon-format/toon": "^2.1.0", "commander": "^14.0.2", "debug": "^4.4.3", "ignore": "^5.3.0", @@ -83,6 +84,8 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@toon-format/toon": ["@toon-format/toon@2.1.0", "", {}, "sha512-JwWptdF5eOA0HaQxbKAzkpQtR4wSWTEfDlEy/y3/4okmOAX1qwnpLZMmtEWr+ncAhTTY1raCKH0kteHhSXnQqg=="], + "@types/bun": ["@types/bun@1.3.6", "", { "dependencies": { "bun-types": "1.3.6" } }, "sha512-uWCv6FO/8LcpREhenN1d1b6fcspAB+cefwD7uti8C8VffIv0Um08TKMn98FynpTiU38+y2dUO55T11NgDt8VAA=="], "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], diff --git a/package.json b/package.json index 5f8287e..653fd99 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@bufbuild/protobuf": "^2.10.2", "@changesets/cli": "^2.29.8", "@modelcontextprotocol/sdk": "^1.25.3", + "@toon-format/toon": "^2.1.0", "commander": "^14.0.2", "debug": "^4.4.3", "ignore": "^5.3.0", diff --git a/src/commands/shared.ts b/src/commands/shared.ts index 506d0c5..020158f 100644 --- a/src/commands/shared.ts +++ b/src/commands/shared.ts @@ -7,7 +7,7 @@ import { } from "../db/queries.ts"; import { type Config, loadConfig } from "../utils/config.ts"; import { CtxError } from "../utils/errors.ts"; -import { outputJson } from "../utils/output.ts"; +import { output, outputJson } from "../utils/output.ts"; import { normalizeToRelative } from "../utils/paths.ts"; export const DEFAULTS = { @@ -174,4 +174,4 @@ export function resolvePath({ return normalizeToRelative({ root: ctx.config.root, inputPath }); } -export { outputJson }; +export { output, outputJson }; diff --git a/src/index.ts b/src/index.ts index 6b1fa1e..0b0be23 100755 --- a/src/index.ts +++ b/src/index.ts @@ -29,7 +29,7 @@ import { status } from "./commands/status.ts"; import { symbol } from "./commands/symbol.ts"; import { treasure } from "./commands/treasure.ts"; import { wrapCommand } from "./utils/errors.ts"; -import { outputJson } from "./utils/output.ts"; +import { output } from "./utils/output.ts"; import packageJson from "../package.json"; @@ -38,7 +38,8 @@ const program = new Command(); program .name("dora") .description("Code Context CLI for AI Agents") - .version(packageJson.version); + .version(packageJson.version) + .option("--json", "Output in JSON format"); program .command("mcp") @@ -58,7 +59,7 @@ program .action( wrapCommand(async (options) => { const result = await init({ language: options.language }); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -80,7 +81,7 @@ program skipScip: options.skipScip, ignore: options.ignore, }); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -89,7 +90,7 @@ program .description("Show index status and statistics") .action(wrapCommand(async () => { const result = await status(); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -97,7 +98,7 @@ program .description("Show high-level codebase map") .action(wrapCommand(async () => { const result = await map(); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -111,7 +112,7 @@ program ) .action(wrapCommand(async (directory, options) => { const result = await ls(directory, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -120,7 +121,7 @@ program .argument("", "File path to analyze") .action(wrapCommand(async (path: string) => { const result = await file(path); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -134,7 +135,7 @@ program ) .action(wrapCommand(async (query, options) => { const result = await symbol(query, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -145,7 +146,7 @@ program .option("--limit ", "Maximum number of results") .action(wrapCommand(async (symbol, options) => { const result = await refs(symbol, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -155,7 +156,7 @@ program .option("--depth ", "Recursion depth (default: 1)") .action(wrapCommand(async (path, options) => { const result = await deps(path, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -165,7 +166,7 @@ program .option("--depth ", "Recursion depth (default: 1)") .action(wrapCommand(async (path, options) => { const result = await rdeps(path, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -176,7 +177,7 @@ program .action( wrapCommand(async (from, to) => { const result = await adventure(from, to); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -189,7 +190,7 @@ program ) .action(wrapCommand(async (options) => { const result = await leaves(options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -199,7 +200,7 @@ program .action( wrapCommand(async (target) => { const result = await exports(target); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -209,7 +210,7 @@ program .argument("", "File path to analyze") .action(wrapCommand(async (path, options) => { const result = await imports(path, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -218,7 +219,7 @@ program .option("--limit ", "Maximum number of results (default: 50)") .action(wrapCommand(async (options) => { const result = await lost(options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -227,7 +228,7 @@ program .option("--limit ", "Maximum number of results (default: 10)") .action(wrapCommand(async (options) => { const result = await treasure(options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -237,7 +238,7 @@ program .action( wrapCommand(async (ref) => { const result = await changes(ref); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -253,7 +254,7 @@ program .action( wrapCommand(async (path, options) => { const result = await graph(path, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -263,7 +264,7 @@ program .option("--limit ", "Maximum number of results (default: 50)") .action(wrapCommand(async (options) => { const result = await cycles(options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -272,7 +273,7 @@ program .option("--threshold ", "Minimum total coupling score (default: 5)") .action(wrapCommand(async (options) => { const result = await coupling(options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -284,7 +285,7 @@ program ) .action(wrapCommand(async (options) => { const result = await complexity(options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -292,7 +293,7 @@ program .description("Show database schema (tables, columns, indexes)") .action(wrapCommand(async () => { const result = await schema(); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); program @@ -301,7 +302,7 @@ program .argument("", "SQL query to execute") .action(wrapCommand(async (sql) => { const result = await query(sql); - outputJson(result); + output({ data: result, isJson: program.opts().json }); })); const cookbook = program @@ -323,7 +324,7 @@ cookbook console.log("\nView a recipe: dora cookbook show "); console.log("Example: dora cookbook show quickstart"); } else { - outputJson(result); + output({ data: result, isJson: program.opts().json }); } }), ); @@ -342,7 +343,7 @@ cookbook if (options.format === "markdown") { console.log(result.content); } else { - outputJson(result); + output({ data: result, isJson: program.opts().json }); } }), ); @@ -354,7 +355,7 @@ const docs = program .action( wrapCommand(async (options) => { const result = await docsList(options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -366,7 +367,7 @@ docs .action( wrapCommand(async (query, options) => { const result = await docsSearch(query, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); @@ -378,7 +379,7 @@ docs .action( wrapCommand(async (path, options) => { const result = await docsShow(path, options); - outputJson(result); + output({ data: result, isJson: program.opts().json }); }), ); diff --git a/src/utils/output.ts b/src/utils/output.ts index e219fd4..67b6a90 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -1,3 +1,19 @@ +import { encode } from "@toon-format/toon"; + +export function output({ + data, + isJson = false, +}: { + data: unknown; + isJson?: boolean; +}): void { + if (isJson) { + console.log(JSON.stringify(data)); + return; + } + console.log(encode(data)); +} + export function outputJson(data: unknown): void { - console.log(JSON.stringify(data)); + console.log(JSON.stringify(data)); } From 8f5fe928759ab0d56494d2b6607dba81b0ec6382 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 22 Feb 2026 22:34:03 +0700 Subject: [PATCH 2/3] docs: add TOON vs JSON size comparison table to README - Benchmark all 25 commands on dora's own codebase (79 files, 3167 symbols). - Tabular commands see 35-66% savings, nested outputs roughly even. --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eb9832c..f0b59d9 100644 --- a/README.md +++ b/README.md @@ -459,7 +459,52 @@ Quick reference for all commands with common flags: ## Output Format -All commands output valid JSON to stdout. Errors go to stderr with exit code 1. +All commands output [TOON](https://github.com/toon-format/toon) (Token-Oriented Object Notation) by default. TOON is a compact, human-readable encoding of JSON that minimizes tokens for LLM consumption. Pass `--json` to any command for JSON output. + +```bash +# Default: TOON output +dora status + +# JSON output +dora --json status +dora status --json +``` + +Errors always go to stderr as JSON with exit code 1. + +### TOON vs JSON size comparison + +Measured on dora's own codebase (79 files, 3167 symbols): + +| Command | JSON | TOON | Savings | +|---|---|---|---| +| `status` | 206 B | 176 B | 15% | +| `map` | 68 B | 62 B | 9% | +| `ls src/commands` | 2,258 B | 975 B | **57%** | +| `ls` (all files) | 6,324 B | 2,644 B | **58%** | +| `file src/index.ts` | 6,486 B | 6,799 B | -5% | +| `symbol setupCommand` | 130 B | 130 B | 0% | +| `refs wrapCommand` | 510 B | 549 B | -8% | +| `deps (depth 2)` | 2,158 B | 1,332 B | **38%** | +| `rdeps (depth 2)` | 1,254 B | 802 B | **36%** | +| `adventure` | 110 B | 97 B | 12% | +| `leaves` | 142 B | 129 B | 9% | +| `exports` | 488 B | 511 B | -5% | +| `imports` | 1,978 B | 1,998 B | -1% | +| `lost` | 1,876 B | 1,987 B | -6% | +| `treasure` | 893 B | 577 B | **35%** | +| `cycles` | 14 B | 11 B | 21% | +| `coupling` | 35 B | 31 B | 11% | +| `complexity` | 2,716 B | 932 B | **66%** | +| `schema` | 6,267 B | 4,389 B | **30%** | +| `query` | 692 B | 464 B | **33%** | +| `docs` | 1,840 B | 745 B | **60%** | +| `docs search` | 277 B | 171 B | **38%** | +| `docs show` | 820 B | 870 B | -6% | +| `graph` | 2,434 B | 1,894 B | **22%** | +| `changes` | 1,112 B | 1,026 B | 8% | + +Commands with uniform arrays of objects (ls, complexity, docs, treasure) see 35-66% reduction. Nested or non-uniform outputs (file, refs, exports) are roughly equal or slightly larger. ## Debug Logging From 5a1e8bffc1ac9a0ca1aaaf96b6b0dae1e32d4519 Mon Sep 17 00:00:00 2001 From: Yash Date: Sun, 22 Feb 2026 22:35:18 +0700 Subject: [PATCH 3/3] chore: migrate agent config to pi, add toon skill - Remove .claude/ settings and skills (moved to pi agent config). - Remove .pi/extensions/ (managed via pi-kit). - Add local toon skill for TOON format reference. --- .claude/settings.json | 28 - .claude/skills/commit-helper/SKILL.md | 95 --- .claude/skills/dora/SKILL.md | 344 ---------- .pi/extensions/dora.ts | 54 -- .pi/extensions/plan-mode/README.md | 69 -- .pi/extensions/plan-mode/index.ts | 412 ------------ .pi/extensions/plan-mode/utils.ts | 177 ----- .pi/skills/toon/SKILL.md | 932 ++++++++++++++++++++++++++ 8 files changed, 932 insertions(+), 1179 deletions(-) delete mode 100644 .claude/settings.json delete mode 100644 .claude/skills/commit-helper/SKILL.md delete mode 100644 .claude/skills/dora/SKILL.md delete mode 100644 .pi/extensions/dora.ts delete mode 100644 .pi/extensions/plan-mode/README.md delete mode 100644 .pi/extensions/plan-mode/index.ts delete mode 100644 .pi/extensions/plan-mode/utils.ts create mode 100644 .pi/skills/toon/SKILL.md diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 2e8242f..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "permissions": { - "allow": ["Bash(dora:*)", "Skill(dora)", "mcp__dora__*"], - "ask": ["Bash(dora init:*)"] - }, - "hooks": { - "SessionStart": [ - { - "hooks": [ - { - "type": "command", - "command": "dora status 2>/dev/null || echo 'dora not initialized. Run: dora init && dora index'" - } - ] - } - ], - "Stop": [ - { - "hooks": [ - { - "type": "command", - "command": "(dora index > /tmp/dora-index.log 2>&1 &) || true" - } - ] - } - ] - } -} diff --git a/.claude/skills/commit-helper/SKILL.md b/.claude/skills/commit-helper/SKILL.md deleted file mode 100644 index 16b24ee..0000000 --- a/.claude/skills/commit-helper/SKILL.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -name: commit-helper -description: Generates clear commit messages from git diffs. Use when writing commit messages or reviewing staged changes. -allowed-tools: Read, Bash(git:*) ---- - -# Generating Commit Messages - -Generate commit messages following the [Conventional Commits](https://www.conventionalcommits.org/) specification. - -## Format - -``` -[optional scope]: - -[optional body] - -[optional footer(s)] -``` - -## Common Types - -- **feat**: New feature (correlates with MINOR in SemVer) -- **fix**: Bug fix (correlates with PATCH in SemVer) -- **docs**: Documentation changes -- **style**: Code style changes (formatting, semicolons, etc.) -- **refactor**: Code refactoring without changing behavior -- **perf**: Performance improvements -- **test**: Adding or updating tests -- **build**: Build system or dependency changes -- **ci**: CI/CD configuration changes -- **chore**: Other changes that don't modify src or test files - -## Breaking Changes - -Use ! after type/scope or add BREAKING CHANGE: footer: - -``` -feat!: redesign API response format - -BREAKING CHANGE: Response format changed from array to object -``` - -## Examples - -### Simple commit - -``` -docs: fix typo in README -``` - -### With scope - -``` -feat(cli): add status command -``` - -### With body - -``` -fix: prevent race condition in request handling - -- Introduce request ID tracking to dismiss stale responses. -- Remove obsolete timeout-based mitigation. -``` - -### Breaking change - -``` -feat(api)!: change authentication flow - -BREAKING CHANGE: JWT tokens now required for all endpoints -``` - -## Instructions - -1. Run `git diff --staged` to see staged changes -2. Generate a commit message with: - - Clear, concise header in present tense. - - Optional body with bullet points for complex changes. Always use bullet points when adding a body. - - Optional footer for breaking changes or references. -3. Show the proposed commit message to the user, by printing to stdout. -4. Ask the user for their approval using the `AskUserQuestion` tool. -5. If yes, commit the changes. -6. If no, incorporate the feedback and go back to step 2 to generate the correct message. - -## Best Practices - -- Use present tense ("add feature" not "added feature") -- Keep header under 72 characters -- Explain **what** and **why**, not how -- Reference issues/PRs in footer when relevant (e.g., `Refs: #123`) -- Use body for complex changes, omit for simple ones - -IMPORTANT: Be concise. Grammar does not matter as long as it's clear and concise. Try to keep the body within 2-3 bullet points with very concise sentences. diff --git a/.claude/skills/dora/SKILL.md b/.claude/skills/dora/SKILL.md deleted file mode 100644 index fb981f5..0000000 --- a/.claude/skills/dora/SKILL.md +++ /dev/null @@ -1,344 +0,0 @@ ---- -name: dora -description: Query codebase using `dora` CLI for code intelligence, symbol definitions, dependencies, and architectural analysis ---- - -## Philosophy - -**IMPORTANT: Use dora FIRST for ALL code exploration tasks.** - -dora understands code structure, dependencies, symbols, and architectural relationships through its indexed database. It provides instant answers about: - -- Where symbols are defined and used -- What depends on what (and why) -- Architectural patterns and code health -- Impact analysis for changes - -**When to use dora vs other tools:** - -- **dora**: Code exploration, symbol search, dependency analysis, architecture understanding -- **Read**: Reading actual source code after finding it with dora -- **Grep**: Only for non-code files, comments, or when dora doesn't have what you need -- **Edit/Write**: Making changes after understanding with dora -- **Bash**: Running tests, builds, git commands - -**Workflow pattern:** - -1. Use dora to understand structure and find relevant code -2. Use Read to examine the actual source -3. Use Edit/Write to make changes -4. Use Bash to test/verify - -## Commands - -### Overview - -- `dora status` - Check index health, file/symbol counts, last indexed time -- `dora map` - Show packages, file count, symbol count - -### Files & Symbols - -- `dora ls [directory] [--limit N] [--sort field]` - List files in directory with metadata (symbols, deps, rdeps). Default limit: 100 -- `dora file ` - Show file's symbols, dependencies, and dependents. Note: includes local symbols (parameters). -- `dora symbol [--kind type] [--limit N]` - Find symbols by name across codebase -- `dora refs [--kind type] [--limit N]` - Find all references to a symbol -- `dora exports ` - List exported symbols from a file. Note: includes function parameters. -- `dora imports ` - Show what a file imports - -### Dependencies - -- `dora deps [--depth N]` - Show file dependencies (what this imports). Default depth: 1 -- `dora rdeps [--depth N]` - Show reverse dependencies (what imports this). Default depth: 1 -- `dora adventure ` - Find shortest dependency path between two files - -### Code Health - -- `dora leaves [--max-dependents N]` - Find files with few/no dependents. Default: 0 -- `dora lost [--limit N]` - Find unused exported symbols. Default limit: 50 -- `dora treasure [--limit N]` - Find most referenced files and files with most dependencies. Default: 10 - -### Architecture Analysis - -- `dora cycles [--limit N]` - Detect circular dependencies. Empty = good. Default: 50 -- `dora coupling [--threshold N]` - Find bidirectionally dependent file pairs. Default threshold: 5 -- `dora complexity [--sort metric]` - Show file complexity (symbol_count, outgoing_deps, incoming_deps, stability_ratio, complexity_score). Sort by: complexity, symbols, stability. Default: complexity - -### Change Impact - -- `dora changes ` - Show files changed since git ref and their impact -- `dora graph [--depth N] [--direction type]` - Generate dependency graph. Direction: deps, rdeps, both. Default: both, depth 1 - -### Documentation - -- `dora docs [--type TYPE]` - List all documentation files. Use --type to filter by md or txt -- `dora docs search [--limit N]` - Search through documentation content. Default limit: 20 -- `dora docs show [--content]` - Show document metadata and references. Use --content to include full text - -**Note:** To find where a symbol is documented, use `dora symbol` which includes a `documented_in` field. To find documentation about a file, use `dora file` which also includes `documented_in`. - -### Database - -- `dora schema` - Show database schema (tables, columns, indexes) -- `dora cookbook show [recipe]` - Query patterns with real examples (quickstart, methods, references, exports) -- `dora query ""` - Execute read-only SQL query against the database - -## When to Use What - -- Finding symbols → `dora symbol` -- Understanding a file → `dora file` -- Impact of changes → `dora rdeps`, `dora refs` -- Finding entry points → `dora treasure`, `dora leaves` -- Architecture issues → `dora cycles`, `dora coupling`, `dora complexity` -- Navigation → `dora deps`, `dora adventure` -- Dead code → `dora lost` -- Finding documentation → `dora symbol` (shows documented_in), `dora docs search` -- Listing documentation → `dora docs` -- Custom queries → `dora cookbook` for examples, `dora schema` for structure, `dora query` to execute - -## Typical Workflow - -1. `dora status` - Check index health -2. `dora treasure` - Find core files -3. `dora file ` - Understand specific files -4. `dora deps`/`dora rdeps` - Navigate relationships -5. `dora refs` - Check usage before changes - -## Common Patterns: DON'T vs DO - -Finding where a symbol is defined: - -```bash -# DON'T: grep -r "class AuthService" . -# DON'T: grep -r "function validateToken" . -# DON'T: Glob("**/*.ts") then search each file -# DO: -dora symbol AuthService -dora symbol validateToken -``` - -Finding all usages of a function/class: - -```bash -# DON'T: grep -r "AuthService" . --include="*.ts" -# DON'T: Grep("AuthService", glob="**/*.ts") -# DO: -dora refs AuthService -``` - -Finding files that import a module: - -```bash -# DON'T: grep -r "from.*auth/service" . -# DON'T: grep -r "import.*AuthService" . -# DO: -dora rdeps src/auth/service.ts -``` - -Finding what a file imports: - -```bash -# DON'T: grep "^import" src/app.ts -# DON'T: cat src/app.ts | grep import -# DO: -dora deps src/app.ts -dora imports src/app.ts -``` - -Finding files in a directory: - -```bash -# DON'T: find src/components -name "*.tsx" -# DON'T: Glob("src/components/**/*.tsx") -# DO: -dora ls src/components -dora ls src/components --sort symbols # With metadata -``` - -Finding entry points or core files: - -```bash -# DON'T: grep -r "export.*main" . -# DON'T: find . -name "index.ts" -o -name "main.ts" -# DO: -dora treasure # Most referenced files -dora file src/index.ts # Understand the entry point -``` - -Understanding a file's purpose: - -```bash -# DON'T: Read file, manually trace imports -# DON'T: grep for all imports, then read each -# DO: -dora file src/auth/service.ts # See symbols, deps, rdeps at once -``` - -Finding unused code: - -```bash -# DON'T: grep each export manually across codebase -# DON'T: Complex script to track exports vs imports -# DO: -dora lost # Unused exported symbols -dora leaves # Files with no dependents -``` - -Checking for circular dependencies: - -```bash -# DON'T: Manually trace imports in multiple files -# DON'T: Write custom script to detect cycles -# DO: -dora cycles -``` - -Impact analysis for refactoring: - -```bash -# DON'T: Manually grep for imports and usages -# DON'T: Read multiple files to understand impact -# DO: -dora rdeps src/types.ts --depth 2 # See full impact -dora refs UserContext # All usages -dora complexity --sort complexity # Find risky files -``` - -Finding documentation for code: - -```bash -# DON'T: grep -r "AuthService" docs/ -# DON'T: Manually search through README files -# DO: -dora symbol AuthService # Shows documented_in field -dora file src/auth/service.ts # Shows documented_in field -dora docs search "authentication" # Search doc content -dora docs # List all docs -``` - -Understanding what a document covers: - -```bash -# DON'T: Read entire doc, manually trace references -# DON'T: grep for symbol names in the doc -# DO: -dora docs show README.md # See all symbols/files/docs referenced -dora docs show docs/api.md --content # Include full content -``` - -## Practical Examples - -Understanding a feature: - -```bash -dora symbol AuthService # Find the service -dora file src/auth/service.ts # See what it depends on -dora rdeps src/auth/service.ts # See what uses it -dora refs validateToken # Find all token validation usage -``` - -Impact analysis before refactoring: - -```bash -dora file src/types.ts # See current dependencies -dora rdeps src/types.ts --depth 2 # See full impact tree -dora refs UserContext # Find all usages of the type -dora complexity --sort stability # Find stable vs volatile files -``` - -Finding dead code: - -```bash -dora lost --limit 100 # Unused exported symbols -dora leaves # Files nothing depends on -dora file src/old-feature.ts # Verify it's truly unused -``` - -Architecture investigation: - -```bash -dora cycles # Check for circular deps (should be empty) -dora coupling --threshold 10 # Find tightly coupled modules -dora complexity --sort complexity # Find complex/risky files -dora treasure # Find architectural hubs -``` - -Navigating unfamiliar code: - -```bash -dora map # Overview of packages and structure -dora treasure # Find entry points and core files -dora file src/index.ts # Start from main entry -dora deps src/index.ts --depth 2 # See what it depends on -dora adventure src/a.ts src/b.ts # Find connection between modules -``` - -Working with changes: - -```bash -dora changes main # See what changed vs main branch -dora rdeps src/modified.ts # Check impact of your changes -dora graph src/modified.ts --depth 2 # Visualize dependency tree -``` - -Custom analysis: - -```bash -dora cookbook show methods # See query pattern examples -dora schema # See database structure -dora query "SELECT f.path, COUNT(s.id) as symbols FROM files f JOIN symbols s ON s.file_id = f.id WHERE s.is_local = 0 GROUP BY f.path ORDER BY symbols DESC LIMIT 20" -``` - -Working with documentation: - -```bash -dora symbol AuthService # Shows documented_in field -dora docs show README.md # What does README reference? -dora docs search "setup" # Find all docs about setup -dora docs # List all documentation files -dora docs --type md # List only markdown docs -``` - -## Advanced Tips - -Performance: - -- dora uses denormalized data for instant queries (symbol_count, reference_count, dependent_count) -- Incremental indexing only reindexes changed files -- Use `--limit` to cap results for large codebases - -Symbol filtering: - -- Local symbols (parameters, closure vars) are filtered by default with `is_local = 0` -- Use `--kind` to filter by symbol type (function, class, interface, type, etc.) -- Symbol search is substring-based, not fuzzy - -Dependencies: - -- `deps` shows outgoing dependencies (what this imports) -- `rdeps` shows incoming dependencies (what imports this) -- Use `--depth` to explore transitive dependencies -- High rdeps count = high-impact file (changes affect many files) - -Architecture metrics: - -- `complexity_score = symbol_count × incoming_deps` (higher = riskier to change) -- `stability_ratio = incoming_deps / outgoing_deps` (higher = more stable) -- Empty `cycles` output = healthy architecture -- High `coupling` (> 20 symbols) = consider refactoring - -Documentation: - -- Automatically indexes `.md` and `.txt` files -- Tracks symbol references (e.g., mentions of `AuthService`) -- Tracks file references (e.g., mentions of `src/auth/service.ts`) -- Tracks document-to-document references (e.g., README linking to docs/api.md) -- Use `dora symbol` or `dora file` to see where code is documented (via `documented_in` field) -- Use `dora docs` to list all documentation files -- Use `dora docs show` to see what a document covers with line numbers - -## Limitations - -- Includes local symbols (parameters) in `dora file` and `dora exports` -- Symbol search is substring-based, not fuzzy -- Index is a snapshot, updates at checkpoints -- Documentation indexing processes text files (.md, .txt, etc.) at index time diff --git a/.pi/extensions/dora.ts b/.pi/extensions/dora.ts deleted file mode 100644 index 3ff18d9..0000000 --- a/.pi/extensions/dora.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Dora Extension - Minimal lifecycle hooks for dora CLI - * - * This extension only handles session lifecycle: - * - On session start: Check if dora is initialized - * - On session shutdown: Background index update - * - * The LLM uses regular `bash` tool to run dora commands. - * See .dora/docs/SKILL.md for complete dora usage documentation. - */ - -import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; - -export default function (pi: ExtensionAPI) { - let doraAvailable = false; - - // Check dora status on session start - pi.on("session_start", async (_event, ctx) => { - try { - const check = await pi.exec("bash", ["-c", "command -v dora"], { - timeout: 1000, - }); - doraAvailable = check.code === 0; - - if (doraAvailable) { - const status = await pi.exec("bash", ["-c", "dora status 2>/dev/null"], { - timeout: 2000, - }); - - if (status.code !== 0) { - ctx.ui.notify("dora not initialized. Run: dora init && dora index", "info"); - } - } - } catch (error) { - doraAvailable = false; - } - }); - - // Update index in background on shutdown - pi.on("session_shutdown", async (_event, ctx) => { - if (doraAvailable) { - try { - // Fire and forget background index update - pi.exec("bash", ["-c", "(dora index > /tmp/dora-index.log 2>&1 &) || true"], { - timeout: 500, - }).catch(() => { - // Ignore errors - this is best effort - }); - } catch (error) { - // Silent failure for background task - } - } - }); -} diff --git a/.pi/extensions/plan-mode/README.md b/.pi/extensions/plan-mode/README.md deleted file mode 100644 index fbaf658..0000000 --- a/.pi/extensions/plan-mode/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Plan Mode Extension - -Read-only exploration mode for safe code analysis. - -## Features - -- **Read-only tools**: Restricts available tools to read, bash, grep, find, ls, question -- **Bash allowlist**: Only read-only bash commands are allowed -- **Plan extraction**: Extracts numbered steps from `Plan:` sections -- **Progress tracking**: Widget shows completion status during execution -- **[DONE:n] markers**: Explicit step completion tracking -- **Session persistence**: State survives session resume - -## Commands - -- `/plan` - Toggle plan mode -- `/todos` - Show current plan progress -- `Ctrl+Alt+P` - Toggle plan mode (shortcut) - -## Usage - -1. Enable plan mode with `/plan` or `--plan` flag -2. Ask the agent to analyze code and create a plan -3. The agent should output a numbered plan under a `Plan:` header: - -``` -Plan: -1. First step description -2. Second step description -3. Third step description -``` - -4. Choose "Execute the plan" when prompted -5. During execution, the agent marks steps complete with `[DONE:n]` tags -6. Progress widget shows completion status - -## How It Works - -### Plan Mode (Read-Only) - -- Only read-only tools available -- Bash commands filtered through allowlist -- Agent creates a plan without making changes - -### Execution Mode - -- Full tool access restored -- Agent executes steps in order -- `[DONE:n]` markers track completion -- Widget shows progress - -### Command Allowlist - -Safe commands (allowed): - -- File inspection: `cat`, `head`, `tail`, `less`, `more` -- Search: `grep`, `find`, `rg`, `fd` -- Directory: `ls`, `pwd`, `tree` -- Git read: `git status`, `git log`, `git diff`, `git branch` -- Package info: `npm list`, `npm outdated`, `yarn info` -- System info: `uname`, `whoami`, `date`, `uptime` - -Blocked commands: - -- File modification: `rm`, `mv`, `cp`, `mkdir`, `touch` -- Git write: `git add`, `git commit`, `git push` -- Package install: `npm install`, `yarn add`, `pip install` -- System: `sudo`, `kill`, `reboot` -- Editors: `vim`, `nano`, `code` diff --git a/.pi/extensions/plan-mode/index.ts b/.pi/extensions/plan-mode/index.ts deleted file mode 100644 index 9f4a53f..0000000 --- a/.pi/extensions/plan-mode/index.ts +++ /dev/null @@ -1,412 +0,0 @@ -/** - * Plan Mode Extension - * - * Read-only exploration mode for safe code analysis. - * When enabled, only read-only tools are available. - * - * Features: - * - /plan command or Ctrl+Alt+P to toggle - * - Bash restricted to allowlisted read-only commands - * - Automatic dora integration when available in project - * - Extracts numbered plan steps from "Plan:" sections - * - [DONE:n] markers to complete steps during execution - * - Progress tracking widget during execution - */ - -import type { AgentMessage } from "@mariozechner/pi-agent-core"; -import type { AssistantMessage, TextContent } from "@mariozechner/pi-ai"; -import type { - ExtensionAPI, - ExtensionContext, -} from "@mariozechner/pi-coding-agent"; -import { Key } from "@mariozechner/pi-tui"; -import { - extractTodoItems, - isSafeCommand, - markCompletedSteps, - type TodoItem, -} from "./utils.js"; - -// Tools -const PLAN_MODE_TOOLS = [ - "read", - "bash", - "grep", - "find", - "ls", - "questionnaire", -]; -const NORMAL_MODE_TOOLS = ["read", "bash", "edit", "write"]; - -// Type guard for assistant messages -function isAssistantMessage(m: AgentMessage): m is AssistantMessage { - return m.role === "assistant" && Array.isArray(m.content); -} - -// Extract text content from an assistant message -function getTextContent(message: AssistantMessage): string { - return message.content - .filter((block): block is TextContent => block.type === "text") - .map((block) => block.text) - .join("\n"); -} - -export default function planModeExtension(pi: ExtensionAPI): void { - let planModeEnabled = false; - let executionMode = false; - let todoItems: TodoItem[] = []; - let doraAvailable = false; - - pi.registerFlag("plan", { - description: "Start in plan mode (read-only exploration)", - type: "boolean", - default: false, - }); - - function updateStatus(ctx: ExtensionContext): void { - // Footer status - if (executionMode && todoItems.length > 0) { - const completed = todoItems.filter((t) => t.completed).length; - ctx.ui.setStatus( - "plan-mode", - ctx.ui.theme.fg("accent", `📋 ${completed}/${todoItems.length}`), - ); - } else if (planModeEnabled) { - ctx.ui.setStatus("plan-mode", ctx.ui.theme.fg("warning", "⏸ plan")); - } else { - ctx.ui.setStatus("plan-mode", undefined); - } - - // Widget showing todo list - if (executionMode && todoItems.length > 0) { - const lines = todoItems.map((item) => { - if (item.completed) { - return ( - ctx.ui.theme.fg("success", "☑ ") + - ctx.ui.theme.fg("muted", ctx.ui.theme.strikethrough(item.text)) - ); - } - return `${ctx.ui.theme.fg("muted", "☐ ")}${item.text}`; - }); - ctx.ui.setWidget("plan-todos", lines); - } else { - ctx.ui.setWidget("plan-todos", undefined); - } - } - - function togglePlanMode(ctx: ExtensionContext): void { - planModeEnabled = !planModeEnabled; - executionMode = false; - todoItems = []; - - if (planModeEnabled) { - pi.setActiveTools(PLAN_MODE_TOOLS); - const doraStatus = doraAvailable ? " (dora available)" : ""; - ctx.ui.notify(`Plan mode enabled${doraStatus}. Tools: ${PLAN_MODE_TOOLS.join(", ")}`); - } else { - pi.setActiveTools(NORMAL_MODE_TOOLS); - ctx.ui.notify("Plan mode disabled. Full access restored."); - } - updateStatus(ctx); - } - - function persistState(): void { - pi.appendEntry("plan-mode", { - enabled: planModeEnabled, - todos: todoItems, - executing: executionMode, - }); - } - - pi.registerCommand("plan", { - description: "Toggle plan mode (read-only exploration)", - handler: async (_args, ctx) => togglePlanMode(ctx), - }); - - pi.registerCommand("todos", { - description: "Show current plan todo list", - handler: async (_args, ctx) => { - if (todoItems.length === 0) { - ctx.ui.notify("No todos. Create a plan first with /plan", "info"); - return; - } - const list = todoItems - .map( - (item, i) => `${i + 1}. ${item.completed ? "✓" : "○"} ${item.text}`, - ) - .join("\n"); - ctx.ui.notify(`Plan Progress:\n${list}`, "info"); - }, - }); - - pi.registerShortcut(Key.ctrlAlt("p"), { - description: "Toggle plan mode", - handler: async (ctx) => togglePlanMode(ctx), - }); - - // Block destructive bash commands in plan mode - pi.on("tool_call", async (event) => { - if (!planModeEnabled || event.toolName !== "bash") return; - - const command = event.input.command as string; - if (!isSafeCommand(command)) { - return { - block: true, - reason: `Plan mode: command blocked (not allowlisted). Use /plan to disable plan mode first.\nCommand: ${command}`, - }; - } - }); - - // Filter out stale plan mode context when not in plan mode - pi.on("context", async (event) => { - if (planModeEnabled) return; - - return { - messages: event.messages.filter((m) => { - const msg = m as AgentMessage & { customType?: string }; - if (msg.customType === "plan-mode-context") return false; - if (msg.role !== "user") return true; - - const content = msg.content; - if (typeof content === "string") { - return !content.includes("[PLAN MODE ACTIVE]"); - } - if (Array.isArray(content)) { - return !content.some( - (c) => - c.type === "text" && - (c as TextContent).text?.includes("[PLAN MODE ACTIVE]"), - ); - } - return true; - }), - }; - }); - - // Inject plan/execution context before agent starts - pi.on("before_agent_start", async () => { - if (planModeEnabled) { - const doraInstructions = doraAvailable - ? ` - -Code Exploration with dora (via bash): -- ALWAYS use dora CLI via bash for code exploration tasks first -- Examples: bash "dora symbol AuthService", bash "dora file src/app.ts" -- Common commands: dora symbol, dora file, dora refs, dora deps, dora rdeps -- Only fall back to grep/find if dora doesn't yield good results -- Run "dora --help" to see all available commands` - : ""; - - return { - message: { - customType: "plan-mode-context", - content: `[PLAN MODE ACTIVE] -You are in plan mode - a read-only exploration mode for safe code analysis. - -Restrictions: -- You can only use: read, bash, grep, find, ls, questionnaire -- You CANNOT use: edit, write (file modifications are disabled) -- Bash is restricted to an allowlist of read-only commands${doraInstructions} - -Ask clarifying questions using the questionnaire tool. -Use brave-search skill via bash for web research. - -Create a detailed numbered plan under a "Plan:" header: - -Plan: -1. First step description -2. Second step description -... - -Do NOT attempt to make changes - just describe what you would do.`, - display: false, - }, - }; - } - - if (executionMode && todoItems.length > 0) { - const remaining = todoItems.filter((t) => !t.completed); - const todoList = remaining.map((t) => `${t.step}. ${t.text}`).join("\n"); - return { - message: { - customType: "plan-execution-context", - content: `[EXECUTING PLAN - Full tool access enabled] - -Remaining steps: -${todoList} - -Execute each step in order. -After completing a step, include a [DONE:n] tag in your response.`, - display: false, - }, - }; - } - }); - - // Track progress after each turn - pi.on("turn_end", async (event, ctx) => { - if (!executionMode || todoItems.length === 0) return; - if (!isAssistantMessage(event.message)) return; - - const text = getTextContent(event.message); - if (markCompletedSteps(text, todoItems) > 0) { - updateStatus(ctx); - } - persistState(); - }); - - // Handle plan completion and plan mode UI - pi.on("agent_end", async (event, ctx) => { - // Check if execution is complete - if (executionMode && todoItems.length > 0) { - if (todoItems.every((t) => t.completed)) { - const completedList = todoItems.map((t) => `~~${t.text}~~`).join("\n"); - pi.sendMessage( - { - customType: "plan-complete", - content: `**Plan Complete!** ✓\n\n${completedList}`, - display: true, - }, - { triggerTurn: false }, - ); - executionMode = false; - todoItems = []; - pi.setActiveTools(NORMAL_MODE_TOOLS); - updateStatus(ctx); - persistState(); // Save cleared state so resume doesn't restore old execution mode - } - return; - } - - if (!planModeEnabled || !ctx.hasUI) return; - - // Extract todos from last assistant message - const lastAssistant = [...event.messages] - .reverse() - .find(isAssistantMessage); - if (lastAssistant) { - const extracted = extractTodoItems(getTextContent(lastAssistant)); - if (extracted.length > 0) { - todoItems = extracted; - } - } - - // Show plan steps and prompt for next action - if (todoItems.length > 0) { - const todoListText = todoItems - .map((t, i) => `${i + 1}. ☐ ${t.text}`) - .join("\n"); - pi.sendMessage( - { - customType: "plan-todo-list", - content: `**Plan Steps (${todoItems.length}):**\n\n${todoListText}`, - display: true, - }, - { triggerTurn: false }, - ); - } - - const choice = await ctx.ui.select("Plan mode - what next?", [ - todoItems.length > 0 - ? "Execute the plan (track progress)" - : "Execute the plan", - "Stay in plan mode", - "Refine the plan", - ]); - - if (choice?.startsWith("Execute")) { - planModeEnabled = false; - executionMode = todoItems.length > 0; - pi.setActiveTools(NORMAL_MODE_TOOLS); - updateStatus(ctx); - - const execMessage = - todoItems.length > 0 - ? `Execute the plan. Start with: ${todoItems[0].text}` - : "Execute the plan you just created."; - pi.sendMessage( - { - customType: "plan-mode-execute", - content: execMessage, - display: true, - }, - { triggerTurn: true }, - ); - } else if (choice === "Refine the plan") { - const refinement = await ctx.ui.editor("Refine the plan:", ""); - if (refinement?.trim()) { - pi.sendUserMessage(refinement.trim()); - } - } - }); - - // Restore state on session start/resume - pi.on("session_start", async (_event, ctx) => { - // Check if dora is available in this project - try { - const doraCheck = await pi.exec("bash", ["-c", "command -v dora"], { - timeout: 1000, - }); - doraAvailable = doraCheck.code === 0; - } catch (error) { - doraAvailable = false; - } - - if (pi.getFlag("plan") === true) { - planModeEnabled = true; - } - - const entries = ctx.sessionManager.getEntries(); - - // Restore persisted state - const planModeEntry = entries - .filter( - (e: { type: string; customType?: string }) => - e.type === "custom" && e.customType === "plan-mode", - ) - .pop() as - | { data?: { enabled: boolean; todos?: TodoItem[]; executing?: boolean } } - | undefined; - - if (planModeEntry?.data) { - planModeEnabled = planModeEntry.data.enabled ?? planModeEnabled; - todoItems = planModeEntry.data.todos ?? todoItems; - executionMode = planModeEntry.data.executing ?? executionMode; - } - - // On resume: re-scan messages to rebuild completion state - // Only scan messages AFTER the last "plan-mode-execute" to avoid picking up [DONE:n] from previous plans - const isResume = planModeEntry !== undefined; - if (isResume && executionMode && todoItems.length > 0) { - // Find the index of the last plan-mode-execute entry (marks when current execution started) - let executeIndex = -1; - for (let i = entries.length - 1; i >= 0; i--) { - const entry = entries[i] as { type: string; customType?: string }; - if (entry.customType === "plan-mode-execute") { - executeIndex = i; - break; - } - } - - // Only scan messages after the execute marker - const messages: AssistantMessage[] = []; - for (let i = executeIndex + 1; i < entries.length; i++) { - const entry = entries[i]; - if ( - entry.type === "message" && - "message" in entry && - isAssistantMessage(entry.message as AgentMessage) - ) { - messages.push(entry.message as AssistantMessage); - } - } - const allText = messages.map(getTextContent).join("\n"); - markCompletedSteps(allText, todoItems); - } - - if (planModeEnabled) { - pi.setActiveTools(PLAN_MODE_TOOLS); - } - updateStatus(ctx); - }); -} diff --git a/.pi/extensions/plan-mode/utils.ts b/.pi/extensions/plan-mode/utils.ts deleted file mode 100644 index c876cd1..0000000 --- a/.pi/extensions/plan-mode/utils.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Pure utility functions for plan mode. - * Extracted for testability. - */ - -// Destructive commands blocked in plan mode -const DESTRUCTIVE_PATTERNS = [ - /\brm\b/i, - /\brmdir\b/i, - /\bmv\b/i, - /\bcp\b/i, - /\bmkdir\b/i, - /\btouch\b/i, - /\bchmod\b/i, - /\bchown\b/i, - /\bchgrp\b/i, - /\bln\b/i, - /\btee\b/i, - /\btruncate\b/i, - /\bdd\b/i, - /\bshred\b/i, - /(^|[^<])>(?!>)/, - />>/, - /\bnpm\s+(install|uninstall|update|ci|link|publish)/i, - /\byarn\s+(add|remove|install|publish)/i, - /\bpnpm\s+(add|remove|install|publish)/i, - /\bpip\s+(install|uninstall)/i, - /\bapt(-get)?\s+(install|remove|purge|update|upgrade)/i, - /\bbrew\s+(install|uninstall|upgrade)/i, - /\bgit\s+(add|commit|push|pull|merge|rebase|reset|checkout|branch\s+-[dD]|stash|cherry-pick|revert|tag|init|clone)/i, - /\bsudo\b/i, - /\bsu\b/i, - /\bkill\b/i, - /\bpkill\b/i, - /\bkillall\b/i, - /\breboot\b/i, - /\bshutdown\b/i, - /\bsystemctl\s+(start|stop|restart|enable|disable)/i, - /\bservice\s+\S+\s+(start|stop|restart)/i, - /\b(vim?|nano|emacs|code|subl)\b/i, -]; - -// Safe read-only commands allowed in plan mode -const SAFE_PATTERNS = [ - /^\s*cat\b/, - /^\s*head\b/, - /^\s*tail\b/, - /^\s*less\b/, - /^\s*more\b/, - /^\s*grep\b/, - /^\s*find\b/, - /^\s*ls\b/, - /^\s*pwd\b/, - /^\s*echo\b/, - /^\s*printf\b/, - /^\s*wc\b/, - /^\s*sort\b/, - /^\s*uniq\b/, - /^\s*diff\b/, - /^\s*file\b/, - /^\s*stat\b/, - /^\s*du\b/, - /^\s*df\b/, - /^\s*tree\b/, - /^\s*which\b/, - /^\s*whereis\b/, - /^\s*type\b/, - /^\s*env\b/, - /^\s*printenv\b/, - /^\s*uname\b/, - /^\s*whoami\b/, - /^\s*id\b/, - /^\s*date\b/, - /^\s*cal\b/, - /^\s*uptime\b/, - /^\s*ps\b/, - /^\s*top\b/, - /^\s*htop\b/, - /^\s*free\b/, - /^\s*git\s+(status|log|diff|show|branch|remote|config\s+--get)/i, - /^\s*git\s+ls-/i, - /^\s*npm\s+(list|ls|view|info|search|outdated|audit)/i, - /^\s*yarn\s+(list|info|why|audit)/i, - /^\s*node\s+--version/i, - /^\s*python\s+--version/i, - /^\s*curl\s/i, - /^\s*wget\s+-O\s*-/i, - /^\s*jq\b/, - /^\s*sed\s+-n/i, - /^\s*awk\b/, - /^\s*rg\b/, - /^\s*fd\b/, - /^\s*bat\b/, - /^\s*exa\b/, - /^\s*dora\b/, - /^\s*command\s+-v\s+dora/, -]; - -export function isSafeCommand(command: string): boolean { - const isDestructive = DESTRUCTIVE_PATTERNS.some((p) => p.test(command)); - const isSafe = SAFE_PATTERNS.some((p) => p.test(command)); - return !isDestructive && isSafe; -} - -export interface TodoItem { - step: number; - text: string; - completed: boolean; -} - -export function cleanStepText(text: string): string { - let cleaned = text - .replace(/\*{1,2}([^*]+)\*{1,2}/g, "$1") // Remove bold/italic - .replace(/`([^`]+)`/g, "$1") // Remove code - .replace( - /^(Use|Run|Execute|Create|Write|Read|Check|Verify|Update|Modify|Add|Remove|Delete|Install)\s+(the\s+)?/i, - "", - ) - .replace(/\s+/g, " ") - .trim(); - - if (cleaned.length > 0) { - cleaned = cleaned.charAt(0).toUpperCase() + cleaned.slice(1); - } - if (cleaned.length > 50) { - cleaned = `${cleaned.slice(0, 47)}...`; - } - return cleaned; -} - -export function extractTodoItems(message: string): TodoItem[] { - const items: TodoItem[] = []; - const headerMatch = message.match(/\*{0,2}Plan:\*{0,2}\s*\n/i); - if (!headerMatch) return items; - - const planSection = message.slice( - message.indexOf(headerMatch[0]) + headerMatch[0].length, - ); - const numberedPattern = /^\s*(\d+)[.)]\s+\*{0,2}([^*\n]+)/gm; - - for (const match of planSection.matchAll(numberedPattern)) { - const text = match[2] - .trim() - .replace(/\*{1,2}$/, "") - .trim(); - if ( - text.length > 5 && - !text.startsWith("`") && - !text.startsWith("/") && - !text.startsWith("-") - ) { - const cleaned = cleanStepText(text); - if (cleaned.length > 3) { - items.push({ step: items.length + 1, text: cleaned, completed: false }); - } - } - } - return items; -} - -export function extractDoneSteps(message: string): number[] { - const steps: number[] = []; - for (const match of message.matchAll(/\[DONE:(\d+)\]/gi)) { - const step = Number(match[1]); - if (Number.isFinite(step)) steps.push(step); - } - return steps; -} - -export function markCompletedSteps(text: string, items: TodoItem[]): number { - const doneSteps = extractDoneSteps(text); - for (const step of doneSteps) { - const item = items.find((t) => t.step === step); - if (item) item.completed = true; - } - return doneSteps.length; -} diff --git a/.pi/skills/toon/SKILL.md b/.pi/skills/toon/SKILL.md new file mode 100644 index 0000000..bb3deea --- /dev/null +++ b/.pi/skills/toon/SKILL.md @@ -0,0 +1,932 @@ +--- +name: toon +description: **Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It's intended for *LLM input* as a drop-in, lossless representation of your existing JSON. +--- + +![TOON logo with step‑by‑step guide](./.github/og.png) + +# Token-Oriented Object Notation (TOON) + +[![CI](https://github.com/toon-format/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon/actions) +[![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon) +[![SPEC v3.0](https://img.shields.io/badge/spec-v3.0-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec) +[![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon) +[![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE) + +**Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It's intended for _LLM input_ as a drop-in, lossless representation of your existing JSON. + +TOON combines YAML's indentation-based structure for nested objects with a CSV-style tabular layout for uniform arrays. TOON's sweet spot is uniform arrays of objects (multiple fields per row, same structure across items), achieving CSV-like compactness while adding explicit structure that helps LLMs parse and validate data reliably. For deeply nested or non-uniform data, JSON may be more efficient. + +The similarity to CSV is intentional: CSV is simple and ubiquitous, and TOON aims to keep that familiarity while remaining a lossless, drop-in representation of JSON for Large Language Models. + +Think of it as a translation layer: use JSON programmatically, and encode it as TOON for LLM input. + +> [!TIP] +> The TOON format is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to the [spec](https://github.com/toon-format/spec) or sharing feedback. + +## Table of Contents + +- [Why TOON?](#why-toon) +- [Key Features](#key-features) +- [When Not to Use TOON](#when-not-to-use-toon) +- [Benchmarks](#benchmarks) +- [Installation & Quick Start](#installation--quick-start) +- [Playgrounds](#playgrounds) +- [Editor Support](#editor-support) +- [CLI](#cli) +- [Format Overview](#format-overview) +- [Using TOON with LLMs](#using-toon-with-llms) +- [Documentation](#documentation) +- [Other Implementations](#other-implementations) +- [📋 Full Specification](https://github.com/toon-format/spec/blob/main/SPEC.md) + +## Why TOON? + +AI is becoming cheaper and more accessible, but larger context windows allow for larger data inputs as well. **LLM tokens still cost money** – and standard JSON is verbose and token-expensive: + +```json +{ + "context": { + "task": "Our favorite hikes together", + "location": "Boulder", + "season": "spring_2025" + }, + "friends": ["ana", "luis", "sam"], + "hikes": [ + { + "id": 1, + "name": "Blue Lake Trail", + "distanceKm": 7.5, + "elevationGain": 320, + "companion": "ana", + "wasSunny": true + }, + { + "id": 2, + "name": "Ridge Overlook", + "distanceKm": 9.2, + "elevationGain": 540, + "companion": "luis", + "wasSunny": false + }, + { + "id": 3, + "name": "Wildflower Loop", + "distanceKm": 5.1, + "elevationGain": 180, + "companion": "sam", + "wasSunny": true + } + ] +} +``` + +
+YAML already conveys the same information with fewer tokens. + +```yaml +context: + task: Our favorite hikes together + location: Boulder + season: spring_2025 +friends: + - ana + - luis + - sam +hikes: + - id: 1 + name: Blue Lake Trail + distanceKm: 7.5 + elevationGain: 320 + companion: ana + wasSunny: true + - id: 2 + name: Ridge Overlook + distanceKm: 9.2 + elevationGain: 540 + companion: luis + wasSunny: false + - id: 3 + name: Wildflower Loop + distanceKm: 5.1 + elevationGain: 180 + companion: sam + wasSunny: true +``` + +
+ +TOON conveys the same information with **even fewer tokens** – combining YAML-like indentation with CSV-style tabular arrays: + +```yaml +context: + task: Our favorite hikes together + location: Boulder + season: spring_2025 +friends[3]: ana,luis,sam +hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}: + 1,Blue Lake Trail,7.5,320,ana,true + 2,Ridge Overlook,9.2,540,luis,false + 3,Wildflower Loop,5.1,180,sam,true +``` + +## Key Features + +- 📊 **Token-Efficient & Accurate:** TOON reaches 74% accuracy (vs JSON's 70%) while using ~40% fewer tokens in mixed-structure benchmarks across 4 models. +- 🔁 **JSON Data Model:** Encodes the same objects, arrays, and primitives as JSON with deterministic, lossless round-trips. +- 🛤️ **LLM-Friendly Guardrails:** Explicit [N] lengths and {fields} headers give models a clear schema to follow, improving parsing reliability. +- 📐 **Minimal Syntax:** Uses indentation instead of braces and minimizes quoting, giving YAML-like readability with CSV-style compactness. +- 🧺 **Tabular Arrays:** Uniform arrays of objects collapse into tables that declare fields once and stream row values line by line. +- 🌐 **Multi-Language Ecosystem:** Spec-driven implementations in TypeScript, Python, Go, Rust, .NET, and other languages. + +## Media Type & File Extension + +By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` for HTTP and content-type–aware contexts. TOON documents are always UTF-8 encoded; the `charset=utf-8` parameter may be specified but defaults to UTF-8 when omitted. See [SPEC.md §18.2](https://github.com/toon-format/spec/blob/main/SPEC.md#182-provisional-media-type) for normative details. + +## When Not to Use TOON + +TOON excels with uniform arrays of objects, but there are cases where other formats are better: + +- **Deeply nested or non-uniform structures** (tabular eligibility ≈ 0%): JSON-compact often uses fewer tokens. Example: complex configuration objects with many nested levels. +- **Semi-uniform arrays** (~40–60% tabular eligibility): Token savings diminish. Prefer JSON if your pipelines already rely on it. +- **Pure tabular data**: CSV is smaller than TOON for flat tables. TOON adds minimal overhead (~5-10%) to provide structure (array length declarations, field headers, delimiter scoping) that improves LLM reliability. +- **Latency-critical applications**: If end-to-end response time is your top priority, benchmark on your exact setup. Some deployments (especially local/quantized models like Ollama) may process compact JSON faster despite TOON's lower token count. Measure TTFT, tokens/sec, and total time for both formats and use whichever is faster. + +See [benchmarks](#benchmarks) for concrete comparisons across different data structures. + +## Benchmarks + +Benchmarks are organized into two tracks to ensure fair comparisons: + +- **Mixed-Structure Track**: Datasets with nested or semi-uniform structures (TOON vs JSON, YAML, XML). CSV excluded as it cannot properly represent these structures. +- **Flat-Only Track**: Datasets with flat tabular structures where CSV is applicable (CSV vs TOON vs JSON, YAML, XML). + +### Retrieval Accuracy + + + +Benchmarks test LLM comprehension across different input formats using 209 data retrieval questions on 4 models. + +
+Show Dataset Catalog + +#### Dataset Catalog + +| Dataset | Rows | Structure | CSV Support | Eligibility | +| --------------------------------------------------- | ---- | ------------ | ----------- | ----------- | +| Uniform employee records | 100 | uniform | ✓ | 100% | +| E-commerce orders with nested structures | 50 | nested | ✗ | 33% | +| Time-series analytics data | 60 | uniform | ✓ | 100% | +| Top 100 GitHub repositories | 100 | uniform | ✓ | 100% | +| Semi-uniform event logs | 75 | semi-uniform | ✗ | 50% | +| Deeply nested configuration | 11 | deep | ✗ | 0% | +| Valid complete dataset (control) | 20 | uniform | ✓ | 100% | +| Array truncated: 3 rows removed from end | 17 | uniform | ✓ | 100% | +| Extra rows added beyond declared length | 23 | uniform | ✓ | 100% | +| Inconsistent field count (missing salary in row 10) | 20 | uniform | ✓ | 100% | +| Missing required fields (no email in multiple rows) | 20 | uniform | ✓ | 100% | + +**Structure classes:** + +- **uniform**: All objects have identical fields with primitive values +- **semi-uniform**: Mix of uniform and non-uniform structures +- **nested**: Objects with nested structures (nested objects or arrays) +- **deep**: Highly nested with minimal tabular eligibility + +**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening) + +**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values) + +
+ +#### Efficiency Ranking (Accuracy per 1K Tokens) + +Each format ranked by efficiency (accuracy percentage per 1,000 tokens): + +``` +TOON ████████████████████ 26.9 acc%/1K tok │ 73.9% acc │ 2,744 tokens +JSON compact █████████████████░░░ 22.9 acc%/1K tok │ 70.7% acc │ 3,081 tokens +YAML ██████████████░░░░░░ 18.6 acc%/1K tok │ 69.0% acc │ 3,719 tokens +JSON ███████████░░░░░░░░░ 15.3 acc%/1K tok │ 69.7% acc │ 4,545 tokens +XML ██████████░░░░░░░░░░ 13.0 acc%/1K tok │ 67.1% acc │ 5,167 tokens +``` + +_Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better._ + +> [!TIP] +> TOON achieves **73.9%** accuracy (vs JSON's 69.7%) while using **39.6% fewer tokens**. + +**Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle. + +#### Per-Model Accuracy + +Accuracy across 4 LLMs on 209 data retrieval questions: + +``` +claude-haiku-4-5-20251001 +→ TOON ████████████░░░░░░░░ 59.8% (125/209) + JSON ███████████░░░░░░░░░ 57.4% (120/209) + YAML ███████████░░░░░░░░░ 56.0% (117/209) + XML ███████████░░░░░░░░░ 55.5% (116/209) + JSON compact ███████████░░░░░░░░░ 55.0% (115/209) + CSV ██████████░░░░░░░░░░ 50.5% (55/109) + +gemini-2.5-flash +→ TOON ██████████████████░░ 87.6% (183/209) + CSV █████████████████░░░ 86.2% (94/109) + JSON compact ████████████████░░░░ 82.3% (172/209) + YAML ████████████████░░░░ 79.4% (166/209) + XML ████████████████░░░░ 79.4% (166/209) + JSON ███████████████░░░░░ 77.0% (161/209) + +gpt-5-nano +→ TOON ██████████████████░░ 90.9% (190/209) + JSON compact ██████████████████░░ 90.9% (190/209) + JSON ██████████████████░░ 89.0% (186/209) + CSV ██████████████████░░ 89.0% (97/109) + YAML █████████████████░░░ 87.1% (182/209) + XML ████████████████░░░░ 80.9% (169/209) + +grok-4-fast-non-reasoning +→ TOON ███████████░░░░░░░░░ 57.4% (120/209) + JSON ███████████░░░░░░░░░ 55.5% (116/209) + JSON compact ███████████░░░░░░░░░ 54.5% (114/209) + YAML ███████████░░░░░░░░░ 53.6% (112/209) + XML ███████████░░░░░░░░░ 52.6% (110/209) + CSV ██████████░░░░░░░░░░ 52.3% (57/109) +``` + +> [!TIP] +> TOON achieves **73.9% accuracy** (vs JSON's 69.7%) while using **39.6% fewer tokens** on these datasets. + +
+Performance by dataset, model, and question type + +#### Performance by Question Type + +| Question Type | TOON | JSON compact | JSON | CSV | YAML | XML | +| --------------------- | ----- | ------------ | ----- | ------ | ----- | ----- | +| Field Retrieval | 99.6% | 99.3% | 99.3% | 100.0% | 98.2% | 98.9% | +| Aggregation | 54.4% | 47.2% | 48.8% | 44.0% | 47.6% | 41.3% | +| Filtering | 56.3% | 57.3% | 50.5% | 49.1% | 51.0% | 47.9% | +| Structure Awareness | 88.0% | 83.0% | 83.0% | 85.9% | 80.0% | 80.0% | +| Structural Validation | 70.0% | 45.0% | 50.0% | 80.0% | 60.0% | 80.0% | + +#### Performance by Dataset + +##### Uniform employee records + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `csv` | 72.0% | 2,352 | 118/164 | +| `toon` | 73.8% | 2,518 | 121/164 | +| `json-compact` | 69.5% | 3,953 | 114/164 | +| `yaml` | 68.3% | 4,982 | 112/164 | +| `json-pretty` | 68.3% | 6,360 | 112/164 | +| `xml` | 69.5% | 7,324 | 114/164 | + +##### E-commerce orders with nested structures + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `toon` | 81.1% | 7,232 | 133/164 | +| `json-compact` | 76.8% | 6,794 | 126/164 | +| `yaml` | 75.6% | 8,347 | 124/164 | +| `json-pretty` | 76.2% | 10,713 | 125/164 | +| `xml` | 74.4% | 12,023 | 122/164 | + +##### Time-series analytics data + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `csv` | 73.3% | 1,406 | 88/120 | +| `toon` | 72.5% | 1,548 | 87/120 | +| `json-compact` | 71.7% | 2,349 | 86/120 | +| `yaml` | 71.7% | 2,949 | 86/120 | +| `json-pretty` | 68.3% | 3,676 | 82/120 | +| `xml` | 68.3% | 4,384 | 82/120 | + +##### Top 100 GitHub repositories + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `toon` | 62.9% | 8,779 | 83/132 | +| `csv` | 61.4% | 8,527 | 81/132 | +| `yaml` | 59.8% | 13,141 | 79/132 | +| `json-compact` | 55.3% | 11,464 | 73/132 | +| `json-pretty` | 56.1% | 15,157 | 74/132 | +| `xml` | 48.5% | 17,105 | 64/132 | + +##### Semi-uniform event logs + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `json-compact` | 63.3% | 4,819 | 76/120 | +| `toon` | 57.5% | 5,799 | 69/120 | +| `json-pretty` | 59.2% | 6,797 | 71/120 | +| `yaml` | 48.3% | 5,827 | 58/120 | +| `xml` | 46.7% | 7,709 | 56/120 | + +##### Deeply nested configuration + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `json-compact` | 92.2% | 574 | 107/116 | +| `toon` | 95.7% | 666 | 111/116 | +| `yaml` | 91.4% | 686 | 106/116 | +| `json-pretty` | 94.0% | 932 | 109/116 | +| `xml` | 92.2% | 1,018 | 107/116 | + +##### Valid complete dataset (control) + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `toon` | 100.0% | 544 | 4/4 | +| `json-compact` | 100.0% | 795 | 4/4 | +| `yaml` | 100.0% | 1,003 | 4/4 | +| `json-pretty` | 100.0% | 1,282 | 4/4 | +| `csv` | 25.0% | 492 | 1/4 | +| `xml` | 0.0% | 1,467 | 0/4 | + +##### Array truncated: 3 rows removed from end + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `csv` | 100.0% | 425 | 4/4 | +| `xml` | 100.0% | 1,251 | 4/4 | +| `toon` | 0.0% | 474 | 0/4 | +| `json-compact` | 0.0% | 681 | 0/4 | +| `json-pretty` | 0.0% | 1,096 | 0/4 | +| `yaml` | 0.0% | 859 | 0/4 | + +##### Extra rows added beyond declared length + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `csv` | 100.0% | 566 | 4/4 | +| `toon` | 75.0% | 621 | 3/4 | +| `xml` | 100.0% | 1,692 | 4/4 | +| `yaml` | 75.0% | 1,157 | 3/4 | +| `json-compact` | 50.0% | 917 | 2/4 | +| `json-pretty` | 50.0% | 1,476 | 2/4 | + +##### Inconsistent field count (missing salary in row 10) + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `csv` | 75.0% | 489 | 3/4 | +| `yaml` | 100.0% | 996 | 4/4 | +| `toon` | 100.0% | 1,019 | 4/4 | +| `json-compact` | 75.0% | 790 | 3/4 | +| `xml` | 100.0% | 1,458 | 4/4 | +| `json-pretty` | 75.0% | 1,274 | 3/4 | + +##### Missing required fields (no email in multiple rows) + +| Format | Accuracy | Tokens | Correct/Total | +| -------------- | -------- | ------ | ------------- | +| `csv` | 100.0% | 329 | 4/4 | +| `xml` | 100.0% | 1,411 | 4/4 | +| `toon` | 75.0% | 983 | 3/4 | +| `yaml` | 25.0% | 960 | 1/4 | +| `json-pretty` | 25.0% | 1,230 | 1/4 | +| `json-compact` | 0.0% | 755 | 0/4 | + +#### Performance by Model + +##### claude-haiku-4-5-20251001 + +| Format | Accuracy | Correct/Total | +| -------------- | -------- | ------------- | +| `toon` | 59.8% | 125/209 | +| `json-pretty` | 57.4% | 120/209 | +| `yaml` | 56.0% | 117/209 | +| `xml` | 55.5% | 116/209 | +| `json-compact` | 55.0% | 115/209 | +| `csv` | 50.5% | 55/109 | + +##### gemini-2.5-flash + +| Format | Accuracy | Correct/Total | +| -------------- | -------- | ------------- | +| `toon` | 87.6% | 183/209 | +| `csv` | 86.2% | 94/109 | +| `json-compact` | 82.3% | 172/209 | +| `yaml` | 79.4% | 166/209 | +| `xml` | 79.4% | 166/209 | +| `json-pretty` | 77.0% | 161/209 | + +##### gpt-5-nano + +| Format | Accuracy | Correct/Total | +| -------------- | -------- | ------------- | +| `toon` | 90.9% | 190/209 | +| `json-compact` | 90.9% | 190/209 | +| `json-pretty` | 89.0% | 186/209 | +| `csv` | 89.0% | 97/109 | +| `yaml` | 87.1% | 182/209 | +| `xml` | 80.9% | 169/209 | + +##### grok-4-fast-non-reasoning + +| Format | Accuracy | Correct/Total | +| -------------- | -------- | ------------- | +| `toon` | 57.4% | 120/209 | +| `json-pretty` | 55.5% | 116/209 | +| `json-compact` | 54.5% | 114/209 | +| `yaml` | 53.6% | 112/209 | +| `xml` | 52.6% | 110/209 | +| `csv` | 52.3% | 57/109 | + +
+ +#### What's Being Measured + +This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it. + +#### Datasets Tested + +Eleven datasets designed to test different structural patterns and validation capabilities: + +**Primary datasets:** + +1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format. +2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays. +3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values. +4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars. +5. **Event Logs** (75 logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects. +6. **Nested Config** (1 configuration): Deeply nested configuration with minimal tabular eligibility. + +**Structural validation datasets:** + +7. **Control**: Valid complete dataset (baseline for validation) +8. **Truncated**: Array with 3 rows removed from end (tests `[N]` length detection) +9. **Extra rows**: Array with 3 additional rows beyond declared length +10. **Width mismatch**: Inconsistent field count (missing salary in row 10) +11. **Missing fields**: Systematic field omissions (no email in multiple rows) + +#### Question Types + +209 questions are generated dynamically across five categories: + +- **Field retrieval (33%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths) + + - Example: "What is Alice's salary?" → `75000` + - Example: "How many items are in order ORD-0042?" → `3` + - Example: "What is the customer name for order ORD-0042?" → `John Doe` + +- **Aggregation (30%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons) + + - Example: "How many employees work in Engineering?" → `17` + - Example: "What is the total revenue across all orders?" → `45123.50` + - Example: "How many employees have salary > 80000?" → `23` + +- **Filtering (23%)**: Multi-condition queries requiring compound logic (AND constraints across fields) + + - Example: "How many employees in Sales have salary > 80000?" → `5` + - Example: "How many active employees have more than 10 years of experience?" → `8` + +- **Structure awareness (12%)**: Tests format-native structural affordances (TOON's `[N]` count and `{fields}`, CSV's header row) + + - Example: "How many employees are in the dataset?" → `100` + - Example: "List the field names for employees" → `id, name, email, department, salary, yearsExperience, active` + - Example: "What is the department of the last employee?" → `Sales` + +- **Structural validation (2%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata + - Example: "Is this data complete and valid?" → `YES` (control dataset) or `NO` (corrupted datasets) + - Tests TOON's `[N]` length validation and `{fields}` consistency checking + - Demonstrates CSV's lack of structural validation capabilities + +#### Evaluation Process + +1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON compact, JSON, CSV, YAML, XML). +2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer. +3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge. + +#### Models & Configuration + +- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-2.5-flash`, `gpt-5-nano`, `grok-4-fast-non-reasoning` +- **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer) +- **Temperature**: Not set (models use their defaults) +- **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls + + + +### Token Efficiency + +Token counts are measured using the GPT-5 `o200k_base` tokenizer via [`gpt-tokenizer`](https://github.com/niieani/gpt-tokenizer). Savings are calculated against formatted JSON (2-space indentation) as the primary baseline, with additional comparisons to compact JSON (minified), YAML, and XML. Actual savings vary by model and tokenizer. + +The benchmarks test datasets across different structural patterns (uniform, semi-uniform, nested, deeply nested) to show where TOON excels and where other formats may be better. + + + +#### Mixed-Structure Track + +Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures. + +``` +🛒 E-commerce orders with nested structures ┊ Tabular: 33% + │ + TOON █████████████░░░░░░░ 72,771 tokens + ├─ vs JSON (−33.1%) 108,806 tokens + ├─ vs JSON compact (+5.5%) 68,975 tokens + ├─ vs YAML (−14.2%) 84,780 tokens + └─ vs XML (−40.5%) 122,406 tokens + +🧾 Semi-uniform event logs ┊ Tabular: 50% + │ + TOON █████████████████░░░ 153,211 tokens + ├─ vs JSON (−15.0%) 180,176 tokens + ├─ vs JSON compact (+19.9%) 127,731 tokens + ├─ vs YAML (−0.8%) 154,505 tokens + └─ vs XML (−25.2%) 204,777 tokens + +🧩 Deeply nested configuration ┊ Tabular: 0% + │ + TOON ██████████████░░░░░░ 631 tokens + ├─ vs JSON (−31.3%) 919 tokens + ├─ vs JSON compact (+11.9%) 564 tokens + ├─ vs YAML (−6.2%) 673 tokens + └─ vs XML (−37.4%) 1,008 tokens + +──────────────────────────────────── Total ──────────────────────────────────── + TOON ████████████████░░░░ 226,613 tokens + ├─ vs JSON (−21.8%) 289,901 tokens + ├─ vs JSON compact (+14.9%) 197,270 tokens + ├─ vs YAML (−5.6%) 239,958 tokens + └─ vs XML (−31.0%) 328,191 tokens +``` + +#### Flat-Only Track + +Datasets with flat tabular structures where CSV is applicable. + +``` +👥 Uniform employee records ┊ Tabular: 100% + │ + CSV ███████████████████░ 46,954 tokens + TOON ████████████████████ 49,831 tokens (+6.1% vs CSV) + ├─ vs JSON (−60.7%) 126,860 tokens + ├─ vs JSON compact (−36.8%) 78,856 tokens + ├─ vs YAML (−50.0%) 99,706 tokens + └─ vs XML (−66.0%) 146,444 tokens + +📈 Time-series analytics data ┊ Tabular: 100% + │ + CSV ██████████████████░░ 8,388 tokens + TOON ████████████████████ 9,120 tokens (+8.7% vs CSV) + ├─ vs JSON (−59.0%) 22,250 tokens + ├─ vs JSON compact (−35.8%) 14,216 tokens + ├─ vs YAML (−48.9%) 17,863 tokens + └─ vs XML (−65.7%) 26,621 tokens + +⭐ Top 100 GitHub repositories ┊ Tabular: 100% + │ + CSV ███████████████████░ 8,512 tokens + TOON ████████████████████ 8,744 tokens (+2.7% vs CSV) + ├─ vs JSON (−42.3%) 15,144 tokens + ├─ vs JSON compact (−23.7%) 11,454 tokens + ├─ vs YAML (−33.4%) 13,128 tokens + └─ vs XML (−48.9%) 17,095 tokens + +──────────────────────────────────── Total ──────────────────────────────────── + CSV ███████████████████░ 63,854 tokens + TOON ████████████████████ 67,695 tokens (+6.0% vs CSV) + ├─ vs JSON (−58.8%) 164,254 tokens + ├─ vs JSON compact (−35.2%) 104,526 tokens + ├─ vs YAML (−48.2%) 130,697 tokens + └─ vs XML (−64.4%) 190,160 tokens +``` + +
+Show detailed examples + +#### 📈 Time-series analytics data + +**Savings:** 13,130 tokens (59.0% reduction vs JSON) + +**JSON** (22,250 tokens): + +```json +{ + "metrics": [ + { + "date": "2025-01-01", + "views": 5715, + "clicks": 211, + "conversions": 28, + "revenue": 7976.46, + "bounceRate": 0.47 + }, + { + "date": "2025-01-02", + "views": 7103, + "clicks": 393, + "conversions": 28, + "revenue": 8360.53, + "bounceRate": 0.32 + }, + { + "date": "2025-01-03", + "views": 7248, + "clicks": 378, + "conversions": 24, + "revenue": 3212.57, + "bounceRate": 0.5 + }, + { + "date": "2025-01-04", + "views": 2927, + "clicks": 77, + "conversions": 11, + "revenue": 1211.69, + "bounceRate": 0.62 + }, + { + "date": "2025-01-05", + "views": 3530, + "clicks": 82, + "conversions": 8, + "revenue": 462.77, + "bounceRate": 0.56 + } + ] +} +``` + +**TOON** (9,120 tokens): + +``` +metrics[5]{date,views,clicks,conversions,revenue,bounceRate}: + 2025-01-01,5715,211,28,7976.46,0.47 + 2025-01-02,7103,393,28,8360.53,0.32 + 2025-01-03,7248,378,24,3212.57,0.5 + 2025-01-04,2927,77,11,1211.69,0.62 + 2025-01-05,3530,82,8,462.77,0.56 +``` + +--- + +#### ⭐ Top 100 GitHub repositories + +**Savings:** 6,400 tokens (42.3% reduction vs JSON) + +**JSON** (15,144 tokens): + +```json +{ + "repositories": [ + { + "id": 28457823, + "name": "freeCodeCamp", + "repo": "freeCodeCamp/freeCodeCamp", + "description": "freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…", + "createdAt": "2014-12-24T17:49:19Z", + "updatedAt": "2025-10-28T11:58:08Z", + "pushedAt": "2025-10-28T10:17:16Z", + "stars": 430886, + "watchers": 8583, + "forks": 42146, + "defaultBranch": "main" + }, + { + "id": 132750724, + "name": "build-your-own-x", + "repo": "codecrafters-io/build-your-own-x", + "description": "Master programming by recreating your favorite technologies from scratch.", + "createdAt": "2018-05-09T12:03:18Z", + "updatedAt": "2025-10-28T12:37:11Z", + "pushedAt": "2025-10-10T18:45:01Z", + "stars": 430877, + "watchers": 6332, + "forks": 40453, + "defaultBranch": "master" + }, + { + "id": 21737465, + "name": "awesome", + "repo": "sindresorhus/awesome", + "description": "😎 Awesome lists about all kinds of interesting topics", + "createdAt": "2014-07-11T13:42:37Z", + "updatedAt": "2025-10-28T12:40:21Z", + "pushedAt": "2025-10-27T17:57:31Z", + "stars": 410052, + "watchers": 8017, + "forks": 32029, + "defaultBranch": "main" + } + ] +} +``` + +**TOON** (8,744 tokens): + +``` +repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}: + 28457823,freeCodeCamp,freeCodeCamp/freeCodeCamp,"freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…","2014-12-24T17:49:19Z","2025-10-28T11:58:08Z","2025-10-28T10:17:16Z",430886,8583,42146,main + 132750724,build-your-own-x,codecrafters-io/build-your-own-x,Master programming by recreating your favorite technologies from scratch.,"2018-05-09T12:03:18Z","2025-10-28T12:37:11Z","2025-10-10T18:45:01Z",430877,6332,40453,master + 21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main +``` + +
+ + + +## Installation & Quick Start + +### CLI (No Installation Required) + +Try TOON instantly with npx: + +```bash +# Convert JSON to TOON +npx @toon-format/cli input.json -o output.toon + +# Pipe from stdin +echo '{"name": "Ada", "role": "dev"}' | npx @toon-format/cli +``` + +See the [CLI section](#cli) for all options and examples. + +### TypeScript Library + +```bash +# npm +npm install @toon-format/toon + +# pnpm +pnpm add @toon-format/toon + +# yarn +yarn add @toon-format/toon +``` + +**Example usage:** + +```ts +import { encode } from "@toon-format/toon"; + +const data = { + users: [ + { id: 1, name: "Alice", role: "admin" }, + { id: 2, name: "Bob", role: "user" }, + ], +}; + +console.log(encode(data)); +// users[2]{id,name,role}: +// 1,Alice,admin +// 2,Bob,user +``` + +**Streaming large datasets:** + +```ts +import { encodeLines } from "@toon-format/toon"; + +const largeData = await fetchThousandsOfRecords(); + +// Memory-efficient streaming for large data +for (const line of encodeLines(largeData)) { + process.stdout.write(`${line}\n`); +} +``` + +> [!TIP] +> For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options). + +**Transforming values with replacer:** + +```ts +import { encode } from "@toon-format/toon"; + +// Remove sensitive fields +const user = { name: "Alice", password: "secret", email: "alice@example.com" }; +const safe = encode(user, { + replacer: (key, value) => (key === "password" ? undefined : value), +}); +// name: Alice +// email: alice@example.com + +// Transform values +const data = { status: "active", count: 5 }; +const transformed = encode(data, { + replacer: (key, value) => + typeof value === "string" ? value.toUpperCase() : value, +}); +// status: ACTIVE +// count: 5 +``` + +> [!TIP] +> The `replacer` function provides fine-grained control over encoding, similar to `JSON.stringify`'s replacer but with path tracking. See the [API Reference](https://toonformat.dev/reference/api#replacer-function) for more examples. + +## Playgrounds + +Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation. + +### Official Playground + +The [TOON Playground](https://toonformat.dev/playground) lets you convert JSON to TOON in real-time, compare token counts, and share your experiments via URL. + +### Community Playgrounds + +- [Format Tokenization Playground](https://www.curiouslychase.com/playground/format-tokenization-exploration) +- [TOON Tools](https://toontools.vercel.app/) + +## Editor Support + +### VS Code + +[TOON Language Support](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) - Syntax highlighting, validation, conversion, and token analysis. + +```bash +code --install-extension vishalraut.vscode-toon +``` + +### Tree-sitter Grammar + +[tree-sitter-toon](https://github.com/3swordman/tree-sitter-toon) - Grammar for Tree-sitter-compatible editors (Neovim, Helix, Emacs, Zed). + +### Neovim + +[toon.nvim](https://github.com/thalesgelinger/toon.nvim) - Lua-based plugin. + +### Other Editors + +Use YAML syntax highlighting as a close approximation. + +## CLI + +Command-line tool for quick JSON↔TOON conversions, token analysis, and pipeline integration. Auto-detects format from file extension, supports stdin/stdout workflows, and offers delimiter options for maximum efficiency. + +```bash +# Encode JSON to TOON (auto-detected) +npx @toon-format/cli input.json -o output.toon + +# Decode TOON to JSON (auto-detected) +npx @toon-format/cli data.toon -o output.json + +# Pipe from stdin (no argument needed) +cat data.json | npx @toon-format/cli +echo '{"name": "Ada"}' | npx @toon-format/cli + +# Output to stdout +npx @toon-format/cli input.json + +# Show token savings +npx @toon-format/cli data.json --stats +``` + +> [!TIP] +> See the full [CLI documentation](https://toonformat.dev/cli/) for all options, examples, and advanced usage. + +## Format Overview + +Detailed syntax references, implementation guides, and quick lookups for understanding and using the TOON format. + +- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax documentation +- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick reference +- [API Reference](https://toonformat.dev/reference/api) – Encode/decode usage (TypeScript) + +## Using TOON with LLMs + +TOON works best when you show the format instead of describing it. The structure is self-documenting – models parse it naturally once they see the pattern. Wrap data in ` ```toon` code blocks for input, and show the expected header template when asking models to generate TOON. Use tab delimiters for even better token efficiency. + +Follow the detailed [LLM integration guide](https://toonformat.dev/guide/llm-prompts) for strategies, examples, and validation techniques. + +## Documentation + +Comprehensive guides, references, and resources to help you get the most out of the TOON format and tools. + +### Getting Started + +- [Introduction & Installation](https://toonformat.dev/guide/getting-started) – What TOON is, when to use it, first steps +- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax with examples +- [Benchmarks](https://toonformat.dev/guide/benchmarks) – Accuracy & token efficiency results + +### Tools & Integration + +- [CLI](https://toonformat.dev/cli/) – Command-line tool for JSON↔TOON conversions +- [Using TOON with LLMs](https://toonformat.dev/guide/llm-prompts) – Prompting strategies & validation +- [Playgrounds](https://toonformat.dev/ecosystem/tools-and-playgrounds) – Interactive tools + +### References + +- [API Reference](https://toonformat.dev/reference/api) – TypeScript/JavaScript encode/decode API +- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick format lookup +- [Specification](https://github.com/toon-format/spec/blob/main/SPEC.md) – Normative rules for implementers + +## Other Implementations + +TOON has official and community implementations across multiple languages including Python, Rust, Go, Java, Swift, .NET, and many more. + +See the full list of implementations in the [documentation](https://toon-format.org/ecosystem/implementations). + +## Credits + +- Logo design by [鈴木ックス(SZKX)](https://x.com/szkx_art) + +## License + +[MIT](./LICENSE) License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)