From 1b26269562b3e2eba5de49f1290cf781de523e83 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sat, 31 Jan 2026 17:25:17 +0100 Subject: [PATCH 01/63] feat: add initial implementation of pipeline UI package - Created `README.md` for documentation. - Added `package.json` with dependencies and scripts. - Configured TypeScript with `tsconfig.json` and `tsconfig.build.json`. - Set up `tsdown` for building the package. - Updated `pnpm-lock.yaml` to include new package dependencies. - Modified `pnpm-workspace.yaml` to include all packages under `packages/**`. --- packages/pipelines/pipeline-core/README.md | 27 ++ packages/pipelines/pipeline-core/package.json | 60 +++ .../pipeline-core/tsconfig.build.json | 5 + .../pipelines/pipeline-core/tsconfig.json | 8 + .../pipelines/pipeline-core/tsdown.config.ts | 7 + .../pipelines/pipeline-executor/README.md | 27 ++ .../pipelines/pipeline-executor/package.json | 60 +++ .../pipeline-executor/tsconfig.build.json | 5 + .../pipelines/pipeline-executor/tsconfig.json | 9 + .../pipeline-executor/tsdown.config.ts | 7 + packages/pipelines/pipeline-graph/README.md | 27 ++ .../pipelines/pipeline-graph/package.json | 60 +++ .../pipeline-graph/tsconfig.build.json | 5 + .../pipelines/pipeline-graph/tsconfig.json | 9 + .../pipelines/pipeline-graph/tsdown.config.ts | 7 + packages/pipelines/pipeline-loader/README.md | 27 ++ .../pipelines/pipeline-loader/package.json | 60 +++ .../pipeline-loader/tsconfig.build.json | 5 + .../pipelines/pipeline-loader/tsconfig.json | 9 + .../pipeline-loader/tsdown.config.ts | 7 + packages/pipelines/pipeline-ui/README.md | 27 ++ packages/pipelines/pipeline-ui/package.json | 60 +++ .../pipelines/pipeline-ui/tsconfig.build.json | 5 + packages/pipelines/pipeline-ui/tsconfig.json | 9 + .../pipelines/pipeline-ui/tsdown.config.ts | 7 + pnpm-lock.yaml | 401 +++++++++++++++--- 26 files changed, 875 insertions(+), 65 deletions(-) create mode 100644 packages/pipelines/pipeline-core/README.md create mode 100644 packages/pipelines/pipeline-core/package.json create mode 100644 packages/pipelines/pipeline-core/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-core/tsconfig.json create mode 100644 packages/pipelines/pipeline-core/tsdown.config.ts create mode 100644 packages/pipelines/pipeline-executor/README.md create mode 100644 packages/pipelines/pipeline-executor/package.json create mode 100644 packages/pipelines/pipeline-executor/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-executor/tsconfig.json create mode 100644 packages/pipelines/pipeline-executor/tsdown.config.ts create mode 100644 packages/pipelines/pipeline-graph/README.md create mode 100644 packages/pipelines/pipeline-graph/package.json create mode 100644 packages/pipelines/pipeline-graph/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-graph/tsconfig.json create mode 100644 packages/pipelines/pipeline-graph/tsdown.config.ts create mode 100644 packages/pipelines/pipeline-loader/README.md create mode 100644 packages/pipelines/pipeline-loader/package.json create mode 100644 packages/pipelines/pipeline-loader/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-loader/tsconfig.json create mode 100644 packages/pipelines/pipeline-loader/tsdown.config.ts create mode 100644 packages/pipelines/pipeline-ui/README.md create mode 100644 packages/pipelines/pipeline-ui/package.json create mode 100644 packages/pipelines/pipeline-ui/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-ui/tsconfig.json create mode 100644 packages/pipelines/pipeline-ui/tsdown.config.ts diff --git a/packages/pipelines/pipeline-core/README.md b/packages/pipelines/pipeline-core/README.md new file mode 100644 index 000000000..494d64b1c --- /dev/null +++ b/packages/pipelines/pipeline-core/README.md @@ -0,0 +1,27 @@ +# @ucdjs/pipelines-core + +[![npm version][npm-version-src]][npm-version-href] +[![npm downloads][npm-downloads-src]][npm-downloads-href] +[![codecov][codecov-src]][codecov-href] + +> [!IMPORTANT] +> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk. + +A collection of core pipeline functionalities for the UCD project. + +## Installation + +```bash +npm install @ucdjs/pipelines-core +``` + +## 📄 License + +Published under [MIT License](./LICENSE). + +[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/pipelines-core?style=flat&colorA=18181B&colorB=4169E1 +[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-core +[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-core?style=flat&colorA=18181B&colorB=4169E1 +[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-core +[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1 +[codecov-href]: https://codecov.io/gh/ucdjs/ucd diff --git a/packages/pipelines/pipeline-core/package.json b/packages/pipelines/pipeline-core/package.json new file mode 100644 index 000000000..879238002 --- /dev/null +++ b/packages/pipelines/pipeline-core/package.json @@ -0,0 +1,60 @@ +{ + "name": "@ucdjs/pipelines-core", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-core" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + ".": "./dist/index.mjs", + "./package.json": "./package.json" + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "tsdown --tsconfig=./tsconfig.build.json", + "dev": "tsdown --watch", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs-internal/shared": "workspace:*", + "picomatch": "catalog:prod", + "zod": "catalog:prod" + }, + "devDependencies": { + "@luxass/eslint-config": "catalog:linting", + "@types/picomatch": "catalog:types", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "eslint": "catalog:linting", + "publint": "catalog:dev", + "tsdown": "catalog:dev", + "tsx": "catalog:dev", + "typescript": "catalog:dev" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-core/tsconfig.build.json b/packages/pipelines/pipeline-core/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-core/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-core/tsconfig.json b/packages/pipelines/pipeline-core/tsconfig.json new file mode 100644 index 000000000..89e9ed10b --- /dev/null +++ b/packages/pipelines/pipeline-core/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test", + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-core/tsdown.config.ts b/packages/pipelines/pipeline-core/tsdown.config.ts new file mode 100644 index 000000000..dee0149e6 --- /dev/null +++ b/packages/pipelines/pipeline-core/tsdown.config.ts @@ -0,0 +1,7 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/index.ts", + ], +}); diff --git a/packages/pipelines/pipeline-executor/README.md b/packages/pipelines/pipeline-executor/README.md new file mode 100644 index 000000000..045525964 --- /dev/null +++ b/packages/pipelines/pipeline-executor/README.md @@ -0,0 +1,27 @@ +# @ucdjs/pipelines-executor + +[![npm version][npm-version-src]][npm-version-href] +[![npm downloads][npm-downloads-src]][npm-downloads-href] +[![codecov][codecov-src]][codecov-href] + +> [!IMPORTANT] +> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk. + +A collection of core pipeline functionalities for the UCD project. + +## Installation + +```bash +npm install @ucdjs/pipelines-executor +``` + +## 📄 License + +Published under [MIT License](./LICENSE). + +[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/pipelines-executor?style=flat&colorA=18181B&colorB=4169E1 +[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-executor +[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-executor?style=flat&colorA=18181B&colorB=4169E1 +[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-executor +[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1 +[codecov-href]: https://codecov.io/gh/ucdjs/ucd diff --git a/packages/pipelines/pipeline-executor/package.json b/packages/pipelines/pipeline-executor/package.json new file mode 100644 index 000000000..ea44d2730 --- /dev/null +++ b/packages/pipelines/pipeline-executor/package.json @@ -0,0 +1,60 @@ +{ + "name": "@ucdjs/pipelines-executor", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-executor" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + ".": "./dist/index.mjs", + "./package.json": "./package.json" + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "tsdown --tsconfig=./tsconfig.build.json", + "dev": "tsdown --watch", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs-internal/shared": "workspace:*", + "picomatch": "catalog:prod", + "zod": "catalog:prod" + }, + "devDependencies": { + "@luxass/eslint-config": "catalog:linting", + "@types/picomatch": "catalog:types", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "eslint": "catalog:linting", + "publint": "catalog:dev", + "tsdown": "catalog:dev", + "tsx": "catalog:dev", + "typescript": "catalog:dev" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-executor/tsconfig.build.json b/packages/pipelines/pipeline-executor/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-executor/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-executor/tsconfig.json b/packages/pipelines/pipeline-executor/tsconfig.json new file mode 100644 index 000000000..07edf31d8 --- /dev/null +++ b/packages/pipelines/pipeline-executor/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test", + "playgrounds" + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-executor/tsdown.config.ts b/packages/pipelines/pipeline-executor/tsdown.config.ts new file mode 100644 index 000000000..dee0149e6 --- /dev/null +++ b/packages/pipelines/pipeline-executor/tsdown.config.ts @@ -0,0 +1,7 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/index.ts", + ], +}); diff --git a/packages/pipelines/pipeline-graph/README.md b/packages/pipelines/pipeline-graph/README.md new file mode 100644 index 000000000..b19a872fe --- /dev/null +++ b/packages/pipelines/pipeline-graph/README.md @@ -0,0 +1,27 @@ +# @ucdjs/pipelines-graph + +[![npm version][npm-version-src]][npm-version-href] +[![npm downloads][npm-downloads-src]][npm-downloads-href] +[![codecov][codecov-src]][codecov-href] + +> [!IMPORTANT] +> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk. + +A collection of core pipeline functionalities for the UCD project. + +## Installation + +```bash +npm install @ucdjs/pipelines-graph +``` + +## 📄 License + +Published under [MIT License](./LICENSE). + +[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/pipelines-graph?style=flat&colorA=18181B&colorB=4169E1 +[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-graph +[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-graph?style=flat&colorA=18181B&colorB=4169E1 +[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-graph +[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1 +[codecov-href]: https://codecov.io/gh/ucdjs/ucd diff --git a/packages/pipelines/pipeline-graph/package.json b/packages/pipelines/pipeline-graph/package.json new file mode 100644 index 000000000..3b579ebb9 --- /dev/null +++ b/packages/pipelines/pipeline-graph/package.json @@ -0,0 +1,60 @@ +{ + "name": "@ucdjs/pipelines-graph", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-graph" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + ".": "./dist/index.mjs", + "./package.json": "./package.json" + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "tsdown --tsconfig=./tsconfig.build.json", + "dev": "tsdown --watch", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs-internal/shared": "workspace:*", + "picomatch": "catalog:prod", + "zod": "catalog:prod" + }, + "devDependencies": { + "@luxass/eslint-config": "catalog:linting", + "@types/picomatch": "catalog:types", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "eslint": "catalog:linting", + "publint": "catalog:dev", + "tsdown": "catalog:dev", + "tsx": "catalog:dev", + "typescript": "catalog:dev" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-graph/tsconfig.build.json b/packages/pipelines/pipeline-graph/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-graph/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-graph/tsconfig.json b/packages/pipelines/pipeline-graph/tsconfig.json new file mode 100644 index 000000000..07edf31d8 --- /dev/null +++ b/packages/pipelines/pipeline-graph/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test", + "playgrounds" + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-graph/tsdown.config.ts b/packages/pipelines/pipeline-graph/tsdown.config.ts new file mode 100644 index 000000000..dee0149e6 --- /dev/null +++ b/packages/pipelines/pipeline-graph/tsdown.config.ts @@ -0,0 +1,7 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/index.ts", + ], +}); diff --git a/packages/pipelines/pipeline-loader/README.md b/packages/pipelines/pipeline-loader/README.md new file mode 100644 index 000000000..5f1710618 --- /dev/null +++ b/packages/pipelines/pipeline-loader/README.md @@ -0,0 +1,27 @@ +# @ucdjs/pipelines-loader + +[![npm version][npm-version-src]][npm-version-href] +[![npm downloads][npm-downloads-src]][npm-downloads-href] +[![codecov][codecov-src]][codecov-href] + +> [!IMPORTANT] +> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk. + +A collection of core pipeline functionalities for the UCD project. + +## Installation + +```bash +npm install @ucdjs/pipelines-loader +``` + +## 📄 License + +Published under [MIT License](./LICENSE). + +[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/pipelines-loader?style=flat&colorA=18181B&colorB=4169E1 +[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-loader +[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-loader?style=flat&colorA=18181B&colorB=4169E1 +[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-loader +[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1 +[codecov-href]: https://codecov.io/gh/ucdjs/ucd diff --git a/packages/pipelines/pipeline-loader/package.json b/packages/pipelines/pipeline-loader/package.json new file mode 100644 index 000000000..b765e2ab1 --- /dev/null +++ b/packages/pipelines/pipeline-loader/package.json @@ -0,0 +1,60 @@ +{ + "name": "@ucdjs/pipelines-loader", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-loader" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + ".": "./dist/index.mjs", + "./package.json": "./package.json" + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "tsdown --tsconfig=./tsconfig.build.json", + "dev": "tsdown --watch", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs-internal/shared": "workspace:*", + "picomatch": "catalog:prod", + "zod": "catalog:prod" + }, + "devDependencies": { + "@luxass/eslint-config": "catalog:linting", + "@types/picomatch": "catalog:types", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "eslint": "catalog:linting", + "publint": "catalog:dev", + "tsdown": "catalog:dev", + "tsx": "catalog:dev", + "typescript": "catalog:dev" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-loader/tsconfig.build.json b/packages/pipelines/pipeline-loader/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-loader/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-loader/tsconfig.json b/packages/pipelines/pipeline-loader/tsconfig.json new file mode 100644 index 000000000..07edf31d8 --- /dev/null +++ b/packages/pipelines/pipeline-loader/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test", + "playgrounds" + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-loader/tsdown.config.ts b/packages/pipelines/pipeline-loader/tsdown.config.ts new file mode 100644 index 000000000..dee0149e6 --- /dev/null +++ b/packages/pipelines/pipeline-loader/tsdown.config.ts @@ -0,0 +1,7 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/index.ts", + ], +}); diff --git a/packages/pipelines/pipeline-ui/README.md b/packages/pipelines/pipeline-ui/README.md new file mode 100644 index 000000000..5032f7fab --- /dev/null +++ b/packages/pipelines/pipeline-ui/README.md @@ -0,0 +1,27 @@ +# @ucdjs/pipelines-ui + +[![npm version][npm-version-src]][npm-version-href] +[![npm downloads][npm-downloads-src]][npm-downloads-href] +[![codecov][codecov-src]][codecov-href] + +> [!IMPORTANT] +> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk. + +A collection of core pipeline functionalities for the UCD project. + +## Installation + +```bash +npm install @ucdjs/pipelines-ui +``` + +## 📄 License + +Published under [MIT License](./LICENSE). + +[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/pipelines-ui?style=flat&colorA=18181B&colorB=4169E1 +[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-ui +[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-ui?style=flat&colorA=18181B&colorB=4169E1 +[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-ui +[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1 +[codecov-href]: https://codecov.io/gh/ucdjs/ucd diff --git a/packages/pipelines/pipeline-ui/package.json b/packages/pipelines/pipeline-ui/package.json new file mode 100644 index 000000000..57ee99589 --- /dev/null +++ b/packages/pipelines/pipeline-ui/package.json @@ -0,0 +1,60 @@ +{ + "name": "@ucdjs/pipelines-ui", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-ui" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + ".": "./dist/index.mjs", + "./package.json": "./package.json" + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "tsdown --tsconfig=./tsconfig.build.json", + "dev": "tsdown --watch", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs-internal/shared": "workspace:*", + "picomatch": "catalog:prod", + "zod": "catalog:prod" + }, + "devDependencies": { + "@luxass/eslint-config": "catalog:linting", + "@types/picomatch": "catalog:types", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "eslint": "catalog:linting", + "publint": "catalog:dev", + "tsdown": "catalog:dev", + "tsx": "catalog:dev", + "typescript": "catalog:dev" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-ui/tsconfig.build.json b/packages/pipelines/pipeline-ui/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-ui/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-ui/tsconfig.json b/packages/pipelines/pipeline-ui/tsconfig.json new file mode 100644 index 000000000..07edf31d8 --- /dev/null +++ b/packages/pipelines/pipeline-ui/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test", + "playgrounds" + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-ui/tsdown.config.ts b/packages/pipelines/pipeline-ui/tsdown.config.ts new file mode 100644 index 000000000..dee0149e6 --- /dev/null +++ b/packages/pipelines/pipeline-ui/tsdown.config.ts @@ -0,0 +1,7 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/index.ts", + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 667fa01ac..8f6d4026a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -375,7 +375,7 @@ importers: devDependencies: '@cloudflare/vitest-pool-workers': specifier: catalog:testing - version: https://pkg.pr.new/@cloudflare/vitest-pool-workers@11632(@vitest/runner@4.1.0-beta.2)(@vitest/snapshot@4.1.0-beta.2)(vitest@4.1.0-beta.1) + version: https://pkg.pr.new/@cloudflare/vitest-pool-workers@11632(@vitest/runner@4.1.0-beta.1)(@vitest/snapshot@4.1.0-beta.1)(vitest@4.1.0-beta.1) '@luxass/eslint-config': specifier: catalog:linting version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.1.0-beta.1) @@ -931,6 +931,206 @@ importers: specifier: catalog:testing version: 4.4.2(vitest@4.1.0-beta.1) + packages/pipelines/pipeline-core: + dependencies: + '@ucdjs-internal/shared': + specifier: workspace:* + version: link:../../shared + picomatch: + specifier: catalog:prod + version: 4.0.3 + zod: + specifier: catalog:prod + version: 4.3.6 + devDependencies: + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@types/picomatch': + specifier: catalog:types + version: 4.0.2 + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + publint: + specifier: catalog:dev + version: 0.3.17 + tsdown: + specifier: catalog:dev + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) + tsx: + specifier: catalog:dev + version: 4.21.0 + typescript: + specifier: catalog:dev + version: 5.9.3 + + packages/pipelines/pipeline-executor: + dependencies: + '@ucdjs-internal/shared': + specifier: workspace:* + version: link:../../shared + picomatch: + specifier: catalog:prod + version: 4.0.3 + zod: + specifier: catalog:prod + version: 4.3.6 + devDependencies: + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@types/picomatch': + specifier: catalog:types + version: 4.0.2 + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + publint: + specifier: catalog:dev + version: 0.3.17 + tsdown: + specifier: catalog:dev + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) + tsx: + specifier: catalog:dev + version: 4.21.0 + typescript: + specifier: catalog:dev + version: 5.9.3 + + packages/pipelines/pipeline-graph: + dependencies: + '@ucdjs-internal/shared': + specifier: workspace:* + version: link:../../shared + picomatch: + specifier: catalog:prod + version: 4.0.3 + zod: + specifier: catalog:prod + version: 4.3.6 + devDependencies: + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@types/picomatch': + specifier: catalog:types + version: 4.0.2 + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + publint: + specifier: catalog:dev + version: 0.3.17 + tsdown: + specifier: catalog:dev + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) + tsx: + specifier: catalog:dev + version: 4.21.0 + typescript: + specifier: catalog:dev + version: 5.9.3 + + packages/pipelines/pipeline-loader: + dependencies: + '@ucdjs-internal/shared': + specifier: workspace:* + version: link:../../shared + picomatch: + specifier: catalog:prod + version: 4.0.3 + zod: + specifier: catalog:prod + version: 4.3.6 + devDependencies: + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@types/picomatch': + specifier: catalog:types + version: 4.0.2 + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + publint: + specifier: catalog:dev + version: 0.3.17 + tsdown: + specifier: catalog:dev + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) + tsx: + specifier: catalog:dev + version: 4.21.0 + typescript: + specifier: catalog:dev + version: 5.9.3 + + packages/pipelines/pipeline-ui: + dependencies: + '@ucdjs-internal/shared': + specifier: workspace:* + version: link:../../shared + picomatch: + specifier: catalog:prod + version: 4.0.3 + zod: + specifier: catalog:prod + version: 4.3.6 + devDependencies: + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@types/picomatch': + specifier: catalog:types + version: 4.0.2 + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + publint: + specifier: catalog:dev + version: 0.3.17 + tsdown: + specifier: catalog:dev + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) + tsx: + specifier: catalog:dev + version: 4.21.0 + typescript: + specifier: catalog:dev + version: 5.9.3 + packages/schema-gen: dependencies: '@ai-sdk/openai': @@ -2395,8 +2595,8 @@ packages: resolution: {integrity: sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/config-helpers@0.5.1': - resolution: {integrity: sha512-QN8067dXsXAl9HIvqws7STEviheRFojX3zek5OpC84oBxDGqizW9731ByF/ASxqQihbWrVDdZXS+Ihnsckm9dg==} + '@eslint/config-helpers@0.5.2': + resolution: {integrity: sha512-a5MxrdDXEvqnIq+LisyCX6tQMPF/dSJpCfBgBauY+pNZ28yCtSsTvyTYrMhaI+LK26bVyCJfJkT0u8KIj2i1dQ==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} '@eslint/core@0.17.0': @@ -2407,6 +2607,10 @@ packages: resolution: {integrity: sha512-r18fEAj9uCk+VjzGt2thsbOmychS+4kxI14spVNibUO2vqKX7obOG+ymZljAwuPZl+S3clPGwCwTDtrdqTiY6Q==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} + '@eslint/core@1.1.0': + resolution: {integrity: sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + '@eslint/eslintrc@3.3.1': resolution: {integrity: sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -4601,16 +4805,32 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/project-service@8.53.1': + resolution: {integrity: sha512-WYC4FB5Ra0xidsmlPb+1SsnaSKPmS3gsjIARwbEkHkoWloQmuzcfypljaJcR78uyLA1h8sHdWWPHSLDI+MtNog==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/project-service@8.54.0': resolution: {integrity: sha512-YPf+rvJ1s7MyiWM4uTRhE4DvBXrEV+d8oC3P9Y2eT7S+HBS0clybdMIPnhiATi9vZOYDc7OQ1L/i6ga6NFYK/g==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/scope-manager@8.53.1': + resolution: {integrity: sha512-Lu23yw1uJMFY8cUeq7JlrizAgeQvWugNQzJp8C3x8Eo5Jw5Q2ykMdiiTB9vBVOOUBysMzmRRmUfwFrZuI2C4SQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/scope-manager@8.54.0': resolution: {integrity: sha512-27rYVQku26j/PbHYcVfRPonmOlVI6gihHtXFbTdB5sb6qA0wdAQAbyXFVarQ5t4HRojIz64IV90YtsjQSSGlQg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/tsconfig-utils@8.53.1': + resolution: {integrity: sha512-qfvLXS6F6b1y43pnf0pPbXJ+YoXIC7HKg0UGZ27uMIemKMKA6XH2DTxsEDdpdN29D+vHV07x/pnlPNVLhdhWiA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/tsconfig-utils@8.54.0': resolution: {integrity: sha512-dRgOyT2hPk/JwxNMZDsIXDgyl9axdJI3ogZ2XWhBPsnZUv+hPesa5iuhdYt2gzwA9t8RE5ytOJ6xB0moV0Ujvw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -4624,16 +4844,33 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/types@8.53.1': + resolution: {integrity: sha512-jr/swrr2aRmUAUjW5/zQHbMaui//vQlsZcJKijZf3M26bnmLj8LyZUpj8/Rd6uzaek06OWsqdofN/Thenm5O8A==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/types@8.54.0': resolution: {integrity: sha512-PDUI9R1BVjqu7AUDsRBbKMtwmjWcn4J3le+5LpcFgWULN3LvHC5rkc9gCVxbrsrGmO1jfPybN5s6h4Jy+OnkAA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/typescript-estree@8.53.1': + resolution: {integrity: sha512-RGlVipGhQAG4GxV1s34O91cxQ/vWiHJTDHbXRr0li2q/BGg3RR/7NM8QDWgkEgrwQYCvmJV9ichIwyoKCQ+DTg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/typescript-estree@8.54.0': resolution: {integrity: sha512-BUwcskRaPvTk6fzVWgDPdUndLjB87KYDrN5EYGetnktoeAvPtO4ONHlAZDnj5VFnUANg0Sjm7j4usBlnoVMHwA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/utils@8.53.1': + resolution: {integrity: sha512-c4bMvGVWW4hv6JmDUEG7fSYlWOl3II2I4ylt0NM+seinYQlZMQIaKaXIIVJWt9Ofh6whrpM+EdDQXKXjNovvrg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/utils@8.54.0': resolution: {integrity: sha512-9Cnda8GS57AQakvRyG0PTejJNlA2xhvyNtEVIMlDWOOeEyBkYWhGPnfrIAnqxLMTSTo6q8g12XVjjev5l1NvMA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -4641,6 +4878,10 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/visitor-keys@8.53.1': + resolution: {integrity: sha512-oy+wV7xDKFPRyNggmXuZQSBzvoLnpmJs+GhzRhPjrxl2b/jIlyjVokzm47CZCDUdXKr2zd7ZLodPfOBpOPyPlg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/visitor-keys@8.54.0': resolution: {integrity: sha512-VFlhGSl4opC0bprJiItPQ1RfUhGDIBokcPwaFH4yiBCaNPeld/9VeXbiPO1cLyorQi1G1vL+ecBk1x8o1axORA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -4726,27 +4967,18 @@ packages: '@vitest/pretty-format@4.1.0-beta.1': resolution: {integrity: sha512-CeI3uthjV/XKA6KBCr/B5HlCQaFdCgprdl7gBg/sUExQPary8BBhYoVWJeAPTeg9u+ppT9S4v/sYjjNjn3Qsrw==} - '@vitest/pretty-format@4.1.0-beta.2': - resolution: {integrity: sha512-Ms6NWhaLbJC8UaxLg+OLneAbGFoaAgTvN0RDSjso3QOwcwv6Aqy9QWXl1NT07+BRrK+EOSjgRJQNb8f1pkyLqw==} - '@vitest/runner@4.0.17': resolution: {integrity: sha512-JmuQyf8aMWoo/LmNFppdpkfRVHJcsgzkbCA+/Bk7VfNH7RE6Ut2qxegeyx2j3ojtJtKIbIGy3h+KxGfYfk28YQ==} '@vitest/runner@4.1.0-beta.1': resolution: {integrity: sha512-oE0nFu+0zT6IhhAu8Z9wWCWWy63a7btZLvq4zUkrGwJ9U4sabXHWzYakBE6ZDLXpI8aDv796+0AMej2AJ7m3tw==} - '@vitest/runner@4.1.0-beta.2': - resolution: {integrity: sha512-5ZU0gHEVEV0414RSj57ov1AWVLtTtQzH/BEDDafv94E92frTGoOLKHRPkLRiw7efBxD+R1L3tUphBvw0R5mNJw==} - '@vitest/snapshot@4.0.17': resolution: {integrity: sha512-npPelD7oyL+YQM2gbIYvlavlMVWUfNNGZPcu0aEUQXt7FXTuqhmgiYupPnAanhKvyP6Srs2pIbWo30K0RbDtRQ==} '@vitest/snapshot@4.1.0-beta.1': resolution: {integrity: sha512-wSt0PAy1QCZjzPUgpIYXBtZFTFXPw65GIQxz9mjhl0yjkAE+wnRU08+w3R3X5hrCKYVhTS3HHV8zs6Yin2K0Dw==} - '@vitest/snapshot@4.1.0-beta.2': - resolution: {integrity: sha512-wqepqIP9VDfD5dTDq2x3J1hW2Qd2BOlhBGrFS0Ye0pN0I3cFpX2+d10GbbRxN5NilofXaZnSo8avW/0zboDMtQ==} - '@vitest/spy@4.0.17': resolution: {integrity: sha512-I1bQo8QaP6tZlTomQNWKJE6ym4SHf3oLS7ceNjozxxgzavRAgZDc06T7kD8gb9bXKEgcLNt00Z+kZO6KaJ62Ew==} @@ -4769,9 +5001,6 @@ packages: '@vitest/utils@4.1.0-beta.1': resolution: {integrity: sha512-IUCsqDFj8E8WJq3wGRQ7MiMb2571tjTnjyrJ1oy+0HODutA2TpZGRqBA8ziLCIWTOL/e4RArE2k6eZh/jXgk9A==} - '@vitest/utils@4.1.0-beta.2': - resolution: {integrity: sha512-c5ZnkcNaSANcFn70YNwGNb8vXCWjFKGbIJ1tFVUf5+oBXNFfNqksTOqHDPydS2vi3IPVqYjA2n5UqKZus20qDw==} - '@vscode/vsce-sign-alpine-arm64@2.0.6': resolution: {integrity: sha512-wKkJBsvKF+f0GfsUuGT0tSW0kZL87QggEiqNqK6/8hvqsXvpx8OsTEc3mnE1kejkh5r+qUyQ7PtF8jZYN0mo8Q==} cpu: [arm64] @@ -5266,6 +5495,10 @@ packages: resolution: {integrity: sha512-buhp5kePrmda3vhc5B9t7pUQXAb2Tnd0qgpkIhPhkHXxJpiPJ11H0ZEU0oBpJ2QztSbzG/ZxMj/CHsYJqRHmyg==} engines: {node: '>= 12.0.0'} + comment-parser@1.4.4: + resolution: {integrity: sha512-0D6qSQ5IkeRrGJFHRClzaMOenMeT0gErz3zIw3AprKMqhRN6LNU2jQOdkPG/FZ+8bCgXE1VidrgSzlBBDZRr8A==} + engines: {node: '>= 12.0.0'} + comment-parser@1.4.5: resolution: {integrity: sha512-aRDkn3uyIlCFfk5NUA+VdwMmMsh8JGhc4hapfV4yxymHGQ3BVskMQfoXGpCo5IoBuQ9tS5iiVKhCpTcB4pW4qw==} engines: {node: '>= 12.0.0'} @@ -6010,6 +6243,10 @@ packages: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} + expect-type@1.2.2: + resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} + engines: {node: '>=12.0.0'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -7356,7 +7593,7 @@ packages: hasBin: true miniflare@https://pkg.pr.new/cloudflare/workers-sdk/miniflare@79a1932: - resolution: {tarball: https://pkg.pr.new/cloudflare/workers-sdk/miniflare@79a1932} + resolution: {integrity: sha512-AzGelopZDQwEVqzT1spQKyBn0J12EsD52GlkJUB0HaQ6oNWLRM66hiQJ66sqpIzwQ7+DC2Qtgv4gqTyED3IBRw==, tarball: https://pkg.pr.new/cloudflare/workers-sdk/miniflare@79a1932} version: 4.20260120.0 engines: {node: '>=18.0.0'} hasBin: true @@ -8746,8 +8983,8 @@ packages: resolution: {integrity: sha512-d9CwU93nN0IA1QL+GSNDdwLAu1Ew5ZjTwupvedwg3WdfoH6pIDvYQ2hV0Uc2nKBLPq7NB5apCx57MLS5qlmO5g==} engines: {node: '>=20'} - type-fest@5.4.2: - resolution: {integrity: sha512-FLEenlVYf7Zcd34ISMLo3ZzRE1gRjY1nMDTp+bQRBiPsaKyIW8K3Zr99ioHDUgA9OGuGGJPyYpNcffGmBhJfGg==} + type-fest@5.4.3: + resolution: {integrity: sha512-AXSAQJu79WGc79/3e9/CR77I/KQgeY1AhNvcShIH4PTcGYyC4xv6H4R4AUOwkPS5799KlVDAu8zExeCrkGquiA==} engines: {node: '>=20'} type-is@2.0.1: @@ -8800,10 +9037,6 @@ packages: undici-types@7.10.0: resolution: {integrity: sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==} - undici@7.16.0: - resolution: {integrity: sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==} - engines: {node: '>=20.18.1'} - undici@7.18.2: resolution: {integrity: sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw==} engines: {node: '>=20.18.1'} @@ -9828,10 +10061,10 @@ snapshots: optionalDependencies: workerd: 1.20260128.0 - '@cloudflare/vitest-pool-workers@https://pkg.pr.new/@cloudflare/vitest-pool-workers@11632(@vitest/runner@4.1.0-beta.2)(@vitest/snapshot@4.1.0-beta.2)(vitest@4.1.0-beta.1)': + '@cloudflare/vitest-pool-workers@https://pkg.pr.new/@cloudflare/vitest-pool-workers@11632(@vitest/runner@4.1.0-beta.1)(@vitest/snapshot@4.1.0-beta.1)(vitest@4.1.0-beta.1)': dependencies: - '@vitest/runner': 4.1.0-beta.2 - '@vitest/snapshot': 4.1.0-beta.2 + '@vitest/runner': 4.1.0-beta.1 + '@vitest/snapshot': 4.1.0-beta.1 cjs-module-lexer: 1.4.3 esbuild: 0.27.0 miniflare: https://pkg.pr.new/cloudflare/workers-sdk/miniflare@79a1932 @@ -9969,7 +10202,7 @@ snapshots: '@effect/sql': 0.48.6(@effect/experimental@0.57.11(@effect/platform@0.93.6(effect@3.19.9))(effect@3.19.9)(ioredis@5.8.2))(@effect/platform@0.93.6(effect@3.19.9))(effect@3.19.9) effect: 3.19.9 mime: 3.0.0 - undici: 7.16.0 + undici: 7.18.2 ws: 8.18.3 transitivePeerDependencies: - bufferutil @@ -10021,7 +10254,7 @@ snapshots: '@es-joy/jsdoccomment@0.78.0': dependencies: '@types/estree': 1.0.8 - '@typescript-eslint/types': 8.54.0 + '@typescript-eslint/types': 8.53.1 comment-parser: 1.4.1 esquery: 1.7.0 jsdoc-type-pratt-parser: 7.0.0 @@ -10029,7 +10262,7 @@ snapshots: '@es-joy/jsdoccomment@0.83.0': dependencies: '@types/estree': 1.0.8 - '@typescript-eslint/types': 8.54.0 + '@typescript-eslint/types': 8.53.1 comment-parser: 1.4.5 esquery: 1.7.0 jsdoc-type-pratt-parser: 7.1.0 @@ -10294,9 +10527,9 @@ snapshots: dependencies: '@eslint/core': 0.17.0 - '@eslint/config-helpers@0.5.1': + '@eslint/config-helpers@0.5.2': dependencies: - '@eslint/core': 1.0.1 + '@eslint/core': 1.1.0 '@eslint/core@0.17.0': dependencies: @@ -10306,6 +10539,10 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 + '@eslint/core@1.1.0': + dependencies: + '@types/json-schema': 7.0.15 + '@eslint/eslintrc@3.3.1': dependencies: ajv: 6.12.6 @@ -11602,7 +11839,7 @@ snapshots: dependencies: '@scalar/helpers': 0.2.10 nanoid: 5.1.6 - type-fest: 5.4.2 + type-fest: 5.4.3 zod: 4.3.6 '@sec-ant/readable-stream@0.4.1': {} @@ -12069,7 +12306,7 @@ snapshots: '@stylistic/eslint-plugin@5.7.1(eslint@9.39.2(jiti@2.6.1))': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) - '@typescript-eslint/types': 8.54.0 + '@typescript-eslint/types': 8.53.1 eslint: 9.39.2(jiti@2.6.1) eslint-visitor-keys: 4.2.1 espree: 10.4.0 @@ -12634,6 +12871,15 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/project-service@8.53.1(typescript@5.9.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.53.1(typescript@5.9.3) + '@typescript-eslint/types': 8.53.1 + debug: 4.4.3(supports-color@10.2.2) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/project-service@8.54.0(typescript@5.9.3)': dependencies: '@typescript-eslint/tsconfig-utils': 8.54.0(typescript@5.9.3) @@ -12643,11 +12889,20 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/scope-manager@8.53.1': + dependencies: + '@typescript-eslint/types': 8.53.1 + '@typescript-eslint/visitor-keys': 8.53.1 + '@typescript-eslint/scope-manager@8.54.0': dependencies: '@typescript-eslint/types': 8.54.0 '@typescript-eslint/visitor-keys': 8.54.0 + '@typescript-eslint/tsconfig-utils@8.53.1(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + '@typescript-eslint/tsconfig-utils@8.54.0(typescript@5.9.3)': dependencies: typescript: 5.9.3 @@ -12664,8 +12919,25 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/types@8.53.1': {} + '@typescript-eslint/types@8.54.0': {} + '@typescript-eslint/typescript-estree@8.53.1(typescript@5.9.3)': + dependencies: + '@typescript-eslint/project-service': 8.53.1(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.53.1(typescript@5.9.3) + '@typescript-eslint/types': 8.53.1 + '@typescript-eslint/visitor-keys': 8.53.1 + debug: 4.4.3(supports-color@10.2.2) + minimatch: 9.0.5 + semver: 7.7.3 + tinyglobby: 0.2.15 + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/typescript-estree@8.54.0(typescript@5.9.3)': dependencies: '@typescript-eslint/project-service': 8.54.0(typescript@5.9.3) @@ -12681,6 +12953,17 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/utils@8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + '@typescript-eslint/scope-manager': 8.53.1 + '@typescript-eslint/types': 8.53.1 + '@typescript-eslint/typescript-estree': 8.53.1(typescript@5.9.3) + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/utils@8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) @@ -12692,6 +12975,11 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/visitor-keys@8.53.1': + dependencies: + '@typescript-eslint/types': 8.53.1 + eslint-visitor-keys: 4.2.1 + '@typescript-eslint/visitor-keys@8.54.0': dependencies: '@typescript-eslint/types': 8.54.0 @@ -12774,8 +13062,8 @@ snapshots: '@vitest/eslint-plugin@1.6.6(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17)': dependencies: - '@typescript-eslint/scope-manager': 8.54.0 - '@typescript-eslint/utils': 8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.53.1 + '@typescript-eslint/utils': 8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) optionalDependencies: typescript: 5.9.3 @@ -12785,8 +13073,8 @@ snapshots: '@vitest/eslint-plugin@1.6.6(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.1.0-beta.1)': dependencies: - '@typescript-eslint/scope-manager': 8.54.0 - '@typescript-eslint/utils': 8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.53.1 + '@typescript-eslint/utils': 8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) optionalDependencies: typescript: 5.9.3 @@ -12850,10 +13138,6 @@ snapshots: dependencies: tinyrainbow: 3.0.3 - '@vitest/pretty-format@4.1.0-beta.2': - dependencies: - tinyrainbow: 3.0.3 - '@vitest/runner@4.0.17': dependencies: '@vitest/utils': 4.0.17 @@ -12865,11 +13149,6 @@ snapshots: '@vitest/utils': 4.1.0-beta.1 pathe: 2.0.3 - '@vitest/runner@4.1.0-beta.2': - dependencies: - '@vitest/utils': 4.1.0-beta.2 - pathe: 2.0.3 - '@vitest/snapshot@4.0.17': dependencies: '@vitest/pretty-format': 4.0.17 @@ -12883,12 +13162,6 @@ snapshots: magic-string: 0.30.21 pathe: 2.0.3 - '@vitest/snapshot@4.1.0-beta.2': - dependencies: - '@vitest/pretty-format': 4.1.0-beta.2 - magic-string: 0.30.21 - pathe: 2.0.3 - '@vitest/spy@4.0.17': optional: true @@ -12928,11 +13201,6 @@ snapshots: '@vitest/pretty-format': 4.1.0-beta.1 tinyrainbow: 3.0.3 - '@vitest/utils@4.1.0-beta.2': - dependencies: - '@vitest/pretty-format': 4.1.0-beta.2 - tinyrainbow: 3.0.3 - '@vscode/vsce-sign-alpine-arm64@2.0.6': optional: true @@ -13369,7 +13637,7 @@ snapshots: parse5: 7.3.0 parse5-htmlparser2-tree-adapter: 7.1.0 parse5-parser-stream: 7.1.2 - undici: 7.16.0 + undici: 7.18.2 whatwg-mimetype: 4.0.0 chokidar@3.6.0: @@ -13471,6 +13739,8 @@ snapshots: comment-parser@1.4.1: {} + comment-parser@1.4.4: {} + comment-parser@1.4.5: {} commit-parser@1.3.0: @@ -13954,7 +14224,7 @@ snapshots: eslint-flat-config-utils@3.0.0: dependencies: - '@eslint/config-helpers': 0.5.1 + '@eslint/config-helpers': 0.5.2 pathe: 2.0.3 eslint-formatting-reporter@0.0.0(eslint@9.39.2(jiti@2.6.1)): @@ -14058,7 +14328,7 @@ snapshots: eslint-plugin-perfectionist@5.4.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3): dependencies: - '@typescript-eslint/utils': 8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/utils': 8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) natural-orderby: 5.0.0 transitivePeerDependencies: @@ -14189,9 +14459,9 @@ snapshots: dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) '@eslint-community/regexpp': 4.12.2 - comment-parser: 1.4.5 + comment-parser: 1.4.4 eslint: 9.39.2(jiti@2.6.1) - jsdoc-type-pratt-parser: 7.1.0 + jsdoc-type-pratt-parser: 7.0.0 refa: 0.12.1 regexp-ast-analysis: 0.7.1 scslre: 0.3.0 @@ -14431,6 +14701,9 @@ snapshots: expand-template@2.0.3: optional: true + expect-type@1.2.2: + optional: true + expect-type@1.3.0: {} express-rate-limit@7.5.1(express@5.2.1): @@ -17827,7 +18100,7 @@ snapshots: dependencies: tagged-tag: 1.0.0 - type-fest@5.4.2: + type-fest@5.4.3: dependencies: tagged-tag: 1.0.0 @@ -17900,8 +18173,6 @@ snapshots: undici-types@7.10.0: {} - undici@7.16.0: {} - undici@7.18.2: {} unenv@2.0.0-rc.24: @@ -18119,7 +18390,7 @@ snapshots: '@vitest/spy': 4.0.17 '@vitest/utils': 4.0.17 es-module-lexer: 1.7.0 - expect-type: 1.3.0 + expect-type: 1.2.2 magic-string: 0.30.21 obug: 2.1.1 pathe: 2.0.3 From f3e42a7ea7e2ef900ccfb97d77e755cda3157799 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 18 Jan 2026 09:39:02 +0100 Subject: [PATCH 02/63] feat(pipelines): implement cache key and store interfaces --- packages/pipelines/pipeline-core/src/cache.ts | 136 ++++++++++++++++++ packages/pipelines/pipeline-core/src/index.ts | 0 .../pipelines/pipeline-executor/src/index.ts | 0 .../pipelines/pipeline-graph/src/index.ts | 0 .../pipelines/pipeline-loader/src/index.ts | 0 packages/pipelines/pipeline-ui/src/index.ts | 0 6 files changed, 136 insertions(+) create mode 100644 packages/pipelines/pipeline-core/src/cache.ts create mode 100644 packages/pipelines/pipeline-core/src/index.ts create mode 100644 packages/pipelines/pipeline-executor/src/index.ts create mode 100644 packages/pipelines/pipeline-graph/src/index.ts create mode 100644 packages/pipelines/pipeline-loader/src/index.ts create mode 100644 packages/pipelines/pipeline-ui/src/index.ts diff --git a/packages/pipelines/pipeline-core/src/cache.ts b/packages/pipelines/pipeline-core/src/cache.ts new file mode 100644 index 000000000..4a6b67176 --- /dev/null +++ b/packages/pipelines/pipeline-core/src/cache.ts @@ -0,0 +1,136 @@ +export interface CacheKey { + /** + * The route ID that produced this cache entry. + */ + routeId: string; + + /** + * The Unicode version being processed. + */ + version: string; + + /** + * Hash of the input file content. + */ + inputHash: string; + + /** + * Hashes of artifact dependencies used by this route. + * Key is artifact ID, value is the artifact's content hash. + */ + artifactHashes: Record; +} + +export type SerializedCacheKey = `${string}|${string}|${string}|${string}`; + +export interface CacheEntry { + /** + * The cache key that identifies this entry. + */ + key: CacheKey; + + /** + * The cached output data. + */ + output: TOutput[]; + + /** + * Artifacts produced by this route during execution. + * Key is artifact ID, value is the serialized artifact. + */ + producedArtifacts: Record; + + /** + * Timestamp when this entry was created (ISO 8601). + */ + createdAt: string; + + /** + * Optional metadata about the cache entry. + */ + meta?: Record; +} + +export interface CacheStore { + /** + * Retrieve a cached entry by key. + * Returns undefined if not found. + */ + get: (key: CacheKey) => Promise; + + /** + * Store a cache entry. + */ + set: (entry: CacheEntry) => Promise; + + /** + * Check if a cache entry exists for the given key. + */ + has: (key: CacheKey) => Promise; + + /** + * Delete a cache entry by key. + * Returns true if the entry existed and was deleted. + */ + delete: (key: CacheKey) => Promise; + + /** + * Clear all cache entries. + */ + clear: () => Promise; +} + +export function serializeCacheKey(key: CacheKey): SerializedCacheKey { + const artifactHashStr = Object.entries(key.artifactHashes) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, hash]) => `${id}:${hash}`) + .join(","); + + return `${key.routeId}|${key.version}|${key.inputHash}|${artifactHashStr}` as SerializedCacheKey; +} + +export function deserializeCacheKey(serialized: SerializedCacheKey): CacheKey { + const [routeId, version, inputHash, artifactHashStr] = serialized.split("|") as [ + string, + string, + string, + string + ] + + const artifactHashes: Record = {}; + if (artifactHashStr) { + for (const pair of artifactHashStr.split(",")) { + const [id, hash] = pair.split(":"); + artifactHashes[id!] = hash!; + } + } + + return { + routeId, + version, + inputHash, + artifactHashes, + }; +} + +export function areCacheKeysEqual(a: CacheKey, b: CacheKey): boolean { + if (a.routeId !== b.routeId) return false; + if (a.version !== b.version) return false; + if (a.inputHash !== b.inputHash) return false; + + const aKeys = Object.keys(a.artifactHashes).sort(); + const bKeys = Object.keys(b.artifactHashes).sort(); + if (aKeys.length !== bKeys.length) return false; + + for (let i = 0; i < aKeys.length; i++) { + const key = aKeys[i]!; + if (key !== bKeys[i]) return false; + if (a.artifactHashes[key] !== b.artifactHashes[key]) return false; + } + + return true; +} + +export function defineCacheStore(store: CacheStore): CacheStore { + return store; +} diff --git a/packages/pipelines/pipeline-core/src/index.ts b/packages/pipelines/pipeline-core/src/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/pipelines/pipeline-executor/src/index.ts b/packages/pipelines/pipeline-executor/src/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/pipelines/pipeline-graph/src/index.ts b/packages/pipelines/pipeline-graph/src/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/pipelines/pipeline-loader/src/index.ts b/packages/pipelines/pipeline-loader/src/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/pipelines/pipeline-ui/src/index.ts b/packages/pipelines/pipeline-ui/src/index.ts new file mode 100644 index 000000000..e69de29bb From a44fa5c6e5de1951867eac9b9ada69ecae60704d Mon Sep 17 00:00:00 2001 From: Lucas Date: Sat, 31 Jan 2026 17:26:07 +0100 Subject: [PATCH 03/63] WIP --- .../pipelines/pipeline-artifacts/README.md | 27 + .../pipelines/pipeline-artifacts/package.json | 57 ++ .../pipeline-artifacts/src/definition.ts | 42 + .../pipelines/pipeline-artifacts/src/index.ts | 26 + .../pipeline-artifacts/src/schema.ts | 68 ++ .../test/definition.test.ts | 404 ++++++++ .../pipeline-artifacts/test/schema.test.ts | 278 ++++++ .../pipeline-artifacts/tsconfig.build.json | 5 + .../pipeline-artifacts/tsconfig.json | 8 + .../pipeline-artifacts/tsdown.config.ts | 7 + packages/pipelines/pipeline-core/src/cache.ts | 136 --- .../pipeline-core/src/dependencies.ts | 70 ++ .../pipelines/pipeline-core/src/events.ts | 215 ++++ .../pipelines/pipeline-core/src/filters.ts | 62 ++ packages/pipelines/pipeline-core/src/index.ts | 141 +++ .../pipelines/pipeline-core/src/pipeline.ts | 269 +++++ packages/pipelines/pipeline-core/src/route.ts | 97 ++ .../pipelines/pipeline-core/src/source.ts | 87 ++ .../pipelines/pipeline-core/src/transform.ts | 144 +++ packages/pipelines/pipeline-core/src/types.ts | 288 ++++++ .../pipeline-core/test/dependencies.test.ts | 304 ++++++ .../pipeline-core/test/filters.test.ts | 445 +++++++++ .../pipeline-core/test/transform.test.ts | 330 ++++++ .../pipelines/pipeline-executor/package.json | 8 +- .../pipelines/pipeline-executor/src/cache.ts | 136 +++ .../pipeline-executor/src/executor.ts | 936 ++++++++++++++++++ .../pipelines/pipeline-executor/src/index.ts | 28 + .../pipeline-executor/src/results.ts | 28 + .../pipelines/pipeline-graph/package.json | 6 +- packages/pipelines/pipeline-graph/src/dag.ts | 225 +++++ .../pipelines/pipeline-graph/src/index.ts | 11 + .../pipelines/pipeline-loader/package.json | 6 +- .../pipelines/pipeline-loader/src/index.ts | 11 + .../pipelines/pipeline-loader/src/loader.ts | 160 +++ packages/pipelines/pipeline-ui/package.json | 7 +- packages/pipelines/pipeline-ui/src/index.ts | 32 + pnpm-lock.yaml | 113 +-- tooling/tsconfig/base.json | 8 + vitest.config.ts | 8 +- 39 files changed, 5012 insertions(+), 221 deletions(-) create mode 100644 packages/pipelines/pipeline-artifacts/README.md create mode 100644 packages/pipelines/pipeline-artifacts/package.json create mode 100644 packages/pipelines/pipeline-artifacts/src/definition.ts create mode 100644 packages/pipelines/pipeline-artifacts/src/index.ts create mode 100644 packages/pipelines/pipeline-artifacts/src/schema.ts create mode 100644 packages/pipelines/pipeline-artifacts/test/definition.test.ts create mode 100644 packages/pipelines/pipeline-artifacts/test/schema.test.ts create mode 100644 packages/pipelines/pipeline-artifacts/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-artifacts/tsconfig.json create mode 100644 packages/pipelines/pipeline-artifacts/tsdown.config.ts delete mode 100644 packages/pipelines/pipeline-core/src/cache.ts create mode 100644 packages/pipelines/pipeline-core/src/dependencies.ts create mode 100644 packages/pipelines/pipeline-core/src/events.ts create mode 100644 packages/pipelines/pipeline-core/src/filters.ts create mode 100644 packages/pipelines/pipeline-core/src/pipeline.ts create mode 100644 packages/pipelines/pipeline-core/src/route.ts create mode 100644 packages/pipelines/pipeline-core/src/source.ts create mode 100644 packages/pipelines/pipeline-core/src/transform.ts create mode 100644 packages/pipelines/pipeline-core/src/types.ts create mode 100644 packages/pipelines/pipeline-core/test/dependencies.test.ts create mode 100644 packages/pipelines/pipeline-core/test/filters.test.ts create mode 100644 packages/pipelines/pipeline-core/test/transform.test.ts create mode 100644 packages/pipelines/pipeline-executor/src/cache.ts create mode 100644 packages/pipelines/pipeline-executor/src/executor.ts create mode 100644 packages/pipelines/pipeline-executor/src/results.ts create mode 100644 packages/pipelines/pipeline-graph/src/dag.ts create mode 100644 packages/pipelines/pipeline-loader/src/loader.ts diff --git a/packages/pipelines/pipeline-artifacts/README.md b/packages/pipelines/pipeline-artifacts/README.md new file mode 100644 index 000000000..77accc96a --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/README.md @@ -0,0 +1,27 @@ +# @ucdjs/pipelines-artifacts + +[![npm version][npm-version-src]][npm-version-href] +[![npm downloads][npm-downloads-src]][npm-downloads-href] +[![codecov][codecov-src]][codecov-href] + +> [!IMPORTANT] +> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk. + +A collection of core pipeline functionalities for the UCD project. + +## Installation + +```bash +npm install @ucdjs/pipelines-artifacts +``` + +## 📄 License + +Published under [MIT License](./LICENSE). + +[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/pipelines-artifacts?style=flat&colorA=18181B&colorB=4169E1 +[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-artifacts +[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-artifacts?style=flat&colorA=18181B&colorB=4169E1 +[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-artifacts +[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1 +[codecov-href]: https://codecov.io/gh/ucdjs/ucd diff --git a/packages/pipelines/pipeline-artifacts/package.json b/packages/pipelines/pipeline-artifacts/package.json new file mode 100644 index 000000000..c0ab8812a --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/package.json @@ -0,0 +1,57 @@ +{ + "name": "@ucdjs/pipelines-artifacts", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-artifacts" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + ".": "./dist/index.mjs", + "./package.json": "./package.json" + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "tsdown --tsconfig=./tsconfig.build.json", + "dev": "tsdown --watch", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs/pipelines-core": "workspace:*", + "zod": "catalog:prod" + }, + "devDependencies": { + "@luxass/eslint-config": "catalog:linting", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "eslint": "catalog:linting", + "publint": "catalog:dev", + "tsdown": "catalog:dev", + "typescript": "catalog:dev" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-artifacts/src/definition.ts b/packages/pipelines/pipeline-artifacts/src/definition.ts new file mode 100644 index 000000000..13eec1841 --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/src/definition.ts @@ -0,0 +1,42 @@ +import type { ParseContext, ParsedRow, PipelineFilter } from "@ucdjs/pipelines-core"; + +export interface ArtifactBuildContext { + version: string; +} + +export interface PipelineArtifactDefinition< + TId extends string = string, + TValue = unknown, +> { + id: TId; + filter?: PipelineFilter; + parser?: (ctx: ParseContext) => AsyncIterable; + build: (ctx: ArtifactBuildContext, rows?: AsyncIterable) => Promise; +} + +export function definePipelineArtifact< + const TId extends string, + TValue, +>( + definition: PipelineArtifactDefinition, +): PipelineArtifactDefinition { + return definition; +} + +export type InferArtifactId = T extends PipelineArtifactDefinition ? TId : never; +export type InferArtifactValue = T extends PipelineArtifactDefinition ? TValue : never; + +export type InferArtifactsMap = { + [K in T[number] as InferArtifactId]: InferArtifactValue; +}; + +export function isPipelineArtifactDefinition(value: unknown): value is PipelineArtifactDefinition { + return ( + typeof value === "object" + && value !== null + && "id" in value + && "build" in value + && typeof (value as { id: unknown }).id === "string" + && typeof (value as { build: unknown }).build === "function" + ); +} diff --git a/packages/pipelines/pipeline-artifacts/src/index.ts b/packages/pipelines/pipeline-artifacts/src/index.ts new file mode 100644 index 000000000..ee22f42d4 --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/src/index.ts @@ -0,0 +1,26 @@ +export type { + Artifact, + GlobalArtifact, + ArtifactDefinition, + InferArtifactSchemaType, + InferEmittedArtifacts, +} from "./schema"; + +export { + artifact, + isGlobalArtifact, + isVersionArtifact, +} from "./schema"; + +export type { + ArtifactBuildContext, + PipelineArtifactDefinition, + InferArtifactId, + InferArtifactValue, + InferArtifactsMap, +} from "./definition"; + +export { + definePipelineArtifact, + isPipelineArtifactDefinition, +} from "./definition"; diff --git a/packages/pipelines/pipeline-artifacts/src/schema.ts b/packages/pipelines/pipeline-artifacts/src/schema.ts new file mode 100644 index 000000000..42cd5fdd5 --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/src/schema.ts @@ -0,0 +1,68 @@ +import type { z } from "zod"; + +export interface Artifact { + _type: "artifact"; + schema: TSchema; + scope: "version"; +} + +export interface GlobalArtifact { + _type: "global-artifact"; + schema: TSchema; + scope: "global"; +} + +export type ArtifactDefinition = + | Artifact + | GlobalArtifact; + +export function artifact( + schema: TSchema +): Artifact; + +export function artifact( + schema: TSchema, + scope: "version" +): Artifact; + +export function artifact( + schema: TSchema, + scope: "global" +): GlobalArtifact; + +export function artifact( + schema: TSchema, + scope?: "version" | "global", +): ArtifactDefinition { + if (scope === "global") { + return { + _type: "global-artifact", + schema, + scope: "global", + }; + } + return { + _type: "artifact", + schema, + scope: "version", + }; +} + +export type InferArtifactSchemaType = + T extends ArtifactDefinition ? z.infer : never; + +export type InferEmittedArtifacts> = { + [K in keyof TEmits]: InferArtifactSchemaType; +}; + +export function isGlobalArtifact( + def: ArtifactDefinition, +): def is GlobalArtifact { + return def._type === "global-artifact"; +} + +export function isVersionArtifact( + def: ArtifactDefinition, +): def is Artifact { + return def._type === "artifact"; +} diff --git a/packages/pipelines/pipeline-artifacts/test/definition.test.ts b/packages/pipelines/pipeline-artifacts/test/definition.test.ts new file mode 100644 index 000000000..3d279b638 --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/test/definition.test.ts @@ -0,0 +1,404 @@ +import { describe, expect, it, vi } from "vitest"; +import type { ParseContext, ParsedRow, PipelineFilter } from "@ucdjs/pipelines-core"; +import { + definePipelineArtifact, + isPipelineArtifactDefinition, + type ArtifactBuildContext, + type InferArtifactId, + type InferArtifactsMap, + type InferArtifactValue, + type PipelineArtifactDefinition, +} from "../src/definition"; + +describe("definePipelineArtifact", () => { + it("should define a minimal artifact", () => { + const build = vi.fn().mockResolvedValue("result"); + const artifact = definePipelineArtifact({ + id: "test-artifact", + build, + }); + + expect(artifact).toEqual({ + id: "test-artifact", + build, + }); + }); + + it("should define artifact with filter", () => { + const build = vi.fn().mockResolvedValue({ count: 42 }); + const filter: PipelineFilter = (ctx) => ctx.file.name.endsWith(".txt"); + + const artifact = definePipelineArtifact({ + id: "filtered-artifact", + filter, + build, + }); + + expect(artifact.id).toBe("filtered-artifact"); + expect(artifact.filter).toBe(filter); + expect(artifact.build).toBe(build); + }); + + it("should define artifact with parser", async () => { + const mockRows: ParsedRow[] = [ + { sourceFile: "test.txt", kind: "point", codePoint: "0041", value: "a" }, + { sourceFile: "test.txt", kind: "range", start: "0042", end: "0043", value: "b" }, + ]; + + async function* parser(_ctx: ParseContext): AsyncIterable { + for (const row of mockRows) { + yield row; + } + } + + const build = vi.fn().mockResolvedValue([]); + + const artifact = definePipelineArtifact({ + id: "parsed-artifact", + parser, + build, + }); + + expect(artifact.parser).toBe(parser); + }); + + it("should define artifact with all properties", async () => { + const filter: PipelineFilter = (ctx) => ctx.file.ext === ".txt"; + async function* parser(_ctx: ParseContext): AsyncIterable { + yield { sourceFile: "test.txt", kind: "point", codePoint: "0041", value: "test" }; + } + const build = vi.fn().mockResolvedValue({ data: "processed" }); + + const artifact = definePipelineArtifact({ + id: "complete-artifact", + filter, + parser, + build, + }); + + expect(artifact).toEqual({ + id: "complete-artifact", + filter, + parser, + build, + }); + }); + + it("should preserve build function signature", async () => { + interface CustomResult { + version: string; + count: number; + } + + const build = async (ctx: ArtifactBuildContext): Promise => { + return { + version: ctx.version, + count: 42, + }; + }; + + const artifact = definePipelineArtifact({ + id: "typed-artifact", + build, + }); + + const context: ArtifactBuildContext = { version: "16.0.0" }; + const result = await artifact.build(context); + + expect(result).toEqual({ + version: "16.0.0", + count: 42, + }); + }); + + it("should work with async build processing rows", async () => { + async function* mockParser(): AsyncIterable { + yield { sourceFile: "test.txt", kind: "point", codePoint: "0041", value: "a" }; + yield { sourceFile: "test.txt", kind: "point", codePoint: "0042", value: "b" }; + } + + const build = async ( + ctx: ArtifactBuildContext, + rows?: AsyncIterable, + ): Promise => { + let count = 0; + if (rows) { + for await (const _row of rows) { + count++; + } + } + return count; + }; + + const artifact = definePipelineArtifact({ + id: "counting-artifact", + parser: mockParser, + build, + }); + + const context: ArtifactBuildContext = { version: "16.0.0" }; + const rows = mockParser(); + const result = await artifact.build(context, rows); + + expect(result).toBe(2); + }); +}); + +describe("isPipelineArtifactDefinition", () => { + it("should return true for valid artifact definition", () => { + const valid: PipelineArtifactDefinition = { + id: "test", + build: async () => "result", + }; + + expect(isPipelineArtifactDefinition(valid)).toBe(true); + }); + + it("should return true for artifact with all properties", () => { + const valid: PipelineArtifactDefinition = { + id: "test", + filter: (ctx) => ctx.file.ext === ".txt", + parser: async function* () { + yield { sourceFile: "test.txt", kind: "point", codePoint: "0041" }; + }, + build: async () => "result", + }; + + expect(isPipelineArtifactDefinition(valid)).toBe(true); + }); + + it("should return false for null", () => { + expect(isPipelineArtifactDefinition(null)).toBe(false); + }); + + it("should return false for undefined", () => { + expect(isPipelineArtifactDefinition(undefined)).toBe(false); + }); + + it("should return false for primitive types", () => { + expect(isPipelineArtifactDefinition("string")).toBe(false); + expect(isPipelineArtifactDefinition(123)).toBe(false); + expect(isPipelineArtifactDefinition(true)).toBe(false); + }); + + it("should return false for empty object", () => { + expect(isPipelineArtifactDefinition({})).toBe(false); + }); + + it("should return false for object missing id", () => { + const invalid = { + build: async () => "result", + }; + + expect(isPipelineArtifactDefinition(invalid)).toBe(false); + }); + + it("should return false for object missing build", () => { + const invalid = { + id: "test", + }; + + expect(isPipelineArtifactDefinition(invalid)).toBe(false); + }); + + it("should return false for object with non-string id", () => { + const invalid = { + id: 123, + build: async () => "result", + }; + + expect(isPipelineArtifactDefinition(invalid)).toBe(false); + }); + + it("should return false for object with non-function build", () => { + const invalid = { + id: "test", + build: "not a function", + }; + + expect(isPipelineArtifactDefinition(invalid)).toBe(false); + }); + + it("should return false for array", () => { + expect(isPipelineArtifactDefinition([])).toBe(false); + expect(isPipelineArtifactDefinition([{ id: "test", build: async () => {} }])).toBe(false); + }); + + it("should work as type guard", () => { + const unknown: unknown = { + id: "test", + build: async () => "result", + }; + + if (isPipelineArtifactDefinition(unknown)) { + expect(unknown.id).toBe("test"); + expect(typeof unknown.build).toBe("function"); + } else { + throw new Error("Expected valid artifact definition"); + } + }); +}); + +describe("type inference", () => { + describe("InferArtifactId", () => { + it("should infer artifact id", () => { + const artifact = definePipelineArtifact({ + id: "my-artifact", + build: async () => "result", + }); + + type Id = InferArtifactId; + const id: Id = "my-artifact"; + + expect(id).toBe("my-artifact"); + }); + + it("should work with const assertion", () => { + const artifact = definePipelineArtifact({ + id: "specific-id" as const, + build: async () => 123, + }); + + type Id = InferArtifactId; + const id: Id = "specific-id"; + + expect(id).toBe("specific-id"); + }); + }); + + describe("InferArtifactValue", () => { + it("should infer artifact value type", () => { + const artifact = definePipelineArtifact({ + id: "test", + build: async (): Promise<{ count: number }> => ({ count: 42 }), + }); + + type Value = InferArtifactValue; + const value: Value = { count: 42 }; + + expect(value).toEqual({ count: 42 }); + }); + + it("should work with primitive return types", () => { + const stringArtifact = definePipelineArtifact({ + id: "string-artifact", + build: async (): Promise => "result", + }); + + type StringValue = InferArtifactValue; + const str: StringValue = "test"; + + expect(str).toBe("test"); + }); + }); + + describe("InferArtifactsMap", () => { + it("should infer map of artifact ids to values", () => { + const artifacts = [ + definePipelineArtifact({ + id: "counts", + build: async (): Promise => 42, + }), + definePipelineArtifact({ + id: "names", + build: async (): Promise => ["a", "b"], + }), + ] as const; + + type ArtifactsMap = InferArtifactsMap; + + const map: ArtifactsMap = { + counts: 42, + names: ["a", "b"], + }; + + expect(map).toEqual({ + counts: 42, + names: ["a", "b"], + }); + }); + + it("should work with complex value types", () => { + const artifacts = [ + definePipelineArtifact({ + id: "data", + build: async (): Promise<{ items: string[]; total: number }> => ({ + items: ["x", "y"], + total: 2, + }), + }), + definePipelineArtifact({ + id: "enabled", + build: async (): Promise => true, + }), + ] as const; + + type ArtifactsMap = InferArtifactsMap; + + const map: ArtifactsMap = { + data: { items: ["x", "y"], total: 2 }, + enabled: true, + }; + + expect(map.data.items).toHaveLength(2); + expect(map.enabled).toBe(true); + }); + }); +}); + +describe("build context", () => { + it("should receive version in build context", async () => { + const build = vi.fn(async (ctx: ArtifactBuildContext) => { + return { processedVersion: ctx.version }; + }); + + const artifact = definePipelineArtifact({ + id: "version-aware", + build, + }); + + const context: ArtifactBuildContext = { version: "16.0.0" }; + const result = await artifact.build(context); + + expect(build).toHaveBeenCalledWith(context, undefined); + expect(result).toEqual({ processedVersion: "16.0.0" }); + }); + + it("should receive rows when parser is provided", async () => { + const mockRows: ParsedRow[] = [ + { sourceFile: "test.txt", kind: "point", codePoint: "0041", value: "a" }, + { sourceFile: "test.txt", kind: "point", codePoint: "0042", value: "b" }, + ]; + + async function* mockParser(): AsyncIterable { + for (const row of mockRows) { + yield row; + } + } + + const build = vi.fn(async (ctx: ArtifactBuildContext, rows?: AsyncIterable) => { + const collected: ParsedRow[] = []; + if (rows) { + for await (const row of rows) { + collected.push(row); + } + } + return { version: ctx.version, rowCount: collected.length }; + }); + + const artifact = definePipelineArtifact({ + id: "row-processor", + parser: mockParser, + build, + }); + + const context: ArtifactBuildContext = { version: "16.0.0" }; + const rows = mockParser(); + const result = await artifact.build(context, rows); + + expect(result).toEqual({ + version: "16.0.0", + rowCount: 2, + }); + }); +}); diff --git a/packages/pipelines/pipeline-artifacts/test/schema.test.ts b/packages/pipelines/pipeline-artifacts/test/schema.test.ts new file mode 100644 index 000000000..aa92edde1 --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/test/schema.test.ts @@ -0,0 +1,278 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { + artifact, + isGlobalArtifact, + isVersionArtifact, + type Artifact, + type ArtifactDefinition, + type GlobalArtifact, + type InferArtifactSchemaType, + type InferEmittedArtifacts, +} from "../src/schema"; + +describe("artifact", () => { + it("should create version artifact by default", () => { + const schema = z.string(); + const result = artifact(schema); + + expect(result).toEqual({ + _type: "artifact", + schema, + scope: "version", + }); + }); + + it("should create version artifact when explicitly specified", () => { + const schema = z.number(); + const result = artifact(schema, "version"); + + expect(result).toEqual({ + _type: "artifact", + schema, + scope: "version", + }); + }); + + it("should create global artifact when scope is global", () => { + const schema = z.boolean(); + const result = artifact(schema, "global"); + + expect(result).toEqual({ + _type: "global-artifact", + schema, + scope: "global", + }); + }); + + it("should work with complex schemas", () => { + const schema = z.object({ + id: z.string(), + count: z.number(), + metadata: z.record(z.string(), z.unknown()), + }); + const result = artifact(schema, "version"); + + expect(result.schema).toBe(schema); + expect(result.scope).toBe("version"); + }); + + it("should work with array schemas", () => { + const schema = z.array(z.string()); + const result = artifact(schema); + + expect(result.schema).toBe(schema); + expect(result._type).toBe("artifact"); + }); + + it("should work with union schemas", () => { + const schema = z.union([z.string(), z.number()]); + const result = artifact(schema, "global"); + + expect(result.schema).toBe(schema); + expect(result._type).toBe("global-artifact"); + }); +}); + +describe("isGlobalArtifact", () => { + it("should return true for global artifacts", () => { + const schema = z.string(); + const globalArt: ArtifactDefinition = { + _type: "global-artifact", + schema, + scope: "global", + }; + + expect(isGlobalArtifact(globalArt)).toBe(true); + }); + + it("should return false for version artifacts", () => { + const schema = z.string(); + const versionArt = artifact(schema, "version"); + + expect(isGlobalArtifact(versionArt)).toBe(false); + }); + + it("should work as type guard", () => { + const schema = z.object({ value: z.string() }); + const art: ArtifactDefinition = artifact(schema, "global"); + + if (isGlobalArtifact(art)) { + expect(art._type).toBe("global-artifact"); + expect(art.scope).toBe("global"); + } else { + throw new Error("Expected global artifact"); + } + }); +}); + +describe("isVersionArtifact", () => { + it("should return true for version artifacts", () => { + const schema = z.number(); + const versionArt: Artifact = { + _type: "artifact", + schema, + scope: "version", + }; + + expect(isVersionArtifact(versionArt)).toBe(true); + }); + + it("should return false for global artifacts", () => { + const schema = z.number(); + const globalArt: GlobalArtifact = { + _type: "global-artifact", + schema, + scope: "global", + }; + + expect(isVersionArtifact(globalArt)).toBe(false); + }); + + it("should work as type guard", () => { + const schema = z.object({ count: z.number() }); + const art: ArtifactDefinition = artifact(schema, "version"); + + if (isVersionArtifact(art)) { + expect(art._type).toBe("artifact"); + expect(art.scope).toBe("version"); + } else { + throw new Error("Expected version artifact"); + } + }); +}); + +describe("type inference", () => { + describe("InferArtifactSchemaType", () => { + it("should infer schema type correctly", () => { + const schema = z.object({ + id: z.string(), + count: z.number(), + }); + const art = artifact(schema); + + type Inferred = InferArtifactSchemaType; + type Expected = { id: string; count: number }; + + const assertType: Inferred = { id: "test", count: 42 }; + const _checkType: Expected = assertType; + + expect(assertType).toEqual({ id: "test", count: 42 }); + }); + + it("should work with primitive types", () => { + const stringArt = artifact(z.string()); + type StringType = InferArtifactSchemaType; + + const value: StringType = "hello"; + expect(value).toBe("hello"); + }); + }); + + describe("InferEmittedArtifacts", () => { + it("should infer multiple artifact types", () => { + const emits = { + result: artifact(z.object({ value: z.string() })), + count: artifact(z.number()), + enabled: artifact(z.boolean()), + } as const; + + type Inferred = InferEmittedArtifacts; + + const artifacts: Inferred = { + result: { value: "test" }, + count: 42, + enabled: true, + }; + + expect(artifacts).toEqual({ + result: { value: "test" }, + count: 42, + enabled: true, + }); + }); + + it("should work with global and version artifacts", () => { + const emits = { + global: artifact(z.string(), "global"), + version: artifact(z.number(), "version"), + } as const; + + type Inferred = InferEmittedArtifacts; + + const artifacts: Inferred = { + global: "test", + version: 123, + }; + + expect(artifacts).toEqual({ + global: "test", + version: 123, + }); + }); + + it("should work with complex nested schemas", () => { + const emits = { + data: artifact( + z.object({ + items: z.array(z.string()), + metadata: z.record(z.string(), z.unknown()), + }), + ), + } as const; + + type Inferred = InferEmittedArtifacts; + + const artifacts: Inferred = { + data: { + items: ["a", "b", "c"], + metadata: { key: "value" }, + }, + }; + + expect(artifacts.data.items).toHaveLength(3); + }); + }); +}); + +describe("schema validation", () => { + it("should validate data with artifact schema", () => { + const schema = z.object({ + name: z.string(), + age: z.number(), + }); + const art = artifact(schema); + + const validData = { name: "John", age: 30 }; + const result = art.schema.parse(validData); + + expect(result).toEqual(validData); + }); + + it("should reject invalid data", () => { + const schema = z.object({ + name: z.string(), + age: z.number(), + }); + const art = artifact(schema); + + const invalidData = { name: "John", age: "thirty" }; + const result = art.schema.safeParse(invalidData); + + expect(result.success).toBe(false); + }); + + it("should work with optional fields", () => { + const schema = z.object({ + required: z.string(), + optional: z.number().optional(), + }); + const art = artifact(schema); + + const data1 = { required: "test" }; + const data2 = { required: "test", optional: 42 }; + + expect(art.schema.safeParse(data1).success).toBe(true); + expect(art.schema.safeParse(data2).success).toBe(true); + }); +}); diff --git a/packages/pipelines/pipeline-artifacts/tsconfig.build.json b/packages/pipelines/pipeline-artifacts/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-artifacts/tsconfig.json b/packages/pipelines/pipeline-artifacts/tsconfig.json new file mode 100644 index 000000000..89e9ed10b --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test", + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-artifacts/tsdown.config.ts b/packages/pipelines/pipeline-artifacts/tsdown.config.ts new file mode 100644 index 000000000..dee0149e6 --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/tsdown.config.ts @@ -0,0 +1,7 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/index.ts", + ], +}); diff --git a/packages/pipelines/pipeline-core/src/cache.ts b/packages/pipelines/pipeline-core/src/cache.ts deleted file mode 100644 index 4a6b67176..000000000 --- a/packages/pipelines/pipeline-core/src/cache.ts +++ /dev/null @@ -1,136 +0,0 @@ -export interface CacheKey { - /** - * The route ID that produced this cache entry. - */ - routeId: string; - - /** - * The Unicode version being processed. - */ - version: string; - - /** - * Hash of the input file content. - */ - inputHash: string; - - /** - * Hashes of artifact dependencies used by this route. - * Key is artifact ID, value is the artifact's content hash. - */ - artifactHashes: Record; -} - -export type SerializedCacheKey = `${string}|${string}|${string}|${string}`; - -export interface CacheEntry { - /** - * The cache key that identifies this entry. - */ - key: CacheKey; - - /** - * The cached output data. - */ - output: TOutput[]; - - /** - * Artifacts produced by this route during execution. - * Key is artifact ID, value is the serialized artifact. - */ - producedArtifacts: Record; - - /** - * Timestamp when this entry was created (ISO 8601). - */ - createdAt: string; - - /** - * Optional metadata about the cache entry. - */ - meta?: Record; -} - -export interface CacheStore { - /** - * Retrieve a cached entry by key. - * Returns undefined if not found. - */ - get: (key: CacheKey) => Promise; - - /** - * Store a cache entry. - */ - set: (entry: CacheEntry) => Promise; - - /** - * Check if a cache entry exists for the given key. - */ - has: (key: CacheKey) => Promise; - - /** - * Delete a cache entry by key. - * Returns true if the entry existed and was deleted. - */ - delete: (key: CacheKey) => Promise; - - /** - * Clear all cache entries. - */ - clear: () => Promise; -} - -export function serializeCacheKey(key: CacheKey): SerializedCacheKey { - const artifactHashStr = Object.entries(key.artifactHashes) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([id, hash]) => `${id}:${hash}`) - .join(","); - - return `${key.routeId}|${key.version}|${key.inputHash}|${artifactHashStr}` as SerializedCacheKey; -} - -export function deserializeCacheKey(serialized: SerializedCacheKey): CacheKey { - const [routeId, version, inputHash, artifactHashStr] = serialized.split("|") as [ - string, - string, - string, - string - ] - - const artifactHashes: Record = {}; - if (artifactHashStr) { - for (const pair of artifactHashStr.split(",")) { - const [id, hash] = pair.split(":"); - artifactHashes[id!] = hash!; - } - } - - return { - routeId, - version, - inputHash, - artifactHashes, - }; -} - -export function areCacheKeysEqual(a: CacheKey, b: CacheKey): boolean { - if (a.routeId !== b.routeId) return false; - if (a.version !== b.version) return false; - if (a.inputHash !== b.inputHash) return false; - - const aKeys = Object.keys(a.artifactHashes).sort(); - const bKeys = Object.keys(b.artifactHashes).sort(); - if (aKeys.length !== bKeys.length) return false; - - for (let i = 0; i < aKeys.length; i++) { - const key = aKeys[i]!; - if (key !== bKeys[i]) return false; - if (a.artifactHashes[key] !== b.artifactHashes[key]) return false; - } - - return true; -} - -export function defineCacheStore(store: CacheStore): CacheStore { - return store; -} diff --git a/packages/pipelines/pipeline-core/src/dependencies.ts b/packages/pipelines/pipeline-core/src/dependencies.ts new file mode 100644 index 000000000..7f07f608e --- /dev/null +++ b/packages/pipelines/pipeline-core/src/dependencies.ts @@ -0,0 +1,70 @@ +type RouteDependency = `route:${string}`; +type ArtifactDependency = `artifact:${string}:${string}`; + +export type PipelineDependency = RouteDependency | ArtifactDependency; + +export interface ParsedRouteDependency { + type: "route"; + routeId: string; +} + +export interface ParsedArtifactDependency { + type: "artifact"; + routeId: string; + artifactName: string; +} + +export type ParsedDependency = ParsedRouteDependency | ParsedArtifactDependency; + +export type ParseDependencyType = + T extends `route:${infer RouteId}` + ? { type: "route"; routeId: RouteId } + : T extends `artifact:${infer RouteId}:${infer ArtifactName}` + ? { type: "artifact"; routeId: RouteId; artifactName: ArtifactName } + : never; + +export type ExtractRouteDependencies = { + [K in keyof T]: T[K] extends `route:${infer RouteId}` ? RouteId : never; +}[number]; + +export type ExtractArtifactDependencies = { + [K in keyof T]: T[K] extends `artifact:${infer RouteId}:${infer ArtifactName}` + ? { routeId: RouteId; artifactName: ArtifactName } + : never; +}[number]; + +export type ExtractArtifactKeys = { + [K in keyof T]: T[K] extends `artifact:${infer RouteId}:${infer ArtifactName}` + ? `${RouteId}:${ArtifactName}` + : never; +}[number]; + +export function parseDependency(dep: PipelineDependency): ParsedDependency { + const parts = dep.split(":"); + + if (parts[0] === "route" && parts[1]) { + return { type: "route", routeId: parts[1] }; + } + + if (parts[0] === "artifact" && parts[1] && parts[2]) { + return { type: "artifact", routeId: parts[1], artifactName: parts[2] }; + } + + throw new Error(`Invalid dependency format: ${dep}. Expected "route:" or "artifact::"`); +} + +export function isRouteDependency(dep: PipelineDependency): dep is RouteDependency { + return dep.startsWith("route:"); +} + +export function isArtifactDependency(dep: PipelineDependency): dep is ArtifactDependency { + return dep.startsWith("artifact:"); +} + +export function createRouteDependency(routeId: string): RouteDependency { + return `route:${routeId}`; +} + +export function createArtifactDependency(routeId: string, artifactName: string): ArtifactDependency { + return `artifact:${routeId}:${artifactName}`; +} diff --git a/packages/pipelines/pipeline-core/src/events.ts b/packages/pipelines/pipeline-core/src/events.ts new file mode 100644 index 000000000..3bf9079d9 --- /dev/null +++ b/packages/pipelines/pipeline-core/src/events.ts @@ -0,0 +1,215 @@ +import type { FileContext } from "./types"; + +export type PipelineEventType = + | "pipeline:start" + | "pipeline:end" + | "version:start" + | "version:end" + | "artifact:start" + | "artifact:end" + | "artifact:produced" + | "artifact:consumed" + | "file:matched" + | "file:skipped" + | "file:fallback" + | "parse:start" + | "parse:end" + | "resolve:start" + | "resolve:end" + | "cache:hit" + | "cache:miss" + | "cache:store" + | "error"; + +export type PipelineStartEvent = { + type: "pipeline:start"; + versions: string[]; + timestamp: number; +}; + +export type PipelineEndEvent = { + type: "pipeline:end"; + durationMs: number; + timestamp: number; +}; + +export type VersionStartEvent = { + type: "version:start"; + version: string; + timestamp: number; +}; + +export type VersionEndEvent = { + type: "version:end"; + version: string; + durationMs: number; + timestamp: number; +}; + +export type ArtifactStartEvent = { + type: "artifact:start"; + artifactId: string; + version: string; + timestamp: number; +}; + +export type ArtifactEndEvent = { + type: "artifact:end"; + artifactId: string; + version: string; + durationMs: number; + timestamp: number; +}; + +export type ArtifactProducedEvent = { + type: "artifact:produced"; + artifactId: string; + routeId: string; + version: string; + timestamp: number; +}; + +export type ArtifactConsumedEvent = { + type: "artifact:consumed"; + artifactId: string; + routeId: string; + version: string; + timestamp: number; +}; + +export type FileMatchedEvent = { + type: "file:matched"; + file: FileContext; + routeId: string; + timestamp: number; +}; + +export type FileSkippedEvent = { + type: "file:skipped"; + file: FileContext; + reason: "no-match" | "filtered"; + timestamp: number; +}; + +export type FileFallbackEvent = { + type: "file:fallback"; + file: FileContext; + timestamp: number; +}; + +export type ParseStartEvent = { + type: "parse:start"; + file: FileContext; + routeId: string; + timestamp: number; +}; + +export type ParseEndEvent = { + type: "parse:end"; + file: FileContext; + routeId: string; + rowCount: number; + durationMs: number; + timestamp: number; +}; + +export type ResolveStartEvent = { + type: "resolve:start"; + file: FileContext; + routeId: string; + timestamp: number; +}; + +export type ResolveEndEvent = { + type: "resolve:end"; + file: FileContext; + routeId: string; + outputCount: number; + durationMs: number; + timestamp: number; +}; + +export type CacheHitEvent = { + type: "cache:hit"; + routeId: string; + file: FileContext; + version: string; + timestamp: number; +}; + +export type CacheMissEvent = { + type: "cache:miss"; + routeId: string; + file: FileContext; + version: string; + timestamp: number; +}; + +export type CacheStoreEvent = { + type: "cache:store"; + routeId: string; + file: FileContext; + version: string; + timestamp: number; +}; + +export type PipelineErrorEvent = { + type: "error"; + error: PipelineError; + timestamp: number; +}; + +export type PipelineEvent = + | PipelineStartEvent + | PipelineEndEvent + | VersionStartEvent + | VersionEndEvent + | ArtifactStartEvent + | ArtifactEndEvent + | ArtifactProducedEvent + | ArtifactConsumedEvent + | FileMatchedEvent + | FileSkippedEvent + | FileFallbackEvent + | ParseStartEvent + | ParseEndEvent + | ResolveStartEvent + | ResolveEndEvent + | CacheHitEvent + | CacheMissEvent + | CacheStoreEvent + | PipelineErrorEvent; + +export type PipelineErrorScope = "pipeline" | "version" | "file" | "route" | "artifact"; + +export interface PipelineError { + scope: PipelineErrorScope; + message: string; + error?: unknown; + file?: FileContext; + routeId?: string; + artifactId?: string; + version?: string; +} + +export type PipelineGraphNodeType = "source" | "file" | "route" | "artifact" | "output"; + +export type PipelineGraphNode = + | { id: string; type: "source"; version: string } + | { id: string; type: "file"; file: FileContext } + | { id: string; type: "route"; routeId: string } + | { id: string; type: "artifact"; artifactId: string } + | { id: string; type: "output"; outputIndex: number; property?: string }; + +export type PipelineGraphEdgeType = "provides" | "matched" | "parsed" | "resolved" | "uses-artifact"; + +export interface PipelineGraphEdge { + from: string; + to: string; + type: PipelineGraphEdgeType; +} + +export interface PipelineGraph { + nodes: PipelineGraphNode[]; + edges: PipelineGraphEdge[]; +} diff --git a/packages/pipelines/pipeline-core/src/filters.ts b/packages/pipelines/pipeline-core/src/filters.ts new file mode 100644 index 000000000..7adb1c7a0 --- /dev/null +++ b/packages/pipelines/pipeline-core/src/filters.ts @@ -0,0 +1,62 @@ +import type { FileContext, PipelineFilter } from "./types"; +import picomatch from "picomatch"; + +export function byName(name: string): PipelineFilter { + return (ctx) => ctx.file.name === name; +} + +export function byDir(dir: FileContext["dir"]): PipelineFilter { + return (ctx) => ctx.file.dir === dir; +} + +export function byExt(ext: string): PipelineFilter { + if (ext === "") { + return (ctx) => ctx.file.ext === ""; + } + const normalizedExt = ext.startsWith(".") ? ext : `.${ext}`; + return (ctx) => ctx.file.ext === normalizedExt; +} + +export function byGlob(pattern: string): PipelineFilter { + const matcher = picomatch(pattern); + return (ctx) => matcher(ctx.file.path); +} + +export function byPath(pathPattern: string | RegExp): PipelineFilter { + if (typeof pathPattern === "string") { + return (ctx) => ctx.file.path === pathPattern; + } + return (ctx) => pathPattern.test(ctx.file.path); +} + +export function byProp(pattern: string | RegExp): PipelineFilter { + if (typeof pattern === "string") { + return (ctx) => ctx.row?.property === pattern; + } + return (ctx) => !!ctx.row?.property && pattern.test(ctx.row.property); +} + +export function bySource(sourceIds: string | string[]): PipelineFilter { + const ids = Array.isArray(sourceIds) ? sourceIds : [sourceIds]; + return (ctx) => ctx.source != null && ids.includes(ctx.source.id); +} + +export function and(...filters: PipelineFilter[]): PipelineFilter { + return (ctx) => filters.every((f) => f(ctx)); +} + +export function or(...filters: PipelineFilter[]): PipelineFilter { + return (ctx) => filters.some((f) => f(ctx)); +} + +export function not(filter: PipelineFilter): PipelineFilter { + return (ctx) => !filter(ctx); +} + +export function always(): PipelineFilter { + return () => true; +} + +export function never(): PipelineFilter { + return () => false; +} diff --git a/packages/pipelines/pipeline-core/src/index.ts b/packages/pipelines/pipeline-core/src/index.ts index e69de29bb..5f7abf61b 100644 --- a/packages/pipelines/pipeline-core/src/index.ts +++ b/packages/pipelines/pipeline-core/src/index.ts @@ -0,0 +1,141 @@ +export type { + FileContext, + RowContext, + FilterContext, + PipelineFilter, + ParsedRow, + ParseContext, + ParserFn, + ResolvedEntry, + DefaultRange, + PropertyJson, + ResolveContext, + ResolverFn, + RouteOutput, +} from "./types"; + +export { + byName, + byDir, + byExt, + byGlob, + byPath, + byProp, + bySource, + and, + or, + not, + always, + never, +} from "./filters"; + +export type { + PipelineDependency, + ParsedRouteDependency, + ParsedArtifactDependency, + ParsedDependency, + ParseDependencyType, + ExtractRouteDependencies, + ExtractArtifactDependencies, + ExtractArtifactKeys, +} from "./dependencies"; + +export { + parseDependency, + isRouteDependency, + isArtifactDependency, + createRouteDependency, + createArtifactDependency, +} from "./dependencies"; + +export type { + PipelineEventType, + PipelineStartEvent, + PipelineEndEvent, + VersionStartEvent, + VersionEndEvent, + ArtifactStartEvent, + ArtifactEndEvent, + ArtifactProducedEvent, + ArtifactConsumedEvent, + FileMatchedEvent, + FileSkippedEvent, + FileFallbackEvent, + ParseStartEvent, + ParseEndEvent, + ResolveStartEvent, + ResolveEndEvent, + CacheHitEvent, + CacheMissEvent, + CacheStoreEvent, + PipelineErrorEvent, + PipelineEvent, + PipelineErrorScope, + PipelineError, + PipelineGraphNodeType, + PipelineGraphNode, + PipelineGraphEdgeType, + PipelineGraphEdge, + PipelineGraph, +} from "./events"; + +export type { + StreamOptions, + FileMetadata, + SourceBackend, + PipelineSourceDefinition, + SourceFileContext, + InferSourceId, + InferSourceIds, +} from "./source"; + +export { + definePipelineSource, + resolveSourceFiles, + resolveMultipleSourceFiles, +} from "./source"; + +export type { + TransformContext, + PipelineTransformDefinition, + InferTransformInput, + InferTransformOutput, + ChainTransforms, +} from "./transform"; + +export { + definePipelineTransform, + applyTransforms, +} from "./transform"; + +export type { + ArtifactDefinition, + InferArtifactType, + RouteResolveContext, + PipelineRouteDefinition, + InferRouteId, + InferRouteDepends, + InferRouteEmits, + InferRouteTransforms, + InferRouteOutput, + InferRoutesOutput, + InferEmittedArtifactsFromRoute, +} from "./route"; + +export { definePipelineRoute } from "./route"; + +export type { + FallbackRouteDefinition, + PipelineDefinitionOptions, + PipelineDefinition, + InferPipelineOutput, + InferPipelineSourceIds, + InferPipelineRouteIds, +} from "./pipeline"; + +export { + definePipeline, + isPipelineDefinition, + getPipelineRouteIds, + getPipelineSourceIds, +} from "./pipeline"; diff --git a/packages/pipelines/pipeline-core/src/pipeline.ts b/packages/pipelines/pipeline-core/src/pipeline.ts new file mode 100644 index 000000000..4018d43fd --- /dev/null +++ b/packages/pipelines/pipeline-core/src/pipeline.ts @@ -0,0 +1,269 @@ +import type { PipelineEvent } from "./events"; +import type { PipelineRouteDefinition, InferRoutesOutput } from "./route"; +import type { PipelineSourceDefinition, InferSourceIds } from "./source"; +import type { PipelineFilter, ParseContext, ParsedRow, ResolveContext } from "./types"; + +/** + * Fallback route definition for files that don't match any explicit route. + */ +export interface FallbackRouteDefinition< + TArtifacts extends Record = Record, + TOutput = unknown, +> { + /** + * Optional filter to restrict which unmatched files the fallback handles. + */ + filter?: PipelineFilter; + + /** + * Parser function that yields parsed rows from file content. + */ + parser: (ctx: ParseContext) => AsyncIterable; + + /** + * Resolver function that transforms parsed rows into output. + */ + resolver: (ctx: ResolveContext, rows: AsyncIterable) => Promise; +} + +/** + * Options for defining a pipeline. + * This is a pure data structure with no execution logic. + */ +export interface PipelineDefinitionOptions< + TSources extends readonly PipelineSourceDefinition[] = readonly PipelineSourceDefinition[], + TRoutes extends readonly PipelineRouteDefinition[] = readonly PipelineRouteDefinition[], + TFallback extends FallbackRouteDefinition | undefined = undefined, +> { + /** + * Unique identifier for the pipeline. + */ + id: string; + + /** + * Human-readable name for the pipeline. + */ + name?: string; + + /** + * Description of what this pipeline does. + */ + description?: string; + + /** + * Unicode versions this pipeline processes. + */ + versions: string[]; + + /** + * Input sources that provide files to the pipeline. + */ + inputs: TSources; + + /** + * Routes that process matched files. + */ + routes: TRoutes; + + /** + * Global filter to include/exclude files before routing. + */ + include?: PipelineFilter; + + /** + * If true, throw error for files with no matching route (and no fallback). + * @default false + */ + strict?: boolean; + + /** + * Maximum concurrent route executions. + * @default 4 + */ + concurrency?: number; + + /** + * Fallback handler for files that don't match any route. + */ + fallback?: TFallback; + + /** + * Event handler for pipeline events. + * Note: This is stored but not invoked by the definition itself. + * The executor is responsible for calling this. + */ + onEvent?: (event: PipelineEvent) => void | Promise; +} + +/** + * A pipeline definition is a pure data structure that describes + * how to process Unicode Character Database files. + * + * It does NOT contain execution logic - that is handled by the executor. + */ +export interface PipelineDefinition< + TId extends string = string, + TSources extends readonly PipelineSourceDefinition[] = readonly PipelineSourceDefinition[], + TRoutes extends readonly PipelineRouteDefinition[] = readonly PipelineRouteDefinition[], + TFallback extends FallbackRouteDefinition | undefined = undefined, +> { + /** + * Marker to identify this as a pipeline definition. + */ + readonly _type: "pipeline-definition"; + + /** + * Unique identifier for the pipeline. + */ + readonly id: TId; + + /** + * Human-readable name for the pipeline. + */ + readonly name?: string; + + /** + * Description of what this pipeline does. + */ + readonly description?: string; + + /** + * Unicode versions this pipeline processes. + */ + readonly versions: string[]; + + /** + * Input sources that provide files to the pipeline. + */ + readonly inputs: TSources; + + /** + * Routes that process matched files. + */ + readonly routes: TRoutes; + + /** + * Global filter to include/exclude files before routing. + */ + readonly include?: PipelineFilter; + + /** + * If true, throw error for files with no matching route (and no fallback). + */ + readonly strict: boolean; + + /** + * Maximum concurrent route executions. + */ + readonly concurrency: number; + + /** + * Fallback handler for files that don't match any route. + */ + readonly fallback?: TFallback; + + /** + * Event handler for pipeline events. + */ + readonly onEvent?: (event: PipelineEvent) => void | Promise; +} + +/** + * Infer the output type of a pipeline definition. + */ +export type InferPipelineOutput< + TRoutes extends readonly PipelineRouteDefinition[], + TFallback extends FallbackRouteDefinition | undefined, +> = TFallback extends FallbackRouteDefinition + ? InferRoutesOutput | TFallbackOutput + : InferRoutesOutput; + +/** + * Infer the source IDs from a pipeline definition. + */ +export type InferPipelineSourceIds = T extends PipelineDefinition + ? InferSourceIds + : never; + +/** + * Infer the route IDs from a pipeline definition. + */ +export type InferPipelineRouteIds = T extends PipelineDefinition + ? TRoutes[number] extends PipelineRouteDefinition + ? TId + : never + : never; + +/** + * Define a pipeline configuration. + * + * This returns a pure data structure describing the pipeline. + * To execute the pipeline, pass it to a pipeline executor. + * + * @example + * ```ts + * const pipeline = definePipeline({ + * id: "my-pipeline", + * versions: ["16.0.0"], + * inputs: [mySource], + * routes: [myRoute], + * }); + * + * // Execute with an executor + * const executor = createPipelineExecutor({ pipelines: [pipeline] }); + * const result = await executor.run(); + * ``` + */ +export function definePipeline< + const TId extends string, + const TSources extends readonly PipelineSourceDefinition[], + const TRoutes extends readonly PipelineRouteDefinition[], + TFallback extends FallbackRouteDefinition | undefined = undefined, +>( + options: PipelineDefinitionOptions & { id: TId }, +): PipelineDefinition { + return { + _type: "pipeline-definition", + id: options.id, + name: options.name, + description: options.description, + versions: options.versions, + inputs: options.inputs, + routes: options.routes, + include: options.include, + strict: options.strict ?? false, + concurrency: options.concurrency ?? 4, + fallback: options.fallback, + onEvent: options.onEvent, + }; +} + +/** + * Type guard to check if a value is a pipeline definition. + */ +export function isPipelineDefinition(value: unknown): value is PipelineDefinition { + return ( + typeof value === "object" + && value !== null + && "_type" in value + && (value as { _type: unknown })._type === "pipeline-definition" + ); +} + +/** + * Extract route IDs from a pipeline definition. + */ +export function getPipelineRouteIds( + pipeline: T, +): string[] { + return pipeline.routes.map((route) => route.id); +} + +/** + * Extract source IDs from a pipeline definition. + */ +export function getPipelineSourceIds( + pipeline: T, +): string[] { + return pipeline.inputs.map((source) => source.id); +} diff --git a/packages/pipelines/pipeline-core/src/route.ts b/packages/pipelines/pipeline-core/src/route.ts new file mode 100644 index 000000000..32b78a9bf --- /dev/null +++ b/packages/pipelines/pipeline-core/src/route.ts @@ -0,0 +1,97 @@ +import type { z } from "zod"; +import type { ExtractArtifactKeys, PipelineDependency } from "./dependencies"; +import type { ChainTransforms, PipelineTransformDefinition } from "./transform"; +import type { + FileContext, + ParserFn, + ParsedRow, + PipelineFilter, + PropertyJson, + RouteOutput, +} from "./types"; + +export interface ArtifactDefinition { + _type: "artifact" | "global-artifact"; + schema: TSchema; + scope: "version" | "global"; +} + +export type InferArtifactType = + T extends ArtifactDefinition ? z.infer : never; + +export interface RouteResolveContext< + TArtifactKeys extends string = string, + TEmits extends Record = Record, +> { + version: string; + file: FileContext; + getArtifact: (key: K) => unknown; + emitArtifact: ( + key: K, + value: InferArtifactType, + ) => void; + normalizeEntries: (entries: Array<{ range?: string; codePoint?: string; sequence?: string[]; value: string | string[] }>) => Array<{ range?: string; codePoint?: string; sequence?: string[]; value: string | string[] }>; + now: () => string; +} + +export interface PipelineRouteDefinition< + TId extends string = string, + TDepends extends readonly PipelineDependency[] = readonly PipelineDependency[], + TEmits extends Record = Record, + TTransforms extends readonly PipelineTransformDefinition[] = readonly PipelineTransformDefinition[], + TOutput = PropertyJson[], +> { + id: TId; + filter: PipelineFilter; + depends?: TDepends; + emits?: TEmits; + parser: ParserFn; + transforms?: TTransforms; + resolver: ( + ctx: RouteResolveContext, TEmits>, + rows: AsyncIterable>, + ) => Promise; + out?: RouteOutput; + cache?: boolean; +} + +export function definePipelineRoute< + const TId extends string, + const TDepends extends readonly PipelineDependency[] = readonly [], + const TEmits extends Record = Record, + const TTransforms extends readonly PipelineTransformDefinition[] = readonly [], + TOutput = PropertyJson[], +>( + definition: PipelineRouteDefinition, +): PipelineRouteDefinition { + return definition; +} + +export type InferRouteId = T extends PipelineRouteDefinition + ? TId + : never; + +export type InferRouteDepends = T extends PipelineRouteDefinition + ? TDepends + : never; + +export type InferRouteEmits = T extends PipelineRouteDefinition + ? TEmits + : never; + +export type InferRouteTransforms = T extends PipelineRouteDefinition + ? TTransforms + : never; + +export type InferRouteOutput = T extends PipelineRouteDefinition + ? TOutput + : never; + +export type InferRoutesOutput[]> = + T[number] extends PipelineRouteDefinition + ? TOutput extends unknown[] ? TOutput[number] : TOutput + : never; + +export type InferEmittedArtifactsFromRoute = T extends PipelineRouteDefinition + ? { [K in keyof TEmits]: TEmits[K] extends ArtifactDefinition ? z.infer : never } + : never; diff --git a/packages/pipelines/pipeline-core/src/source.ts b/packages/pipelines/pipeline-core/src/source.ts new file mode 100644 index 000000000..a94553b11 --- /dev/null +++ b/packages/pipelines/pipeline-core/src/source.ts @@ -0,0 +1,87 @@ +import type { FileContext, PipelineFilter } from "./types"; + +export interface StreamOptions { + chunkSize?: number; + start?: number; + end?: number; +} + +export interface FileMetadata { + size: number; + hash?: string; + lastModified?: string; +} + +export interface SourceBackend { + listFiles(version: string): Promise; + readFile(file: FileContext): Promise; + readFileStream?(file: FileContext, options?: StreamOptions): AsyncIterable; + getMetadata?(file: FileContext): Promise; +} + +export interface PipelineSourceDefinition { + id: TId; + backend: SourceBackend; + includes?: PipelineFilter; + excludes?: PipelineFilter; +} + +export interface SourceFileContext extends FileContext { + source: { + id: string; + }; +} + +export function definePipelineSource( + definition: PipelineSourceDefinition, +): PipelineSourceDefinition { + return definition; +} + +export async function resolveSourceFiles( + source: PipelineSourceDefinition, + version: string, +): Promise { + const allFiles = await source.backend.listFiles(version); + + const filteredFiles = allFiles.filter((file) => { + const ctx = { file }; + + if (source.includes && !source.includes(ctx)) { + return false; + } + + if (source.excludes && source.excludes(ctx)) { + return false; + } + + return true; + }); + + return filteredFiles.map((file) => ({ + ...file, + source: { id: source.id }, + })); +} + +export async function resolveMultipleSourceFiles( + sources: PipelineSourceDefinition[], + version: string, +): Promise { + const filesByPath = new Map(); + + for (const source of sources) { + const files = await resolveSourceFiles(source, version); + for (const file of files) { + filesByPath.set(file.path, file); + } + } + + return Array.from(filesByPath.values()); +} + +export type InferSourceId = T extends PipelineSourceDefinition ? TId : never; + +export type InferSourceIds = { + [K in keyof T]: InferSourceId; +}[number]; diff --git a/packages/pipelines/pipeline-core/src/transform.ts b/packages/pipelines/pipeline-core/src/transform.ts new file mode 100644 index 000000000..0c3725471 --- /dev/null +++ b/packages/pipelines/pipeline-core/src/transform.ts @@ -0,0 +1,144 @@ +import type { FileContext } from "./types"; + +export interface TransformContext { + version: string; + file: FileContext; +} + +export interface PipelineTransformDefinition { + id: string; + fn: (ctx: TransformContext, rows: AsyncIterable) => AsyncIterable; +} + +export function definePipelineTransform( + definition: PipelineTransformDefinition, +): PipelineTransformDefinition { + return definition; +} + +export type InferTransformInput = + T extends PipelineTransformDefinition ? TInput : never; + +export type InferTransformOutput = + T extends PipelineTransformDefinition ? TOutput : never; + +type ChainTwo = T1 extends PipelineTransformDefinition + ? T2 extends PipelineTransformDefinition + ? O2 + : never + : never; + +type ChainThree = T1 extends PipelineTransformDefinition + ? T2 extends PipelineTransformDefinition + ? T3 extends PipelineTransformDefinition + ? O3 + : never + : never + : never; + +type ChainFour = T1 extends PipelineTransformDefinition + ? T2 extends PipelineTransformDefinition + ? T3 extends PipelineTransformDefinition + ? T4 extends PipelineTransformDefinition + ? O4 + : never + : never + : never + : never; + +type ChainFive = T1 extends PipelineTransformDefinition + ? T2 extends PipelineTransformDefinition + ? T3 extends PipelineTransformDefinition + ? T4 extends PipelineTransformDefinition + ? T5 extends PipelineTransformDefinition + ? O5 + : never + : never + : never + : never + : never; + +type ChainSix = T1 extends PipelineTransformDefinition + ? T2 extends PipelineTransformDefinition + ? T3 extends PipelineTransformDefinition + ? T4 extends PipelineTransformDefinition + ? T5 extends PipelineTransformDefinition + ? T6 extends PipelineTransformDefinition + ? O6 + : never + : never + : never + : never + : never + : never; + +type ChainSeven = T1 extends PipelineTransformDefinition + ? T2 extends PipelineTransformDefinition + ? T3 extends PipelineTransformDefinition + ? T4 extends PipelineTransformDefinition + ? T5 extends PipelineTransformDefinition + ? T6 extends PipelineTransformDefinition + ? T7 extends PipelineTransformDefinition + ? O7 + : never + : never + : never + : never + : never + : never + : never; + +type ChainEight = T1 extends PipelineTransformDefinition + ? T2 extends PipelineTransformDefinition + ? T3 extends PipelineTransformDefinition + ? T4 extends PipelineTransformDefinition + ? T5 extends PipelineTransformDefinition + ? T6 extends PipelineTransformDefinition + ? T7 extends PipelineTransformDefinition + ? T8 extends PipelineTransformDefinition + ? O8 + : never + : never + : never + : never + : never + : never + : never + : never; + +export type ChainTransforms< + TInput, + TTransforms extends readonly PipelineTransformDefinition[], +> = TTransforms extends readonly [] + ? TInput + : TTransforms extends readonly [infer T1 extends PipelineTransformDefinition] + ? O1 + : TTransforms extends readonly [infer T1, infer T2] + ? ChainTwo + : TTransforms extends readonly [infer T1, infer T2, infer T3] + ? ChainThree + : TTransforms extends readonly [infer T1, infer T2, infer T3, infer T4] + ? ChainFour + : TTransforms extends readonly [infer T1, infer T2, infer T3, infer T4, infer T5] + ? ChainFive + : TTransforms extends readonly [infer T1, infer T2, infer T3, infer T4, infer T5, infer T6] + ? ChainSix + : TTransforms extends readonly [infer T1, infer T2, infer T3, infer T4, infer T5, infer T6, infer T7] + ? ChainSeven + : TTransforms extends readonly [infer T1, infer T2, infer T3, infer T4, infer T5, infer T6, infer T7, infer T8] + ? ChainEight + : unknown; + +export async function* applyTransforms( + ctx: TransformContext, + rows: AsyncIterable, + transforms: readonly PipelineTransformDefinition[], +): AsyncIterable { + let current: AsyncIterable = rows; + + for (const transform of transforms) { + current = transform.fn(ctx, current); + } + + yield* current; +} diff --git a/packages/pipelines/pipeline-core/src/types.ts b/packages/pipelines/pipeline-core/src/types.ts new file mode 100644 index 000000000..ff880642e --- /dev/null +++ b/packages/pipelines/pipeline-core/src/types.ts @@ -0,0 +1,288 @@ +/** + * Represents the context of a file being processed in the pipeline. + */ +export interface FileContext { + /** + * The Unicode version being processed (e.g., "16.0.0"). + */ + version: string; + + /** + * The directory category of the file. + */ + dir: "ucd" | "extracted" | "auxiliary" | "emoji" | "unihan" | string; + + /** + * The relative path from the version root (e.g., "ucd/LineBreak.txt"). + */ + path: string; + + /** + * The file name (e.g., "LineBreak.txt"). + */ + name: string; + + /** + * The file extension (e.g., ".txt"). + */ + ext: string; +} + +/** + * Context for a specific row/line within a file. + * Used during row-level filtering in multi-property files. + */ +export interface RowContext { + /** + * The property name for multi-property files (e.g., "NFKC_Casefold"). + */ + property?: string; +} + +/** + * Combined context passed to filter predicates. + * During file routing, only `file` is defined. + * During row filtering, both `file` and `row` are defined. + */ +export interface FilterContext { + /** + * The file context. + */ + file: FileContext; + + /** + * The row context (only defined during row-level filtering). + */ + row?: RowContext; + + /** + * The source context (only defined when using multiple sources). + */ + source?: { + /** + * The source ID. + */ + id: string; + }; +} + +/** + * A predicate function that determines if a file or row should be processed. + */ +export type PipelineFilter = (ctx: FilterContext) => boolean; + +/** + * A parsed row from a UCD file. + */ +export interface ParsedRow { + /** + * The source file path relative to the version root. + */ + sourceFile: string; + + /** + * The kind of entry. + */ + kind: "range" | "point" | "sequence" | "alias"; + + /** + * Start of range (hex string, e.g., "0041"). + */ + start?: string; + + /** + * End of range (hex string, e.g., "005A"). + */ + end?: string; + + /** + * Single code point (hex string). + */ + codePoint?: string; + + /** + * Sequence of code points (hex strings). + */ + sequence?: string[]; + + /** + * Property name for multi-property files. + */ + property?: string; + + /** + * The value(s) associated with this entry. + */ + value?: string | string[]; + + /** + * Additional metadata (comments, line numbers, etc.). + */ + meta?: Record; +} + +/** + * Context passed to parser functions. + */ +export interface ParseContext { + /** + * The file being parsed. + */ + file: FileContext; + + /** + * Read the raw content of the file. + */ + readContent: () => Promise; + + /** + * Read the file line by line. + */ + readLines: () => AsyncIterable; + + /** + * Check if a line is a comment. + */ + isComment: (line: string) => boolean; +} + +/** + * A parser function that converts file content to parsed rows. + */ +export type ParserFn = (ctx: ParseContext) => AsyncIterable; + +/** + * A resolved entry in the output JSON. + */ +export interface ResolvedEntry { + /** + * Range in "XXXX..YYYY" format (hex, inclusive). + */ + range?: `${string}..${string}`; + + /** + * Single code point in hex. + */ + codePoint?: string; + + /** + * Sequence of code points. + */ + sequence?: string[]; + + /** + * The value(s) for this entry. + */ + value: string | string[]; +} + +/** + * A default range from @missing declarations. + */ +export interface DefaultRange { + /** + * The range this default applies to. + */ + range: `${string}..${string}`; + + /** + * The default value. + */ + value: string | string[]; +} + +/** + * The standardized JSON output for a property. + */ +export interface PropertyJson { + /** + * The Unicode version (e.g., "16.0.0"). + */ + version: string; + + /** + * The property name (e.g., "Line_Break"). + */ + property: string; + + /** + * The source file name (e.g., "LineBreak.txt"). + */ + file: string; + + /** + * The resolved entries. + */ + entries: ResolvedEntry[]; + + /** + * Default ranges from @missing (in encounter order). + */ + defaults?: DefaultRange[]; + + /** + * Additional metadata. + */ + meta?: Record; +} + +/** + * Context passed to resolver functions. + */ +export interface ResolveContext = Record> { + /** + * The Unicode version being processed. + */ + version: string; + + /** + * The file being resolved. + */ + file: FileContext; + + /** + * Get an artifact by ID. + */ + getArtifact: (id: K) => TArtifacts[K]; + + /** + * Emit an artifact for subsequent routes. + */ + emitArtifact: (id: K, value: V) => void; + + /** + * Normalize and sort entries by code point range. + */ + normalizeEntries: (entries: ResolvedEntry[]) => ResolvedEntry[]; + + /** + * Get current timestamp in ISO 8601 format. + */ + now: () => string; +} + +/** + * A resolver function that converts parsed rows to property JSON. + */ +export type ResolverFn< + TArtifacts extends Record = Record, + TOutput = PropertyJson[], +> = ( + ctx: ResolveContext, + rows: AsyncIterable, +) => Promise; + +/** + * Output configuration for a route. + */ +export interface RouteOutput { + /** + * Custom output directory. + */ + dir?: string; + + /** + * Custom file name generator. + */ + fileName?: (pj: PropertyJson) => string; +} diff --git a/packages/pipelines/pipeline-core/test/dependencies.test.ts b/packages/pipelines/pipeline-core/test/dependencies.test.ts new file mode 100644 index 000000000..bbc03addf --- /dev/null +++ b/packages/pipelines/pipeline-core/test/dependencies.test.ts @@ -0,0 +1,304 @@ +import { describe, expect, it } from "vitest"; +import { + createArtifactDependency, + createRouteDependency, + isArtifactDependency, + isRouteDependency, + parseDependency, + type ExtractArtifactDependencies, + type ExtractArtifactKeys, + type ExtractRouteDependencies, + type ParsedArtifactDependency, + type ParsedDependency, + type ParsedRouteDependency, + type PipelineDependency, +} from "../src/dependencies"; + +describe("parseDependency", () => { + it("should parse route dependency", () => { + const result = parseDependency("route:my-route"); + + expect(result).toEqual({ + type: "route", + routeId: "my-route", + }); + }); + + it("should parse artifact dependency", () => { + const result = parseDependency("artifact:my-route:my-artifact"); + + expect(result).toEqual({ + type: "artifact", + routeId: "my-route", + artifactName: "my-artifact", + }); + }); + + it("should parse route with hyphens and underscores", () => { + const result = parseDependency("route:unicode-data_processor"); + + expect(result).toEqual({ + type: "route", + routeId: "unicode-data_processor", + }); + }); + + it("should parse artifact with complex names", () => { + const result = parseDependency("artifact:data-processor:normalized_output"); + + expect(result).toEqual({ + type: "artifact", + routeId: "data-processor", + artifactName: "normalized_output", + }); + }); + + it("should throw error for invalid format", () => { + expect(() => parseDependency("invalid" as PipelineDependency)).toThrow( + 'Invalid dependency format: invalid. Expected "route:" or "artifact::"', + ); + }); + + it("should throw error for route without id", () => { + expect(() => parseDependency("route:" as PipelineDependency)).toThrow( + 'Invalid dependency format: route:', + ); + }); + + it("should throw error for artifact without name", () => { + expect(() => parseDependency("artifact:my-route:" as PipelineDependency)).toThrow( + 'Invalid dependency format: artifact:my-route:', + ); + }); + + it("should throw error for artifact without route id", () => { + expect(() => parseDependency("artifact::my-artifact" as PipelineDependency)).toThrow( + 'Invalid dependency format: artifact::my-artifact', + ); + }); + + it("should throw error for unknown dependency type", () => { + expect(() => parseDependency("unknown:value" as PipelineDependency)).toThrow( + 'Invalid dependency format: unknown:value', + ); + }); +}); + +describe("isRouteDependency", () => { + it("should return true for route dependency", () => { + expect(isRouteDependency("route:my-route")).toBe(true); + }); + + it("should return false for artifact dependency", () => { + expect(isRouteDependency("artifact:route:artifact")).toBe(false); + }); + + it("should work as type guard", () => { + const dep: PipelineDependency = "route:test" as const; + + if (isRouteDependency(dep)) { + expect(dep).toBe("route:test"); + } else { + throw new Error("Expected route dependency"); + } + }); +}); + +describe("isArtifactDependency", () => { + it("should return true for artifact dependency", () => { + expect(isArtifactDependency("artifact:route:artifact")).toBe(true); + }); + + it("should return false for route dependency", () => { + expect(isArtifactDependency("route:my-route")).toBe(false); + }); + + it("should work as type guard", () => { + const dep: PipelineDependency = "artifact:route:artifact" as const; + + if (isArtifactDependency(dep)) { + expect(dep).toBe("artifact:route:artifact"); + } else { + throw new Error("Expected artifact dependency"); + } + }); +}); + +describe("createRouteDependency", () => { + it("should create route dependency", () => { + const dep = createRouteDependency("my-route"); + + expect(dep).toBe("route:my-route"); + }); + + it("should create route dependency with complex id", () => { + const dep = createRouteDependency("unicode-data_processor"); + + expect(dep).toBe("route:unicode-data_processor"); + }); + + it("should be parseable", () => { + const dep = createRouteDependency("test-route"); + const parsed = parseDependency(dep); + + expect(parsed).toEqual({ + type: "route", + routeId: "test-route", + }); + }); +}); + +describe("createArtifactDependency", () => { + it("should create artifact dependency", () => { + const dep = createArtifactDependency("my-route", "my-artifact"); + + expect(dep).toBe("artifact:my-route:my-artifact"); + }); + + it("should create artifact dependency with complex names", () => { + const dep = createArtifactDependency("data-processor", "normalized_output"); + + expect(dep).toBe("artifact:data-processor:normalized_output"); + }); + + it("should be parseable", () => { + const dep = createArtifactDependency("test-route", "test-artifact"); + const parsed = parseDependency(dep); + + expect(parsed).toEqual({ + type: "artifact", + routeId: "test-route", + artifactName: "test-artifact", + }); + }); +}); + +describe("ParsedDependency types", () => { + it("should handle route dependency types", () => { + const parsed: ParsedRouteDependency = { + type: "route", + routeId: "my-route", + }; + + expect(parsed.type).toBe("route"); + expect(parsed.routeId).toBe("my-route"); + }); + + it("should handle artifact dependency types", () => { + const parsed: ParsedArtifactDependency = { + type: "artifact", + routeId: "my-route", + artifactName: "my-artifact", + }; + + expect(parsed.type).toBe("artifact"); + expect(parsed.routeId).toBe("my-route"); + expect(parsed.artifactName).toBe("my-artifact"); + }); + + it("should handle union type correctly", () => { + const routeDep: ParsedDependency = { + type: "route", + routeId: "test", + }; + + const artifactDep: ParsedDependency = { + type: "artifact", + routeId: "test", + artifactName: "artifact", + }; + + expect(routeDep.type).toBe("route"); + expect(artifactDep.type).toBe("artifact"); + }); +}); + +describe("type inference", () => { + describe("ExtractRouteDependencies", () => { + it("should extract route ids from dependency array", () => { + const deps = [ + "route:parser", + "route:normalizer", + "artifact:other:data", + ] as const; + + type RouteIds = ExtractRouteDependencies; + + const id1: RouteIds = "parser"; + const id2: RouteIds = "normalizer"; + + expect(id1).toBe("parser"); + expect(id2).toBe("normalizer"); + }); + + it("should extract never type for empty array", () => { + const deps = [] as const; + + type RouteIds = ExtractRouteDependencies; + + const neverValue: RouteIds = undefined as never; + expect(neverValue).toBeUndefined(); + }); + }); + + describe("ExtractArtifactDependencies", () => { + it("should extract artifact info from dependency array", () => { + const deps = [ + "artifact:parser:result", + "artifact:normalizer:data", + "route:other", + ] as const; + + type ArtifactDeps = ExtractArtifactDependencies; + + const dep1: ArtifactDeps = { routeId: "parser", artifactName: "result" }; + const dep2: ArtifactDeps = { routeId: "normalizer", artifactName: "data" }; + + expect(dep1).toEqual({ routeId: "parser", artifactName: "result" }); + expect(dep2).toEqual({ routeId: "normalizer", artifactName: "data" }); + }); + }); + + describe("ExtractArtifactKeys", () => { + it("should extract artifact keys from dependency array", () => { + const deps = [ + "artifact:parser:result", + "artifact:normalizer:data", + "route:other", + ] as const; + + type ArtifactKeys = ExtractArtifactKeys; + + const key1: ArtifactKeys = "parser:result"; + const key2: ArtifactKeys = "normalizer:data"; + + expect(key1).toBe("parser:result"); + expect(key2).toBe("normalizer:data"); + }); + }); +}); + +describe("roundtrip parsing", () => { + it("should roundtrip route dependency", () => { + const original = createRouteDependency("test-route"); + const parsed = parseDependency(original); + const reconstructed = createRouteDependency(parsed.routeId); + + expect(reconstructed).toBe(original); + }); + + it("should roundtrip artifact dependency", () => { + const original = createArtifactDependency("test-route", "test-artifact"); + const parsed = parseDependency(original); + + if (parsed.type === "artifact") { + const reconstructed = createArtifactDependency( + parsed.routeId, + parsed.artifactName, + ); + expect(reconstructed).toBe(original); + } else { + throw new Error("Expected artifact dependency"); + } + }); +}); diff --git a/packages/pipelines/pipeline-core/test/filters.test.ts b/packages/pipelines/pipeline-core/test/filters.test.ts new file mode 100644 index 000000000..1d7aacde0 --- /dev/null +++ b/packages/pipelines/pipeline-core/test/filters.test.ts @@ -0,0 +1,445 @@ +import { describe, expect, it } from "vitest"; +import type { FileContext, FilterContext } from "../src/types"; +import { + always, + and, + byDir, + byExt, + byGlob, + byName, + byPath, + byProp, + bySource, + never, + not, + or, +} from "../src/filters"; + +function createFileContext(overrides: Partial = {}): FileContext { + return { + version: "16.0.0", + dir: "ucd", + path: "ucd/LineBreak.txt", + name: "LineBreak.txt", + ext: ".txt", + ...overrides, + }; +} + +function createFilterContext( + fileOverrides: Partial = {}, + rowProperty?: string, + sourceId?: string, +): FilterContext { + return { + file: createFileContext(fileOverrides), + row: rowProperty ? { property: rowProperty } : undefined, + source: sourceId ? { id: sourceId } : undefined, + }; +} + +describe("byName", () => { + it("should match exact file name", () => { + const filter = byName("LineBreak.txt"); + const ctx = createFilterContext({ name: "LineBreak.txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match different file name", () => { + const filter = byName("LineBreak.txt"); + const ctx = createFilterContext({ name: "PropList.txt" }); + + expect(filter(ctx)).toBe(false); + }); + + it("should be case-sensitive", () => { + const filter = byName("LineBreak.txt"); + const ctx = createFilterContext({ name: "linebreak.txt" }); + + expect(filter(ctx)).toBe(false); + }); +}); + +describe("byDir", () => { + it("should match ucd directory", () => { + const filter = byDir("ucd"); + const ctx = createFilterContext({ dir: "ucd" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match extracted directory", () => { + const filter = byDir("extracted"); + const ctx = createFilterContext({ dir: "extracted" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match auxiliary directory", () => { + const filter = byDir("auxiliary"); + const ctx = createFilterContext({ dir: "auxiliary" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match emoji directory", () => { + const filter = byDir("emoji"); + const ctx = createFilterContext({ dir: "emoji" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match unihan directory", () => { + const filter = byDir("unihan"); + const ctx = createFilterContext({ dir: "unihan" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match different directory", () => { + const filter = byDir("ucd"); + const ctx = createFilterContext({ dir: "emoji" }); + + expect(filter(ctx)).toBe(false); + }); + + it("should match custom directory", () => { + const filter = byDir("custom-dir"); + const ctx = createFilterContext({ dir: "custom-dir" }); + + expect(filter(ctx)).toBe(true); + }); +}); + +describe("byExt", () => { + it("should match extension with dot", () => { + const filter = byExt(".txt"); + const ctx = createFilterContext({ ext: ".txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match extension without dot", () => { + const filter = byExt("txt"); + const ctx = createFilterContext({ ext: ".txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match different extension", () => { + const filter = byExt(".txt"); + const ctx = createFilterContext({ ext: ".xml" }); + + expect(filter(ctx)).toBe(false); + }); + + it("should match empty extension", () => { + const filter = byExt(""); + const ctx = createFilterContext({ ext: "" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match non-empty extension when filter is empty", () => { + const filter = byExt(""); + const ctx = createFilterContext({ ext: ".txt" }); + + expect(filter(ctx)).toBe(false); + }); +}); + +describe("byGlob", () => { + it("should match wildcard pattern", () => { + const filter = byGlob("ucd/*.txt"); + const ctx = createFilterContext({ path: "ucd/LineBreak.txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match double-star pattern", () => { + const filter = byGlob("**/*.txt"); + const ctx = createFilterContext({ path: "ucd/extracted/LineBreak.txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match specific file pattern", () => { + const filter = byGlob("ucd/LineBreak.txt"); + const ctx = createFilterContext({ path: "ucd/LineBreak.txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match excluded pattern", () => { + const filter = byGlob("ucd/*.txt"); + const ctx = createFilterContext({ path: "emoji/data.txt" }); + + expect(filter(ctx)).toBe(false); + }); + + it("should match multiple extensions with brace expansion", () => { + const filter = byGlob("ucd/*.{txt,xml}"); + + expect(filter(createFilterContext({ path: "ucd/file.txt" }))).toBe(true); + expect(filter(createFilterContext({ path: "ucd/file.xml" }))).toBe(true); + expect(filter(createFilterContext({ path: "ucd/file.json" }))).toBe(false); + }); +}); + +describe("byPath", () => { + it("should match exact path string", () => { + const filter = byPath("ucd/LineBreak.txt"); + const ctx = createFilterContext({ path: "ucd/LineBreak.txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match different path", () => { + const filter = byPath("ucd/LineBreak.txt"); + const ctx = createFilterContext({ path: "ucd/PropList.txt" }); + + expect(filter(ctx)).toBe(false); + }); + + it("should match regex pattern", () => { + const filter = byPath(/LineBreak/); + const ctx = createFilterContext({ path: "ucd/LineBreak.txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should match complex regex", () => { + const filter = byPath(/^ucd\/.*\.txt$/); + + expect(filter(createFilterContext({ path: "ucd/LineBreak.txt" }))).toBe(true); + expect(filter(createFilterContext({ path: "ucd/file.txt" }))).toBe(true); + expect(filter(createFilterContext({ path: "emoji/file.txt" }))).toBe(false); + expect(filter(createFilterContext({ path: "ucd/file.xml" }))).toBe(false); + }); +}); + +describe("byProp", () => { + it("should match exact property name", () => { + const filter = byProp("NFKC_Casefold"); + const ctx = createFilterContext({}, "NFKC_Casefold"); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match different property", () => { + const filter = byProp("NFKC_Casefold"); + const ctx = createFilterContext({}, "Line_Break"); + + expect(filter(ctx)).toBe(false); + }); + + it("should return false when no row context", () => { + const filter = byProp("NFKC_Casefold"); + const ctx = createFilterContext(); + + expect(filter(ctx)).toBe(false); + }); + + it("should match regex pattern", () => { + const filter = byProp(/^NFKC_/); + const ctx = createFilterContext({}, "NFKC_Casefold"); + + expect(filter(ctx)).toBe(true); + }); + + it("should not match when regex does not match", () => { + const filter = byProp(/^NFKC_/); + const ctx = createFilterContext({}, "Line_Break"); + + expect(filter(ctx)).toBe(false); + }); + + it("should return false for regex when no property", () => { + const filter = byProp(/^NFKC_/); + const ctx = createFilterContext(); + + expect(filter(ctx)).toBe(false); + }); +}); + +describe("bySource", () => { + it("should match single source id", () => { + const filter = bySource("unicode"); + const ctx = createFilterContext({}, undefined, "unicode"); + + expect(filter(ctx)).toBe(true); + }); + + it("should match array of source ids", () => { + const filter = bySource(["unicode", "cldr"]); + + expect(filter(createFilterContext({}, undefined, "unicode"))).toBe(true); + expect(filter(createFilterContext({}, undefined, "cldr"))).toBe(true); + expect(filter(createFilterContext({}, undefined, "other"))).toBe(false); + }); + + it("should not match when source is undefined", () => { + const filter = bySource("unicode"); + const ctx = createFilterContext(); + + expect(filter(ctx)).toBe(false); + }); + + it("should not match different source", () => { + const filter = bySource("unicode"); + const ctx = createFilterContext({}, undefined, "cldr"); + + expect(filter(ctx)).toBe(false); + }); +}); + +describe("and", () => { + it("should return true when all filters pass", () => { + const filter = and( + byDir("ucd"), + byExt(".txt"), + byName("LineBreak.txt"), + ); + const ctx = createFilterContext({ + dir: "ucd", + ext: ".txt", + name: "LineBreak.txt", + }); + + expect(filter(ctx)).toBe(true); + }); + + it("should return false when any filter fails", () => { + const filter = and( + byDir("ucd"), + byExt(".txt"), + byName("PropList.txt"), + ); + const ctx = createFilterContext({ + dir: "ucd", + ext: ".txt", + name: "LineBreak.txt", + }); + + expect(filter(ctx)).toBe(false); + }); + + it("should return true for empty filter array", () => { + const filter = and(); + const ctx = createFilterContext(); + + expect(filter(ctx)).toBe(true); + }); + + it("should work with single filter", () => { + const filter = and(byDir("ucd")); + const ctx = createFilterContext({ dir: "ucd" }); + + expect(filter(ctx)).toBe(true); + }); +}); + +describe("or", () => { + it("should return true when any filter passes", () => { + const filter = or( + byName("LineBreak.txt"), + byName("PropList.txt"), + ); + const ctx = createFilterContext({ name: "LineBreak.txt" }); + + expect(filter(ctx)).toBe(true); + }); + + it("should return false when all filters fail", () => { + const filter = or( + byName("LineBreak.txt"), + byName("PropList.txt"), + ); + const ctx = createFilterContext({ name: "UnicodeData.txt" }); + + expect(filter(ctx)).toBe(false); + }); + + it("should return false for empty filter array", () => { + const filter = or(); + const ctx = createFilterContext(); + + expect(filter(ctx)).toBe(false); + }); + + it("should work with single filter", () => { + const filter = or(byDir("ucd")); + + expect(filter(createFilterContext({ dir: "ucd" }))).toBe(true); + expect(filter(createFilterContext({ dir: "emoji" }))).toBe(false); + }); +}); + +describe("not", () => { + it("should invert filter result", () => { + const filter = not(byDir("ucd")); + + expect(filter(createFilterContext({ dir: "ucd" }))).toBe(false); + expect(filter(createFilterContext({ dir: "emoji" }))).toBe(true); + }); + + it("should work with complex filters", () => { + const filter = not(and(byDir("ucd"), byExt(".txt"))); + + expect(filter(createFilterContext({ dir: "ucd", ext: ".txt" }))).toBe(false); + expect(filter(createFilterContext({ dir: "ucd", ext: ".xml" }))).toBe(true); + expect(filter(createFilterContext({ dir: "emoji", ext: ".txt" }))).toBe(true); + }); +}); + +describe("always", () => { + it("should always return true", () => { + const filter = always(); + + expect(filter(createFilterContext())).toBe(true); + expect(filter(createFilterContext({ dir: "ucd" }))).toBe(true); + expect(filter(createFilterContext({ dir: "emoji" }))).toBe(true); + }); +}); + +describe("never", () => { + it("should always return false", () => { + const filter = never(); + + expect(filter(createFilterContext())).toBe(false); + expect(filter(createFilterContext({ dir: "ucd" }))).toBe(false); + expect(filter(createFilterContext({ dir: "emoji" }))).toBe(false); + }); +}); + +describe("complex filter combinations", () => { + it("should combine and/or/not filters", () => { + const filter = and( + byDir("ucd"), + or( + byExt(".txt"), + byExt(".xml"), + ), + not(byName("ReadMe.txt")), + ); + + expect(filter(createFilterContext({ dir: "ucd", ext: ".txt", name: "LineBreak.txt" }))).toBe(true); + expect(filter(createFilterContext({ dir: "ucd", ext: ".xml", name: "data.xml" }))).toBe(true); + expect(filter(createFilterContext({ dir: "ucd", ext: ".txt", name: "ReadMe.txt" }))).toBe(false); + expect(filter(createFilterContext({ dir: "emoji", ext: ".txt", name: "data.txt" }))).toBe(false); + expect(filter(createFilterContext({ dir: "ucd", ext: ".json", name: "data.json" }))).toBe(false); + }); + + it("should handle nested logic", () => { + const filter = or( + and(byDir("ucd"), byExt(".txt")), + and(byDir("emoji"), byExt(".txt")), + ); + + expect(filter(createFilterContext({ dir: "ucd", ext: ".txt" }))).toBe(true); + expect(filter(createFilterContext({ dir: "emoji", ext: ".txt" }))).toBe(true); + expect(filter(createFilterContext({ dir: "ucd", ext: ".xml" }))).toBe(false); + expect(filter(createFilterContext({ dir: "auxiliary", ext: ".txt" }))).toBe(false); + }); +}); diff --git a/packages/pipelines/pipeline-core/test/transform.test.ts b/packages/pipelines/pipeline-core/test/transform.test.ts new file mode 100644 index 000000000..23685187e --- /dev/null +++ b/packages/pipelines/pipeline-core/test/transform.test.ts @@ -0,0 +1,330 @@ +import { describe, expect, it } from "vitest"; +import { + applyTransforms, + definePipelineTransform, + type InferTransformInput, + type InferTransformOutput, + type PipelineTransformDefinition, + type TransformContext, +} from "../src/transform"; +import type { FileContext } from "../src/types"; + +function createTransformContext(): TransformContext { + const file: FileContext = { + version: "16.0.0", + dir: "ucd", + path: "ucd/LineBreak.txt", + name: "LineBreak.txt", + ext: ".txt", + }; + + return { + version: "16.0.0", + file, + }; +} + +async function* createAsyncIterable(items: T[]): AsyncIterable { + for (const item of items) { + yield item; + } +} + +async function collectAsync(iter: AsyncIterable): Promise { + const result: T[] = []; + for await (const item of iter) { + result.push(item); + } + return result; +} + +describe("definePipelineTransform", () => { + it("should define a simple transform", () => { + const transform = definePipelineTransform({ + id: "uppercase", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield (row as string).toUpperCase(); + } + }, + }); + + expect(transform.id).toBe("uppercase"); + expect(typeof transform.fn).toBe("function"); + }); + + it("should define a transform with type parameters", () => { + const transform = definePipelineTransform({ + id: "string-length", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row.length; + } + }, + }); + + expect(transform.id).toBe("string-length"); + }); + + it("should preserve transform function", async () => { + const transform = definePipelineTransform({ + id: "double", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row * 2; + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable([1, 2, 3]); + const result = await collectAsync(transform.fn(ctx, input)); + + expect(result).toEqual([2, 4, 6]); + }); +}); + +describe("applyTransforms", () => { + it("should apply single transform", async () => { + const uppercase = definePipelineTransform({ + id: "uppercase", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row.toUpperCase(); + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable(["hello", "world"]); + const result = await collectAsync(applyTransforms(ctx, input, [uppercase])); + + expect(result).toEqual(["HELLO", "WORLD"]); + }); + + it("should chain multiple transforms", async () => { + const uppercase = definePipelineTransform({ + id: "uppercase", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row.toUpperCase(); + } + }, + }); + + const exclaim = definePipelineTransform({ + id: "exclaim", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield `${row}!`; + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable(["hello", "world"]); + const result = await collectAsync(applyTransforms(ctx, input, [uppercase, exclaim])); + + expect(result).toEqual(["HELLO!", "WORLD!"]); + }); + + it("should handle type transformations", async () => { + const toLength = definePipelineTransform({ + id: "to-length", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row.length; + } + }, + }); + + const double = definePipelineTransform({ + id: "double", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row * 2; + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable(["a", "ab", "abc"]); + const result = await collectAsync(applyTransforms(ctx, input, [toLength, double])); + + expect(result).toEqual([2, 4, 6]); + }); + + it("should apply transforms in order", async () => { + const append = (suffix: string) => + definePipelineTransform({ + id: `append-${suffix}`, + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield `${row}${suffix}`; + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable(["x"]); + const result = await collectAsync( + applyTransforms(ctx, input, [append("1"), append("2"), append("3")]), + ); + + expect(result).toEqual(["x123"]); + }); + + it("should handle empty transform array", async () => { + const ctx = createTransformContext(); + const input = createAsyncIterable(["a", "b", "c"]); + const result = await collectAsync(applyTransforms(ctx, input, [])); + + expect(result).toEqual(["a", "b", "c"]); + }); + + it("should handle empty input", async () => { + const uppercase = definePipelineTransform({ + id: "uppercase", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row.toUpperCase(); + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable([]); + const result = await collectAsync(applyTransforms(ctx, input, [uppercase])); + + expect(result).toEqual([]); + }); + + it("should pass context to transforms", async () => { + let capturedVersion: string | undefined; + let capturedFileName: string | undefined; + + const captureContext = definePipelineTransform({ + id: "capture", + fn: async function* (ctx, rows) { + capturedVersion = ctx.version; + capturedFileName = ctx.file.name; + for await (const row of rows) { + yield row; + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable(["test"]); + await collectAsync(applyTransforms(ctx, input, [captureContext])); + + expect(capturedVersion).toBe("16.0.0"); + expect(capturedFileName).toBe("LineBreak.txt"); + }); + + it("should handle object transformations", async () => { + interface Person { + name: string; + age: number; + } + + interface PersonWithId extends Person { + id: string; + } + + const addId = definePipelineTransform({ + id: "add-id", + fn: async function* (_ctx, rows) { + let counter = 0; + for await (const row of rows) { + yield { ...row, id: `person-${counter++}` }; + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable([ + { name: "Alice", age: 30 }, + { name: "Bob", age: 25 }, + ]); + const result = await collectAsync(applyTransforms(ctx, input, [addId])); + + expect(result).toEqual([ + { name: "Alice", age: 30, id: "person-0" }, + { name: "Bob", age: 25, id: "person-1" }, + ]); + }); + + it("should handle filter transformations", async () => { + const filterEven = definePipelineTransform({ + id: "filter-even", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + if (row % 2 === 0) { + yield row; + } + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable([1, 2, 3, 4, 5, 6]); + const result = await collectAsync(applyTransforms(ctx, input, [filterEven])); + + expect(result).toEqual([2, 4, 6]); + }); + + it("should handle aggregation transformations", async () => { + const toArray = definePipelineTransform({ + id: "to-array", + fn: async function* (_ctx, rows) { + const arr: number[] = []; + for await (const row of rows) { + arr.push(row); + } + yield arr; + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable([1, 2, 3]); + const result = await collectAsync(applyTransforms(ctx, input, [toArray])); + + expect(result).toEqual([[1, 2, 3]]); + }); +}); + +describe("type inference", () => { + it("should preserve transform types", async () => { + const stringToNumber = definePipelineTransform({ + id: "length", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row.length; + } + }, + }); + + const ctx = createTransformContext(); + const input = createAsyncIterable(["hello", "world"]); + const result = await collectAsync(stringToNumber.fn(ctx, input)); + + expect(result).toEqual([5, 5]); + }); +}); + +describe("PipelineTransformDefinition", () => { + it("should create valid transform definition", () => { + const def: PipelineTransformDefinition = { + id: "test", + fn: async function* (_ctx, rows) { + for await (const row of rows) { + yield row.length; + } + }, + }; + + expect(def.id).toBe("test"); + expect(typeof def.fn).toBe("function"); + }); +}); diff --git a/packages/pipelines/pipeline-executor/package.json b/packages/pipelines/pipeline-executor/package.json index ea44d2730..f51ead4c5 100644 --- a/packages/pipelines/pipeline-executor/package.json +++ b/packages/pipelines/pipeline-executor/package.json @@ -39,19 +39,17 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { - "@ucdjs-internal/shared": "workspace:*", - "picomatch": "catalog:prod", - "zod": "catalog:prod" + "@ucdjs/pipelines-core": "workspace:*", + "@ucdjs/pipelines-artifacts": "workspace:*", + "@ucdjs/pipelines-graph": "workspace:*" }, "devDependencies": { "@luxass/eslint-config": "catalog:linting", - "@types/picomatch": "catalog:types", "@ucdjs-tooling/tsconfig": "workspace:*", "@ucdjs-tooling/tsdown-config": "workspace:*", "eslint": "catalog:linting", "publint": "catalog:dev", "tsdown": "catalog:dev", - "tsx": "catalog:dev", "typescript": "catalog:dev" }, "publishConfig": { diff --git a/packages/pipelines/pipeline-executor/src/cache.ts b/packages/pipelines/pipeline-executor/src/cache.ts new file mode 100644 index 000000000..c8bd1ec20 --- /dev/null +++ b/packages/pipelines/pipeline-executor/src/cache.ts @@ -0,0 +1,136 @@ +export interface CacheKey { + routeId: string; + version: string; + inputHash: string; + artifactHashes: Record; +} + +export interface CacheEntry { + key: CacheKey; + output: TOutput[]; + producedArtifacts: Record; + createdAt: string; + meta?: Record; +} + +export function serializeCacheKey(key: CacheKey): string { + const artifactHashStr = Object.entries(key.artifactHashes) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, hash]) => `${id}:${hash}`) + .join(","); + + return `${key.routeId}|${key.version}|${key.inputHash}|${artifactHashStr}`; +} + +export interface CacheStore { + get: (key: CacheKey) => Promise; + set: (entry: CacheEntry) => Promise; + has: (key: CacheKey) => Promise; + delete: (key: CacheKey) => Promise; + clear: () => Promise; + stats?: () => Promise; +} + +export interface CacheStats { + entries: number; + sizeBytes?: number; + hits?: number; + misses?: number; +} + +export interface CacheOptions { + enabled?: boolean; + hashFn?: (content: string) => string; +} + +export function createMemoryCacheStore(): CacheStore { + const cache = new Map(); + let hits = 0; + let misses = 0; + + return { + async get(key: CacheKey): Promise { + const serialized = serializeCacheKey(key); + const entry = cache.get(serialized); + if (entry) { + hits++; + } else { + misses++; + } + return entry; + }, + + async set(entry: CacheEntry): Promise { + const serialized = serializeCacheKey(entry.key); + cache.set(serialized, entry); + }, + + async has(key: CacheKey): Promise { + const serialized = serializeCacheKey(key); + return cache.has(serialized); + }, + + async delete(key: CacheKey): Promise { + const serialized = serializeCacheKey(key); + return cache.delete(serialized); + }, + + async clear(): Promise { + cache.clear(); + hits = 0; + misses = 0; + }, + + async stats(): Promise { + return { + entries: cache.size, + hits, + misses, + }; + }, + }; +} + +export function defaultHashFn(content: string): string { + let hash = 5381; + for (let i = 0; i < content.length; i++) { + hash = ((hash << 5) + hash) ^ content.charCodeAt(i); + } + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +export function hashArtifact(value: unknown): string { + if (value === null || value === undefined) { + return "null"; + } + + if (typeof value === "string") { + return defaultHashFn(value); + } + + if (value instanceof Map) { + const entries = Array.from(value.entries()) + .sort(([a], [b]) => String(a).localeCompare(String(b))) + .map(([k, v]) => `${String(k)}=${String(v)}`) + .join(";"); + return defaultHashFn(entries); + } + + if (value instanceof Set) { + const entries = Array.from(value) + .map(String) + .sort() + .join(";"); + return defaultHashFn(entries); + } + + if (Array.isArray(value)) { + return defaultHashFn(JSON.stringify(value)); + } + + if (typeof value === "object") { + return defaultHashFn(JSON.stringify(value)); + } + + return defaultHashFn(String(value)); +} diff --git a/packages/pipelines/pipeline-executor/src/executor.ts b/packages/pipelines/pipeline-executor/src/executor.ts new file mode 100644 index 000000000..255a3cadd --- /dev/null +++ b/packages/pipelines/pipeline-executor/src/executor.ts @@ -0,0 +1,936 @@ +import type { + FileContext, + ParseContext, + ParsedRow, + PipelineDefinition, + PipelineEvent, + PipelineFilter, + PipelineGraphEdge, + PipelineGraphNode, + PipelineRouteDefinition, + RouteResolveContext, + SourceBackend, + SourceFileContext, +} from "@ucdjs/pipelines-core"; +import { applyTransforms, resolveMultipleSourceFiles } from "@ucdjs/pipelines-core"; +import type { ArtifactDefinition, PipelineArtifactDefinition } from "@ucdjs/pipelines-artifacts"; +import { isGlobalArtifact } from "@ucdjs/pipelines-artifacts"; +import { buildDAG, getExecutionLayers } from "@ucdjs/pipelines-graph"; +import type { CacheEntry, CacheKey, CacheStore } from "./cache"; +import { defaultHashFn, hashArtifact } from "./cache"; +import type { MultiplePipelineRunResult, PipelineRunResult, PipelineSummary } from "./results"; + +interface SourceAdapter { + listFiles: (version: string) => Promise; + readFile: (file: FileContext) => Promise; +} + +export interface PipelineExecutorOptions { + pipelines: PipelineDefinition[]; + artifacts?: PipelineArtifactDefinition[]; + cacheStore?: CacheStore; + onEvent?: (event: PipelineEvent) => void | Promise; +} + +export interface PipelineExecutorRunOptions { + cache?: boolean; + versions?: string[]; + pipelines?: string[]; +} + +export interface PipelineExecutor { + run: (options?: PipelineExecutorRunOptions) => Promise; + runSingle: (pipelineId: string, options?: Omit) => Promise; +} + +export function createPipelineExecutor(options: PipelineExecutorOptions): PipelineExecutor { + const { + pipelines, + artifacts: globalArtifacts = [], + cacheStore, + onEvent, + } = options; + + const pipelinesById = new Map(pipelines.map((p) => [p.id, p])); + + async function emit(event: PipelineEvent): Promise { + if (onEvent) { + await onEvent(event); + } + } + + async function runSinglePipeline( + pipeline: PipelineDefinition, + runOptions: Omit = {}, + ): Promise { + const { cache: enableCache = true, versions: runVersions } = runOptions; + const useCache = enableCache && cacheStore != null; + const versionsToRun = runVersions ?? pipeline.versions; + + const effectiveSource = createSourceAdapter(pipeline); + + const startTime = performance.now(); + const graphNodes: PipelineGraphNode[] = []; + const graphEdges: PipelineGraphEdge[] = []; + const allOutputs: unknown[] = []; + const errors: PipelineRunResult["errors"] = []; + + let totalFiles = 0; + let matchedFiles = 0; + let skippedFiles = 0; + let fallbackFiles = 0; + + const dagResult = buildDAG(pipeline.routes); + if (!dagResult.valid) { + throw new Error(`Pipeline DAG validation failed:\n${dagResult.errors.map((e) => ` - ${e.message}`).join("\n")}`); + } + const dag = dagResult.dag!; + + await emit({ type: "pipeline:start", versions: versionsToRun, timestamp: Date.now() }); + + for (const version of versionsToRun) { + const versionStartTime = performance.now(); + await emit({ type: "version:start", version, timestamp: Date.now() }); + + const sourceNodeId = `source:${version}`; + graphNodes.push({ id: sourceNodeId, type: "source", version }); + + const artifactsMap: Record = {}; + const globalArtifactsMap: Record = {}; + + for (const artifactDef of globalArtifacts) { + const artifactStartTime = performance.now(); + await emit({ + type: "artifact:start", + artifactId: artifactDef.id, + version, + timestamp: Date.now(), + }); + + const artifactNodeId = `artifact:${version}:${artifactDef.id}`; + graphNodes.push({ id: artifactNodeId, type: "artifact", artifactId: artifactDef.id }); + graphEdges.push({ from: sourceNodeId, to: artifactNodeId, type: "provides" }); + + try { + let rows: AsyncIterable | undefined; + + if (artifactDef.filter && artifactDef.parser) { + const files = await effectiveSource.listFiles(version); + for (const file of files) { + if (artifactDef.filter({ file })) { + const parseCtx = createParseContext(file, effectiveSource); + rows = artifactDef.parser(parseCtx); + break; + } + } + } + + const value = await artifactDef.build({ version }, rows); + artifactsMap[artifactDef.id] = value; + } catch (err) { + const pipelineError = { + scope: "artifact" as const, + message: err instanceof Error ? err.message : String(err), + error: err, + artifactId: artifactDef.id, + version, + }; + errors.push(pipelineError); + await emit({ + type: "error", + error: pipelineError, + timestamp: Date.now(), + }); + } + + await emit({ + type: "artifact:end", + artifactId: artifactDef.id, + version, + durationMs: performance.now() - artifactStartTime, + timestamp: Date.now(), + }); + } + + const files = await effectiveSource.listFiles(version); + totalFiles += files.length; + + const filesToProcess = pipeline.include + ? files.filter((file) => pipeline.include!({ file })) + : files; + + const executionLayers = getExecutionLayers(dag); + + for (const layer of executionLayers) { + const processingQueue = createProcessingQueue(pipeline.concurrency); + const layerRoutes = pipeline.routes.filter((r) => layer.includes(r.id)); + + for (const route of layerRoutes) { + const matchingFiles = filesToProcess.filter((file) => { + const sourceFile = file as SourceFileContext; + const filterCtx = { + file, + source: sourceFile.source, + }; + return route.filter(filterCtx); + }); + + for (const file of matchingFiles) { + await processingQueue.add(async () => { + const fileNodeId = `file:${version}:${file.path}`; + if (!graphNodes.some((n) => n.id === fileNodeId)) { + graphNodes.push({ id: fileNodeId, type: "file", file }); + graphEdges.push({ from: sourceNodeId, to: fileNodeId, type: "provides" }); + } + + matchedFiles++; + const routeNodeId = `route:${version}:${route.id}`; + + if (!graphNodes.some((n) => n.id === routeNodeId)) { + graphNodes.push({ id: routeNodeId, type: "route", routeId: route.id }); + } + + graphEdges.push({ from: fileNodeId, to: routeNodeId, type: "matched" }); + + await emit({ + type: "file:matched", + file, + routeId: route.id, + timestamp: Date.now(), + }); + + try { + const routeCacheEnabled = useCache && route.cache !== false; + let result: ProcessRouteResult | null = null; + let cacheHit = false; + + if (routeCacheEnabled && cacheStore) { + const fileContent = await effectiveSource.readFile(file); + const inputHash = defaultHashFn(fileContent); + + const partialKey: CacheKey = { + routeId: route.id, + version, + inputHash, + artifactHashes: {}, + }; + + const cachedEntry = await cacheStore.get(partialKey); + + if (cachedEntry) { + const currentArtifactHashes: Record = {}; + for (const id of Object.keys(cachedEntry.key.artifactHashes)) { + const combinedMap = { ...artifactsMap, ...globalArtifactsMap }; + if (id in combinedMap) { + currentArtifactHashes[id] = hashArtifact(combinedMap[id]); + } + } + + const artifactHashesMatch = Object.keys(cachedEntry.key.artifactHashes).every( + (id) => currentArtifactHashes[id] === cachedEntry.key.artifactHashes[id], + ); + + if (artifactHashesMatch) { + cacheHit = true; + result = { + outputs: cachedEntry.output, + emittedArtifacts: cachedEntry.producedArtifacts, + consumedArtifactIds: Object.keys(cachedEntry.key.artifactHashes), + }; + + await emit({ + type: "cache:hit", + routeId: route.id, + file, + version, + timestamp: Date.now(), + }); + } + } + + if (!cacheHit) { + await emit({ + type: "cache:miss", + routeId: route.id, + file, + version, + timestamp: Date.now(), + }); + } + } + + if (!result) { + result = await processRoute( + file, + route, + { ...artifactsMap, ...globalArtifactsMap }, + effectiveSource, + version, + emit, + ); + + if (routeCacheEnabled && cacheStore) { + const fileContent = await effectiveSource.readFile(file); + const combinedMap = { ...artifactsMap, ...globalArtifactsMap }; + const cacheKey = await buildCacheKey( + route.id, + version, + fileContent, + combinedMap, + result.consumedArtifactIds, + ); + + const cacheEntry: CacheEntry = { + key: cacheKey, + output: result.outputs, + producedArtifacts: result.emittedArtifacts, + createdAt: new Date().toISOString(), + }; + + await cacheStore.set(cacheEntry); + + await emit({ + type: "cache:store", + routeId: route.id, + file, + version, + timestamp: Date.now(), + }); + } + } + + for (const [artifactName, artifactValue] of Object.entries(result.emittedArtifacts)) { + const prefixedKey = `${route.id}:${artifactName}`; + const artifactDef = route.emits?.[artifactName]; + + if (artifactDef && isGlobalArtifact(artifactDef)) { + globalArtifactsMap[prefixedKey] = artifactValue; + } else { + artifactsMap[prefixedKey] = artifactValue; + } + } + + for (const output of result.outputs) { + const outputIndex = allOutputs.length; + allOutputs.push(output); + + const outputNodeId = `output:${version}:${outputIndex}`; + graphNodes.push({ + id: outputNodeId, + type: "output", + outputIndex, + property: (output as { property?: string }).property, + }); + graphEdges.push({ from: routeNodeId, to: outputNodeId, type: "resolved" }); + } + } catch (err) { + const pipelineError = { + scope: "route" as const, + message: err instanceof Error ? err.message : String(err), + error: err, + file, + routeId: route.id, + version, + }; + errors.push(pipelineError); + await emit({ + type: "error", + error: pipelineError, + timestamp: Date.now(), + }); + } + }); + } + } + + await processingQueue.drain(); + } + + const processedFiles = new Set(); + for (const route of pipeline.routes) { + for (const file of filesToProcess) { + const sourceFile = file as SourceFileContext; + const filterCtx = { file, source: sourceFile.source }; + if (route.filter(filterCtx)) { + processedFiles.add(file.path); + } + } + } + + for (const file of filesToProcess) { + if (processedFiles.has(file.path)) continue; + + if (pipeline.fallback) { + const fallback = pipeline.fallback as FallbackRouteDefinition; + const shouldUseFallback = !fallback.filter || fallback.filter({ file }); + + if (shouldUseFallback) { + fallbackFiles++; + + const fileNodeId = `file:${version}:${file.path}`; + if (!graphNodes.some((n) => n.id === fileNodeId)) { + graphNodes.push({ id: fileNodeId, type: "file", file }); + graphEdges.push({ from: sourceNodeId, to: fileNodeId, type: "provides" }); + } + + await emit({ + type: "file:fallback", + file, + timestamp: Date.now(), + }); + + try { + const outputs = await processFallback( + file, + fallback, + { ...artifactsMap, ...globalArtifactsMap }, + effectiveSource, + version, + emit, + ); + + for (const output of outputs) { + const outputIndex = allOutputs.length; + allOutputs.push(output); + + const outputNodeId = `output:${version}:${outputIndex}`; + graphNodes.push({ + id: outputNodeId, + type: "output", + outputIndex, + property: (output as { property?: string }).property, + }); + graphEdges.push({ from: fileNodeId, to: outputNodeId, type: "resolved" }); + } + } catch (err) { + const pipelineError = { + scope: "file" as const, + message: err instanceof Error ? err.message : String(err), + error: err, + file, + version, + }; + errors.push(pipelineError); + await emit({ + type: "error", + error: pipelineError, + timestamp: Date.now(), + }); + } + } else { + skippedFiles++; + await emit({ + type: "file:skipped", + file, + reason: "filtered", + timestamp: Date.now(), + }); + } + } else { + skippedFiles++; + + if (pipeline.strict) { + const pipelineError = { + scope: "file" as const, + message: `No matching route for file: ${file.path}`, + file, + version, + }; + errors.push(pipelineError); + await emit({ + type: "error", + error: pipelineError, + timestamp: Date.now(), + }); + } else { + await emit({ + type: "file:skipped", + file, + reason: "no-match", + timestamp: Date.now(), + }); + } + } + } + + await emit({ + type: "version:end", + version, + durationMs: performance.now() - versionStartTime, + timestamp: Date.now(), + }); + } + + const durationMs = performance.now() - startTime; + + await emit({ + type: "pipeline:end", + durationMs, + timestamp: Date.now(), + }); + + const summary: PipelineSummary = { + versions: versionsToRun, + totalFiles, + matchedFiles, + skippedFiles, + fallbackFiles, + totalOutputs: allOutputs.length, + durationMs, + }; + + return { + data: allOutputs, + graph: { nodes: graphNodes, edges: graphEdges }, + errors, + summary, + }; + } + + async function run(runOptions: PipelineExecutorRunOptions = {}): Promise { + const startTime = performance.now(); + const pipelinesToRun = runOptions.pipelines + ? pipelines.filter((p) => runOptions.pipelines!.includes(p.id)) + : pipelines; + + const results = new Map(); + let successfulPipelines = 0; + let failedPipelines = 0; + + for (const pipeline of pipelinesToRun) { + try { + const result = await runSinglePipeline(pipeline, runOptions); + results.set(pipeline.id, result); + if (result.errors.length === 0) { + successfulPipelines++; + } else { + failedPipelines++; + } + } catch (err) { + failedPipelines++; + results.set(pipeline.id, { + data: [], + graph: { nodes: [], edges: [] }, + errors: [{ + scope: "pipeline", + message: err instanceof Error ? err.message : String(err), + error: err, + }], + summary: { + versions: pipeline.versions, + totalFiles: 0, + matchedFiles: 0, + skippedFiles: 0, + fallbackFiles: 0, + totalOutputs: 0, + durationMs: 0, + }, + }); + } + } + + return { + results, + summary: { + totalPipelines: pipelinesToRun.length, + successfulPipelines, + failedPipelines, + durationMs: performance.now() - startTime, + }, + }; + } + + return { + run, + runSingle: (pipelineId, options) => { + const pipeline = pipelinesById.get(pipelineId); + if (!pipeline) { + throw new Error(`Pipeline "${pipelineId}" not found`); + } + return runSinglePipeline(pipeline, options); + }, + }; +} + +function createSourceAdapter(pipeline: PipelineDefinition): SourceAdapter { + if (pipeline.inputs.length === 0) { + throw new Error("Pipeline requires at least one input source"); + } + + const backends = new Map(); + for (const input of pipeline.inputs) { + backends.set(input.id, input.backend); + } + + return { + listFiles: async (version: string) => { + return resolveMultipleSourceFiles(pipeline.inputs as any, version); + }, + readFile: async (file: FileContext) => { + const sourceFile = file as SourceFileContext; + if (sourceFile.source) { + const backend = backends.get(sourceFile.source.id); + if (backend) { + return backend.readFile(file); + } + } + const firstBackend = backends.values().next().value; + if (firstBackend) { + return firstBackend.readFile(file); + } + throw new Error(`No backend found for file: ${file.path}`); + }, + }; +} + +function createParseContext(file: FileContext, source: SourceAdapter): ParseContext { + let cachedContent: string | null = null; + + return { + file, + readContent: async () => { + if (cachedContent === null) { + cachedContent = await source.readFile(file); + } + return cachedContent!; + }, + readLines: async function* () { + const content = await source.readFile(file); + const lines = content.split(/\r?\n/); + for (const line of lines) { + yield line; + } + }, + isComment: (line: string) => line.startsWith("#") || line.trim() === "", + }; +} + +interface ResolveContextOptions { + version: string; + file: FileContext; + routeId: string; + artifactsMap: Record; + emittedArtifacts: Record; + emitsDefinition?: Record; + onArtifactEmit?: (id: string, value: unknown) => void; + onArtifactGet?: (id: string) => void; +} + +function createRouteResolveContext( + options: ResolveContextOptions, +): RouteResolveContext> { + const { version, file, routeId, artifactsMap, emittedArtifacts, emitsDefinition, onArtifactEmit, onArtifactGet } = options; + + return { + version, + file, + getArtifact: (key: K): unknown => { + if (!(key in artifactsMap)) { + throw new Error(`Artifact "${key}" not found. Make sure a route that produces this artifact runs before route "${routeId}".`); + } + onArtifactGet?.(key); + return artifactsMap[key]; + }, + emitArtifact: (id: K, value: unknown): void => { + if (emitsDefinition) { + const def = emitsDefinition[id]; + if (def) { + const result = def.schema.safeParse(value); + if (!result.success) { + throw new Error(`Artifact "${id}" validation failed: ${result.error.message}`); + } + } + } + emittedArtifacts[id] = value; + onArtifactEmit?.(id, value); + }, + normalizeEntries: (entries) => { + return entries.sort((a, b) => { + const aStart = a.range?.split("..")[0] ?? a.codePoint ?? ""; + const bStart = b.range?.split("..")[0] ?? b.codePoint ?? ""; + return aStart.localeCompare(bStart); + }); + }, + now: () => new Date().toISOString(), + }; +} + +interface ProcessRouteResult { + outputs: unknown[]; + emittedArtifacts: Record; + consumedArtifactIds: string[]; +} + +async function processRoute( + file: FileContext, + route: PipelineRouteDefinition, + artifactsMap: Record, + source: SourceAdapter, + version: string, + emit: (event: PipelineEvent) => Promise, +): Promise { + const parseStartTime = performance.now(); + await emit({ + type: "parse:start", + file, + routeId: route.id, + timestamp: Date.now(), + }); + + const parseCtx = createParseContext(file, source); + let rows: AsyncIterable = route.parser(parseCtx); + + const collectedRows: ParsedRow[] = []; + const filteredRows = filterRows(rows as AsyncIterable, file, route.filter, collectedRows); + + if (route.transforms && route.transforms.length > 0) { + rows = applyTransforms( + { version, file }, + filteredRows, + route.transforms, + ); + } else { + rows = filteredRows; + } + + await emit({ + type: "parse:end", + file, + routeId: route.id, + rowCount: collectedRows.length, + durationMs: performance.now() - parseStartTime, + timestamp: Date.now(), + }); + + const resolveStartTime = performance.now(); + await emit({ + type: "resolve:start", + file, + routeId: route.id, + timestamp: Date.now(), + }); + + const emittedArtifacts: Record = {}; + const consumedArtifactIds: string[] = []; + + const resolveCtx = createRouteResolveContext({ + version, + file, + routeId: route.id, + artifactsMap, + emittedArtifacts, + emitsDefinition: route.emits, + onArtifactEmit: async (id) => { + await emit({ + type: "artifact:produced", + artifactId: `${route.id}:${id}`, + routeId: route.id, + version, + timestamp: Date.now(), + }); + }, + onArtifactGet: async (id) => { + if (!consumedArtifactIds.includes(id)) { + consumedArtifactIds.push(id); + await emit({ + type: "artifact:consumed", + artifactId: id, + routeId: route.id, + version, + timestamp: Date.now(), + }); + } + }, + }); + + const outputs = await route.resolver(resolveCtx, rows); + + const outputArray = Array.isArray(outputs) ? outputs : [outputs]; + + await emit({ + type: "resolve:end", + file, + routeId: route.id, + outputCount: outputArray.length, + durationMs: performance.now() - resolveStartTime, + timestamp: Date.now(), + }); + + return { outputs: outputArray, emittedArtifacts, consumedArtifactIds }; +} + +interface FallbackRouteDefinition = Record, TOutput = unknown> { + filter?: PipelineFilter; + parser: (ctx: ParseContext) => AsyncIterable; + resolver: (ctx: { version: string; file: FileContext; getArtifact: (id: K) => TArtifacts[K]; emitArtifact: (id: K, value: V) => void; normalizeEntries: (entries: any[]) => any[]; now: () => string }, rows: AsyncIterable) => Promise; +} + +async function processFallback( + file: FileContext, + fallback: FallbackRouteDefinition, + artifactsMap: Record, + source: SourceAdapter, + version: string, + emit: (event: PipelineEvent) => Promise, +): Promise { + const parseStartTime = performance.now(); + await emit({ + type: "parse:start", + file, + routeId: "__fallback__", + timestamp: Date.now(), + }); + + const parseCtx = createParseContext(file, source); + const rows = fallback.parser(parseCtx); + + const collectedRows: ParsedRow[] = []; + const filteredRows = filterRows(rows, file, fallback.filter, collectedRows); + + await emit({ + type: "parse:end", + file, + routeId: "__fallback__", + rowCount: collectedRows.length, + durationMs: performance.now() - parseStartTime, + timestamp: Date.now(), + }); + + const resolveStartTime = performance.now(); + await emit({ + type: "resolve:start", + file, + routeId: "__fallback__", + timestamp: Date.now(), + }); + + const emittedArtifacts: Record = {}; + + const resolveCtx = { + version, + file, + getArtifact: (id: K): unknown => { + if (!(id in artifactsMap)) { + throw new Error(`Artifact "${String(id)}" not found.`); + } + return artifactsMap[id]; + }, + emitArtifact: (id: K, value: V): void => { + emittedArtifacts[id] = value; + }, + normalizeEntries: (entries: any[]) => { + return entries.sort((a: any, b: any) => { + const aStart = a.range?.split("..")[0] ?? a.codePoint ?? ""; + const bStart = b.range?.split("..")[0] ?? b.codePoint ?? ""; + return aStart.localeCompare(bStart); + }); + }, + now: () => new Date().toISOString(), + }; + const outputs = await fallback.resolver(resolveCtx, filteredRows); + + const outputArray = Array.isArray(outputs) ? outputs : [outputs]; + + await emit({ + type: "resolve:end", + file, + routeId: "__fallback__", + outputCount: outputArray.length, + durationMs: performance.now() - resolveStartTime, + timestamp: Date.now(), + }); + + return outputArray; +} + +async function* filterRows( + rows: AsyncIterable, + file: FileContext, + filter: PipelineFilter | undefined, + collector: ParsedRow[], +): AsyncIterable { + for await (const row of rows) { + collector.push(row); + + if (!filter) { + yield row; + continue; + } + + const shouldInclude = filter({ + file, + row: { property: row.property }, + }); + + if (shouldInclude) { + yield row; + } + } +} + +async function buildCacheKey( + routeId: string, + version: string, + fileContent: string, + artifactsMap: Record, + consumedArtifactIds: string[], +): Promise { + const artifactHashes: Record = {}; + for (const id of consumedArtifactIds) { + if (id in artifactsMap) { + artifactHashes[id] = hashArtifact(artifactsMap[id]); + } + } + + return { + routeId, + version, + inputHash: defaultHashFn(fileContent), + artifactHashes, + }; +} + +interface ProcessingQueue { + add: (task: () => Promise) => Promise; + drain: () => Promise; +} + +function createProcessingQueue(concurrency: number): ProcessingQueue { + const queue: (() => Promise)[] = []; + let running = 0; + let resolveIdle: (() => void) | null = null; + + async function runNext(): Promise { + if (running >= concurrency || queue.length === 0) { + if (running === 0 && queue.length === 0 && resolveIdle) { + resolveIdle(); + } + return; + } + + running++; + const task = queue.shift()!; + + try { + await task(); + } finally { + running--; + runNext(); + } + } + + return { + add: async (task) => { + queue.push(task); + runNext(); + }, + drain: () => { + if (running === 0 && queue.length === 0) { + return Promise.resolve(); + } + return new Promise((resolve) => { + resolveIdle = resolve; + }); + }, + }; +} diff --git a/packages/pipelines/pipeline-executor/src/index.ts b/packages/pipelines/pipeline-executor/src/index.ts index e69de29bb..407897fbb 100644 --- a/packages/pipelines/pipeline-executor/src/index.ts +++ b/packages/pipelines/pipeline-executor/src/index.ts @@ -0,0 +1,28 @@ +export type { + CacheKey, + CacheEntry, + CacheStore, + CacheStats, + CacheOptions, +} from "./cache"; + +export { + serializeCacheKey, + createMemoryCacheStore, + defaultHashFn, + hashArtifact, +} from "./cache"; + +export type { + PipelineSummary, + PipelineRunResult, + MultiplePipelineRunResult, +} from "./results"; + +export type { + PipelineExecutorOptions, + PipelineExecutorRunOptions, + PipelineExecutor, +} from "./executor"; + +export { createPipelineExecutor } from "./executor"; diff --git a/packages/pipelines/pipeline-executor/src/results.ts b/packages/pipelines/pipeline-executor/src/results.ts new file mode 100644 index 000000000..eb4ac6325 --- /dev/null +++ b/packages/pipelines/pipeline-executor/src/results.ts @@ -0,0 +1,28 @@ +import type { PipelineError, PipelineGraph } from "@ucdjs/pipelines-core"; + +export interface PipelineSummary { + versions: string[]; + totalFiles: number; + matchedFiles: number; + skippedFiles: number; + fallbackFiles: number; + totalOutputs: number; + durationMs: number; +} + +export interface PipelineRunResult { + data: TData[]; + graph: PipelineGraph; + errors: PipelineError[]; + summary: PipelineSummary; +} + +export interface MultiplePipelineRunResult { + results: Map>; + summary: { + totalPipelines: number; + successfulPipelines: number; + failedPipelines: number; + durationMs: number; + }; +} diff --git a/packages/pipelines/pipeline-graph/package.json b/packages/pipelines/pipeline-graph/package.json index 3b579ebb9..f070e15b1 100644 --- a/packages/pipelines/pipeline-graph/package.json +++ b/packages/pipelines/pipeline-graph/package.json @@ -39,19 +39,15 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { - "@ucdjs-internal/shared": "workspace:*", - "picomatch": "catalog:prod", - "zod": "catalog:prod" + "@ucdjs/pipelines-core": "workspace:*" }, "devDependencies": { "@luxass/eslint-config": "catalog:linting", - "@types/picomatch": "catalog:types", "@ucdjs-tooling/tsconfig": "workspace:*", "@ucdjs-tooling/tsdown-config": "workspace:*", "eslint": "catalog:linting", "publint": "catalog:dev", "tsdown": "catalog:dev", - "tsx": "catalog:dev", "typescript": "catalog:dev" }, "publishConfig": { diff --git a/packages/pipelines/pipeline-graph/src/dag.ts b/packages/pipelines/pipeline-graph/src/dag.ts new file mode 100644 index 000000000..f01cdbfc9 --- /dev/null +++ b/packages/pipelines/pipeline-graph/src/dag.ts @@ -0,0 +1,225 @@ +import type { PipelineRouteDefinition } from "@ucdjs/pipelines-core"; +import { isArtifactDependency, isRouteDependency, parseDependency } from "@ucdjs/pipelines-core"; + +export interface DAGNode { + id: string; + dependencies: Set; + dependents: Set; + emittedArtifacts: Set; +} + +export interface DAG { + nodes: Map; + executionOrder: string[]; +} + +export interface DAGValidationError { + type: "cycle" | "missing-route" | "missing-artifact"; + message: string; + details: { + routeId?: string; + dependencyId?: string; + cycle?: string[]; + }; +} + +export interface DAGValidationResult { + valid: boolean; + errors: DAGValidationError[]; + dag?: DAG; +} + +export function buildDAG(routes: readonly PipelineRouteDefinition[]): DAGValidationResult { + const errors: DAGValidationError[] = []; + const nodes = new Map(); + const routeIds = new Set(routes.map((r) => r.id)); + const artifactsByRoute = new Map>(); + + for (const route of routes) { + const emittedArtifacts = new Set(); + if (route.emits) { + for (const artifactName of Object.keys(route.emits)) { + emittedArtifacts.add(`${route.id}:${artifactName}`); + } + } + artifactsByRoute.set(route.id, emittedArtifacts); + + nodes.set(route.id, { + id: route.id, + dependencies: new Set(), + dependents: new Set(), + emittedArtifacts, + }); + } + + for (const route of routes) { + const node = nodes.get(route.id)!; + + if (!route.depends) continue; + + for (const dep of route.depends) { + const parsed = parseDependency(dep); + + if (isRouteDependency(dep)) { + if (!routeIds.has(parsed.routeId)) { + errors.push({ + type: "missing-route", + message: `Route "${route.id}" depends on non-existent route "${parsed.routeId}"`, + details: { routeId: route.id, dependencyId: parsed.routeId }, + }); + continue; + } + node.dependencies.add(parsed.routeId); + nodes.get(parsed.routeId)!.dependents.add(route.id); + } else if (isArtifactDependency(dep)) { + const artifactParsed = parseDependency(dep); + if (artifactParsed.type !== "artifact") continue; + + if (!routeIds.has(artifactParsed.routeId)) { + errors.push({ + type: "missing-route", + message: `Route "${route.id}" depends on artifact from non-existent route "${artifactParsed.routeId}"`, + details: { routeId: route.id, dependencyId: artifactParsed.routeId }, + }); + continue; + } + + const routeArtifacts = artifactsByRoute.get(artifactParsed.routeId); + const artifactKey = `${artifactParsed.routeId}:${artifactParsed.artifactName}`; + if (!routeArtifacts?.has(artifactKey)) { + errors.push({ + type: "missing-artifact", + message: `Route "${route.id}" depends on non-existent artifact "${artifactParsed.artifactName}" from route "${artifactParsed.routeId}"`, + details: { routeId: route.id, dependencyId: artifactKey }, + }); + continue; + } + + node.dependencies.add(artifactParsed.routeId); + nodes.get(artifactParsed.routeId)!.dependents.add(route.id); + } + } + } + + const cycleResult = detectCycle(nodes); + if (cycleResult) { + errors.push({ + type: "cycle", + message: `Circular dependency detected: ${cycleResult.join(" -> ")}`, + details: { cycle: cycleResult }, + }); + } + + if (errors.length > 0) { + return { valid: false, errors }; + } + + const executionOrder = topologicalSort(nodes); + + return { + valid: true, + errors: [], + dag: { nodes, executionOrder }, + }; +} + +function detectCycle(nodes: Map): string[] | null { + const visited = new Set(); + const recursionStack = new Set(); + const path: string[] = []; + + function dfs(nodeId: string): string[] | null { + visited.add(nodeId); + recursionStack.add(nodeId); + path.push(nodeId); + + const node = nodes.get(nodeId); + if (node) { + for (const depId of node.dependencies) { + if (!visited.has(depId)) { + const cycle = dfs(depId); + if (cycle) return cycle; + } else if (recursionStack.has(depId)) { + const cycleStart = path.indexOf(depId); + return [...path.slice(cycleStart), depId]; + } + } + } + + path.pop(); + recursionStack.delete(nodeId); + return null; + } + + for (const nodeId of nodes.keys()) { + if (!visited.has(nodeId)) { + const cycle = dfs(nodeId); + if (cycle) return cycle; + } + } + + return null; +} + +function topologicalSort(nodes: Map): string[] { + const result: string[] = []; + const visited = new Set(); + const temp = new Set(); + + function visit(nodeId: string): void { + if (temp.has(nodeId)) return; + if (visited.has(nodeId)) return; + + temp.add(nodeId); + + const node = nodes.get(nodeId); + if (node) { + for (const depId of node.dependencies) { + visit(depId); + } + } + + temp.delete(nodeId); + visited.add(nodeId); + result.push(nodeId); + } + + for (const nodeId of nodes.keys()) { + if (!visited.has(nodeId)) { + visit(nodeId); + } + } + + return result; +} + +export function getExecutionLayers(dag: DAG): string[][] { + const layers: string[][] = []; + const scheduled = new Set(); + const remaining = new Set(dag.nodes.keys()); + + while (remaining.size > 0) { + const layer: string[] = []; + + for (const nodeId of remaining) { + const node = dag.nodes.get(nodeId)!; + const allDepsScheduled = [...node.dependencies].every((dep) => scheduled.has(dep)); + if (allDepsScheduled) { + layer.push(nodeId); + } + } + + if (layer.length === 0) { + break; + } + + for (const nodeId of layer) { + remaining.delete(nodeId); + scheduled.add(nodeId); + } + + layers.push(layer); + } + + return layers; +} diff --git a/packages/pipelines/pipeline-graph/src/index.ts b/packages/pipelines/pipeline-graph/src/index.ts index e69de29bb..a02490739 100644 --- a/packages/pipelines/pipeline-graph/src/index.ts +++ b/packages/pipelines/pipeline-graph/src/index.ts @@ -0,0 +1,11 @@ +export type { + DAGNode, + DAG, + DAGValidationError, + DAGValidationResult, +} from "./dag"; + +export { + buildDAG, + getExecutionLayers, +} from "./dag"; diff --git a/packages/pipelines/pipeline-loader/package.json b/packages/pipelines/pipeline-loader/package.json index b765e2ab1..c1ef37072 100644 --- a/packages/pipelines/pipeline-loader/package.json +++ b/packages/pipelines/pipeline-loader/package.json @@ -39,19 +39,15 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { - "@ucdjs-internal/shared": "workspace:*", - "picomatch": "catalog:prod", - "zod": "catalog:prod" + "@ucdjs/pipelines-core": "workspace:*" }, "devDependencies": { "@luxass/eslint-config": "catalog:linting", - "@types/picomatch": "catalog:types", "@ucdjs-tooling/tsconfig": "workspace:*", "@ucdjs-tooling/tsdown-config": "workspace:*", "eslint": "catalog:linting", "publint": "catalog:dev", "tsdown": "catalog:dev", - "tsx": "catalog:dev", "typescript": "catalog:dev" }, "publishConfig": { diff --git a/packages/pipelines/pipeline-loader/src/index.ts b/packages/pipelines/pipeline-loader/src/index.ts index e69de29bb..9b6f37e29 100644 --- a/packages/pipelines/pipeline-loader/src/index.ts +++ b/packages/pipelines/pipeline-loader/src/index.ts @@ -0,0 +1,11 @@ +export { + loadPipelineFile, + loadPipelines, + loadPipelinesFromPaths, +} from "./loader"; + +export type { + LoadedPipelineFile, + LoadPipelinesOptions, + LoadPipelinesResult, +} from "./loader"; diff --git a/packages/pipelines/pipeline-loader/src/loader.ts b/packages/pipelines/pipeline-loader/src/loader.ts new file mode 100644 index 000000000..743ebbd9f --- /dev/null +++ b/packages/pipelines/pipeline-loader/src/loader.ts @@ -0,0 +1,160 @@ +import type { PipelineDefinition } from "@ucdjs/pipelines-core"; +import { isPipelineDefinition } from "@ucdjs/pipelines-core"; + +/** + * Result of loading pipeline definitions from a file. + */ +export interface LoadedPipelineFile { + /** + * The file path that was loaded. + */ + filePath: string; + + /** + * Pipeline definitions found in the file. + */ + pipelines: PipelineDefinition[]; + + /** + * Export names that contained pipeline definitions. + */ + exportNames: string[]; +} + +/** + * Result of loading multiple pipeline files. + */ +export interface LoadPipelinesResult { + /** + * All pipeline definitions found across all files. + */ + pipelines: PipelineDefinition[]; + + /** + * Details about each file that was loaded. + */ + files: LoadedPipelineFile[]; + + /** + * Files that failed to load. + */ + errors: Array<{ + filePath: string; + error: Error; + }>; +} + +/** + * Options for loading pipelines. + */ +export interface LoadPipelinesOptions { + /** + * If true, throw on first error instead of collecting errors. + * @default false + */ + throwOnError?: boolean; +} + +/** + * Load a single pipeline file and extract all PipelineDefinition exports. + * + * @param filePath - Absolute or relative path to the file to load + * @returns The loaded pipeline file with extracted definitions + * + * @example + * ```ts + * const result = await loadPipelineFile("./pipelines/my-pipeline.ts"); + * console.log(result.pipelines); // Array of PipelineDefinition + * ``` + */ +export async function loadPipelineFile(filePath: string): Promise { + const module = await import(filePath); + + const pipelines: PipelineDefinition[] = []; + const exportNames: string[] = []; + + for (const [name, value] of Object.entries(module)) { + if (isPipelineDefinition(value)) { + pipelines.push(value); + exportNames.push(name); + } + } + + return { + filePath, + pipelines, + exportNames, + }; +} + +/** + * Load multiple pipeline files and extract all PipelineDefinition exports. + * + * @param filePaths - Array of file paths to load + * @param options - Loading options + * @returns Combined result with all pipelines and any errors + * + * @example + * ```ts + * const result = await loadPipelines([ + * "./pipelines/blocks.ts", + * "./pipelines/scripts.ts", + * ]); + * + * console.log(result.pipelines); // All pipelines from all files + * console.log(result.errors); // Any files that failed to load + * ``` + */ +export async function loadPipelines( + filePaths: string[], + options: LoadPipelinesOptions = {}, +): Promise { + const { throwOnError = false } = options; + + const pipelines: PipelineDefinition[] = []; + const files: LoadedPipelineFile[] = []; + const errors: Array<{ filePath: string; error: Error }> = []; + + for (const filePath of filePaths) { + try { + const result = await loadPipelineFile(filePath); + files.push(result); + pipelines.push(...result.pipelines); + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + + if (throwOnError) { + throw new Error(`Failed to load pipeline file: ${filePath}`, { cause: error }); + } + + errors.push({ filePath, error }); + } + } + + return { + pipelines, + files, + errors, + }; +} + +/** + * Load pipelines from a directory by matching file patterns. + * This is a convenience wrapper that combines glob matching with loading. + * + * Note: This function requires the caller to provide the resolved file paths. + * Use a glob library to find files first, then pass them to loadPipelines. + * + * @param filePaths - Pre-resolved file paths (use a glob library to find them) + * @param options - Loading options + * @returns Combined result with all pipelines + * + * @example + * ```ts + * import { glob } from "glob"; + * + * const files = await glob("./pipelines/*.ts"); + * const result = await loadPipelines(files); + * ``` + */ +export { loadPipelines as loadPipelinesFromPaths }; diff --git a/packages/pipelines/pipeline-ui/package.json b/packages/pipelines/pipeline-ui/package.json index 57ee99589..db174e5b0 100644 --- a/packages/pipelines/pipeline-ui/package.json +++ b/packages/pipelines/pipeline-ui/package.json @@ -39,19 +39,16 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { - "@ucdjs-internal/shared": "workspace:*", - "picomatch": "catalog:prod", - "zod": "catalog:prod" + "@ucdjs/pipelines-core": "workspace:*", + "@ucdjs/pipelines-graph": "workspace:*" }, "devDependencies": { "@luxass/eslint-config": "catalog:linting", - "@types/picomatch": "catalog:types", "@ucdjs-tooling/tsconfig": "workspace:*", "@ucdjs-tooling/tsdown-config": "workspace:*", "eslint": "catalog:linting", "publint": "catalog:dev", "tsdown": "catalog:dev", - "tsx": "catalog:dev", "typescript": "catalog:dev" }, "publishConfig": { diff --git a/packages/pipelines/pipeline-ui/src/index.ts b/packages/pipelines/pipeline-ui/src/index.ts index e69de29bb..3d9d474a1 100644 --- a/packages/pipelines/pipeline-ui/src/index.ts +++ b/packages/pipelines/pipeline-ui/src/index.ts @@ -0,0 +1,32 @@ +import type { PipelineDefinition } from "@ucdjs/pipelines-core"; +import type { DAG } from "@ucdjs/pipelines-graph"; + +export interface GraphVisualizationOptions { + format?: "mermaid" | "dot" | "json"; +} + +export function visualizeGraph( + _dag: DAG, + _options?: GraphVisualizationOptions, +): string { + throw new Error("Not implemented: visualizeGraph is a placeholder for future functionality"); +} + +export interface EventLogOptions { + maxEvents?: number; + filter?: (event: unknown) => boolean; +} + +export function createEventLogger(_options?: EventLogOptions): { + log: (event: unknown) => void; + getEvents: () => unknown[]; + clear: () => void; +} { + throw new Error("Not implemented: createEventLogger is a placeholder for future functionality"); +} + +export function renderPipelineSummary( + _pipelines: PipelineDefinition[], +): string { + throw new Error("Not implemented: renderPipelineSummary is a placeholder for future functionality"); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f6d4026a..a83c30feb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -931,14 +931,11 @@ importers: specifier: catalog:testing version: 4.4.2(vitest@4.1.0-beta.1) - packages/pipelines/pipeline-core: + packages/pipelines/pipeline-artifacts: dependencies: - '@ucdjs-internal/shared': + '@ucdjs/pipelines-core': specifier: workspace:* - version: link:../../shared - picomatch: - specifier: catalog:prod - version: 4.0.3 + version: link:../pipeline-core zod: specifier: catalog:prod version: 4.3.6 @@ -946,9 +943,6 @@ importers: '@luxass/eslint-config': specifier: catalog:linting version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) - '@types/picomatch': - specifier: catalog:types - version: 4.0.2 '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig @@ -964,14 +958,11 @@ importers: tsdown: specifier: catalog:dev version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) - tsx: - specifier: catalog:dev - version: 4.21.0 typescript: specifier: catalog:dev version: 5.9.3 - packages/pipelines/pipeline-executor: + packages/pipelines/pipeline-core: dependencies: '@ucdjs-internal/shared': specifier: workspace:* @@ -1011,24 +1002,21 @@ importers: specifier: catalog:dev version: 5.9.3 - packages/pipelines/pipeline-graph: + packages/pipelines/pipeline-executor: dependencies: - '@ucdjs-internal/shared': + '@ucdjs/pipelines-artifacts': specifier: workspace:* - version: link:../../shared - picomatch: - specifier: catalog:prod - version: 4.0.3 - zod: - specifier: catalog:prod - version: 4.3.6 + version: link:../pipeline-artifacts + '@ucdjs/pipelines-core': + specifier: workspace:* + version: link:../pipeline-core + '@ucdjs/pipelines-graph': + specifier: workspace:* + version: link:../pipeline-graph devDependencies: '@luxass/eslint-config': specifier: catalog:linting version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) - '@types/picomatch': - specifier: catalog:types - version: 4.0.2 '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig @@ -1044,31 +1032,47 @@ importers: tsdown: specifier: catalog:dev version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) - tsx: + typescript: specifier: catalog:dev - version: 4.21.0 + version: 5.9.3 + + packages/pipelines/pipeline-graph: + dependencies: + '@ucdjs/pipelines-core': + specifier: workspace:* + version: link:../pipeline-core + devDependencies: + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + publint: + specifier: catalog:dev + version: 0.3.17 + tsdown: + specifier: catalog:dev + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) typescript: specifier: catalog:dev version: 5.9.3 packages/pipelines/pipeline-loader: dependencies: - '@ucdjs-internal/shared': + '@ucdjs/pipelines-core': specifier: workspace:* - version: link:../../shared - picomatch: - specifier: catalog:prod - version: 4.0.3 - zod: - specifier: catalog:prod - version: 4.3.6 + version: link:../pipeline-core devDependencies: '@luxass/eslint-config': specifier: catalog:linting version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) - '@types/picomatch': - specifier: catalog:types - version: 4.0.2 '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig @@ -1084,31 +1088,22 @@ importers: tsdown: specifier: catalog:dev version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) - tsx: - specifier: catalog:dev - version: 4.21.0 typescript: specifier: catalog:dev version: 5.9.3 packages/pipelines/pipeline-ui: dependencies: - '@ucdjs-internal/shared': + '@ucdjs/pipelines-core': specifier: workspace:* - version: link:../../shared - picomatch: - specifier: catalog:prod - version: 4.0.3 - zod: - specifier: catalog:prod - version: 4.3.6 + version: link:../pipeline-core + '@ucdjs/pipelines-graph': + specifier: workspace:* + version: link:../pipeline-graph devDependencies: '@luxass/eslint-config': specifier: catalog:linting version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) - '@types/picomatch': - specifier: catalog:types - version: 4.0.2 '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig @@ -1124,9 +1119,6 @@ importers: tsdown: specifier: catalog:dev version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) - tsx: - specifier: catalog:dev - version: 4.21.0 typescript: specifier: catalog:dev version: 5.9.3 @@ -6243,10 +6235,6 @@ packages: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} - expect-type@1.2.2: - resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} - engines: {node: '>=12.0.0'} - expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -7593,7 +7581,7 @@ packages: hasBin: true miniflare@https://pkg.pr.new/cloudflare/workers-sdk/miniflare@79a1932: - resolution: {integrity: sha512-AzGelopZDQwEVqzT1spQKyBn0J12EsD52GlkJUB0HaQ6oNWLRM66hiQJ66sqpIzwQ7+DC2Qtgv4gqTyED3IBRw==, tarball: https://pkg.pr.new/cloudflare/workers-sdk/miniflare@79a1932} + resolution: {tarball: https://pkg.pr.new/cloudflare/workers-sdk/miniflare@79a1932} version: 4.20260120.0 engines: {node: '>=18.0.0'} hasBin: true @@ -14701,9 +14689,6 @@ snapshots: expand-template@2.0.3: optional: true - expect-type@1.2.2: - optional: true - expect-type@1.3.0: {} express-rate-limit@7.5.1(express@5.2.1): @@ -18390,7 +18375,7 @@ snapshots: '@vitest/spy': 4.0.17 '@vitest/utils': 4.0.17 es-module-lexer: 1.7.0 - expect-type: 1.2.2 + expect-type: 1.3.0 magic-string: 0.30.21 obug: 2.1.1 pathe: 2.0.3 diff --git a/tooling/tsconfig/base.json b/tooling/tsconfig/base.json index c2f5080f5..3dc7a87db 100644 --- a/tooling/tsconfig/base.json +++ b/tooling/tsconfig/base.json @@ -52,6 +52,14 @@ "./packages/fs-bridge/src/bridges/node.ts" ], + // pipeline packages + "@ucdjs/pipelines-core": ["./packages/pipelines/pipeline-core/src/index.ts"], + "@ucdjs/pipelines-artifacts": ["./packages/pipelines/pipeline-artifacts/src/index.ts"], + "@ucdjs/pipelines-graph": ["./packages/pipelines/pipeline-graph/src/index.ts"], + "@ucdjs/pipelines-executor": ["./packages/pipelines/pipeline-executor/src/index.ts"], + "@ucdjs/pipelines-loader": ["./packages/pipelines/pipeline-loader/src/index.ts"], + "@ucdjs/pipelines-ui": ["./packages/pipelines/pipeline-ui/src/index.ts"], + // Test utils "@ucdjs/test-utils": ["./packages/test-utils/src/index.ts"], // we use alias for test utils to avoid having to build it during development diff --git a/vitest.config.ts b/vitest.config.ts index cb2a0525e..573d9701c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -48,6 +48,8 @@ async function createProjects(root: string): Promise } const packageProjects = await createProjects("packages"); +const pipelinePackageProjects = await createProjects("packages/pipelines"); + const appProjects = await createProjects("apps"); const hiddenLogs: string[] = []; @@ -72,7 +74,11 @@ export default defineConfig({ return false; }, - projects: [...packageProjects, ...appProjects], + projects: [ + ...packageProjects, + ...pipelinePackageProjects, + ...appProjects + ] }, esbuild: { target: "es2020" }, resolve: { alias: aliases }, From 787478bbd3a0c0269669e0d585443d62d8aad4ab Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 19 Jan 2026 06:42:31 +0100 Subject: [PATCH 04/63] feat(pipeline-presets): add standard and unicode data parsers - Introduced `standard.ts` for parsing standard Unicode data. - Added `unicode-data.ts` for parsing detailed Unicode character metadata. - Created pipelines for basic, emoji, and full Unicode data processing. - Implemented resolvers for grouped and property JSON outputs. - Added various transforms including deduplication, range expansion, and filtering. - Established HTTP and memory sources for data retrieval. - Configured TypeScript settings for building and testing. --- .../pipeline-presets/eslint.config.js | 7 ++ .../pipelines/pipeline-presets/package.json | 63 +++++++++++ .../pipelines/pipeline-presets/src/index.ts | 79 +++++++++++++ .../pipeline-presets/src/parsers/index.ts | 23 ++++ .../src/parsers/multi-property.ts | 71 ++++++++++++ .../pipeline-presets/src/parsers/sequence.ts | 63 +++++++++++ .../pipeline-presets/src/parsers/standard.ts | 63 +++++++++++ .../src/parsers/unicode-data.ts | 106 +++++++++++++++++ .../pipeline-presets/src/pipelines/basic.ts | 37 ++++++ .../pipeline-presets/src/pipelines/emoji.ts | 37 ++++++ .../pipeline-presets/src/pipelines/full.ts | 37 ++++++ .../pipeline-presets/src/pipelines/index.ts | 14 +++ .../pipeline-presets/src/resolvers/grouped.ts | 75 ++++++++++++ .../pipeline-presets/src/resolvers/index.ts | 10 ++ .../src/resolvers/property-json.ts | 63 +++++++++++ .../pipeline-presets/src/routes/common.ts | 107 ++++++++++++++++++ .../pipeline-presets/src/routes/index.ts | 13 +++ .../pipeline-presets/src/sources/http.ts | 107 ++++++++++++++++++ .../pipeline-presets/src/sources/index.ts | 17 +++ .../pipeline-presets/src/sources/memory.ts | 95 ++++++++++++++++ .../src/transforms/deduplicate.ts | 74 ++++++++++++ .../src/transforms/expand-ranges.ts | 72 ++++++++++++ .../pipeline-presets/src/transforms/filter.ts | 71 ++++++++++++ .../pipeline-presets/src/transforms/index.ts | 30 +++++ .../src/transforms/normalize.ts | 66 +++++++++++ .../pipeline-presets/src/transforms/sort.ts | 61 ++++++++++ .../pipeline-presets/tsconfig.build.json | 5 + .../pipelines/pipeline-presets/tsconfig.json | 8 ++ .../pipeline-presets/tsdown.config.ts | 13 +++ pnpm-lock.yaml | 31 +++++ 30 files changed, 1518 insertions(+) create mode 100644 packages/pipelines/pipeline-presets/eslint.config.js create mode 100644 packages/pipelines/pipeline-presets/package.json create mode 100644 packages/pipelines/pipeline-presets/src/index.ts create mode 100644 packages/pipelines/pipeline-presets/src/parsers/index.ts create mode 100644 packages/pipelines/pipeline-presets/src/parsers/multi-property.ts create mode 100644 packages/pipelines/pipeline-presets/src/parsers/sequence.ts create mode 100644 packages/pipelines/pipeline-presets/src/parsers/standard.ts create mode 100644 packages/pipelines/pipeline-presets/src/parsers/unicode-data.ts create mode 100644 packages/pipelines/pipeline-presets/src/pipelines/basic.ts create mode 100644 packages/pipelines/pipeline-presets/src/pipelines/emoji.ts create mode 100644 packages/pipelines/pipeline-presets/src/pipelines/full.ts create mode 100644 packages/pipelines/pipeline-presets/src/pipelines/index.ts create mode 100644 packages/pipelines/pipeline-presets/src/resolvers/grouped.ts create mode 100644 packages/pipelines/pipeline-presets/src/resolvers/index.ts create mode 100644 packages/pipelines/pipeline-presets/src/resolvers/property-json.ts create mode 100644 packages/pipelines/pipeline-presets/src/routes/common.ts create mode 100644 packages/pipelines/pipeline-presets/src/routes/index.ts create mode 100644 packages/pipelines/pipeline-presets/src/sources/http.ts create mode 100644 packages/pipelines/pipeline-presets/src/sources/index.ts create mode 100644 packages/pipelines/pipeline-presets/src/sources/memory.ts create mode 100644 packages/pipelines/pipeline-presets/src/transforms/deduplicate.ts create mode 100644 packages/pipelines/pipeline-presets/src/transforms/expand-ranges.ts create mode 100644 packages/pipelines/pipeline-presets/src/transforms/filter.ts create mode 100644 packages/pipelines/pipeline-presets/src/transforms/index.ts create mode 100644 packages/pipelines/pipeline-presets/src/transforms/normalize.ts create mode 100644 packages/pipelines/pipeline-presets/src/transforms/sort.ts create mode 100644 packages/pipelines/pipeline-presets/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-presets/tsconfig.json create mode 100644 packages/pipelines/pipeline-presets/tsdown.config.ts diff --git a/packages/pipelines/pipeline-presets/eslint.config.js b/packages/pipelines/pipeline-presets/eslint.config.js new file mode 100644 index 000000000..d9c0ca1ec --- /dev/null +++ b/packages/pipelines/pipeline-presets/eslint.config.js @@ -0,0 +1,7 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, +}); diff --git a/packages/pipelines/pipeline-presets/package.json b/packages/pipelines/pipeline-presets/package.json new file mode 100644 index 000000000..a428e626d --- /dev/null +++ b/packages/pipelines/pipeline-presets/package.json @@ -0,0 +1,63 @@ +{ + "name": "@ucdjs/pipeline-presets", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-presets" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + ".": "./dist/index.mjs", + "./parsers": "./dist/parsers/index.mjs", + "./transforms": "./dist/transforms/index.mjs", + "./resolvers": "./dist/resolvers/index.mjs", + "./sources": "./dist/sources/index.mjs", + "./routes": "./dist/routes/index.mjs", + "./pipelines": "./dist/pipelines/index.mjs", + "./package.json": "./package.json" + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "tsdown --tsconfig=./tsconfig.build.json", + "dev": "tsdown --watch", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs/pipelines-core": "workspace:*", + "zod": "catalog:prod" + }, + "devDependencies": { + "@luxass/eslint-config": "catalog:linting", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "eslint": "catalog:linting", + "publint": "catalog:dev", + "tsdown": "catalog:dev", + "typescript": "catalog:dev" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-presets/src/index.ts b/packages/pipelines/pipeline-presets/src/index.ts new file mode 100644 index 000000000..9b9ef99f6 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/index.ts @@ -0,0 +1,79 @@ +export { + createMultiPropertyParser, + createSequenceParser, + createStandardParser, + multiPropertyParser, + type MultiPropertyParserOptions, + sequenceParser, + type SequenceParserOptions, + standardParser, + type StandardParserOptions, + type UnicodeDataMeta, + unicodeDataParser, + type UnicodeDataRow, +} from "./parsers"; + +export { + type BasicPipelineOptions, + createBasicPipeline, + createEmojiPipeline, + createFullPipeline, + type EmojiPipelineOptions, + type FullPipelineOptions, +} from "./pipelines"; + +export { + createGroupedResolver, + createPropertyJsonResolver, + type GroupedResolverOptions, + propertyJsonResolver, + type PropertyJsonResolverOptions, +} from "./resolvers"; + +export { + allRoutes, + blocksRoute, + coreRoutes, + derivedCorePropertiesRoute, + emojiDataRoute, + emojiRoutes, + generalCategoryRoute, + lineBreakRoute, + propListRoute, + scriptsRoute, + unicodeDataRoute, +} from "./routes"; + +export { + createHttpBackend, + createHttpSource, + createMemoryBackend, + createMemorySource, + createUnicodeOrgSource, + type HttpBackendOptions, + type HttpSourceOptions, + type MemoryBackendOptions, + type MemoryFile, + type MemorySourceOptions, + UNICODE_ORG_BASE_URL, + unicodeOrgSource, +} from "./sources"; + +export { + createDeduplicateTransform, + createExpandRangesTransform, + createFilterByPipelineFilter, + createNormalizeTransform, + createRowFilter, + createSortTransform, + type DeduplicateOptions, + deduplicateRows, + type DeduplicateStrategy, + expandRanges, + type ExpandRangesOptions, + normalizeCodePoints, + type RowFilterOptions, + sortByCodePoint, + type SortDirection, + type SortOptions, +} from "./transforms"; diff --git a/packages/pipelines/pipeline-presets/src/parsers/index.ts b/packages/pipelines/pipeline-presets/src/parsers/index.ts new file mode 100644 index 000000000..740d4a2c7 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/parsers/index.ts @@ -0,0 +1,23 @@ +export { + createMultiPropertyParser, + multiPropertyParser, + type MultiPropertyParserOptions, +} from "./multi-property"; + +export { + createSequenceParser, + sequenceParser, + type SequenceParserOptions, +} from "./sequence"; + +export { + createStandardParser, + standardParser, + type StandardParserOptions, +} from "./standard"; + +export { + type UnicodeDataMeta, + unicodeDataParser, + type UnicodeDataRow, +} from "./unicode-data"; diff --git a/packages/pipelines/pipeline-presets/src/parsers/multi-property.ts b/packages/pipelines/pipeline-presets/src/parsers/multi-property.ts new file mode 100644 index 000000000..e32d5468f --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/parsers/multi-property.ts @@ -0,0 +1,71 @@ +import type { ParseContext, ParsedRow, ParserFn } from "@ucdjs/pipelines-core"; + +export interface MultiPropertyParserOptions { + delimiter?: string; + propertyMarker?: string; + trimFields?: boolean; +} + +function parseCodePointOrRange(field: string): { kind: ParsedRow["kind"]; start?: string; end?: string; codePoint?: string } { + const trimmed = field.trim(); + + if (trimmed.includes("..")) { + const [start, end] = trimmed.split(".."); + return { kind: "range", start: start.trim(), end: end.trim() }; + } + + return { kind: "point", codePoint: trimmed }; +} + +export function createMultiPropertyParser(options: MultiPropertyParserOptions = {}): ParserFn { + const { delimiter = ";", propertyMarker = "@", trimFields = true } = options; + + return async function* multiPropertyParser(ctx: ParseContext): AsyncIterable { + let currentProperty: string | undefined; + + for await (const line of ctx.readLines()) { + const trimmedLine = line.trim(); + + if (trimmedLine.startsWith(`# ${propertyMarker}`)) { + const match = trimmedLine.match(/# @(\w+)=(\w+)/); + if (match) { + currentProperty = match[2]; + } + continue; + } + + if (ctx.isComment(line) || trimmedLine === "") { + continue; + } + + const commentIndex = trimmedLine.indexOf("#"); + const dataLine = commentIndex >= 0 ? trimmedLine.slice(0, commentIndex) : trimmedLine; + + if (dataLine.trim() === "") { + continue; + } + + const fields = dataLine.split(delimiter); + if (fields.length < 2) { + continue; + } + + const codePointField = trimFields ? fields[0].trim() : fields[0]; + const valueField = trimFields ? fields[1].trim() : fields[1]; + + const { kind, start, end, codePoint } = parseCodePointOrRange(codePointField); + + yield { + sourceFile: ctx.file.path, + kind, + start, + end, + codePoint, + property: currentProperty || valueField, + value: valueField, + }; + } + }; +} + +export const multiPropertyParser = createMultiPropertyParser(); diff --git a/packages/pipelines/pipeline-presets/src/parsers/sequence.ts b/packages/pipelines/pipeline-presets/src/parsers/sequence.ts new file mode 100644 index 000000000..28a403df7 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/parsers/sequence.ts @@ -0,0 +1,63 @@ +import type { ParseContext, ParsedRow, ParserFn } from "@ucdjs/pipelines-core"; + +export interface SequenceParserOptions { + delimiter?: string; + sequenceDelimiter?: string; + trimFields?: boolean; +} + +export function createSequenceParser(options: SequenceParserOptions = {}): ParserFn { + const { delimiter = ";", sequenceDelimiter = " ", trimFields = true } = options; + + return async function* sequenceParser(ctx: ParseContext): AsyncIterable { + for await (const line of ctx.readLines()) { + if (ctx.isComment(line)) { + continue; + } + + const trimmedLine = line.trim(); + if (trimmedLine === "") { + continue; + } + + const commentIndex = trimmedLine.indexOf("#"); + const dataLine = commentIndex >= 0 ? trimmedLine.slice(0, commentIndex) : trimmedLine; + + if (dataLine.trim() === "") { + continue; + } + + const fields = dataLine.split(delimiter); + if (fields.length < 2) { + continue; + } + + const sequenceField = trimFields ? fields[0].trim() : fields[0]; + const valueField = trimFields ? fields[1].trim() : fields[1]; + + const codePoints = sequenceField.split(sequenceDelimiter).filter(Boolean); + + if (codePoints.length === 0) { + continue; + } + + if (codePoints.length === 1) { + yield { + sourceFile: ctx.file.path, + kind: "point", + codePoint: codePoints[0], + value: valueField, + }; + } else { + yield { + sourceFile: ctx.file.path, + kind: "sequence", + sequence: codePoints, + value: valueField, + }; + } + } + }; +} + +export const sequenceParser = createSequenceParser(); diff --git a/packages/pipelines/pipeline-presets/src/parsers/standard.ts b/packages/pipelines/pipeline-presets/src/parsers/standard.ts new file mode 100644 index 000000000..7d5044b61 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/parsers/standard.ts @@ -0,0 +1,63 @@ +import type { ParseContext, ParsedRow, ParserFn } from "@ucdjs/pipelines-core"; + +export interface StandardParserOptions { + delimiter?: string; + trimFields?: boolean; + skipEmpty?: boolean; +} + +function parseCodePointOrRange(field: string): { kind: ParsedRow["kind"]; start?: string; end?: string; codePoint?: string } { + const trimmed = field.trim(); + + if (trimmed.includes("..")) { + const [start, end] = trimmed.split(".."); + return { kind: "range", start: start.trim(), end: end.trim() }; + } + + return { kind: "point", codePoint: trimmed }; +} + +export function createStandardParser(options: StandardParserOptions = {}): ParserFn { + const { delimiter = ";", trimFields = true, skipEmpty = true } = options; + + return async function* standardParser(ctx: ParseContext): AsyncIterable { + for await (const line of ctx.readLines()) { + if (ctx.isComment(line)) { + continue; + } + + const trimmedLine = line.trim(); + if (skipEmpty && trimmedLine === "") { + continue; + } + + const commentIndex = trimmedLine.indexOf("#"); + const dataLine = commentIndex >= 0 ? trimmedLine.slice(0, commentIndex) : trimmedLine; + + if (dataLine.trim() === "") { + continue; + } + + const fields = dataLine.split(delimiter); + if (fields.length < 2) { + continue; + } + + const codePointField = trimFields ? fields[0].trim() : fields[0]; + const valueField = trimFields ? fields[1].trim() : fields[1]; + + const { kind, start, end, codePoint } = parseCodePointOrRange(codePointField); + + yield { + sourceFile: ctx.file.path, + kind, + start, + end, + codePoint, + value: valueField, + }; + } + }; +} + +export const standardParser = createStandardParser(); diff --git a/packages/pipelines/pipeline-presets/src/parsers/unicode-data.ts b/packages/pipelines/pipeline-presets/src/parsers/unicode-data.ts new file mode 100644 index 000000000..a2e8d13e0 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/parsers/unicode-data.ts @@ -0,0 +1,106 @@ +import type { ParseContext, ParsedRow, ParserFn } from "@ucdjs/pipelines-core"; + +export interface UnicodeDataMeta { + characterName: string; + generalCategory: string; + canonicalCombiningClass: string; + bidiClass: string; + decompositionMapping: string; + numericType: string; + numericValue: string; + bidiMirrored: string; + unicode1Name: string; + isoComment: string; + simpleUppercaseMapping: string; + simpleLowercaseMapping: string; + simpleTitlecaseMapping: string; +} + +export type UnicodeDataRow = ParsedRow & { meta: UnicodeDataMeta }; + +export const unicodeDataParser: ParserFn = async function* ( + ctx: ParseContext, +): AsyncIterable { + let rangeStart: string | null = null; + let rangeName: string | null = null; + + for await (const line of ctx.readLines()) { + if (ctx.isComment(line)) { + continue; + } + + const trimmedLine = line.trim(); + if (trimmedLine === "") { + continue; + } + + const fields = trimmedLine.split(";"); + if (fields.length < 14) { + continue; + } + + const codePoint = fields[0].trim(); + const characterName = fields[1].trim(); + const generalCategory = fields[2].trim(); + + if (characterName.endsWith(", First>")) { + rangeStart = codePoint; + rangeName = characterName.replace(", First>", "").replace("<", ""); + continue; + } + + if (characterName.endsWith(", Last>") && rangeStart !== null) { + const row: UnicodeDataRow = { + sourceFile: ctx.file.path, + kind: "range", + start: rangeStart, + end: codePoint, + value: generalCategory, + meta: { + characterName: rangeName || "", + generalCategory, + canonicalCombiningClass: fields[3].trim(), + bidiClass: fields[4].trim(), + decompositionMapping: fields[5].trim(), + numericType: fields[6].trim(), + numericValue: fields[7].trim(), + bidiMirrored: fields[9].trim(), + unicode1Name: fields[10].trim(), + isoComment: fields[11].trim(), + simpleUppercaseMapping: fields[12].trim(), + simpleLowercaseMapping: fields[13].trim(), + simpleTitlecaseMapping: fields[14]?.trim() || "", + }, + }; + + rangeStart = null; + rangeName = null; + yield row; + continue; + } + + const row: UnicodeDataRow = { + sourceFile: ctx.file.path, + kind: "point", + codePoint, + value: generalCategory, + meta: { + characterName, + generalCategory, + canonicalCombiningClass: fields[3].trim(), + bidiClass: fields[4].trim(), + decompositionMapping: fields[5].trim(), + numericType: fields[6].trim(), + numericValue: fields[7].trim(), + bidiMirrored: fields[9].trim(), + unicode1Name: fields[10].trim(), + isoComment: fields[11].trim(), + simpleUppercaseMapping: fields[12].trim(), + simpleLowercaseMapping: fields[13].trim(), + simpleTitlecaseMapping: fields[14]?.trim() || "", + }, + }; + + yield row; + } +}; diff --git a/packages/pipelines/pipeline-presets/src/pipelines/basic.ts b/packages/pipelines/pipeline-presets/src/pipelines/basic.ts new file mode 100644 index 000000000..5a604053a --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/pipelines/basic.ts @@ -0,0 +1,37 @@ +import type { PipelineDefinition } from "@ucdjs/pipelines-core"; +import { byExt, definePipeline } from "@ucdjs/pipelines-core"; +import { standardParser } from "../parsers/standard"; +import { propertyJsonResolver } from "../resolvers/property-json"; +import { coreRoutes } from "../routes/common"; + +export interface BasicPipelineOptions { + id?: string; + versions: string[]; + concurrency?: number; + strict?: boolean; +} + +export function createBasicPipeline(options: BasicPipelineOptions) { + const { + id = "basic-ucd", + versions, + concurrency = 4, + strict = false, + } = options; + + return definePipeline({ + id, + name: "Basic UCD Pipeline", + description: "Processes core Unicode Character Database files", + versions, + inputs: [], + routes: [...coreRoutes], + include: byExt(".txt"), + concurrency, + strict, + fallback: { + parser: standardParser, + resolver: propertyJsonResolver, + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/src/pipelines/emoji.ts b/packages/pipelines/pipeline-presets/src/pipelines/emoji.ts new file mode 100644 index 000000000..f9fdb17e8 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/pipelines/emoji.ts @@ -0,0 +1,37 @@ +import type { PipelineDefinition } from "@ucdjs/pipelines-core"; +import { and, byDir, byExt, definePipeline } from "@ucdjs/pipelines-core"; +import { sequenceParser } from "../parsers/sequence"; +import { propertyJsonResolver } from "../resolvers/property-json"; +import { emojiRoutes } from "../routes/common"; + +export interface EmojiPipelineOptions { + id?: string; + versions: string[]; + concurrency?: number; + strict?: boolean; +} + +export function createEmojiPipeline(options: EmojiPipelineOptions) { + const { + id = "emoji", + versions, + concurrency = 4, + strict = false, + } = options; + + return definePipeline({ + id, + name: "Emoji Pipeline", + description: "Processes Unicode emoji data files", + versions, + inputs: [], + routes: [...emojiRoutes], + include: and(byDir("emoji"), byExt(".txt")), + concurrency, + strict, + fallback: { + parser: sequenceParser, + resolver: propertyJsonResolver, + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/src/pipelines/full.ts b/packages/pipelines/pipeline-presets/src/pipelines/full.ts new file mode 100644 index 000000000..08cb17e1a --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/pipelines/full.ts @@ -0,0 +1,37 @@ +import type { PipelineDefinition } from "@ucdjs/pipelines-core"; +import { byExt, definePipeline } from "@ucdjs/pipelines-core"; +import { standardParser } from "../parsers/standard"; +import { propertyJsonResolver } from "../resolvers/property-json"; +import { allRoutes } from "../routes/common"; + +export interface FullPipelineOptions { + id?: string; + versions: string[]; + concurrency?: number; + strict?: boolean; +} + +export function createFullPipeline(options: FullPipelineOptions) { + const { + id = "full-ucd", + versions, + concurrency = 4, + strict = false, + } = options; + + return definePipeline({ + id, + name: "Full UCD Pipeline", + description: "Processes all Unicode Character Database files", + versions, + inputs: [], + routes: [...allRoutes], + include: byExt(".txt"), + concurrency, + strict, + fallback: { + parser: standardParser, + resolver: propertyJsonResolver, + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/src/pipelines/index.ts b/packages/pipelines/pipeline-presets/src/pipelines/index.ts new file mode 100644 index 000000000..c7f2b4ae7 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/pipelines/index.ts @@ -0,0 +1,14 @@ +export { + type BasicPipelineOptions, + createBasicPipeline, +} from "./basic"; + +export { + createEmojiPipeline, + type EmojiPipelineOptions, +} from "./emoji"; + +export { + createFullPipeline, + type FullPipelineOptions, +} from "./full"; diff --git a/packages/pipelines/pipeline-presets/src/resolvers/grouped.ts b/packages/pipelines/pipeline-presets/src/resolvers/grouped.ts new file mode 100644 index 000000000..91d869f14 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/resolvers/grouped.ts @@ -0,0 +1,75 @@ +import type { ParsedRow, PropertyJson, ResolvedEntry, RouteResolveContext } from "@ucdjs/pipelines-core"; + +export interface GroupedResolverOptions { + groupBy: "property" | "value" | ((row: ParsedRow) => string); + propertyNameFn?: (groupKey: string, ctx: RouteResolveContext) => string; +} + +function rowToResolvedEntry(row: ParsedRow): ResolvedEntry | null { + const value = row.value; + if (value === undefined) { + return null; + } + + if (row.kind === "point" && row.codePoint) { + return { codePoint: row.codePoint, value }; + } + + if (row.kind === "range" && row.start && row.end) { + return { range: `${row.start}..${row.end}`, value }; + } + + if (row.kind === "sequence" && row.sequence) { + return { sequence: row.sequence, value }; + } + + return null; +} + +export function createGroupedResolver(options: GroupedResolverOptions) { + const { groupBy, propertyNameFn } = options; + + const getGroupKey = typeof groupBy === "function" + ? groupBy + : groupBy === "property" + ? (row: ParsedRow) => row.property || "unknown" + : (row: ParsedRow) => { + const v = row.value; + return Array.isArray(v) ? v.join(",") : v || "unknown"; + }; + + return async function groupedResolver( + ctx: RouteResolveContext, + rows: AsyncIterable, + ): Promise { + const groups = new Map(); + + for await (const row of rows) { + const key = getGroupKey(row); + const entry = rowToResolvedEntry(row); + + if (entry) { + const existing = groups.get(key) || []; + existing.push(entry); + groups.set(key, existing); + } + } + + const results: PropertyJson[] = []; + + for (const [key, entries] of groups) { + const propertyName = propertyNameFn + ? propertyNameFn(key, ctx) + : key; + + results.push({ + version: ctx.version, + property: propertyName, + file: ctx.file.name, + entries: ctx.normalizeEntries(entries), + }); + } + + return results; + }; +} diff --git a/packages/pipelines/pipeline-presets/src/resolvers/index.ts b/packages/pipelines/pipeline-presets/src/resolvers/index.ts new file mode 100644 index 000000000..cba8b9d1b --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/resolvers/index.ts @@ -0,0 +1,10 @@ +export { + createGroupedResolver, + type GroupedResolverOptions, +} from "./grouped"; + +export { + createPropertyJsonResolver, + propertyJsonResolver, + type PropertyJsonResolverOptions, +} from "./property-json"; diff --git a/packages/pipelines/pipeline-presets/src/resolvers/property-json.ts b/packages/pipelines/pipeline-presets/src/resolvers/property-json.ts new file mode 100644 index 000000000..dcc6216f8 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/resolvers/property-json.ts @@ -0,0 +1,63 @@ +import type { ParsedRow, PropertyJson, ResolvedEntry, RouteResolveContext } from "@ucdjs/pipelines-core"; + +export interface PropertyJsonResolverOptions { + property?: string; + includeDefaults?: boolean; +} + +function rowToResolvedEntry(row: ParsedRow): ResolvedEntry | null { + const value = row.value; + if (value === undefined) { + return null; + } + + if (row.kind === "point" && row.codePoint) { + return { + codePoint: row.codePoint, + value, + }; + } + + if (row.kind === "range" && row.start && row.end) { + return { + range: `${row.start}..${row.end}`, + value, + }; + } + + if (row.kind === "sequence" && row.sequence) { + return { + sequence: row.sequence, + value, + }; + } + + return null; +} + +export function createPropertyJsonResolver(options: PropertyJsonResolverOptions = {}) { + return async function propertyJsonResolver( + ctx: RouteResolveContext, + rows: AsyncIterable, + ): Promise { + const entries: ResolvedEntry[] = []; + + for await (const row of rows) { + const entry = rowToResolvedEntry(row); + if (entry) { + entries.push(entry); + } + } + + const propertyName = options.property || ctx.file.name.replace(/\.txt$/, ""); + + return [{ + version: ctx.version, + property: propertyName, + file: ctx.file.name, + entries: ctx.normalizeEntries(entries), + }]; + }; +} + +export const propertyJsonResolver = createPropertyJsonResolver(); diff --git a/packages/pipelines/pipeline-presets/src/routes/common.ts b/packages/pipelines/pipeline-presets/src/routes/common.ts new file mode 100644 index 000000000..13e2b3a33 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/routes/common.ts @@ -0,0 +1,107 @@ +import type { PipelineRouteDefinition } from "@ucdjs/pipelines-core"; +import { + byDir, + byGlob, + byName, + definePipelineRoute, + +} from "@ucdjs/pipelines-core"; +import { multiPropertyParser } from "../parsers/multi-property"; +import { standardParser } from "../parsers/standard"; +import { unicodeDataParser } from "../parsers/unicode-data"; +import { createGroupedResolver } from "../resolvers/grouped"; +import { propertyJsonResolver } from "../resolvers/property-json"; +import { normalizeCodePoints } from "../transforms/normalize"; +import { sortByCodePoint } from "../transforms/sort"; + +export const lineBreakRoute = definePipelineRoute({ + id: "line-break", + filter: byName("LineBreak.txt"), + parser: standardParser, + transforms: [normalizeCodePoints, sortByCodePoint], + resolver: propertyJsonResolver, +}); + +export const scriptsRoute = definePipelineRoute({ + id: "scripts", + filter: byName("Scripts.txt"), + parser: standardParser, + transforms: [normalizeCodePoints, sortByCodePoint], + resolver: propertyJsonResolver, +}); + +export const blocksRoute = definePipelineRoute({ + id: "blocks", + filter: byName("Blocks.txt"), + parser: standardParser, + transforms: [normalizeCodePoints, sortByCodePoint], + resolver: propertyJsonResolver, +}); + +export const generalCategoryRoute = definePipelineRoute({ + id: "general-category", + filter: byName("extracted/DerivedGeneralCategory.txt"), + parser: standardParser, + transforms: [normalizeCodePoints, sortByCodePoint], + resolver: propertyJsonResolver, +}); + +export const propListRoute = definePipelineRoute({ + id: "prop-list", + filter: byName("PropList.txt"), + parser: standardParser, + transforms: [normalizeCodePoints, sortByCodePoint], + resolver: createGroupedResolver({ + groupBy: "value", + propertyNameFn: (value) => value, + }), +}); + +export const derivedCorePropertiesRoute = definePipelineRoute({ + id: "derived-core-properties", + filter: byName("DerivedCoreProperties.txt"), + parser: standardParser, + transforms: [normalizeCodePoints, sortByCodePoint], + resolver: createGroupedResolver({ + groupBy: "value", + propertyNameFn: (value) => value, + }), +}); + +export const emojiDataRoute = definePipelineRoute({ + id: "emoji-data", + filter: byGlob("emoji/emoji-data.txt"), + parser: standardParser, + transforms: [normalizeCodePoints, sortByCodePoint], + resolver: createGroupedResolver({ + groupBy: "value", + propertyNameFn: (value) => `Emoji_${value}`, + }), +}); + +export const unicodeDataRoute = definePipelineRoute({ + id: "unicode-data", + filter: byName("UnicodeData.txt"), + parser: unicodeDataParser, + transforms: [normalizeCodePoints], + resolver: propertyJsonResolver, +}); + +export const coreRoutes = [ + lineBreakRoute, + scriptsRoute, + blocksRoute, + generalCategoryRoute, + propListRoute, + derivedCorePropertiesRoute, + unicodeDataRoute, +] as const; + +export const emojiRoutes = [ + emojiDataRoute, +] as const; + +export const allRoutes = [ + ...coreRoutes, + ...emojiRoutes, +] as const; diff --git a/packages/pipelines/pipeline-presets/src/routes/index.ts b/packages/pipelines/pipeline-presets/src/routes/index.ts new file mode 100644 index 000000000..139b580b2 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/routes/index.ts @@ -0,0 +1,13 @@ +export { + allRoutes, + blocksRoute, + coreRoutes, + derivedCorePropertiesRoute, + emojiDataRoute, + emojiRoutes, + generalCategoryRoute, + lineBreakRoute, + propListRoute, + scriptsRoute, + unicodeDataRoute, +} from "./common"; diff --git a/packages/pipelines/pipeline-presets/src/sources/http.ts b/packages/pipelines/pipeline-presets/src/sources/http.ts new file mode 100644 index 000000000..91b569298 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/sources/http.ts @@ -0,0 +1,107 @@ +import type { + FileContext, + FileMetadata, + PipelineSourceDefinition, + SourceBackend, +} from "@ucdjs/pipelines-core"; +import { definePipelineSource } from "@ucdjs/pipelines-core"; + +export interface HttpBackendOptions { + baseUrl: string; + headers?: Record; + timeout?: number; +} + +export function createHttpBackend(options: HttpBackendOptions): SourceBackend { + const { baseUrl, headers = {}, timeout = 30000 } = options; + + const normalizedBaseUrl = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; + + return { + async listFiles(version: string): Promise { + throw new Error( + `HTTP backend does not support listing files. ` + + `Use a file manifest or provide explicit file list for version ${version}.`, + ); + }, + + async readFile(file: FileContext): Promise { + const url = `${normalizedBaseUrl}${file.version}/${file.path}`; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const response = await fetch(url, { + headers, + signal: controller.signal, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); + } + + return response.text(); + } finally { + clearTimeout(timeoutId); + } + }, + + async getMetadata(file: FileContext): Promise { + const url = `${normalizedBaseUrl}${file.version}/${file.path}`; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const response = await fetch(url, { + method: "HEAD", + headers, + signal: controller.signal, + }); + + if (!response.ok) { + throw new Error(`Failed to get metadata for ${url}: ${response.status}`); + } + + const contentLength = response.headers.get("content-length"); + const lastModified = response.headers.get("last-modified"); + + return { + size: contentLength ? Number.parseInt(contentLength, 10) : 0, + lastModified: lastModified || undefined, + }; + } finally { + clearTimeout(timeoutId); + } + }, + }; +} + +export interface HttpSourceOptions extends HttpBackendOptions { + id?: string; +} + +export function createHttpSource( + options: HttpSourceOptions & { id?: TId }, +): PipelineSourceDefinition { + const { id = "http" as TId, ...backendOptions } = options; + + return definePipelineSource({ + id: id as TId extends undefined ? "http" : TId, + backend: createHttpBackend(backendOptions), + }); +} + +export const UNICODE_ORG_BASE_URL = "https://www.unicode.org/Public/"; + +export function createUnicodeOrgSource( + id?: TId, +): PipelineSourceDefinition { + return createHttpSource({ + id: (id ?? "unicode-org") as TId extends undefined ? "unicode-org" : TId, + baseUrl: UNICODE_ORG_BASE_URL, + }); +} + +export const unicodeOrgSource = createUnicodeOrgSource(); diff --git a/packages/pipelines/pipeline-presets/src/sources/index.ts b/packages/pipelines/pipeline-presets/src/sources/index.ts new file mode 100644 index 000000000..e72bedba1 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/sources/index.ts @@ -0,0 +1,17 @@ +export { + createHttpBackend, + createHttpSource, + createUnicodeOrgSource, + type HttpBackendOptions, + type HttpSourceOptions, + UNICODE_ORG_BASE_URL, + unicodeOrgSource, +} from "./http"; + +export { + createMemoryBackend, + createMemorySource, + type MemoryBackendOptions, + type MemoryFile, + type MemorySourceOptions, +} from "./memory"; diff --git a/packages/pipelines/pipeline-presets/src/sources/memory.ts b/packages/pipelines/pipeline-presets/src/sources/memory.ts new file mode 100644 index 000000000..297ae29f6 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/sources/memory.ts @@ -0,0 +1,95 @@ +import type { + FileContext, + FileMetadata, + PipelineSourceDefinition, + SourceBackend, +} from "@ucdjs/pipelines-core"; +import { definePipelineSource } from "@ucdjs/pipelines-core"; + +export interface MemoryFile { + path: string; + content: string; + dir?: FileContext["dir"]; +} + +export interface MemoryBackendOptions { + files: Record; +} + +function getFileContext(version: string, file: MemoryFile): FileContext { + const path = file.path; + const parts = path.split("/"); + const name = parts[parts.length - 1]; + const extIndex = name.lastIndexOf("."); + const ext = extIndex >= 0 ? name.slice(extIndex) : ""; + const dir = file.dir || parts[0] || "ucd"; + + return { + version, + dir, + path, + name, + ext, + }; +} + +export function createMemoryBackend(options: MemoryBackendOptions): SourceBackend { + const { files } = options; + + return { + async listFiles(version: string): Promise { + const versionFiles = files[version]; + if (!versionFiles) { + return []; + } + + return versionFiles.map((f) => getFileContext(version, f)); + }, + + async readFile(file: FileContext): Promise { + const versionFiles = files[file.version]; + if (!versionFiles) { + throw new Error(`Version ${file.version} not found in memory backend`); + } + + const memFile = versionFiles.find((f) => f.path === file.path); + if (!memFile) { + throw new Error(`File ${file.path} not found in version ${file.version}`); + } + + return memFile.content; + }, + + async getMetadata(file: FileContext): Promise { + const versionFiles = files[file.version]; + if (!versionFiles) { + throw new Error(`Version ${file.version} not found in memory backend`); + } + + const memFile = versionFiles.find((f) => f.path === file.path); + if (!memFile) { + throw new Error(`File ${file.path} not found in version ${file.version}`); + } + + return { + size: new TextEncoder().encode(memFile.content).length, + }; + }, + }; +} + +export interface MemorySourceOptions { + id?: string; + files: Record; +} + +export function createMemorySource( + options: MemorySourceOptions & { id?: TId }, +): PipelineSourceDefinition { + const { id = "memory" as TId, files } = options; + + return definePipelineSource({ + id: id as TId extends undefined ? "memory" : TId, + backend: createMemoryBackend({ files }), + }); +} diff --git a/packages/pipelines/pipeline-presets/src/transforms/deduplicate.ts b/packages/pipelines/pipeline-presets/src/transforms/deduplicate.ts new file mode 100644 index 000000000..dd82bb10b --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/transforms/deduplicate.ts @@ -0,0 +1,74 @@ +import type { ParsedRow } from "@ucdjs/pipelines-core"; +import { definePipelineTransform } from "@ucdjs/pipelines-core"; + +function getRowKey(row: ParsedRow): string { + if (row.kind === "point" && row.codePoint) { + return `point:${row.codePoint}`; + } + if (row.kind === "range" && row.start && row.end) { + return `range:${row.start}..${row.end}`; + } + if (row.kind === "sequence" && row.sequence) { + return `seq:${row.sequence.join("-")}`; + } + return `unknown:${JSON.stringify(row)}`; +} + +export const deduplicateRows = definePipelineTransform({ + id: "deduplicate-rows", + async* fn(_ctx, rows) { + const seen = new Set(); + + for await (const row of rows) { + const key = getRowKey(row); + + if (!seen.has(key)) { + seen.add(key); + yield row; + } + } + }, +}); + +export type DeduplicateStrategy = "first" | "last" | "merge"; + +export interface DeduplicateOptions { + strategy?: DeduplicateStrategy; + keyFn?: (row: ParsedRow) => string; +} + +export function createDeduplicateTransform(options: DeduplicateOptions = {}) { + const { strategy = "first", keyFn = getRowKey } = options; + + if (strategy === "last") { + return definePipelineTransform({ + id: "deduplicate-rows-last", + async* fn(_ctx, rows) { + const byKey = new Map(); + + for await (const row of rows) { + const key = keyFn(row); + byKey.set(key, row); + } + + yield* byKey.values(); + }, + }); + } + + return definePipelineTransform({ + id: "deduplicate-rows-first", + async* fn(_ctx, rows) { + const seen = new Set(); + + for await (const row of rows) { + const key = keyFn(row); + + if (!seen.has(key)) { + seen.add(key); + yield row; + } + } + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/src/transforms/expand-ranges.ts b/packages/pipelines/pipeline-presets/src/transforms/expand-ranges.ts new file mode 100644 index 000000000..5068dfbbe --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/transforms/expand-ranges.ts @@ -0,0 +1,72 @@ +import type { ParsedRow } from "@ucdjs/pipelines-core"; +import { definePipelineTransform } from "@ucdjs/pipelines-core"; + +function hexToNumber(hex: string): number { + return Number.parseInt(hex, 16); +} + +function numberToHex(num: number): string { + return num.toString(16).toUpperCase().padStart(4, "0"); +} + +export const expandRanges = definePipelineTransform({ + id: "expand-ranges", + async* fn(_ctx, rows) { + for await (const row of rows) { + if (row.kind === "range" && row.start && row.end) { + const start = hexToNumber(row.start); + const end = hexToNumber(row.end); + + for (let i = start; i <= end; i++) { + yield { + ...row, + kind: "point", + codePoint: numberToHex(i), + start: undefined, + end: undefined, + }; + } + } else { + yield row; + } + } + }, +}); + +export interface ExpandRangesOptions { + maxExpansion?: number; +} + +export function createExpandRangesTransform(options: ExpandRangesOptions = {}) { + const { maxExpansion = 10000 } = options; + + return definePipelineTransform({ + id: "expand-ranges-limited", + async* fn(_ctx, rows) { + for await (const row of rows) { + if (row.kind === "range" && row.start && row.end) { + const start = hexToNumber(row.start); + const end = hexToNumber(row.end); + const size = end - start + 1; + + if (size > maxExpansion) { + yield row; + continue; + } + + for (let i = start; i <= end; i++) { + yield { + ...row, + kind: "point", + codePoint: numberToHex(i), + start: undefined, + end: undefined, + }; + } + } else { + yield row; + } + } + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/src/transforms/filter.ts b/packages/pipelines/pipeline-presets/src/transforms/filter.ts new file mode 100644 index 000000000..ac1332dcd --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/transforms/filter.ts @@ -0,0 +1,71 @@ +import type { ParsedRow, PipelineFilter } from "@ucdjs/pipelines-core"; +import { definePipelineTransform } from "@ucdjs/pipelines-core"; + +export interface RowFilterOptions { + property?: string | RegExp; + value?: string | RegExp; + kind?: ParsedRow["kind"] | ParsedRow["kind"][]; +} + +export function createRowFilter(options: RowFilterOptions) { + return definePipelineTransform({ + id: "row-filter", + async* fn(ctx, rows) { + for await (const row of rows) { + if (options.property) { + if (!row.property) { + continue; + } + if (typeof options.property === "string") { + if (row.property !== options.property) { + continue; + } + } else if (!options.property.test(row.property)) { + continue; + } + } + + if (options.value) { + const rowValue = Array.isArray(row.value) ? row.value.join(",") : row.value; + if (!rowValue) { + continue; + } + if (typeof options.value === "string") { + if (rowValue !== options.value) { + continue; + } + } else if (!options.value.test(rowValue)) { + continue; + } + } + + if (options.kind) { + const kinds = Array.isArray(options.kind) ? options.kind : [options.kind]; + if (!kinds.includes(row.kind)) { + continue; + } + } + + yield row; + } + }, + }); +} + +export function createFilterByPipelineFilter(filter: PipelineFilter) { + return definePipelineTransform({ + id: "filter-by-pipeline-filter", + async* fn(ctx, rows) { + for await (const row of rows) { + const filterCtx = { + file: ctx.file, + row: row.property ? { property: row.property } : undefined, + }; + + if (filter(filterCtx)) { + yield row; + } + } + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/src/transforms/index.ts b/packages/pipelines/pipeline-presets/src/transforms/index.ts new file mode 100644 index 000000000..53fddc6b3 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/transforms/index.ts @@ -0,0 +1,30 @@ +export { + createDeduplicateTransform, + type DeduplicateOptions, + deduplicateRows, + type DeduplicateStrategy, +} from "./deduplicate"; + +export { + createExpandRangesTransform, + expandRanges, + type ExpandRangesOptions, +} from "./expand-ranges"; + +export { + createFilterByPipelineFilter, + createRowFilter, + type RowFilterOptions, +} from "./filter"; + +export { + createNormalizeTransform, + normalizeCodePoints, +} from "./normalize"; + +export { + createSortTransform, + sortByCodePoint, + type SortDirection, + type SortOptions, +} from "./sort"; diff --git a/packages/pipelines/pipeline-presets/src/transforms/normalize.ts b/packages/pipelines/pipeline-presets/src/transforms/normalize.ts new file mode 100644 index 000000000..a9c80fcf7 --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/transforms/normalize.ts @@ -0,0 +1,66 @@ +import type { ParsedRow, PipelineTransformDefinition } from "@ucdjs/pipelines-core"; +import { definePipelineTransform } from "@ucdjs/pipelines-core"; + +function normalizeHex(hex: string): string { + return hex.toUpperCase().replace(/^0+/, "") || "0"; +} + +function padHex(hex: string, length: number = 4): string { + return hex.toUpperCase().padStart(length, "0"); +} + +export const normalizeCodePoints = definePipelineTransform({ + id: "normalize-code-points", + async* fn(_ctx, rows) { + for await (const row of rows) { + const normalized = { ...row }; + + if (normalized.codePoint) { + normalized.codePoint = padHex(normalizeHex(normalized.codePoint)); + } + + if (normalized.start) { + normalized.start = padHex(normalizeHex(normalized.start)); + } + + if (normalized.end) { + normalized.end = padHex(normalizeHex(normalized.end)); + } + + if (normalized.sequence) { + normalized.sequence = normalized.sequence.map((cp) => padHex(normalizeHex(cp))); + } + + yield normalized; + } + }, +}); + +export function createNormalizeTransform(padLength: number = 4): PipelineTransformDefinition { + return definePipelineTransform({ + id: `normalize-code-points-${padLength}`, + async* fn(_ctx, rows) { + for await (const row of rows) { + const normalized = { ...row }; + + if (normalized.codePoint) { + normalized.codePoint = padHex(normalizeHex(normalized.codePoint), padLength); + } + + if (normalized.start) { + normalized.start = padHex(normalizeHex(normalized.start), padLength); + } + + if (normalized.end) { + normalized.end = padHex(normalizeHex(normalized.end), padLength); + } + + if (normalized.sequence) { + normalized.sequence = normalized.sequence.map((cp) => padHex(normalizeHex(cp), padLength)); + } + + yield normalized; + } + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/src/transforms/sort.ts b/packages/pipelines/pipeline-presets/src/transforms/sort.ts new file mode 100644 index 000000000..2174488ab --- /dev/null +++ b/packages/pipelines/pipeline-presets/src/transforms/sort.ts @@ -0,0 +1,61 @@ +import type { ParsedRow } from "@ucdjs/pipelines-core"; +import { definePipelineTransform } from "@ucdjs/pipelines-core"; + +function hexToNumber(hex: string): number { + return Number.parseInt(hex, 16); +} + +function getRowSortKey(row: ParsedRow): number { + if (row.codePoint) { + return hexToNumber(row.codePoint); + } + if (row.start) { + return hexToNumber(row.start); + } + if (row.sequence && row.sequence.length > 0) { + return hexToNumber(row.sequence[0]); + } + return 0; +} + +export const sortByCodePoint = definePipelineTransform({ + id: "sort-by-code-point", + async* fn(_ctx, rows) { + const collected: ParsedRow[] = []; + + for await (const row of rows) { + collected.push(row); + } + + collected.sort((a, b) => getRowSortKey(a) - getRowSortKey(b)); + + yield* collected; + }, +}); + +export type SortDirection = "asc" | "desc"; + +export interface SortOptions { + direction?: SortDirection; + keyFn?: (row: ParsedRow) => number; +} + +export function createSortTransform(options: SortOptions = {}) { + const { direction = "asc", keyFn = getRowSortKey } = options; + const multiplier = direction === "asc" ? 1 : -1; + + return definePipelineTransform({ + id: `sort-${direction}`, + async* fn(_ctx, rows) { + const collected: ParsedRow[] = []; + + for await (const row of rows) { + collected.push(row); + } + + collected.sort((a, b) => multiplier * (keyFn(a) - keyFn(b))); + + yield* collected; + }, + }); +} diff --git a/packages/pipelines/pipeline-presets/tsconfig.build.json b/packages/pipelines/pipeline-presets/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-presets/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-presets/tsconfig.json b/packages/pipelines/pipeline-presets/tsconfig.json new file mode 100644 index 000000000..9c6dd744b --- /dev/null +++ b/packages/pipelines/pipeline-presets/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test" + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-presets/tsdown.config.ts b/packages/pipelines/pipeline-presets/tsdown.config.ts new file mode 100644 index 000000000..27f9d71b5 --- /dev/null +++ b/packages/pipelines/pipeline-presets/tsdown.config.ts @@ -0,0 +1,13 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/index.ts", + "./src/parsers/index.ts", + "./src/transforms/index.ts", + "./src/resolvers/index.ts", + "./src/sources/index.ts", + "./src/routes/index.ts", + "./src/pipelines/index.ts", + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a83c30feb..c5b56b89d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1092,6 +1092,37 @@ importers: specifier: catalog:dev version: 5.9.3 + packages/pipelines/pipeline-presets: + dependencies: + '@ucdjs/pipelines-core': + specifier: workspace:* + version: link:../pipeline-core + zod: + specifier: catalog:prod + version: 4.3.5 + devDependencies: + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0-beta.1(@eslint-react/eslint-plugin@2.3.12(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.24(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + publint: + specifier: catalog:dev + version: 0.3.16 + tsdown: + specifier: catalog:dev + version: 0.19.0(publint@0.3.16)(synckit@0.11.11)(typescript@5.9.3) + typescript: + specifier: catalog:dev + version: 5.9.3 + packages/pipelines/pipeline-ui: dependencies: '@ucdjs/pipelines-core': From cd7ea18f7fbab1eb30e855925e344b29cb335c90 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 19 Jan 2026 06:42:53 +0100 Subject: [PATCH 05/63] chore: add ESLint configuration files --- packages/pipelines/pipeline-artifacts/eslint.config.js | 7 +++++++ packages/pipelines/pipeline-core/eslint.config.js | 7 +++++++ packages/pipelines/pipeline-executor/eslint.config.js | 7 +++++++ packages/pipelines/pipeline-graph/eslint.config.js | 7 +++++++ packages/pipelines/pipeline-loader/eslint.config.js | 7 +++++++ packages/pipelines/pipeline-ui/eslint.config.js | 7 +++++++ 6 files changed, 42 insertions(+) create mode 100644 packages/pipelines/pipeline-artifacts/eslint.config.js create mode 100644 packages/pipelines/pipeline-core/eslint.config.js create mode 100644 packages/pipelines/pipeline-executor/eslint.config.js create mode 100644 packages/pipelines/pipeline-graph/eslint.config.js create mode 100644 packages/pipelines/pipeline-loader/eslint.config.js create mode 100644 packages/pipelines/pipeline-ui/eslint.config.js diff --git a/packages/pipelines/pipeline-artifacts/eslint.config.js b/packages/pipelines/pipeline-artifacts/eslint.config.js new file mode 100644 index 000000000..d9c0ca1ec --- /dev/null +++ b/packages/pipelines/pipeline-artifacts/eslint.config.js @@ -0,0 +1,7 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, +}); diff --git a/packages/pipelines/pipeline-core/eslint.config.js b/packages/pipelines/pipeline-core/eslint.config.js new file mode 100644 index 000000000..d9c0ca1ec --- /dev/null +++ b/packages/pipelines/pipeline-core/eslint.config.js @@ -0,0 +1,7 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, +}); diff --git a/packages/pipelines/pipeline-executor/eslint.config.js b/packages/pipelines/pipeline-executor/eslint.config.js new file mode 100644 index 000000000..d9c0ca1ec --- /dev/null +++ b/packages/pipelines/pipeline-executor/eslint.config.js @@ -0,0 +1,7 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, +}); diff --git a/packages/pipelines/pipeline-graph/eslint.config.js b/packages/pipelines/pipeline-graph/eslint.config.js new file mode 100644 index 000000000..d9c0ca1ec --- /dev/null +++ b/packages/pipelines/pipeline-graph/eslint.config.js @@ -0,0 +1,7 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, +}); diff --git a/packages/pipelines/pipeline-loader/eslint.config.js b/packages/pipelines/pipeline-loader/eslint.config.js new file mode 100644 index 000000000..d9c0ca1ec --- /dev/null +++ b/packages/pipelines/pipeline-loader/eslint.config.js @@ -0,0 +1,7 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, +}); diff --git a/packages/pipelines/pipeline-ui/eslint.config.js b/packages/pipelines/pipeline-ui/eslint.config.js new file mode 100644 index 000000000..d9c0ca1ec --- /dev/null +++ b/packages/pipelines/pipeline-ui/eslint.config.js @@ -0,0 +1,7 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, +}); From 1f1e5628cdf3783e0ac4574eb276b5f5faf81eaf Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 19 Jan 2026 06:47:01 +0100 Subject: [PATCH 06/63] chore: lint --- .../test/definition.test.ts | 22 +- .../pipeline-artifacts/test/schema.test.ts | 18 +- .../pipeline-core/src/dependencies.ts | 4 +- .../pipelines/pipeline-core/src/events.ts | 172 ++-- packages/pipelines/pipeline-core/src/index.ts | 198 ++-- .../pipelines/pipeline-core/src/pipeline.ts | 6 +- packages/pipelines/pipeline-core/src/route.ts | 12 +- .../pipelines/pipeline-core/src/source.ts | 8 +- .../pipelines/pipeline-core/src/transform.ts | 8 +- .../pipeline-core/test/dependencies.test.ts | 34 +- .../pipeline-core/test/filters.test.ts | 2 +- .../pipeline-core/test/pipeline.test.ts | 547 ++++++++++ .../pipeline-core/test/route.test.ts | 460 +++++++++ .../pipeline-core/test/source.test.ts | 350 +++++++ .../pipeline-core/test/transform.test.ts | 46 +- .../pipeline-executor/src/executor.ts | 10 +- .../pipelines/pipeline-executor/src/index.ts | 22 +- .../pipeline-executor/test/cache.test.ts | 484 +++++++++ .../pipeline-executor/test/executor.test.ts | 939 ++++++++++++++++++ .../pipelines/pipeline-graph/src/index.ts | 2 +- 20 files changed, 3066 insertions(+), 278 deletions(-) create mode 100644 packages/pipelines/pipeline-core/test/pipeline.test.ts create mode 100644 packages/pipelines/pipeline-core/test/route.test.ts create mode 100644 packages/pipelines/pipeline-core/test/source.test.ts create mode 100644 packages/pipelines/pipeline-executor/test/cache.test.ts create mode 100644 packages/pipelines/pipeline-executor/test/executor.test.ts diff --git a/packages/pipelines/pipeline-artifacts/test/definition.test.ts b/packages/pipelines/pipeline-artifacts/test/definition.test.ts index 3d279b638..3e38385a2 100644 --- a/packages/pipelines/pipeline-artifacts/test/definition.test.ts +++ b/packages/pipelines/pipeline-artifacts/test/definition.test.ts @@ -1,13 +1,15 @@ -import { describe, expect, it, vi } from "vitest"; +import type { + ArtifactBuildContext, + InferArtifactId, + InferArtifactsMap, + InferArtifactValue, + PipelineArtifactDefinition, +} from "../src/definition"; import type { ParseContext, ParsedRow, PipelineFilter } from "@ucdjs/pipelines-core"; +import { describe, expect, it, vi } from "vitest"; import { definePipelineArtifact, isPipelineArtifactDefinition, - type ArtifactBuildContext, - type InferArtifactId, - type InferArtifactsMap, - type InferArtifactValue, - type PipelineArtifactDefinition, } from "../src/definition"; describe("definePipelineArtifact", () => { @@ -158,7 +160,7 @@ describe("isPipelineArtifactDefinition", () => { const valid: PipelineArtifactDefinition = { id: "test", filter: (ctx) => ctx.file.ext === ".txt", - parser: async function* () { + async* parser() { yield { sourceFile: "test.txt", kind: "point", codePoint: "0041" }; }, build: async () => "result", @@ -240,7 +242,7 @@ describe("isPipelineArtifactDefinition", () => { }); describe("type inference", () => { - describe("InferArtifactId", () => { + describe("inferArtifactId", () => { it("should infer artifact id", () => { const artifact = definePipelineArtifact({ id: "my-artifact", @@ -266,7 +268,7 @@ describe("type inference", () => { }); }); - describe("InferArtifactValue", () => { + describe("inferArtifactValue", () => { it("should infer artifact value type", () => { const artifact = definePipelineArtifact({ id: "test", @@ -292,7 +294,7 @@ describe("type inference", () => { }); }); - describe("InferArtifactsMap", () => { + describe("inferArtifactsMap", () => { it("should infer map of artifact ids to values", () => { const artifacts = [ definePipelineArtifact({ diff --git a/packages/pipelines/pipeline-artifacts/test/schema.test.ts b/packages/pipelines/pipeline-artifacts/test/schema.test.ts index aa92edde1..452116f85 100644 --- a/packages/pipelines/pipeline-artifacts/test/schema.test.ts +++ b/packages/pipelines/pipeline-artifacts/test/schema.test.ts @@ -1,14 +1,16 @@ +import type { + Artifact, + ArtifactDefinition, + GlobalArtifact, + InferArtifactSchemaType, + InferEmittedArtifacts, +} from "../src/schema"; import { describe, expect, it } from "vitest"; import { z } from "zod"; import { artifact, isGlobalArtifact, isVersionArtifact, - type Artifact, - type ArtifactDefinition, - type GlobalArtifact, - type InferArtifactSchemaType, - type InferEmittedArtifacts, } from "../src/schema"; describe("artifact", () => { @@ -143,7 +145,7 @@ describe("isVersionArtifact", () => { }); describe("type inference", () => { - describe("InferArtifactSchemaType", () => { + describe("inferArtifactSchemaType", () => { it("should infer schema type correctly", () => { const schema = z.object({ id: z.string(), @@ -152,7 +154,7 @@ describe("type inference", () => { const art = artifact(schema); type Inferred = InferArtifactSchemaType; - type Expected = { id: string; count: number }; + interface Expected { id: string; count: number } const assertType: Inferred = { id: "test", count: 42 }; const _checkType: Expected = assertType; @@ -169,7 +171,7 @@ describe("type inference", () => { }); }); - describe("InferEmittedArtifacts", () => { + describe("inferEmittedArtifacts", () => { it("should infer multiple artifact types", () => { const emits = { result: artifact(z.object({ value: z.string() })), diff --git a/packages/pipelines/pipeline-core/src/dependencies.ts b/packages/pipelines/pipeline-core/src/dependencies.ts index 7f07f608e..43aff2445 100644 --- a/packages/pipelines/pipeline-core/src/dependencies.ts +++ b/packages/pipelines/pipeline-core/src/dependencies.ts @@ -16,8 +16,8 @@ export interface ParsedArtifactDependency { export type ParsedDependency = ParsedRouteDependency | ParsedArtifactDependency; -export type ParseDependencyType = - T extends `route:${infer RouteId}` +export type ParseDependencyType + = T extends `route:${infer RouteId}` ? { type: "route"; routeId: RouteId } : T extends `artifact:${infer RouteId}:${infer ArtifactName}` ? { type: "artifact"; routeId: RouteId; artifactName: ArtifactName } diff --git a/packages/pipelines/pipeline-core/src/events.ts b/packages/pipelines/pipeline-core/src/events.ts index 3bf9079d9..70efcdee0 100644 --- a/packages/pipelines/pipeline-core/src/events.ts +++ b/packages/pipelines/pipeline-core/src/events.ts @@ -1,184 +1,184 @@ import type { FileContext } from "./types"; -export type PipelineEventType = - | "pipeline:start" - | "pipeline:end" - | "version:start" - | "version:end" - | "artifact:start" - | "artifact:end" - | "artifact:produced" - | "artifact:consumed" - | "file:matched" - | "file:skipped" - | "file:fallback" - | "parse:start" - | "parse:end" - | "resolve:start" - | "resolve:end" - | "cache:hit" - | "cache:miss" - | "cache:store" - | "error"; - -export type PipelineStartEvent = { +export type PipelineEventType + = | "pipeline:start" + | "pipeline:end" + | "version:start" + | "version:end" + | "artifact:start" + | "artifact:end" + | "artifact:produced" + | "artifact:consumed" + | "file:matched" + | "file:skipped" + | "file:fallback" + | "parse:start" + | "parse:end" + | "resolve:start" + | "resolve:end" + | "cache:hit" + | "cache:miss" + | "cache:store" + | "error"; + +export interface PipelineStartEvent { type: "pipeline:start"; versions: string[]; timestamp: number; -}; +} -export type PipelineEndEvent = { +export interface PipelineEndEvent { type: "pipeline:end"; durationMs: number; timestamp: number; -}; +} -export type VersionStartEvent = { +export interface VersionStartEvent { type: "version:start"; version: string; timestamp: number; -}; +} -export type VersionEndEvent = { +export interface VersionEndEvent { type: "version:end"; version: string; durationMs: number; timestamp: number; -}; +} -export type ArtifactStartEvent = { +export interface ArtifactStartEvent { type: "artifact:start"; artifactId: string; version: string; timestamp: number; -}; +} -export type ArtifactEndEvent = { +export interface ArtifactEndEvent { type: "artifact:end"; artifactId: string; version: string; durationMs: number; timestamp: number; -}; +} -export type ArtifactProducedEvent = { +export interface ArtifactProducedEvent { type: "artifact:produced"; artifactId: string; routeId: string; version: string; timestamp: number; -}; +} -export type ArtifactConsumedEvent = { +export interface ArtifactConsumedEvent { type: "artifact:consumed"; artifactId: string; routeId: string; version: string; timestamp: number; -}; +} -export type FileMatchedEvent = { +export interface FileMatchedEvent { type: "file:matched"; file: FileContext; routeId: string; timestamp: number; -}; +} -export type FileSkippedEvent = { +export interface FileSkippedEvent { type: "file:skipped"; file: FileContext; reason: "no-match" | "filtered"; timestamp: number; -}; +} -export type FileFallbackEvent = { +export interface FileFallbackEvent { type: "file:fallback"; file: FileContext; timestamp: number; -}; +} -export type ParseStartEvent = { +export interface ParseStartEvent { type: "parse:start"; file: FileContext; routeId: string; timestamp: number; -}; +} -export type ParseEndEvent = { +export interface ParseEndEvent { type: "parse:end"; file: FileContext; routeId: string; rowCount: number; durationMs: number; timestamp: number; -}; +} -export type ResolveStartEvent = { +export interface ResolveStartEvent { type: "resolve:start"; file: FileContext; routeId: string; timestamp: number; -}; +} -export type ResolveEndEvent = { +export interface ResolveEndEvent { type: "resolve:end"; file: FileContext; routeId: string; outputCount: number; durationMs: number; timestamp: number; -}; +} -export type CacheHitEvent = { +export interface CacheHitEvent { type: "cache:hit"; routeId: string; file: FileContext; version: string; timestamp: number; -}; +} -export type CacheMissEvent = { +export interface CacheMissEvent { type: "cache:miss"; routeId: string; file: FileContext; version: string; timestamp: number; -}; +} -export type CacheStoreEvent = { +export interface CacheStoreEvent { type: "cache:store"; routeId: string; file: FileContext; version: string; timestamp: number; -}; +} -export type PipelineErrorEvent = { +export interface PipelineErrorEvent { type: "error"; error: PipelineError; timestamp: number; -}; - -export type PipelineEvent = - | PipelineStartEvent - | PipelineEndEvent - | VersionStartEvent - | VersionEndEvent - | ArtifactStartEvent - | ArtifactEndEvent - | ArtifactProducedEvent - | ArtifactConsumedEvent - | FileMatchedEvent - | FileSkippedEvent - | FileFallbackEvent - | ParseStartEvent - | ParseEndEvent - | ResolveStartEvent - | ResolveEndEvent - | CacheHitEvent - | CacheMissEvent - | CacheStoreEvent - | PipelineErrorEvent; +} + +export type PipelineEvent + = | PipelineStartEvent + | PipelineEndEvent + | VersionStartEvent + | VersionEndEvent + | ArtifactStartEvent + | ArtifactEndEvent + | ArtifactProducedEvent + | ArtifactConsumedEvent + | FileMatchedEvent + | FileSkippedEvent + | FileFallbackEvent + | ParseStartEvent + | ParseEndEvent + | ResolveStartEvent + | ResolveEndEvent + | CacheHitEvent + | CacheMissEvent + | CacheStoreEvent + | PipelineErrorEvent; export type PipelineErrorScope = "pipeline" | "version" | "file" | "route" | "artifact"; @@ -194,12 +194,12 @@ export interface PipelineError { export type PipelineGraphNodeType = "source" | "file" | "route" | "artifact" | "output"; -export type PipelineGraphNode = - | { id: string; type: "source"; version: string } - | { id: string; type: "file"; file: FileContext } - | { id: string; type: "route"; routeId: string } - | { id: string; type: "artifact"; artifactId: string } - | { id: string; type: "output"; outputIndex: number; property?: string }; +export type PipelineGraphNode + = | { id: string; type: "source"; version: string } + | { id: string; type: "file"; file: FileContext } + | { id: string; type: "route"; routeId: string } + | { id: string; type: "artifact"; artifactId: string } + | { id: string; type: "output"; outputIndex: number; property?: string }; export type PipelineGraphEdgeType = "provides" | "matched" | "parsed" | "resolved" | "uses-artifact"; diff --git a/packages/pipelines/pipeline-core/src/index.ts b/packages/pipelines/pipeline-core/src/index.ts index 5f7abf61b..5ef3e95df 100644 --- a/packages/pipelines/pipeline-core/src/index.ts +++ b/packages/pipelines/pipeline-core/src/index.ts @@ -1,141 +1,141 @@ export type { - FileContext, - RowContext, - FilterContext, - PipelineFilter, - ParsedRow, - ParseContext, - ParserFn, - ResolvedEntry, - DefaultRange, - PropertyJson, - ResolveContext, - ResolverFn, - RouteOutput, -} from "./types"; - -export { - byName, - byDir, - byExt, - byGlob, - byPath, - byProp, - bySource, - and, - or, - not, - always, - never, -} from "./filters"; - -export type { - PipelineDependency, - ParsedRouteDependency, + ExtractArtifactDependencies, + ExtractArtifactKeys, + ExtractRouteDependencies, ParsedArtifactDependency, ParsedDependency, ParseDependencyType, - ExtractRouteDependencies, - ExtractArtifactDependencies, - ExtractArtifactKeys, + ParsedRouteDependency, + PipelineDependency, } from "./dependencies"; export { - parseDependency, - isRouteDependency, - isArtifactDependency, - createRouteDependency, createArtifactDependency, + createRouteDependency, + isArtifactDependency, + isRouteDependency, + parseDependency, } from "./dependencies"; export type { - PipelineEventType, - PipelineStartEvent, - PipelineEndEvent, - VersionStartEvent, - VersionEndEvent, - ArtifactStartEvent, + ArtifactConsumedEvent, ArtifactEndEvent, ArtifactProducedEvent, - ArtifactConsumedEvent, - FileMatchedEvent, - FileSkippedEvent, - FileFallbackEvent, - ParseStartEvent, - ParseEndEvent, - ResolveStartEvent, - ResolveEndEvent, + ArtifactStartEvent, CacheHitEvent, CacheMissEvent, CacheStoreEvent, + FileFallbackEvent, + FileMatchedEvent, + FileSkippedEvent, + ParseEndEvent, + ParseStartEvent, + PipelineEndEvent, + PipelineError, PipelineErrorEvent, - PipelineEvent, PipelineErrorScope, - PipelineError, - PipelineGraphNodeType, - PipelineGraphNode, - PipelineGraphEdgeType, - PipelineGraphEdge, + PipelineEvent, + PipelineEventType, PipelineGraph, + PipelineGraphEdge, + PipelineGraphEdgeType, + PipelineGraphNode, + PipelineGraphNodeType, + PipelineStartEvent, + ResolveEndEvent, + ResolveStartEvent, + VersionEndEvent, + VersionStartEvent, } from "./events"; +export { + always, + and, + byDir, + byExt, + byGlob, + byName, + byPath, + byProp, + bySource, + never, + not, + or, +} from "./filters"; + +export type { + FallbackRouteDefinition, + InferPipelineOutput, + InferPipelineRouteIds, + InferPipelineSourceIds, + PipelineDefinition, + PipelineDefinitionOptions, +} from "./pipeline"; + +export { + definePipeline, + getPipelineRouteIds, + getPipelineSourceIds, + isPipelineDefinition, +} from "./pipeline"; + +export type { + ArtifactDefinition, + InferArtifactType, + InferEmittedArtifactsFromRoute, + InferRouteDepends, + InferRouteEmits, + InferRouteId, + InferRouteOutput, + InferRoutesOutput, + InferRouteTransforms, + PipelineRouteDefinition, + RouteResolveContext, +} from "./route"; + +export { definePipelineRoute } from "./route"; + export type { - StreamOptions, FileMetadata, - SourceBackend, - PipelineSourceDefinition, - SourceFileContext, InferSourceId, InferSourceIds, + PipelineSourceDefinition, + SourceBackend, + SourceFileContext, + StreamOptions, } from "./source"; export { definePipelineSource, - resolveSourceFiles, resolveMultipleSourceFiles, + resolveSourceFiles, } from "./source"; export type { - TransformContext, - PipelineTransformDefinition, + ChainTransforms, InferTransformInput, InferTransformOutput, - ChainTransforms, + PipelineTransformDefinition, + TransformContext, } from "./transform"; export { - definePipelineTransform, applyTransforms, + definePipelineTransform, } from "./transform"; export type { - ArtifactDefinition, - InferArtifactType, - RouteResolveContext, - PipelineRouteDefinition, - InferRouteId, - InferRouteDepends, - InferRouteEmits, - InferRouteTransforms, - InferRouteOutput, - InferRoutesOutput, - InferEmittedArtifactsFromRoute, -} from "./route"; - -export { definePipelineRoute } from "./route"; - -export type { - FallbackRouteDefinition, - PipelineDefinitionOptions, - PipelineDefinition, - InferPipelineOutput, - InferPipelineSourceIds, - InferPipelineRouteIds, -} from "./pipeline"; - -export { - definePipeline, - isPipelineDefinition, - getPipelineRouteIds, - getPipelineSourceIds, -} from "./pipeline"; + DefaultRange, + FileContext, + FilterContext, + ParseContext, + ParsedRow, + ParserFn, + PipelineFilter, + PropertyJson, + ResolveContext, + ResolvedEntry, + ResolverFn, + RouteOutput, + RowContext, +} from "./types"; diff --git a/packages/pipelines/pipeline-core/src/pipeline.ts b/packages/pipelines/pipeline-core/src/pipeline.ts index 4018d43fd..54b6877e3 100644 --- a/packages/pipelines/pipeline-core/src/pipeline.ts +++ b/packages/pipelines/pipeline-core/src/pipeline.ts @@ -1,7 +1,7 @@ import type { PipelineEvent } from "./events"; -import type { PipelineRouteDefinition, InferRoutesOutput } from "./route"; -import type { PipelineSourceDefinition, InferSourceIds } from "./source"; -import type { PipelineFilter, ParseContext, ParsedRow, ResolveContext } from "./types"; +import type { InferRoutesOutput, PipelineRouteDefinition } from "./route"; +import type { InferSourceIds, PipelineSourceDefinition } from "./source"; +import type { ParseContext, ParsedRow, PipelineFilter, ResolveContext } from "./types"; /** * Fallback route definition for files that don't match any explicit route. diff --git a/packages/pipelines/pipeline-core/src/route.ts b/packages/pipelines/pipeline-core/src/route.ts index 32b78a9bf..992356a3d 100644 --- a/packages/pipelines/pipeline-core/src/route.ts +++ b/packages/pipelines/pipeline-core/src/route.ts @@ -1,14 +1,14 @@ -import type { z } from "zod"; import type { ExtractArtifactKeys, PipelineDependency } from "./dependencies"; import type { ChainTransforms, PipelineTransformDefinition } from "./transform"; import type { FileContext, - ParserFn, ParsedRow, + ParserFn, PipelineFilter, PropertyJson, RouteOutput, } from "./types"; +import type { z } from "zod"; export interface ArtifactDefinition { _type: "artifact" | "global-artifact"; @@ -16,8 +16,8 @@ export interface ArtifactDefinition { scope: "version" | "global"; } -export type InferArtifactType = - T extends ArtifactDefinition ? z.infer : never; +export type InferArtifactType + = T extends ArtifactDefinition ? z.infer : never; export interface RouteResolveContext< TArtifactKeys extends string = string, @@ -87,8 +87,8 @@ export type InferRouteOutput = T extends PipelineRouteDefinition[]> = - T[number] extends PipelineRouteDefinition +export type InferRoutesOutput[]> + = T[number] extends PipelineRouteDefinition ? TOutput extends unknown[] ? TOutput[number] : TOutput : never; diff --git a/packages/pipelines/pipeline-core/src/source.ts b/packages/pipelines/pipeline-core/src/source.ts index a94553b11..717a38c28 100644 --- a/packages/pipelines/pipeline-core/src/source.ts +++ b/packages/pipelines/pipeline-core/src/source.ts @@ -13,10 +13,10 @@ export interface FileMetadata { } export interface SourceBackend { - listFiles(version: string): Promise; - readFile(file: FileContext): Promise; - readFileStream?(file: FileContext, options?: StreamOptions): AsyncIterable; - getMetadata?(file: FileContext): Promise; + listFiles: (version: string) => Promise; + readFile: (file: FileContext) => Promise; + readFileStream?: (file: FileContext, options?: StreamOptions) => AsyncIterable; + getMetadata?: (file: FileContext) => Promise; } export interface PipelineSourceDefinition { diff --git a/packages/pipelines/pipeline-core/src/transform.ts b/packages/pipelines/pipeline-core/src/transform.ts index 0c3725471..5fe4092e2 100644 --- a/packages/pipelines/pipeline-core/src/transform.ts +++ b/packages/pipelines/pipeline-core/src/transform.ts @@ -16,11 +16,11 @@ export function definePipelineTransform( return definition; } -export type InferTransformInput = - T extends PipelineTransformDefinition ? TInput : never; +export type InferTransformInput + = T extends PipelineTransformDefinition ? TInput : never; -export type InferTransformOutput = - T extends PipelineTransformDefinition ? TOutput : never; +export type InferTransformOutput + = T extends PipelineTransformDefinition ? TOutput : never; type ChainTwo = T1 extends PipelineTransformDefinition ? T2 extends PipelineTransformDefinition diff --git a/packages/pipelines/pipeline-core/test/dependencies.test.ts b/packages/pipelines/pipeline-core/test/dependencies.test.ts index bbc03addf..8fa95c735 100644 --- a/packages/pipelines/pipeline-core/test/dependencies.test.ts +++ b/packages/pipelines/pipeline-core/test/dependencies.test.ts @@ -1,3 +1,12 @@ +import type { + ExtractArtifactDependencies, + ExtractArtifactKeys, + ExtractRouteDependencies, + ParsedArtifactDependency, + ParsedDependency, + ParsedRouteDependency, + PipelineDependency, +} from "../src/dependencies"; import { describe, expect, it } from "vitest"; import { createArtifactDependency, @@ -5,13 +14,6 @@ import { isArtifactDependency, isRouteDependency, parseDependency, - type ExtractArtifactDependencies, - type ExtractArtifactKeys, - type ExtractRouteDependencies, - type ParsedArtifactDependency, - type ParsedDependency, - type ParsedRouteDependency, - type PipelineDependency, } from "../src/dependencies"; describe("parseDependency", () => { @@ -55,31 +57,31 @@ describe("parseDependency", () => { it("should throw error for invalid format", () => { expect(() => parseDependency("invalid" as PipelineDependency)).toThrow( - 'Invalid dependency format: invalid. Expected "route:" or "artifact::"', + "Invalid dependency format: invalid. Expected \"route:\" or \"artifact::\"", ); }); it("should throw error for route without id", () => { expect(() => parseDependency("route:" as PipelineDependency)).toThrow( - 'Invalid dependency format: route:', + "Invalid dependency format: route:", ); }); it("should throw error for artifact without name", () => { expect(() => parseDependency("artifact:my-route:" as PipelineDependency)).toThrow( - 'Invalid dependency format: artifact:my-route:', + "Invalid dependency format: artifact:my-route:", ); }); it("should throw error for artifact without route id", () => { expect(() => parseDependency("artifact::my-artifact" as PipelineDependency)).toThrow( - 'Invalid dependency format: artifact::my-artifact', + "Invalid dependency format: artifact::my-artifact", ); }); it("should throw error for unknown dependency type", () => { expect(() => parseDependency("unknown:value" as PipelineDependency)).toThrow( - 'Invalid dependency format: unknown:value', + "Invalid dependency format: unknown:value", ); }); }); @@ -173,7 +175,7 @@ describe("createArtifactDependency", () => { }); }); -describe("ParsedDependency types", () => { +describe("parsedDependency types", () => { it("should handle route dependency types", () => { const parsed: ParsedRouteDependency = { type: "route", @@ -214,7 +216,7 @@ describe("ParsedDependency types", () => { }); describe("type inference", () => { - describe("ExtractRouteDependencies", () => { + describe("extractRouteDependencies", () => { it("should extract route ids from dependency array", () => { const deps = [ "route:parser", @@ -241,7 +243,7 @@ describe("type inference", () => { }); }); - describe("ExtractArtifactDependencies", () => { + describe("extractArtifactDependencies", () => { it("should extract artifact info from dependency array", () => { const deps = [ "artifact:parser:result", @@ -259,7 +261,7 @@ describe("type inference", () => { }); }); - describe("ExtractArtifactKeys", () => { + describe("extractArtifactKeys", () => { it("should extract artifact keys from dependency array", () => { const deps = [ "artifact:parser:result", diff --git a/packages/pipelines/pipeline-core/test/filters.test.ts b/packages/pipelines/pipeline-core/test/filters.test.ts index 1d7aacde0..2f95b69f8 100644 --- a/packages/pipelines/pipeline-core/test/filters.test.ts +++ b/packages/pipelines/pipeline-core/test/filters.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest"; import type { FileContext, FilterContext } from "../src/types"; +import { describe, expect, it } from "vitest"; import { always, and, diff --git a/packages/pipelines/pipeline-core/test/pipeline.test.ts b/packages/pipelines/pipeline-core/test/pipeline.test.ts new file mode 100644 index 000000000..0b3eddd87 --- /dev/null +++ b/packages/pipelines/pipeline-core/test/pipeline.test.ts @@ -0,0 +1,547 @@ +import type { FallbackRouteDefinition, PipelineDefinition } from "../src/pipeline"; +import type { SourceBackend } from "../src/source"; +import type { FileContext, ParsedRow, PropertyJson } from "../src/types"; +import { describe, expect, it, vi } from "vitest"; +import { + definePipeline, + getPipelineRouteIds, + getPipelineSourceIds, + isPipelineDefinition, +} from "../src/pipeline"; +import { definePipelineRoute } from "../src/route"; +import { definePipelineSource } from "../src/source"; + +function createMockBackend(): SourceBackend { + return { + listFiles: vi.fn().mockResolvedValue([]), + readFile: vi.fn().mockResolvedValue(""), + }; +} + +function createMockSource(id: string) { + return definePipelineSource({ + id, + backend: createMockBackend(), + }); +} + +async function* mockParser(): AsyncIterable { + yield { sourceFile: "test.txt", kind: "point", codePoint: "0041", value: "A" }; +} + +function createMockRoute(id: string) { + return definePipelineRoute({ + id, + filter: () => true, + parser: mockParser, + resolver: async () => [], + }); +} + +describe("definePipeline", () => { + it("should define a minimal pipeline", () => { + const pipeline = definePipeline({ + id: "test-pipeline", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + expect(pipeline._type).toBe("pipeline-definition"); + expect(pipeline.id).toBe("test-pipeline"); + expect(pipeline.versions).toEqual(["16.0.0"]); + expect(pipeline.inputs).toEqual([]); + expect(pipeline.routes).toEqual([]); + }); + + it("should define a pipeline with name and description", () => { + const pipeline = definePipeline({ + id: "named-pipeline", + name: "My Pipeline", + description: "A test pipeline", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + expect(pipeline.name).toBe("My Pipeline"); + expect(pipeline.description).toBe("A test pipeline"); + }); + + it("should define a pipeline with multiple versions", () => { + const pipeline = definePipeline({ + id: "multi-version", + versions: ["16.0.0", "15.1.0", "15.0.0"], + inputs: [], + routes: [], + }); + + expect(pipeline.versions).toEqual(["16.0.0", "15.1.0", "15.0.0"]); + }); + + it("should define a pipeline with inputs", () => { + const source1 = createMockSource("source1"); + const source2 = createMockSource("source2"); + + const pipeline = definePipeline({ + id: "with-inputs", + versions: ["16.0.0"], + inputs: [source1, source2], + routes: [], + }); + + expect(pipeline.inputs).toHaveLength(2); + expect(pipeline.inputs[0].id).toBe("source1"); + expect(pipeline.inputs[1].id).toBe("source2"); + }); + + it("should define a pipeline with routes", () => { + const route1 = createMockRoute("route1"); + const route2 = createMockRoute("route2"); + + const pipeline = definePipeline({ + id: "with-routes", + versions: ["16.0.0"], + inputs: [], + routes: [route1, route2], + }); + + expect(pipeline.routes).toHaveLength(2); + expect(pipeline.routes[0].id).toBe("route1"); + expect(pipeline.routes[1].id).toBe("route2"); + }); + + it("should define a pipeline with include filter", () => { + const include = vi.fn().mockReturnValue(true); + + const pipeline = definePipeline({ + id: "filtered", + versions: ["16.0.0"], + inputs: [], + routes: [], + include, + }); + + expect(pipeline.include).toBe(include); + }); + + it("should default strict to false", () => { + const pipeline = definePipeline({ + id: "default-strict", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + expect(pipeline.strict).toBe(false); + }); + + it("should allow setting strict to true", () => { + const pipeline = definePipeline({ + id: "strict-pipeline", + versions: ["16.0.0"], + inputs: [], + routes: [], + strict: true, + }); + + expect(pipeline.strict).toBe(true); + }); + + it("should default concurrency to 4", () => { + const pipeline = definePipeline({ + id: "default-concurrency", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + expect(pipeline.concurrency).toBe(4); + }); + + it("should allow setting custom concurrency", () => { + const pipeline = definePipeline({ + id: "custom-concurrency", + versions: ["16.0.0"], + inputs: [], + routes: [], + concurrency: 8, + }); + + expect(pipeline.concurrency).toBe(8); + }); + + it("should define a pipeline with fallback", () => { + const fallback: FallbackRouteDefinition = { + parser: mockParser, + resolver: async () => [], + }; + + const pipeline = definePipeline({ + id: "with-fallback", + versions: ["16.0.0"], + inputs: [], + routes: [], + fallback, + }); + + expect(pipeline.fallback).toBe(fallback); + }); + + it("should define a pipeline with fallback filter", () => { + const fallback: FallbackRouteDefinition = { + filter: (ctx) => ctx.file.ext === ".txt", + parser: mockParser, + resolver: async () => [], + }; + + const pipeline = definePipeline({ + id: "filtered-fallback", + versions: ["16.0.0"], + inputs: [], + routes: [], + fallback, + }); + + expect(pipeline.fallback?.filter).toBeDefined(); + }); + + it("should define a pipeline with onEvent handler", () => { + const onEvent = vi.fn(); + + const pipeline = definePipeline({ + id: "with-events", + versions: ["16.0.0"], + inputs: [], + routes: [], + onEvent, + }); + + expect(pipeline.onEvent).toBe(onEvent); + }); + + it("should define a complete pipeline with all options", () => { + const source = createMockSource("source"); + const route = createMockRoute("route"); + const include = vi.fn().mockReturnValue(true); + const onEvent = vi.fn(); + const fallback: FallbackRouteDefinition = { + parser: mockParser, + resolver: async () => [], + }; + + const pipeline = definePipeline({ + id: "complete-pipeline", + name: "Complete Pipeline", + description: "A fully configured pipeline", + versions: ["16.0.0", "15.1.0"], + inputs: [source], + routes: [route], + include, + strict: true, + concurrency: 2, + fallback, + onEvent, + }); + + expect(pipeline._type).toBe("pipeline-definition"); + expect(pipeline.id).toBe("complete-pipeline"); + expect(pipeline.name).toBe("Complete Pipeline"); + expect(pipeline.description).toBe("A fully configured pipeline"); + expect(pipeline.versions).toHaveLength(2); + expect(pipeline.inputs).toHaveLength(1); + expect(pipeline.routes).toHaveLength(1); + expect(pipeline.include).toBe(include); + expect(pipeline.strict).toBe(true); + expect(pipeline.concurrency).toBe(2); + expect(pipeline.fallback).toBe(fallback); + expect(pipeline.onEvent).toBe(onEvent); + }); +}); + +describe("isPipelineDefinition", () => { + it("should return true for valid pipeline definition", () => { + const pipeline = definePipeline({ + id: "test", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + expect(isPipelineDefinition(pipeline)).toBe(true); + }); + + it("should return false for null", () => { + expect(isPipelineDefinition(null)).toBe(false); + }); + + it("should return false for undefined", () => { + expect(isPipelineDefinition(undefined)).toBe(false); + }); + + it("should return false for primitive types", () => { + expect(isPipelineDefinition("string")).toBe(false); + expect(isPipelineDefinition(123)).toBe(false); + expect(isPipelineDefinition(true)).toBe(false); + }); + + it("should return false for empty object", () => { + expect(isPipelineDefinition({})).toBe(false); + }); + + it("should return false for object without _type", () => { + const notPipeline = { + id: "test", + versions: ["16.0.0"], + inputs: [], + routes: [], + }; + + expect(isPipelineDefinition(notPipeline)).toBe(false); + }); + + it("should return false for object with wrong _type", () => { + const notPipeline = { + _type: "not-a-pipeline", + id: "test", + versions: ["16.0.0"], + inputs: [], + routes: [], + }; + + expect(isPipelineDefinition(notPipeline)).toBe(false); + }); + + it("should return false for array", () => { + expect(isPipelineDefinition([])).toBe(false); + }); + + it("should work as type guard", () => { + const unknown: unknown = definePipeline({ + id: "guarded", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + if (isPipelineDefinition(unknown)) { + expect(unknown.id).toBe("guarded"); + expect(unknown._type).toBe("pipeline-definition"); + } else { + throw new Error("Expected valid pipeline definition"); + } + }); +}); + +describe("getPipelineRouteIds", () => { + it("should return empty array for pipeline with no routes", () => { + const pipeline = definePipeline({ + id: "no-routes", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + const routeIds = getPipelineRouteIds(pipeline); + + expect(routeIds).toEqual([]); + }); + + it("should return route ids for pipeline with routes", () => { + const route1 = createMockRoute("route-a"); + const route2 = createMockRoute("route-b"); + const route3 = createMockRoute("route-c"); + + const pipeline = definePipeline({ + id: "with-routes", + versions: ["16.0.0"], + inputs: [], + routes: [route1, route2, route3], + }); + + const routeIds = getPipelineRouteIds(pipeline); + + expect(routeIds).toEqual(["route-a", "route-b", "route-c"]); + }); + + it("should preserve route order", () => { + const routes = [ + createMockRoute("third"), + createMockRoute("first"), + createMockRoute("second"), + ]; + + const pipeline = definePipeline({ + id: "ordered", + versions: ["16.0.0"], + inputs: [], + routes, + }); + + const routeIds = getPipelineRouteIds(pipeline); + + expect(routeIds).toEqual(["third", "first", "second"]); + }); +}); + +describe("getPipelineSourceIds", () => { + it("should return empty array for pipeline with no sources", () => { + const pipeline = definePipeline({ + id: "no-sources", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + const sourceIds = getPipelineSourceIds(pipeline); + + expect(sourceIds).toEqual([]); + }); + + it("should return source ids for pipeline with sources", () => { + const source1 = createMockSource("source-a"); + const source2 = createMockSource("source-b"); + + const pipeline = definePipeline({ + id: "with-sources", + versions: ["16.0.0"], + inputs: [source1, source2], + routes: [], + }); + + const sourceIds = getPipelineSourceIds(pipeline); + + expect(sourceIds).toEqual(["source-a", "source-b"]); + }); + + it("should preserve source order", () => { + const sources = [ + createMockSource("z-source"), + createMockSource("a-source"), + createMockSource("m-source"), + ]; + + const pipeline = definePipeline({ + id: "ordered", + versions: ["16.0.0"], + inputs: sources, + routes: [], + }); + + const sourceIds = getPipelineSourceIds(pipeline); + + expect(sourceIds).toEqual(["z-source", "a-source", "m-source"]); + }); +}); + +describe("pipeline definition properties", () => { + it("should have readonly _type property", () => { + const pipeline = definePipeline({ + id: "readonly-test", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + expect(pipeline._type).toBe("pipeline-definition"); + }); + + it("should include all provided inputs", () => { + const sources = [ + createMockSource("unicode"), + createMockSource("cldr"), + createMockSource("local"), + ]; + + const pipeline = definePipeline({ + id: "multi-source", + versions: ["16.0.0"], + inputs: sources, + routes: [], + }); + + expect(pipeline.inputs).toHaveLength(3); + sources.forEach((source, i) => { + expect(pipeline.inputs[i]).toBe(source); + }); + }); + + it("should include all provided routes", () => { + const routes = [ + createMockRoute("line-break"), + createMockRoute("scripts"), + createMockRoute("blocks"), + ]; + + const pipeline = definePipeline({ + id: "multi-route", + versions: ["16.0.0"], + inputs: [], + routes, + }); + + expect(pipeline.routes).toHaveLength(3); + routes.forEach((route, i) => { + expect(pipeline.routes[i]).toBe(route); + }); + }); +}); + +describe("fallback route", () => { + it("should have parser and resolver", () => { + const fallback: FallbackRouteDefinition = { + parser: mockParser, + resolver: async () => [], + }; + + const pipeline = definePipeline({ + id: "fallback-test", + versions: ["16.0.0"], + inputs: [], + routes: [], + fallback, + }); + + expect(typeof pipeline.fallback?.parser).toBe("function"); + expect(typeof pipeline.fallback?.resolver).toBe("function"); + }); + + it("should support custom output type in resolver", async () => { + interface CustomOutput { + raw: string; + } + + const fallback: FallbackRouteDefinition, CustomOutput> = { + parser: mockParser, + resolver: async () => ({ raw: "data" }), + }; + + const pipeline = definePipeline({ + id: "custom-fallback", + versions: ["16.0.0"], + inputs: [], + routes: [], + fallback, + }); + + expect(pipeline.fallback).toBeDefined(); + }); +}); + +describe("type inference", () => { + it("should preserve const types for sources and routes", () => { + const source = createMockSource("my-source"); + const route = createMockRoute("my-route"); + + const pipeline = definePipeline({ + id: "typed-pipeline" as const, + versions: ["16.0.0"], + inputs: [source] as const, + routes: [route] as const, + }); + + expect(pipeline.id).toBe("typed-pipeline"); + expect(pipeline.inputs).toHaveLength(1); + expect(pipeline.routes).toHaveLength(1); + }); +}); diff --git a/packages/pipelines/pipeline-core/test/route.test.ts b/packages/pipelines/pipeline-core/test/route.test.ts new file mode 100644 index 000000000..f8691c4ba --- /dev/null +++ b/packages/pipelines/pipeline-core/test/route.test.ts @@ -0,0 +1,460 @@ +import type { + ArtifactDefinition, + InferRouteDepends, + InferRouteEmits, + InferRouteId, + InferRouteOutput, + PipelineRouteDefinition, + RouteResolveContext, +} from "../src/route"; +import type { FileContext, ParsedRow, PropertyJson } from "../src/types"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import { + definePipelineRoute, +} from "../src/route"; +import { definePipelineTransform } from "../src/transform"; + +function createFileContext(): FileContext { + return { + version: "16.0.0", + dir: "ucd", + path: "ucd/LineBreak.txt", + name: "LineBreak.txt", + ext: ".txt", + }; +} + +async function* mockParser(): AsyncIterable { + yield { sourceFile: "test.txt", kind: "point", codePoint: "0041", value: "A" }; + yield { sourceFile: "test.txt", kind: "point", codePoint: "0042", value: "B" }; +} + +function createMockResolveContext(): RouteResolveContext { + return { + version: "16.0.0", + file: createFileContext(), + getArtifact: vi.fn(), + emitArtifact: vi.fn(), + normalizeEntries: vi.fn((entries) => entries), + now: vi.fn(() => "2024-01-01T00:00:00Z"), + }; +} + +describe("definePipelineRoute", () => { + it("should define a minimal route", () => { + const route = definePipelineRoute({ + id: "test-route", + filter: () => true, + parser: mockParser, + resolver: async () => [], + }); + + expect(route.id).toBe("test-route"); + expect(typeof route.filter).toBe("function"); + expect(typeof route.parser).toBe("function"); + expect(typeof route.resolver).toBe("function"); + }); + + it("should define a route with dependencies", () => { + const route = definePipelineRoute({ + id: "dependent-route", + filter: () => true, + depends: ["route:other-route", "artifact:source:data"] as const, + parser: mockParser, + resolver: async () => [], + }); + + expect(route.depends).toEqual(["route:other-route", "artifact:source:data"]); + }); + + it("should define a route with emits", () => { + const emits = { + result: { + _type: "artifact" as const, + schema: z.string(), + scope: "version" as const, + }, + }; + + const route = definePipelineRoute({ + id: "emitting-route", + filter: () => true, + emits, + parser: mockParser, + resolver: async () => [], + }); + + expect(route.emits).toBe(emits); + }); + + it("should define a route with transforms", () => { + const transform = definePipelineTransform({ + id: "test-transform", + async* fn(_ctx, rows) { + for await (const row of rows) { + yield row; + } + }, + }); + + const route = definePipelineRoute({ + id: "transformed-route", + filter: () => true, + parser: mockParser, + transforms: [transform] as const, + resolver: async () => [], + }); + + expect(route.transforms).toHaveLength(1); + }); + + it("should define a route with output configuration", () => { + const route = definePipelineRoute({ + id: "output-route", + filter: () => true, + parser: mockParser, + resolver: async () => [], + out: { + dir: "custom-dir", + fileName: (pj) => `${pj.property}.json`, + }, + }); + + expect(route.out?.dir).toBe("custom-dir"); + expect(typeof route.out?.fileName).toBe("function"); + }); + + it("should define a route with cache option", () => { + const route = definePipelineRoute({ + id: "cached-route", + filter: () => true, + parser: mockParser, + resolver: async () => [], + cache: true, + }); + + expect(route.cache).toBe(true); + }); + + it("should define a route with all options", () => { + const transform = definePipelineTransform({ + id: "transform", + async* fn(_ctx, rows) { + yield* rows; + }, + }); + + const emits = { + data: { + _type: "artifact" as const, + schema: z.number(), + scope: "version" as const, + }, + }; + + const route = definePipelineRoute({ + id: "full-route", + filter: (ctx) => ctx.file.ext === ".txt", + depends: ["route:dependency"] as const, + emits, + parser: mockParser, + transforms: [transform] as const, + resolver: async () => [], + out: { dir: "output" }, + cache: true, + }); + + expect(route.id).toBe("full-route"); + expect(route.depends).toHaveLength(1); + expect(route.emits).toBe(emits); + expect(route.transforms).toHaveLength(1); + expect(route.out?.dir).toBe("output"); + expect(route.cache).toBe(true); + }); +}); + +describe("route filter", () => { + it("should filter by file extension", () => { + const route = definePipelineRoute({ + id: "txt-only", + filter: (ctx) => ctx.file.ext === ".txt", + parser: mockParser, + resolver: async () => [], + }); + + expect(route.filter({ file: { ...createFileContext(), ext: ".txt" } })).toBe(true); + expect(route.filter({ file: { ...createFileContext(), ext: ".xml" } })).toBe(false); + }); + + it("should filter by file name", () => { + const route = definePipelineRoute({ + id: "specific-file", + filter: (ctx) => ctx.file.name === "LineBreak.txt", + parser: mockParser, + resolver: async () => [], + }); + + expect(route.filter({ file: createFileContext() })).toBe(true); + expect(route.filter({ file: { ...createFileContext(), name: "Other.txt" } })).toBe(false); + }); + + it("should filter by directory", () => { + const route = definePipelineRoute({ + id: "ucd-only", + filter: (ctx) => ctx.file.dir === "ucd", + parser: mockParser, + resolver: async () => [], + }); + + expect(route.filter({ file: { ...createFileContext(), dir: "ucd" } })).toBe(true); + expect(route.filter({ file: { ...createFileContext(), dir: "emoji" } })).toBe(false); + }); +}); + +describe("route resolver", () => { + it("should call resolver with context and rows", async () => { + const resolver = vi.fn().mockResolvedValue([]); + + const route = definePipelineRoute({ + id: "test", + filter: () => true, + parser: mockParser, + resolver, + }); + + const ctx = createMockResolveContext(); + const rows = mockParser(); + + await route.resolver(ctx, rows); + + expect(resolver).toHaveBeenCalledWith(ctx, rows); + }); + + it("should return PropertyJson array", async () => { + const expectedOutput: PropertyJson[] = [{ + version: "16.0.0", + property: "Test", + file: "test.txt", + entries: [], + }]; + + const route = definePipelineRoute({ + id: "test", + filter: () => true, + parser: mockParser, + resolver: async () => expectedOutput, + }); + + const result = await route.resolver(createMockResolveContext(), mockParser()); + + expect(result).toEqual(expectedOutput); + }); + + it("should support custom output types", async () => { + interface CustomOutput { + count: number; + data: string[]; + } + + const route = definePipelineRoute<"custom", readonly [], Record, readonly [], CustomOutput>({ + id: "custom", + filter: () => true, + parser: mockParser, + resolver: async (_ctx, rows) => { + const data: string[] = []; + for await (const row of rows) { + if (row.value && typeof row.value === "string") { + data.push(row.value); + } + } + return { count: data.length, data }; + }, + }); + + const result = await route.resolver(createMockResolveContext(), mockParser()); + + expect(result.count).toBe(2); + expect(result.data).toEqual(["A", "B"]); + }); +}); + +describe("route context methods", () => { + it("should provide getArtifact in resolver context", async () => { + const getArtifact = vi.fn().mockReturnValue("artifact-value"); + + const route = definePipelineRoute({ + id: "test", + filter: () => true, + depends: ["artifact:source:data"] as const, + parser: mockParser, + resolver: async (ctx) => { + const value = ctx.getArtifact("source:data"); + return [{ value }] as any; + }, + }); + + const ctx: RouteResolveContext = { + ...createMockResolveContext(), + getArtifact, + }; + + await route.resolver(ctx, mockParser()); + + expect(getArtifact).toHaveBeenCalledWith("source:data"); + }); + + it("should provide emitArtifact in resolver context", async () => { + const emitArtifact = vi.fn(); + + const route = definePipelineRoute({ + id: "test", + filter: () => true, + emits: { + result: { + _type: "artifact", + schema: z.string(), + scope: "version", + }, + }, + parser: mockParser, + resolver: async (ctx) => { + ctx.emitArtifact("result", "emitted-value"); + return []; + }, + }); + + const ctx: RouteResolveContext = { + ...createMockResolveContext(), + emitArtifact, + }; + + await route.resolver(ctx, mockParser()); + + expect(emitArtifact).toHaveBeenCalledWith("result", "emitted-value"); + }); + + it("should provide normalizeEntries in resolver context", async () => { + const normalizeEntries = vi.fn((entries) => entries.sort()); + + const route = definePipelineRoute({ + id: "test", + filter: () => true, + parser: mockParser, + resolver: async (ctx) => { + const entries = [{ codePoint: "0042", value: "B" }, { codePoint: "0041", value: "A" }]; + ctx.normalizeEntries(entries); + return []; + }, + }); + + const ctx: RouteResolveContext = { + ...createMockResolveContext(), + normalizeEntries, + }; + + await route.resolver(ctx, mockParser()); + + expect(normalizeEntries).toHaveBeenCalled(); + }); + + it("should provide now in resolver context", async () => { + const now = vi.fn().mockReturnValue("2024-06-15T12:00:00Z"); + + const route = definePipelineRoute({ + id: "test", + filter: () => true, + parser: mockParser, + resolver: async (ctx) => { + const timestamp = ctx.now(); + return [{ timestamp }] as any; + }, + }); + + const ctx: RouteResolveContext = { + ...createMockResolveContext(), + now, + }; + + const result = await route.resolver(ctx, mockParser()); + + expect(now).toHaveBeenCalled(); + expect(result[0].timestamp).toBe("2024-06-15T12:00:00Z"); + }); +}); + +describe("type inference", () => { + describe("inferRouteId", () => { + it("should infer route id", () => { + const route = definePipelineRoute({ + id: "my-route", + filter: () => true, + parser: mockParser, + resolver: async () => [], + }); + + type RouteId = InferRouteId; + const id: RouteId = "my-route"; + + expect(id).toBe("my-route"); + }); + }); + + describe("inferRouteDepends", () => { + it("should infer route dependencies", () => { + const route = definePipelineRoute({ + id: "test", + filter: () => true, + depends: ["route:dep1", "artifact:route:artifact"] as const, + parser: mockParser, + resolver: async () => [], + }); + + type Depends = InferRouteDepends; + const deps: Depends = ["route:dep1", "artifact:route:artifact"]; + + expect(deps).toHaveLength(2); + }); + }); + + describe("inferRouteEmits", () => { + it("should infer route emits", () => { + const emits = { + data: { + _type: "artifact" as const, + schema: z.string(), + scope: "version" as const, + }, + } satisfies Record; + + const route = definePipelineRoute({ + id: "test", + filter: () => true, + emits, + parser: mockParser, + resolver: async () => [], + }); + + type Emits = InferRouteEmits; + const routeEmits: Emits = route.emits!; + + expect(routeEmits.data._type).toBe("artifact"); + }); + }); + + describe("inferRouteOutput", () => { + it("should infer route output type", () => { + const route = definePipelineRoute({ + id: "test", + filter: () => true, + parser: mockParser, + resolver: async (): Promise => [], + }); + + type Output = InferRouteOutput; + const output: Output = []; + + expect(output).toEqual([]); + }); + }); +}); diff --git a/packages/pipelines/pipeline-core/test/source.test.ts b/packages/pipelines/pipeline-core/test/source.test.ts new file mode 100644 index 000000000..f226c75cf --- /dev/null +++ b/packages/pipelines/pipeline-core/test/source.test.ts @@ -0,0 +1,350 @@ +import type { + InferSourceId, + InferSourceIds, + PipelineSourceDefinition, + SourceBackend, + SourceFileContext, +} from "../src/source"; +import type { FileContext } from "../src/types"; +import { describe, expect, it, vi } from "vitest"; +import { + definePipelineSource, + resolveMultipleSourceFiles, + resolveSourceFiles, +} from "../src/source"; + +function createMockBackend(files: FileContext[]): SourceBackend { + return { + listFiles: vi.fn().mockResolvedValue(files), + readFile: vi.fn().mockResolvedValue("file content"), + }; +} + +function createFileContext(overrides: Partial = {}): FileContext { + return { + version: "16.0.0", + dir: "ucd", + path: "ucd/LineBreak.txt", + name: "LineBreak.txt", + ext: ".txt", + ...overrides, + }; +} + +describe("definePipelineSource", () => { + describe("basic functionality", () => { + it("should define a source with id and backend", () => { + const backend = createMockBackend([]); + const source = definePipelineSource({ + id: "unicode", + backend, + }); + + expect(source.id).toBe("unicode"); + expect(source.backend).toBe(backend); + }); + + it("should define a source with includes filter", () => { + const backend = createMockBackend([]); + const includes = vi.fn().mockReturnValue(true); + + const source = definePipelineSource({ + id: "filtered", + backend, + includes, + }); + + expect(source.includes).toBe(includes); + }); + + it("should define a source with excludes filter", () => { + const backend = createMockBackend([]); + const excludes = vi.fn().mockReturnValue(false); + + const source = definePipelineSource({ + id: "filtered", + backend, + excludes, + }); + + expect(source.excludes).toBe(excludes); + }); + + it("should define a source with both includes and excludes", () => { + const backend = createMockBackend([]); + const includes = vi.fn().mockReturnValue(true); + const excludes = vi.fn().mockReturnValue(false); + + const source = definePipelineSource({ + id: "dual-filtered", + backend, + includes, + excludes, + }); + + expect(source.includes).toBe(includes); + expect(source.excludes).toBe(excludes); + }); + }); +}); + +describe("resolveSourceFiles", () => { + it("should resolve all files from backend", async () => { + const files = [ + createFileContext({ path: "ucd/LineBreak.txt", name: "LineBreak.txt" }), + createFileContext({ path: "ucd/Scripts.txt", name: "Scripts.txt" }), + ]; + const backend = createMockBackend(files); + const source = definePipelineSource({ id: "test", backend }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result).toHaveLength(2); + expect(backend.listFiles).toHaveBeenCalledWith("16.0.0"); + }); + + it("should add source id to each file", async () => { + const files = [createFileContext()]; + const backend = createMockBackend(files); + const source = definePipelineSource({ id: "my-source", backend }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result[0]?.source).toEqual({ id: "my-source" }); + }); + + it("should filter files using includes", async () => { + const files = [ + createFileContext({ path: "ucd/LineBreak.txt", name: "LineBreak.txt" }), + createFileContext({ path: "ucd/Scripts.txt", name: "Scripts.txt" }), + createFileContext({ path: "emoji/data.txt", name: "data.txt", dir: "emoji" }), + ]; + const backend = createMockBackend(files); + const includes = (ctx: { file: FileContext }) => ctx.file.dir === "ucd"; + + const source = definePipelineSource({ id: "test", backend, includes }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result).toHaveLength(2); + expect(result.every((f) => f.dir === "ucd")).toBe(true); + }); + + it("should filter files using excludes", async () => { + const files = [ + createFileContext({ path: "ucd/LineBreak.txt", name: "LineBreak.txt" }), + createFileContext({ path: "ucd/ReadMe.txt", name: "ReadMe.txt" }), + createFileContext({ path: "ucd/Scripts.txt", name: "Scripts.txt" }), + ]; + const backend = createMockBackend(files); + const excludes = (ctx: { file: FileContext }) => ctx.file.name === "ReadMe.txt"; + + const source = definePipelineSource({ id: "test", backend, excludes }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result).toHaveLength(2); + expect(result.some((f) => f.name === "ReadMe.txt")).toBe(false); + }); + + it("should apply both includes and excludes", async () => { + const files = [ + createFileContext({ path: "ucd/LineBreak.txt", name: "LineBreak.txt" }), + createFileContext({ path: "ucd/ReadMe.txt", name: "ReadMe.txt" }), + createFileContext({ path: "emoji/data.txt", name: "data.txt", dir: "emoji" }), + ]; + const backend = createMockBackend(files); + const includes = (ctx: { file: FileContext }) => ctx.file.dir === "ucd"; + const excludes = (ctx: { file: FileContext }) => ctx.file.name === "ReadMe.txt"; + + const source = definePipelineSource({ id: "test", backend, includes, excludes }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result).toHaveLength(1); + expect(result[0]?.name).toBe("LineBreak.txt"); + }); + + it("should return empty array when no files match", async () => { + const files = [createFileContext({ dir: "emoji" })]; + const backend = createMockBackend(files); + const includes = (ctx: { file: FileContext }) => ctx.file.dir === "ucd"; + + const source = definePipelineSource({ id: "test", backend, includes }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result).toHaveLength(0); + }); + + it("should return empty array when backend returns empty", async () => { + const backend = createMockBackend([]); + const source = definePipelineSource({ id: "test", backend }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result).toHaveLength(0); + }); + + it("should preserve all file properties", async () => { + const file = createFileContext({ + version: "16.0.0", + dir: "ucd", + path: "ucd/LineBreak.txt", + name: "LineBreak.txt", + ext: ".txt", + }); + const backend = createMockBackend([file]); + const source = definePipelineSource({ id: "test", backend }); + + const result = await resolveSourceFiles(source, "16.0.0"); + + expect(result[0]).toBeDefined(); + expect(result[0]).toMatchObject({ + version: "16.0.0", + dir: "ucd", + path: "ucd/LineBreak.txt", + name: "LineBreak.txt", + ext: ".txt", + source: { id: "test" }, + }); + }); +}); + +describe("resolveMultipleSourceFiles", () => { + it("should resolve files from multiple sources", async () => { + const source1Files = [createFileContext({ path: "ucd/LineBreak.txt" })]; + const source2Files = [createFileContext({ path: "emoji/data.txt", dir: "emoji" })]; + + const source1 = definePipelineSource({ + id: "source1", + backend: createMockBackend(source1Files), + }); + const source2 = definePipelineSource({ + id: "source2", + backend: createMockBackend(source2Files), + }); + + const result = await resolveMultipleSourceFiles([source1, source2], "16.0.0"); + + expect(result).toHaveLength(2); + }); + + it("should deduplicate files by path (last source wins)", async () => { + const source1Files = [ + createFileContext({ path: "ucd/LineBreak.txt", name: "LineBreak.txt" }), + ]; + const source2Files = [ + createFileContext({ path: "ucd/LineBreak.txt", name: "LineBreak.txt" }), + ]; + + const source1 = definePipelineSource({ + id: "source1", + backend: createMockBackend(source1Files), + }); + const source2 = definePipelineSource({ + id: "source2", + backend: createMockBackend(source2Files), + }); + + const result = await resolveMultipleSourceFiles([source1, source2], "16.0.0"); + + expect(result).toHaveLength(1); + expect(result[0]?.source.id).toBe("source2"); + }); + + it("should return empty array for empty sources array", async () => { + const result = await resolveMultipleSourceFiles([], "16.0.0"); + + expect(result).toHaveLength(0); + }); + + it("should handle sources with no files", async () => { + const source1 = definePipelineSource({ + id: "empty1", + backend: createMockBackend([]), + }); + const source2 = definePipelineSource({ + id: "empty2", + backend: createMockBackend([]), + }); + + const result = await resolveMultipleSourceFiles([source1, source2], "16.0.0"); + + expect(result).toHaveLength(0); + }); + + it("should apply filters from each source independently", async () => { + const source1Files = [ + createFileContext({ path: "ucd/LineBreak.txt", dir: "ucd" }), + createFileContext({ path: "ucd/ReadMe.txt", dir: "ucd", name: "ReadMe.txt" }), + ]; + const source2Files = [ + createFileContext({ path: "emoji/data.txt", dir: "emoji" }), + ]; + + const source1 = definePipelineSource({ + id: "source1", + backend: createMockBackend(source1Files), + excludes: (ctx) => ctx.file.name === "ReadMe.txt", + }); + const source2 = definePipelineSource({ + id: "source2", + backend: createMockBackend(source2Files), + }); + + const result = await resolveMultipleSourceFiles([source1, source2], "16.0.0"); + + expect(result).toHaveLength(2); + expect(result.some((f) => f.name === "ReadMe.txt")).toBe(false); + }); +}); + +describe("type inference", () => { + describe("inferSourceId", () => { + it("should infer source id type", () => { + const source = definePipelineSource({ + id: "my-source", + backend: createMockBackend([]), + }); + + type SourceId = InferSourceId; + const id: SourceId = "my-source"; + + expect(id).toBe("my-source"); + }); + }); + + describe("inferSourceIds", () => { + it("should infer multiple source ids", () => { + const sources = [ + definePipelineSource({ id: "source1", backend: createMockBackend([]) }), + definePipelineSource({ id: "source2", backend: createMockBackend([]) }), + ] as const; + + type SourceIds = InferSourceIds; + const id1: SourceIds = "source1"; + const id2: SourceIds = "source2"; + + expect(id1).toBe("source1"); + expect(id2).toBe("source2"); + }); + }); +}); + +describe("sourceFileContext", () => { + it("should extend FileContext with source info", async () => { + const files = [createFileContext()]; + const backend = createMockBackend(files); + const source = definePipelineSource({ id: "test-source", backend }); + + const result = await resolveSourceFiles(source, "16.0.0"); + expect(result[0]).toBeDefined(); + const file = result[0]!; + + expect(file.source).toBeDefined(); + expect(file.source.id).toBe("test-source"); + expect(file.version).toBe("16.0.0"); + expect(file.dir).toBe("ucd"); + }); +}); diff --git a/packages/pipelines/pipeline-core/test/transform.test.ts b/packages/pipelines/pipeline-core/test/transform.test.ts index 23685187e..0fe5eebe9 100644 --- a/packages/pipelines/pipeline-core/test/transform.test.ts +++ b/packages/pipelines/pipeline-core/test/transform.test.ts @@ -1,13 +1,15 @@ +import type { + InferTransformInput, + InferTransformOutput, + PipelineTransformDefinition, + TransformContext, +} from "../src/transform"; +import type { FileContext } from "../src/types"; import { describe, expect, it } from "vitest"; import { applyTransforms, definePipelineTransform, - type InferTransformInput, - type InferTransformOutput, - type PipelineTransformDefinition, - type TransformContext, } from "../src/transform"; -import type { FileContext } from "../src/types"; function createTransformContext(): TransformContext { const file: FileContext = { @@ -42,7 +44,7 @@ describe("definePipelineTransform", () => { it("should define a simple transform", () => { const transform = definePipelineTransform({ id: "uppercase", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield (row as string).toUpperCase(); } @@ -56,7 +58,7 @@ describe("definePipelineTransform", () => { it("should define a transform with type parameters", () => { const transform = definePipelineTransform({ id: "string-length", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row.length; } @@ -69,7 +71,7 @@ describe("definePipelineTransform", () => { it("should preserve transform function", async () => { const transform = definePipelineTransform({ id: "double", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row * 2; } @@ -88,7 +90,7 @@ describe("applyTransforms", () => { it("should apply single transform", async () => { const uppercase = definePipelineTransform({ id: "uppercase", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row.toUpperCase(); } @@ -105,7 +107,7 @@ describe("applyTransforms", () => { it("should chain multiple transforms", async () => { const uppercase = definePipelineTransform({ id: "uppercase", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row.toUpperCase(); } @@ -114,7 +116,7 @@ describe("applyTransforms", () => { const exclaim = definePipelineTransform({ id: "exclaim", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield `${row}!`; } @@ -131,7 +133,7 @@ describe("applyTransforms", () => { it("should handle type transformations", async () => { const toLength = definePipelineTransform({ id: "to-length", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row.length; } @@ -140,7 +142,7 @@ describe("applyTransforms", () => { const double = definePipelineTransform({ id: "double", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row * 2; } @@ -158,7 +160,7 @@ describe("applyTransforms", () => { const append = (suffix: string) => definePipelineTransform({ id: `append-${suffix}`, - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield `${row}${suffix}`; } @@ -185,7 +187,7 @@ describe("applyTransforms", () => { it("should handle empty input", async () => { const uppercase = definePipelineTransform({ id: "uppercase", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row.toUpperCase(); } @@ -205,7 +207,7 @@ describe("applyTransforms", () => { const captureContext = definePipelineTransform({ id: "capture", - fn: async function* (ctx, rows) { + async* fn(ctx, rows) { capturedVersion = ctx.version; capturedFileName = ctx.file.name; for await (const row of rows) { @@ -234,7 +236,7 @@ describe("applyTransforms", () => { const addId = definePipelineTransform({ id: "add-id", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { let counter = 0; for await (const row of rows) { yield { ...row, id: `person-${counter++}` }; @@ -258,7 +260,7 @@ describe("applyTransforms", () => { it("should handle filter transformations", async () => { const filterEven = definePipelineTransform({ id: "filter-even", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { if (row % 2 === 0) { yield row; @@ -277,7 +279,7 @@ describe("applyTransforms", () => { it("should handle aggregation transformations", async () => { const toArray = definePipelineTransform({ id: "to-array", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { const arr: number[] = []; for await (const row of rows) { arr.push(row); @@ -298,7 +300,7 @@ describe("type inference", () => { it("should preserve transform types", async () => { const stringToNumber = definePipelineTransform({ id: "length", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row.length; } @@ -313,11 +315,11 @@ describe("type inference", () => { }); }); -describe("PipelineTransformDefinition", () => { +describe("pipelineTransformDefinition", () => { it("should create valid transform definition", () => { const def: PipelineTransformDefinition = { id: "test", - fn: async function* (_ctx, rows) { + async* fn(_ctx, rows) { for await (const row of rows) { yield row.length; } diff --git a/packages/pipelines/pipeline-executor/src/executor.ts b/packages/pipelines/pipeline-executor/src/executor.ts index 255a3cadd..cc661f039 100644 --- a/packages/pipelines/pipeline-executor/src/executor.ts +++ b/packages/pipelines/pipeline-executor/src/executor.ts @@ -1,3 +1,6 @@ +import type { CacheEntry, CacheKey, CacheStore } from "./cache"; +import type { MultiplePipelineRunResult, PipelineRunResult, PipelineSummary } from "./results"; +import type { ArtifactDefinition, PipelineArtifactDefinition } from "@ucdjs/pipelines-artifacts"; import type { FileContext, ParseContext, @@ -12,13 +15,10 @@ import type { SourceBackend, SourceFileContext, } from "@ucdjs/pipelines-core"; -import { applyTransforms, resolveMultipleSourceFiles } from "@ucdjs/pipelines-core"; -import type { ArtifactDefinition, PipelineArtifactDefinition } from "@ucdjs/pipelines-artifacts"; import { isGlobalArtifact } from "@ucdjs/pipelines-artifacts"; +import { applyTransforms, resolveMultipleSourceFiles } from "@ucdjs/pipelines-core"; import { buildDAG, getExecutionLayers } from "@ucdjs/pipelines-graph"; -import type { CacheEntry, CacheKey, CacheStore } from "./cache"; import { defaultHashFn, hashArtifact } from "./cache"; -import type { MultiplePipelineRunResult, PipelineRunResult, PipelineSummary } from "./results"; interface SourceAdapter { listFiles: (version: string) => Promise; @@ -594,7 +594,7 @@ function createParseContext(file: FileContext, source: SourceAdapter): ParseCont } return cachedContent!; }, - readLines: async function* () { + async* readLines() { const content = await source.readFile(file); const lines = content.split(/\r?\n/); for (const line of lines) { diff --git a/packages/pipelines/pipeline-executor/src/index.ts b/packages/pipelines/pipeline-executor/src/index.ts index 407897fbb..df4193bd5 100644 --- a/packages/pipelines/pipeline-executor/src/index.ts +++ b/packages/pipelines/pipeline-executor/src/index.ts @@ -1,28 +1,28 @@ export type { - CacheKey, CacheEntry, - CacheStore, - CacheStats, + CacheKey, CacheOptions, + CacheStats, + CacheStore, } from "./cache"; export { - serializeCacheKey, createMemoryCacheStore, defaultHashFn, hashArtifact, + serializeCacheKey, } from "./cache"; export type { - PipelineSummary, - PipelineRunResult, - MultiplePipelineRunResult, -} from "./results"; - -export type { + PipelineExecutor, PipelineExecutorOptions, PipelineExecutorRunOptions, - PipelineExecutor, } from "./executor"; export { createPipelineExecutor } from "./executor"; + +export type { + MultiplePipelineRunResult, + PipelineRunResult, + PipelineSummary, +} from "./results"; diff --git a/packages/pipelines/pipeline-executor/test/cache.test.ts b/packages/pipelines/pipeline-executor/test/cache.test.ts new file mode 100644 index 000000000..a8fca00bb --- /dev/null +++ b/packages/pipelines/pipeline-executor/test/cache.test.ts @@ -0,0 +1,484 @@ +import type { CacheEntry, CacheKey, CacheStore } from "../src/cache"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + + createMemoryCacheStore, + defaultHashFn, + hashArtifact, + serializeCacheKey, +} from "../src/cache"; + +describe("serializeCacheKey", () => { + it("should serialize a simple cache key", () => { + const key: CacheKey = { + routeId: "my-route", + version: "16.0.0", + inputHash: "abc123", + artifactHashes: {}, + }; + + const serialized = serializeCacheKey(key); + + expect(serialized).toBe("my-route|16.0.0|abc123|"); + }); + + it("should serialize cache key with artifact hashes", () => { + const key: CacheKey = { + routeId: "my-route", + version: "16.0.0", + inputHash: "abc123", + artifactHashes: { + "route1:artifact1": "hash1", + "route2:artifact2": "hash2", + }, + }; + + const serialized = serializeCacheKey(key); + + expect(serialized).toContain("my-route|16.0.0|abc123|"); + expect(serialized).toContain("route1:artifact1:hash1"); + expect(serialized).toContain("route2:artifact2:hash2"); + }); + + it("should sort artifact hashes alphabetically", () => { + const key: CacheKey = { + routeId: "route", + version: "16.0.0", + inputHash: "hash", + artifactHashes: { + zebra: "z", + alpha: "a", + beta: "b", + }, + }; + + const serialized = serializeCacheKey(key); + + expect(serialized).toBe("route|16.0.0|hash|alpha:a,beta:b,zebra:z"); + }); + + it("should produce same result for same key", () => { + const key: CacheKey = { + routeId: "route", + version: "16.0.0", + inputHash: "hash", + artifactHashes: { a: "1", b: "2" }, + }; + + expect(serializeCacheKey(key)).toBe(serializeCacheKey(key)); + }); + + it("should produce different results for different keys", () => { + const key1: CacheKey = { + routeId: "route1", + version: "16.0.0", + inputHash: "hash", + artifactHashes: {}, + }; + const key2: CacheKey = { + routeId: "route2", + version: "16.0.0", + inputHash: "hash", + artifactHashes: {}, + }; + + expect(serializeCacheKey(key1)).not.toBe(serializeCacheKey(key2)); + }); +}); + +describe("defaultHashFn", () => { + it("should hash a string", () => { + const hash = defaultHashFn("hello world"); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should produce same hash for same input", () => { + const input = "test string"; + + expect(defaultHashFn(input)).toBe(defaultHashFn(input)); + }); + + it("should produce different hashes for different inputs", () => { + expect(defaultHashFn("hello")).not.toBe(defaultHashFn("world")); + }); + + it("should handle empty string", () => { + const hash = defaultHashFn(""); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should handle long strings", () => { + const longString = "a".repeat(10000); + const hash = defaultHashFn(longString); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should handle unicode strings", () => { + const hash = defaultHashFn("こんにちは世界"); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); +}); + +describe("hashArtifact", () => { + it("should hash null as 'null'", () => { + expect(hashArtifact(null)).toBe("null"); + }); + + it("should hash undefined as 'null'", () => { + expect(hashArtifact(undefined)).toBe("null"); + }); + + it("should hash strings using defaultHashFn", () => { + const value = "test string"; + expect(hashArtifact(value)).toBe(defaultHashFn(value)); + }); + + it("should hash numbers", () => { + const hash = hashArtifact(42); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should hash booleans", () => { + const trueHash = hashArtifact(true); + const falseHash = hashArtifact(false); + + expect(trueHash).not.toBe(falseHash); + }); + + it("should hash arrays", () => { + const hash = hashArtifact([1, 2, 3]); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should hash objects", () => { + const hash = hashArtifact({ a: 1, b: 2 }); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should hash Maps", () => { + const map = new Map([ + ["key1", "value1"], + ["key2", "value2"], + ]); + const hash = hashArtifact(map); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should hash Sets", () => { + const set = new Set(["a", "b", "c"]); + const hash = hashArtifact(set); + + expect(typeof hash).toBe("string"); + expect(hash.length).toBe(8); + }); + + it("should produce same hash for equivalent objects", () => { + const obj1 = { a: 1, b: 2 }; + const obj2 = { a: 1, b: 2 }; + + expect(hashArtifact(obj1)).toBe(hashArtifact(obj2)); + }); + + it("should produce different hashes for different objects", () => { + const obj1 = { a: 1 }; + const obj2 = { a: 2 }; + + expect(hashArtifact(obj1)).not.toBe(hashArtifact(obj2)); + }); + + it("should sort Map entries for consistent hashing", () => { + const map1 = new Map([ + ["b", "2"], + ["a", "1"], + ]); + const map2 = new Map([ + ["a", "1"], + ["b", "2"], + ]); + + expect(hashArtifact(map1)).toBe(hashArtifact(map2)); + }); + + it("should sort Set entries for consistent hashing", () => { + const set1 = new Set(["c", "a", "b"]); + const set2 = new Set(["a", "b", "c"]); + + expect(hashArtifact(set1)).toBe(hashArtifact(set2)); + }); +}); + +describe("createMemoryCacheStore", () => { + let store: CacheStore; + + beforeEach(() => { + store = createMemoryCacheStore(); + }); + + function createCacheKey(overrides: Partial = {}): CacheKey { + return { + routeId: "test-route", + version: "16.0.0", + inputHash: "testhash", + artifactHashes: {}, + ...overrides, + }; + } + + function createCacheEntry(key: CacheKey, output: unknown[] = []): CacheEntry { + return { + key, + output, + producedArtifacts: {}, + createdAt: new Date().toISOString(), + }; + } + + describe("get", () => { + it("should return undefined for non-existent key", async () => { + const key = createCacheKey(); + const result = await store.get(key); + + expect(result).toBeUndefined(); + }); + + it("should return entry for existing key", async () => { + const key = createCacheKey(); + const entry = createCacheEntry(key, ["output"]); + + await store.set(entry); + const result = await store.get(key); + + expect(result).toEqual(entry); + }); + }); + + describe("set", () => { + it("should store an entry", async () => { + const key = createCacheKey(); + const entry = createCacheEntry(key, ["data"]); + + await store.set(entry); + const result = await store.get(key); + + expect(result).toEqual(entry); + }); + + it("should overwrite existing entry", async () => { + const key = createCacheKey(); + const entry1 = createCacheEntry(key, ["first"]); + const entry2 = createCacheEntry(key, ["second"]); + + await store.set(entry1); + await store.set(entry2); + const result = await store.get(key); + + expect(result?.output).toEqual(["second"]); + }); + + it("should store entries with different keys separately", async () => { + const key1 = createCacheKey({ routeId: "route1" }); + const key2 = createCacheKey({ routeId: "route2" }); + const entry1 = createCacheEntry(key1, ["data1"]); + const entry2 = createCacheEntry(key2, ["data2"]); + + await store.set(entry1); + await store.set(entry2); + + expect((await store.get(key1))?.output).toEqual(["data1"]); + expect((await store.get(key2))?.output).toEqual(["data2"]); + }); + }); + + describe("has", () => { + it("should return false for non-existent key", async () => { + const key = createCacheKey(); + + expect(await store.has(key)).toBe(false); + }); + + it("should return true for existing key", async () => { + const key = createCacheKey(); + const entry = createCacheEntry(key); + + await store.set(entry); + + expect(await store.has(key)).toBe(true); + }); + }); + + describe("delete", () => { + it("should return false for non-existent key", async () => { + const key = createCacheKey(); + + expect(await store.delete(key)).toBe(false); + }); + + it("should return true and remove existing key", async () => { + const key = createCacheKey(); + const entry = createCacheEntry(key); + + await store.set(entry); + const deleted = await store.delete(key); + + expect(deleted).toBe(true); + expect(await store.has(key)).toBe(false); + }); + }); + + describe("clear", () => { + it("should remove all entries", async () => { + const key1 = createCacheKey({ routeId: "route1" }); + const key2 = createCacheKey({ routeId: "route2" }); + + await store.set(createCacheEntry(key1)); + await store.set(createCacheEntry(key2)); + await store.clear(); + + expect(await store.has(key1)).toBe(false); + expect(await store.has(key2)).toBe(false); + }); + + it("should reset stats", async () => { + const key = createCacheKey(); + await store.set(createCacheEntry(key)); + await store.get(key); + await store.get(createCacheKey({ routeId: "nonexistent" })); + + await store.clear(); + + const stats = await store.stats?.(); + expect(stats?.entries).toBe(0); + expect(stats?.hits).toBe(0); + expect(stats?.misses).toBe(0); + }); + }); + + describe("stats", () => { + it("should track entry count", async () => { + const key1 = createCacheKey({ routeId: "route1" }); + const key2 = createCacheKey({ routeId: "route2" }); + + await store.set(createCacheEntry(key1)); + await store.set(createCacheEntry(key2)); + + const stats = await store.stats?.(); + expect(stats?.entries).toBe(2); + }); + + it("should track cache hits", async () => { + const key = createCacheKey(); + await store.set(createCacheEntry(key)); + + await store.get(key); + await store.get(key); + + const stats = await store.stats?.(); + expect(stats?.hits).toBe(2); + }); + + it("should track cache misses", async () => { + await store.get(createCacheKey({ routeId: "missing1" })); + await store.get(createCacheKey({ routeId: "missing2" })); + + const stats = await store.stats?.(); + expect(stats?.misses).toBe(2); + }); + + it("should track hits and misses together", async () => { + const existingKey = createCacheKey({ routeId: "existing" }); + await store.set(createCacheEntry(existingKey)); + + await store.get(existingKey); + await store.get(createCacheKey({ routeId: "missing" })); + await store.get(existingKey); + + const stats = await store.stats?.(); + expect(stats?.hits).toBe(2); + expect(stats?.misses).toBe(1); + }); + }); +}); + +describe("cache key matching", () => { + it("should match keys with same artifact hashes", async () => { + const store = createMemoryCacheStore(); + + const key: CacheKey = { + routeId: "route", + version: "16.0.0", + inputHash: "input", + artifactHashes: { + dep1: "hash1", + dep2: "hash2", + }, + }; + + await store.set({ + key, + output: ["result"], + producedArtifacts: {}, + createdAt: new Date().toISOString(), + }); + + const sameKey: CacheKey = { + routeId: "route", + version: "16.0.0", + inputHash: "input", + artifactHashes: { + dep1: "hash1", + dep2: "hash2", + }, + }; + + const result = await store.get(sameKey); + expect(result?.output).toEqual(["result"]); + }); + + it("should not match keys with different artifact hashes", async () => { + const store = createMemoryCacheStore(); + + const key: CacheKey = { + routeId: "route", + version: "16.0.0", + inputHash: "input", + artifactHashes: { + dep1: "hash1", + }, + }; + + await store.set({ + key, + output: ["result"], + producedArtifacts: {}, + createdAt: new Date().toISOString(), + }); + + const differentKey: CacheKey = { + routeId: "route", + version: "16.0.0", + inputHash: "input", + artifactHashes: { + dep1: "different-hash", + }, + }; + + const result = await store.get(differentKey); + expect(result).toBeUndefined(); + }); +}); diff --git a/packages/pipelines/pipeline-executor/test/executor.test.ts b/packages/pipelines/pipeline-executor/test/executor.test.ts new file mode 100644 index 000000000..d3f0eff78 --- /dev/null +++ b/packages/pipelines/pipeline-executor/test/executor.test.ts @@ -0,0 +1,939 @@ +import type { CacheStore } from "../src/cache"; +import type { PipelineExecutor } from "../src/executor"; +import type { + FileContext, + ParseContext, + ParsedRow, + PipelineEvent, + PipelineRouteDefinition, + PipelineSourceDefinition, + SourceBackend, +} from "@ucdjs/pipelines-core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createMemoryCacheStore } from "../src/cache"; +import { createPipelineExecutor } from "../src/executor"; +import { definePipeline, definePipelineRoute, definePipelineSource } from "@ucdjs/pipelines-core"; + +function createMockFile(name: string, dir: string = "ucd"): FileContext { + return { + version: "16.0.0", + dir, + path: `${dir}/${name}`, + name, + ext: name.includes(".") ? `.${name.split(".").pop()}` : "", + }; +} + +function createMockBackend(files: FileContext[], contents: Record = {}): SourceBackend { + return { + listFiles: vi.fn().mockResolvedValue(files), + readFile: vi.fn().mockImplementation((file: FileContext) => { + return Promise.resolve(contents[file.path] ?? ""); + }), + }; +} + +async function* mockParser(ctx: ParseContext): AsyncIterable { + const content = await ctx.readContent(); + const lines = content.split("\n").filter((line) => !ctx.isComment(line)); + + for (const line of lines) { + const [codePoint, value] = line.split(";").map((s) => s.trim()); + if (codePoint && value) { + yield { + sourceFile: ctx.file.path, + kind: "point", + codePoint, + value, + }; + } + } +} + +function createTestRoute( + id: string, + filter: (ctx: { file: FileContext }) => boolean, +): PipelineRouteDefinition { + return definePipelineRoute({ + id, + filter, + parser: mockParser, + resolver: async (ctx, rows) => { + const entries: Array<{ codePoint: string; value: string }> = []; + for await (const row of rows) { + entries.push({ + codePoint: row.codePoint!, + value: row.value as string, + }); + } + return { + version: ctx.version, + file: ctx.file.name, + entries, + }; + }, + }); +} + +function createTestSource(files: FileContext[], contents: Record = {}): PipelineSourceDefinition { + return definePipelineSource({ + id: "test-source", + backend: createMockBackend(files, contents), + }); +} + +describe("createPipelineExecutor", () => { + it("should create an executor with run and runSingle methods", () => { + const executor = createPipelineExecutor({ + pipelines: [], + }); + + expect(executor).toHaveProperty("run"); + expect(executor).toHaveProperty("runSingle"); + expect(typeof executor.run).toBe("function"); + expect(typeof executor.runSingle).toBe("function"); + }); + + it("should accept pipelines and optional artifacts", () => { + const pipeline = definePipeline({ + id: "test", + versions: ["16.0.0"], + inputs: [createTestSource([])], + routes: [], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + artifacts: [], + }); + + expect(executor).toBeDefined(); + }); + + it("should accept optional cache store", () => { + const cacheStore = createMemoryCacheStore(); + + const executor = createPipelineExecutor({ + pipelines: [], + cacheStore, + }); + + expect(executor).toBeDefined(); + }); + + it("should accept optional event handler", () => { + const onEvent = vi.fn(); + + const executor = createPipelineExecutor({ + pipelines: [], + onEvent, + }); + + expect(executor).toBeDefined(); + }); +}); + +describe("executor.run", () => { + let executor: PipelineExecutor; + let files: FileContext[]; + let contents: Record; + + beforeEach(() => { + files = [ + createMockFile("LineBreak.txt"), + createMockFile("Scripts.txt"), + ]; + + contents = { + "ucd/LineBreak.txt": "0041;AL\n0042;AL", + "ucd/Scripts.txt": "0041;Latin\n0042;Latin", + }; + + const source = createTestSource(files, contents); + const routes = [ + createTestRoute("line-break", (ctx) => ctx.file.name === "LineBreak.txt"), + createTestRoute("scripts", (ctx) => ctx.file.name === "Scripts.txt"), + ]; + + const pipeline = definePipeline({ + id: "test-pipeline", + versions: ["16.0.0"], + inputs: [source], + routes, + }); + + executor = createPipelineExecutor({ + pipelines: [pipeline], + }); + }); + + it("should run all pipelines and return results", async () => { + const result = await executor.run(); + + expect(result.results).toBeInstanceOf(Map); + expect(result.results.size).toBe(1); + expect(result.results.has("test-pipeline")).toBe(true); + }); + + it("should return summary with pipeline counts", async () => { + const result = await executor.run(); + + expect(result.summary).toEqual({ + totalPipelines: 1, + successfulPipelines: 1, + failedPipelines: 0, + durationMs: expect.any(Number), + }); + }); + + it("should process files matching routes", async () => { + const result = await executor.run(); + const pipelineResult = result.results.get("test-pipeline")!; + + expect(pipelineResult.data.length).toBe(2); + }); + + it("should filter pipelines by id when specified", async () => { + const result = await executor.run({ pipelines: ["test-pipeline"] }); + + expect(result.results.has("test-pipeline")).toBe(true); + }); + + it("should skip pipelines not in filter list", async () => { + const result = await executor.run({ pipelines: ["non-existent"] }); + + expect(result.results.size).toBe(0); + }); + + it("should filter versions when specified", async () => { + const result = await executor.run({ versions: ["16.0.0"] }); + const pipelineResult = result.results.get("test-pipeline")!; + + expect(pipelineResult.summary.versions).toEqual(["16.0.0"]); + }); +}); + +describe("executor.runSingle", () => { + let executor: PipelineExecutor; + let files: FileContext[]; + let contents: Record; + + beforeEach(() => { + files = [createMockFile("LineBreak.txt")]; + contents = { "ucd/LineBreak.txt": "0041;AL" }; + + const source = createTestSource(files, contents); + const route = createTestRoute("line-break", (ctx) => ctx.file.name === "LineBreak.txt"); + + const pipeline = definePipeline({ + id: "test-pipeline", + versions: ["16.0.0"], + inputs: [source], + routes: [route], + }); + + executor = createPipelineExecutor({ + pipelines: [pipeline], + }); + }); + + it("should run a single pipeline by id", async () => { + const result = await executor.runSingle("test-pipeline"); + + expect(result).toBeDefined(); + expect(result.data).toBeDefined(); + expect(result.graph).toBeDefined(); + expect(result.errors).toBeDefined(); + expect(result.summary).toBeDefined(); + }); + + it("should throw error for unknown pipeline id", async () => { + await expect(executor.runSingle("unknown")).rejects.toThrow( + 'Pipeline "unknown" not found', + ); + }); + + it("should return pipeline run result", async () => { + const result = await executor.runSingle("test-pipeline"); + + expect(result.data.length).toBeGreaterThan(0); + expect(result.errors).toEqual([]); + }); + + it("should accept version filter", async () => { + const result = await executor.runSingle("test-pipeline", { + versions: ["16.0.0"], + }); + + expect(result.summary.versions).toEqual(["16.0.0"]); + }); + + it("should respect cache option", async () => { + const cacheStore = createMemoryCacheStore(); + + const executor = createPipelineExecutor({ + pipelines: [ + definePipeline({ + id: "cached-pipeline", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("line-break", (ctx) => ctx.file.name === "LineBreak.txt")], + }), + ], + cacheStore, + }); + + await executor.runSingle("cached-pipeline", { cache: true }); + const stats = await cacheStore.stats?.(); + + expect(stats?.entries).toBeGreaterThanOrEqual(0); + }); +}); + +describe("pipeline events", () => { + it("should emit pipeline:start and pipeline:end events", async () => { + const events: PipelineEvent[] = []; + const onEvent = vi.fn((event: PipelineEvent) => { + events.push(event); + }); + + const pipeline = definePipeline({ + id: "event-test", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + onEvent, + }); + + await executor.runSingle("event-test"); + + const eventTypes = events.map((e) => e.type); + expect(eventTypes).toContain("pipeline:start"); + expect(eventTypes).toContain("pipeline:end"); + }); + + it("should emit version:start and version:end events", async () => { + const events: PipelineEvent[] = []; + + const pipeline = definePipeline({ + id: "version-events", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("version-events"); + + const eventTypes = events.map((e) => e.type); + expect(eventTypes).toContain("version:start"); + expect(eventTypes).toContain("version:end"); + }); + + it("should emit parse and resolve events", async () => { + const events: PipelineEvent[] = []; + + const pipeline = definePipeline({ + id: "parse-events", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("parse-events"); + + const eventTypes = events.map((e) => e.type); + expect(eventTypes).toContain("parse:start"); + expect(eventTypes).toContain("parse:end"); + expect(eventTypes).toContain("resolve:start"); + expect(eventTypes).toContain("resolve:end"); + }); + + it("should emit file:matched event when file matches route", async () => { + const events: PipelineEvent[] = []; + + const pipeline = definePipeline({ + id: "file-matched", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("file-matched"); + + const matchedEvents = events.filter((e) => e.type === "file:matched"); + expect(matchedEvents.length).toBeGreaterThan(0); + }); +}); + +describe("pipeline graph", () => { + it("should build graph with source nodes", async () => { + const pipeline = definePipeline({ + id: "graph-test", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("graph-test"); + + const sourceNodes = result.graph.nodes.filter((n) => n.type === "source"); + expect(sourceNodes.length).toBe(1); + }); + + it("should build graph with file nodes", async () => { + const pipeline = definePipeline({ + id: "graph-files", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("graph-files"); + + const fileNodes = result.graph.nodes.filter((n) => n.type === "file"); + expect(fileNodes.length).toBe(1); + }); + + it("should build graph with route nodes", async () => { + const pipeline = definePipeline({ + id: "graph-routes", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("graph-routes"); + + const routeNodes = result.graph.nodes.filter((n) => n.type === "route"); + expect(routeNodes.length).toBe(1); + }); + + it("should build graph with output nodes", async () => { + const pipeline = definePipeline({ + id: "graph-outputs", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("graph-outputs"); + + const outputNodes = result.graph.nodes.filter((n) => n.type === "output"); + expect(outputNodes.length).toBeGreaterThan(0); + }); + + it("should build graph with edges", async () => { + const pipeline = definePipeline({ + id: "graph-edges", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("graph-edges"); + + expect(result.graph.edges.length).toBeGreaterThan(0); + }); +}); + +describe("pipeline summary", () => { + it("should track total files", async () => { + const files = [ + createMockFile("File1.txt"), + createMockFile("File2.txt"), + createMockFile("File3.txt"), + ]; + const contents = { + "ucd/File1.txt": "0041;A", + "ucd/File2.txt": "0042;B", + "ucd/File3.txt": "0043;C", + }; + + const pipeline = definePipeline({ + id: "summary-total", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("all", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("summary-total"); + + expect(result.summary.totalFiles).toBe(3); + }); + + it("should track matched files", async () => { + const files = [ + createMockFile("Match1.txt"), + createMockFile("Match2.txt"), + createMockFile("NoMatch.txt"), + ]; + const contents = { + "ucd/Match1.txt": "0041;A", + "ucd/Match2.txt": "0042;B", + "ucd/NoMatch.txt": "0043;C", + }; + + const pipeline = definePipeline({ + id: "summary-matched", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("match", (ctx) => ctx.file.name.startsWith("Match"))], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("summary-matched"); + + expect(result.summary.matchedFiles).toBe(2); + }); + + it("should track skipped files", async () => { + const files = [ + createMockFile("Process.txt"), + createMockFile("Skip.txt"), + ]; + const contents = { + "ucd/Process.txt": "0041;A", + "ucd/Skip.txt": "0042;B", + }; + + const pipeline = definePipeline({ + id: "summary-skipped", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("process", (ctx) => ctx.file.name === "Process.txt")], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("summary-skipped"); + + expect(result.summary.skippedFiles).toBe(1); + }); + + it("should track duration", async () => { + const pipeline = definePipeline({ + id: "summary-duration", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("summary-duration"); + + expect(result.summary.durationMs).toBeGreaterThanOrEqual(0); + }); + + it("should track total outputs", async () => { + const files = [ + createMockFile("File1.txt"), + createMockFile("File2.txt"), + ]; + const contents = { + "ucd/File1.txt": "0041;A", + "ucd/File2.txt": "0042;B", + }; + + const pipeline = definePipeline({ + id: "summary-outputs", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("all", () => true)], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("summary-outputs"); + + expect(result.summary.totalOutputs).toBe(2); + }); +}); + +describe("error handling", () => { + it("should capture route errors without stopping execution", async () => { + const failingRoute = definePipelineRoute({ + id: "failing", + filter: () => true, + parser: mockParser, + resolver: async () => { + throw new Error("Route failed"); + }, + }); + + const pipeline = definePipeline({ + id: "error-test", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [failingRoute], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("error-test"); + + expect(result.errors.length).toBeGreaterThan(0); + expect(result.errors[0].scope).toBe("route"); + expect(result.errors[0].message).toContain("Route failed"); + }); + + it("should emit error events", async () => { + const events: PipelineEvent[] = []; + + const failingRoute = definePipelineRoute({ + id: "failing", + filter: () => true, + parser: mockParser, + resolver: async () => { + throw new Error("Test error"); + }, + }); + + const pipeline = definePipeline({ + id: "error-events", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [failingRoute], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("error-events"); + + const errorEvents = events.filter((e) => e.type === "error"); + expect(errorEvents.length).toBeGreaterThan(0); + }); + + it("should handle pipeline without inputs gracefully", async () => { + const pipeline = definePipeline({ + id: "no-inputs", + versions: ["16.0.0"], + inputs: [], + routes: [], + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + + await expect(executor.runSingle("no-inputs")).rejects.toThrow( + "Pipeline requires at least one input source", + ); + }); +}); + +describe("caching", () => { + let cacheStore: CacheStore; + + beforeEach(() => { + cacheStore = createMemoryCacheStore(); + }); + + it("should use cache when enabled", async () => { + const events: PipelineEvent[] = []; + + const pipeline = definePipeline({ + id: "cache-test", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + cacheStore, + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("cache-test", { cache: true }); + + const cacheEvents = events.filter((e) => + e.type === "cache:hit" || e.type === "cache:miss" || e.type === "cache:store", + ); + expect(cacheEvents.length).toBeGreaterThan(0); + }); + + it("should hit cache on second run", async () => { + const events: PipelineEvent[] = []; + + const pipeline = definePipeline({ + id: "cache-hit-test", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + cacheStore, + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("cache-hit-test", { cache: true }); + + events.length = 0; + + await executor.runSingle("cache-hit-test", { cache: true }); + + const hitEvents = events.filter((e) => e.type === "cache:hit"); + expect(hitEvents.length).toBeGreaterThan(0); + }); + + it("should skip cache when disabled", async () => { + const events: PipelineEvent[] = []; + + const pipeline = definePipeline({ + id: "cache-disabled", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Test.txt")], { "ucd/Test.txt": "0041;A" })], + routes: [createTestRoute("test", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + cacheStore, + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("cache-disabled", { cache: false }); + + const cacheEvents = events.filter((e) => + e.type === "cache:hit" || e.type === "cache:miss" || e.type === "cache:store", + ); + expect(cacheEvents).toEqual([]); + }); +}); + +describe("multiple pipelines", () => { + it("should run multiple pipelines", async () => { + const files = [createMockFile("Test.txt")]; + const contents = { "ucd/Test.txt": "0041;A" }; + + const pipeline1 = definePipeline({ + id: "pipeline-1", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("route-1", () => true)], + }); + + const pipeline2 = definePipeline({ + id: "pipeline-2", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("route-2", () => true)], + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline1, pipeline2], + }); + + const result = await executor.run(); + + expect(result.results.size).toBe(2); + expect(result.results.has("pipeline-1")).toBe(true); + expect(result.results.has("pipeline-2")).toBe(true); + }); + + it("should track successful and failed pipelines", async () => { + const files = [createMockFile("Test.txt")]; + const contents = { "ucd/Test.txt": "0041;A" }; + + const successPipeline = definePipeline({ + id: "success", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("ok", () => true)], + }); + + const failPipeline = definePipeline({ + id: "fail", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [ + definePipelineRoute({ + id: "fail-route", + filter: () => true, + parser: mockParser, + resolver: async () => { + throw new Error("Intentional failure"); + }, + }), + ], + }); + + const executor = createPipelineExecutor({ + pipelines: [successPipeline, failPipeline], + }); + + const result = await executor.run(); + + expect(result.summary.successfulPipelines).toBe(1); + expect(result.summary.failedPipelines).toBe(1); + }); +}); + +describe("strict mode", () => { + it("should error on unmatched files in strict mode", async () => { + const files = [ + createMockFile("Matched.txt"), + createMockFile("Unmatched.txt"), + ]; + const contents = { + "ucd/Matched.txt": "0041;A", + "ucd/Unmatched.txt": "0042;B", + }; + + const pipeline = definePipeline({ + id: "strict-test", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("matched", (ctx) => ctx.file.name === "Matched.txt")], + strict: true, + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("strict-test"); + + const fileErrors = result.errors.filter((e) => e.scope === "file"); + expect(fileErrors.length).toBe(1); + expect(fileErrors[0].message).toContain("No matching route"); + }); + + it("should not error on unmatched files when not strict", async () => { + const files = [ + createMockFile("Matched.txt"), + createMockFile("Unmatched.txt"), + ]; + const contents = { + "ucd/Matched.txt": "0041;A", + "ucd/Unmatched.txt": "0042;B", + }; + + const pipeline = definePipeline({ + id: "non-strict-test", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("matched", (ctx) => ctx.file.name === "Matched.txt")], + strict: false, + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("non-strict-test"); + + const fileErrors = result.errors.filter((e) => e.scope === "file"); + expect(fileErrors).toEqual([]); + }); +}); + +describe("fallback route", () => { + it("should use fallback for unmatched files", async () => { + const files = [ + createMockFile("Matched.txt"), + createMockFile("Fallback.txt"), + ]; + const contents = { + "ucd/Matched.txt": "0041;A", + "ucd/Fallback.txt": "0042;B", + }; + + const pipeline = definePipeline({ + id: "fallback-test", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("matched", (ctx) => ctx.file.name === "Matched.txt")], + fallback: { + parser: mockParser, + resolver: async (ctx, rows) => { + const entries: Array<{ codePoint: string; value: string }> = []; + for await (const row of rows) { + entries.push({ codePoint: row.codePoint!, value: row.value as string }); + } + return { type: "fallback", file: ctx.file.name, entries }; + }, + }, + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("fallback-test"); + + expect(result.summary.fallbackFiles).toBe(1); + expect(result.data.length).toBe(2); + + const fallbackOutput = result.data.find((d: any) => d.type === "fallback"); + expect(fallbackOutput).toBeDefined(); + }); + + it("should emit file:fallback event", async () => { + const events: PipelineEvent[] = []; + + const pipeline = definePipeline({ + id: "fallback-event", + versions: ["16.0.0"], + inputs: [createTestSource([createMockFile("Unmatched.txt")], { "ucd/Unmatched.txt": "0041;A" })], + routes: [], + fallback: { + parser: mockParser, + resolver: async () => ({ fallback: true }), + }, + }); + + const executor = createPipelineExecutor({ + pipelines: [pipeline], + onEvent: (event) => events.push(event), + }); + + await executor.runSingle("fallback-event"); + + const fallbackEvents = events.filter((e) => e.type === "file:fallback"); + expect(fallbackEvents.length).toBe(1); + }); +}); + +describe("include filter", () => { + it("should only process files matching include filter", async () => { + const files = [ + createMockFile("Include.txt"), + createMockFile("Exclude.txt"), + ]; + const contents = { + "ucd/Include.txt": "0041;A", + "ucd/Exclude.txt": "0042;B", + }; + + const pipeline = definePipeline({ + id: "include-test", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("all", () => true)], + include: (ctx) => ctx.file.name.startsWith("Include"), + }); + + const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const result = await executor.runSingle("include-test"); + + expect(result.summary.matchedFiles).toBe(1); + expect(result.data.length).toBe(1); + }); +}); diff --git a/packages/pipelines/pipeline-graph/src/index.ts b/packages/pipelines/pipeline-graph/src/index.ts index a02490739..683b235cf 100644 --- a/packages/pipelines/pipeline-graph/src/index.ts +++ b/packages/pipelines/pipeline-graph/src/index.ts @@ -1,6 +1,6 @@ export type { - DAGNode, DAG, + DAGNode, DAGValidationError, DAGValidationResult, } from "./dag"; From 802a15570ce3e2f20a6e85bca0d957c20ef9b8ed Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 19 Jan 2026 06:49:45 +0100 Subject: [PATCH 07/63] chore: lint --- packages/pipelines/pipeline-core/src/route.ts | 14 +++++++++++++- .../pipeline-core/test/dependencies.test.ts | 9 +++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/pipelines/pipeline-core/src/route.ts b/packages/pipelines/pipeline-core/src/route.ts index 992356a3d..e4e50071b 100644 --- a/packages/pipelines/pipeline-core/src/route.ts +++ b/packages/pipelines/pipeline-core/src/route.ts @@ -30,7 +30,19 @@ export interface RouteResolveContext< key: K, value: InferArtifactType, ) => void; - normalizeEntries: (entries: Array<{ range?: string; codePoint?: string; sequence?: string[]; value: string | string[] }>) => Array<{ range?: string; codePoint?: string; sequence?: string[]; value: string | string[] }>; + normalizeEntries: (entries: Array< + { + range?: string; + codePoint?: string; + sequence?: string[]; + value: string | string[]; + } + >) => Array<{ + range?: string; + codePoint?: string; + sequence?: string[]; + value: string | string[]; + }>; now: () => string; } diff --git a/packages/pipelines/pipeline-core/test/dependencies.test.ts b/packages/pipelines/pipeline-core/test/dependencies.test.ts index 8fa95c735..0f8cab000 100644 --- a/packages/pipelines/pipeline-core/test/dependencies.test.ts +++ b/packages/pipelines/pipeline-core/test/dependencies.test.ts @@ -218,13 +218,11 @@ describe("parsedDependency types", () => { describe("type inference", () => { describe("extractRouteDependencies", () => { it("should extract route ids from dependency array", () => { - const deps = [ + type RouteIds = ExtractRouteDependencies<[ "route:parser", "route:normalizer", "artifact:other:data", - ] as const; - - type RouteIds = ExtractRouteDependencies; + ]>; const id1: RouteIds = "parser"; const id2: RouteIds = "normalizer"; @@ -234,6 +232,7 @@ describe("type inference", () => { }); it("should extract never type for empty array", () => { + // eslint-disable-next-line unused-imports/no-unused-vars const deps = [] as const; type RouteIds = ExtractRouteDependencies; @@ -245,6 +244,7 @@ describe("type inference", () => { describe("extractArtifactDependencies", () => { it("should extract artifact info from dependency array", () => { + // eslint-disable-next-line unused-imports/no-unused-vars const deps = [ "artifact:parser:result", "artifact:normalizer:data", @@ -263,6 +263,7 @@ describe("type inference", () => { describe("extractArtifactKeys", () => { it("should extract artifact keys from dependency array", () => { + // eslint-disable-next-line unused-imports/no-unused-vars const deps = [ "artifact:parser:result", "artifact:normalizer:data", From ecfda3fdbc2ebc113783ac86a3a583c2c332b34c Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 20 Jan 2026 12:54:39 +0100 Subject: [PATCH 08/63] chore: update build and dev script filters Modified the `build` and `dev` script filters in `package.json` to use double asterisks (`**`) for better matching of package directories. From ef318dcfbbd226a472109c4c4afd92e936c7fe89 Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 20 Jan 2026 12:55:05 +0100 Subject: [PATCH 09/63] chore: update eslint config to beta.2 and add tinyglobby Updated `@luxass/eslint-config` from version `7.0.0-beta.1` to `7.0.0-beta.2` for improved linting rules. Added `tinyglobby` package at version `0.2.15` to enhance globbing capabilities in the project. --- pnpm-lock.yaml | 2 +- pnpm-workspace.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c5b56b89d..e54b80163 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1103,7 +1103,7 @@ importers: devDependencies: '@luxass/eslint-config': specifier: catalog:linting - version: 7.0.0-beta.1(@eslint-react/eslint-plugin@2.3.12(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.24(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + version: 7.0.0-beta.2(@eslint-react/eslint-plugin@2.7.2(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 87356dce8..79d8aaf19 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -65,6 +65,7 @@ catalogs: "@luxass/msw-utils": 0.6.0 hookable: 6.0.1 cac: 6.7.14 + tinyglobby: 0.2.15 build: tsdown: 0.20.1 From d0177c9bdcec8327000092b3726e460a27979a80 Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 20 Jan 2026 12:55:57 +0100 Subject: [PATCH 10/63] WIP --- packages/cli/package.json | 1 + packages/cli/src/cli-utils.ts | 17 +- packages/cli/src/cmd/pipelines/list.ts | 95 +++++++ packages/cli/src/cmd/pipelines/root.ts | 64 +++++ packages/cli/src/cmd/pipelines/run.ts | 29 ++ .../pipelines/pipeline-artifacts/package.json | 2 - .../pipelines/pipeline-artifacts/src/index.ts | 28 +- .../pipeline-artifacts/src/schema.ts | 16 +- .../test/definition.test.ts | 2 +- .../pipeline-artifacts/tsconfig.json | 2 +- packages/pipelines/pipeline-core/package.json | 2 - .../src/dag.ts | 29 +- packages/pipelines/pipeline-core/src/index.ts | 12 + .../pipelines/pipeline-core/src/pipeline.ts | 16 ++ packages/pipelines/pipeline-core/src/route.ts | 17 +- .../pipelines/pipeline-core/src/transform.ts | 1 + .../pipelines/pipeline-core/test/dag.test.ts | 252 +++++++++++++++++ .../pipeline-core/test/pipeline.test.ts | 4 +- .../pipeline-core/test/route.test.ts | 12 +- .../pipeline-core/test/source.test.ts | 2 - .../pipeline-core/test/transform.test.ts | 2 - .../pipelines/pipeline-core/tsconfig.json | 2 +- .../pipelines/pipeline-executor/package.json | 3 +- .../pipeline-executor/src/executor.ts | 33 +-- .../pipeline-executor/test/executor.test.ts | 256 +++++++++--------- .../pipelines/pipeline-graph/package.json | 2 - .../pipelines/pipeline-graph/src/builder.ts | 106 ++++++++ .../pipelines/pipeline-graph/src/index.ts | 14 +- .../pipelines/pipeline-loader/package.json | 5 +- .../pipelines/pipeline-loader/src/find.ts | 24 ++ .../pipelines/pipeline-loader/src/index.ts | 2 + .../pipeline-playground/package.json | 40 +++ .../src/sequence.ucd-pipeline.ts | 28 ++ .../src/simple.ucd-pipeline.ts | 28 ++ .../pipeline-playground/tsconfig.build.json | 5 + .../pipeline-playground/tsconfig.json | 7 + .../pipeline-playground/tsdown.config.ts | 9 + .../pipelines/pipeline-presets/package.json | 10 +- .../pipeline-presets/src/parsers/standard.ts | 8 +- .../pipeline-presets/src/pipelines/basic.ts | 1 - .../pipeline-presets/src/pipelines/emoji.ts | 1 - .../pipeline-presets/src/pipelines/full.ts | 1 - .../src/resolvers/property-json.ts | 9 +- .../pipeline-presets/src/routes/common.ts | 3 - packages/pipelines/pipeline-ui/package.json | 3 +- packages/pipelines/pipeline-ui/src/index.ts | 3 +- tooling/tsconfig/base.json | 1 + 47 files changed, 950 insertions(+), 259 deletions(-) create mode 100644 packages/cli/src/cmd/pipelines/list.ts create mode 100644 packages/cli/src/cmd/pipelines/root.ts create mode 100644 packages/cli/src/cmd/pipelines/run.ts rename packages/pipelines/{pipeline-graph => pipeline-core}/src/dag.ts (89%) create mode 100644 packages/pipelines/pipeline-core/test/dag.test.ts create mode 100644 packages/pipelines/pipeline-graph/src/builder.ts create mode 100644 packages/pipelines/pipeline-loader/src/find.ts create mode 100644 packages/pipelines/pipeline-playground/package.json create mode 100644 packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts create mode 100644 packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts create mode 100644 packages/pipelines/pipeline-playground/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-playground/tsconfig.json create mode 100644 packages/pipelines/pipeline-playground/tsdown.config.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 87b326a39..f32341a9b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -48,6 +48,7 @@ "@ucdjs/env": "workspace:*", "@ucdjs/fs-bridge": "workspace:*", "@ucdjs/lockfile": "workspace:*", + "@ucdjs/pipelines-loader": "workspace:*", "@ucdjs/schema-gen": "workspace:*", "@ucdjs/schemas": "workspace:*", "@ucdjs/ucd-store": "workspace:*", diff --git a/packages/cli/src/cli-utils.ts b/packages/cli/src/cli-utils.ts index c0ffc74b7..d9007dec0 100644 --- a/packages/cli/src/cli-utils.ts +++ b/packages/cli/src/cli-utils.ts @@ -2,6 +2,8 @@ import type { Prettify, RemoveIndexSignature } from "@luxass/utils"; import type { Arguments } from "yargs-parser"; import type { CLICodegenCmdOptions } from "./cmd/codegen/root"; import type { CLIFilesCmdOptions } from "./cmd/files/root"; +import type { CLILockfileCmdOptions } from "./cmd/lockfile/root"; +import type { CLIPipelinesCmdOptions } from "./cmd/pipelines/root"; import type { CLIStoreCmdOptions } from "./cmd/store/root"; import process from "node:process"; import { @@ -22,13 +24,15 @@ type CLICommand | "codegen" | "store" | "files" - | "lockfile"; + | "lockfile" + | "pipelines"; const SUPPORTED_COMMANDS = new Set([ "codegen", "store", "files", "lockfile", + "pipelines", ]); export interface GlobalCLIFlags { @@ -181,6 +185,7 @@ export async function runCommand(cmd: CLICommand, flags: Arguments): Promise>; +} + +export async function runListPipelines({ flags }: CLIPipelinesRunCmdOptions) { + if (flags?.help || flags?.h) { + printHelp({ + headline: "List Pipelines", + commandName: "ucd pipelines list", + usage: "[...flags]", + tables: { + Flags: [ + ["--cwd ", "Search for pipeline files from this directory."], + ["--help (-h)", "See all available flags."], + ], + }, + }); + return; + } + + const cwd = flags?.cwd ?? process.cwd(); + + output.info("Searching for pipeline files..."); + const files = await findPipelineFiles(["**/*.ucd-pipeline.ts"], cwd); + + if (files.length === 0) { + output.info("No pipeline files found (pattern: **/*.ucd-pipeline.ts)."); + return; + } + + const result = await loadPipelinesFromPaths(files); + const totalPipelines = result.pipelines.length; + + header("Pipelines"); + keyValue("Files", String(result.files.length)); + keyValue("Pipelines", String(totalPipelines)); + blankLine(); + + for (const f of result.files) { + const rel = path.relative(cwd, f.filePath); + output.info(`${dim("•")} ${cyan(rel)}`); + + if (f.exportNames.length === 0) { + output.info(` ${dim("(no pipeline exports found)")}`); + continue; + } + + const items = f.exportNames.map((name, i) => { + const pipeline = f.pipelines[i]; + const displayName = pipeline?.name ?? name; + const id = pipeline?.id; + const idLabel = id && id !== name && id !== displayName ? ` ${dim(`[${id}]`)}` : ""; + const routesCount = Array.isArray(pipeline?.routes) ? pipeline.routes.length : 0; + const sourcesCount = Array.isArray(pipeline?.inputs) ? pipeline.inputs.length : 0; + const details = ` ${dim("·")} ${routesCount} route(s) ${dim("·")} ${sourcesCount} source(s)`; + const description = pipeline?.description ? ` ${dim("·")} ${pipeline.description}` : ""; + + return `${bold(displayName)}${idLabel}${details}${description}`; + }); + + items.forEach((item, index) => { + const isLast = index === items.length - 1; + const prefix = isLast ? "└" : "├"; + output.info(` ${dim(prefix)} ${item}`); + }); + } + + if (result.errors.length > 0) { + blankLine(); + header("Errors"); + for (const e of result.errors) { + const rel = path.relative(process.cwd(), e.filePath); + output.error(` ${yellow("•")} ${rel}: ${e.error.message}`); + } + } +} diff --git a/packages/cli/src/cmd/pipelines/root.ts b/packages/cli/src/cmd/pipelines/root.ts new file mode 100644 index 000000000..d55c119f1 --- /dev/null +++ b/packages/cli/src/cmd/pipelines/root.ts @@ -0,0 +1,64 @@ +import type { CLIArguments } from "../../cli-utils"; +import { printHelp } from "../../cli-utils"; + +export interface CLIPipelinesCmdOptions { + flags: CLIArguments<{ + ui: boolean; + }>; +} + +const PIPELINES_SUBCOMMANDS = [ + "run", + "list", + "create", +] as const; +export type Subcommand = (typeof PIPELINES_SUBCOMMANDS)[number]; + +function isValidSubcommand(subcommand: string): subcommand is Subcommand { + return PIPELINES_SUBCOMMANDS.includes(subcommand as Subcommand); +} + +export async function runPipelinesRoot(subcommand: string, { flags }: CLIPipelinesCmdOptions) { + const isValidSub = isValidSubcommand(subcommand); + const requestsHelp = flags?.help || flags?.h; + + if (!isValidSub || (requestsHelp && !isValidSub)) { + printHelp({ + commandName: "ucd pipelines", + usage: "[command] [...flags]", + tables: { + Commands: [ + ["run", "Run a pipeline from the command line."], + ["list", "List available pipelines."], + ["create", "Create a new pipeline scaffold."], + ], + Flags: [ + ["--help (-h)", "See all available flags."], + ], + }, + }); + return; + } + + if (subcommand === "run") { + const { runPipelinesRun } = await import("./run"); + await runPipelinesRun({ + flags, + }); + return; + } + + if (subcommand === "list") { + const { runListPipelines } = await import("./list"); + await runListPipelines({ flags }); + return; + } + + // if (subcommand === "create") { + // const { runVerifyStore } = await import("./create"); + // await runVerifyStore({ flags, versions }); + // return; + // } + + throw new Error(`Invalid subcommand: ${subcommand}`); +} diff --git a/packages/cli/src/cmd/pipelines/run.ts b/packages/cli/src/cmd/pipelines/run.ts new file mode 100644 index 000000000..81cc21bc9 --- /dev/null +++ b/packages/cli/src/cmd/pipelines/run.ts @@ -0,0 +1,29 @@ +import type { Prettify } from "@luxass/utils"; +import type { CLIArguments } from "../../cli-utils"; +import { printHelp } from "../../cli-utils"; +import { output } from "../../output"; + +export interface CLIPipelinesRunCmdOptions { + flags: CLIArguments>; +} + +export async function runPipelinesRun({ flags }: CLIPipelinesRunCmdOptions) { + if (flags?.help || flags?.h) { + printHelp({ + headline: "Run Pipelines", + commandName: "ucd pipelines run", + usage: "[...pipelines] [...flags]", + tables: { + Flags: [ + ["--ui", "Run the pipeline with a UI."], + ["--help (-h)", "See all available flags."], + ], + }, + }); + return; + } + + output.info("Running pipelines..."); +} diff --git a/packages/pipelines/pipeline-artifacts/package.json b/packages/pipelines/pipeline-artifacts/package.json index c0ab8812a..e0ac3f374 100644 --- a/packages/pipelines/pipeline-artifacts/package.json +++ b/packages/pipelines/pipeline-artifacts/package.json @@ -22,8 +22,6 @@ ".": "./dist/index.mjs", "./package.json": "./package.json" }, - "main": "./dist/index.mjs", - "module": "./dist/index.mjs", "types": "./dist/index.d.mts", "files": [ "dist" diff --git a/packages/pipelines/pipeline-artifacts/src/index.ts b/packages/pipelines/pipeline-artifacts/src/index.ts index ee22f42d4..9e5da954a 100644 --- a/packages/pipelines/pipeline-artifacts/src/index.ts +++ b/packages/pipelines/pipeline-artifacts/src/index.ts @@ -1,7 +1,20 @@ +export type { + ArtifactBuildContext, + InferArtifactId, + InferArtifactsMap, + InferArtifactValue, + PipelineArtifactDefinition, +} from "./definition"; + +export { + definePipelineArtifact, + isPipelineArtifactDefinition, +} from "./definition"; + export type { Artifact, - GlobalArtifact, ArtifactDefinition, + GlobalArtifact, InferArtifactSchemaType, InferEmittedArtifacts, } from "./schema"; @@ -11,16 +24,3 @@ export { isGlobalArtifact, isVersionArtifact, } from "./schema"; - -export type { - ArtifactBuildContext, - PipelineArtifactDefinition, - InferArtifactId, - InferArtifactValue, - InferArtifactsMap, -} from "./definition"; - -export { - definePipelineArtifact, - isPipelineArtifactDefinition, -} from "./definition"; diff --git a/packages/pipelines/pipeline-artifacts/src/schema.ts b/packages/pipelines/pipeline-artifacts/src/schema.ts index 42cd5fdd5..05e3d690f 100644 --- a/packages/pipelines/pipeline-artifacts/src/schema.ts +++ b/packages/pipelines/pipeline-artifacts/src/schema.ts @@ -12,22 +12,22 @@ export interface GlobalArtifact { scope: "global"; } -export type ArtifactDefinition = - | Artifact - | GlobalArtifact; +export type ArtifactDefinition + = | Artifact + | GlobalArtifact; export function artifact( - schema: TSchema + schema: TSchema, ): Artifact; export function artifact( schema: TSchema, - scope: "version" + scope: "version", ): Artifact; export function artifact( schema: TSchema, - scope: "global" + scope: "global", ): GlobalArtifact; export function artifact( @@ -48,8 +48,8 @@ export function artifact( }; } -export type InferArtifactSchemaType = - T extends ArtifactDefinition ? z.infer : never; +export type InferArtifactSchemaType + = T extends ArtifactDefinition ? z.infer : never; export type InferEmittedArtifacts> = { [K in keyof TEmits]: InferArtifactSchemaType; diff --git a/packages/pipelines/pipeline-artifacts/test/definition.test.ts b/packages/pipelines/pipeline-artifacts/test/definition.test.ts index 3e38385a2..acaa7b6e7 100644 --- a/packages/pipelines/pipeline-artifacts/test/definition.test.ts +++ b/packages/pipelines/pipeline-artifacts/test/definition.test.ts @@ -1,3 +1,4 @@ +import type { ParseContext, ParsedRow, PipelineFilter } from "@ucdjs/pipelines-core"; import type { ArtifactBuildContext, InferArtifactId, @@ -5,7 +6,6 @@ import type { InferArtifactValue, PipelineArtifactDefinition, } from "../src/definition"; -import type { ParseContext, ParsedRow, PipelineFilter } from "@ucdjs/pipelines-core"; import { describe, expect, it, vi } from "vitest"; import { definePipelineArtifact, diff --git a/packages/pipelines/pipeline-artifacts/tsconfig.json b/packages/pipelines/pipeline-artifacts/tsconfig.json index 89e9ed10b..9c6dd744b 100644 --- a/packages/pipelines/pipeline-artifacts/tsconfig.json +++ b/packages/pipelines/pipeline-artifacts/tsconfig.json @@ -2,7 +2,7 @@ "extends": "@ucdjs-tooling/tsconfig/base", "include": [ "src", - "test", + "test" ], "exclude": ["dist"] } diff --git a/packages/pipelines/pipeline-core/package.json b/packages/pipelines/pipeline-core/package.json index 879238002..ae6274c32 100644 --- a/packages/pipelines/pipeline-core/package.json +++ b/packages/pipelines/pipeline-core/package.json @@ -22,8 +22,6 @@ ".": "./dist/index.mjs", "./package.json": "./package.json" }, - "main": "./dist/index.mjs", - "module": "./dist/index.mjs", "types": "./dist/index.d.mts", "files": [ "dist" diff --git a/packages/pipelines/pipeline-graph/src/dag.ts b/packages/pipelines/pipeline-core/src/dag.ts similarity index 89% rename from packages/pipelines/pipeline-graph/src/dag.ts rename to packages/pipelines/pipeline-core/src/dag.ts index f01cdbfc9..4ed489253 100644 --- a/packages/pipelines/pipeline-graph/src/dag.ts +++ b/packages/pipelines/pipeline-core/src/dag.ts @@ -1,5 +1,5 @@ -import type { PipelineRouteDefinition } from "@ucdjs/pipelines-core"; -import { isArtifactDependency, isRouteDependency, parseDependency } from "@ucdjs/pipelines-core"; +import type { PipelineRouteDefinition } from "./route"; +import { isArtifactDependency, isRouteDependency, parseDependency } from "./dependencies"; export interface DAGNode { id: string; @@ -14,7 +14,7 @@ export interface DAG { } export interface DAGValidationError { - type: "cycle" | "missing-route" | "missing-artifact"; + type: "cycle" | "missing-route" | "missing-artifact" | "duplicate-route"; message: string; details: { routeId?: string; @@ -32,9 +32,30 @@ export interface DAGValidationResult { export function buildDAG(routes: readonly PipelineRouteDefinition[]): DAGValidationResult { const errors: DAGValidationError[] = []; const nodes = new Map(); - const routeIds = new Set(routes.map((r) => r.id)); const artifactsByRoute = new Map>(); + const seenIds = new Map(); + for (let i = 0; i < routes.length; i++) { + const route = routes[i]; + if (!route) continue; + const id = route.id; + if (seenIds.has(id)) { + errors.push({ + type: "duplicate-route", + message: `Duplicate route ID "${id}" found at index ${seenIds.get(id)} and ${i}`, + details: { routeId: id }, + }); + } else { + seenIds.set(id, i); + } + } + + if (errors.length > 0) { + return { valid: false, errors }; + } + + const routeIds = new Set(routes.map((r) => r.id)); + for (const route of routes) { const emittedArtifacts = new Set(); if (route.emits) { diff --git a/packages/pipelines/pipeline-core/src/index.ts b/packages/pipelines/pipeline-core/src/index.ts index 5ef3e95df..31c07764e 100644 --- a/packages/pipelines/pipeline-core/src/index.ts +++ b/packages/pipelines/pipeline-core/src/index.ts @@ -1,3 +1,15 @@ +export type { + DAG, + DAGNode, + DAGValidationError, + DAGValidationResult, +} from "./dag"; + +export { + buildDAG, + getExecutionLayers, +} from "./dag"; + export type { ExtractArtifactDependencies, ExtractArtifactKeys, diff --git a/packages/pipelines/pipeline-core/src/pipeline.ts b/packages/pipelines/pipeline-core/src/pipeline.ts index 54b6877e3..3f0767aaf 100644 --- a/packages/pipelines/pipeline-core/src/pipeline.ts +++ b/packages/pipelines/pipeline-core/src/pipeline.ts @@ -1,7 +1,9 @@ +import type { DAG } from "./dag"; import type { PipelineEvent } from "./events"; import type { InferRoutesOutput, PipelineRouteDefinition } from "./route"; import type { InferSourceIds, PipelineSourceDefinition } from "./source"; import type { ParseContext, ParsedRow, PipelineFilter, ResolveContext } from "./types"; +import { buildDAG } from "./dag"; /** * Fallback route definition for files that don't match any explicit route. @@ -166,6 +168,12 @@ export interface PipelineDefinition< * Event handler for pipeline events. */ readonly onEvent?: (event: PipelineEvent) => void | Promise; + + /** + * Precomputed DAG (Directed Acyclic Graph) for route execution order. + * Built at definition time from route dependencies. + */ + readonly dag: DAG; } /** @@ -222,6 +230,13 @@ export function definePipeline< >( options: PipelineDefinitionOptions & { id: TId }, ): PipelineDefinition { + const dagResult = buildDAG(options.routes); + + if (!dagResult.valid) { + const errorMessages = dagResult.errors.map((e) => e.message).join("\n "); + throw new Error(`Pipeline "${options.id}" has invalid route dependencies:\n ${errorMessages}`); + } + return { _type: "pipeline-definition", id: options.id, @@ -235,6 +250,7 @@ export function definePipeline< concurrency: options.concurrency ?? 4, fallback: options.fallback, onEvent: options.onEvent, + dag: dagResult.dag!, }; } diff --git a/packages/pipelines/pipeline-core/src/route.ts b/packages/pipelines/pipeline-core/src/route.ts index e4e50071b..8aa718b55 100644 --- a/packages/pipelines/pipeline-core/src/route.ts +++ b/packages/pipelines/pipeline-core/src/route.ts @@ -1,3 +1,4 @@ +import type { z } from "zod"; import type { ExtractArtifactKeys, PipelineDependency } from "./dependencies"; import type { ChainTransforms, PipelineTransformDefinition } from "./transform"; import type { @@ -6,9 +7,9 @@ import type { ParserFn, PipelineFilter, PropertyJson, + ResolvedEntry, RouteOutput, } from "./types"; -import type { z } from "zod"; export interface ArtifactDefinition { _type: "artifact" | "global-artifact"; @@ -30,19 +31,7 @@ export interface RouteResolveContext< key: K, value: InferArtifactType, ) => void; - normalizeEntries: (entries: Array< - { - range?: string; - codePoint?: string; - sequence?: string[]; - value: string | string[]; - } - >) => Array<{ - range?: string; - codePoint?: string; - sequence?: string[]; - value: string | string[]; - }>; + normalizeEntries: (entries: ResolvedEntry[]) => ResolvedEntry[]; now: () => string; } diff --git a/packages/pipelines/pipeline-core/src/transform.ts b/packages/pipelines/pipeline-core/src/transform.ts index 5fe4092e2..76ad5a298 100644 --- a/packages/pipelines/pipeline-core/src/transform.ts +++ b/packages/pipelines/pipeline-core/src/transform.ts @@ -111,6 +111,7 @@ export type ChainTransforms< TTransforms extends readonly PipelineTransformDefinition[], > = TTransforms extends readonly [] ? TInput + // eslint-disable-next-line unused-imports/no-unused-vars : TTransforms extends readonly [infer T1 extends PipelineTransformDefinition] ? O1 : TTransforms extends readonly [infer T1, infer T2] diff --git a/packages/pipelines/pipeline-core/test/dag.test.ts b/packages/pipelines/pipeline-core/test/dag.test.ts new file mode 100644 index 000000000..95439106c --- /dev/null +++ b/packages/pipelines/pipeline-core/test/dag.test.ts @@ -0,0 +1,252 @@ +import type { ParsedRow, PipelineRouteDefinition } from "../src"; +import { describe, expect, it } from "vitest"; +import { buildDAG, definePipelineRoute, getExecutionLayers } from "../src"; + +function createMockParser() { + return async function* (): AsyncIterable { + yield { + sourceFile: "test.txt", + kind: "point" as const, + codePoint: "0041", + value: "test", + }; + }; +} + +function createRoute( + id: string, + depends?: (`route:${string}` | `artifact:${string}:${string}`)[], +): PipelineRouteDefinition { + return definePipelineRoute({ + id, + filter: () => true, + parser: createMockParser(), + resolver: async () => [], + depends, + }); +} + +describe("buildDAG", () => { + it("should build DAG from independent routes", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b"), + createRoute("route-c"), + ]; + + const result = buildDAG(routes); + + expect(result.valid).toBe(true); + expect(result.errors).toEqual([]); + expect(result.dag).toBeDefined(); + expect(result.dag!.nodes.size).toBe(3); + }); + + it("should build DAG with route dependencies", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b", ["route:route-a"]), + createRoute("route-c", ["route:route-b"]), + ]; + + const result = buildDAG(routes); + + expect(result.valid).toBe(true); + expect(result.dag!.nodes.get("route-b")!.dependencies.has("route-a")).toBe(true); + expect(result.dag!.nodes.get("route-c")!.dependencies.has("route-b")).toBe(true); + }); + + it("should detect circular dependencies", () => { + const routes = [ + createRoute("route-a", ["route:route-c"]), + createRoute("route-b", ["route:route-a"]), + createRoute("route-c", ["route:route-b"]), + ]; + + const result = buildDAG(routes); + + expect(result.valid).toBe(false); + expect(result.errors.length).toBeGreaterThan(0); + expect(result.errors[0].type).toBe("cycle"); + expect(result.errors[0].details.cycle).toBeDefined(); + }); + + it("should detect missing route dependencies", () => { + const routes = [ + createRoute("route-a", ["route:missing"]), + ]; + + const result = buildDAG(routes); + + expect(result.valid).toBe(false); + expect(result.errors[0].type).toBe("missing-route"); + expect(result.errors[0].details.dependencyId).toBe("missing"); + }); + + it("should detect duplicate route IDs", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b"), + createRoute("route-a"), + ]; + + const result = buildDAG(routes); + + expect(result.valid).toBe(false); + expect(result.errors.length).toBe(1); + expect(result.errors[0].type).toBe("duplicate-route"); + expect(result.errors[0].details.routeId).toBe("route-a"); + expect(result.errors[0].message).toContain("index 0 and 2"); + }); + + it("should track dependents", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b", ["route:route-a"]), + createRoute("route-c", ["route:route-a"]), + ]; + + const result = buildDAG(routes); + + const nodeA = result.dag!.nodes.get("route-a"); + expect(nodeA!.dependents.has("route-b")).toBe(true); + expect(nodeA!.dependents.has("route-c")).toBe(true); + }); + + it("should generate correct execution order", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b", ["route:route-a"]), + createRoute("route-c", ["route:route-b"]), + ]; + + const result = buildDAG(routes); + + expect(result.valid).toBe(true); + const order = result.dag!.executionOrder; + const indexA = order.indexOf("route-a"); + const indexB = order.indexOf("route-b"); + const indexC = order.indexOf("route-c"); + + expect(indexA).toBeLessThan(indexB); + expect(indexB).toBeLessThan(indexC); + }); +}); + +describe("getExecutionLayers", () => { + it("should put independent routes in same layer", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b"), + createRoute("route-c"), + ]; + + const result = buildDAG(routes); + const layers = getExecutionLayers(result.dag!); + + expect(layers.length).toBe(1); + expect(layers[0]).toHaveLength(3); + expect(new Set(layers[0])).toEqual(new Set(["route-a", "route-b", "route-c"])); + }); + + it("should create layers based on dependencies", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b", ["route:route-a"]), + createRoute("route-c", ["route:route-b"]), + ]; + + const result = buildDAG(routes); + const layers = getExecutionLayers(result.dag!); + + expect(layers.length).toBe(3); + expect(layers[0]).toEqual(["route-a"]); + expect(layers[1]).toEqual(["route-b"]); + expect(layers[2]).toEqual(["route-c"]); + }); + + it("should handle fan-in dependencies", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b"), + createRoute("route-c", ["route:route-a", "route:route-b"]), + ]; + + const result = buildDAG(routes); + const layers = getExecutionLayers(result.dag!); + + expect(layers.length).toBe(2); + expect(layers[0].sort()).toEqual(["route-a", "route-b"]); + expect(layers[1]).toEqual(["route-c"]); + }); + + it("should handle fan-out dependencies", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b", ["route:route-a"]), + createRoute("route-c", ["route:route-a"]), + ]; + + const result = buildDAG(routes); + const layers = getExecutionLayers(result.dag!); + + expect(layers.length).toBe(2); + expect(layers[0]).toEqual(["route-a"]); + expect(layers[1].sort()).toEqual(["route-b", "route-c"]); + }); + + it("should handle complex DAG", () => { + const routes = [ + createRoute("route-a"), + createRoute("route-b"), + createRoute("route-c", ["route:route-a"]), + createRoute("route-d", ["route:route-b"]), + createRoute("route-e", ["route:route-c", "route:route-d"]), + ]; + + const result = buildDAG(routes); + const layers = getExecutionLayers(result.dag!); + + expect(layers.length).toBe(3); + expect(new Set(layers[0])).toEqual(new Set(["route-a", "route-b"])); + expect(new Set(layers[1])).toEqual(new Set(["route-c", "route-d"])); + expect(layers[2]).toEqual(["route-e"]); + }); +}); + +describe("DAGNode structure", () => { + it("should have correct node structure", () => { + const routes = [createRoute("test-route")]; + + const result = buildDAG(routes); + const node = result.dag!.nodes.get("test-route"); + + expect(node).toEqual({ + id: "test-route", + dependencies: expect.any(Set), + dependents: expect.any(Set), + emittedArtifacts: expect.any(Set), + }); + }); + + it("should have empty dependencies for independent routes", () => { + const routes = [createRoute("independent")]; + + const result = buildDAG(routes); + const node = result.dag!.nodes.get("independent"); + + expect(node!.dependencies.size).toBe(0); + }); + + it("should have empty dependents for leaf nodes", () => { + const routes = [ + createRoute("root"), + createRoute("leaf", ["route:root"]), + ]; + + const result = buildDAG(routes); + const node = result.dag!.nodes.get("leaf"); + + expect(node!.dependents.size).toBe(0); + }); +}); diff --git a/packages/pipelines/pipeline-core/test/pipeline.test.ts b/packages/pipelines/pipeline-core/test/pipeline.test.ts index 0b3eddd87..42eab5075 100644 --- a/packages/pipelines/pipeline-core/test/pipeline.test.ts +++ b/packages/pipelines/pipeline-core/test/pipeline.test.ts @@ -1,6 +1,6 @@ -import type { FallbackRouteDefinition, PipelineDefinition } from "../src/pipeline"; +import type { FallbackRouteDefinition } from "../src/pipeline"; import type { SourceBackend } from "../src/source"; -import type { FileContext, ParsedRow, PropertyJson } from "../src/types"; +import type { ParsedRow } from "../src/types"; import { describe, expect, it, vi } from "vitest"; import { definePipeline, diff --git a/packages/pipelines/pipeline-core/test/route.test.ts b/packages/pipelines/pipeline-core/test/route.test.ts index f8691c4ba..4ff52ef38 100644 --- a/packages/pipelines/pipeline-core/test/route.test.ts +++ b/packages/pipelines/pipeline-core/test/route.test.ts @@ -4,11 +4,10 @@ import type { InferRouteEmits, InferRouteId, InferRouteOutput, - PipelineRouteDefinition, RouteResolveContext, } from "../src/route"; import type { FileContext, ParsedRow, PropertyJson } from "../src/types"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, expectTypeOf, it, vi } from "vitest"; import { z } from "zod"; import { definePipelineRoute, @@ -386,17 +385,18 @@ describe("route context methods", () => { describe("type inference", () => { describe("inferRouteId", () => { it("should infer route id", () => { + const routeId = "my-route" as const; + // eslint-disable-next-line unused-imports/no-unused-vars const route = definePipelineRoute({ - id: "my-route", + id: routeId, filter: () => true, parser: mockParser, resolver: async () => [], }); type RouteId = InferRouteId; - const id: RouteId = "my-route"; - - expect(id).toBe("my-route"); + expectTypeOf().toBeString(); + expectTypeOf().toEqualTypeOf(); }); }); diff --git a/packages/pipelines/pipeline-core/test/source.test.ts b/packages/pipelines/pipeline-core/test/source.test.ts index f226c75cf..2b3cb497b 100644 --- a/packages/pipelines/pipeline-core/test/source.test.ts +++ b/packages/pipelines/pipeline-core/test/source.test.ts @@ -1,9 +1,7 @@ import type { InferSourceId, InferSourceIds, - PipelineSourceDefinition, SourceBackend, - SourceFileContext, } from "../src/source"; import type { FileContext } from "../src/types"; import { describe, expect, it, vi } from "vitest"; diff --git a/packages/pipelines/pipeline-core/test/transform.test.ts b/packages/pipelines/pipeline-core/test/transform.test.ts index 0fe5eebe9..21c058cdb 100644 --- a/packages/pipelines/pipeline-core/test/transform.test.ts +++ b/packages/pipelines/pipeline-core/test/transform.test.ts @@ -1,6 +1,4 @@ import type { - InferTransformInput, - InferTransformOutput, PipelineTransformDefinition, TransformContext, } from "../src/transform"; diff --git a/packages/pipelines/pipeline-core/tsconfig.json b/packages/pipelines/pipeline-core/tsconfig.json index 89e9ed10b..9c6dd744b 100644 --- a/packages/pipelines/pipeline-core/tsconfig.json +++ b/packages/pipelines/pipeline-core/tsconfig.json @@ -2,7 +2,7 @@ "extends": "@ucdjs-tooling/tsconfig/base", "include": [ "src", - "test", + "test" ], "exclude": ["dist"] } diff --git a/packages/pipelines/pipeline-executor/package.json b/packages/pipelines/pipeline-executor/package.json index f51ead4c5..70da5e1df 100644 --- a/packages/pipelines/pipeline-executor/package.json +++ b/packages/pipelines/pipeline-executor/package.json @@ -39,9 +39,8 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { - "@ucdjs/pipelines-core": "workspace:*", "@ucdjs/pipelines-artifacts": "workspace:*", - "@ucdjs/pipelines-graph": "workspace:*" + "@ucdjs/pipelines-core": "workspace:*" }, "devDependencies": { "@luxass/eslint-config": "catalog:linting", diff --git a/packages/pipelines/pipeline-executor/src/executor.ts b/packages/pipelines/pipeline-executor/src/executor.ts index cc661f039..b35d9a706 100644 --- a/packages/pipelines/pipeline-executor/src/executor.ts +++ b/packages/pipelines/pipeline-executor/src/executor.ts @@ -1,5 +1,3 @@ -import type { CacheEntry, CacheKey, CacheStore } from "./cache"; -import type { MultiplePipelineRunResult, PipelineRunResult, PipelineSummary } from "./results"; import type { ArtifactDefinition, PipelineArtifactDefinition } from "@ucdjs/pipelines-artifacts"; import type { FileContext, @@ -15,9 +13,10 @@ import type { SourceBackend, SourceFileContext, } from "@ucdjs/pipelines-core"; +import type { CacheEntry, CacheKey, CacheStore } from "./cache"; +import type { MultiplePipelineRunResult, PipelineRunResult, PipelineSummary } from "./results"; import { isGlobalArtifact } from "@ucdjs/pipelines-artifacts"; -import { applyTransforms, resolveMultipleSourceFiles } from "@ucdjs/pipelines-core"; -import { buildDAG, getExecutionLayers } from "@ucdjs/pipelines-graph"; +import { applyTransforms, getExecutionLayers, resolveMultipleSourceFiles } from "@ucdjs/pipelines-core"; import { defaultHashFn, hashArtifact } from "./cache"; interface SourceAdapter { @@ -26,7 +25,6 @@ interface SourceAdapter { } export interface PipelineExecutorOptions { - pipelines: PipelineDefinition[]; artifacts?: PipelineArtifactDefinition[]; cacheStore?: CacheStore; onEvent?: (event: PipelineEvent) => void | Promise; @@ -35,24 +33,19 @@ export interface PipelineExecutorOptions { export interface PipelineExecutorRunOptions { cache?: boolean; versions?: string[]; - pipelines?: string[]; } export interface PipelineExecutor { - run: (options?: PipelineExecutorRunOptions) => Promise; - runSingle: (pipelineId: string, options?: Omit) => Promise; + run: (pipelines: PipelineDefinition[], options?: PipelineExecutorRunOptions) => Promise; } export function createPipelineExecutor(options: PipelineExecutorOptions): PipelineExecutor { const { - pipelines, artifacts: globalArtifacts = [], cacheStore, onEvent, } = options; - const pipelinesById = new Map(pipelines.map((p) => [p.id, p])); - async function emit(event: PipelineEvent): Promise { if (onEvent) { await onEvent(event); @@ -80,11 +73,7 @@ export function createPipelineExecutor(options: PipelineExecutorOptions): Pipeli let skippedFiles = 0; let fallbackFiles = 0; - const dagResult = buildDAG(pipeline.routes); - if (!dagResult.valid) { - throw new Error(`Pipeline DAG validation failed:\n${dagResult.errors.map((e) => ` - ${e.message}`).join("\n")}`); - } - const dag = dagResult.dag!; + const dag = pipeline.dag; await emit({ type: "pipeline:start", versions: versionsToRun, timestamp: Date.now() }); @@ -487,11 +476,8 @@ export function createPipelineExecutor(options: PipelineExecutorOptions): Pipeli }; } - async function run(runOptions: PipelineExecutorRunOptions = {}): Promise { + async function run(pipelinesToRun: PipelineDefinition[], runOptions: PipelineExecutorRunOptions = {}): Promise { const startTime = performance.now(); - const pipelinesToRun = runOptions.pipelines - ? pipelines.filter((p) => runOptions.pipelines!.includes(p.id)) - : pipelines; const results = new Map(); let successfulPipelines = 0; @@ -542,13 +528,6 @@ export function createPipelineExecutor(options: PipelineExecutorOptions): Pipeli return { run, - runSingle: (pipelineId, options) => { - const pipeline = pipelinesById.get(pipelineId); - if (!pipeline) { - throw new Error(`Pipeline "${pipelineId}" not found`); - } - return runSinglePipeline(pipeline, options); - }, }; } diff --git a/packages/pipelines/pipeline-executor/test/executor.test.ts b/packages/pipelines/pipeline-executor/test/executor.test.ts index d3f0eff78..4959cb78c 100644 --- a/packages/pipelines/pipeline-executor/test/executor.test.ts +++ b/packages/pipelines/pipeline-executor/test/executor.test.ts @@ -1,5 +1,3 @@ -import type { CacheStore } from "../src/cache"; -import type { PipelineExecutor } from "../src/executor"; import type { FileContext, ParseContext, @@ -9,10 +7,12 @@ import type { PipelineSourceDefinition, SourceBackend, } from "@ucdjs/pipelines-core"; +import type { CacheStore } from "../src/cache"; +import type { PipelineExecutor } from "../src/executor"; +import { definePipeline, definePipelineRoute, definePipelineSource } from "@ucdjs/pipelines-core"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { createMemoryCacheStore } from "../src/cache"; import { createPipelineExecutor } from "../src/executor"; -import { definePipeline, definePipelineRoute, definePipelineSource } from "@ucdjs/pipelines-core"; function createMockFile(name: string, dir: string = "ucd"): FileContext { return { @@ -60,7 +60,7 @@ function createTestRoute( parser: mockParser, resolver: async (ctx, rows) => { const entries: Array<{ codePoint: string; value: string }> = []; - for await (const row of rows) { + for await (const row of rows as AsyncIterable) { entries.push({ codePoint: row.codePoint!, value: row.value as string, @@ -83,15 +83,11 @@ function createTestSource(files: FileContext[], contents: Record } describe("createPipelineExecutor", () => { - it("should create an executor with run and runSingle methods", () => { - const executor = createPipelineExecutor({ - pipelines: [], - }); + it("should create an executor with run method", () => { + const executor = createPipelineExecutor({}); expect(executor).toHaveProperty("run"); - expect(executor).toHaveProperty("runSingle"); expect(typeof executor.run).toBe("function"); - expect(typeof executor.runSingle).toBe("function"); }); it("should accept pipelines and optional artifacts", () => { @@ -103,7 +99,6 @@ describe("createPipelineExecutor", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], artifacts: [], }); @@ -114,7 +109,6 @@ describe("createPipelineExecutor", () => { const cacheStore = createMemoryCacheStore(); const executor = createPipelineExecutor({ - pipelines: [], cacheStore, }); @@ -125,7 +119,6 @@ describe("createPipelineExecutor", () => { const onEvent = vi.fn(); const executor = createPipelineExecutor({ - pipelines: [], onEvent, }); @@ -137,6 +130,7 @@ describe("executor.run", () => { let executor: PipelineExecutor; let files: FileContext[]; let contents: Record; + let pipeline: ReturnType; beforeEach(() => { files = [ @@ -155,20 +149,18 @@ describe("executor.run", () => { createTestRoute("scripts", (ctx) => ctx.file.name === "Scripts.txt"), ]; - const pipeline = definePipeline({ + pipeline = definePipeline({ id: "test-pipeline", versions: ["16.0.0"], inputs: [source], routes, }); - executor = createPipelineExecutor({ - pipelines: [pipeline], - }); + executor = createPipelineExecutor({}); }); it("should run all pipelines and return results", async () => { - const result = await executor.run(); + const result = await executor.run([pipeline as any]); expect(result.results).toBeInstanceOf(Map); expect(result.results.size).toBe(1); @@ -176,7 +168,7 @@ describe("executor.run", () => { }); it("should return summary with pipeline counts", async () => { - const result = await executor.run(); + const result = await executor.run([pipeline as any]); expect(result.summary).toEqual({ totalPipelines: 1, @@ -187,36 +179,37 @@ describe("executor.run", () => { }); it("should process files matching routes", async () => { - const result = await executor.run(); + const result = await executor.run([pipeline as any]); const pipelineResult = result.results.get("test-pipeline")!; expect(pipelineResult.data.length).toBe(2); }); - it("should filter pipelines by id when specified", async () => { - const result = await executor.run({ pipelines: ["test-pipeline"] }); + it("should run provided pipelines", async () => { + const result = await executor.run([pipeline as any]); expect(result.results.has("test-pipeline")).toBe(true); }); - it("should skip pipelines not in filter list", async () => { - const result = await executor.run({ pipelines: ["non-existent"] }); + it("should return empty results when no pipelines provided", async () => { + const result = await executor.run([]); expect(result.results.size).toBe(0); }); it("should filter versions when specified", async () => { - const result = await executor.run({ versions: ["16.0.0"] }); + const result = await executor.run([pipeline as any], { versions: ["16.0.0"] }); const pipelineResult = result.results.get("test-pipeline")!; expect(pipelineResult.summary.versions).toEqual(["16.0.0"]); }); }); -describe("executor.runSingle", () => { +describe("running single pipeline via run()", () => { let executor: PipelineExecutor; let files: FileContext[]; let contents: Record; + let pipeline: ReturnType; beforeEach(() => { files = [createMockFile("LineBreak.txt")]; @@ -225,20 +218,19 @@ describe("executor.runSingle", () => { const source = createTestSource(files, contents); const route = createTestRoute("line-break", (ctx) => ctx.file.name === "LineBreak.txt"); - const pipeline = definePipeline({ + pipeline = definePipeline({ id: "test-pipeline", versions: ["16.0.0"], inputs: [source], routes: [route], }); - executor = createPipelineExecutor({ - pipelines: [pipeline], - }); + executor = createPipelineExecutor({}); }); - it("should run a single pipeline by id", async () => { - const result = await executor.runSingle("test-pipeline"); + it("should run a single pipeline", async () => { + const multi = await executor.run([pipeline]); + const result = multi.results.get("test-pipeline")!; expect(result).toBeDefined(); expect(result.data).toBeDefined(); @@ -247,23 +239,17 @@ describe("executor.runSingle", () => { expect(result.summary).toBeDefined(); }); - it("should throw error for unknown pipeline id", async () => { - await expect(executor.runSingle("unknown")).rejects.toThrow( - 'Pipeline "unknown" not found', - ); - }); - it("should return pipeline run result", async () => { - const result = await executor.runSingle("test-pipeline"); + const multi = await executor.run([pipeline]); + const result = multi.results.get("test-pipeline")!; expect(result.data.length).toBeGreaterThan(0); expect(result.errors).toEqual([]); }); it("should accept version filter", async () => { - const result = await executor.runSingle("test-pipeline", { - versions: ["16.0.0"], - }); + const multi = await executor.run([pipeline], { versions: ["16.0.0"] }); + const result = multi.results.get("test-pipeline")!; expect(result.summary.versions).toEqual(["16.0.0"]); }); @@ -271,23 +257,25 @@ describe("executor.runSingle", () => { it("should respect cache option", async () => { const cacheStore = createMemoryCacheStore(); - const executor = createPipelineExecutor({ - pipelines: [ - definePipeline({ - id: "cached-pipeline", - versions: ["16.0.0"], - inputs: [createTestSource(files, contents)], - routes: [createTestRoute("line-break", (ctx) => ctx.file.name === "LineBreak.txt")], - }), - ], - cacheStore, + const cachedPipeline = definePipeline({ + id: "cached-pipeline", + versions: ["16.0.0"], + inputs: [createTestSource(files, contents)], + routes: [createTestRoute("line-break", (ctx) => ctx.file.name === "LineBreak.txt")], }); - await executor.runSingle("cached-pipeline", { cache: true }); + const ex = createPipelineExecutor({ cacheStore }); + + await ex.run([cachedPipeline], { cache: true }); const stats = await cacheStore.stats?.(); expect(stats?.entries).toBeGreaterThanOrEqual(0); }); + + it("should return empty results for unknown pipeline", async () => { + const multi = await executor.run([]); + expect(multi.results.size).toBe(0); + }); }); describe("pipeline events", () => { @@ -304,12 +292,9 @@ describe("pipeline events", () => { routes: [createTestRoute("test", () => true)], }); - const executor = createPipelineExecutor({ - pipelines: [pipeline], - onEvent, - }); + const executor = createPipelineExecutor({ onEvent }); - await executor.runSingle("event-test"); + await executor.run([pipeline as any]); const eventTypes = events.map((e) => e.type); expect(eventTypes).toContain("pipeline:start"); @@ -327,11 +312,10 @@ describe("pipeline events", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], - onEvent: (event) => events.push(event), + onEvent: (event) => { events.push(event); return undefined; }, }); - await executor.runSingle("version-events"); + await executor.run([pipeline as any]); const eventTypes = events.map((e) => e.type); expect(eventTypes).toContain("version:start"); @@ -349,11 +333,10 @@ describe("pipeline events", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], - onEvent: (event) => events.push(event), + onEvent: (event) => { events.push(event); return undefined; }, }); - await executor.runSingle("parse-events"); + await executor.run([pipeline as any]); const eventTypes = events.map((e) => e.type); expect(eventTypes).toContain("parse:start"); @@ -373,11 +356,10 @@ describe("pipeline events", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], - onEvent: (event) => events.push(event), + onEvent: (event) => { events.push(event); return undefined; }, }); - await executor.runSingle("file-matched"); + await executor.run([pipeline as any]); const matchedEvents = events.filter((e) => e.type === "file:matched"); expect(matchedEvents.length).toBeGreaterThan(0); @@ -393,8 +375,9 @@ describe("pipeline graph", () => { routes: [createTestRoute("test", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("graph-test"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("graph-test")!; const sourceNodes = result.graph.nodes.filter((n) => n.type === "source"); expect(sourceNodes.length).toBe(1); @@ -408,8 +391,9 @@ describe("pipeline graph", () => { routes: [createTestRoute("test", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("graph-files"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("graph-files")!; const fileNodes = result.graph.nodes.filter((n) => n.type === "file"); expect(fileNodes.length).toBe(1); @@ -423,8 +407,9 @@ describe("pipeline graph", () => { routes: [createTestRoute("test", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("graph-routes"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("graph-routes")!; const routeNodes = result.graph.nodes.filter((n) => n.type === "route"); expect(routeNodes.length).toBe(1); @@ -438,8 +423,9 @@ describe("pipeline graph", () => { routes: [createTestRoute("test", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("graph-outputs"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("graph-outputs")!; const outputNodes = result.graph.nodes.filter((n) => n.type === "output"); expect(outputNodes.length).toBeGreaterThan(0); @@ -453,8 +439,9 @@ describe("pipeline graph", () => { routes: [createTestRoute("test", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("graph-edges"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("graph-edges")!; expect(result.graph.edges.length).toBeGreaterThan(0); }); @@ -480,8 +467,9 @@ describe("pipeline summary", () => { routes: [createTestRoute("all", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("summary-total"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("summary-total")!; expect(result.summary.totalFiles).toBe(3); }); @@ -505,8 +493,9 @@ describe("pipeline summary", () => { routes: [createTestRoute("match", (ctx) => ctx.file.name.startsWith("Match"))], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("summary-matched"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("summary-matched")!; expect(result.summary.matchedFiles).toBe(2); }); @@ -528,8 +517,9 @@ describe("pipeline summary", () => { routes: [createTestRoute("process", (ctx) => ctx.file.name === "Process.txt")], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("summary-skipped"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("summary-skipped")!; expect(result.summary.skippedFiles).toBe(1); }); @@ -542,8 +532,9 @@ describe("pipeline summary", () => { routes: [createTestRoute("test", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("summary-duration"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("summary-duration")!; expect(result.summary.durationMs).toBeGreaterThanOrEqual(0); }); @@ -565,8 +556,9 @@ describe("pipeline summary", () => { routes: [createTestRoute("all", () => true)], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("summary-outputs"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("summary-outputs")!; expect(result.summary.totalOutputs).toBe(2); }); @@ -590,12 +582,14 @@ describe("error handling", () => { routes: [failingRoute], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("error-test"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("error-test")!; expect(result.errors.length).toBeGreaterThan(0); - expect(result.errors[0].scope).toBe("route"); - expect(result.errors[0].message).toContain("Route failed"); + const firstError = result.errors[0]!; + expect(firstError.scope).toBe("route"); + expect(firstError.message).toContain("Route failed"); }); it("should emit error events", async () => { @@ -618,11 +612,10 @@ describe("error handling", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], - onEvent: (event) => events.push(event), + onEvent: (event) => { events.push(event); return undefined; }, }); - await executor.runSingle("error-events"); + await executor.run([pipeline]); const errorEvents = events.filter((e) => e.type === "error"); expect(errorEvents.length).toBeGreaterThan(0); @@ -636,11 +629,13 @@ describe("error handling", () => { routes: [], }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); + const executor = createPipelineExecutor({}); - await expect(executor.runSingle("no-inputs")).rejects.toThrow( - "Pipeline requires at least one input source", - ); + const multi = await executor.run([pipeline as any]); + const result = multi.results.get("no-inputs")!; + expect(result.errors.length).toBeGreaterThan(0); + const firstError = result.errors[0]!; + expect(firstError.message).toContain("Pipeline requires at least one input source"); }); }); @@ -662,12 +657,11 @@ describe("caching", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], cacheStore, - onEvent: (event) => events.push(event), + onEvent: (event) => { events.push(event); return undefined; }, }); - await executor.runSingle("cache-test", { cache: true }); + await executor.run([pipeline], { cache: true }); const cacheEvents = events.filter((e) => e.type === "cache:hit" || e.type === "cache:miss" || e.type === "cache:store", @@ -686,16 +680,15 @@ describe("caching", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], cacheStore, - onEvent: (event) => events.push(event), + onEvent: (event) => { events.push(event); return undefined; }, }); - await executor.runSingle("cache-hit-test", { cache: true }); + await executor.run([pipeline], { cache: true }); events.length = 0; - await executor.runSingle("cache-hit-test", { cache: true }); + await executor.run([pipeline], { cache: true }); const hitEvents = events.filter((e) => e.type === "cache:hit"); expect(hitEvents.length).toBeGreaterThan(0); @@ -712,12 +705,11 @@ describe("caching", () => { }); const executor = createPipelineExecutor({ - pipelines: [pipeline], cacheStore, - onEvent: (event) => events.push(event), + onEvent: (event) => { events.push(event); return undefined; }, }); - await executor.runSingle("cache-disabled", { cache: false }); + await executor.run([pipeline as any], { cache: false }); const cacheEvents = events.filter((e) => e.type === "cache:hit" || e.type === "cache:miss" || e.type === "cache:store", @@ -745,11 +737,9 @@ describe("multiple pipelines", () => { routes: [createTestRoute("route-2", () => true)], }); - const executor = createPipelineExecutor({ - pipelines: [pipeline1, pipeline2], - }); + const executor = createPipelineExecutor({}); - const result = await executor.run(); + const result = await executor.run([pipeline1 as any, pipeline2 as any]); expect(result.results.size).toBe(2); expect(result.results.has("pipeline-1")).toBe(true); @@ -783,11 +773,9 @@ describe("multiple pipelines", () => { ], }); - const executor = createPipelineExecutor({ - pipelines: [successPipeline, failPipeline], - }); + const executor = createPipelineExecutor({}); - const result = await executor.run(); + const result = await executor.run([successPipeline as any, failPipeline as any]); expect(result.summary.successfulPipelines).toBe(1); expect(result.summary.failedPipelines).toBe(1); @@ -813,12 +801,14 @@ describe("strict mode", () => { strict: true, }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("strict-test"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("strict-test")!; const fileErrors = result.errors.filter((e) => e.scope === "file"); expect(fileErrors.length).toBe(1); - expect(fileErrors[0].message).toContain("No matching route"); + const firstFileError = fileErrors[0]!; + expect(firstFileError.message).toContain("No matching route"); }); it("should not error on unmatched files when not strict", async () => { @@ -839,8 +829,9 @@ describe("strict mode", () => { strict: false, }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("non-strict-test"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("non-strict-test")!; const fileErrors = result.errors.filter((e) => e.scope === "file"); expect(fileErrors).toEqual([]); @@ -865,7 +856,7 @@ describe("fallback route", () => { routes: [createTestRoute("matched", (ctx) => ctx.file.name === "Matched.txt")], fallback: { parser: mockParser, - resolver: async (ctx, rows) => { + resolver: async (ctx: any, rows: AsyncIterable) => { const entries: Array<{ codePoint: string; value: string }> = []; for await (const row of rows) { entries.push({ codePoint: row.codePoint!, value: row.value as string }); @@ -873,10 +864,11 @@ describe("fallback route", () => { return { type: "fallback", file: ctx.file.name, entries }; }, }, - }); + }) as any; - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("fallback-test"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("fallback-test")!; expect(result.summary.fallbackFiles).toBe(1); expect(result.data.length).toBe(2); @@ -897,14 +889,11 @@ describe("fallback route", () => { parser: mockParser, resolver: async () => ({ fallback: true }), }, - }); + }) as any; - const executor = createPipelineExecutor({ - pipelines: [pipeline], - onEvent: (event) => events.push(event), - }); + const executor = createPipelineExecutor({ onEvent: (event) => { events.push(event); return undefined; } }); - await executor.runSingle("fallback-event"); + await executor.run([pipeline as any]); const fallbackEvents = events.filter((e) => e.type === "file:fallback"); expect(fallbackEvents.length).toBe(1); @@ -930,8 +919,9 @@ describe("include filter", () => { include: (ctx) => ctx.file.name.startsWith("Include"), }); - const executor = createPipelineExecutor({ pipelines: [pipeline] }); - const result = await executor.runSingle("include-test"); + const executor = createPipelineExecutor({}); + const multi = await executor.run([pipeline]); + const result = multi.results.get("include-test")!; expect(result.summary.matchedFiles).toBe(1); expect(result.data.length).toBe(1); diff --git a/packages/pipelines/pipeline-graph/package.json b/packages/pipelines/pipeline-graph/package.json index f070e15b1..a1ead8b4f 100644 --- a/packages/pipelines/pipeline-graph/package.json +++ b/packages/pipelines/pipeline-graph/package.json @@ -22,8 +22,6 @@ ".": "./dist/index.mjs", "./package.json": "./package.json" }, - "main": "./dist/index.mjs", - "module": "./dist/index.mjs", "types": "./dist/index.d.mts", "files": [ "dist" diff --git a/packages/pipelines/pipeline-graph/src/builder.ts b/packages/pipelines/pipeline-graph/src/builder.ts new file mode 100644 index 000000000..d63a8c053 --- /dev/null +++ b/packages/pipelines/pipeline-graph/src/builder.ts @@ -0,0 +1,106 @@ +import type { + DAG, + FileContext, + PipelineDefinition, + PipelineGraph, + PipelineGraphEdge, + PipelineGraphNode, +} from "@ucdjs/pipelines-core"; + +export interface GraphBuilderOptions { + includeArtifacts?: boolean; +} + +export class PipelineGraphBuilder { + private nodes: Map = new Map(); + private edges: PipelineGraphEdge[] = []; + + addSourceNode(version: string): string { + const id = `source:${version}`; + if (!this.nodes.has(id)) { + this.nodes.set(id, { id, type: "source", version }); + } + return id; + } + + addFileNode(file: FileContext): string { + const id = `file:${file.version}:${file.path}`; + if (!this.nodes.has(id)) { + this.nodes.set(id, { id, type: "file", file }); + } + return id; + } + + addRouteNode(routeId: string, version: string): string { + const id = `route:${version}:${routeId}`; + if (!this.nodes.has(id)) { + this.nodes.set(id, { id, type: "route", routeId }); + } + return id; + } + + addArtifactNode(artifactId: string, version: string): string { + const id = `artifact:${version}:${artifactId}`; + if (!this.nodes.has(id)) { + this.nodes.set(id, { id, type: "artifact", artifactId }); + } + return id; + } + + addOutputNode(outputIndex: number, version: string, property?: string): string { + const id = `output:${version}:${outputIndex}`; + if (!this.nodes.has(id)) { + this.nodes.set(id, { id, type: "output", outputIndex, property }); + } + return id; + } + + addEdge(from: string, to: string, type: PipelineGraphEdge["type"]): void { + const exists = this.edges.some((e) => e.from === from && e.to === to && e.type === type); + if (!exists) { + this.edges.push({ from, to, type }); + } + } + + build(): PipelineGraph { + return { + nodes: Array.from(this.nodes.values()), + edges: [...this.edges], + }; + } + + clear(): void { + this.nodes.clear(); + this.edges.length = 0; + } +} + +export function buildRouteGraph( + pipeline: PipelineDefinition, + dag: DAG, +): PipelineGraph { + const builder = new PipelineGraphBuilder(); + + for (const route of pipeline.routes) { + const routeNode = dag.nodes.get(route.id); + if (!routeNode) continue; + + builder.addRouteNode(route.id, "static"); + + for (const depId of routeNode.dependencies) { + builder.addRouteNode(depId, "static"); + builder.addEdge(`route:static:${depId}`, `route:static:${route.id}`, "provides"); + } + + for (const artifactId of routeNode.emittedArtifacts) { + builder.addArtifactNode(artifactId, "static"); + builder.addEdge(`route:static:${route.id}`, `artifact:static:${artifactId}`, "resolved"); + } + } + + return builder.build(); +} + +export function createPipelineGraphBuilder(): PipelineGraphBuilder { + return new PipelineGraphBuilder(); +} diff --git a/packages/pipelines/pipeline-graph/src/index.ts b/packages/pipelines/pipeline-graph/src/index.ts index 683b235cf..3f4f3058a 100644 --- a/packages/pipelines/pipeline-graph/src/index.ts +++ b/packages/pipelines/pipeline-graph/src/index.ts @@ -1,11 +1,9 @@ export type { - DAG, - DAGNode, - DAGValidationError, - DAGValidationResult, -} from "./dag"; + GraphBuilderOptions, +} from "./builder"; export { - buildDAG, - getExecutionLayers, -} from "./dag"; + buildRouteGraph, + createPipelineGraphBuilder, + PipelineGraphBuilder, +} from "./builder"; diff --git a/packages/pipelines/pipeline-loader/package.json b/packages/pipelines/pipeline-loader/package.json index c1ef37072..63c013b58 100644 --- a/packages/pipelines/pipeline-loader/package.json +++ b/packages/pipelines/pipeline-loader/package.json @@ -22,8 +22,6 @@ ".": "./dist/index.mjs", "./package.json": "./package.json" }, - "main": "./dist/index.mjs", - "module": "./dist/index.mjs", "types": "./dist/index.d.mts", "files": [ "dist" @@ -39,7 +37,8 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { - "@ucdjs/pipelines-core": "workspace:*" + "@ucdjs/pipelines-core": "workspace:*", + "tinyglobby": "catalog:prod" }, "devDependencies": { "@luxass/eslint-config": "catalog:linting", diff --git a/packages/pipelines/pipeline-loader/src/find.ts b/packages/pipelines/pipeline-loader/src/find.ts new file mode 100644 index 000000000..00b1feb24 --- /dev/null +++ b/packages/pipelines/pipeline-loader/src/find.ts @@ -0,0 +1,24 @@ +import { glob } from "tinyglobby"; + +/** + * Find pipeline files on disk. + * + * By default matches files named `*.ucd-pipeline.ts` (the repository standard). + * + * @param {string | string[]} patterns glob string or array of glob strings + * @param {string} cwd optional working directory (defaults to process.cwd()) + */ +export async function findPipelineFiles( + patterns: string | string[] = ["**/*.ucd-pipeline.ts"], + cwd?: string, +): Promise { + const p = Array.isArray(patterns) ? patterns : [patterns]; + + return glob(p, { + // eslint-disable-next-line node/prefer-global/process + cwd: cwd ?? process.cwd(), + ignore: ["node_modules/**", "**/node_modules/**", "**/dist/**", "**/build/**", "**/.git/**"], + absolute: true, + onlyFiles: true, + }); +} diff --git a/packages/pipelines/pipeline-loader/src/index.ts b/packages/pipelines/pipeline-loader/src/index.ts index 9b6f37e29..64aec795e 100644 --- a/packages/pipelines/pipeline-loader/src/index.ts +++ b/packages/pipelines/pipeline-loader/src/index.ts @@ -1,3 +1,5 @@ +export { findPipelineFiles } from "./find"; + export { loadPipelineFile, loadPipelines, diff --git a/packages/pipelines/pipeline-playground/package.json b/packages/pipelines/pipeline-playground/package.json new file mode 100644 index 000000000..3f3767720 --- /dev/null +++ b/packages/pipelines/pipeline-playground/package.json @@ -0,0 +1,40 @@ +{ + "name": "@ucdjs/pipelines-playground", + "version": "0.0.0", + "private": true, + "type": "module", + "author": { + "name": "Lucas Nørgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.28.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-playground" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "exports": { + "./package.json": "./package.json" + }, + "publishConfig": { + "access": "restricted" + }, + "files": [], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "pipelines:list": "ucd pipelines list --cwd ." + }, + "dependencies": { + "@ucdjs/pipelines-core": "workspace:*", + "@ucdjs/pipelines-presets": "workspace:*" + }, + "devDependencies": {} +} diff --git a/packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts b/packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts new file mode 100644 index 000000000..7e12d3965 --- /dev/null +++ b/packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts @@ -0,0 +1,28 @@ +import { byName, definePipeline } from "@ucdjs/pipelines-core"; +import { createMemorySource, sequenceParser, propertyJsonResolver } from "@ucdjs/pipelines-presets"; + +export default definePipeline({ + id: "playground-sequence", + name: "Playground Sequence", + versions: ["16.0.0"], + inputs: [ + createMemorySource({ + files: { + "16.0.0": [ + { + path: "ucd/Sequences.txt", + content: "0041 0308; A_DIAERESIS\n006F 0308; O_DIAERESIS\n", + }, + ], + }, + }), + ], + routes: [ + { + id: "sequence-route", + filter: byName("Sequences.txt"), + parser: sequenceParser, + resolver: propertyJsonResolver, + }, + ], +}); diff --git a/packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts b/packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts new file mode 100644 index 000000000..427f6f992 --- /dev/null +++ b/packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts @@ -0,0 +1,28 @@ +import { always, definePipeline, definePipelineRoute } from "@ucdjs/pipelines-core"; +import { createMemorySource, standardParser, propertyJsonResolver } from "@ucdjs/pipelines-presets"; + +export default definePipeline({ + id: "playground-simple", + name: "Playground Simple", + versions: ["16.0.0"], + inputs: [ + createMemorySource({ + files: { + "16.0.0": [ + { + path: "ucd/Hello.txt", + content: "0048; H\n0065; e\n006C; l\n006C; l\n006F; o\n", + }, + ], + }, + }), + ], + routes: [ + definePipelineRoute({ + id: "hello", + filter: always(), + parser: standardParser, + resolver: propertyJsonResolver, + }), + ], +}); diff --git a/packages/pipelines/pipeline-playground/tsconfig.build.json b/packages/pipelines/pipeline-playground/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-playground/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-playground/tsconfig.json b/packages/pipelines/pipeline-playground/tsconfig.json new file mode 100644 index 000000000..ebce38306 --- /dev/null +++ b/packages/pipelines/pipeline-playground/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src" + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-playground/tsdown.config.ts b/packages/pipelines/pipeline-playground/tsdown.config.ts new file mode 100644 index 000000000..324776b47 --- /dev/null +++ b/packages/pipelines/pipeline-playground/tsdown.config.ts @@ -0,0 +1,9 @@ +import { createTsdownConfig } from "@ucdjs-tooling/tsdown-config"; + +export default createTsdownConfig({ + entry: [ + "./src/simple.ucd-pipeline.ts", + "./src/memory.ucd-pipeline.ts", + "./src/sequence.ucd-pipeline.ts", + ], +}); diff --git a/packages/pipelines/pipeline-presets/package.json b/packages/pipelines/pipeline-presets/package.json index a428e626d..0e50df06d 100644 --- a/packages/pipelines/pipeline-presets/package.json +++ b/packages/pipelines/pipeline-presets/package.json @@ -1,5 +1,5 @@ { - "name": "@ucdjs/pipeline-presets", + "name": "@ucdjs/pipelines-presets", "version": "0.0.1", "type": "module", "author": { @@ -21,15 +21,13 @@ "exports": { ".": "./dist/index.mjs", "./parsers": "./dist/parsers/index.mjs", - "./transforms": "./dist/transforms/index.mjs", + "./pipelines": "./dist/pipelines/index.mjs", "./resolvers": "./dist/resolvers/index.mjs", - "./sources": "./dist/sources/index.mjs", "./routes": "./dist/routes/index.mjs", - "./pipelines": "./dist/pipelines/index.mjs", + "./sources": "./dist/sources/index.mjs", + "./transforms": "./dist/transforms/index.mjs", "./package.json": "./package.json" }, - "main": "./dist/index.mjs", - "module": "./dist/index.mjs", "types": "./dist/index.d.mts", "files": [ "dist" diff --git a/packages/pipelines/pipeline-presets/src/parsers/standard.ts b/packages/pipelines/pipeline-presets/src/parsers/standard.ts index 7d5044b61..bb540fa21 100644 --- a/packages/pipelines/pipeline-presets/src/parsers/standard.ts +++ b/packages/pipelines/pipeline-presets/src/parsers/standard.ts @@ -11,7 +11,7 @@ function parseCodePointOrRange(field: string): { kind: ParsedRow["kind"]; start? if (trimmed.includes("..")) { const [start, end] = trimmed.split(".."); - return { kind: "range", start: start.trim(), end: end.trim() }; + return { kind: "range", start: start!.trim(), end: end!.trim() }; } return { kind: "point", codePoint: trimmed }; @@ -43,10 +43,10 @@ export function createStandardParser(options: StandardParserOptions = {}): Parse continue; } - const codePointField = trimFields ? fields[0].trim() : fields[0]; - const valueField = trimFields ? fields[1].trim() : fields[1]; + const codePointField = trimFields ? fields[0]!.trim() : fields[0]; + const valueField = trimFields ? fields[1]!.trim() : fields[1]; - const { kind, start, end, codePoint } = parseCodePointOrRange(codePointField); + const { kind, start, end, codePoint } = parseCodePointOrRange(codePointField!); yield { sourceFile: ctx.file.path, diff --git a/packages/pipelines/pipeline-presets/src/pipelines/basic.ts b/packages/pipelines/pipeline-presets/src/pipelines/basic.ts index 5a604053a..227b2ebf0 100644 --- a/packages/pipelines/pipeline-presets/src/pipelines/basic.ts +++ b/packages/pipelines/pipeline-presets/src/pipelines/basic.ts @@ -1,4 +1,3 @@ -import type { PipelineDefinition } from "@ucdjs/pipelines-core"; import { byExt, definePipeline } from "@ucdjs/pipelines-core"; import { standardParser } from "../parsers/standard"; import { propertyJsonResolver } from "../resolvers/property-json"; diff --git a/packages/pipelines/pipeline-presets/src/pipelines/emoji.ts b/packages/pipelines/pipeline-presets/src/pipelines/emoji.ts index f9fdb17e8..949d304cf 100644 --- a/packages/pipelines/pipeline-presets/src/pipelines/emoji.ts +++ b/packages/pipelines/pipeline-presets/src/pipelines/emoji.ts @@ -1,4 +1,3 @@ -import type { PipelineDefinition } from "@ucdjs/pipelines-core"; import { and, byDir, byExt, definePipeline } from "@ucdjs/pipelines-core"; import { sequenceParser } from "../parsers/sequence"; import { propertyJsonResolver } from "../resolvers/property-json"; diff --git a/packages/pipelines/pipeline-presets/src/pipelines/full.ts b/packages/pipelines/pipeline-presets/src/pipelines/full.ts index 08cb17e1a..256fae8d8 100644 --- a/packages/pipelines/pipeline-presets/src/pipelines/full.ts +++ b/packages/pipelines/pipeline-presets/src/pipelines/full.ts @@ -1,4 +1,3 @@ -import type { PipelineDefinition } from "@ucdjs/pipelines-core"; import { byExt, definePipeline } from "@ucdjs/pipelines-core"; import { standardParser } from "../parsers/standard"; import { propertyJsonResolver } from "../resolvers/property-json"; diff --git a/packages/pipelines/pipeline-presets/src/resolvers/property-json.ts b/packages/pipelines/pipeline-presets/src/resolvers/property-json.ts index dcc6216f8..945bf436a 100644 --- a/packages/pipelines/pipeline-presets/src/resolvers/property-json.ts +++ b/packages/pipelines/pipeline-presets/src/resolvers/property-json.ts @@ -1,4 +1,4 @@ -import type { ParsedRow, PropertyJson, ResolvedEntry, RouteResolveContext } from "@ucdjs/pipelines-core"; +import type { ArtifactDefinition, ParsedRow, PropertyJson, ResolvedEntry, RouteResolveContext } from "@ucdjs/pipelines-core"; export interface PropertyJsonResolverOptions { property?: string; @@ -36,8 +36,11 @@ function rowToResolvedEntry(row: ParsedRow): ResolvedEntry | null { } export function createPropertyJsonResolver(options: PropertyJsonResolverOptions = {}) { - return async function propertyJsonResolver( - ctx: RouteResolveContext, + return async function propertyJsonResolver< + TArtifactKeys extends string, + TEmits extends Record, + >( + ctx: RouteResolveContext, rows: AsyncIterable, ): Promise { const entries: ResolvedEntry[] = []; diff --git a/packages/pipelines/pipeline-presets/src/routes/common.ts b/packages/pipelines/pipeline-presets/src/routes/common.ts index 13e2b3a33..1e307fcf7 100644 --- a/packages/pipelines/pipeline-presets/src/routes/common.ts +++ b/packages/pipelines/pipeline-presets/src/routes/common.ts @@ -1,12 +1,9 @@ -import type { PipelineRouteDefinition } from "@ucdjs/pipelines-core"; import { - byDir, byGlob, byName, definePipelineRoute, } from "@ucdjs/pipelines-core"; -import { multiPropertyParser } from "../parsers/multi-property"; import { standardParser } from "../parsers/standard"; import { unicodeDataParser } from "../parsers/unicode-data"; import { createGroupedResolver } from "../resolvers/grouped"; diff --git a/packages/pipelines/pipeline-ui/package.json b/packages/pipelines/pipeline-ui/package.json index db174e5b0..1897ab133 100644 --- a/packages/pipelines/pipeline-ui/package.json +++ b/packages/pipelines/pipeline-ui/package.json @@ -39,8 +39,7 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { - "@ucdjs/pipelines-core": "workspace:*", - "@ucdjs/pipelines-graph": "workspace:*" + "@ucdjs/pipelines-core": "workspace:*" }, "devDependencies": { "@luxass/eslint-config": "catalog:linting", diff --git a/packages/pipelines/pipeline-ui/src/index.ts b/packages/pipelines/pipeline-ui/src/index.ts index 3d9d474a1..57da5a903 100644 --- a/packages/pipelines/pipeline-ui/src/index.ts +++ b/packages/pipelines/pipeline-ui/src/index.ts @@ -1,5 +1,4 @@ -import type { PipelineDefinition } from "@ucdjs/pipelines-core"; -import type { DAG } from "@ucdjs/pipelines-graph"; +import type { DAG, PipelineDefinition } from "@ucdjs/pipelines-core"; export interface GraphVisualizationOptions { format?: "mermaid" | "dot" | "json"; diff --git a/tooling/tsconfig/base.json b/tooling/tsconfig/base.json index 3dc7a87db..b335cbcf4 100644 --- a/tooling/tsconfig/base.json +++ b/tooling/tsconfig/base.json @@ -59,6 +59,7 @@ "@ucdjs/pipelines-executor": ["./packages/pipelines/pipeline-executor/src/index.ts"], "@ucdjs/pipelines-loader": ["./packages/pipelines/pipeline-loader/src/index.ts"], "@ucdjs/pipelines-ui": ["./packages/pipelines/pipeline-ui/src/index.ts"], + "@ucdjs/pipelines-presets": ["./packages/pipelines/pipeline-presets/src/index.ts"], // Test utils "@ucdjs/test-utils": ["./packages/test-utils/src/index.ts"], From 135cf05fe2eec8c7fd8fcce1a0ebabe8f1eeaa9a Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 20 Jan 2026 13:08:26 +0100 Subject: [PATCH 11/63] feat: enhance pipeline definition options and type inference - Updated `definePipeline` to use `Omit` for better type safety. - Adjusted type inference tests to validate expected types. - Added `definePipelineRoute` to the playground example for improved routing functionality. --- packages/pipelines/pipeline-core/src/pipeline.ts | 11 ++++++++--- packages/pipelines/pipeline-core/src/route.ts | 2 +- .../pipelines/pipeline-core/test/route.test.ts | 10 ++++------ .../pipelines/pipeline-core/test/source.test.ts | 14 +++++--------- .../src/sequence.ucd-pipeline.ts | 6 +++--- .../pipeline-playground/src/simple.ucd-pipeline.ts | 1 + 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/pipelines/pipeline-core/src/pipeline.ts b/packages/pipelines/pipeline-core/src/pipeline.ts index 3f0767aaf..cd8548a09 100644 --- a/packages/pipelines/pipeline-core/src/pipeline.ts +++ b/packages/pipelines/pipeline-core/src/pipeline.ts @@ -228,7 +228,12 @@ export function definePipeline< const TRoutes extends readonly PipelineRouteDefinition[], TFallback extends FallbackRouteDefinition | undefined = undefined, >( - options: PipelineDefinitionOptions & { id: TId }, + options: Omit, "inputs" | "routes"> + & { + id: TId; + inputs: readonly [...TSources]; + routes: readonly [...TRoutes]; + }, ): PipelineDefinition { const dagResult = buildDAG(options.routes); @@ -243,8 +248,8 @@ export function definePipeline< name: options.name, description: options.description, versions: options.versions, - inputs: options.inputs, - routes: options.routes, + inputs: options.inputs as TSources, + routes: options.routes as TRoutes, include: options.include, strict: options.strict ?? false, concurrency: options.concurrency ?? 4, diff --git a/packages/pipelines/pipeline-core/src/route.ts b/packages/pipelines/pipeline-core/src/route.ts index 8aa718b55..59ba48e4b 100644 --- a/packages/pipelines/pipeline-core/src/route.ts +++ b/packages/pipelines/pipeline-core/src/route.ts @@ -39,7 +39,7 @@ export interface PipelineRouteDefinition< TId extends string = string, TDepends extends readonly PipelineDependency[] = readonly PipelineDependency[], TEmits extends Record = Record, - TTransforms extends readonly PipelineTransformDefinition[] = readonly PipelineTransformDefinition[], + TTransforms extends readonly PipelineTransformDefinition[] = readonly [], TOutput = PropertyJson[], > { id: TId; diff --git a/packages/pipelines/pipeline-core/test/route.test.ts b/packages/pipelines/pipeline-core/test/route.test.ts index 4ff52ef38..f44596abd 100644 --- a/packages/pipelines/pipeline-core/test/route.test.ts +++ b/packages/pipelines/pipeline-core/test/route.test.ts @@ -402,6 +402,7 @@ describe("type inference", () => { describe("inferRouteDepends", () => { it("should infer route dependencies", () => { + // eslint-disable-next-line unused-imports/no-unused-vars const route = definePipelineRoute({ id: "test", filter: () => true, @@ -411,9 +412,7 @@ describe("type inference", () => { }); type Depends = InferRouteDepends; - const deps: Depends = ["route:dep1", "artifact:route:artifact"]; - - expect(deps).toHaveLength(2); + expectTypeOf().toEqualTypeOf(); }); }); @@ -444,6 +443,7 @@ describe("type inference", () => { describe("inferRouteOutput", () => { it("should infer route output type", () => { + // eslint-disable-next-line unused-imports/no-unused-vars const route = definePipelineRoute({ id: "test", filter: () => true, @@ -452,9 +452,7 @@ describe("type inference", () => { }); type Output = InferRouteOutput; - const output: Output = []; - - expect(output).toEqual([]); + expectTypeOf().toEqualTypeOf(); }); }); }); diff --git a/packages/pipelines/pipeline-core/test/source.test.ts b/packages/pipelines/pipeline-core/test/source.test.ts index 2b3cb497b..1dce1fc53 100644 --- a/packages/pipelines/pipeline-core/test/source.test.ts +++ b/packages/pipelines/pipeline-core/test/source.test.ts @@ -4,7 +4,7 @@ import type { SourceBackend, } from "../src/source"; import type { FileContext } from "../src/types"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, expectTypeOf, it, vi } from "vitest"; import { definePipelineSource, resolveMultipleSourceFiles, @@ -301,31 +301,27 @@ describe("resolveMultipleSourceFiles", () => { describe("type inference", () => { describe("inferSourceId", () => { it("should infer source id type", () => { + // eslint-disable-next-line unused-imports/no-unused-vars const source = definePipelineSource({ id: "my-source", backend: createMockBackend([]), }); type SourceId = InferSourceId; - const id: SourceId = "my-source"; - - expect(id).toBe("my-source"); + expectTypeOf().toEqualTypeOf<"my-source">(); }); }); describe("inferSourceIds", () => { it("should infer multiple source ids", () => { + // eslint-disable-next-line unused-imports/no-unused-vars const sources = [ definePipelineSource({ id: "source1", backend: createMockBackend([]) }), definePipelineSource({ id: "source2", backend: createMockBackend([]) }), ] as const; type SourceIds = InferSourceIds; - const id1: SourceIds = "source1"; - const id2: SourceIds = "source2"; - - expect(id1).toBe("source1"); - expect(id2).toBe("source2"); + expectTypeOf().toEqualTypeOf<"source1" | "source2">(); }); }); }); diff --git a/packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts b/packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts index 7e12d3965..a0917be93 100644 --- a/packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts +++ b/packages/pipelines/pipeline-playground/src/sequence.ucd-pipeline.ts @@ -1,4 +1,4 @@ -import { byName, definePipeline } from "@ucdjs/pipelines-core"; +import { byName, definePipeline, definePipelineRoute } from "@ucdjs/pipelines-core"; import { createMemorySource, sequenceParser, propertyJsonResolver } from "@ucdjs/pipelines-presets"; export default definePipeline({ @@ -18,11 +18,11 @@ export default definePipeline({ }), ], routes: [ - { + definePipelineRoute({ id: "sequence-route", filter: byName("Sequences.txt"), parser: sequenceParser, resolver: propertyJsonResolver, - }, + }), ], }); diff --git a/packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts b/packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts index 427f6f992..33264d465 100644 --- a/packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts +++ b/packages/pipelines/pipeline-playground/src/simple.ucd-pipeline.ts @@ -26,3 +26,4 @@ export default definePipeline({ }), ], }); + From b7701388b494ff6f7584e682de714816f83fda7e Mon Sep 17 00:00:00 2001 From: Lucas Date: Sat, 31 Jan 2026 19:11:04 +0100 Subject: [PATCH 12/63] chore: remove main and module fields from package.json --- .../pipelines/pipeline-executor/package.json | 2 - packages/pipelines/pipeline-ui/package.json | 2 - pnpm-lock.yaml | 198 ++++-------------- 3 files changed, 43 insertions(+), 159 deletions(-) diff --git a/packages/pipelines/pipeline-executor/package.json b/packages/pipelines/pipeline-executor/package.json index 70da5e1df..6476afd7f 100644 --- a/packages/pipelines/pipeline-executor/package.json +++ b/packages/pipelines/pipeline-executor/package.json @@ -22,8 +22,6 @@ ".": "./dist/index.mjs", "./package.json": "./package.json" }, - "main": "./dist/index.mjs", - "module": "./dist/index.mjs", "types": "./dist/index.d.mts", "files": [ "dist" diff --git a/packages/pipelines/pipeline-ui/package.json b/packages/pipelines/pipeline-ui/package.json index 1897ab133..543cae955 100644 --- a/packages/pipelines/pipeline-ui/package.json +++ b/packages/pipelines/pipeline-ui/package.json @@ -22,8 +22,6 @@ ".": "./dist/index.mjs", "./package.json": "./package.json" }, - "main": "./dist/index.mjs", - "module": "./dist/index.mjs", "types": "./dist/index.d.mts", "files": [ "dist" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e54b80163..b43fb7b9c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -131,6 +131,9 @@ catalogs: picomatch: specifier: 4.0.3 version: 4.0.3 + tinyglobby: + specifier: 0.2.15 + version: 0.2.15 yargs-parser: specifier: 22.0.0 version: 22.0.0 @@ -666,6 +669,9 @@ importers: '@ucdjs/lockfile': specifier: workspace:* version: link:../lockfile + '@ucdjs/pipelines-loader': + specifier: workspace:* + version: link:../pipelines/pipeline-loader '@ucdjs/schema-gen': specifier: workspace:* version: link:../schema-gen @@ -1010,9 +1016,6 @@ importers: '@ucdjs/pipelines-core': specifier: workspace:* version: link:../pipeline-core - '@ucdjs/pipelines-graph': - specifier: workspace:* - version: link:../pipeline-graph devDependencies: '@luxass/eslint-config': specifier: catalog:linting @@ -1069,6 +1072,9 @@ importers: '@ucdjs/pipelines-core': specifier: workspace:* version: link:../pipeline-core + tinyglobby: + specifier: catalog:prod + version: 0.2.15 devDependencies: '@luxass/eslint-config': specifier: catalog:linting @@ -1092,6 +1098,15 @@ importers: specifier: catalog:dev version: 5.9.3 + packages/pipelines/pipeline-playground: + dependencies: + '@ucdjs/pipelines-core': + specifier: workspace:* + version: link:../pipeline-core + '@ucdjs/pipelines-presets': + specifier: workspace:* + version: link:../pipeline-presets + packages/pipelines/pipeline-presets: dependencies: '@ucdjs/pipelines-core': @@ -1099,11 +1114,11 @@ importers: version: link:../pipeline-core zod: specifier: catalog:prod - version: 4.3.5 + version: 4.3.6 devDependencies: '@luxass/eslint-config': specifier: catalog:linting - version: 7.0.0-beta.2(@eslint-react/eslint-plugin@2.7.2(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig @@ -1115,10 +1130,10 @@ importers: version: 9.39.2(jiti@2.6.1) publint: specifier: catalog:dev - version: 0.3.16 + version: 0.3.17 tsdown: specifier: catalog:dev - version: 0.19.0(publint@0.3.16)(synckit@0.11.11)(typescript@5.9.3) + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) typescript: specifier: catalog:dev version: 5.9.3 @@ -1128,9 +1143,6 @@ importers: '@ucdjs/pipelines-core': specifier: workspace:* version: link:../pipeline-core - '@ucdjs/pipelines-graph': - specifier: workspace:* - version: link:../pipeline-graph devDependencies: '@luxass/eslint-config': specifier: catalog:linting @@ -2023,7 +2035,7 @@ packages: optional: true '@cloudflare/vitest-pool-workers@https://pkg.pr.new/@cloudflare/vitest-pool-workers@11632': - resolution: {integrity: sha512-NmzXfKgEcR1I9xQMeQGZODB9N4wBzTbl9PJMxLzeq5smYatje18ZRDxeN2P4bS3pRCa6XI/aRNTAEYyAR4C4ew==, tarball: https://pkg.pr.new/@cloudflare/vitest-pool-workers@11632} + resolution: {tarball: https://pkg.pr.new/@cloudflare/vitest-pool-workers@11632} version: 0.12.6 peerDependencies: '@vitest/runner': 4.1.0-beta.1 @@ -2626,10 +2638,6 @@ packages: resolution: {integrity: sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/core@1.0.1': - resolution: {integrity: sha512-r18fEAj9uCk+VjzGt2thsbOmychS+4kxI14spVNibUO2vqKX7obOG+ymZljAwuPZl+S3clPGwCwTDtrdqTiY6Q==} - engines: {node: ^20.19.0 || ^22.13.0 || >=24} - '@eslint/core@1.1.0': resolution: {integrity: sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} @@ -4828,32 +4836,16 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/project-service@8.53.1': - resolution: {integrity: sha512-WYC4FB5Ra0xidsmlPb+1SsnaSKPmS3gsjIARwbEkHkoWloQmuzcfypljaJcR78uyLA1h8sHdWWPHSLDI+MtNog==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - peerDependencies: - typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/project-service@8.54.0': resolution: {integrity: sha512-YPf+rvJ1s7MyiWM4uTRhE4DvBXrEV+d8oC3P9Y2eT7S+HBS0clybdMIPnhiATi9vZOYDc7OQ1L/i6ga6NFYK/g==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/scope-manager@8.53.1': - resolution: {integrity: sha512-Lu23yw1uJMFY8cUeq7JlrizAgeQvWugNQzJp8C3x8Eo5Jw5Q2ykMdiiTB9vBVOOUBysMzmRRmUfwFrZuI2C4SQ==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/scope-manager@8.54.0': resolution: {integrity: sha512-27rYVQku26j/PbHYcVfRPonmOlVI6gihHtXFbTdB5sb6qA0wdAQAbyXFVarQ5t4HRojIz64IV90YtsjQSSGlQg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/tsconfig-utils@8.53.1': - resolution: {integrity: sha512-qfvLXS6F6b1y43pnf0pPbXJ+YoXIC7HKg0UGZ27uMIemKMKA6XH2DTxsEDdpdN29D+vHV07x/pnlPNVLhdhWiA==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - peerDependencies: - typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/tsconfig-utils@8.54.0': resolution: {integrity: sha512-dRgOyT2hPk/JwxNMZDsIXDgyl9axdJI3ogZ2XWhBPsnZUv+hPesa5iuhdYt2gzwA9t8RE5ytOJ6xB0moV0Ujvw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -4867,33 +4859,16 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/types@8.53.1': - resolution: {integrity: sha512-jr/swrr2aRmUAUjW5/zQHbMaui//vQlsZcJKijZf3M26bnmLj8LyZUpj8/Rd6uzaek06OWsqdofN/Thenm5O8A==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/types@8.54.0': resolution: {integrity: sha512-PDUI9R1BVjqu7AUDsRBbKMtwmjWcn4J3le+5LpcFgWULN3LvHC5rkc9gCVxbrsrGmO1jfPybN5s6h4Jy+OnkAA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/typescript-estree@8.53.1': - resolution: {integrity: sha512-RGlVipGhQAG4GxV1s34O91cxQ/vWiHJTDHbXRr0li2q/BGg3RR/7NM8QDWgkEgrwQYCvmJV9ichIwyoKCQ+DTg==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - peerDependencies: - typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/typescript-estree@8.54.0': resolution: {integrity: sha512-BUwcskRaPvTk6fzVWgDPdUndLjB87KYDrN5EYGetnktoeAvPtO4ONHlAZDnj5VFnUANg0Sjm7j4usBlnoVMHwA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/utils@8.53.1': - resolution: {integrity: sha512-c4bMvGVWW4hv6JmDUEG7fSYlWOl3II2I4ylt0NM+seinYQlZMQIaKaXIIVJWt9Ofh6whrpM+EdDQXKXjNovvrg==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - peerDependencies: - eslint: ^8.57.0 || ^9.0.0 - typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/utils@8.54.0': resolution: {integrity: sha512-9Cnda8GS57AQakvRyG0PTejJNlA2xhvyNtEVIMlDWOOeEyBkYWhGPnfrIAnqxLMTSTo6q8g12XVjjev5l1NvMA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -4901,10 +4876,6 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/visitor-keys@8.53.1': - resolution: {integrity: sha512-oy+wV7xDKFPRyNggmXuZQSBzvoLnpmJs+GhzRhPjrxl2b/jIlyjVokzm47CZCDUdXKr2zd7ZLodPfOBpOPyPlg==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@typescript-eslint/visitor-keys@8.54.0': resolution: {integrity: sha512-VFlhGSl4opC0bprJiItPQ1RfUhGDIBokcPwaFH4yiBCaNPeld/9VeXbiPO1cLyorQi1G1vL+ecBk1x8o1axORA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -5518,10 +5489,6 @@ packages: resolution: {integrity: sha512-buhp5kePrmda3vhc5B9t7pUQXAb2Tnd0qgpkIhPhkHXxJpiPJ11H0ZEU0oBpJ2QztSbzG/ZxMj/CHsYJqRHmyg==} engines: {node: '>= 12.0.0'} - comment-parser@1.4.4: - resolution: {integrity: sha512-0D6qSQ5IkeRrGJFHRClzaMOenMeT0gErz3zIw3AprKMqhRN6LNU2jQOdkPG/FZ+8bCgXE1VidrgSzlBBDZRr8A==} - engines: {node: '>= 12.0.0'} - comment-parser@1.4.5: resolution: {integrity: sha512-aRDkn3uyIlCFfk5NUA+VdwMmMsh8JGhc4hapfV4yxymHGQ3BVskMQfoXGpCo5IoBuQ9tS5iiVKhCpTcB4pW4qw==} engines: {node: '>= 12.0.0'} @@ -8998,10 +8965,6 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} - type-fest@5.3.0: - resolution: {integrity: sha512-d9CwU93nN0IA1QL+GSNDdwLAu1Ew5ZjTwupvedwg3WdfoH6pIDvYQ2hV0Uc2nKBLPq7NB5apCx57MLS5qlmO5g==} - engines: {node: '>=20'} - type-fest@5.4.3: resolution: {integrity: sha512-AXSAQJu79WGc79/3e9/CR77I/KQgeY1AhNvcShIH4PTcGYyC4xv6H4R4AUOwkPS5799KlVDAu8zExeCrkGquiA==} engines: {node: '>=20'} @@ -9092,9 +9055,6 @@ packages: unist-util-visit-parents@6.0.2: resolution: {integrity: sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==} - unist-util-visit@5.0.0: - resolution: {integrity: sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==} - unist-util-visit@5.1.0: resolution: {integrity: sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==} @@ -9647,9 +9607,6 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.3.5: - resolution: {integrity: sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==} - zod@4.3.6: resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} @@ -10273,7 +10230,7 @@ snapshots: '@es-joy/jsdoccomment@0.78.0': dependencies: '@types/estree': 1.0.8 - '@typescript-eslint/types': 8.53.1 + '@typescript-eslint/types': 8.54.0 comment-parser: 1.4.1 esquery: 1.7.0 jsdoc-type-pratt-parser: 7.0.0 @@ -10281,7 +10238,7 @@ snapshots: '@es-joy/jsdoccomment@0.83.0': dependencies: '@types/estree': 1.0.8 - '@typescript-eslint/types': 8.53.1 + '@typescript-eslint/types': 8.54.0 comment-parser: 1.4.5 esquery: 1.7.0 jsdoc-type-pratt-parser: 7.1.0 @@ -10554,10 +10511,6 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/core@1.0.1': - dependencies: - '@types/json-schema': 7.0.15 - '@eslint/core@1.1.0': dependencies: '@types/json-schema': 7.0.15 @@ -10601,7 +10554,7 @@ snapshots: '@eslint/plugin-kit@0.5.1': dependencies: - '@eslint/core': 1.0.1 + '@eslint/core': 1.1.0 levn: 0.4.1 '@floating-ui/core@1.7.3': @@ -11011,7 +10964,7 @@ snapshots: unified: 11.0.5 unist-util-position-from-estree: 2.0.0 unist-util-stringify-position: 4.0.0 - unist-util-visit: 5.0.0 + unist-util-visit: 5.1.0 vfile: 6.0.3 transitivePeerDependencies: - supports-color @@ -12325,7 +12278,7 @@ snapshots: '@stylistic/eslint-plugin@5.7.1(eslint@9.39.2(jiti@2.6.1))': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) - '@typescript-eslint/types': 8.53.1 + '@typescript-eslint/types': 8.54.0 eslint: 9.39.2(jiti@2.6.1) eslint-visitor-keys: 4.2.1 espree: 10.4.0 @@ -12890,15 +12843,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.53.1(typescript@5.9.3)': - dependencies: - '@typescript-eslint/tsconfig-utils': 8.53.1(typescript@5.9.3) - '@typescript-eslint/types': 8.53.1 - debug: 4.4.3(supports-color@10.2.2) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/project-service@8.54.0(typescript@5.9.3)': dependencies: '@typescript-eslint/tsconfig-utils': 8.54.0(typescript@5.9.3) @@ -12908,20 +12852,11 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/scope-manager@8.53.1': - dependencies: - '@typescript-eslint/types': 8.53.1 - '@typescript-eslint/visitor-keys': 8.53.1 - '@typescript-eslint/scope-manager@8.54.0': dependencies: '@typescript-eslint/types': 8.54.0 '@typescript-eslint/visitor-keys': 8.54.0 - '@typescript-eslint/tsconfig-utils@8.53.1(typescript@5.9.3)': - dependencies: - typescript: 5.9.3 - '@typescript-eslint/tsconfig-utils@8.54.0(typescript@5.9.3)': dependencies: typescript: 5.9.3 @@ -12938,25 +12873,8 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/types@8.53.1': {} - '@typescript-eslint/types@8.54.0': {} - '@typescript-eslint/typescript-estree@8.53.1(typescript@5.9.3)': - dependencies: - '@typescript-eslint/project-service': 8.53.1(typescript@5.9.3) - '@typescript-eslint/tsconfig-utils': 8.53.1(typescript@5.9.3) - '@typescript-eslint/types': 8.53.1 - '@typescript-eslint/visitor-keys': 8.53.1 - debug: 4.4.3(supports-color@10.2.2) - minimatch: 9.0.5 - semver: 7.7.3 - tinyglobby: 0.2.15 - ts-api-utils: 2.4.0(typescript@5.9.3) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/typescript-estree@8.54.0(typescript@5.9.3)': dependencies: '@typescript-eslint/project-service': 8.54.0(typescript@5.9.3) @@ -12972,17 +12890,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': - dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) - '@typescript-eslint/scope-manager': 8.53.1 - '@typescript-eslint/types': 8.53.1 - '@typescript-eslint/typescript-estree': 8.53.1(typescript@5.9.3) - eslint: 9.39.2(jiti@2.6.1) - typescript: 5.9.3 - transitivePeerDependencies: - - supports-color - '@typescript-eslint/utils@8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) @@ -12994,11 +12901,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/visitor-keys@8.53.1': - dependencies: - '@typescript-eslint/types': 8.53.1 - eslint-visitor-keys: 4.2.1 - '@typescript-eslint/visitor-keys@8.54.0': dependencies: '@typescript-eslint/types': 8.54.0 @@ -13081,8 +12983,8 @@ snapshots: '@vitest/eslint-plugin@1.6.6(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17)': dependencies: - '@typescript-eslint/scope-manager': 8.53.1 - '@typescript-eslint/utils': 8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.54.0 + '@typescript-eslint/utils': 8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) optionalDependencies: typescript: 5.9.3 @@ -13092,8 +12994,8 @@ snapshots: '@vitest/eslint-plugin@1.6.6(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.1.0-beta.1)': dependencies: - '@typescript-eslint/scope-manager': 8.53.1 - '@typescript-eslint/utils': 8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.54.0 + '@typescript-eslint/utils': 8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) optionalDependencies: typescript: 5.9.3 @@ -13758,8 +13660,6 @@ snapshots: comment-parser@1.4.1: {} - comment-parser@1.4.4: {} - comment-parser@1.4.5: {} commit-parser@1.3.0: @@ -14347,7 +14247,7 @@ snapshots: eslint-plugin-perfectionist@5.4.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3): dependencies: - '@typescript-eslint/utils': 8.53.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/utils': 8.54.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) natural-orderby: 5.0.0 transitivePeerDependencies: @@ -14478,16 +14378,16 @@ snapshots: dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) '@eslint-community/regexpp': 4.12.2 - comment-parser: 1.4.4 + comment-parser: 1.4.5 eslint: 9.39.2(jiti@2.6.1) - jsdoc-type-pratt-parser: 7.0.0 + jsdoc-type-pratt-parser: 7.1.0 refa: 0.12.1 regexp-ast-analysis: 0.7.1 scslre: 0.3.0 eslint-plugin-toml@1.0.3(eslint@9.39.2(jiti@2.6.1)): dependencies: - '@eslint/core': 1.0.1 + '@eslint/core': 1.1.0 '@eslint/plugin-kit': 0.5.1 debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.2(jiti@2.6.1) @@ -14539,7 +14439,7 @@ snapshots: eslint-plugin-yml@3.0.0(eslint@9.39.2(jiti@2.6.1)): dependencies: - '@eslint/core': 1.0.1 + '@eslint/core': 1.1.0 '@eslint/plugin-kit': 0.5.1 debug: 4.4.3(supports-color@10.2.2) diff-sequences: 29.6.3 @@ -14963,9 +14863,9 @@ snapshots: tinyglobby: 0.2.15 unified: 11.0.5 unist-util-remove-position: 5.0.0 - unist-util-visit: 5.0.0 + unist-util-visit: 5.1.0 vfile: 6.0.3 - zod: 4.3.5 + zod: 4.3.6 optionalDependencies: '@types/react': 19.2.10 react: 19.2.4 @@ -16036,7 +15936,7 @@ snapshots: mdast-util-to-string: 4.0.0 micromark-util-classify-character: 2.0.1 micromark-util-decode-string: 2.0.1 - unist-util-visit: 5.0.0 + unist-util-visit: 5.1.0 zwitch: 2.0.4 mdast-util-to-string@4.0.0: @@ -16449,7 +16349,7 @@ snapshots: statuses: 2.0.2 strict-event-emitter: 0.5.1 tough-cookie: 6.0.0 - type-fest: 5.3.0 + type-fest: 5.4.3 until-async: 3.0.2 yargs: 17.7.2 optionalDependencies: @@ -16474,7 +16374,7 @@ snapshots: statuses: 2.0.2 strict-event-emitter: 0.5.1 tough-cookie: 6.0.0 - type-fest: 5.3.0 + type-fest: 5.4.3 until-async: 3.0.2 yargs: 17.7.2 optionalDependencies: @@ -18112,10 +18012,6 @@ snapshots: type-fest@4.41.0: {} - type-fest@5.3.0: - dependencies: - tagged-tag: 1.0.0 - type-fest@5.4.3: dependencies: tagged-tag: 1.0.0 @@ -18224,7 +18120,7 @@ snapshots: unist-util-remove-position@5.0.0: dependencies: '@types/unist': 3.0.3 - unist-util-visit: 5.0.0 + unist-util-visit: 5.1.0 unist-util-stringify-position@4.0.0: dependencies: @@ -18235,12 +18131,6 @@ snapshots: '@types/unist': 3.0.3 unist-util-is: 6.0.0 - unist-util-visit@5.0.0: - dependencies: - '@types/unist': 3.0.3 - unist-util-is: 6.0.0 - unist-util-visit-parents: 6.0.2 - unist-util-visit@5.1.0: dependencies: '@types/unist': 3.0.3 @@ -18775,8 +18665,6 @@ snapshots: zod@3.25.76: {} - zod@4.3.5: {} - zod@4.3.6: {} zwitch@2.0.4: {} From d971cc0bc7fb51e074ac73ead4887fb219c59b69 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sat, 31 Jan 2026 19:11:13 +0100 Subject: [PATCH 13/63] refactor(pipeline-loader): remove unused loadPipelines function --- .../pipelines/pipeline-loader/src/index.ts | 1 - .../pipelines/pipeline-loader/src/loader.ts | 98 +++++++++---------- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/packages/pipelines/pipeline-loader/src/index.ts b/packages/pipelines/pipeline-loader/src/index.ts index 64aec795e..ab8100ad4 100644 --- a/packages/pipelines/pipeline-loader/src/index.ts +++ b/packages/pipelines/pipeline-loader/src/index.ts @@ -2,7 +2,6 @@ export { findPipelineFiles } from "./find"; export { loadPipelineFile, - loadPipelines, loadPipelinesFromPaths, } from "./loader"; diff --git a/packages/pipelines/pipeline-loader/src/loader.ts b/packages/pipelines/pipeline-loader/src/loader.ts index 743ebbd9f..d73a88a86 100644 --- a/packages/pipelines/pipeline-loader/src/loader.ts +++ b/packages/pipelines/pipeline-loader/src/loader.ts @@ -1,9 +1,6 @@ import type { PipelineDefinition } from "@ucdjs/pipelines-core"; import { isPipelineDefinition } from "@ucdjs/pipelines-core"; -/** - * Result of loading pipeline definitions from a file. - */ export interface LoadedPipelineFile { /** * The file path that was loaded. @@ -21,9 +18,6 @@ export interface LoadedPipelineFile { exportNames: string[]; } -/** - * Result of loading multiple pipeline files. - */ export interface LoadPipelinesResult { /** * All pipeline definitions found across all files. @@ -38,15 +32,14 @@ export interface LoadPipelinesResult { /** * Files that failed to load. */ - errors: Array<{ - filePath: string; - error: Error; - }>; + errors: Array; +} + +export interface PipelineLoadError { + filePath: string; + error: Error; } -/** - * Options for loading pipelines. - */ export interface LoadPipelinesOptions { /** * If true, throw on first error instead of collecting errors. @@ -58,8 +51,8 @@ export interface LoadPipelinesOptions { /** * Load a single pipeline file and extract all PipelineDefinition exports. * - * @param filePath - Absolute or relative path to the file to load - * @returns The loaded pipeline file with extracted definitions + * @param {string} filePath - Absolute or relative path to the file to load + * @returns {Promise} The loaded pipeline file with extracted definitions * * @example * ```ts @@ -90,13 +83,13 @@ export async function loadPipelineFile(filePath: string): Promise { const { throwOnError = false } = options; - const pipelines: PipelineDefinition[] = []; - const files: LoadedPipelineFile[] = []; - const errors: Array<{ filePath: string; error: Error }> = []; + if (throwOnError) { + const wrapped = filePaths.map((filePath) => + loadPipelineFile(filePath).catch((err) => { + const error = err instanceof Error ? err : new Error(String(err)); + throw new Error(`Failed to load pipeline file: ${filePath}`, { cause: error }); + }), + ); - for (const filePath of filePaths) { - try { - const result = await loadPipelineFile(filePath); - files.push(result); - pipelines.push(...result.pipelines); - } catch (err) { - const error = err instanceof Error ? err : new Error(String(err)); + const results = await Promise.all(wrapped); + const pipelines = results.flatMap((r) => r.pipelines); - if (throwOnError) { - throw new Error(`Failed to load pipeline file: ${filePath}`, { cause: error }); - } + return { + pipelines, + files: results, + errors: [], + }; + } + + // Collect errors instead of throwing; preserve order. + const settled = await Promise.allSettled(filePaths.map((fp) => loadPipelineFile(fp))); - errors.push({ filePath, error }); + const files: LoadedPipelineFile[] = []; + const errors: Array = []; + + for (const [i, result] of settled.entries()) { + if (result.status === "fulfilled") { + files.push(result.value); + continue; } + + const error = result.reason instanceof Error + ? result.reason + : new Error(String(result.reason)); + errors.push({ filePath: filePaths[i]!, error }); } + const pipelines = files.flatMap((f) => f.pipelines); + return { pipelines, files, errors, }; } - -/** - * Load pipelines from a directory by matching file patterns. - * This is a convenience wrapper that combines glob matching with loading. - * - * Note: This function requires the caller to provide the resolved file paths. - * Use a glob library to find files first, then pass them to loadPipelines. - * - * @param filePaths - Pre-resolved file paths (use a glob library to find them) - * @param options - Loading options - * @returns Combined result with all pipelines - * - * @example - * ```ts - * import { glob } from "glob"; - * - * const files = await glob("./pipelines/*.ts"); - * const result = await loadPipelines(files); - * ``` - */ -export { loadPipelines as loadPipelinesFromPaths }; From 840f3618cd595fe2b7d621efbd303c2ca32bbde7 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sat, 31 Jan 2026 20:19:51 +0100 Subject: [PATCH 14/63] feat(pipeline-loader): add vitest-testdirs for testing support - Introduced `vitest-testdirs` in `package.json` for improved testing capabilities. - Added comprehensive tests for pipeline file handling in `loader.test.ts`. --- .../pipelines/pipeline-loader/package.json | 3 +- .../pipeline-loader/test/loader.test.ts | 160 ++++++++++++++++++ pnpm-lock.yaml | 5 +- 3 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 packages/pipelines/pipeline-loader/test/loader.test.ts diff --git a/packages/pipelines/pipeline-loader/package.json b/packages/pipelines/pipeline-loader/package.json index 63c013b58..dce62b021 100644 --- a/packages/pipelines/pipeline-loader/package.json +++ b/packages/pipelines/pipeline-loader/package.json @@ -47,7 +47,8 @@ "eslint": "catalog:linting", "publint": "catalog:dev", "tsdown": "catalog:dev", - "typescript": "catalog:dev" + "typescript": "catalog:dev", + "vitest-testdirs": "catalog:testing" }, "publishConfig": { "access": "public" diff --git a/packages/pipelines/pipeline-loader/test/loader.test.ts b/packages/pipelines/pipeline-loader/test/loader.test.ts new file mode 100644 index 000000000..23eb51fc5 --- /dev/null +++ b/packages/pipelines/pipeline-loader/test/loader.test.ts @@ -0,0 +1,160 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { testdir } from "vitest-testdirs"; +import { findPipelineFiles, loadPipelineFile, loadPipelinesFromPaths } from "../src"; + +function pipelineModuleSource({ named = [], defaultId, extraExports }: { + named?: string[]; + defaultId?: string; + extraExports?: string; +}) { + const baseDefinition = (id: string) => `({ + _type: "pipeline-definition", + id: "${id}", + versions: ["16.0.0"], + inputs: [], + routes: [], +})`; + + const namedExports = named + .map((name) => `export const ${name} = ${baseDefinition(name)};`) + .join("\n\n"); + + const defaultExport = defaultId + ? `export default ${baseDefinition(defaultId)};` + : ""; + + return [namedExports, defaultExport, extraExports ?? ""] + .filter(Boolean) + .join("\n\n"); +} + +describe("findPipelineFiles", () => { + it("should find pipeline files and ignore node_modules and dist", async () => { + const root = await testdir({ + pipelines: { + "alpha.ucd-pipeline.ts": pipelineModuleSource({ named: ["alpha"] }), + "nested": { + "beta.ucd-pipeline.ts": pipelineModuleSource({ named: ["beta"] }), + }, + }, + node_modules: { + "ignored.ucd-pipeline.ts": pipelineModuleSource({ named: ["ignored"] }), + }, + dist: { + "built.ucd-pipeline.ts": pipelineModuleSource({ named: ["built"] }), + }, + }); + + const files = await findPipelineFiles("**/*.ucd-pipeline.ts", root); + const expected = [ + path.join(root, "pipelines", "alpha.ucd-pipeline.ts"), + path.join(root, "pipelines", "nested", "beta.ucd-pipeline.ts"), + ]; + + expect(files.sort()).toEqual(expected.sort()); + expect(files.every((file) => path.isAbsolute(file))).toBe(true); + }); + + it("should support custom patterns with a cwd", async () => { + const root = await testdir({ + pipelines: { + "gamma.ucd-pipeline.ts": pipelineModuleSource({ named: ["gamma"] }), + "notes.txt": "not a pipeline", + }, + other: { + "delta.ucd-pipeline.ts": pipelineModuleSource({ named: ["delta"] }), + }, + }); + + const cwd = path.join(root, "pipelines"); + const files = await findPipelineFiles(["**/*.ucd-pipeline.ts"], cwd); + + expect(files).toEqual([path.join(cwd, "gamma.ucd-pipeline.ts")]); + }); +}); + +describe("loadPipelineFile", () => { + it("should load pipeline definitions and export names", async () => { + const root = await testdir({ + "demo.ucd-pipeline.ts": pipelineModuleSource({ + named: ["alpha"], + defaultId: "default-pipeline", + extraExports: "export const config = { name: \"pipeline\" };", + }), + }); + const filePath = path.join(root, "demo.ucd-pipeline.ts"); + + const result = await loadPipelineFile(filePath); + + expect(result.filePath).toBe(filePath); + expect(result.exportNames).toEqual(expect.arrayContaining(["alpha", "default"])); + expect(result.exportNames).toHaveLength(2); + expect(result.pipelines.map((pipeline) => pipeline.id).sort()) + .toEqual(["alpha", "default-pipeline"]); + }); + + it("should return empty arrays when no pipelines are exported", async () => { + const root = await testdir({ + "empty.ucd-pipeline.ts": "export const config = { ok: true };", + }); + const filePath = path.join(root, "empty.ucd-pipeline.ts"); + + const result = await loadPipelineFile(filePath); + + expect(result.pipelines).toEqual([]); + expect(result.exportNames).toEqual([]); + }); +}); + +describe("loadPipelinesFromPaths", () => { + it("should merge pipelines and file metadata", async () => { + const root = await testdir({ + "alpha.ucd-pipeline.ts": pipelineModuleSource({ named: ["alpha"] }), + "beta.ucd-pipeline.ts": pipelineModuleSource({ named: ["beta"] }), + }); + + const alphaPath = path.join(root, "alpha.ucd-pipeline.ts"); + const betaPath = path.join(root, "beta.ucd-pipeline.ts"); + + const result = await loadPipelinesFromPaths([alphaPath, betaPath]); + + expect(result.errors).toEqual([]); + expect(result.files.map((file) => file.filePath)).toEqual([alphaPath, betaPath]); + expect(result.pipelines.map((pipeline) => pipeline.id).sort()).toEqual(["alpha", "beta"]); + }); + + it("should collect errors when files fail to load", async () => { + const root = await testdir({ + "alpha.ucd-pipeline.ts": pipelineModuleSource({ named: ["alpha"] }), + }); + + const alphaPath = path.join(root, "alpha.ucd-pipeline.ts"); + const missingPath = path.join(root, "missing.ucd-pipeline.ts"); + + const result = await loadPipelinesFromPaths([alphaPath, missingPath]); + + expect(result.files).toHaveLength(1); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]?.filePath).toBe(missingPath); + expect(result.errors[0]?.error).toBeInstanceOf(Error); + expect(result.pipelines.map((pipeline) => pipeline.id)).toEqual(["alpha"]); + }); + + it("should throw when throwOnError is enabled", async () => { + const root = await testdir({ + "alpha.ucd-pipeline.ts": pipelineModuleSource({ named: ["alpha"] }), + }); + + const missingPath = path.join(root, "missing.ucd-pipeline.ts"); + + try { + await loadPipelinesFromPaths([missingPath], { throwOnError: true }); + throw new Error("Expected loadPipelinesFromPaths to throw"); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain(`Failed to load pipeline file: ${missingPath}`); + expect((error as Error).cause).toBeInstanceOf(Error); + } + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b43fb7b9c..595641c4d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1078,7 +1078,7 @@ importers: devDependencies: '@luxass/eslint-config': specifier: catalog:linting - version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.1.0-beta.1) '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig @@ -1097,6 +1097,9 @@ importers: typescript: specifier: catalog:dev version: 5.9.3 + vitest-testdirs: + specifier: catalog:testing + version: 4.4.2(vitest@4.1.0-beta.1) packages/pipelines/pipeline-playground: dependencies: From 4fc2fb4f3f1195fee0f4bc70b5b04e09d96a9f81 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 1 Feb 2026 08:33:30 +0100 Subject: [PATCH 15/63] WIP: first draft of pipeline ui --- apps/web/package.json | 1 + .../components/layout/sidebar/app-sidebar.tsx | 3 +- apps/web/src/routeTree.gen.ts | 21 ++ apps/web/src/routes/pipeline-graph.tsx | 82 ++++++ .../pipelines/pipeline-ui/eslint.config.js | 3 + packages/pipelines/pipeline-ui/package.json | 23 +- .../pipeline-ui/src/components/details.tsx | 184 ++++++++++++++ .../pipeline-ui/src/components/filters.tsx | 143 +++++++++++ .../pipeline-ui/src/components/node-types.ts | 16 ++ .../pipeline-ui/src/components/nodes.tsx | 210 ++++++++++++++++ .../src/components/pipeline-graph.tsx | 237 ++++++++++++++++++ packages/pipelines/pipeline-ui/src/index.ts | 52 ++-- .../pipelines/pipeline-ui/src/lib/adapter.ts | 111 ++++++++ .../pipelines/pipeline-ui/src/lib/colors.ts | 15 ++ .../pipelines/pipeline-ui/src/lib/layout.ts | 114 +++++++++ .../pipelines/pipeline-ui/src/lib/utils.ts | 7 + .../pipelines/pipeline-ui/tsdown.config.ts | 6 + pnpm-lock.yaml | 212 ++++++++++++++++ pnpm-workspace.yaml | 1 + 19 files changed, 1408 insertions(+), 33 deletions(-) create mode 100644 apps/web/src/routes/pipeline-graph.tsx create mode 100644 packages/pipelines/pipeline-ui/src/components/details.tsx create mode 100644 packages/pipelines/pipeline-ui/src/components/filters.tsx create mode 100644 packages/pipelines/pipeline-ui/src/components/node-types.ts create mode 100644 packages/pipelines/pipeline-ui/src/components/nodes.tsx create mode 100644 packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx create mode 100644 packages/pipelines/pipeline-ui/src/lib/adapter.ts create mode 100644 packages/pipelines/pipeline-ui/src/lib/colors.ts create mode 100644 packages/pipelines/pipeline-ui/src/lib/layout.ts create mode 100644 packages/pipelines/pipeline-ui/src/lib/utils.ts diff --git a/apps/web/package.json b/apps/web/package.json index f5f70135f..48bb8389e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -29,6 +29,7 @@ "@tanstack/react-router-ssr-query": "catalog:web", "@tanstack/react-start": "catalog:web", "@ucdjs-internal/shared-ui": "workspace:*", + "@ucdjs/pipelines-ui": "workspace:*", "@ucdjs/schemas": "workspace:*", "@ucdjs/ucd-store": "workspace:*", "@unicode-utils/core": "catalog:prod", diff --git a/apps/web/src/components/layout/sidebar/app-sidebar.tsx b/apps/web/src/components/layout/sidebar/app-sidebar.tsx index c93054303..8f9362862 100644 --- a/apps/web/src/components/layout/sidebar/app-sidebar.tsx +++ b/apps/web/src/components/layout/sidebar/app-sidebar.tsx @@ -12,7 +12,7 @@ import { SidebarMenuItem, SidebarRail, } from "@ucdjs-internal/shared-ui/ui/sidebar"; -import { BookOpen, ExternalLink, Grid3X3, Lightbulb, Search, Type } from "lucide-react"; +import { BookOpen, ExternalLink, GitGraph, Grid3X3, Lightbulb, Search, Type } from "lucide-react"; import { UcdLogo } from "../../ucd-logo"; import { VersionSwitcher } from "../../version-switcher"; @@ -33,6 +33,7 @@ const VERSION_ITEMS = [ const TOOLS_ITEMS = [ { to: "/file-explorer/$", params: { _splat: "" }, icon: BookOpen, label: "File Explorer" }, + { to: "/pipeline-graph", icon: GitGraph, label: "Pipeline Graph" }, ] as const; export function AppSidebar({ ...props }: ComponentProps) { diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index 117671496..943cd67f7 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -10,6 +10,7 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as SearchRouteImport } from './routes/search' +import { Route as PipelineGraphRouteImport } from './routes/pipeline-graph' import { Route as CodepointInspectorRouteImport } from './routes/codepoint-inspector' import { Route as FileExplorerRouteRouteImport } from './routes/file-explorer/route' import { Route as IndexRouteImport } from './routes/index' @@ -31,6 +32,11 @@ const SearchRoute = SearchRouteImport.update({ path: '/search', getParentRoute: () => rootRouteImport, } as any) +const PipelineGraphRoute = PipelineGraphRouteImport.update({ + id: '/pipeline-graph', + path: '/pipeline-graph', + getParentRoute: () => rootRouteImport, +} as any) const CodepointInspectorRoute = CodepointInspectorRouteImport.update({ id: '/codepoint-inspector', path: '/codepoint-inspector', @@ -113,6 +119,7 @@ export interface FileRoutesByFullPath { '/': typeof IndexRoute '/file-explorer': typeof FileExplorerRouteRouteWithChildren '/codepoint-inspector': typeof CodepointInspectorRoute + '/pipeline-graph': typeof PipelineGraphRoute '/search': typeof SearchRoute '/v/$version': typeof VVersionRouteRouteWithChildren '/file-explorer/$': typeof FileExplorerSplatRoute @@ -131,6 +138,7 @@ export interface FileRoutesByTo { '/': typeof IndexRoute '/file-explorer': typeof FileExplorerRouteRouteWithChildren '/codepoint-inspector': typeof CodepointInspectorRoute + '/pipeline-graph': typeof PipelineGraphRoute '/search': typeof SearchRoute '/file-explorer/$': typeof FileExplorerSplatRoute '/v': typeof VIndexRoute @@ -149,6 +157,7 @@ export interface FileRoutesById { '/': typeof IndexRoute '/file-explorer': typeof FileExplorerRouteRouteWithChildren '/codepoint-inspector': typeof CodepointInspectorRoute + '/pipeline-graph': typeof PipelineGraphRoute '/search': typeof SearchRoute '/v/$version': typeof VVersionRouteRouteWithChildren '/file-explorer/$': typeof FileExplorerSplatRoute @@ -169,6 +178,7 @@ export interface FileRouteTypes { | '/' | '/file-explorer' | '/codepoint-inspector' + | '/pipeline-graph' | '/search' | '/v/$version' | '/file-explorer/$' @@ -187,6 +197,7 @@ export interface FileRouteTypes { | '/' | '/file-explorer' | '/codepoint-inspector' + | '/pipeline-graph' | '/search' | '/file-explorer/$' | '/v' @@ -204,6 +215,7 @@ export interface FileRouteTypes { | '/' | '/file-explorer' | '/codepoint-inspector' + | '/pipeline-graph' | '/search' | '/v/$version' | '/file-explorer/$' @@ -223,6 +235,7 @@ export interface RootRouteChildren { IndexRoute: typeof IndexRoute FileExplorerRouteRoute: typeof FileExplorerRouteRouteWithChildren CodepointInspectorRoute: typeof CodepointInspectorRoute + PipelineGraphRoute: typeof PipelineGraphRoute SearchRoute: typeof SearchRoute VVersionRouteRoute: typeof VVersionRouteRouteWithChildren VIndexRoute: typeof VIndexRoute @@ -237,6 +250,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof SearchRouteImport parentRoute: typeof rootRouteImport } + '/pipeline-graph': { + id: '/pipeline-graph' + path: '/pipeline-graph' + fullPath: '/pipeline-graph' + preLoaderRoute: typeof PipelineGraphRouteImport + parentRoute: typeof rootRouteImport + } '/codepoint-inspector': { id: '/codepoint-inspector' path: '/codepoint-inspector' @@ -388,6 +408,7 @@ const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, FileExplorerRouteRoute: FileExplorerRouteRouteWithChildren, CodepointInspectorRoute: CodepointInspectorRoute, + PipelineGraphRoute: PipelineGraphRoute, SearchRoute: SearchRoute, VVersionRouteRoute: VVersionRouteRouteWithChildren, VIndexRoute: VIndexRoute, diff --git a/apps/web/src/routes/pipeline-graph.tsx b/apps/web/src/routes/pipeline-graph.tsx new file mode 100644 index 000000000..43dfd73fb --- /dev/null +++ b/apps/web/src/routes/pipeline-graph.tsx @@ -0,0 +1,82 @@ +import type { PipelineGraph as PipelineGraphType } from "@ucdjs/pipelines-core"; +import { createFileRoute, Link } from "@tanstack/react-router"; +import { Breadcrumb, BreadcrumbItem, BreadcrumbLink, BreadcrumbList, BreadcrumbPage, BreadcrumbSeparator } from "@ucdjs-internal/shared-ui/ui/breadcrumb"; +import { PipelineGraph } from "@ucdjs/pipelines-ui"; + +export const Route = createFileRoute("/pipeline-graph")({ + component: PipelineGraphPage, +}); + +// Fake data for testing the visualization +const fakeGraph: PipelineGraphType = { + nodes: [ + { id: "source-16.0.0", type: "source", version: "16.0.0" }, + { id: "file-ucd-linebreak", type: "file", file: { version: "16.0.0", dir: "ucd", path: "ucd/LineBreak.txt", name: "LineBreak.txt", ext: ".txt" } }, + { id: "file-ucd-unicodedata", type: "file", file: { version: "16.0.0", dir: "ucd", path: "ucd/UnicodeData.txt", name: "UnicodeData.txt", ext: ".txt" } }, + { id: "file-ucd-blocks", type: "file", file: { version: "16.0.0", dir: "ucd", path: "ucd/Blocks.txt", name: "Blocks.txt", ext: ".txt" } }, + { id: "file-emoji-data", type: "file", file: { version: "16.0.0", dir: "emoji", path: "emoji/emoji-data.txt", name: "emoji-data.txt", ext: ".txt" } }, + { id: "route-linebreak", type: "route", routeId: "linebreak" }, + { id: "route-unicodedata", type: "route", routeId: "unicode-data" }, + { id: "route-blocks", type: "route", routeId: "blocks" }, + { id: "route-emoji", type: "route", routeId: "emoji-properties" }, + { id: "artifact-linebreak-json", type: "artifact", artifactId: "linebreak.json" }, + { id: "artifact-characters-json", type: "artifact", artifactId: "characters.json" }, + { id: "artifact-blocks-json", type: "artifact", artifactId: "blocks.json" }, + { id: "artifact-emoji-json", type: "artifact", artifactId: "emoji.json" }, + { id: "output-0", type: "output", outputIndex: 0 }, + { id: "output-1", type: "output", outputIndex: 1 }, + { id: "output-2", type: "output", outputIndex: 2 }, + { id: "output-3", type: "output", outputIndex: 3 }, + ], + edges: [ + { from: "source-16.0.0", to: "file-ucd-linebreak", type: "provides" }, + { from: "source-16.0.0", to: "file-ucd-unicodedata", type: "provides" }, + { from: "source-16.0.0", to: "file-ucd-blocks", type: "provides" }, + { from: "source-16.0.0", to: "file-emoji-data", type: "provides" }, + { from: "file-ucd-linebreak", to: "route-linebreak", type: "matched" }, + { from: "file-ucd-unicodedata", to: "route-unicodedata", type: "matched" }, + { from: "file-ucd-blocks", to: "route-blocks", type: "matched" }, + { from: "file-emoji-data", to: "route-emoji", type: "matched" }, + { from: "route-linebreak", to: "artifact-linebreak-json", type: "parsed" }, + { from: "route-unicodedata", to: "artifact-characters-json", type: "parsed" }, + { from: "route-blocks", to: "artifact-blocks-json", type: "parsed" }, + { from: "route-emoji", to: "artifact-emoji-json", type: "parsed" }, + { from: "artifact-linebreak-json", to: "output-0", type: "resolved" }, + { from: "artifact-characters-json", to: "output-1", type: "resolved" }, + { from: "artifact-blocks-json", to: "output-2", type: "resolved" }, + { from: "artifact-emoji-json", to: "output-3", type: "resolved" }, + // Cross-artifact dependency example + { from: "artifact-characters-json", to: "artifact-emoji-json", type: "uses-artifact" }, + ], +}; + +function PipelineGraphPage() { + return ( +
+
+ + + + Home} /> + + + + Pipeline Graph + + + +
+ +
+ { + if (node) { + console.log("Selected node:", node); + } + }} + /> +
+
+ ); +} diff --git a/packages/pipelines/pipeline-ui/eslint.config.js b/packages/pipelines/pipeline-ui/eslint.config.js index d9c0ca1ec..e08216c94 100644 --- a/packages/pipelines/pipeline-ui/eslint.config.js +++ b/packages/pipelines/pipeline-ui/eslint.config.js @@ -4,4 +4,7 @@ import { luxass } from "@luxass/eslint-config"; export default luxass({ type: "lib", pnpm: true, + react: true, +}).overrideRules({ + "ts/explicit-function-return-type": "off", }); diff --git a/packages/pipelines/pipeline-ui/package.json b/packages/pipelines/pipeline-ui/package.json index 543cae955..a6491fd51 100644 --- a/packages/pipelines/pipeline-ui/package.json +++ b/packages/pipelines/pipeline-ui/package.json @@ -18,6 +18,12 @@ "bugs": { "url": "https://github.com/ucdjs/ucd/issues" }, + "sideEffects": false, + "imports": { + "#components/*": "./src/components/*.tsx", + "#lib/*": "./src/lib/*.ts", + "#hooks/*": "./src/hooks/*.ts" + }, "exports": { ".": "./dist/index.mjs", "./package.json": "./package.json" @@ -36,15 +42,30 @@ "lint": "eslint .", "typecheck": "tsc --noEmit -p tsconfig.build.json" }, + "peerDependencies": { + "react": ">=18.0.0", + "react-dom": ">=18.0.0" + }, "dependencies": { - "@ucdjs/pipelines-core": "workspace:*" + "@ucdjs/pipelines-core": "workspace:*", + "@xyflow/react": "catalog:web", + "clsx": "catalog:web", + "tailwind-merge": "catalog:web" }, "devDependencies": { + "@eslint-react/eslint-plugin": "catalog:linting", "@luxass/eslint-config": "catalog:linting", + "@types/react": "catalog:types", + "@types/react-dom": "catalog:types", "@ucdjs-tooling/tsconfig": "workspace:*", "@ucdjs-tooling/tsdown-config": "workspace:*", "eslint": "catalog:linting", + "eslint-plugin-react-hooks": "catalog:linting", + "eslint-plugin-react-refresh": "catalog:linting", "publint": "catalog:dev", + "react": "catalog:web", + "react-dom": "catalog:web", + "tailwindcss": "catalog:web", "tsdown": "catalog:dev", "typescript": "catalog:dev" }, diff --git a/packages/pipelines/pipeline-ui/src/components/details.tsx b/packages/pipelines/pipeline-ui/src/components/details.tsx new file mode 100644 index 000000000..804430c87 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/details.tsx @@ -0,0 +1,184 @@ +import type { PipelineGraphNode } from "@ucdjs/pipelines-core"; +import type { CSSProperties } from "react"; +import { memo } from "react"; + +export interface PipelineGraphDetailsProps { + node: PipelineGraphNode | null; + onClose: () => void; +} + +// Hoisted static styles +const containerStyle: CSSProperties = { + width: "280px", + backgroundColor: "#ffffff", + borderLeft: "1px solid #e5e7eb", + display: "flex", + flexDirection: "column", + height: "100%", + boxShadow: "-2px 0 8px rgba(0,0,0,0.05)", + fontFamily: "system-ui, -apple-system, sans-serif", +}; + +const headerStyle: CSSProperties = { + display: "flex", + alignItems: "center", + justifyContent: "space-between", + padding: "12px", + borderBottom: "1px solid #e5e7eb", +}; + +const closeButtonStyle: CSSProperties = { + padding: "4px", + color: "#9ca3af", + background: "none", + border: "none", + borderRadius: "4px", + cursor: "pointer", + display: "flex", + alignItems: "center", + justifyContent: "center", +}; + +const contentStyle: CSSProperties = { + flex: 1, + padding: "12px", + overflowY: "auto", +}; + +const detailsContainerStyle: CSSProperties = { + display: "flex", + flexDirection: "column", + gap: "12px", +}; + +const detailRowStyle: CSSProperties = { + display: "flex", + flexDirection: "column", + gap: "2px", +}; + +const detailLabelStyle: CSSProperties = { + fontSize: "11px", + fontWeight: 500, + color: "#6b7280", + textTransform: "uppercase", + letterSpacing: "0.025em", +}; + +const detailValueStyle: CSSProperties = { + fontSize: "13px", + color: "#111827", + fontFamily: "ui-monospace, monospace", + wordBreak: "break-all", +}; + +// Badge style cache +const badgeStyleCache = new Map(); + +function getBadgeStyle(type: string): CSSProperties { + let cached = badgeStyleCache.get(type); + if (!cached) { + const colors: Record = { + source: { bg: "#eef2ff", color: "#4f46e5" }, + file: { bg: "#ecfdf5", color: "#059669" }, + route: { bg: "#fffbeb", color: "#d97706" }, + artifact: { bg: "#f5f3ff", color: "#7c3aed" }, + output: { bg: "#f0f9ff", color: "#0284c7" }, + }; + const c = colors[type] ?? { bg: "#f3f4f6", color: "#6b7280" }; + cached = { + padding: "2px 8px", + fontSize: "11px", + fontWeight: 600, + borderRadius: "4px", + textTransform: "uppercase", + letterSpacing: "0.025em", + backgroundColor: c.bg, + color: c.color, + }; + badgeStyleCache.set(type, cached); + } + return cached; +} + +// Memoized detail row component +const DetailRow = memo(({ label, value }: { label: string; value: string }) => { + return ( +
+ {label} + {value} +
+ ); +}); + +// Hoisted close icon SVG - static content +const closeIcon = ( + + + +); + +// Memoized node details renderer +const NodeDetails = memo(({ node }: { node: PipelineGraphNode }) => { + switch (node.type) { + case "source": + return ; + case "file": + return ( + <> + + + + + + + ); + case "route": + return ; + case "artifact": + return ; + case "output": + return ( + <> + + {node.property && } + + ); + } +}); + +export const PipelineGraphDetails = memo(({ + node, + onClose, +}: PipelineGraphDetailsProps) => { + if (!node) { + return null; + } + + return ( +
+ {/* Header */} +
+ + {node.type} + + +
+ + {/* Content */} +
+
+ + +
+
+
+ ); +}); diff --git a/packages/pipelines/pipeline-ui/src/components/filters.tsx b/packages/pipelines/pipeline-ui/src/components/filters.tsx new file mode 100644 index 000000000..c90f16d52 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/filters.tsx @@ -0,0 +1,143 @@ +import type { PipelineGraphNodeType } from "@ucdjs/pipelines-core"; +import type { CSSProperties } from "react"; +import { memo, useCallback } from "react"; + +interface NodeTypeConfig { + label: string; + color: string; +} + +// Hoisted outside component - never changes +const nodeTypeLabels: Record = { + source: { label: "Source", color: "#6366f1" }, + file: { label: "File", color: "#10b981" }, + route: { label: "Route", color: "#f59e0b" }, + artifact: { label: "Artifact", color: "#8b5cf6" }, + output: { label: "Output", color: "#0ea5e9" }, +}; + +// Hoisted array - stable reference +const allNodeTypes: readonly PipelineGraphNodeType[] = ["source", "file", "route", "artifact", "output"] as const; + +// Hoisted static styles +const containerStyle: CSSProperties = { + display: "flex", + alignItems: "center", + gap: "6px", + padding: "8px 12px", + backgroundColor: "#ffffff", + borderRadius: "10px", + boxShadow: "0 1px 3px rgba(0,0,0,0.1), 0 1px 2px rgba(0,0,0,0.06)", + fontFamily: "system-ui, -apple-system, sans-serif", + border: "1px solid #e5e7eb", +}; + +const labelStyle: CSSProperties = { + fontSize: "11px", + fontWeight: 500, + color: "#6b7280", + marginRight: "4px", + textTransform: "uppercase", + letterSpacing: "0.05em", +}; + +// Cache for button styles to avoid recreation on each render +const buttonStyleCache = new Map(); +const dotStyleCache = new Map(); + +function getButtonStyle(color: string, isVisible: boolean): CSSProperties { + const key = `${color}-${isVisible}`; + let cached = buttonStyleCache.get(key); + if (!cached) { + cached = { + display: "flex", + alignItems: "center", + gap: "6px", + padding: "4px 10px", + fontSize: "12px", + fontWeight: 500, + borderRadius: "6px", + border: "none", + cursor: "pointer", + transition: "all 0.15s ease", + backgroundColor: isVisible ? `${color}15` : "#f3f4f6", + color: isVisible ? color : "#9ca3af", + opacity: isVisible ? 1 : 0.6, + }; + buttonStyleCache.set(key, cached); + } + return cached; +} + +function getDotStyle(color: string, isVisible: boolean): CSSProperties { + const key = `${color}-${isVisible}`; + let cached = dotStyleCache.get(key); + if (!cached) { + cached = { + width: "8px", + height: "8px", + borderRadius: "50%", + backgroundColor: color, + opacity: isVisible ? 1 : 0.3, + transition: "opacity 0.15s ease", + }; + dotStyleCache.set(key, cached); + } + return cached; +} + +export interface PipelineGraphFiltersProps { + visibleTypes: Set; + onToggleType: (type: PipelineGraphNodeType) => void; +} + +// Memoized individual filter button to prevent re-renders +interface FilterButtonProps { + type: PipelineGraphNodeType; + config: NodeTypeConfig; + isVisible: boolean; + onToggle: (type: PipelineGraphNodeType) => void; +} + +const FilterButton = memo(({ + type, + config, + isVisible, + onToggle, +}: FilterButtonProps) => { + const handleClick = useCallback(() => { + onToggle(type); + }, [onToggle, type]); + + return ( + + ); +}); + +export const PipelineGraphFilters = memo(({ + visibleTypes, + onToggleType, +}: PipelineGraphFiltersProps) => { + return ( +
+ Show: + {allNodeTypes.map((type) => ( + + ))} +
+ ); +}); diff --git a/packages/pipelines/pipeline-ui/src/components/node-types.ts b/packages/pipelines/pipeline-ui/src/components/node-types.ts new file mode 100644 index 000000000..ce435e21e --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/node-types.ts @@ -0,0 +1,16 @@ +import { + ArtifactNode, + FileNode, + OutputNode, + RouteNode, + SourceNode, +} from "./nodes"; + +// Hoisted outside component to maintain stable reference +export const nodeTypes = { + source: SourceNode, + file: FileNode, + route: RouteNode, + artifact: ArtifactNode, + output: OutputNode, +} as const; diff --git a/packages/pipelines/pipeline-ui/src/components/nodes.tsx b/packages/pipelines/pipeline-ui/src/components/nodes.tsx new file mode 100644 index 000000000..a7044f82c --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/nodes.tsx @@ -0,0 +1,210 @@ +import type { PipelineGraphNode } from "@ucdjs/pipelines-core"; +import type { NodeProps } from "@xyflow/react"; +import type { CSSProperties } from "react"; +import { Handle, Position } from "@xyflow/react"; +import { memo } from "react"; + +export interface PipelineNodeData { + pipelineNode: PipelineGraphNode; + label: string; +} + +interface NodeTypeStyle { + bg: string; + border: string; + iconBg: string; + icon: string; +} + +// Hoisted outside component - these never change +const nodeTypeStyles: Record = { + source: { + bg: "#eef2ff", + border: "#a5b4fc", + iconBg: "#6366f1", + icon: "S", + }, + file: { + bg: "#ecfdf5", + border: "#6ee7b7", + iconBg: "#10b981", + icon: "F", + }, + route: { + bg: "#fffbeb", + border: "#fcd34d", + iconBg: "#f59e0b", + icon: "R", + }, + artifact: { + bg: "#f5f3ff", + border: "#c4b5fd", + iconBg: "#8b5cf6", + icon: "A", + }, + output: { + bg: "#f0f9ff", + border: "#7dd3fc", + iconBg: "#0ea5e9", + icon: "O", + }, +}; + +const defaultStyle: NodeTypeStyle = { + bg: "#f9fafb", + border: "#d1d5db", + iconBg: "#6b7280", + icon: "?", +}; + +// Hoisted static styles - reused across all nodes +const flexCenterStyle: CSSProperties = { + display: "flex", + alignItems: "center", +}; + +const labelContainerStyle: CSSProperties = { + display: "flex", + flexDirection: "column", + overflow: "hidden", + marginLeft: "10px", +}; + +const typeStyle: CSSProperties = { + fontSize: "10px", + textTransform: "uppercase", + letterSpacing: "0.05em", + color: "#6b7280", + marginBottom: "1px", +}; + +const labelStyle: CSSProperties = { + fontSize: "13px", + fontWeight: 500, + color: "#111827", + whiteSpace: "nowrap", + overflow: "hidden", + textOverflow: "ellipsis", +}; + +// Cache for computed styles to avoid recreation +const containerStyleCache = new Map(); +const iconStyleCache = new Map(); +const handleStyleCache = new Map(); + +function getContainerStyle(styles: NodeTypeStyle, selected: boolean): CSSProperties { + const key = `${styles.bg}-${styles.border}-${selected}`; + let cached = containerStyleCache.get(key); + if (!cached) { + cached = { + backgroundColor: styles.bg, + border: `2px solid ${styles.border}`, + borderRadius: "10px", + padding: "10px 14px", + minWidth: "150px", + maxWidth: "220px", + boxShadow: selected + ? `0 0 0 2px #3b82f6, 0 1px 3px rgba(0,0,0,0.1)` + : "0 1px 3px rgba(0,0,0,0.1)", + transition: "box-shadow 0.15s ease", + fontFamily: "system-ui, -apple-system, sans-serif", + }; + containerStyleCache.set(key, cached); + } + return cached; +} + +function getIconStyle(iconBg: string): CSSProperties { + let cached = iconStyleCache.get(iconBg); + if (!cached) { + cached = { + display: "flex", + alignItems: "center", + justifyContent: "center", + width: "28px", + height: "28px", + borderRadius: "6px", + backgroundColor: iconBg, + color: "#ffffff", + fontSize: "12px", + fontWeight: 700, + flexShrink: 0, + }; + iconStyleCache.set(iconBg, cached); + } + return cached; +} + +function getHandleStyle(border: string): CSSProperties { + let cached = handleStyleCache.get(border); + if (!cached) { + cached = { + width: "8px", + height: "8px", + backgroundColor: border, + border: "none", + }; + handleStyleCache.set(border, cached); + } + return cached; +} + +// Base node component - memoized to prevent re-renders when parent updates +const BaseNode = memo(({ + data, + selected = false, + type, +}: NodeProps & { data: PipelineNodeData; type: string }) => { + const styles = nodeTypeStyles[type] ?? defaultStyle; + + return ( +
+ + +
+ + {styles.icon} + +
+ + {type} + + + {data.label} + +
+
+ + +
+ ); +}); + +// Individual node type components - memoized +export const SourceNode = memo((props: NodeProps & { data: PipelineNodeData }) => { + return ; +}); + +export const FileNode = memo((props: NodeProps & { data: PipelineNodeData }) => { + return ; +}); + +export const RouteNode = memo((props: NodeProps & { data: PipelineNodeData }) => { + return ; +}); + +export const ArtifactNode = memo((props: NodeProps & { data: PipelineNodeData }) => { + return ; +}); + +export const OutputNode = memo((props: NodeProps & { data: PipelineNodeData }) => { + return ; +}); diff --git a/packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx b/packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx new file mode 100644 index 000000000..354cf56be --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx @@ -0,0 +1,237 @@ +import type { PipelineFlowNode } from "#lib/adapter"; +import type { + PipelineGraphNode, + PipelineGraphNodeType, + PipelineGraph as PipelineGraphType, +} from "@ucdjs/pipelines-core"; +import type { NodeMouseHandler, OnNodesChange } from "@xyflow/react"; +import type { CSSProperties } from "react"; +import { filterNodesByType, pipelineGraphToFlow } from "#lib/adapter"; +import { getNodeColor } from "#lib/colors"; +import { applyLayout } from "#lib/layout"; +import { + applyNodeChanges, + Background, + Controls, + MiniMap, + ReactFlow, +} from "@xyflow/react"; +import { memo, useCallback, useMemo, useRef, useState } from "react"; +import { PipelineGraphDetails } from "./details"; +import { PipelineGraphFilters } from "./filters"; +import { nodeTypes } from "./node-types"; + +import "@xyflow/react/dist/style.css"; + +// Hoisted outside component - stable references +const defaultVisibleTypes: Set = new Set([ + "source", + "file", + "route", + "artifact", + "output", +]); + +// Hoisted static styles +const containerStyle: CSSProperties = { + display: "flex", + height: "100%", + width: "100%", +}; + +const graphContainerStyle: CSSProperties = { + flex: 1, + display: "flex", + flexDirection: "column", + position: "relative", +}; + +const filtersContainerStyle: CSSProperties = { + position: "absolute", + top: "12px", + left: "12px", + zIndex: 10, +}; + +// Hoisted fitView options - stable reference +const fitViewOptions = { padding: 0.2 } as const; + +// Hoisted proOptions - stable reference +const proOptions = { hideAttribution: true } as const; + +// Hoisted minimap mask color +const minimapMaskColor = "rgba(0, 0, 0, 0.1)"; + +export interface PipelineGraphProps { + /** The pipeline graph to visualize */ + graph: PipelineGraphType; + /** Callback when a node is selected/deselected */ + onNodeSelect?: (node: PipelineGraphNode | null) => void; + /** Show the filter toolbar (default: true) */ + showFilters?: boolean; + /** Show the details panel (default: true) */ + showDetails?: boolean; + /** Show the minimap (default: true) */ + showMinimap?: boolean; +} + +export const PipelineGraph = memo(({ + graph, + onNodeSelect, + showFilters = true, + showDetails = true, + showMinimap = true, +}: PipelineGraphProps) => { + // Convert pipeline graph to React Flow format - memoized + const { allNodes, allEdges } = useMemo(() => { + const { nodes, edges } = pipelineGraphToFlow(graph); + return { allNodes: nodes, allEdges: edges }; + }, [graph]); + + // Filter state - use lazy initializer for Set + const [visibleTypes, setVisibleTypes] = useState>( + () => new Set(defaultVisibleTypes), + ); + + // Selected node state + const [selectedNode, setSelectedNode] = useState(null); + + // Apply filtering and layout - memoized + const { layoutedNodes, layoutedEdges } = useMemo(() => { + const { nodes: filteredNodes, edges: filteredEdges } = filterNodesByType( + allNodes, + allEdges, + visibleTypes, + ); + const positioned = applyLayout(filteredNodes, filteredEdges); + return { layoutedNodes: positioned, layoutedEdges: filteredEdges }; + }, [allNodes, allEdges, visibleTypes]); + + // Maintain actual node state for React Flow + // Initialize from layouted nodes and update when they change + const [nodes, setNodes] = useState([]); + + // Track which layout we've synced to avoid unnecessary updates + const layoutKeyRef = useRef(""); + + // Sync nodes when layout changes (e.g., filter changes) + // We use a key based on node IDs to detect layout changes + const currentLayoutKey = layoutedNodes.map((n) => n.id).join(","); + if (currentLayoutKey !== layoutKeyRef.current) { + layoutKeyRef.current = currentLayoutKey; + setNodes(layoutedNodes); + } + + // Handle node changes (for drag operations and initialization) + const onNodesChange: OnNodesChange = useCallback((changes) => { + setNodes((nds) => applyNodeChanges(changes, nds)); + }, []); + + // Track if we're currently dragging to prevent showing details on drag + const isDraggingRef = useRef(false); + + // Handle filter toggle - uses functional setState for stable callback + const handleToggleType = useCallback((type: PipelineGraphNodeType) => { + setVisibleTypes((prev) => { + const next = new Set(prev); + if (next.has(type)) { + // Don't allow hiding all types + if (next.size > 1) { + next.delete(type); + } + } else { + next.add(type); + } + return next; + }); + }, []); + + // Handle node drag start - mark as dragging + const handleNodeDragStart = useCallback(() => { + isDraggingRef.current = true; + }, []); + + // Handle node drag stop - reset dragging flag after a short delay + // The delay ensures the click handler sees the dragging state + const handleNodeDragStop = useCallback(() => { + // Use setTimeout to reset after click event has fired + setTimeout(() => { + isDraggingRef.current = false; + }, 0); + }, []); + + // Handle node click - only show details if not dragging + const handleNodeClick: NodeMouseHandler = useCallback( + (_event, node) => { + if (isDraggingRef.current) { + return; + } + const pipelineNode = node.data?.pipelineNode ?? null; + setSelectedNode(pipelineNode); + onNodeSelect?.(pipelineNode); + }, + [onNodeSelect], + ); + + // Handle pane click - deselect node when clicking empty area + const handlePaneClick = useCallback(() => { + setSelectedNode(null); + onNodeSelect?.(null); + }, [onNodeSelect]); + + // Handle close details - stable callback + const handleCloseDetails = useCallback(() => { + setSelectedNode(null); + onNodeSelect?.(null); + }, [onNodeSelect]); + + return ( +
+
+ {/* Filters toolbar */} + {showFilters && ( +
+ +
+ )} + + {/* Main graph */} + + + + {showMinimap && ( + + )} + +
+ + {/* Details panel */} + {showDetails && selectedNode && ( + + )} +
+ ); +}); diff --git a/packages/pipelines/pipeline-ui/src/index.ts b/packages/pipelines/pipeline-ui/src/index.ts index 57da5a903..6a13f270b 100644 --- a/packages/pipelines/pipeline-ui/src/index.ts +++ b/packages/pipelines/pipeline-ui/src/index.ts @@ -1,31 +1,21 @@ -import type { DAG, PipelineDefinition } from "@ucdjs/pipelines-core"; - -export interface GraphVisualizationOptions { - format?: "mermaid" | "dot" | "json"; -} - -export function visualizeGraph( - _dag: DAG, - _options?: GraphVisualizationOptions, -): string { - throw new Error("Not implemented: visualizeGraph is a placeholder for future functionality"); -} - -export interface EventLogOptions { - maxEvents?: number; - filter?: (event: unknown) => boolean; -} - -export function createEventLogger(_options?: EventLogOptions): { - log: (event: unknown) => void; - getEvents: () => unknown[]; - clear: () => void; -} { - throw new Error("Not implemented: createEventLogger is a placeholder for future functionality"); -} - -export function renderPipelineSummary( - _pipelines: PipelineDefinition[], -): string { - throw new Error("Not implemented: renderPipelineSummary is a placeholder for future functionality"); -} +export { PipelineGraphDetails, type PipelineGraphDetailsProps } from "./components/details"; +export { PipelineGraphFilters, type PipelineGraphFiltersProps } from "./components/filters"; +export { nodeTypes } from "./components/node-types"; +export { + ArtifactNode, + FileNode, + OutputNode, + type PipelineNodeData, + RouteNode, + SourceNode, +} from "./components/nodes"; +export { PipelineGraph, type PipelineGraphProps } from "./components/pipeline-graph"; +export { + filterNodesByType, + type PipelineFlowEdge, + type PipelineFlowNode, + pipelineGraphToFlow, +} from "./lib/adapter"; +export { getNodeColor, nodeTypeColors } from "./lib/colors"; +export { applyLayout, NODE_HEIGHT, NODE_WIDTH } from "./lib/layout"; +export { cn } from "./lib/utils"; diff --git a/packages/pipelines/pipeline-ui/src/lib/adapter.ts b/packages/pipelines/pipeline-ui/src/lib/adapter.ts new file mode 100644 index 000000000..96f30b18e --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/lib/adapter.ts @@ -0,0 +1,111 @@ +import type { + PipelineGraph, + PipelineGraphEdge, + PipelineGraphNode, + PipelineGraphNodeType, +} from "@ucdjs/pipelines-core"; +import type { Edge, Node } from "@xyflow/react"; + +export interface PipelineFlowNode extends Node { + data: { + pipelineNode: PipelineGraphNode; + label: string; + }; + type: PipelineGraphNodeType; +} + +export interface PipelineFlowEdge extends Edge { + data: { + pipelineEdge: PipelineGraphEdge; + }; +} + +/** + * Get a human-readable label for a pipeline graph node + */ +function getNodeLabel(node: PipelineGraphNode): string { + switch (node.type) { + case "source": + return `v${node.version}`; + case "file": + return node.file.name; + case "route": + return node.routeId; + case "artifact": + return node.artifactId; + case "output": + return node.property + ? `Output[${node.outputIndex}].${node.property}` + : `Output[${node.outputIndex}]`; + } +} + +/** + * Get edge style based on edge type + */ +function getEdgeStyle(edgeType: PipelineGraphEdge["type"]): Pick { + const baseStyle = { strokeWidth: 2 }; + + switch (edgeType) { + case "provides": + return { style: { ...baseStyle, stroke: "#6366f1" } }; // indigo + case "matched": + return { style: { ...baseStyle, stroke: "#22c55e" } }; // green + case "parsed": + return { style: { ...baseStyle, stroke: "#f59e0b" } }; // amber + case "resolved": + return { style: { ...baseStyle, stroke: "#3b82f6" } }; // blue + case "uses-artifact": + return { style: { ...baseStyle, stroke: "#8b5cf6" }, animated: true }; // violet, animated + default: + return { style: baseStyle }; + } +} + +/** + * Convert a PipelineGraph to React Flow nodes and edges + */ +export function pipelineGraphToFlow( + graph: PipelineGraph, +): { nodes: PipelineFlowNode[]; edges: PipelineFlowEdge[] } { + const nodes: PipelineFlowNode[] = graph.nodes.map((node) => ({ + id: node.id, + type: node.type, + position: { x: 0, y: 0 }, // Will be set by layout + data: { + pipelineNode: node, + label: getNodeLabel(node), + }, + })); + + const edges: PipelineFlowEdge[] = graph.edges.map((edge, index) => ({ + id: `edge-${index}-${edge.from}-${edge.to}`, + source: edge.from, + target: edge.to, + label: edge.type, + ...getEdgeStyle(edge.type), + data: { + pipelineEdge: edge, + }, + })); + + return { nodes, edges }; +} + +/** + * Filter nodes by type + */ +export function filterNodesByType( + nodes: PipelineFlowNode[], + edges: PipelineFlowEdge[], + visibleTypes: Set, +): { nodes: PipelineFlowNode[]; edges: PipelineFlowEdge[] } { + const filteredNodes = nodes.filter((node) => visibleTypes.has(node.type as PipelineGraphNodeType)); + const filteredNodeIds = new Set(filteredNodes.map((n) => n.id)); + + const filteredEdges = edges.filter( + (edge) => filteredNodeIds.has(edge.source) && filteredNodeIds.has(edge.target), + ); + + return { nodes: filteredNodes, edges: filteredEdges }; +} diff --git a/packages/pipelines/pipeline-ui/src/lib/colors.ts b/packages/pipelines/pipeline-ui/src/lib/colors.ts new file mode 100644 index 000000000..c8f9835b6 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/lib/colors.ts @@ -0,0 +1,15 @@ +// Node type colors for minimap and external use +export const nodeTypeColors: Record = { + source: "#6366f1", + file: "#10b981", + route: "#f59e0b", + artifact: "#8b5cf6", + output: "#0ea5e9", + default: "#6b7280", +}; + +// Hoisted nodeColor function for minimap - stable reference +export function getNodeColor(node: { type?: string }): string { + const color = nodeTypeColors[node.type ?? ""]; + return color ?? nodeTypeColors.default ?? "#6b7280"; +} diff --git a/packages/pipelines/pipeline-ui/src/lib/layout.ts b/packages/pipelines/pipeline-ui/src/lib/layout.ts new file mode 100644 index 000000000..59722e0ef --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/lib/layout.ts @@ -0,0 +1,114 @@ +import type { PipelineFlowEdge, PipelineFlowNode } from "./adapter"; + +const NODE_WIDTH = 180; +const NODE_HEIGHT = 60; +const HORIZONTAL_GAP = 80; +const VERTICAL_GAP = 40; + +/** + * Simple layered layout for DAG visualization + * Assigns nodes to layers based on their distance from root nodes + */ +export function applyLayout( + nodes: PipelineFlowNode[], + edges: PipelineFlowEdge[], +): PipelineFlowNode[] { + if (nodes.length === 0) { + return nodes; + } + + // Build adjacency list for topological sorting + const incomingEdges = new Map>(); + const outgoingEdges = new Map>(); + + for (const node of nodes) { + incomingEdges.set(node.id, new Set()); + outgoingEdges.set(node.id, new Set()); + } + + const nodeIds = new Set(nodes.map((n) => n.id)); + + for (const edge of edges) { + // Only process edges where both nodes exist + if (nodeIds.has(edge.source) && nodeIds.has(edge.target)) { + incomingEdges.get(edge.target)?.add(edge.source); + outgoingEdges.get(edge.source)?.add(edge.target); + } + } + + // Calculate layer for each node using BFS from root nodes + const layers = new Map(); + const rootNodes = nodes.filter((n) => incomingEdges.get(n.id)?.size === 0); + + // If no root nodes, use first node as root + const firstNode = nodes[0]; + const queue: Array<{ id: string; layer: number }> = rootNodes.length > 0 + ? rootNodes.map((n) => ({ id: n.id, layer: 0 })) + : firstNode ? [{ id: firstNode.id, layer: 0 }] : []; + + while (queue.length > 0) { + const item = queue.shift(); + if (!item) { + continue; + } + const { id, layer } = item; + + // Only update if we found a deeper layer (ensures longest path) + if (!layers.has(id) || layers.get(id)! < layer) { + layers.set(id, layer); + + // Add all children to queue + for (const childId of outgoingEdges.get(id) || []) { + queue.push({ id: childId, layer: layer + 1 }); + } + } + } + + // Handle nodes not reachable from roots (disconnected components) + for (const node of nodes) { + if (!layers.has(node.id)) { + layers.set(node.id, 0); + } + } + + // Group nodes by layer + const layerGroups = new Map(); + for (const node of nodes) { + const layer = layers.get(node.id) ?? 0; + if (!layerGroups.has(layer)) { + layerGroups.set(layer, []); + } + layerGroups.get(layer)!.push(node); + } + + // Sort layers + const sortedLayers = Array.from(layerGroups.entries()).sort((a, b) => a[0] - b[0]); + + // Calculate positions + const positionedNodes = new Map(); + + for (const [layerIndex, layerNodes] of sortedLayers) { + const x = layerIndex * (NODE_WIDTH + HORIZONTAL_GAP); + const layerHeight = layerNodes.length * (NODE_HEIGHT + VERTICAL_GAP) - VERTICAL_GAP; + const startY = -layerHeight / 2; + + for (let i = 0; i < layerNodes.length; i++) { + const node = layerNodes[i]; + if (!node) { + continue; + } + const y = startY + i * (NODE_HEIGHT + VERTICAL_GAP); + + const positionedNode: PipelineFlowNode = { + ...node, + position: { x, y }, + }; + positionedNodes.set(node.id, positionedNode); + } + } + + // Return nodes in original order + return nodes.map((n) => positionedNodes.get(n.id) ?? n); +} + +export { NODE_HEIGHT, NODE_WIDTH }; diff --git a/packages/pipelines/pipeline-ui/src/lib/utils.ts b/packages/pipelines/pipeline-ui/src/lib/utils.ts new file mode 100644 index 000000000..88283f013 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/lib/utils.ts @@ -0,0 +1,7 @@ +import type { ClassValue } from "clsx"; +import { clsx } from "clsx"; +import { twMerge } from "tailwind-merge"; + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} diff --git a/packages/pipelines/pipeline-ui/tsdown.config.ts b/packages/pipelines/pipeline-ui/tsdown.config.ts index dee0149e6..ae388ddeb 100644 --- a/packages/pipelines/pipeline-ui/tsdown.config.ts +++ b/packages/pipelines/pipeline-ui/tsdown.config.ts @@ -4,4 +4,10 @@ export default createTsdownConfig({ entry: [ "./src/index.ts", ], + format: "esm", + inputOptions: { + transform: { + jsx: "react-jsx", + }, + }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 595641c4d..5df956c86 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -243,6 +243,9 @@ catalogs: '@vitejs/plugin-react': specifier: 5.1.2 version: 5.1.2 + '@xyflow/react': + specifier: 12.10.0 + version: 12.10.0 babel-plugin-react-compiler: specifier: 1.0.0 version: 1.0.0 @@ -542,6 +545,9 @@ importers: '@ucdjs-internal/shared-ui': specifier: workspace:* version: link:../../packages/shared-ui + '@ucdjs/pipelines-ui': + specifier: workspace:* + version: link:../../packages/pipelines/pipeline-ui '@ucdjs/schemas': specifier: workspace:* version: link:../../packages/schemas @@ -1146,10 +1152,28 @@ importers: '@ucdjs/pipelines-core': specifier: workspace:* version: link:../pipeline-core + '@xyflow/react': + specifier: catalog:web + version: 12.10.0(@types/react@19.2.10)(immer@9.0.21)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + clsx: + specifier: catalog:web + version: 2.1.1 + tailwind-merge: + specifier: catalog:web + version: 3.4.0 devDependencies: + '@eslint-react/eslint-plugin': + specifier: catalog:linting + version: 2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) '@luxass/eslint-config': specifier: catalog:linting version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@types/react': + specifier: catalog:types + version: 19.2.10 + '@types/react-dom': + specifier: catalog:types + version: 19.2.3(@types/react@19.2.10) '@ucdjs-tooling/tsconfig': specifier: workspace:* version: link:../../../tooling/tsconfig @@ -1159,9 +1183,24 @@ importers: eslint: specifier: catalog:linting version: 9.39.2(jiti@2.6.1) + eslint-plugin-react-hooks: + specifier: catalog:linting + version: 7.0.1(eslint@9.39.2(jiti@2.6.1)) + eslint-plugin-react-refresh: + specifier: catalog:linting + version: 0.4.26(eslint@9.39.2(jiti@2.6.1)) publint: specifier: catalog:dev version: 0.3.17 + react: + specifier: catalog:web + version: 19.2.4 + react-dom: + specifier: catalog:web + version: 19.2.4(react@19.2.4) + tailwindcss: + specifier: catalog:web + version: 4.1.18 tsdown: specifier: catalog:dev version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) @@ -4741,6 +4780,24 @@ packages: '@types/chai@5.2.2': resolution: {integrity: sha512-8kB30R7Hwqf40JPiKhVzodJs2Qc1ZJ5zuT3uzw5Hq/dhNCl3G3l83jfpdI1e20BP348+fV7VIL/+FxaXkqBmWg==} + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-drag@3.0.7': + resolution: {integrity: sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-selection@3.0.11': + resolution: {integrity: sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==} + + '@types/d3-transition@3.0.9': + resolution: {integrity: sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==} + + '@types/d3-zoom@3.0.8': + resolution: {integrity: sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==} + '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} @@ -5066,6 +5123,15 @@ packages: '@vue/shared@3.5.25': resolution: {integrity: sha512-AbOPdQQnAnzs58H2FrrDxYj/TJfmeS2jdfEEhgiKINy+bnOANmVizIEgq1r+C5zsbs6l1CCQxtcj71rwNQ4jWg==} + '@xyflow/react@12.10.0': + resolution: {integrity: sha512-eOtz3whDMWrB4KWVatIBrKuxECHqip6PfA8fTpaS2RUGVpiEAe+nqDKsLqkViVWxDGreq0lWX71Xth/SPAzXiw==} + peerDependencies: + react: '>=17' + react-dom: '>=17' + + '@xyflow/system@0.0.74': + resolution: {integrity: sha512-7v7B/PkiVrkdZzSbL+inGAo6tkR/WQHHG0/jhSvLQToCsfa8YubOGmBYd1s08tpKpihdHDZFwzQZeR69QSBb4Q==} + abort-controller@3.0.0: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} @@ -5409,6 +5475,9 @@ packages: class-variance-authority@0.7.1: resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} + classcat@5.0.5: + resolution: {integrity: sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==} + clean-regexp@1.0.0: resolution: {integrity: sha512-GfisEZEJvzKrmGWkvfhgzcz/BllN1USeqD2V6tg14OAOgaCD2Z/PUEuxnAZ/nPvmaHRG7a8y77p1T/IRQ4D1Hw==} engines: {node: '>=4'} @@ -5598,6 +5667,44 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + data-uri-to-buffer@2.0.2: resolution: {integrity: sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==} @@ -9613,6 +9720,21 @@ packages: zod@4.3.6: resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + zustand@4.5.7: + resolution: {integrity: sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==} + engines: {node: '>=12.7.0'} + peerDependencies: + '@types/react': '>=16.8' + immer: '>=9.0.6' + react: '>=16.8' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -12746,6 +12868,27 @@ snapshots: dependencies: '@types/deep-eql': 4.0.2 + '@types/d3-color@3.1.3': {} + + '@types/d3-drag@3.0.7': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-selection@3.0.11': {} + + '@types/d3-transition@3.0.9': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-zoom@3.0.8': + dependencies: + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + '@types/debug@4.1.12': dependencies: '@types/ms': 2.1.0 @@ -13232,6 +13375,29 @@ snapshots: '@vue/shared@3.5.25': {} + '@xyflow/react@12.10.0(@types/react@19.2.10)(immer@9.0.21)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@xyflow/system': 0.0.74 + classcat: 5.0.5 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + zustand: 4.5.7(@types/react@19.2.10)(immer@9.0.21)(react@19.2.4) + transitivePeerDependencies: + - '@types/react' + - immer + + '@xyflow/system@0.0.74': + dependencies: + '@types/d3-drag': 3.0.7 + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + '@types/d3-transition': 3.0.9 + '@types/d3-zoom': 3.0.8 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + abort-controller@3.0.0: dependencies: event-target-shim: 5.0.1 @@ -13591,6 +13757,8 @@ snapshots: dependencies: clsx: 2.1.1 + classcat@5.0.5: {} + clean-regexp@1.0.0: dependencies: escape-string-regexp: 1.0.5 @@ -13754,6 +13922,42 @@ snapshots: csstype@3.2.3: {} + d3-color@3.1.0: {} + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-ease@3.0.1: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-selection@3.0.0: {} + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + data-uri-to-buffer@2.0.2: {} data-uri-to-buffer@4.0.1: {} @@ -18670,4 +18874,12 @@ snapshots: zod@4.3.6: {} + zustand@4.5.7(@types/react@19.2.10)(immer@9.0.21)(react@19.2.4): + dependencies: + use-sync-external-store: 1.6.0(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.10 + immer: 9.0.21 + react: 19.2.4 + zwitch@2.0.4: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 79d8aaf19..ff737f557 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -125,6 +125,7 @@ catalogs: "@fontsource-variable/inter": 5.2.8 shiki: 3.22.0 cmdk: 1.1.1 + "@xyflow/react": 12.10.0 vscode: vscode-ext-gen: 1.5.1 From 5f934aecf303da60cf764ba9009e0ca89e1e5106 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 1 Feb 2026 09:54:30 +0100 Subject: [PATCH 16/63] WIP: first draft of the Pipeline Server --- packages/cli/package.json | 1 + packages/cli/src/cmd/pipelines/root.ts | 1 + packages/cli/src/cmd/pipelines/run.ts | 10 + .../pipeline-server/eslint.config.js | 8 + packages/pipelines/pipeline-server/index.html | 12 + .../pipelines/pipeline-server/package.json | 80 +++++++ .../pipeline-server/src/client/app.tsx | 18 ++ .../pipeline-server/src/client/index.css | 1 + .../pipeline-server/src/client/main.tsx | 10 + .../pipeline-server/src/server/app.ts | 73 ++++++ .../src/server/routes/hello.ts | 10 + .../src/server/routes/pipelines.ts | 12 + .../pipeline-server/tsconfig.build.json | 5 + .../pipelines/pipeline-server/tsconfig.json | 8 + .../pipelines/pipeline-server/vite.config.ts | 67 ++++++ pnpm-lock.yaml | 211 +++++++++++++++++- pnpm-workspace.yaml | 3 + 17 files changed, 526 insertions(+), 4 deletions(-) create mode 100644 packages/pipelines/pipeline-server/eslint.config.js create mode 100644 packages/pipelines/pipeline-server/index.html create mode 100644 packages/pipelines/pipeline-server/package.json create mode 100644 packages/pipelines/pipeline-server/src/client/app.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/index.css create mode 100644 packages/pipelines/pipeline-server/src/client/main.tsx create mode 100644 packages/pipelines/pipeline-server/src/server/app.ts create mode 100644 packages/pipelines/pipeline-server/src/server/routes/hello.ts create mode 100644 packages/pipelines/pipeline-server/src/server/routes/pipelines.ts create mode 100644 packages/pipelines/pipeline-server/tsconfig.build.json create mode 100644 packages/pipelines/pipeline-server/tsconfig.json create mode 100644 packages/pipelines/pipeline-server/vite.config.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index f32341a9b..383f9efa7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -49,6 +49,7 @@ "@ucdjs/fs-bridge": "workspace:*", "@ucdjs/lockfile": "workspace:*", "@ucdjs/pipelines-loader": "workspace:*", + "@ucdjs/pipelines-server": "workspace:*", "@ucdjs/schema-gen": "workspace:*", "@ucdjs/schemas": "workspace:*", "@ucdjs/ucd-store": "workspace:*", diff --git a/packages/cli/src/cmd/pipelines/root.ts b/packages/cli/src/cmd/pipelines/root.ts index d55c119f1..00af65aca 100644 --- a/packages/cli/src/cmd/pipelines/root.ts +++ b/packages/cli/src/cmd/pipelines/root.ts @@ -4,6 +4,7 @@ import { printHelp } from "../../cli-utils"; export interface CLIPipelinesCmdOptions { flags: CLIArguments<{ ui: boolean; + port: number; }>; } diff --git a/packages/cli/src/cmd/pipelines/run.ts b/packages/cli/src/cmd/pipelines/run.ts index 81cc21bc9..e66cd647e 100644 --- a/packages/cli/src/cmd/pipelines/run.ts +++ b/packages/cli/src/cmd/pipelines/run.ts @@ -6,6 +6,7 @@ import { output } from "../../output"; export interface CLIPipelinesRunCmdOptions { flags: CLIArguments>; } @@ -18,6 +19,7 @@ export async function runPipelinesRun({ flags }: CLIPipelinesRunCmdOptions) { tables: { Flags: [ ["--ui", "Run the pipeline with a UI."], + ["--port ", "Port for the UI server (default: 3030)."], ["--help (-h)", "See all available flags."], ], }, @@ -25,5 +27,13 @@ export async function runPipelinesRun({ flags }: CLIPipelinesRunCmdOptions) { return; } + if (flags?.ui) { + const { startServer } = await import("@ucdjs/pipelines-server"); + const port = flags?.port ?? 3030; + output.info(`Starting Pipeline UI on port ${port}...`); + startServer({ port }); + return; + } + output.info("Running pipelines..."); } diff --git a/packages/pipelines/pipeline-server/eslint.config.js b/packages/pipelines/pipeline-server/eslint.config.js new file mode 100644 index 000000000..ab060e8c2 --- /dev/null +++ b/packages/pipelines/pipeline-server/eslint.config.js @@ -0,0 +1,8 @@ +// @ts-check +import { luxass } from "@luxass/eslint-config"; + +export default luxass({ + type: "lib", + pnpm: true, + react: true, +}); diff --git a/packages/pipelines/pipeline-server/index.html b/packages/pipelines/pipeline-server/index.html new file mode 100644 index 000000000..29bf19dcc --- /dev/null +++ b/packages/pipelines/pipeline-server/index.html @@ -0,0 +1,12 @@ + + + + + + UCD Pipeline UI + + +
+ + + diff --git a/packages/pipelines/pipeline-server/package.json b/packages/pipelines/pipeline-server/package.json new file mode 100644 index 000000000..9b25e5044 --- /dev/null +++ b/packages/pipelines/pipeline-server/package.json @@ -0,0 +1,80 @@ +{ + "name": "@ucdjs/pipelines-server", + "version": "0.0.1", + "type": "module", + "author": { + "name": "Lucas Norgaard", + "email": "lucasnrgaard@gmail.com", + "url": "https://luxass.dev" + }, + "packageManager": "pnpm@10.27.0", + "license": "MIT", + "homepage": "https://github.com/ucdjs/ucd", + "repository": { + "type": "git", + "url": "git+https://github.com/ucdjs/ucd.git", + "directory": "packages/pipelines/pipeline-server" + }, + "bugs": { + "url": "https://github.com/ucdjs/ucd/issues" + }, + "sideEffects": false, + "imports": { + "#routes/*": "./src/routes/*.ts" + }, + "exports": { + ".": "./dist/server/app.js", + "./package.json": "./package.json" + }, + "files": [ + "dist" + ], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "build": "vite build", + "dev": "vite dev --port 3031", + "clean": "git clean -xdf dist node_modules", + "lint": "eslint .", + "typecheck": "tsc --noEmit -p tsconfig.build.json" + }, + "dependencies": { + "@ucdjs/pipelines-core": "workspace:*", + "@ucdjs/pipelines-executor": "workspace:*", + "@ucdjs/pipelines-loader": "workspace:*", + "@ucdjs/pipelines-ui": "workspace:*", + "chokidar": "catalog:prod", + "crossws": "catalog:prod", + "h3": "catalog:prod", + "pathe": "catalog:prod" + }, + "devDependencies": { + "@eslint-react/eslint-plugin": "catalog:linting", + "@luxass/eslint-config": "catalog:linting", + "@tailwindcss/vite": "catalog:web", + "@tanstack/react-router": "catalog:web", + "@tanstack/router-plugin": "catalog:web", + "@types/node": "catalog:types", + "@types/react": "catalog:types", + "@types/react-dom": "catalog:types", + "@ucdjs-tooling/tsconfig": "workspace:*", + "@ucdjs-tooling/tsdown-config": "workspace:*", + "@vitejs/plugin-react": "catalog:web", + "clsx": "catalog:web", + "eslint": "catalog:linting", + "eslint-plugin-react-hooks": "catalog:linting", + "eslint-plugin-react-refresh": "catalog:linting", + "publint": "catalog:dev", + "react": "catalog:web", + "react-dom": "catalog:web", + "tailwind-merge": "catalog:web", + "tailwindcss": "catalog:web", + "tsdown": "catalog:dev", + "typescript": "catalog:dev", + "vite": "catalog:web" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/pipelines/pipeline-server/src/client/app.tsx b/packages/pipelines/pipeline-server/src/client/app.tsx new file mode 100644 index 000000000..e2a480e86 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/app.tsx @@ -0,0 +1,18 @@ +import { useEffect, useState } from "react"; + +export function App() { + const [message, setMessage] = useState("Loading..."); + + useEffect(() => { + fetch("/api/hello") + .then((r) => r.json()) + .then((d) => setMessage(d.message)); + }, []); + + return ( +
+

UCD Pipeline UI

+

{message}

+
+ ); +} diff --git a/packages/pipelines/pipeline-server/src/client/index.css b/packages/pipelines/pipeline-server/src/client/index.css new file mode 100644 index 000000000..f1d8c73cd --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/index.css @@ -0,0 +1 @@ +@import "tailwindcss"; diff --git a/packages/pipelines/pipeline-server/src/client/main.tsx b/packages/pipelines/pipeline-server/src/client/main.tsx new file mode 100644 index 000000000..4c7ec85f9 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/main.tsx @@ -0,0 +1,10 @@ +import { StrictMode } from "react"; +import { createRoot } from "react-dom/client"; +import { App } from "./app"; +import "./index.css"; + +createRoot(document.getElementById("root")!).render( + + + , +); diff --git a/packages/pipelines/pipeline-server/src/server/app.ts b/packages/pipelines/pipeline-server/src/server/app.ts new file mode 100644 index 000000000..83b1eb77f --- /dev/null +++ b/packages/pipelines/pipeline-server/src/server/app.ts @@ -0,0 +1,73 @@ +import fs from "node:fs"; +import path from "node:path"; +import { H3, serve, serveStatic } from "h3"; +import helloRouter from "./routes/hello"; +import pipelinesRouter from "./routes/pipelines"; + +export interface ServerOptions { + port?: number; +} + +export function createApp(): H3 { + const app = new H3({ debug: true }); + + // Mount route handlers + app.mount("/api/hello", helloRouter); + app.mount("/api/pipelines", pipelinesRouter); + + return app; +} + +export function startServer(options: ServerOptions = {}): void { + const { port = 3030 } = options; + + const app = createApp(); + + // Static file serving for client assets + const clientDir = path.join(import.meta.dirname, "../client"); + + app.use((event) => { + const url = event.path; + + // Skip API routes + if (url.startsWith("/api")) { + return; + } + + return serveStatic(event, { + getContents: (id) => fs.promises.readFile(path.join(clientDir, id)), + getMeta: async (id) => { + const filePath = path.join(clientDir, id); + try { + const stats = await fs.promises.stat(filePath); + if (!stats.isFile()) return; + return { size: stats.size, mtime: stats.mtime }; + } catch { + // File not found - fall through to SPA fallback + } + }, + }); + }); + + // SPA fallback - serve index.html for client-side routing + app.use((event) => { + const url = event.path; + + // Skip API routes + if (url.startsWith("/api")) { + return; + } + + const indexPath = path.join(clientDir, "index.html"); + return fs.promises.readFile(indexPath, "utf-8").then((html) => { + return new Response(html, { + headers: { "content-type": "text/html" }, + }); + }); + }); + + serve(app, { port }); + + // eslint-disable-next-line no-console + console.log(`Pipeline UI running at http://localhost:${port}`); +} diff --git a/packages/pipelines/pipeline-server/src/server/routes/hello.ts b/packages/pipelines/pipeline-server/src/server/routes/hello.ts new file mode 100644 index 000000000..5449ec97f --- /dev/null +++ b/packages/pipelines/pipeline-server/src/server/routes/hello.ts @@ -0,0 +1,10 @@ +import { H3 } from "h3"; + +const router = new H3(); + +router.get("/", () => ({ + message: "Hello from H3!", + timestamp: Date.now(), +})); + +export default router; diff --git a/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts new file mode 100644 index 000000000..62419fb20 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts @@ -0,0 +1,12 @@ +import { H3 } from "h3"; + +const router = new H3(); + +router.get("/", () => ({ + pipelines: [], +})); + +// router.get("/:id", (event) => { ... }); +// router.post("/:id/execute", (event) => { ... }); + +export default router; diff --git a/packages/pipelines/pipeline-server/tsconfig.build.json b/packages/pipelines/pipeline-server/tsconfig.build.json new file mode 100644 index 000000000..36c889e0c --- /dev/null +++ b/packages/pipelines/pipeline-server/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src"], + "exclude": ["dist", "test"] +} diff --git a/packages/pipelines/pipeline-server/tsconfig.json b/packages/pipelines/pipeline-server/tsconfig.json new file mode 100644 index 000000000..9c6dd744b --- /dev/null +++ b/packages/pipelines/pipeline-server/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "@ucdjs-tooling/tsconfig/base", + "include": [ + "src", + "test" + ], + "exclude": ["dist"] +} diff --git a/packages/pipelines/pipeline-server/vite.config.ts b/packages/pipelines/pipeline-server/vite.config.ts new file mode 100644 index 000000000..8e9ff29c2 --- /dev/null +++ b/packages/pipelines/pipeline-server/vite.config.ts @@ -0,0 +1,67 @@ +import type { Plugin } from "vite"; +import tailwindcss from "@tailwindcss/vite"; +import react from "@vitejs/plugin-react"; +import { defineConfig } from "vite"; +import { createApp } from "./src/server/app"; + +function h3DevServerPlugin(): Plugin { + const app = createApp(); + + return { + name: "h3-dev-server", + configureServer(server) { + // Add middleware BEFORE Vite's internal middleware (no return = pre-hook) + // This ensures /api routes are handled before Vite's SPA fallback + server.middlewares.use(async (req, res, next) => { + if (!req.url?.startsWith("/api")) { + return next(); + } + + try { + const response = await app.fetch( + new Request(new URL(req.url, "http://localhost"), { + method: req.method, + headers: req.headers as HeadersInit, + }), + ); + + res.statusCode = response.status; + response.headers.forEach((value, key) => { + res.setHeader(key, value); + }); + + const body = await response.text(); + res.end(body); + } catch (error) { + next(error); + } + }); + }, + }; +} + +export default defineConfig({ + plugins: [react(), tailwindcss(), h3DevServerPlugin()], + environments: { + client: { + build: { + outDir: "dist/client", + }, + }, + server: { + build: { + outDir: "dist/server", + ssr: true, + rollupOptions: { + input: "src/server/app.ts", + }, + }, + }, + }, + builder: { + async buildApp(builder) { + await builder.build(builder.environments.client); + await builder.build(builder.environments.server); + }, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5df956c86..52d59be3a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,12 +110,21 @@ catalogs: cac: specifier: 6.7.14 version: 6.7.14 + chokidar: + specifier: 5.0.0 + version: 5.0.0 + crossws: + specifier: 0.4.4 + version: 0.4.4 defu: specifier: 6.1.4 version: 6.1.4 farver: specifier: 0.4.2 version: 0.4.2 + h3: + specifier: 2.0.1-rc.11 + version: 2.0.1-rc.11 hookable: specifier: 6.0.1 version: 6.0.1 @@ -523,7 +532,7 @@ importers: version: 0.9.4(@types/react-dom@19.2.3(@types/react@19.2.10))(@types/react@19.2.10)(csstype@3.2.3)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.9) '@tanstack/react-form': specifier: catalog:web - version: 1.28.0(@tanstack/react-start@1.157.17(crossws@0.4.3(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + version: 1.28.0(@tanstack/react-start@1.157.17(crossws@0.4.4(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@tanstack/react-query': specifier: catalog:web version: 5.90.20(react@19.2.4) @@ -541,7 +550,7 @@ importers: version: 1.157.17(@tanstack/query-core@5.90.20)(@tanstack/react-query@5.90.20(react@19.2.4))(@tanstack/react-router@1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(@tanstack/router-core@1.157.16)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@tanstack/react-start': specifier: catalog:web - version: 1.157.17(crossws@0.4.3(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + version: 1.157.17(crossws@0.4.4(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) '@ucdjs-internal/shared-ui': specifier: workspace:* version: link:../../packages/shared-ui @@ -678,6 +687,9 @@ importers: '@ucdjs/pipelines-loader': specifier: workspace:* version: link:../pipelines/pipeline-loader + '@ucdjs/pipelines-server': + specifier: workspace:* + version: link:../pipelines/pipeline-server '@ucdjs/schema-gen': specifier: workspace:* version: link:../schema-gen @@ -1147,6 +1159,103 @@ importers: specifier: catalog:dev version: 5.9.3 + packages/pipelines/pipeline-server: + dependencies: + '@ucdjs/pipelines-core': + specifier: workspace:* + version: link:../pipeline-core + '@ucdjs/pipelines-executor': + specifier: workspace:* + version: link:../pipeline-executor + '@ucdjs/pipelines-loader': + specifier: workspace:* + version: link:../pipeline-loader + '@ucdjs/pipelines-ui': + specifier: workspace:* + version: link:../pipeline-ui + chokidar: + specifier: catalog:prod + version: 5.0.0 + crossws: + specifier: catalog:prod + version: 0.4.4(srvx@0.10.1) + h3: + specifier: catalog:prod + version: 2.0.1-rc.11(crossws@0.4.4(srvx@0.10.1)) + pathe: + specifier: catalog:prod + version: 2.0.3 + devDependencies: + '@eslint-react/eslint-plugin': + specifier: catalog:linting + version: 2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@luxass/eslint-config': + specifier: catalog:linting + version: 7.0.0(@eslint-react/eslint-plugin@2.8.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(@vue/compiler-sfc@3.5.25)(eslint-plugin-format@1.3.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-hooks@7.0.1(eslint@9.39.2(jiti@2.6.1)))(eslint-plugin-react-refresh@0.4.26(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)(vitest@4.0.17) + '@tailwindcss/vite': + specifier: catalog:web + version: 4.1.18(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + '@tanstack/react-router': + specifier: catalog:web + version: 1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tanstack/router-plugin': + specifier: catalog:web + version: 1.157.17(@tanstack/react-router@1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + '@types/node': + specifier: catalog:types + version: 22.18.12 + '@types/react': + specifier: catalog:types + version: 19.2.10 + '@types/react-dom': + specifier: catalog:types + version: 19.2.3(@types/react@19.2.10) + '@ucdjs-tooling/tsconfig': + specifier: workspace:* + version: link:../../../tooling/tsconfig + '@ucdjs-tooling/tsdown-config': + specifier: workspace:* + version: link:../../../tooling/tsdown-config + '@vitejs/plugin-react': + specifier: catalog:web + version: 5.1.2(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + clsx: + specifier: catalog:web + version: 2.1.1 + eslint: + specifier: catalog:linting + version: 9.39.2(jiti@2.6.1) + eslint-plugin-react-hooks: + specifier: catalog:linting + version: 7.0.1(eslint@9.39.2(jiti@2.6.1)) + eslint-plugin-react-refresh: + specifier: catalog:linting + version: 0.4.26(eslint@9.39.2(jiti@2.6.1)) + publint: + specifier: catalog:dev + version: 0.3.17 + react: + specifier: catalog:web + version: 19.2.4 + react-dom: + specifier: catalog:web + version: 19.2.4(react@19.2.4) + tailwind-merge: + specifier: catalog:web + version: 3.4.0 + tailwindcss: + specifier: catalog:web + version: 4.1.18 + tsdown: + specifier: catalog:dev + version: 0.20.1(publint@0.3.17)(synckit@0.11.11)(typescript@5.9.3) + typescript: + specifier: catalog:dev + version: 5.9.3 + vite: + specifier: catalog:web + version: 7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2) + packages/pipelines/pipeline-ui: dependencies: '@ucdjs/pipelines-core': @@ -5644,6 +5753,14 @@ packages: srvx: optional: true + crossws@0.4.4: + resolution: {integrity: sha512-w6c4OdpRNnudVmcgr7brb/+/HmYjMQvYToO/oTrprTwxRUiom3LYWU1PMWuD006okbUWpII1Ea9/+kwpUfmyRg==} + peerDependencies: + srvx: '>=0.7.1' + peerDependenciesMeta: + srvx: + optional: true + css-select@5.2.2: resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==} @@ -12560,13 +12677,13 @@ snapshots: - solid-js - utf-8-validate - '@tanstack/react-form@1.28.0(@tanstack/react-start@1.157.17(crossws@0.4.3(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + '@tanstack/react-form@1.28.0(@tanstack/react-start@1.157.17(crossws@0.4.4(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@tanstack/form-core': 1.28.0 '@tanstack/react-store': 0.8.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react: 19.2.4 optionalDependencies: - '@tanstack/react-start': 1.157.17(crossws@0.4.3(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + '@tanstack/react-start': 1.157.17(crossws@0.4.4(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) transitivePeerDependencies: - react-dom @@ -12636,6 +12753,18 @@ snapshots: transitivePeerDependencies: - crossws + '@tanstack/react-start-server@1.157.17(crossws@0.4.4(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@tanstack/history': 1.154.14 + '@tanstack/react-router': 1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tanstack/router-core': 1.157.16 + '@tanstack/start-client-core': 1.157.16 + '@tanstack/start-server-core': 1.157.16(crossws@0.4.4(srvx@0.10.1)) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + transitivePeerDependencies: + - crossws + '@tanstack/react-start@1.157.17(crossws@0.4.3(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@tanstack/react-router': 1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -12656,6 +12785,26 @@ snapshots: - vite-plugin-solid - webpack + '@tanstack/react-start@1.157.17(crossws@0.4.4(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@tanstack/react-router': 1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tanstack/react-start-client': 1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tanstack/react-start-server': 1.157.17(crossws@0.4.4(srvx@0.10.1))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tanstack/router-utils': 1.154.7 + '@tanstack/start-client-core': 1.157.16 + '@tanstack/start-plugin-core': 1.157.17(@tanstack/react-router@1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(crossws@0.4.4(srvx@0.10.1))(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + '@tanstack/start-server-core': 1.157.16(crossws@0.4.4(srvx@0.10.1)) + pathe: 2.0.3 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + vite: 7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2) + transitivePeerDependencies: + - '@rsbuild/core' + - crossws + - supports-color + - vite-plugin-solid + - webpack + '@tanstack/react-store@0.8.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@tanstack/store': 0.8.0 @@ -12776,6 +12925,37 @@ snapshots: - vite-plugin-solid - webpack + '@tanstack/start-plugin-core@1.157.17(@tanstack/react-router@1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(crossws@0.4.4(srvx@0.10.1))(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@babel/code-frame': 7.27.1 + '@babel/core': 7.28.5 + '@babel/types': 7.28.5 + '@rolldown/pluginutils': 1.0.0-beta.40 + '@tanstack/router-core': 1.157.16 + '@tanstack/router-generator': 1.157.16 + '@tanstack/router-plugin': 1.157.17(@tanstack/react-router@1.157.17(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + '@tanstack/router-utils': 1.154.7 + '@tanstack/start-client-core': 1.157.16 + '@tanstack/start-server-core': 1.157.16(crossws@0.4.4(srvx@0.10.1)) + babel-dead-code-elimination: 1.0.11 + cheerio: 1.1.2 + exsolve: 1.0.8 + pathe: 2.0.3 + srvx: 0.10.1 + tinyglobby: 0.2.15 + ufo: 1.6.1 + vite: 7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2) + vitefu: 1.1.1(vite@7.3.1(@types/node@22.18.12)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.2)) + xmlbuilder2: 4.0.3 + zod: 3.25.76 + transitivePeerDependencies: + - '@rsbuild/core' + - '@tanstack/react-router' + - crossws + - supports-color + - vite-plugin-solid + - webpack + '@tanstack/start-server-core@1.157.16(crossws@0.4.3(srvx@0.10.1))': dependencies: '@tanstack/history': 1.154.14 @@ -12788,6 +12968,18 @@ snapshots: transitivePeerDependencies: - crossws + '@tanstack/start-server-core@1.157.16(crossws@0.4.4(srvx@0.10.1))': + dependencies: + '@tanstack/history': 1.154.14 + '@tanstack/router-core': 1.157.16 + '@tanstack/start-client-core': 1.157.16 + '@tanstack/start-storage-context': 1.157.16 + h3-v2: h3@2.0.1-rc.11(crossws@0.4.4(srvx@0.10.1)) + seroval: 1.5.0 + tiny-invariant: 1.3.3 + transitivePeerDependencies: + - crossws + '@tanstack/start-storage-context@1.157.16': dependencies: '@tanstack/router-core': 1.157.16 @@ -13893,6 +14085,10 @@ snapshots: optionalDependencies: srvx: 0.10.1 + crossws@0.4.4(srvx@0.10.1): + optionalDependencies: + srvx: 0.10.1 + css-select@5.2.2: dependencies: boolbase: 1.0.0 @@ -15248,6 +15444,13 @@ snapshots: optionalDependencies: crossws: 0.4.3(srvx@0.10.1) + h3@2.0.1-rc.11(crossws@0.4.4(srvx@0.10.1)): + dependencies: + rou3: 0.7.12 + srvx: 0.10.1 + optionalDependencies: + crossws: 0.4.4(srvx@0.10.1) + has-bigints@1.1.0: {} has-flag@4.0.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ff737f557..2e93f35aa 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -66,6 +66,9 @@ catalogs: hookable: 6.0.1 cac: 6.7.14 tinyglobby: 0.2.15 + h3: 2.0.1-rc.11 + chokidar: 5.0.0 + crossws: 0.4.4 build: tsdown: 0.20.1 From 42243436c317561e6619cb8f2993096eec806464 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 1 Feb 2026 10:40:51 +0100 Subject: [PATCH 17/63] WIP: doesn't work right now. AI generated most of it. --- packages/cli/src/cmd/pipelines/run.ts | 5 +- .../pipelines/pipeline-loader/src/loader.ts | 1 + .../pipelines/pipeline-server/package.json | 1 + .../pipeline-server/src/client/app.tsx | 18 -- .../pipeline-server/src/client/main.tsx | 13 +- .../src/client/routeTree.gen.ts | 77 +++++ .../src/client/routes/__root.tsx | 102 +++++++ .../src/client/routes/index.tsx | 68 +++++ .../src/client/routes/pipelines.$id.tsx | 152 ++++++++++ .../pipeline-server/src/server/app.ts | 42 ++- .../src/server/routes/execute.ts | 59 ++++ .../src/server/routes/hello.ts | 10 - .../src/server/routes/pipelines.ts | 45 ++- .../src/server/routes/versions.ts | 40 +++ .../pipeline-server/src/server/types.ts | 56 ++++ .../pipelines/pipeline-server/vite.config.ts | 26 +- packages/pipelines/pipeline-ui/package.json | 1 + .../src/components/execution-result.tsx | 171 +++++++++++ .../src/components/pipeline-detail.tsx | 281 ++++++++++++++++++ .../src/components/pipeline-sidebar.tsx | 219 ++++++++++++++ .../pipeline-ui/src/hooks/use-execute.ts | 70 +++++ .../pipeline-ui/src/hooks/use-pipeline.ts | 66 ++++ .../pipeline-ui/src/hooks/use-pipelines.ts | 57 ++++ packages/pipelines/pipeline-ui/src/index.ts | 90 +++++- packages/pipelines/pipeline-ui/src/types.ts | 66 ++++ .../pipelines/pipeline-ui/tsdown.config.ts | 9 + pnpm-lock.yaml | 6 + 27 files changed, 1699 insertions(+), 52 deletions(-) delete mode 100644 packages/pipelines/pipeline-server/src/client/app.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/routeTree.gen.ts create mode 100644 packages/pipelines/pipeline-server/src/client/routes/__root.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/routes/index.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx create mode 100644 packages/pipelines/pipeline-server/src/server/routes/execute.ts delete mode 100644 packages/pipelines/pipeline-server/src/server/routes/hello.ts create mode 100644 packages/pipelines/pipeline-server/src/server/routes/versions.ts create mode 100644 packages/pipelines/pipeline-server/src/server/types.ts create mode 100644 packages/pipelines/pipeline-ui/src/components/execution-result.tsx create mode 100644 packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx create mode 100644 packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx create mode 100644 packages/pipelines/pipeline-ui/src/hooks/use-execute.ts create mode 100644 packages/pipelines/pipeline-ui/src/hooks/use-pipeline.ts create mode 100644 packages/pipelines/pipeline-ui/src/hooks/use-pipelines.ts create mode 100644 packages/pipelines/pipeline-ui/src/types.ts diff --git a/packages/cli/src/cmd/pipelines/run.ts b/packages/cli/src/cmd/pipelines/run.ts index e66cd647e..dc5674a07 100644 --- a/packages/cli/src/cmd/pipelines/run.ts +++ b/packages/cli/src/cmd/pipelines/run.ts @@ -1,5 +1,6 @@ import type { Prettify } from "@luxass/utils"; import type { CLIArguments } from "../../cli-utils"; +import process from "node:process"; import { printHelp } from "../../cli-utils"; import { output } from "../../output"; @@ -30,8 +31,10 @@ export async function runPipelinesRun({ flags }: CLIPipelinesRunCmdOptions) { if (flags?.ui) { const { startServer } = await import("@ucdjs/pipelines-server"); const port = flags?.port ?? 3030; + const cwd = process.cwd(); output.info(`Starting Pipeline UI on port ${port}...`); - startServer({ port }); + output.info(`Looking for pipelines in: ${cwd}`); + await startServer({ port, cwd }); return; } diff --git a/packages/pipelines/pipeline-loader/src/loader.ts b/packages/pipelines/pipeline-loader/src/loader.ts index d73a88a86..0e7ca5fd6 100644 --- a/packages/pipelines/pipeline-loader/src/loader.ts +++ b/packages/pipelines/pipeline-loader/src/loader.ts @@ -61,6 +61,7 @@ export interface LoadPipelinesOptions { * ``` */ export async function loadPipelineFile(filePath: string): Promise { + console.log(`Loading pipeline file: ${filePath}`); const module = await import(filePath); const pipelines: PipelineDefinition[] = []; diff --git a/packages/pipelines/pipeline-server/package.json b/packages/pipelines/pipeline-server/package.json index 9b25e5044..10100d012 100644 --- a/packages/pipelines/pipeline-server/package.json +++ b/packages/pipelines/pipeline-server/package.json @@ -40,6 +40,7 @@ "typecheck": "tsc --noEmit -p tsconfig.build.json" }, "dependencies": { + "@ucdjs-internal/shared-ui": "workspace:*", "@ucdjs/pipelines-core": "workspace:*", "@ucdjs/pipelines-executor": "workspace:*", "@ucdjs/pipelines-loader": "workspace:*", diff --git a/packages/pipelines/pipeline-server/src/client/app.tsx b/packages/pipelines/pipeline-server/src/client/app.tsx deleted file mode 100644 index e2a480e86..000000000 --- a/packages/pipelines/pipeline-server/src/client/app.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { useEffect, useState } from "react"; - -export function App() { - const [message, setMessage] = useState("Loading..."); - - useEffect(() => { - fetch("/api/hello") - .then((r) => r.json()) - .then((d) => setMessage(d.message)); - }, []); - - return ( -
-

UCD Pipeline UI

-

{message}

-
- ); -} diff --git a/packages/pipelines/pipeline-server/src/client/main.tsx b/packages/pipelines/pipeline-server/src/client/main.tsx index 4c7ec85f9..1ac67bde1 100644 --- a/packages/pipelines/pipeline-server/src/client/main.tsx +++ b/packages/pipelines/pipeline-server/src/client/main.tsx @@ -1,10 +1,19 @@ +import { createRouter, RouterProvider } from "@tanstack/react-router"; import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; -import { App } from "./app"; +import { routeTree } from "./routeTree.gen"; import "./index.css"; +const router = createRouter({ routeTree }); + +declare module "@tanstack/react-router" { + interface Register { + router: typeof router; + } +} + createRoot(document.getElementById("root")!).render( - + , ); diff --git a/packages/pipelines/pipeline-server/src/client/routeTree.gen.ts b/packages/pipelines/pipeline-server/src/client/routeTree.gen.ts new file mode 100644 index 000000000..cf7bf7cb7 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routeTree.gen.ts @@ -0,0 +1,77 @@ +/* eslint-disable */ + +// @ts-nocheck + +// noinspection JSUnusedGlobalSymbols + +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. + +import { Route as rootRouteImport } from './routes/__root' +import { Route as IndexRouteImport } from './routes/index' +import { Route as PipelinesIdRouteImport } from './routes/pipelines.$id' + +const IndexRoute = IndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => rootRouteImport, +} as any) +const PipelinesIdRoute = PipelinesIdRouteImport.update({ + id: '/pipelines/$id', + path: '/pipelines/$id', + getParentRoute: () => rootRouteImport, +} as any) + +export interface FileRoutesByFullPath { + '/': typeof IndexRoute + '/pipelines/$id': typeof PipelinesIdRoute +} +export interface FileRoutesByTo { + '/': typeof IndexRoute + '/pipelines/$id': typeof PipelinesIdRoute +} +export interface FileRoutesById { + __root__: typeof rootRouteImport + '/': typeof IndexRoute + '/pipelines/$id': typeof PipelinesIdRoute +} +export interface FileRouteTypes { + fileRoutesByFullPath: FileRoutesByFullPath + fullPaths: '/' | '/pipelines/$id' + fileRoutesByTo: FileRoutesByTo + to: '/' | '/pipelines/$id' + id: '__root__' | '/' | '/pipelines/$id' + fileRoutesById: FileRoutesById +} +export interface RootRouteChildren { + IndexRoute: typeof IndexRoute + PipelinesIdRoute: typeof PipelinesIdRoute +} + +declare module '@tanstack/react-router' { + interface FileRoutesByPath { + '/': { + id: '/' + path: '/' + fullPath: '/' + preLoaderRoute: typeof IndexRouteImport + parentRoute: typeof rootRouteImport + } + '/pipelines/$id': { + id: '/pipelines/$id' + path: '/pipelines/$id' + fullPath: '/pipelines/$id' + preLoaderRoute: typeof PipelinesIdRouteImport + parentRoute: typeof rootRouteImport + } + } +} + +const rootRouteChildren: RootRouteChildren = { + IndexRoute: IndexRoute, + PipelinesIdRoute: PipelinesIdRoute, +} +export const routeTree = rootRouteImport + ._addFileChildren(rootRouteChildren) + ._addFileTypes() diff --git a/packages/pipelines/pipeline-server/src/client/routes/__root.tsx b/packages/pipelines/pipeline-server/src/client/routes/__root.tsx new file mode 100644 index 000000000..fbf533381 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/__root.tsx @@ -0,0 +1,102 @@ +import type { PipelineInfo, PipelinesResponse } from "@ucdjs/pipelines-ui"; +import { + createRootRoute, + Link, + Outlet, + useMatches, +} from "@tanstack/react-router"; +import { + PipelineSidebar, + PipelineSidebarItem, + usePipelines, +} from "@ucdjs/pipelines-ui"; +import { createContext, use, useMemo } from "react"; + +interface PipelinesContextValue { + data: PipelinesResponse | null; + loading: boolean; + error: string | null; + refetch: () => void; +} + +const PipelinesContext = createContext(null); + +export function usePipelinesContext(): PipelinesContextValue { + const ctx = use(PipelinesContext); + if (!ctx) { + throw new Error("usePipelinesContext must be used within PipelinesProvider"); + } + return ctx; +} + +function useCurrentPipelineId(): string | undefined { + const matches = useMatches(); + return useMemo(() => { + for (const match of matches) { + const params = match.params as Record | undefined; + if (params && "id" in params) { + return params.id; + } + } + return undefined; + }, [matches]); +} + +function SidebarItemWithLink({ + pipeline, + isActive, +}: { + pipeline: PipelineInfo; + isActive: boolean; +}) { + return ( + + + + ); +} + +function RootLayout() { + const { data, loading, error, refetch } = usePipelines(); + const currentPipelineId = useCurrentPipelineId(); + + // Memoize context value to prevent unnecessary re-renders + const contextValue = useMemo( + () => ({ + data, + loading, + error, + refetch, + }), + [data, loading, error, refetch], + ); + + return ( + +
+ ( + + )} + /> +
+ +
+
+ + ); +} + +export const Route = createRootRoute({ + component: RootLayout, +}); diff --git a/packages/pipelines/pipeline-server/src/client/routes/index.tsx b/packages/pipelines/pipeline-server/src/client/routes/index.tsx new file mode 100644 index 000000000..64ebfdd9a --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/index.tsx @@ -0,0 +1,68 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { memo } from "react"; +import { usePipelinesContext } from "./__root"; + +const HomePage = memo(() => { + const { data, loading } = usePipelinesContext(); + + if (loading) { + return ( +
+

Loading pipelines...

+
+ ); + } + + const pipelineCount = data?.pipelines.length ?? 0; + + return ( +
+
+
+ + + +
+ + {pipelineCount > 0 + ? ( + <> +

+ Select a Pipeline +

+

+ Choose a pipeline from the sidebar to view its details and execute it. +

+ + ) + : ( + <> +

+ No Pipelines Found +

+

+ Create a pipeline file to get started. +

+ + *.ucd-pipeline.ts + + + )} +
+
+ ); +}); + +export const Route = createFileRoute("/")({ + component: HomePage, +}); diff --git a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx new file mode 100644 index 000000000..9425cdce6 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx @@ -0,0 +1,152 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { + ExecutionResult, + RouteList, + SourceList, + useExecute, + usePipeline, + VersionSelector, +} from "@ucdjs/pipelines-ui"; +import { memo, useCallback, useEffect, useState } from "react"; + +const PipelineDetailPage = memo(() => { + const { id } = Route.useParams(); + const { pipeline, loading, error } = usePipeline(id); + const { execute, executing, result } = useExecute(); + + // Selected versions state + const [selectedVersions, setSelectedVersions] = useState>(new Set()); + + // Sync selected versions when pipeline loads + useEffect(() => { + if (pipeline) { + setSelectedVersions(new Set(pipeline.versions)); + } + }, [pipeline]); + + // Version toggle handler + const handleToggleVersion = useCallback((version: string) => { + setSelectedVersions((prev) => { + const next = new Set(prev); + if (next.has(version)) { + next.delete(version); + } else { + next.add(version); + } + return next; + }); + }, []); + + // Select all versions + const handleSelectAll = useCallback(() => { + if (pipeline) { + setSelectedVersions(new Set(pipeline.versions)); + } + }, [pipeline]); + + // Deselect all versions + const handleDeselectAll = useCallback(() => { + setSelectedVersions(new Set()); + }, []); + + // Execute handler + const handleExecute = useCallback(async () => { + if (!pipeline || selectedVersions.size === 0) return; + await execute(id, Array.from(selectedVersions)); + }, [execute, id, pipeline, selectedVersions]); + + // Loading state + if (loading) { + return ( +
+

Loading pipeline...

+
+ ); + } + + // Error state + if (error) { + return ( +
+
+

{error}

+

+ Pipeline ID: + {" "} + {id} +

+
+
+ ); + } + + // Not found state + if (!pipeline) { + return ( +
+

Pipeline not found

+
+ ); + } + + const canExecute = selectedVersions.size > 0 && !executing; + + return ( +
+ {/* Header */} +
+
+
+

+ {pipeline.name || pipeline.id} +

+ {pipeline.description && ( +

+ {pipeline.description} +

+ )} +
+ +
+ + {/* Version selector */} +
+ +
+
+ + {/* Content */} +
+ {/* Execution result */} + {result && } + + {/* Routes */} + + + {/* Sources */} + +
+
+ ); +}); + +export const Route = createFileRoute("/pipelines/$id")({ + component: PipelineDetailPage, +}); diff --git a/packages/pipelines/pipeline-server/src/server/app.ts b/packages/pipelines/pipeline-server/src/server/app.ts index 83b1eb77f..6d3df4619 100644 --- a/packages/pipelines/pipeline-server/src/server/app.ts +++ b/packages/pipelines/pipeline-server/src/server/app.ts @@ -1,27 +1,51 @@ import fs from "node:fs"; import path from "node:path"; +import process from "node:process"; import { H3, serve, serveStatic } from "h3"; -import helloRouter from "./routes/hello"; -import pipelinesRouter from "./routes/pipelines"; +import { executeRouter } from "./routes/execute"; +import { pipelinesRouter } from "./routes/pipelines"; +import { versionsRouter } from "./routes/versions"; -export interface ServerOptions { +export interface AppOptions { + cwd?: string; +} + +export interface ServerOptions extends AppOptions { port?: number; } -export function createApp(): H3 { +declare module "h3" { + interface H3EventContext { + cwd: string; + } +} + +export function createApp(options: AppOptions = {}): H3 { + const { cwd = process.cwd() } = options; + const app = new H3({ debug: true }); - // Mount route handlers - app.mount("/api/hello", helloRouter); + // Middleware to attach cwd to context + app.use("/**", (event, next) => { + event.context.cwd = cwd; + next(); + }); + + app.get("/api/hello", () => ({ + message: "Hello from H3!", + timestamp: Date.now(), + })); app.mount("/api/pipelines", pipelinesRouter); + app.mount("/api/pipelines/:id/execute", executeRouter); + app.mount("/api/versions", versionsRouter); return app; } -export function startServer(options: ServerOptions = {}): void { - const { port = 3030 } = options; +export async function startServer(options: ServerOptions = {}): Promise { + const { port = 3030, cwd = process.cwd() } = options; - const app = createApp(); + const app = createApp({ cwd }); // Static file serving for client assets const clientDir = path.join(import.meta.dirname, "../client"); diff --git a/packages/pipelines/pipeline-server/src/server/routes/execute.ts b/packages/pipelines/pipeline-server/src/server/routes/execute.ts new file mode 100644 index 000000000..0c63736ac --- /dev/null +++ b/packages/pipelines/pipeline-server/src/server/routes/execute.ts @@ -0,0 +1,59 @@ +import { createPipelineExecutor } from "@ucdjs/pipelines-executor"; +import { findPipelineFiles, loadPipelinesFromPaths } from "@ucdjs/pipelines-loader"; +import { H3, readBody } from "h3"; + +export const executeRouter = new H3(); + +interface ExecuteBody { + versions?: string[]; + cache?: boolean; +} + +executeRouter.post("/", async (event) => { + const { cwd } = event.context; + const id = event.context.params?.id; + + if (!id) { + return { error: "Pipeline ID is required" }; + } + + const files = await findPipelineFiles(cwd); + const result = await loadPipelinesFromPaths(files); + + const pipeline = result.pipelines.find((p) => p.id === id); + + if (!pipeline) { + return { error: `Pipeline "${id}" not found` }; + } + + const body = (await readBody(event).catch(() => null)) ?? {}; + const versions = body.versions ?? pipeline.versions; + const cache = body.cache ?? true; + + const executor = createPipelineExecutor({}); + + try { + const execResult = await executor.run([pipeline], { + versions, + cache, + }); + + const pipelineResult = execResult.results.get(id); + + return { + success: true, + pipelineId: id, + summary: pipelineResult?.summary, + errors: pipelineResult?.errors.map((e) => ({ + scope: e.scope, + message: e.message, + })), + }; + } catch (err) { + return { + success: false, + pipelineId: id, + error: err instanceof Error ? err.message : String(err), + }; + } +}); diff --git a/packages/pipelines/pipeline-server/src/server/routes/hello.ts b/packages/pipelines/pipeline-server/src/server/routes/hello.ts deleted file mode 100644 index 5449ec97f..000000000 --- a/packages/pipelines/pipeline-server/src/server/routes/hello.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { H3 } from "h3"; - -const router = new H3(); - -router.get("/", () => ({ - message: "Hello from H3!", - timestamp: Date.now(), -})); - -export default router; diff --git a/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts index 62419fb20..34587073d 100644 --- a/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts +++ b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts @@ -1,12 +1,43 @@ +import { findPipelineFiles, loadPipelinesFromPaths } from "@ucdjs/pipelines-loader"; import { H3 } from "h3"; +import { toPipelineDetails, toPipelineInfo } from "../types"; -const router = new H3(); +export const pipelinesRouter = new H3(); -router.get("/", () => ({ - pipelines: [], -})); +pipelinesRouter.get("/", async (event) => { + const { cwd } = event.context; -// router.get("/:id", (event) => { ... }); -// router.post("/:id/execute", (event) => { ... }); + const files = await findPipelineFiles(["**/*.ucd-pipeline.ts"], cwd); + const result = await loadPipelinesFromPaths(files); -export default router; + return { + pipelines: result.pipelines.map(toPipelineInfo), + cwd, + errors: result.errors.map((e) => ({ + filePath: e.filePath, + message: e.error.message, + })), + }; +}); + +pipelinesRouter.get("/:id", async (event) => { + const { cwd } = event.context; + const id = event.context.params?.id; + + if (!id) { + return { error: "Pipeline ID is required" }; + } + + const files = await findPipelineFiles(cwd); + const result = await loadPipelinesFromPaths(files); + + const pipeline = result.pipelines.find((p) => p.id === id); + + if (!pipeline) { + return { error: `Pipeline "${id}" not found` }; + } + + return { + pipeline: toPipelineDetails(pipeline), + }; +}); diff --git a/packages/pipelines/pipeline-server/src/server/routes/versions.ts b/packages/pipelines/pipeline-server/src/server/routes/versions.ts new file mode 100644 index 000000000..2aa4ec5a9 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/server/routes/versions.ts @@ -0,0 +1,40 @@ +import { findPipelineFiles, loadPipelinesFromPaths } from "@ucdjs/pipelines-loader"; +import { H3 } from "h3"; + +export const versionsRouter = new H3(); + +versionsRouter.get("/", async (event) => { + const { cwd } = event.context; + + const files = await findPipelineFiles(cwd); + const result = await loadPipelinesFromPaths(files); + + // Collect unique versions from all pipelines + const versionsSet = new Set(); + for (const pipeline of result.pipelines) { + for (const version of pipeline.versions) { + versionsSet.add(version); + } + } + + // Sort versions semantically (newest first) + const versions = Array.from(versionsSet).sort((a, b) => { + const aParts = a.split(".").map(Number); + const bParts = b.split(".").map(Number); + const aMajor = aParts[0] ?? 0; + const aMinor = aParts[1] ?? 0; + const aPatch = aParts[2] ?? 0; + const bMajor = bParts[0] ?? 0; + const bMinor = bParts[1] ?? 0; + const bPatch = bParts[2] ?? 0; + + if (aMajor !== bMajor) return bMajor - aMajor; + if (aMinor !== bMinor) return bMinor - aMinor; + return bPatch - aPatch; + }); + + return { + versions, + count: versions.length, + }; +}); diff --git a/packages/pipelines/pipeline-server/src/server/types.ts b/packages/pipelines/pipeline-server/src/server/types.ts new file mode 100644 index 000000000..ea3be5ba5 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/server/types.ts @@ -0,0 +1,56 @@ +import type { PipelineDefinition } from "@ucdjs/pipelines-core"; + +/** + * Serializable pipeline info for the API. + */ +export interface PipelineInfo { + id: string; + name?: string; + description?: string; + versions: string[]; + routeCount: number; + sourceCount: number; +} + +/** + * Detailed pipeline info including routes. + */ +export interface PipelineDetails extends PipelineInfo { + routes: Array<{ + id: string; + cache: boolean; + }>; + sources: Array<{ + id: string; + }>; +} + +/** + * Convert a PipelineDefinition to serializable PipelineInfo. + */ +export function toPipelineInfo(pipeline: PipelineDefinition): PipelineInfo { + return { + id: pipeline.id, + name: pipeline.name, + description: pipeline.description, + versions: pipeline.versions, + routeCount: pipeline.routes.length, + sourceCount: pipeline.inputs.length, + }; +} + +/** + * Convert a PipelineDefinition to detailed info. + */ +export function toPipelineDetails(pipeline: PipelineDefinition): PipelineDetails { + return { + ...toPipelineInfo(pipeline), + routes: pipeline.routes.map((route) => ({ + id: route.id, + cache: route.cache !== false, + })), + sources: pipeline.inputs.map((source) => ({ + id: source.id, + })), + }; +} diff --git a/packages/pipelines/pipeline-server/vite.config.ts b/packages/pipelines/pipeline-server/vite.config.ts index 8e9ff29c2..6cb2987d3 100644 --- a/packages/pipelines/pipeline-server/vite.config.ts +++ b/packages/pipelines/pipeline-server/vite.config.ts @@ -1,5 +1,6 @@ import type { Plugin } from "vite"; import tailwindcss from "@tailwindcss/vite"; +import { TanStackRouterVite } from "@tanstack/router-plugin/vite"; import react from "@vitejs/plugin-react"; import { defineConfig } from "vite"; import { createApp } from "./src/server/app"; @@ -18,10 +19,21 @@ function h3DevServerPlugin(): Plugin { } try { + // Collect request body for POST/PUT/PATCH + let body: string | undefined; + if (req.method && ["POST", "PUT", "PATCH"].includes(req.method)) { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(chunk); + } + body = Buffer.concat(chunks).toString(); + } + const response = await app.fetch( new Request(new URL(req.url, "http://localhost"), { method: req.method, headers: req.headers as HeadersInit, + body, }), ); @@ -30,8 +42,8 @@ function h3DevServerPlugin(): Plugin { res.setHeader(key, value); }); - const body = await response.text(); - res.end(body); + const responseBody = await response.text(); + res.end(responseBody); } catch (error) { next(error); } @@ -41,7 +53,15 @@ function h3DevServerPlugin(): Plugin { } export default defineConfig({ - plugins: [react(), tailwindcss(), h3DevServerPlugin()], + plugins: [ + TanStackRouterVite({ + routesDirectory: "./src/client/routes", + generatedRouteTree: "./src/client/routeTree.gen.ts", + }), + react(), + tailwindcss(), + h3DevServerPlugin(), + ], environments: { client: { build: { diff --git a/packages/pipelines/pipeline-ui/package.json b/packages/pipelines/pipeline-ui/package.json index a6491fd51..840ab2acb 100644 --- a/packages/pipelines/pipeline-ui/package.json +++ b/packages/pipelines/pipeline-ui/package.json @@ -47,6 +47,7 @@ "react-dom": ">=18.0.0" }, "dependencies": { + "@ucdjs-internal/shared-ui": "workspace:*", "@ucdjs/pipelines-core": "workspace:*", "@xyflow/react": "catalog:web", "clsx": "catalog:web", diff --git a/packages/pipelines/pipeline-ui/src/components/execution-result.tsx b/packages/pipelines/pipeline-ui/src/components/execution-result.tsx new file mode 100644 index 000000000..7a031b325 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/execution-result.tsx @@ -0,0 +1,171 @@ +import type { ExecuteResult } from "../types"; +import { cn } from "#lib/utils"; +import { memo } from "react"; + +export interface ExecutionResultProps { + result: ExecuteResult; + className?: string; +} + +/** + * Displays the result of a pipeline execution + */ +export const ExecutionResult = memo(({ + result, + className, +}: ExecutionResultProps) => { + const isSuccess = result.success; + + return ( +
+ {/* Status indicator */} +
+
+ + {isSuccess ? "Completed" : "Failed"} + +
+ + {/* Summary stats */} + {result.summary && ( +
+
+ Routes + {result.summary.totalRoutes} +
+
+ Success + + {result.summary.successfulRoutes} + +
+
+ Failed + {result.summary.failedRoutes} +
+
+ Time + + {result.summary.totalTime} + ms + +
+
+ )} + + {/* Top-level error */} + {result.error && ( +

{result.error}

+ )} + + {/* Detailed errors list */} + {result.errors && result.errors.length > 0 && ( +
+ {result.errors.map((err, i) => ( +
+ + [ + {err.scope} + ] + + {" "} + {err.message} +
+ ))} +
+ )} +
+ ); +}); + +export interface ExecutionSummaryProps { + totalRoutes: number; + successfulRoutes: number; + failedRoutes: number; + totalTime: number; + className?: string; +} + +/** + * Compact execution summary (without result wrapper) + */ +export const ExecutionSummary = memo(({ + totalRoutes, + successfulRoutes, + failedRoutes, + totalTime, + className, +}: ExecutionSummaryProps) => { + return ( +
+
+ Routes + {totalRoutes} +
+
+ Success + {successfulRoutes} +
+
+ Failed + {failedRoutes} +
+
+ Time + + {totalTime} + ms + +
+
+ ); +}); + +export interface ExecutionErrorsProps { + errors: Array<{ scope: string; message: string }>; + className?: string; +} + +/** + * List of execution errors + */ +export const ExecutionErrors = memo(({ + errors, + className, +}: ExecutionErrorsProps) => { + if (errors.length === 0) return null; + + return ( +
+ {errors.map((err, i) => ( +
+ + [ + {err.scope} + ] + + {" "} + {err.message} +
+ ))} +
+ ); +}); diff --git a/packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx b/packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx new file mode 100644 index 000000000..ae5c7f1ed --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx @@ -0,0 +1,281 @@ +import { cn } from "#lib/utils"; +import { memo, useCallback, useMemo, useState } from "react"; + +// --------------------------------------------------------------------------- +// VersionTag +// --------------------------------------------------------------------------- + +export interface VersionTagProps { + version: string; + selected: boolean; + onToggle?: () => void; + className?: string; +} + +/** + * Selectable version tag/badge + */ +export const VersionTag = memo(({ + version, + selected, + onToggle, + className, +}: VersionTagProps) => { + return ( + + ); +}); + +export interface VersionSelectorProps { + versions: string[]; + selectedVersions: Set; + onToggleVersion: (version: string) => void; + onSelectAll?: () => void; + onDeselectAll?: () => void; + className?: string; +} + +/** + * Version selector with all/none controls + */ +export const VersionSelector = memo(({ + versions, + selectedVersions, + onToggleVersion, + onSelectAll, + onDeselectAll, + className, +}: VersionSelectorProps) => { + // Memoize version toggle handlers + const versionToggles = useMemo(() => { + const map = new Map void>(); + for (const v of versions) { + map.set(v, () => onToggleVersion(v)); + } + return map; + }, [versions, onToggleVersion]); + + return ( +
+
+ + Versions ( + {selectedVersions.size} + / + {versions.length} + ) + + {(onSelectAll || onDeselectAll) && ( +
+ {onSelectAll && ( + + )} + {onDeselectAll && ( + + )} +
+ )} +
+
+ {versions.map((version) => ( + + ))} +
+
+ ); +}); + +// --------------------------------------------------------------------------- +// RouteItem +// --------------------------------------------------------------------------- + +export interface RouteItemProps { + route: { id: string; cache: boolean }; + className?: string; +} + +/** + * Pipeline route list item + */ +export const RouteItem = memo(({ + route, + className, +}: RouteItemProps) => { + return ( +
+ {route.id} + {route.cache && ( + + cached + + )} +
+ ); +}); + +export interface RouteListProps { + routes: Array<{ id: string; cache: boolean }>; + className?: string; +} + +/** + * List of pipeline routes + */ +export const RouteList = memo(({ + routes, + className, +}: RouteListProps) => { + return ( +
+

+ Routes ( + {routes.length} + ) +

+
+ {routes.map((route) => ( + + ))} +
+
+ ); +}); + +// --------------------------------------------------------------------------- +// SourceItem +// --------------------------------------------------------------------------- + +export interface SourceItemProps { + source: { id: string }; + className?: string; +} + +/** + * Pipeline source list item + */ +export const SourceItem = memo(({ + source, + className, +}: SourceItemProps) => { + return ( +
+ {source.id} +
+ ); +}); + +export interface SourceListProps { + sources: Array<{ id: string }>; + className?: string; +} + +/** + * List of pipeline sources + */ +export const SourceList = memo(({ + sources, + className, +}: SourceListProps) => { + return ( +
+

+ Sources ( + {sources.length} + ) +

+
+ {sources.map((source) => ( + + ))} +
+
+ ); +}); + +// --------------------------------------------------------------------------- +// useVersionSelection hook +// --------------------------------------------------------------------------- + +export interface UseVersionSelectionReturn { + selectedVersions: Set; + toggleVersion: (version: string) => void; + selectAll: () => void; + deselectAll: () => void; + setVersions: (versions: string[]) => void; +} + +/** + * Hook to manage version selection state + */ +export function useVersionSelection( + initialVersions: string[] = [], +): UseVersionSelectionReturn { + const [selectedVersions, setSelectedVersions] = useState>( + () => new Set(initialVersions), + ); + + const toggleVersion = useCallback((version: string) => { + setSelectedVersions((prev) => { + const next = new Set(prev); + if (next.has(version)) { + next.delete(version); + } else { + next.add(version); + } + return next; + }); + }, []); + + const selectAll = useCallback(() => { + // Requires knowing the full version list - handled via setVersions + }, []); + + const deselectAll = useCallback(() => { + setSelectedVersions(new Set()); + }, []); + + const setVersions = useCallback((versions: string[]) => { + setSelectedVersions(new Set(versions)); + }, []); + + return { + selectedVersions, + toggleVersion, + selectAll, + deselectAll, + setVersions, + }; +} diff --git a/packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx b/packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx new file mode 100644 index 000000000..708499ac2 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx @@ -0,0 +1,219 @@ +import type { PipelineInfo, PipelinesResponse } from "../types"; +import { cn } from "#lib/utils"; +import { memo } from "react"; + +export interface PipelineSidebarItemProps { + pipeline: PipelineInfo; + isActive: boolean; + onClick?: () => void; + className?: string; +} + +/** + * Individual pipeline item in the sidebar + */ +export const PipelineSidebarItem = memo(({ + pipeline, + isActive, + onClick, + className, +}: PipelineSidebarItemProps) => { + return ( + + ); +}); + +export interface PipelineSidebarListProps { + pipelines: PipelineInfo[]; + currentPipelineId?: string; + onPipelineClick?: (pipeline: PipelineInfo) => void; + renderItem?: (pipeline: PipelineInfo, isActive: boolean) => React.ReactNode; + className?: string; +} + +/** + * List of pipelines in the sidebar + */ +export const PipelineSidebarList = memo(({ + pipelines, + currentPipelineId, + onPipelineClick, + renderItem, + className, +}: PipelineSidebarListProps) => { + if (pipelines.length === 0) { + return ( +
+

No pipelines found

+

+ Create a + {" "} + *.ucd-pipeline.ts + {" "} + file +

+
+ ); + } + + return ( +
    + {pipelines.map((pipeline) => { + const isActive = currentPipelineId === pipeline.id; + return ( +
  • + {renderItem + ? ( + renderItem(pipeline, isActive) + ) + : ( + onPipelineClick?.(pipeline)} + /> + )} +
    • + ); + })} +
+ ); +}); + +export interface PipelineSidebarErrorsProps { + count: number; + className?: string; +} + +/** + * Error indicator shown at the bottom of the sidebar + */ +export const PipelineSidebarErrors = memo(({ + count, + className, +}: PipelineSidebarErrorsProps) => { + if (count === 0) return null; + + return ( +
+

+ {count} + {" "} + load error + {count !== 1 ? "s" : ""} +

+
+ ); +}); + +export interface PipelineSidebarHeaderProps { + title?: string; + cwd?: string; + className?: string; +} + +/** + * Header section of the sidebar + */ +export const PipelineSidebarHeader = memo(({ + title = "UCD Pipelines", + cwd, + className, +}: PipelineSidebarHeaderProps) => { + return ( +
+

{title}

+ {cwd && ( +

+ {cwd} +

+ )} +
+ ); +}); + +export interface PipelineSidebarProps { + /** Pipeline data from the API */ + data: PipelinesResponse | null; + /** Whether data is currently loading */ + loading: boolean; + /** Currently selected pipeline ID */ + currentPipelineId?: string; + /** Callback when a pipeline is clicked */ + onPipelineClick?: (pipeline: PipelineInfo) => void; + /** Custom item renderer */ + renderItem?: (pipeline: PipelineInfo, isActive: boolean) => React.ReactNode; + /** Sidebar title */ + title?: string; + /** Additional className for the container */ + className?: string; +} + +/** + * Complete pipeline sidebar component + */ +export const PipelineSidebar = memo(({ + data, + loading, + currentPipelineId, + onPipelineClick, + renderItem, + title, + className, +}: PipelineSidebarProps) => { + return ( + + ); +}); diff --git a/packages/pipelines/pipeline-ui/src/hooks/use-execute.ts b/packages/pipelines/pipeline-ui/src/hooks/use-execute.ts new file mode 100644 index 000000000..38de3aba4 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/hooks/use-execute.ts @@ -0,0 +1,70 @@ +import type { ExecuteResult } from "../types"; +import { useCallback, useState } from "react"; + +export interface UseExecuteOptions { + /** Base URL for the API (default: "") */ + baseUrl?: string; +} + +export interface UseExecuteReturn { + execute: (pipelineId: string, versions: string[]) => Promise; + executing: boolean; + result: ExecuteResult | null; + error: string | null; + reset: () => void; +} + +/** + * Hook to execute a pipeline with selected versions + */ +export function useExecute(options: UseExecuteOptions = {}): UseExecuteReturn { + const { baseUrl = "" } = options; + + const [executing, setExecuting] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + + const execute = useCallback( + async (pipelineId: string, versions: string[]): Promise => { + setExecuting(true); + setError(null); + setResult(null); + + try { + const res = await fetch(`${baseUrl}/api/pipelines/${pipelineId}/execute`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ versions }), + }); + const data: ExecuteResult = await res.json(); + setResult(data); + return data; + } catch (err) { + const errorResult: ExecuteResult = { + success: false, + pipelineId, + error: err instanceof Error ? err.message : String(err), + }; + setResult(errorResult); + setError(errorResult.error ?? "Execution failed"); + return errorResult; + } finally { + setExecuting(false); + } + }, + [baseUrl], + ); + + const reset = useCallback(() => { + setResult(null); + setError(null); + }, []); + + return { + execute, + executing, + result, + error, + reset, + }; +} diff --git a/packages/pipelines/pipeline-ui/src/hooks/use-pipeline.ts b/packages/pipelines/pipeline-ui/src/hooks/use-pipeline.ts new file mode 100644 index 000000000..39bdb54af --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/hooks/use-pipeline.ts @@ -0,0 +1,66 @@ +import type { PipelineDetails, PipelineResponse } from "../types"; +import { useCallback, useEffect, useState } from "react"; + +export interface UsePipelineOptions { + /** Base URL for the API (default: "") */ + baseUrl?: string; + /** Whether to fetch on mount (default: true) */ + fetchOnMount?: boolean; +} + +export interface UsePipelineReturn { + pipeline: PipelineDetails | null; + loading: boolean; + error: string | null; + refetch: () => void; +} + +/** + * Hook to fetch and manage a single pipeline by ID + */ +export function usePipeline( + pipelineId: string, + options: UsePipelineOptions = {}, +): UsePipelineReturn { + const { baseUrl = "", fetchOnMount = true } = options; + + const [pipeline, setPipeline] = useState(null); + const [loading, setLoading] = useState(fetchOnMount); + const [error, setError] = useState(null); + + const fetchPipeline = useCallback(async () => { + setLoading(true); + setError(null); + try { + const res = await fetch(`${baseUrl}/api/pipelines/${pipelineId}`); + if (!res.ok) { + throw new Error(`HTTP ${res.status}`); + } + const json: PipelineResponse = await res.json(); + if (json.error) { + setError(json.error); + setPipeline(null); + } else { + setPipeline(json.pipeline ?? null); + } + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to load pipeline"); + setPipeline(null); + } finally { + setLoading(false); + } + }, [baseUrl, pipelineId]); + + useEffect(() => { + if (fetchOnMount) { + fetchPipeline(); + } + }, [fetchOnMount, fetchPipeline]); + + return { + pipeline, + loading, + error, + refetch: fetchPipeline, + }; +} diff --git a/packages/pipelines/pipeline-ui/src/hooks/use-pipelines.ts b/packages/pipelines/pipeline-ui/src/hooks/use-pipelines.ts new file mode 100644 index 000000000..a0c778043 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/hooks/use-pipelines.ts @@ -0,0 +1,57 @@ +import type { PipelinesResponse } from "../types"; +import { useCallback, useEffect, useState } from "react"; + +export interface UsePipelinesOptions { + /** Base URL for the API (default: "") */ + baseUrl?: string; + /** Whether to fetch on mount (default: true) */ + fetchOnMount?: boolean; +} + +export interface UsePipelinesReturn { + data: PipelinesResponse | null; + loading: boolean; + error: string | null; + refetch: () => void; +} + +/** + * Hook to fetch and manage the list of all pipelines + */ +export function usePipelines(options: UsePipelinesOptions = {}): UsePipelinesReturn { + const { baseUrl = "", fetchOnMount = true } = options; + + const [data, setData] = useState(null); + const [loading, setLoading] = useState(fetchOnMount); + const [error, setError] = useState(null); + + const fetchPipelines = useCallback(async () => { + setLoading(true); + setError(null); + try { + const res = await fetch(`${baseUrl}/api/pipelines`); + if (!res.ok) { + throw new Error(`HTTP ${res.status}`); + } + const json = await res.json(); + setData(json); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to load pipelines"); + } finally { + setLoading(false); + } + }, [baseUrl]); + + useEffect(() => { + if (fetchOnMount) { + fetchPipelines(); + } + }, [fetchOnMount, fetchPipelines]); + + return { + data, + loading, + error, + refetch: fetchPipelines, + }; +} diff --git a/packages/pipelines/pipeline-ui/src/index.ts b/packages/pipelines/pipeline-ui/src/index.ts index 6a13f270b..93501ade7 100644 --- a/packages/pipelines/pipeline-ui/src/index.ts +++ b/packages/pipelines/pipeline-ui/src/index.ts @@ -1,3 +1,85 @@ +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- +export type { + ExecuteResult, + LoadError, + PipelineDetails, + PipelineInfo, + PipelineResponse, + PipelinesResponse, +} from "./types"; + +// --------------------------------------------------------------------------- +// Hooks +// --------------------------------------------------------------------------- +export { + usePipeline, + type UsePipelineOptions, + type UsePipelineReturn, +} from "./hooks/use-pipeline"; +export { + usePipelines, + type UsePipelinesOptions, + type UsePipelinesReturn, +} from "./hooks/use-pipelines"; +export { + useExecute, + type UseExecuteOptions, + type UseExecuteReturn, +} from "./hooks/use-execute"; + +// --------------------------------------------------------------------------- +// Sidebar Components +// --------------------------------------------------------------------------- +export { + PipelineSidebar, + PipelineSidebarErrors, + PipelineSidebarHeader, + PipelineSidebarItem, + PipelineSidebarList, + type PipelineSidebarErrorsProps, + type PipelineSidebarHeaderProps, + type PipelineSidebarItemProps, + type PipelineSidebarListProps, + type PipelineSidebarProps, +} from "./components/pipeline-sidebar"; + +// --------------------------------------------------------------------------- +// Pipeline Detail Components +// --------------------------------------------------------------------------- +export { + RouteItem, + RouteList, + SourceItem, + SourceList, + VersionSelector, + VersionTag, + useVersionSelection, + type RouteItemProps, + type RouteListProps, + type SourceItemProps, + type SourceListProps, + type UseVersionSelectionReturn, + type VersionSelectorProps, + type VersionTagProps, +} from "./components/pipeline-detail"; + +// --------------------------------------------------------------------------- +// Execution Components +// --------------------------------------------------------------------------- +export { + ExecutionErrors, + ExecutionResult, + ExecutionSummary, + type ExecutionErrorsProps, + type ExecutionResultProps, + type ExecutionSummaryProps, +} from "./components/execution-result"; + +// --------------------------------------------------------------------------- +// Pipeline Graph Components (existing) +// --------------------------------------------------------------------------- export { PipelineGraphDetails, type PipelineGraphDetailsProps } from "./components/details"; export { PipelineGraphFilters, type PipelineGraphFiltersProps } from "./components/filters"; export { nodeTypes } from "./components/node-types"; @@ -5,16 +87,20 @@ export { ArtifactNode, FileNode, OutputNode, - type PipelineNodeData, RouteNode, SourceNode, + type PipelineNodeData, } from "./components/nodes"; export { PipelineGraph, type PipelineGraphProps } from "./components/pipeline-graph"; + +// --------------------------------------------------------------------------- +// Utilities +// --------------------------------------------------------------------------- export { filterNodesByType, + pipelineGraphToFlow, type PipelineFlowEdge, type PipelineFlowNode, - pipelineGraphToFlow, } from "./lib/adapter"; export { getNodeColor, nodeTypeColors } from "./lib/colors"; export { applyLayout, NODE_HEIGHT, NODE_WIDTH } from "./lib/layout"; diff --git a/packages/pipelines/pipeline-ui/src/types.ts b/packages/pipelines/pipeline-ui/src/types.ts new file mode 100644 index 000000000..d9b6a5634 --- /dev/null +++ b/packages/pipelines/pipeline-ui/src/types.ts @@ -0,0 +1,66 @@ +/** + * Summary information about a pipeline for list views + */ +export interface PipelineInfo { + id: string; + name?: string; + description?: string; + versions: string[]; + routeCount: number; + sourceCount: number; +} + +/** + * Detailed pipeline information including routes and sources + */ +export interface PipelineDetails { + id: string; + name?: string; + description?: string; + versions: string[]; + routeCount: number; + sourceCount: number; + routes: Array<{ id: string; cache: boolean }>; + sources: Array<{ id: string }>; +} + +/** + * Error that occurred while loading a pipeline file + */ +export interface LoadError { + filePath: string; + message: string; +} + +/** + * Response from the pipelines list endpoint + */ +export interface PipelinesResponse { + pipelines: PipelineInfo[]; + cwd: string; + errors: LoadError[]; +} + +/** + * Response from the single pipeline endpoint + */ +export interface PipelineResponse { + pipeline?: PipelineDetails; + error?: string; +} + +/** + * Result of pipeline execution + */ +export interface ExecuteResult { + success: boolean; + pipelineId: string; + summary?: { + totalRoutes: number; + successfulRoutes: number; + failedRoutes: number; + totalTime: number; + }; + errors?: Array<{ scope: string; message: string }>; + error?: string; +} diff --git a/packages/pipelines/pipeline-ui/tsdown.config.ts b/packages/pipelines/pipeline-ui/tsdown.config.ts index ae388ddeb..6ee8e0d5e 100644 --- a/packages/pipelines/pipeline-ui/tsdown.config.ts +++ b/packages/pipelines/pipeline-ui/tsdown.config.ts @@ -4,6 +4,15 @@ export default createTsdownConfig({ entry: [ "./src/index.ts", ], + exports: { + customExports(exports) { + exports["./styles/globals.css"] = "./src/styles/globals.css"; + exports["./package.json"] = "./package.json"; + + return exports; + }, + packageJson: false, + }, format: "esm", inputOptions: { transform: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 52d59be3a..0dbf4bdfd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1161,6 +1161,9 @@ importers: packages/pipelines/pipeline-server: dependencies: + '@ucdjs-internal/shared-ui': + specifier: workspace:* + version: link:../../shared-ui '@ucdjs/pipelines-core': specifier: workspace:* version: link:../pipeline-core @@ -1258,6 +1261,9 @@ importers: packages/pipelines/pipeline-ui: dependencies: + '@ucdjs-internal/shared-ui': + specifier: workspace:* + version: link:../../shared-ui '@ucdjs/pipelines-core': specifier: workspace:* version: link:../pipeline-core From feaf778a14497e7bfeda6b678896d1e4723c37cc Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 1 Feb 2026 10:48:31 +0100 Subject: [PATCH 18/63] refactor(pipeline-loader): update findPipelineFiles interface Refactor `findPipelineFiles` to accept an options object for improved flexibility. Update related function calls throughout the codebase to use the new structure. --- packages/cli/src/cmd/pipelines/list.ts | 4 +- .../pipelines/pipeline-loader/src/find.ts | 42 +++++++++++++++---- .../pipelines/pipeline-loader/src/index.ts | 1 + .../pipelines/pipeline-loader/src/loader.ts | 2 - .../pipeline-loader/test/loader.test.ts | 8 +++- .../src/server/routes/execute.ts | 2 +- .../src/server/routes/pipelines.ts | 8 +++- .../src/server/routes/versions.ts | 2 +- 8 files changed, 52 insertions(+), 17 deletions(-) diff --git a/packages/cli/src/cmd/pipelines/list.ts b/packages/cli/src/cmd/pipelines/list.ts index 39de2d8b5..192b71e2d 100644 --- a/packages/cli/src/cmd/pipelines/list.ts +++ b/packages/cli/src/cmd/pipelines/list.ts @@ -40,7 +40,9 @@ export async function runListPipelines({ flags }: CLIPipelinesRunCmdOptions) { const cwd = flags?.cwd ?? process.cwd(); output.info("Searching for pipeline files..."); - const files = await findPipelineFiles(["**/*.ucd-pipeline.ts"], cwd); + const files = await findPipelineFiles({ + cwd, + }); if (files.length === 0) { output.info("No pipeline files found (pattern: **/*.ucd-pipeline.ts)."); diff --git a/packages/pipelines/pipeline-loader/src/find.ts b/packages/pipelines/pipeline-loader/src/find.ts index 00b1feb24..f2485c33b 100644 --- a/packages/pipelines/pipeline-loader/src/find.ts +++ b/packages/pipelines/pipeline-loader/src/find.ts @@ -1,22 +1,48 @@ import { glob } from "tinyglobby"; +export interface FindPipelineFilesOptions { + /** + * Glob patterns to match pipeline files. + * Defaults to `**\/*.ucd-pipeline.ts` + */ + patterns?: string | string[]; + + /** + * Current working directory to resolve patterns against. + * Defaults to `process.cwd()` + */ + cwd?: string; +} + /** * Find pipeline files on disk. * * By default matches files named `*.ucd-pipeline.ts` (the repository standard). * - * @param {string | string[]} patterns glob string or array of glob strings - * @param {string} cwd optional working directory (defaults to process.cwd()) + * @param {FindPipelineFilesOptions} options options for finding pipeline files + * @returns {Promise} Array of matched file paths + * + * @example + * ```ts + * const files = await findPipelineFiles({ cwd: "./pipelines" }); + * console.log(files); // Array of file paths + * ``` */ export async function findPipelineFiles( - patterns: string | string[] = ["**/*.ucd-pipeline.ts"], - cwd?: string, + options: FindPipelineFilesOptions = {}, ): Promise { - const p = Array.isArray(patterns) ? patterns : [patterns]; + let patterns: string[] = ["**/*.ucd-pipeline.ts"]; + // eslint-disable-next-line node/prefer-global/process + const resolvedCwd = options.cwd ?? process.cwd(); + + if (options.patterns) { + patterns = Array.isArray(options.patterns) + ? options.patterns + : [options.patterns]; + } - return glob(p, { - // eslint-disable-next-line node/prefer-global/process - cwd: cwd ?? process.cwd(), + return glob(patterns, { + cwd: resolvedCwd, ignore: ["node_modules/**", "**/node_modules/**", "**/dist/**", "**/build/**", "**/.git/**"], absolute: true, onlyFiles: true, diff --git a/packages/pipelines/pipeline-loader/src/index.ts b/packages/pipelines/pipeline-loader/src/index.ts index ab8100ad4..dbd1a3782 100644 --- a/packages/pipelines/pipeline-loader/src/index.ts +++ b/packages/pipelines/pipeline-loader/src/index.ts @@ -1,3 +1,4 @@ +export type { FindPipelineFilesOptions } from "./find"; export { findPipelineFiles } from "./find"; export { diff --git a/packages/pipelines/pipeline-loader/src/loader.ts b/packages/pipelines/pipeline-loader/src/loader.ts index 0e7ca5fd6..333054890 100644 --- a/packages/pipelines/pipeline-loader/src/loader.ts +++ b/packages/pipelines/pipeline-loader/src/loader.ts @@ -61,7 +61,6 @@ export interface LoadPipelinesOptions { * ``` */ export async function loadPipelineFile(filePath: string): Promise { - console.log(`Loading pipeline file: ${filePath}`); const module = await import(filePath); const pipelines: PipelineDefinition[] = []; @@ -123,7 +122,6 @@ export async function loadPipelinesFromPaths( }; } - // Collect errors instead of throwing; preserve order. const settled = await Promise.allSettled(filePaths.map((fp) => loadPipelineFile(fp))); const files: LoadedPipelineFile[] = []; diff --git a/packages/pipelines/pipeline-loader/test/loader.test.ts b/packages/pipelines/pipeline-loader/test/loader.test.ts index 23eb51fc5..aab1f104d 100644 --- a/packages/pipelines/pipeline-loader/test/loader.test.ts +++ b/packages/pipelines/pipeline-loader/test/loader.test.ts @@ -46,7 +46,9 @@ describe("findPipelineFiles", () => { }, }); - const files = await findPipelineFiles("**/*.ucd-pipeline.ts", root); + const files = await findPipelineFiles({ + cwd: root, + }); const expected = [ path.join(root, "pipelines", "alpha.ucd-pipeline.ts"), path.join(root, "pipelines", "nested", "beta.ucd-pipeline.ts"), @@ -68,7 +70,9 @@ describe("findPipelineFiles", () => { }); const cwd = path.join(root, "pipelines"); - const files = await findPipelineFiles(["**/*.ucd-pipeline.ts"], cwd); + const files = await findPipelineFiles({ + cwd, + }); expect(files).toEqual([path.join(cwd, "gamma.ucd-pipeline.ts")]); }); diff --git a/packages/pipelines/pipeline-server/src/server/routes/execute.ts b/packages/pipelines/pipeline-server/src/server/routes/execute.ts index 0c63736ac..80a48b508 100644 --- a/packages/pipelines/pipeline-server/src/server/routes/execute.ts +++ b/packages/pipelines/pipeline-server/src/server/routes/execute.ts @@ -17,7 +17,7 @@ executeRouter.post("/", async (event) => { return { error: "Pipeline ID is required" }; } - const files = await findPipelineFiles(cwd); + const files = await findPipelineFiles({ cwd }); const result = await loadPipelinesFromPaths(files); const pipeline = result.pipelines.find((p) => p.id === id); diff --git a/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts index 34587073d..a43824bf2 100644 --- a/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts +++ b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts @@ -7,7 +7,9 @@ export const pipelinesRouter = new H3(); pipelinesRouter.get("/", async (event) => { const { cwd } = event.context; - const files = await findPipelineFiles(["**/*.ucd-pipeline.ts"], cwd); + const files = await findPipelineFiles({ + cwd, + }); const result = await loadPipelinesFromPaths(files); return { @@ -28,7 +30,9 @@ pipelinesRouter.get("/:id", async (event) => { return { error: "Pipeline ID is required" }; } - const files = await findPipelineFiles(cwd); + const files = await findPipelineFiles({ + cwd, + }); const result = await loadPipelinesFromPaths(files); const pipeline = result.pipelines.find((p) => p.id === id); diff --git a/packages/pipelines/pipeline-server/src/server/routes/versions.ts b/packages/pipelines/pipeline-server/src/server/routes/versions.ts index 2aa4ec5a9..3e1e51589 100644 --- a/packages/pipelines/pipeline-server/src/server/routes/versions.ts +++ b/packages/pipelines/pipeline-server/src/server/routes/versions.ts @@ -6,7 +6,7 @@ export const versionsRouter = new H3(); versionsRouter.get("/", async (event) => { const { cwd } = event.context; - const files = await findPipelineFiles(cwd); + const files = await findPipelineFiles({ cwd }); const result = await loadPipelinesFromPaths(files); // Collect unique versions from all pipelines From 990056fbc44a1c003deabde528ef31c4a9555e1b Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 4 Feb 2026 19:35:08 +0100 Subject: [PATCH 19/63] feat(pipeline-ui): update styles and components for new design - Refactored `pipeline-detail`, `pipeline-graph`, and `pipeline-sidebar` components to use new color schemes and styles. - Added search functionality to `PipelineSidebar` for improved user experience. - Enhanced `usePipelines` and `useExecute` hooks to support search queries. - Introduced global CSS styles for consistent theming across the application. - Updated types in `types.ts` to reflect changes in pipeline structure and execution results. BREAKING CHANGE: The structure of the `PipelineDetails` and `ExecuteResult` types has been modified to include new fields. --- .../pipeline-server/eslint.config.js | 2 +- .../pipelines/pipeline-server/package.json | 5 +- .../src/client/hooks/pipeline-context.ts | 12 + .../client/hooks/pipeline-detail-context.ts | 12 + .../pipeline-server/src/client/index.css | 2 + .../src/client/routeTree.gen.ts | 136 ++++++- .../src/client/routes/__root.tsx | 53 +-- .../src/client/routes/index.tsx | 122 +++--- .../src/client/routes/pipelines.$id.code.tsx | 102 +++++ .../src/client/routes/pipelines.$id.graph.tsx | 48 +++ .../src/client/routes/pipelines.$id.index.tsx | 91 +++++ .../client/routes/pipelines.$id.inspect.tsx | 324 +++++++++++++++ .../src/client/routes/pipelines.$id.logs.tsx | 105 +++++ .../src/client/routes/pipelines.$id.tsx | 368 +++++++++++++----- .../pipeline-server/src/client/types.ts | 53 +++ .../pipeline-server/src/server/app.ts | 13 +- .../pipeline-server/src/server/code.ts | 119 ++++++ .../src/server/routes/execute.ts | 19 +- .../src/server/routes/pipelines.ts | 61 ++- .../pipeline-server/src/server/types.ts | 40 +- .../pipeline-server/tsconfig.build.json | 10 +- packages/pipelines/pipeline-server/turbo.json | 15 + .../pipelines/pipeline-server/vite.config.ts | 41 +- packages/pipelines/pipeline-ui/package.json | 15 + .../src/components/execution-result.tsx | 96 +++-- .../src/components/pipeline-detail.tsx | 35 +- .../src/components/pipeline-graph.tsx | 5 +- .../src/components/pipeline-sidebar.tsx | 61 ++- .../pipeline-ui/src/hooks/use-execute.ts | 7 +- .../pipeline-ui/src/hooks/use-pipelines.ts | 16 +- packages/pipelines/pipeline-ui/src/index.ts | 141 +++---- .../pipeline-ui/src/styles/globals.css | 3 + packages/pipelines/pipeline-ui/src/types.ts | 27 +- .../pipelines/pipeline-ui/tsdown.config.ts | 8 +- packages/pipelines/pipeline-ui/turbo.json | 15 + pnpm-lock.yaml | 239 ++++++++++++ tooling/tsconfig/base.json | 4 + 37 files changed, 2037 insertions(+), 388 deletions(-) create mode 100644 packages/pipelines/pipeline-server/src/client/hooks/pipeline-context.ts create mode 100644 packages/pipelines/pipeline-server/src/client/hooks/pipeline-detail-context.ts create mode 100644 packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.code.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.graph.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.index.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.inspect.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.logs.tsx create mode 100644 packages/pipelines/pipeline-server/src/client/types.ts create mode 100644 packages/pipelines/pipeline-server/src/server/code.ts create mode 100644 packages/pipelines/pipeline-server/turbo.json create mode 100644 packages/pipelines/pipeline-ui/src/styles/globals.css create mode 100644 packages/pipelines/pipeline-ui/turbo.json diff --git a/packages/pipelines/pipeline-server/eslint.config.js b/packages/pipelines/pipeline-server/eslint.config.js index ab060e8c2..df23f0f0b 100644 --- a/packages/pipelines/pipeline-server/eslint.config.js +++ b/packages/pipelines/pipeline-server/eslint.config.js @@ -2,7 +2,7 @@ import { luxass } from "@luxass/eslint-config"; export default luxass({ - type: "lib", + type: "app", pnpm: true, react: true, }); diff --git a/packages/pipelines/pipeline-server/package.json b/packages/pipelines/pipeline-server/package.json index 10100d012..9e10e8743 100644 --- a/packages/pipelines/pipeline-server/package.json +++ b/packages/pipelines/pipeline-server/package.json @@ -47,7 +47,9 @@ "@ucdjs/pipelines-ui": "workspace:*", "chokidar": "catalog:prod", "crossws": "catalog:prod", + "esrap": "2.2.2", "h3": "catalog:prod", + "oxc-parser": "0.112.0", "pathe": "catalog:prod" }, "devDependencies": { @@ -73,7 +75,8 @@ "tailwindcss": "catalog:web", "tsdown": "catalog:dev", "typescript": "catalog:dev", - "vite": "catalog:web" + "vite": "catalog:web", + "vite-tsconfig-paths": "catalog:web" }, "publishConfig": { "access": "public" diff --git a/packages/pipelines/pipeline-server/src/client/hooks/pipeline-context.ts b/packages/pipelines/pipeline-server/src/client/hooks/pipeline-context.ts new file mode 100644 index 000000000..4eb959b34 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/hooks/pipeline-context.ts @@ -0,0 +1,12 @@ +import type { PipelinesContextValue } from "../types"; +import { createContext, use } from "react"; + +export const PipelinesContext = createContext(null); + +export function usePipelinesContext(): PipelinesContextValue { + const ctx = use(PipelinesContext); + if (!ctx) { + throw new Error("usePipelinesContext must be used within PipelinesProvider"); + } + return ctx; +} diff --git a/packages/pipelines/pipeline-server/src/client/hooks/pipeline-detail-context.ts b/packages/pipelines/pipeline-server/src/client/hooks/pipeline-detail-context.ts new file mode 100644 index 000000000..1207c5314 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/hooks/pipeline-detail-context.ts @@ -0,0 +1,12 @@ +import type { PipelineDetailContextValue } from "../types"; +import { createContext, use } from "react"; + +export const PipelineDetailContext = createContext(null); + +export function usePipelineDetailContext(): PipelineDetailContextValue { + const ctx = use(PipelineDetailContext); + if (!ctx) { + throw new Error("usePipelineDetailContext must be used within PipelineDetailLayout"); + } + return ctx; +} diff --git a/packages/pipelines/pipeline-server/src/client/index.css b/packages/pipelines/pipeline-server/src/client/index.css index f1d8c73cd..a60504aa5 100644 --- a/packages/pipelines/pipeline-server/src/client/index.css +++ b/packages/pipelines/pipeline-server/src/client/index.css @@ -1 +1,3 @@ @import "tailwindcss"; +@import "@ucdjs-internal/shared-ui/styles/globals.css"; +@import "@ucdjs/pipelines-ui/styles.css"; diff --git a/packages/pipelines/pipeline-server/src/client/routeTree.gen.ts b/packages/pipelines/pipeline-server/src/client/routeTree.gen.ts index cf7bf7cb7..59d7eed03 100644 --- a/packages/pipelines/pipeline-server/src/client/routeTree.gen.ts +++ b/packages/pipelines/pipeline-server/src/client/routeTree.gen.ts @@ -11,6 +11,11 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as IndexRouteImport } from './routes/index' import { Route as PipelinesIdRouteImport } from './routes/pipelines.$id' +import { Route as PipelinesIdIndexRouteImport } from './routes/pipelines.$id.index' +import { Route as PipelinesIdLogsRouteImport } from './routes/pipelines.$id.logs' +import { Route as PipelinesIdInspectRouteImport } from './routes/pipelines.$id.inspect' +import { Route as PipelinesIdGraphRouteImport } from './routes/pipelines.$id.graph' +import { Route as PipelinesIdCodeRouteImport } from './routes/pipelines.$id.code' const IndexRoute = IndexRouteImport.update({ id: '/', @@ -22,31 +27,91 @@ const PipelinesIdRoute = PipelinesIdRouteImport.update({ path: '/pipelines/$id', getParentRoute: () => rootRouteImport, } as any) +const PipelinesIdIndexRoute = PipelinesIdIndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => PipelinesIdRoute, +} as any) +const PipelinesIdLogsRoute = PipelinesIdLogsRouteImport.update({ + id: '/logs', + path: '/logs', + getParentRoute: () => PipelinesIdRoute, +} as any) +const PipelinesIdInspectRoute = PipelinesIdInspectRouteImport.update({ + id: '/inspect', + path: '/inspect', + getParentRoute: () => PipelinesIdRoute, +} as any) +const PipelinesIdGraphRoute = PipelinesIdGraphRouteImport.update({ + id: '/graph', + path: '/graph', + getParentRoute: () => PipelinesIdRoute, +} as any) +const PipelinesIdCodeRoute = PipelinesIdCodeRouteImport.update({ + id: '/code', + path: '/code', + getParentRoute: () => PipelinesIdRoute, +} as any) export interface FileRoutesByFullPath { '/': typeof IndexRoute - '/pipelines/$id': typeof PipelinesIdRoute + '/pipelines/$id': typeof PipelinesIdRouteWithChildren + '/pipelines/$id/code': typeof PipelinesIdCodeRoute + '/pipelines/$id/graph': typeof PipelinesIdGraphRoute + '/pipelines/$id/inspect': typeof PipelinesIdInspectRoute + '/pipelines/$id/logs': typeof PipelinesIdLogsRoute + '/pipelines/$id/': typeof PipelinesIdIndexRoute } export interface FileRoutesByTo { '/': typeof IndexRoute - '/pipelines/$id': typeof PipelinesIdRoute + '/pipelines/$id/code': typeof PipelinesIdCodeRoute + '/pipelines/$id/graph': typeof PipelinesIdGraphRoute + '/pipelines/$id/inspect': typeof PipelinesIdInspectRoute + '/pipelines/$id/logs': typeof PipelinesIdLogsRoute + '/pipelines/$id': typeof PipelinesIdIndexRoute } export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute - '/pipelines/$id': typeof PipelinesIdRoute + '/pipelines/$id': typeof PipelinesIdRouteWithChildren + '/pipelines/$id/code': typeof PipelinesIdCodeRoute + '/pipelines/$id/graph': typeof PipelinesIdGraphRoute + '/pipelines/$id/inspect': typeof PipelinesIdInspectRoute + '/pipelines/$id/logs': typeof PipelinesIdLogsRoute + '/pipelines/$id/': typeof PipelinesIdIndexRoute } export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath - fullPaths: '/' | '/pipelines/$id' + fullPaths: + | '/' + | '/pipelines/$id' + | '/pipelines/$id/code' + | '/pipelines/$id/graph' + | '/pipelines/$id/inspect' + | '/pipelines/$id/logs' + | '/pipelines/$id/' fileRoutesByTo: FileRoutesByTo - to: '/' | '/pipelines/$id' - id: '__root__' | '/' | '/pipelines/$id' + to: + | '/' + | '/pipelines/$id/code' + | '/pipelines/$id/graph' + | '/pipelines/$id/inspect' + | '/pipelines/$id/logs' + | '/pipelines/$id' + id: + | '__root__' + | '/' + | '/pipelines/$id' + | '/pipelines/$id/code' + | '/pipelines/$id/graph' + | '/pipelines/$id/inspect' + | '/pipelines/$id/logs' + | '/pipelines/$id/' fileRoutesById: FileRoutesById } export interface RootRouteChildren { IndexRoute: typeof IndexRoute - PipelinesIdRoute: typeof PipelinesIdRoute + PipelinesIdRoute: typeof PipelinesIdRouteWithChildren } declare module '@tanstack/react-router' { @@ -65,12 +130,67 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof PipelinesIdRouteImport parentRoute: typeof rootRouteImport } + '/pipelines/$id/': { + id: '/pipelines/$id/' + path: '/' + fullPath: '/pipelines/$id/' + preLoaderRoute: typeof PipelinesIdIndexRouteImport + parentRoute: typeof PipelinesIdRoute + } + '/pipelines/$id/logs': { + id: '/pipelines/$id/logs' + path: '/logs' + fullPath: '/pipelines/$id/logs' + preLoaderRoute: typeof PipelinesIdLogsRouteImport + parentRoute: typeof PipelinesIdRoute + } + '/pipelines/$id/inspect': { + id: '/pipelines/$id/inspect' + path: '/inspect' + fullPath: '/pipelines/$id/inspect' + preLoaderRoute: typeof PipelinesIdInspectRouteImport + parentRoute: typeof PipelinesIdRoute + } + '/pipelines/$id/graph': { + id: '/pipelines/$id/graph' + path: '/graph' + fullPath: '/pipelines/$id/graph' + preLoaderRoute: typeof PipelinesIdGraphRouteImport + parentRoute: typeof PipelinesIdRoute + } + '/pipelines/$id/code': { + id: '/pipelines/$id/code' + path: '/code' + fullPath: '/pipelines/$id/code' + preLoaderRoute: typeof PipelinesIdCodeRouteImport + parentRoute: typeof PipelinesIdRoute + } } } +interface PipelinesIdRouteChildren { + PipelinesIdCodeRoute: typeof PipelinesIdCodeRoute + PipelinesIdGraphRoute: typeof PipelinesIdGraphRoute + PipelinesIdInspectRoute: typeof PipelinesIdInspectRoute + PipelinesIdLogsRoute: typeof PipelinesIdLogsRoute + PipelinesIdIndexRoute: typeof PipelinesIdIndexRoute +} + +const PipelinesIdRouteChildren: PipelinesIdRouteChildren = { + PipelinesIdCodeRoute: PipelinesIdCodeRoute, + PipelinesIdGraphRoute: PipelinesIdGraphRoute, + PipelinesIdInspectRoute: PipelinesIdInspectRoute, + PipelinesIdLogsRoute: PipelinesIdLogsRoute, + PipelinesIdIndexRoute: PipelinesIdIndexRoute, +} + +const PipelinesIdRouteWithChildren = PipelinesIdRoute._addFileChildren( + PipelinesIdRouteChildren, +) + const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, - PipelinesIdRoute: PipelinesIdRoute, + PipelinesIdRoute: PipelinesIdRouteWithChildren, } export const routeTree = rootRouteImport ._addFileChildren(rootRouteChildren) diff --git a/packages/pipelines/pipeline-server/src/client/routes/__root.tsx b/packages/pipelines/pipeline-server/src/client/routes/__root.tsx index fbf533381..8212aa3ca 100644 --- a/packages/pipelines/pipeline-server/src/client/routes/__root.tsx +++ b/packages/pipelines/pipeline-server/src/client/routes/__root.tsx @@ -1,4 +1,5 @@ -import type { PipelineInfo, PipelinesResponse } from "@ucdjs/pipelines-ui"; +import type { PipelineInfo } from "@ucdjs/pipelines-ui"; +import type { PipelinesContextValue } from "../types"; import { createRootRoute, Link, @@ -10,31 +11,15 @@ import { PipelineSidebarItem, usePipelines, } from "@ucdjs/pipelines-ui"; -import { createContext, use, useMemo } from "react"; - -interface PipelinesContextValue { - data: PipelinesResponse | null; - loading: boolean; - error: string | null; - refetch: () => void; -} - -const PipelinesContext = createContext(null); - -export function usePipelinesContext(): PipelinesContextValue { - const ctx = use(PipelinesContext); - if (!ctx) { - throw new Error("usePipelinesContext must be used within PipelinesProvider"); - } - return ctx; -} +import { useMemo, useState } from "react"; +import { PipelinesContext } from "../hooks/pipeline-context"; function useCurrentPipelineId(): string | undefined { const matches = useMatches(); return useMemo(() => { for (const match of matches) { const params = match.params as Record | undefined; - if (params && "id" in params) { + if (params?.id) { return params.id; } } @@ -42,18 +27,18 @@ function useCurrentPipelineId(): string | undefined { }, [matches]); } -function SidebarItemWithLink({ - pipeline, - isActive, -}: { +interface SidebarItemProps { pipeline: PipelineInfo; isActive: boolean; -}) { +} + +function SidebarItemWithLink({ pipeline, isActive }: SidebarItemProps) { return ( ( () => ({ data, @@ -80,23 +69,21 @@ function RootLayout() { return ( -
+
( )} /> -
+
); } - -export const Route = createRootRoute({ - component: RootLayout, -}); diff --git a/packages/pipelines/pipeline-server/src/client/routes/index.tsx b/packages/pipelines/pipeline-server/src/client/routes/index.tsx index 64ebfdd9a..5d5432825 100644 --- a/packages/pipelines/pipeline-server/src/client/routes/index.tsx +++ b/packages/pipelines/pipeline-server/src/client/routes/index.tsx @@ -1,16 +1,75 @@ import { createFileRoute } from "@tanstack/react-router"; -import { memo } from "react"; -import { usePipelinesContext } from "./__root"; +import { usePipelinesContext } from "../hooks/pipeline-context"; -const HomePage = memo(() => { +export const Route = createFileRoute("/")({ + component: HomePage, +}); + +function EmptyStateIcon() { + return ( +
+ +
+ ); +} + +function SelectPipelinePrompt() { + return ( + <> + +

+ Select a Pipeline +

+

+ Choose a pipeline from the sidebar to view its details and execute it. +

+ + ); +} + +function NoPipelinesFound() { + return ( + <> + +

+ No Pipelines Found +

+

+ Create a pipeline file to get started. +

+ + *.ucd-pipeline.ts + + + ); +} + +function LoadingState() { + return ( +
+

Loading pipelines...

+
+ ); +} + +function HomePage() { const { data, loading } = usePipelinesContext(); if (loading) { - return ( -
-

Loading pipelines...

-
- ); + return ; } const pipelineCount = data?.pipelines.length ?? 0; @@ -18,51 +77,8 @@ const HomePage = memo(() => { return (
-
- - - -
- - {pipelineCount > 0 - ? ( - <> -

- Select a Pipeline -

-

- Choose a pipeline from the sidebar to view its details and execute it. -

- - ) - : ( - <> -

- No Pipelines Found -

-

- Create a pipeline file to get started. -

- - *.ucd-pipeline.ts - - - )} + {pipelineCount > 0 ? : }
); -}); - -export const Route = createFileRoute("/")({ - component: HomePage, -}); +} diff --git a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.code.tsx b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.code.tsx new file mode 100644 index 000000000..8ccbb4cdf --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.code.tsx @@ -0,0 +1,102 @@ +import type { CodeResponse } from "../types"; +import { createFileRoute } from "@tanstack/react-router"; +import { Card, CardContent, CardHeader, CardTitle } from "@ucdjs-internal/shared-ui/ui/card"; + +export const Route = createFileRoute("/pipelines/$id/code")({ + loader: async ({ params }): Promise => { + const res = await fetch(`/api/pipelines/${params.id}/code`); + if (!res.ok) { + throw new Error(`Failed to load code (${res.status})`); + } + return res.json(); + }, + component: PipelineCodePage, +}); + +interface ErrorDisplayProps { + error: string; +} + +function ErrorDisplay({ error }: ErrorDisplayProps) { + return ( + + + Pipeline Code + + +

{error}

+ + + ); +} + +interface CodeDisplayProps { + code: string; + filePath: string; +} + +function CodeDisplay({ code, filePath }: CodeDisplayProps) { + return ( + + + Pipeline Code + + +

{filePath}

+ 
+          {code}
+
+ + + ); +} + +function EmptyCodeDisplay({ pipelineId }: { pipelineId: string }) { + return ( + + + Pipeline Code + + +

{pipelineId}

+ 
+          No code found.
+
+ + + ); +} + +function PipelineCodePage() { + const { id } = Route.useParams(); + const data = Route.useLoaderData(); + + if (data?.error) { + return ; + } + + if (!data?.code) { + return ( +
+ +
+ ); + } + + return ( +
+ +
+ ); +} diff --git a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.graph.tsx b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.graph.tsx new file mode 100644 index 000000000..1314f3e22 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.graph.tsx @@ -0,0 +1,48 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { PipelineGraph } from "@ucdjs/pipelines-ui"; +import { usePipelineDetailContext } from "../hooks/pipeline-detail-context"; + +export const Route = createFileRoute("/pipelines/$id/graph")({ + component: PipelineGraphPage, +}); + +function EmptyGraphState() { + return ( +
+ Run the pipeline to generate the execution graph. +
+ ); +} + +function PipelineGraphPage() { + const { execution } = usePipelineDetailContext(); + + const graph = execution.result?.graph && execution.result.graph.nodes.length > 0 + ? execution.result.graph + : null; + + if (!graph) { + return ; + } + + return ( +
+ +
+ ); +} diff --git a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.index.tsx b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.index.tsx new file mode 100644 index 000000000..674cedb52 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.index.tsx @@ -0,0 +1,91 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { Badge } from "@ucdjs-internal/shared-ui/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@ucdjs-internal/shared-ui/ui/card"; +import { ExecutionResult, RouteList, SourceList } from "@ucdjs/pipelines-ui"; +import { usePipelineDetailContext } from "../hooks/pipeline-detail-context"; + +export const Route = createFileRoute("/pipelines/$id/")({ + component: PipelineOverviewPage, +}); + +function ExecutionStatusCard() { + const { execution } = usePipelineDetailContext(); + + return ( + + + Overview + + + {execution.result + ? ( + + ) + : ( +
+ No runs yet. Execute the pipeline to see results. + Awaiting run +
+ )} + + + ); +} + +function RoutesCard() { + const { pipeline } = usePipelineDetailContext(); + + if (!pipeline) return null; + + return ( + + + Routes + + + ({ + id: route.id, + cache: route.cache, + }))} + /> + + + ); +} + +function SourcesCard() { + const { pipeline } = usePipelineDetailContext(); + + if (!pipeline) return null; + + return ( + + + Sources + + + + + + ); +} + +function PipelineOverviewPage() { + const { pipeline } = usePipelineDetailContext(); + + if (!pipeline) { + return null; + } + + return ( +
+ + +
+ + +
+
+ ); +} diff --git a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.inspect.tsx b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.inspect.tsx new file mode 100644 index 000000000..b9e37bc08 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.inspect.tsx @@ -0,0 +1,324 @@ +import type { PipelineDetails } from "@ucdjs/pipelines-ui"; +import { createFileRoute } from "@tanstack/react-router"; +import { cn } from "@ucdjs-internal/shared-ui/lib/utils"; +import { Badge } from "@ucdjs-internal/shared-ui/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@ucdjs-internal/shared-ui/ui/card"; +import { Input } from "@ucdjs-internal/shared-ui/ui/input"; +import { useEffect, useMemo, useState } from "react"; +import { usePipelineDetailContext } from "../hooks/pipeline-detail-context"; + +export const Route = createFileRoute("/pipelines/$id/inspect")({ + component: PipelineInspectPage, +}); + +type RouteInfo = PipelineDetails["routes"][number]; +type Dependency = RouteInfo["depends"][number]; +type EmittedArtifact = RouteInfo["emits"][number]; +type OutputConfig = RouteInfo["outputs"][number]; + +interface RouteListProps { + routes: RouteInfo[]; + selectedRouteId: string | null; + onSelectRoute: (routeId: string) => void; + searchQuery: string; + onSearchChange: (query: string) => void; +} + +function RouteListCard({ + routes, + selectedRouteId, + onSelectRoute, + searchQuery, + onSearchChange, +}: RouteListProps) { + const filteredRoutes = useMemo(() => { + if (!searchQuery.trim()) return routes; + const value = searchQuery.trim().toLowerCase(); + return routes.filter((route) => route.id.toLowerCase().includes(value)); + }, [searchQuery, routes]); + + return ( + + + Routes + + + onSearchChange(event.target.value)} + placeholder="Search routes" + aria-label="Search routes" + /> +
+ {filteredRoutes.length === 0 + ? ( +

No routes match the search.

+ ) + : ( + filteredRoutes.map((route) => ( + onSelectRoute(route.id)} + /> + )) + )} +
+ + + ); +} + +interface RouteListItemProps { + route: RouteInfo; + isSelected: boolean; + onClick: () => void; +} + +function RouteListItem({ route, isSelected, onClick }: RouteListItemProps) { + return ( + + ); +} + +interface RouteDetailsProps { + route: RouteInfo; +} + +function RouteDetailsCard({ route }: RouteDetailsProps) { + return ( + + + Details + + + + + + + + + ); +} + +interface DependsSectionProps { + depends: readonly Dependency[]; +} + +function DependsSection({ depends }: DependsSectionProps) { + return ( +
+

Depends

+ {depends.length === 0 + ? ( +

No dependencies.

+ ) + : ( +
+ {depends.map((dep, index) => ( + + {dep.type === "route" + ? `route:${dep.routeId}` + : `artifact:${dep.routeId}:${dep.artifactName}`} + + ))} +
+ )} +
+ ); +} + +interface EmitsSectionProps { + emits: readonly EmittedArtifact[]; +} + +function EmitsSection({ emits }: EmitsSectionProps) { + return ( +
+

Emits

+ {emits.length === 0 + ? ( +

No emitted artifacts.

+ ) + : ( +
+ {emits.map((emit) => ( + + {emit.id} + {" "} + {emit.scope} + + ))} +
+ )} +
+ ); +} + +interface OutputsSectionProps { + outputs: readonly OutputConfig[]; +} + +function OutputsSection({ outputs }: OutputsSectionProps) { + return ( +
+

Outputs

+ {outputs.length === 0 + ? ( +

No output configuration.

+ ) + : ( +
+ {outputs.map((output, index) => ( +
+
+ dir: + {" "} + {output.dir ?? "default"} +
+
+ file: + {" "} + {output.fileName ?? "default"} +
+
+ ))} +
+ )} +
+ ); +} + +interface TransformsSectionProps { + transforms: readonly string[]; +} + +function TransformsSection({ transforms }: TransformsSectionProps) { + return ( +
+

Transforms

+ {transforms.length === 0 + ? ( +

No transforms.

+ ) + : ( +
+ {transforms.map((transform, index) => ( + + {transform} + + ))} +
+ )} +
+ ); +} + +function EmptyDetailsCard() { + return ( + + + Details + + +

Select a route to inspect.

+ + + ); +} + +function PipelineInspectPage() { + const { pipeline } = usePipelineDetailContext(); + const [searchQuery, setSearchQuery] = useState(""); + const [selectedRouteId, setSelectedRouteId] = useState(null); + + const routes = pipeline?.routes ?? []; + + const filteredRoutes = useMemo(() => { + if (!searchQuery.trim()) return routes; + const value = searchQuery.trim().toLowerCase(); + return routes.filter((route) => route.id.toLowerCase().includes(value)); + }, [searchQuery, routes]); + + const selectedRoute = useMemo(() => { + return filteredRoutes.find((route) => route.id === selectedRouteId) ?? filteredRoutes[0] ?? null; + }, [filteredRoutes, selectedRouteId]); + + // Reset selected route when filtered routes change + useEffect(() => { + if (filteredRoutes.length === 0) { + setSelectedRouteId(null); + } else if (!selectedRouteId || !filteredRoutes.some((route) => route.id === selectedRouteId)) { + setSelectedRouteId(filteredRoutes[0]!.id); + } + }, [filteredRoutes, selectedRouteId]); + + if (!pipeline) { + return null; + } + + return ( +
+ + + {selectedRoute + ? ( + + ) + : ( + + )} +
+ ); +} diff --git a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.logs.tsx b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.logs.tsx new file mode 100644 index 000000000..0d17cad70 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.logs.tsx @@ -0,0 +1,105 @@ +import type { PipelineEvent } from "@ucdjs/pipelines-core"; +import { createFileRoute } from "@tanstack/react-router"; +import { Badge } from "@ucdjs-internal/shared-ui/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@ucdjs-internal/shared-ui/ui/card"; +import { usePipelineDetailContext } from "../hooks/pipeline-detail-context"; + +export const Route = createFileRoute("/pipelines/$id/logs")({ + component: PipelineLogsPage, +}); + +interface LogEventDisplayProps { + event: PipelineEvent; + index: number; +} + +function LogEventDisplay({ event }: LogEventDisplayProps) { + const timestamp = new Date(event.timestamp).toLocaleTimeString(); + + // Extract optional fields safely + const message = "message" in event ? String(event.message) : undefined; + const version = "version" in event ? event.version : undefined; + const routeId = "routeId" in event ? event.routeId : undefined; + const artifactId = "artifactId" in event ? event.artifactId : undefined; + const durationMs = "durationMs" in event ? event.durationMs : undefined; + + return ( +
+ {event.type} + + + {version && ( + + v + {version} + + )} + + {routeId && {routeId}} + {artifactId && {artifactId}} + + {durationMs != null && ( + + {Math.round(durationMs)} + ms + + )} + + {message && ( + {message} + )} +
+ ); +} + +function EmptyLogsState() { + return ( +

+ No logs yet. Execute the pipeline to see execution logs. +

+ ); +} + +function LogsList({ events }: { events: readonly PipelineEvent[] }) { + return ( +
+ {events.map((event, index) => ( + + ))} +
+ ); +} + +function PipelineLogsPage() { + const { execution } = usePipelineDetailContext(); + + return ( + + + Execution Logs + + + {execution.events.length === 0 + ? ( + + ) + : ( + + )} + + + ); +} diff --git a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx index 9425cdce6..2a1b6487f 100644 --- a/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx +++ b/packages/pipelines/pipeline-server/src/client/routes/pipelines.$id.tsx @@ -1,31 +1,226 @@ -import { createFileRoute } from "@tanstack/react-router"; +import type { PipelineEvent } from "@ucdjs/pipelines-core"; +import type { ExecuteResult, PipelineDetails } from "@ucdjs/pipelines-ui"; +import type { PipelineDetailContextValue, PipelineExecutionState, PipelineTab } from "../types"; +import { createFileRoute, Link, Outlet, useRouterState } from "@tanstack/react-router"; +import { cn } from "@ucdjs-internal/shared-ui/lib/utils"; +import { Badge } from "@ucdjs-internal/shared-ui/ui/badge"; +import { Button } from "@ucdjs-internal/shared-ui/ui/button"; import { - ExecutionResult, - RouteList, - SourceList, useExecute, usePipeline, VersionSelector, } from "@ucdjs/pipelines-ui"; -import { memo, useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { PipelineDetailContext } from "../hooks/pipeline-detail-context"; -const PipelineDetailPage = memo(() => { +const PIPELINE_TABS: readonly PipelineTab[] = [ + { id: "overview", label: "Overview", to: "/pipelines/$id", exact: true }, + { id: "graph", label: "Graph", to: "/pipelines/$id/graph" }, + { id: "inspect", label: "Inspect", to: "/pipelines/$id/inspect" }, + { id: "logs", label: "Logs", to: "/pipelines/$id/logs" }, + { id: "code", label: "Code", to: "/pipelines/$id/code" }, +] as const; + +function useActiveTabId(): string { + const routerState = useRouterState(); + const currentPath = routerState.location.pathname; + + return useMemo(() => { + // Find the most specific matching tab + const matchingTab = [...PIPELINE_TABS] + .reverse() + .find((tab) => { + const pattern = tab.to.replace("$id", "[^/]+"); + const regex = new RegExp(`^${pattern}$`); + return regex.test(currentPath); + }); + + return matchingTab?.id ?? PIPELINE_TABS[0]!.id; + }, [currentPath]); +} + +interface PipelineHeaderProps { + pipeline: PipelineDetails; + selectedVersions: Set; + executionState: PipelineExecutionState; + executing: boolean; + onExecute: () => void; +} + +function PipelineHeader({ + pipeline, + selectedVersions, + executionState, + executing, + onExecute, +}: PipelineHeaderProps) { + const hasResult = executionState.result != null; + const wasSuccess = hasResult && executionState.result?.success; + + return ( +
+
+
+
+

+ {pipeline.name || pipeline.id} +

+ + {pipeline.versions.length} + {" "} + versions + + + {pipeline.routeCount} + {" "} + routes + + + {pipeline.sourceCount} + {" "} + sources + +
+ {pipeline.description && ( +

+ {pipeline.description} +

+ )} +
+
+ {hasResult && ( + + {wasSuccess ? "Last run: success" : "Last run: failed"} + + )} + +
+
+
+ ); +} + +interface VersionSelectorSectionProps { + pipeline: PipelineDetails; + selectedVersions: Set; + onToggleVersion: (version: string) => void; + onSelectAll: () => void; + onDeselectAll: () => void; +} + +function VersionSelectorSection({ + pipeline, + selectedVersions, + onToggleVersion, + onSelectAll, + onDeselectAll, +}: VersionSelectorSectionProps) { + return ( +
+ +
+ ); +} + +interface PipelineTabsProps { + pipelineId: string; + activeTabId: string; +} + +function PipelineTabs({ pipelineId, activeTabId }: PipelineTabsProps) { + return ( + + ); +} + +interface LoadingStateProps { + message?: string; +} + +function LoadingState({ message = "Loading..." }: LoadingStateProps) { + return ( +
+

{message}

+
+ ); +} + +interface ErrorStateProps { + error: string; + context?: string; +} + +function ErrorState({ error, context }: ErrorStateProps) { + return ( +
+
+

{error}

+ {context && ( +

{context}

+ )} +
+
+ ); +} + +export const Route = createFileRoute("/pipelines/$id")({ + component: PipelineDetailLayout, +}); + +function PipelineDetailLayout() { const { id } = Route.useParams(); const { pipeline, loading, error } = usePipeline(id); - const { execute, executing, result } = useExecute(); + const { execute, executing, result, error: executeError, reset } = useExecute(); + const [selectedVersions, setSelectedVersions] = useState>(() => new Set()); + const [events, setEvents] = useState([]); + const activeTabId = useActiveTabId(); - // Selected versions state - const [selectedVersions, setSelectedVersions] = useState>(new Set()); - - // Sync selected versions when pipeline loads + // Reset state when pipeline changes useEffect(() => { if (pipeline) { setSelectedVersions(new Set(pipeline.versions)); + setEvents([]); + reset(); } - }, [pipeline]); + }, [pipeline?.id, reset]); // Only depend on pipeline ID, not the whole object - // Version toggle handler - const handleToggleVersion = useCallback((version: string) => { + const toggleVersion = useCallback((version: string) => { setSelectedVersions((prev) => { const next = new Set(prev); if (next.has(version)) { @@ -37,116 +232,85 @@ const PipelineDetailPage = memo(() => { }); }, []); - // Select all versions - const handleSelectAll = useCallback(() => { + const selectAllVersions = useCallback(() => { if (pipeline) { setSelectedVersions(new Set(pipeline.versions)); } - }, [pipeline]); + }, [pipeline?.versions]); - // Deselect all versions - const handleDeselectAll = useCallback(() => { + const deselectAllVersions = useCallback(() => { setSelectedVersions(new Set()); }, []); - // Execute handler - const handleExecute = useCallback(async () => { + const executePipeline = useCallback(async () => { if (!pipeline || selectedVersions.size === 0) return; - await execute(id, Array.from(selectedVersions)); + + try { + const execResult = await execute(id, Array.from(selectedVersions)); + setEvents(execResult.events ?? []); + } catch (err) { + // Error is handled by useExecute hook + console.error("Pipeline execution failed:", err); + } }, [execute, id, pipeline, selectedVersions]); - // Loading state + const executionState = useMemo(() => ({ + result: result ?? null, + events, + executing, + error: executeError, + }), [events, executeError, executing, result]); + + const contextValue = useMemo(() => ({ + pipeline, + loading, + error, + execution: executionState, + selectedVersions, + setSelectedVersions, + toggleVersion, + selectAllVersions, + deselectAllVersions, + executePipeline, + }), [pipeline, loading, error, executionState, selectedVersions, toggleVersion, selectAllVersions, deselectAllVersions, executePipeline]); + if (loading) { - return ( -
-

Loading pipeline...

-
- ); + return ; } - // Error state if (error) { - return ( -
-
-

{error}

-

- Pipeline ID: - {" "} - {id} -

-
-
- ); + return ; } - // Not found state if (!pipeline) { - return ( -
-

Pipeline not found

-
- ); + return ; } - const canExecute = selectedVersions.size > 0 && !executing; - return ( -
- {/* Header */} -
-
-
-

- {pipeline.name || pipeline.id} -

- {pipeline.description && ( -

- {pipeline.description} -

- )} -
- -
- - {/* Version selector */} -
- -
-
+ +
+ - {/* Content */} -
- {/* Execution result */} - {result && } + - {/* Routes */} - + - {/* Sources */} - +
+ +
-
+ ); -}); - -export const Route = createFileRoute("/pipelines/$id")({ - component: PipelineDetailPage, -}); +} diff --git a/packages/pipelines/pipeline-server/src/client/types.ts b/packages/pipelines/pipeline-server/src/client/types.ts new file mode 100644 index 000000000..ee15cf638 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/client/types.ts @@ -0,0 +1,53 @@ +import type { PipelineEvent } from "@ucdjs/pipelines-core"; +import type { ExecuteResult, PipelineDetails, PipelineInfo, PipelinesResponse } from "@ucdjs/pipelines-ui"; + +export interface PipelinesContextValue { + data: PipelinesResponse | null; + loading: boolean; + error: string | null; + refetch: () => void; +} + +export interface PipelineExecutionState { + result: ExecuteResult | null; + events: PipelineEvent[]; + executing: boolean; + error: string | null; +} + +export interface PipelineDetailContextValue { + pipeline: PipelineDetails | null; + loading: boolean; + error: string | null; + execution: PipelineExecutionState; + selectedVersions: Set; + setSelectedVersions: (versions: Set) => void; + toggleVersion: (version: string) => void; + selectAllVersions: () => void; + deselectAllVersions: () => void; + executePipeline: () => Promise; +} + +export interface PipelineTab { + readonly id: string; + readonly label: string; + readonly to: string; + readonly exact?: boolean; +} + +export interface LogEventDisplayProps { + event: PipelineEvent; + index: number; +} + +export interface RouteListItemProps { + route: PipelineDetails["routes"][number]; + isSelected: boolean; + onClick: () => void; +} + +export interface CodeResponse { + code?: string; + filePath?: string; + error?: string; +} diff --git a/packages/pipelines/pipeline-server/src/server/app.ts b/packages/pipelines/pipeline-server/src/server/app.ts index 6d3df4619..99883c8ca 100644 --- a/packages/pipelines/pipeline-server/src/server/app.ts +++ b/packages/pipelines/pipeline-server/src/server/app.ts @@ -25,9 +25,15 @@ export function createApp(options: AppOptions = {}): H3 { const app = new H3({ debug: true }); + let cwdPath = path.resolve(cwd); + // If we run in development mode, we should set the CWD to the pipeline playground. + if (process.env.NODE_ENV === "development" || (import.meta as any).env.DEV) { + cwdPath = path.join(import.meta.dirname, "../../../pipeline-playground"); + } + // Middleware to attach cwd to context app.use("/**", (event, next) => { - event.context.cwd = cwd; + event.context.cwd = cwdPath; next(); }); @@ -51,7 +57,7 @@ export async function startServer(options: ServerOptions = {}): Promise { const clientDir = path.join(import.meta.dirname, "../client"); app.use((event) => { - const url = event.path; + const url = event.url.pathname; // Skip API routes if (url.startsWith("/api")) { @@ -59,6 +65,7 @@ export async function startServer(options: ServerOptions = {}): Promise { } return serveStatic(event, { + fallthrough: true, getContents: (id) => fs.promises.readFile(path.join(clientDir, id)), getMeta: async (id) => { const filePath = path.join(clientDir, id); @@ -75,7 +82,7 @@ export async function startServer(options: ServerOptions = {}): Promise { // SPA fallback - serve index.html for client-side routing app.use((event) => { - const url = event.path; + const url = event.url.pathname; // Skip API routes if (url.startsWith("/api")) { diff --git a/packages/pipelines/pipeline-server/src/server/code.ts b/packages/pipelines/pipeline-server/src/server/code.ts new file mode 100644 index 000000000..05ea19d48 --- /dev/null +++ b/packages/pipelines/pipeline-server/src/server/code.ts @@ -0,0 +1,119 @@ +import { print } from "esrap"; +import ts from "esrap/languages/ts"; +import { parseSync } from "oxc-parser"; + +interface ExportTarget { + exportName: string; +} + +export function extractDefinePipelineCode(source: string, { exportName }: ExportTarget): string { + const { program } = parseSync("pipeline.ts", source, { + sourceType: "module", + lang: "ts", + } as any); + + const callExpression = findDefinePipelineExpression(program, exportName); + if (!callExpression) { + throw new Error(`definePipeline call not found for export "${exportName}"`); + } + + try { + const { code } = print( + { + type: "Program", + sourceType: "module", + body: [ + { + type: "ExpressionStatement", + expression: callExpression, + }, + ], + } as any, + ts(), + ); + + return code.trim(); + } catch { + const span = callExpression.span as { start: number; end: number } | undefined; + if (span) { + return source.slice(span.start, span.end).trim(); + } + throw new Error("Failed to print definePipeline call"); + } +} + +function findDefinePipelineExpression(program: any, exportName: string): any | null { + const body = program.body as any[]; + + if (exportName === "default") { + for (const node of body) { + if (node.type === "ExportDefaultDeclaration") { + if (isDefinePipelineCall(node.declaration)) { + return node.declaration; + } + if (node.declaration?.type === "Identifier") { + const expr = findVariableInitializer(body, node.declaration.name); + if (expr) return expr; + } + } + } + } + + for (const node of body) { + if (node.type === "ExportNamedDeclaration") { + if (node.declaration) { + const expr = findDefinePipelineInDeclaration(node.declaration, exportName); + if (expr) return expr; + } + + if (node.specifiers?.length) { + for (const spec of node.specifiers) { + if (spec.exported?.name === exportName) { + const localName = spec.local?.name ?? exportName; + const expr = findVariableInitializer(body, localName); + if (expr) return expr; + } + } + } + } + } + + return findVariableInitializer(body, exportName); +} + +function findDefinePipelineInDeclaration(declaration: any, exportName: string): any | null { + if (declaration.type === "VariableDeclaration") { + for (const decl of declaration.declarations ?? []) { + const id = decl.id; + if (id?.type === "Identifier" && id.name === exportName) { + if (isDefinePipelineCall(decl.init)) return decl.init; + } + } + } + + return null; +} + +function findVariableInitializer(body: any[], name: string): any | null { + for (const node of body) { + if (node.type === "VariableDeclaration") { + for (const decl of node.declarations ?? []) { + const id = decl.id; + if (id?.type === "Identifier" && id.name === name) { + if (isDefinePipelineCall(decl.init)) return decl.init; + } + } + } + } + + return null; +} + +function isDefinePipelineCall(node: any): boolean { + return ( + node + && node.type === "CallExpression" + && node.callee?.type === "Identifier" + && node.callee?.name === "definePipeline" + ); +} diff --git a/packages/pipelines/pipeline-server/src/server/routes/execute.ts b/packages/pipelines/pipeline-server/src/server/routes/execute.ts index 80a48b508..708ee47a9 100644 --- a/packages/pipelines/pipeline-server/src/server/routes/execute.ts +++ b/packages/pipelines/pipeline-server/src/server/routes/execute.ts @@ -1,3 +1,4 @@ +import type { PipelineEvent } from "@ucdjs/pipelines-core"; import { createPipelineExecutor } from "@ucdjs/pipelines-executor"; import { findPipelineFiles, loadPipelinesFromPaths } from "@ucdjs/pipelines-loader"; import { H3, readBody } from "h3"; @@ -30,7 +31,12 @@ executeRouter.post("/", async (event) => { const versions = body.versions ?? pipeline.versions; const cache = body.cache ?? true; - const executor = createPipelineExecutor({}); + const events: PipelineEvent[] = []; + const executor = createPipelineExecutor({ + onEvent: (event) => { + events.push(event); + }, + }); try { const execResult = await executor.run([pipeline], { @@ -40,10 +46,21 @@ executeRouter.post("/", async (event) => { const pipelineResult = execResult.results.get(id); + if (pipelineResult) { + // eslint-disable-next-line no-console + console.info("Pipeline run finished:", { + pipelineId: id, + summary: pipelineResult.summary, + errorCount: pipelineResult.errors.length, + }); + } + return { success: true, pipelineId: id, summary: pipelineResult?.summary, + graph: pipelineResult?.graph, + events: events.slice().reverse(), errors: pipelineResult?.errors.map((e) => ({ scope: e.scope, message: e.message, diff --git a/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts index a43824bf2..bafaed4aa 100644 --- a/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts +++ b/packages/pipelines/pipeline-server/src/server/routes/pipelines.ts @@ -1,5 +1,8 @@ +import fs from "node:fs"; +import path from "node:path"; import { findPipelineFiles, loadPipelinesFromPaths } from "@ucdjs/pipelines-loader"; -import { H3 } from "h3"; +import { getQuery, H3 } from "h3"; +import { extractDefinePipelineCode } from "../code"; import { toPipelineDetails, toPipelineInfo } from "../types"; export const pipelinesRouter = new H3(); @@ -11,9 +14,27 @@ pipelinesRouter.get("/", async (event) => { cwd, }); const result = await loadPipelinesFromPaths(files); + const query = getQuery(event).query; + const search = typeof query === "string" ? query.trim().toLowerCase() : ""; + + const pipelines = search + ? result.pipelines.filter((pipeline) => { + const haystack = [ + pipeline.id, + pipeline.name ?? "", + pipeline.description ?? "", + ...pipeline.versions, + ...pipeline.routes.map((route) => route.id), + ...pipeline.inputs.map((input) => input.id), + ] + .join(" ") + .toLowerCase(); + return haystack.includes(search); + }) + : result.pipelines; return { - pipelines: result.pipelines.map(toPipelineInfo), + pipelines: pipelines.map(toPipelineInfo), cwd, errors: result.errors.map((e) => ({ filePath: e.filePath, @@ -45,3 +66,39 @@ pipelinesRouter.get("/:id", async (event) => { pipeline: toPipelineDetails(pipeline), }; }); + +pipelinesRouter.get("/:id/code", async (event) => { + const { cwd } = event.context; + const id = event.context.params?.id; + + if (!id) { + return { error: "Pipeline ID is required" }; + } + + const files = await findPipelineFiles({ + cwd, + }); + const result = await loadPipelinesFromPaths(files); + + const pipeline = result.pipelines.find((p) => p.id === id); + + if (!pipeline) { + return { error: `Pipeline "${id}" not found` }; + } + + const file = result.files.find((f) => f.pipelines.some((p) => p.id === id)); + + if (!file) { + return { error: `Pipeline file for "${id}" not found` }; + } + + const exportIndex = file.pipelines.findIndex((p) => p.id === id); + const exportName = file.exportNames[exportIndex] ?? "default"; + const filePath = path.resolve(cwd, file.filePath); + const source = await fs.promises.readFile(filePath, "utf-8"); + + return { + code: extractDefinePipelineCode(source, { exportName }), + filePath: file.filePath, + }; +}); diff --git a/packages/pipelines/pipeline-server/src/server/types.ts b/packages/pipelines/pipeline-server/src/server/types.ts index ea3be5ba5..4286625da 100644 --- a/packages/pipelines/pipeline-server/src/server/types.ts +++ b/packages/pipelines/pipeline-server/src/server/types.ts @@ -1,4 +1,5 @@ -import type { PipelineDefinition } from "@ucdjs/pipelines-core"; +import type { PipelineDefinition, PipelineRouteDefinition } from "@ucdjs/pipelines-core"; +import { parseDependency } from "@ucdjs/pipelines-core"; /** * Serializable pipeline info for the API. @@ -19,6 +20,13 @@ export interface PipelineDetails extends PipelineInfo { routes: Array<{ id: string; cache: boolean; + depends: Array< + | { type: "route"; routeId: string } + | { type: "artifact"; routeId: string; artifactName: string } + >; + emits: Array<{ id: string; scope: "version" | "global" }>; + outputs: Array<{ dir?: string; fileName?: string }>; + transforms: string[]; }>; sources: Array<{ id: string; @@ -45,12 +53,34 @@ export function toPipelineInfo(pipeline: PipelineDefinition): PipelineInfo { export function toPipelineDetails(pipeline: PipelineDefinition): PipelineDetails { return { ...toPipelineInfo(pipeline), - routes: pipeline.routes.map((route) => ({ - id: route.id, - cache: route.cache !== false, - })), + routes: pipeline.routes.map((route) => toRouteDetails(route)), sources: pipeline.inputs.map((source) => ({ id: source.id, })), }; } + +function toRouteDetails(route: PipelineRouteDefinition): PipelineDetails["routes"][number] { + const depends = (route.depends ?? []).map((dep) => parseDependency(dep)); + const emits = Object.entries(route.emits ?? {}).map(([id, def]) => { + const scope = def.scope === "global" ? "global" : "version"; + return { id, scope } as const; + }); + const outputs = route.out + ? [{ dir: route.out.dir, fileName: typeof route.out.fileName === "function" ? "[fn]" : route.out.fileName }] + : []; + const transformList = (route.transforms ?? []) as Array<{ id?: string }>; + const transforms = transformList.map((transform, index) => { + const id = transform.id; + return id ?? `transform-${index + 1}`; + }); + + return { + id: route.id, + cache: route.cache !== false, + depends, + emits, + outputs, + transforms, + }; +} diff --git a/packages/pipelines/pipeline-server/tsconfig.build.json b/packages/pipelines/pipeline-server/tsconfig.build.json index 36c889e0c..277eca3e1 100644 --- a/packages/pipelines/pipeline-server/tsconfig.build.json +++ b/packages/pipelines/pipeline-server/tsconfig.build.json @@ -1,5 +1,9 @@ { - "extends": "./tsconfig.json", - "include": ["src"], - "exclude": ["dist", "test"] + "extends": "@ucdjs-tooling/tsconfig/base.build", + "include": [ + "src" + ], + "exclude": [ + "dist" + ] } diff --git a/packages/pipelines/pipeline-server/turbo.json b/packages/pipelines/pipeline-server/turbo.json new file mode 100644 index 000000000..ca1c1e9f6 --- /dev/null +++ b/packages/pipelines/pipeline-server/turbo.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://turbo.build/schema.json", + "extends": ["//"], + "tasks": { + "build": { + "outputs": ["dist/**"] + }, + "dev": { + "persistent": true + }, + "typecheck": { + "outputs": [".cache/tsbuildinfo.json"] + } + } +} diff --git a/packages/pipelines/pipeline-server/vite.config.ts b/packages/pipelines/pipeline-server/vite.config.ts index 6cb2987d3..f69e23bf7 100644 --- a/packages/pipelines/pipeline-server/vite.config.ts +++ b/packages/pipelines/pipeline-server/vite.config.ts @@ -1,16 +1,35 @@ +import type { H3 } from "h3"; import type { Plugin } from "vite"; import tailwindcss from "@tailwindcss/vite"; -import { TanStackRouterVite } from "@tanstack/router-plugin/vite"; +import { tanstackRouter } from "@tanstack/router-plugin/vite"; import react from "@vitejs/plugin-react"; import { defineConfig } from "vite"; -import { createApp } from "./src/server/app"; +import viteTsConfigPaths from "vite-tsconfig-paths"; -function h3DevServerPlugin(): Plugin { - const app = createApp(); +const appModuleId = "/src/server/app.ts"; +function h3DevServerPlugin(): Plugin { return { name: "h3-dev-server", configureServer(server) { + let appPromise: Promise

| null = null; + + const getApp = async () => { + if (!appPromise) { + appPromise = server + .ssrLoadModule(appModuleId) + .then((mod) => (mod as typeof import("./src/server/app")).createApp()); + } + + return appPromise; + }; + + server.watcher.on("change", (file) => { + if (file.includes("/src/server/")) { + appPromise = null; + } + }); + // Add middleware BEFORE Vite's internal middleware (no return = pre-hook) // This ensures /api routes are handled before Vite's SPA fallback server.middlewares.use(async (req, res, next) => { @@ -19,13 +38,17 @@ function h3DevServerPlugin(): Plugin { } try { + const app = await getApp(); + // Collect request body for POST/PUT/PATCH let body: string | undefined; if (req.method && ["POST", "PUT", "PATCH"].includes(req.method)) { + // eslint-disable-next-line node/prefer-global/buffer const chunks: Buffer[] = []; for await (const chunk of req) { chunks.push(chunk); } + // eslint-disable-next-line node/prefer-global/buffer body = Buffer.concat(chunks).toString(); } @@ -53,8 +76,14 @@ function h3DevServerPlugin(): Plugin { } export default defineConfig({ + clearScreen: false, plugins: [ - TanStackRouterVite({ + viteTsConfigPaths({ + projects: ["./tsconfig.json"], + loose: true, + projectDiscovery: "lazy", + }), + tanstackRouter({ routesDirectory: "./src/client/routes", generatedRouteTree: "./src/client/routeTree.gen.ts", }), @@ -71,7 +100,7 @@ export default defineConfig({ server: { build: { outDir: "dist/server", - ssr: true, + ssr: false, rollupOptions: { input: "src/server/app.ts", }, diff --git a/packages/pipelines/pipeline-ui/package.json b/packages/pipelines/pipeline-ui/package.json index 840ab2acb..65fbf39a2 100644 --- a/packages/pipelines/pipeline-ui/package.json +++ b/packages/pipelines/pipeline-ui/package.json @@ -26,6 +26,21 @@ }, "exports": { ".": "./dist/index.mjs", + "./components/details": "./dist/components/details.mjs", + "./components/execution-result": "./dist/components/execution-result.mjs", + "./components/filters": "./dist/components/filters.mjs", + "./components/nodes": "./dist/components/nodes.mjs", + "./components/pipeline-detail": "./dist/components/pipeline-detail.mjs", + "./components/pipeline-graph": "./dist/components/pipeline-graph.mjs", + "./components/pipeline-sidebar": "./dist/components/pipeline-sidebar.mjs", + "./hooks/use-execute": "./dist/hooks/use-execute.mjs", + "./hooks/use-pipeline": "./dist/hooks/use-pipeline.mjs", + "./hooks/use-pipelines": "./dist/hooks/use-pipelines.mjs", + "./lib/adapter": "./dist/lib/adapter.mjs", + "./lib/colors": "./dist/lib/colors.mjs", + "./lib/layout": "./dist/lib/layout.mjs", + "./lib/utils": "./dist/lib/utils.mjs", + "./styles.css": "./dist/styles/globals.css", "./package.json": "./package.json" }, "types": "./dist/index.d.mts", diff --git a/packages/pipelines/pipeline-ui/src/components/execution-result.tsx b/packages/pipelines/pipeline-ui/src/components/execution-result.tsx index 7a031b325..daba68c46 100644 --- a/packages/pipelines/pipeline-ui/src/components/execution-result.tsx +++ b/packages/pipelines/pipeline-ui/src/components/execution-result.tsx @@ -21,8 +21,8 @@ export const ExecutionResult = memo(({ className={cn( "rounded-lg border p-4", isSuccess - ? "bg-emerald-950/30 border-emerald-900" - : "bg-red-950/30 border-red-900", + ? "bg-primary/10 border-primary/30" + : "bg-destructive/10 border-destructive/40", className, )} > @@ -31,13 +31,13 @@ export const ExecutionResult = memo(({
{isSuccess ? "Completed" : "Failed"} @@ -46,25 +46,33 @@ export const ExecutionResult = memo(({ {/* Summary stats */} {result.summary && ( -
+
- Routes - {result.summary.totalRoutes} + Files + {result.summary.totalFiles}
- Success - - {result.summary.successfulRoutes} + Matched + + {result.summary.matchedFiles}
- Failed - {result.summary.failedRoutes} + Skipped + {result.summary.skippedFiles}
- Time - - {result.summary.totalTime} + Fallback + {result.summary.fallbackFiles} +
+
+ Outputs + {result.summary.totalOutputs} +
+
+ Time + + {Math.round(result.summary.durationMs)} ms
@@ -73,7 +81,7 @@ export const ExecutionResult = memo(({ {/* Top-level error */} {result.error && ( -

{result.error}

+

{result.error}

)} {/* Detailed errors list */} @@ -81,13 +89,13 @@ export const ExecutionResult = memo(({
{result.errors.map((err, i) => (
- + [ {err.scope} ] {" "} - {err.message} + {err.message}
))}
@@ -97,10 +105,12 @@ export const ExecutionResult = memo(({ }); export interface ExecutionSummaryProps { - totalRoutes: number; - successfulRoutes: number; - failedRoutes: number; - totalTime: number; + totalFiles: number; + matchedFiles: number; + skippedFiles: number; + fallbackFiles: number; + totalOutputs: number; + durationMs: number; className?: string; } @@ -108,30 +118,40 @@ export interface ExecutionSummaryProps { * Compact execution summary (without result wrapper) */ export const ExecutionSummary = memo(({ - totalRoutes, - successfulRoutes, - failedRoutes, - totalTime, + totalFiles, + matchedFiles, + skippedFiles, + fallbackFiles, + totalOutputs, + durationMs, className, }: ExecutionSummaryProps) => { return ( -
+
+
+ Files + {totalFiles} +
+
+ Matched + {matchedFiles} +
- Routes - {totalRoutes} + Skipped + {skippedFiles}
- Success - {successfulRoutes} + Fallback + {fallbackFiles}
- Failed - {failedRoutes} + Outputs + {totalOutputs}
- Time - - {totalTime} + Time + + {Math.round(durationMs)} ms
@@ -157,13 +177,13 @@ export const ExecutionErrors = memo(({
{errors.map((err, i) => (
- + [ {err.scope} ] {" "} - {err.message} + {err.message}
))}
diff --git a/packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx b/packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx index ae5c7f1ed..e175cbfd4 100644 --- a/packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx +++ b/packages/pipelines/pipeline-ui/src/components/pipeline-detail.tsx @@ -28,8 +28,8 @@ export const VersionTag = memo(({ className={cn( "px-2.5 py-1 text-xs rounded transition-colors", selected - ? "bg-zinc-100 text-zinc-900" - : "bg-zinc-800 text-zinc-400 hover:bg-zinc-700", + ? "bg-primary text-primary-foreground" + : "bg-secondary text-secondary-foreground hover:bg-secondary/80", className, )} > @@ -70,7 +70,7 @@ export const VersionSelector = memo(({ return (
- + Versions ( {selectedVersions.size} / @@ -83,7 +83,7 @@ export const VersionSelector = memo(({ @@ -92,7 +92,7 @@ export const VersionSelector = memo(({ @@ -131,14 +131,15 @@ export const RouteItem = memo(({ className, }: RouteItemProps) => { return ( -
- {route.id} + {route.id} {route.cache && ( - + cached )} @@ -160,12 +161,12 @@ export const RouteList = memo(({ }: RouteListProps) => { return (
-

+

Routes ( {routes.length} )

-
+
{routes.map((route) => ( ))} @@ -191,8 +192,8 @@ export const SourceItem = memo(({ className, }: SourceItemProps) => { return ( -
- {source.id} +
+ {source.id}
); }); @@ -211,12 +212,12 @@ export const SourceList = memo(({ }: SourceListProps) => { return (
-

+

Sources ( {sources.length} )

-
+
{sources.map((source) => ( ))} diff --git a/packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx b/packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx index 354cf56be..e0e3bfef7 100644 --- a/packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx +++ b/packages/pipelines/pipeline-ui/src/components/pipeline-graph.tsx @@ -73,6 +73,8 @@ export interface PipelineGraphProps { showDetails?: boolean; /** Show the minimap (default: true) */ showMinimap?: boolean; + /** Additional className for the container */ + className?: string; } export const PipelineGraph = memo(({ @@ -81,6 +83,7 @@ export const PipelineGraph = memo(({ showFilters = true, showDetails = true, showMinimap = true, + className, }: PipelineGraphProps) => { // Convert pipeline graph to React Flow format - memoized const { allNodes, allEdges } = useMemo(() => { @@ -186,7 +189,7 @@ export const PipelineGraph = memo(({ }, [onNodeSelect]); return ( -
+
{/* Filters toolbar */} {showFilters && ( diff --git a/packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx b/packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx index 708499ac2..0c19453c0 100644 --- a/packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx +++ b/packages/pipelines/pipeline-ui/src/components/pipeline-sidebar.tsx @@ -1,5 +1,6 @@ import type { PipelineInfo, PipelinesResponse } from "../types"; import { cn } from "#lib/utils"; +import { Input } from "@ucdjs-internal/shared-ui/ui/input"; import { memo } from "react"; export interface PipelineSidebarItemProps { @@ -23,17 +24,17 @@ export const PipelineSidebarItem = memo(({ type="button" onClick={onClick} className={cn( - "w-full text-left block px-4 py-2 text-sm transition-colors", + "w-full text-left block px-3.5 py-2 text-sm transition-colors rounded-md", isActive - ? "bg-zinc-800 text-zinc-100" - : "text-zinc-400 hover:bg-zinc-900 hover:text-zinc-200", + ? "bg-sidebar-accent text-sidebar-accent-foreground" + : "text-muted-foreground hover:bg-sidebar-accent/80 hover:text-sidebar-accent-foreground", className, )} > {pipeline.name || pipeline.id} - + {pipeline.routeCount} {" "} routes · @@ -67,11 +68,11 @@ export const PipelineSidebarList = memo(({ if (pipelines.length === 0) { return (
-

No pipelines found

-

+

No pipelines found

+

Create a {" "} - *.ucd-pipeline.ts + *.ucd-pipeline.ts {" "} file

@@ -118,8 +119,8 @@ export const PipelineSidebarErrors = memo(({ if (count === 0) return null; return ( -
-

+

+

{count} {" "} load error @@ -144,11 +145,11 @@ export const PipelineSidebarHeader = memo(({ className, }: PipelineSidebarHeaderProps) => { return ( -

-

{title}

+
+

{title}

{cwd && (

{cwd} @@ -171,6 +172,12 @@ export interface PipelineSidebarProps { renderItem?: (pipeline: PipelineInfo, isActive: boolean) => React.ReactNode; /** Sidebar title */ title?: string; + /** Search value */ + searchValue?: string; + /** Search change handler */ + onSearchChange?: (value: string) => void; + /** Placeholder text for search input */ + searchPlaceholder?: string; /** Additional className for the container */ className?: string; } @@ -185,22 +192,40 @@ export const PipelineSidebar = memo(({ onPipelineClick, renderItem, title, + searchValue, + onSearchChange, + searchPlaceholder = "Search pipelines", className, }: PipelineSidebarProps) => { return ( -