From 0bfd6295f21f3f3c81d34cd9b61b6ba494e3a4fe Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 24 Apr 2025 16:35:34 +0200 Subject: [PATCH 1/3] feat: add validation --- packages/cli/src/cmd/validate.ts | 62 ++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/cmd/validate.ts b/packages/cli/src/cmd/validate.ts index dd2fe3cf..319ea338 100644 --- a/packages/cli/src/cmd/validate.ts +++ b/packages/cli/src/cmd/validate.ts @@ -1,5 +1,9 @@ import type { CLIArguments } from "../cli-utils"; -import { green } from "farver/fast"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { sourceHandlers } from "@mojis/adapters"; +import { arktypeParse } from "@mojis/internal-utils"; +import { green, red } from "farver/fast"; import { printHelp } from "../cli-utils"; export interface CLIValidateCmdOptions { @@ -9,7 +13,7 @@ export interface CLIValidateCmdOptions { versions: string[]; } -export async function runValidate({ versions: _providedVersions, flags }: CLIValidateCmdOptions) { +export async function runValidate({ versions, flags }: CLIValidateCmdOptions) { if (flags?.help || flags?.h) { printHelp({ headline: "Validate generated Emoji Data.", @@ -25,6 +29,60 @@ export async function runValidate({ versions: _providedVersions, flags }: CLIVal return; } + const inputDir = flags.inputDir ?? "data"; + const versionsToValidate = versions.length > 0 ? versions : ["15.0"]; // Default to latest version + // eslint-disable-next-line no-console console.log(` ${green("validating emoji data...")}`); + + let hasErrors = false; + + for (const version of versionsToValidate) { + const errors = await validateVersion(version, inputDir); + + if (errors.length > 0) { + hasErrors = true; + console.error(`\n ${red(`✖ validation failed for version ${version}:`)}`); + for (const error of errors) { + console.error(` ${red("•")} ${error}`); + } + } else { + // eslint-disable-next-line no-console + console.log(` ${green(`✓ version ${version} validated successfully`)}`); + } + } + + if (hasErrors) { + // eslint-disable-next-line node/prefer-global/process + process.exit(1); + } +} + +async function validateVersion(version: string, inputDir: string) { + const versionDir = join(inputDir, `v${version}`); + const errors: string[] = []; + + for (const [, adapter] of Object.entries(sourceHandlers)) { + // For each schema defined in the adapter's persistence + for (const [key, schema] of Object.entries(adapter.persistence.schemas)) { + try { + // Handle files that could match the pattern + const files = await readdir(join(versionDir), { recursive: true }); + for (const file of files) { + console.log(join(versionDir, file)); + const content = await readFile(join(versionDir, file), "utf-8"); + const data = JSON.parse(content); + + const result = arktypeParse(data, schema.schema); + if (!result.success) { + errors.push(`${file}: ${result.errors.join(", ")}`); + } + } + } catch (err: any) { + errors.push(`Failed to validate ${key}: ${err.message}`); + } + } + } + + return errors; } From f876ebf08901d48156cccdc6195e6910a250ad4f Mon Sep 17 00:00:00 2001 From: Lucas Date: Sat, 26 Apr 2025 12:33:15 +0200 Subject: [PATCH 2/3] refactor!: switch to ucdjs unicode proxy --- .../adapters/src/handlers/source/metadata.ts | 2 +- .../adapters/src/handlers/source/sequences.ts | 4 ++-- .../src/handlers/source/unicode-names.ts | 12 +++++----- .../src/handlers/source/variations.ts | 4 ++-- .../test/handlers/source/metadata.test.ts | 14 ++++++------ .../test/handlers/source/sequences.test.ts | 8 +++---- .../handlers/source/unicode-names.test.ts | 14 ++++++------ .../test/handlers/source/variations.test.ts | 14 ++++++------ .../parsers/scripts/update-parser-fixtures.ts | 6 ++--- packages/versions/src/api.ts | 8 +++---- packages/versions/test/api.test.ts | 22 +++++++++---------- 11 files changed, 54 insertions(+), 54 deletions(-) diff --git a/packages/adapters/src/handlers/source/metadata.ts b/packages/adapters/src/handlers/source/metadata.ts index 3da7627e..bbd468b4 100644 --- a/packages/adapters/src/handlers/source/metadata.ts +++ b/packages/adapters/src/handlers/source/metadata.ts @@ -61,7 +61,7 @@ export const handler = builder return builder .urls((ctx) => { return { - url: `https://unicode-proxy.mojis.dev/proxy/emoji/${ctx.emoji_version}/emoji-test.txt`, + url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${ctx.emoji_version}/emoji-test.txt`, cacheKey: `v${ctx.emoji_version}/metadata`, }; }) diff --git a/packages/adapters/src/handlers/source/sequences.ts b/packages/adapters/src/handlers/source/sequences.ts index 5f99a8a6..ec1732ed 100644 --- a/packages/adapters/src/handlers/source/sequences.ts +++ b/packages/adapters/src/handlers/source/sequences.ts @@ -72,12 +72,12 @@ export const handler = builder return [ { key: "sequences", - url: `https://unicode-proxy.mojis.dev/proxy/emoji/${emoji_version}/emoji-sequences.txt`, + url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${emoji_version}/emoji-sequences.txt`, cacheKey: `v${emoji_version}/sequences`, }, { key: "zwj", - url: `https://unicode-proxy.mojis.dev/proxy/emoji/${emoji_version}/emoji-zwj-sequences.txt`, + url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${emoji_version}/emoji-zwj-sequences.txt`, cacheKey: `v${emoji_version}/zwj-sequences`, }, ]; diff --git a/packages/adapters/src/handlers/source/unicode-names.ts b/packages/adapters/src/handlers/source/unicode-names.ts index 04bb93a2..eb8c1405 100644 --- a/packages/adapters/src/handlers/source/unicode-names.ts +++ b/packages/adapters/src/handlers/source/unicode-names.ts @@ -3,11 +3,11 @@ import { createSourceAdapter } from "../../builders/source-builder/builder"; import { joinPath } from "../../utils"; const MAPPINGS = { - "1.0": "https://unicode-proxy.mojis.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt", - "2.0": "https://unicode-proxy.mojis.dev/proxy/2.0-Update/UnicodeData-2.0.14.txt", - "3.0": "https://unicode-proxy.mojis.dev/proxy/3.0-Update1/UnicodeData-3.0.1.txt", - "4.0": "https://unicode-proxy.mojis.dev/proxy/4.0-Update1/UnicodeData-4.0.1.txt", - "13.1": "https://unicode-proxy.mojis.dev/proxy/13.0.0/ucd/UnicodeData.txt", + "1.0": "https://unicode-proxy.ucdjs.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt", + "2.0": "https://unicode-proxy.ucdjs.dev/proxy/2.0-Update/UnicodeData-2.0.14.txt", + "3.0": "https://unicode-proxy.ucdjs.dev/proxy/3.0-Update1/UnicodeData-3.0.1.txt", + "4.0": "https://unicode-proxy.ucdjs.dev/proxy/4.0-Update1/UnicodeData-4.0.1.txt", + "13.1": "https://unicode-proxy.ucdjs.dev/proxy/13.0.0/ucd/UnicodeData.txt", } as Record; const builder = createSourceAdapter({ @@ -38,7 +38,7 @@ export const handler = builder (builder) => builder .urls((ctx) => { return { - url: MAPPINGS[ctx.emoji_version] || `https://unicode-proxy.mojis.dev/proxy/${ctx.emoji_version}.0/ucd/UnicodeData.txt`, + url: MAPPINGS[ctx.emoji_version] || `https://unicode-proxy.ucdjs.dev/proxy/${ctx.emoji_version}.0/ucd/UnicodeData.txt`, cacheKey: `v${ctx.emoji_version}/unicode-names`, }; }) diff --git a/packages/adapters/src/handlers/source/variations.ts b/packages/adapters/src/handlers/source/variations.ts index 41ef0fe4..86e8b54c 100644 --- a/packages/adapters/src/handlers/source/variations.ts +++ b/packages/adapters/src/handlers/source/variations.ts @@ -31,13 +31,13 @@ export const handler = builder .urls((ctx) => { if (lte(ctx.unicode_version, "12.1")) { return { - url: `https://unicode-proxy.mojis.dev/proxy/emoji/${ctx.emoji_version}/emoji-variation-sequences.txt`, + url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${ctx.emoji_version}/emoji-variation-sequences.txt`, cacheKey: `v${ctx.emoji_version}/variations`, }; } return { - url: `https://unicode-proxy.mojis.dev/proxy/${ctx.unicode_version}.0/ucd/emoji/emoji-variation-sequences.txt`, + url: `https://unicode-proxy.ucdjs.dev/proxy/${ctx.unicode_version}.0/ucd/emoji/emoji-variation-sequences.txt`, cacheKey: `v${ctx.emoji_version}/variations`, }; }) diff --git a/packages/adapters/test/handlers/source/metadata.test.ts b/packages/adapters/test/handlers/source/metadata.test.ts index e8e94e1f..ff2b3d97 100644 --- a/packages/adapters/test/handlers/source/metadata.test.ts +++ b/packages/adapters/test/handlers/source/metadata.test.ts @@ -22,7 +22,7 @@ describe("metadata adapter handler", () => { }); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], ]); const result = await runSourceAdapter(metadataHandler, mockContext); @@ -143,7 +143,7 @@ describe("metadata adapter handler", () => { `; mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], ]); const result = await runSourceAdapter(metadataHandler, mockContext); @@ -166,7 +166,7 @@ describe("metadata adapter handler", () => { const { runSourceAdapter } = await setupAdapterTest(); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text("")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text("")], ]); const result = await runSourceAdapter(metadataHandler, mockContext); @@ -179,7 +179,7 @@ describe("metadata adapter handler", () => { it("should handle network errors", async () => { const { runSourceAdapter } = await setupAdapterTest(); - mockFetch(`GET https://unicode-proxy.mojis.dev/proxy/emoji/${mockContext.emoji_version}/emoji-test.txt`, () => { + mockFetch(`GET https://unicode-proxy.ucdjs.dev/proxy/emoji/${mockContext.emoji_version}/emoji-test.txt`, () => { return HttpResponse.error(); }); @@ -200,7 +200,7 @@ describe("metadata adapter handler", () => { let fetchCount = 0; mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => { + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => { fetchCount++; return HttpResponse.text(mockEmojiTest); }], @@ -225,7 +225,7 @@ describe("metadata adapter handler", () => { }); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], ]); await expect(runSourceAdapter(metadataHandler, mockContext)) @@ -242,7 +242,7 @@ describe("metadata adapter handler", () => { `; mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)], ]); await expect(runSourceAdapter(metadataHandler, mockContext)) diff --git a/packages/adapters/test/handlers/source/sequences.test.ts b/packages/adapters/test/handlers/source/sequences.test.ts index a8691e92..cab4afc8 100644 --- a/packages/adapters/test/handlers/source/sequences.test.ts +++ b/packages/adapters/test/handlers/source/sequences.test.ts @@ -48,11 +48,11 @@ describe("sequences adapter handler", () => { mockFetch([ [ - "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-sequences.txt", + "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-sequences.txt", () => HttpResponse.text(mockedSequences), ], [ - "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt", + "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt", () => HttpResponse.text(), ], ]); @@ -128,11 +128,11 @@ describe("sequences adapter handler", () => { mockFetch([ [ - "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-sequences.txt", + "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-sequences.txt", () => HttpResponse.text(), ], [ - "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt", + "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt", () => HttpResponse.text(mockedSequences), ], ]); diff --git a/packages/adapters/test/handlers/source/unicode-names.test.ts b/packages/adapters/test/handlers/source/unicode-names.test.ts index 09c8fe2c..0da34e6a 100644 --- a/packages/adapters/test/handlers/source/unicode-names.test.ts +++ b/packages/adapters/test/handlers/source/unicode-names.test.ts @@ -15,7 +15,7 @@ describe("unicode-names adapter handler", () => { const { runSourceAdapter } = await setupAdapterTest(); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600;GRINNING FACE")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600;GRINNING FACE")], ]); const result = await runSourceAdapter(unicodeNamesHandler, mockContext); @@ -28,7 +28,7 @@ describe("unicode-names adapter handler", () => { const { runSourceAdapter } = await setupAdapterTest(); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt", () => HttpResponse.text("1F600;GRINNING FACE")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt", () => HttpResponse.text("1F600;GRINNING FACE")], ]); const result = await runSourceAdapter(unicodeNamesHandler, { ...mockContext, emoji_version: "1.0" }); @@ -41,7 +41,7 @@ describe("unicode-names adapter handler", () => { const { runSourceAdapter } = await setupAdapterTest(); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text( + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text( "1F600;GRINNING FACE\n" + "1F601;GRINNING FACE WITH SMILING EYES\n" + "1F602;FACE WITH TEARS OF JOY", @@ -60,7 +60,7 @@ describe("unicode-names adapter handler", () => { const { runSourceAdapter } = await setupAdapterTest(); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600")], ]); await expect(runSourceAdapter(unicodeNamesHandler, mockContext)) @@ -72,7 +72,7 @@ describe("unicode-names adapter handler", () => { const { runSourceAdapter } = await setupAdapterTest(); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("")], ]); const result = await runSourceAdapter(unicodeNamesHandler, mockContext); @@ -82,7 +82,7 @@ describe("unicode-names adapter handler", () => { it("should handle network errors", async () => { const { runSourceAdapter } = await setupAdapterTest(); - mockFetch(`GET https://unicode-proxy.mojis.dev/proxy/${mockContext.emoji_version}.0/ucd/UnicodeData.txt`, () => { + mockFetch(`GET https://unicode-proxy.ucdjs.dev/proxy/${mockContext.emoji_version}.0/ucd/UnicodeData.txt`, () => { return HttpResponse.error(); }); @@ -97,7 +97,7 @@ describe("unicode-names adapter handler", () => { let fetchCount = 0; mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => { + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => { fetchCount++; return HttpResponse.text("1F600;GRINNING FACE"); }], diff --git a/packages/adapters/test/handlers/source/variations.test.ts b/packages/adapters/test/handlers/source/variations.test.ts index a33b8364..7089437a 100644 --- a/packages/adapters/test/handlers/source/variations.test.ts +++ b/packages/adapters/test/handlers/source/variations.test.ts @@ -22,7 +22,7 @@ describe("variations adapter handler", () => { }); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], ]); const result = await runSourceAdapter(variationsHandler, mockContext); @@ -66,7 +66,7 @@ FE0E ; text style # VS-15 `; mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/12.1/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/12.1/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], ]); const result = await runSourceAdapter(variationsHandler, { @@ -100,7 +100,7 @@ FE0E ; text style # VS-15 const { runSourceAdapter } = await setupAdapterTest(); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text("")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text("")], ]); const result = await runSourceAdapter(variationsHandler, mockContext); @@ -110,7 +110,7 @@ FE0E ; text style # VS-15 it("should handle network errors", async () => { const { runSourceAdapter } = await setupAdapterTest(); - mockFetch(`GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt`, () => { + mockFetch(`GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt`, () => { return HttpResponse.error(); }); @@ -125,7 +125,7 @@ FE0E ; text style # VS-15 let fetchCount = 0; mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => { + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => { fetchCount++; return HttpResponse.text("FE0E ; text style # VS-15"); }], @@ -148,7 +148,7 @@ invalid-line `; mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], ]); await expect(runSourceAdapter(variationsHandler, mockContext)) @@ -166,7 +166,7 @@ invalid-line }); mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], + ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)], ]); await expect(runSourceAdapter(variationsHandler, mockContext)) diff --git a/packages/parsers/scripts/update-parser-fixtures.ts b/packages/parsers/scripts/update-parser-fixtures.ts index 6b9a1590..b06964fa 100644 --- a/packages/parsers/scripts/update-parser-fixtures.ts +++ b/packages/parsers/scripts/update-parser-fixtures.ts @@ -11,7 +11,7 @@ interface Entry { } async function run() { - const rootResponse = await fetch("https://unicode-proxy.mojis.dev/proxy/emoji/"); + const rootResponse = await fetch("https://unicode-proxy.ucdjs.dev/proxy/emoji/"); if (!rootResponse.ok) { throw new Error("failed to fetch root entry"); @@ -21,7 +21,7 @@ async function run() { await mkdir(root, { recursive: true }); async function processDirectory(entry: Entry, basePath: string) { - const dirResponse = await fetch(`https://unicode-proxy.mojis.dev/proxy${entry.path}`); + const dirResponse = await fetch(`https://unicode-proxy.ucdjs.dev/proxy${entry.path}`); const dirEntries: Entry[] = await dirResponse.json(); const fileEntries = dirEntries.filter( @@ -41,7 +41,7 @@ async function run() { await mkdir(new URL(type, root), { recursive: true }); - const content = await fetch(`https://unicode-proxy.mojis.dev/proxy${fullPath}`).then((res) => res.text()); + const content = await fetch(`https://unicode-proxy.ucdjs.dev/proxy${fullPath}`).then((res) => res.text()); await writeFile( new URL(`${type}/v${version.toString()}${fileExt}`, root), content, diff --git a/packages/versions/src/api.ts b/packages/versions/src/api.ts index 947915f8..d58b71d5 100644 --- a/packages/versions/src/api.ts +++ b/packages/versions/src/api.ts @@ -23,8 +23,8 @@ export interface DraftVersion { */ export async function getCurrentDraftVersion(): Promise { const [draftText, emojiText] = await Promise.all([ - "https://unicode-proxy.mojis.dev/proxy/draft/ReadMe.txt", - "https://unicode-proxy.mojis.dev/proxy/draft/emoji/ReadMe.txt", + "https://unicode-proxy.ucdjs.dev/proxy/draft/ReadMe.txt", + "https://unicode-proxy.ucdjs.dev/proxy/draft/emoji/ReadMe.txt", ].map(async (url) => { const res = await fetch(url); @@ -72,8 +72,8 @@ export async function getCurrentDraftVersion(): Promise { */ export async function getAllEmojiVersions(): Promise { const [rootResult, emojiResult] = await Promise.allSettled([ - "https://unicode-proxy.mojis.dev/proxy/", - "https://unicode-proxy.mojis.dev/proxy/emoji/", + "https://unicode-proxy.ucdjs.dev/proxy/", + "https://unicode-proxy.ucdjs.dev/proxy/emoji/", ].map(async (url) => { const res = await fetch(url); diff --git a/packages/versions/test/api.test.ts b/packages/versions/test/api.test.ts index 2506616e..41c6d776 100644 --- a/packages/versions/test/api.test.ts +++ b/packages/versions/test/api.test.ts @@ -19,8 +19,8 @@ describe("all emoji versions", () => { it("should throw if fetch returns invalid data", async () => { mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/", () => HttpResponse.text("Not Found", { status: 400 })], - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/", () => HttpResponse.text("Not Found", { status: 400 })], + ["GET https://unicode-proxy.ucdjs.dev/proxy/", () => HttpResponse.text("Not Found", { status: 400 })], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/", () => HttpResponse.text("Not Found", { status: 400 })], ]); await expect(() => getAllEmojiVersions()).rejects.toThrow("failed to fetch root or emoji page"); @@ -28,8 +28,8 @@ describe("all emoji versions", () => { it("should throw if empty data is returned", async () => { mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/", () => HttpResponse.text("")], - ["GET https://unicode-proxy.mojis.dev/proxy/emoji/", () => HttpResponse.text("")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/", () => HttpResponse.text("")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/", () => HttpResponse.text("")], ]); await expect(() => getAllEmojiVersions()).rejects.toThrow("failed to fetch root or emoji page"); @@ -39,8 +39,8 @@ describe("all emoji versions", () => { describe("draft", () => { it("returns draft versions when fetches succeed and versions match", async () => { mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/draft/ReadMe.txt", () => HttpResponse.text("Version 15.1.0 of the Unicode Standard")], - ["GET https://unicode-proxy.mojis.dev/proxy/draft/emoji/ReadMe.txt", () => HttpResponse.text("Unicode Emoji, Version 15.1")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/draft/ReadMe.txt", () => HttpResponse.text("Version 15.1.0 of the Unicode Standard")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/draft/emoji/ReadMe.txt", () => HttpResponse.text("Unicode Emoji, Version 15.1")], ]); const result = await getCurrentDraftVersion(); @@ -52,8 +52,8 @@ describe("draft", () => { it("should throw when versions do not match", async () => { mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/draft/ReadMe.txt", () => HttpResponse.text("Version 15.1.0 of the Unicode Standard")], - ["GET https://unicode-proxy.mojis.dev/proxy/draft/emoji/ReadMe.txt", () => HttpResponse.text("Unicode Emoji, Version 15.0")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/draft/ReadMe.txt", () => HttpResponse.text("Version 15.1.0 of the Unicode Standard")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/draft/emoji/ReadMe.txt", () => HttpResponse.text("Unicode Emoji, Version 15.0")], ]); await expect(() => getCurrentDraftVersion()).rejects.toThrow("draft versions do not match"); @@ -64,13 +64,13 @@ describe("draft", () => { return new HttpResponse("Not Found", { status: 404 }); }); - await expect(() => getCurrentDraftVersion()).rejects.toThrow("failed to fetch https://unicode-proxy.mojis.dev/proxy/draft/ReadMe.txt: 404"); + await expect(() => getCurrentDraftVersion()).rejects.toThrow("failed to fetch https://unicode-proxy.ucdjs.dev/proxy/draft/ReadMe.txt: 404"); }); it("should throw if emoji version is invalid", async () => { mockFetch([ - ["GET https://unicode-proxy.mojis.dev/proxy/draft/ReadMe.txt", () => HttpResponse.text("")], - ["GET https://unicode-proxy.mojis.dev/proxy/draft/emoji/ReadMe.txt", () => HttpResponse.text("Unicode Emoji, Version 15.0")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/draft/ReadMe.txt", () => HttpResponse.text("")], + ["GET https://unicode-proxy.ucdjs.dev/proxy/draft/emoji/ReadMe.txt", () => HttpResponse.text("Unicode Emoji, Version 15.0")], ]); await expect(() => getCurrentDraftVersion()).rejects.toThrow("failed to extract draft version"); From 043abd2cd3c31c05f8fd3661f635872d2e4ba95c Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 24 Apr 2025 16:35:34 +0200 Subject: [PATCH 3/3] feat: add validation --- packages/cli/src/cmd/validate.ts | 62 ++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/cmd/validate.ts b/packages/cli/src/cmd/validate.ts index dd2fe3cf..319ea338 100644 --- a/packages/cli/src/cmd/validate.ts +++ b/packages/cli/src/cmd/validate.ts @@ -1,5 +1,9 @@ import type { CLIArguments } from "../cli-utils"; -import { green } from "farver/fast"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { sourceHandlers } from "@mojis/adapters"; +import { arktypeParse } from "@mojis/internal-utils"; +import { green, red } from "farver/fast"; import { printHelp } from "../cli-utils"; export interface CLIValidateCmdOptions { @@ -9,7 +13,7 @@ export interface CLIValidateCmdOptions { versions: string[]; } -export async function runValidate({ versions: _providedVersions, flags }: CLIValidateCmdOptions) { +export async function runValidate({ versions, flags }: CLIValidateCmdOptions) { if (flags?.help || flags?.h) { printHelp({ headline: "Validate generated Emoji Data.", @@ -25,6 +29,60 @@ export async function runValidate({ versions: _providedVersions, flags }: CLIVal return; } + const inputDir = flags.inputDir ?? "data"; + const versionsToValidate = versions.length > 0 ? versions : ["15.0"]; // Default to latest version + // eslint-disable-next-line no-console console.log(` ${green("validating emoji data...")}`); + + let hasErrors = false; + + for (const version of versionsToValidate) { + const errors = await validateVersion(version, inputDir); + + if (errors.length > 0) { + hasErrors = true; + console.error(`\n ${red(`✖ validation failed for version ${version}:`)}`); + for (const error of errors) { + console.error(` ${red("•")} ${error}`); + } + } else { + // eslint-disable-next-line no-console + console.log(` ${green(`✓ version ${version} validated successfully`)}`); + } + } + + if (hasErrors) { + // eslint-disable-next-line node/prefer-global/process + process.exit(1); + } +} + +async function validateVersion(version: string, inputDir: string) { + const versionDir = join(inputDir, `v${version}`); + const errors: string[] = []; + + for (const [, adapter] of Object.entries(sourceHandlers)) { + // For each schema defined in the adapter's persistence + for (const [key, schema] of Object.entries(adapter.persistence.schemas)) { + try { + // Handle files that could match the pattern + const files = await readdir(join(versionDir), { recursive: true }); + for (const file of files) { + console.log(join(versionDir, file)); + const content = await readFile(join(versionDir, file), "utf-8"); + const data = JSON.parse(content); + + const result = arktypeParse(data, schema.schema); + if (!result.success) { + errors.push(`${file}: ${result.errors.join(", ")}`); + } + } + } catch (err: any) { + errors.push(`Failed to validate ${key}: ${err.message}`); + } + } + } + + return errors; }