feat: add validation

packages/adapters/src/handlers/source/metadata.ts

@@ -61,7 +61,7 @@ export const handler = builder
       return builder
         .urls((ctx) => {
           return {
-            url: `https://unicode-proxy.mojis.dev/proxy/emoji/${ctx.emoji_version}/emoji-test.txt`,
+            url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${ctx.emoji_version}/emoji-test.txt`,
             cacheKey: `v${ctx.emoji_version}/metadata`,
           };
         })

packages/adapters/src/handlers/source/sequences.ts

@@ -72,12 +72,12 @@ export const handler = builder
         return [
           {
             key: "sequences",
-            url: `https://unicode-proxy.mojis.dev/proxy/emoji/${emoji_version}/emoji-sequences.txt`,
+            url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${emoji_version}/emoji-sequences.txt`,
             cacheKey: `v${emoji_version}/sequences`,
           },
           {
             key: "zwj",
-            url: `https://unicode-proxy.mojis.dev/proxy/emoji/${emoji_version}/emoji-zwj-sequences.txt`,
+            url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${emoji_version}/emoji-zwj-sequences.txt`,
             cacheKey: `v${emoji_version}/zwj-sequences`,
           },
         ];

packages/adapters/src/handlers/source/unicode-names.ts

@@ -3,11 +3,11 @@ import { createSourceAdapter } from "../../builders/source-builder/builder";
 import { joinPath } from "../../utils";
 
 const MAPPINGS = {
-  "1.0": "https://unicode-proxy.mojis.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt",
-  "2.0": "https://unicode-proxy.mojis.dev/proxy/2.0-Update/UnicodeData-2.0.14.txt",
-  "3.0": "https://unicode-proxy.mojis.dev/proxy/3.0-Update1/UnicodeData-3.0.1.txt",
-  "4.0": "https://unicode-proxy.mojis.dev/proxy/4.0-Update1/UnicodeData-4.0.1.txt",
-  "13.1": "https://unicode-proxy.mojis.dev/proxy/13.0.0/ucd/UnicodeData.txt",
+  "1.0": "https://unicode-proxy.ucdjs.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt",
+  "2.0": "https://unicode-proxy.ucdjs.dev/proxy/2.0-Update/UnicodeData-2.0.14.txt",
+  "3.0": "https://unicode-proxy.ucdjs.dev/proxy/3.0-Update1/UnicodeData-3.0.1.txt",
+  "4.0": "https://unicode-proxy.ucdjs.dev/proxy/4.0-Update1/UnicodeData-4.0.1.txt",
+  "13.1": "https://unicode-proxy.ucdjs.dev/proxy/13.0.0/ucd/UnicodeData.txt",
 } as Record<string, string>;
 
 const builder = createSourceAdapter({
@@ -38,7 +38,7 @@ export const handler = builder
     (builder) => builder
       .urls((ctx) => {
         return {
-          url: MAPPINGS[ctx.emoji_version] || `https://unicode-proxy.mojis.dev/proxy/${ctx.emoji_version}.0/ucd/UnicodeData.txt`,
+          url: MAPPINGS[ctx.emoji_version] || `https://unicode-proxy.ucdjs.dev/proxy/${ctx.emoji_version}.0/ucd/UnicodeData.txt`,
           cacheKey: `v${ctx.emoji_version}/unicode-names`,
         };
       })

packages/adapters/src/handlers/source/variations.ts

@@ -31,13 +31,13 @@ export const handler = builder
       .urls((ctx) => {
         if (lte(ctx.unicode_version, "12.1")) {
           return {
-            url: `https://unicode-proxy.mojis.dev/proxy/emoji/${ctx.emoji_version}/emoji-variation-sequences.txt`,
+            url: `https://unicode-proxy.ucdjs.dev/proxy/emoji/${ctx.emoji_version}/emoji-variation-sequences.txt`,
             cacheKey: `v${ctx.emoji_version}/variations`,
           };
         }
 
         return {
-          url: `https://unicode-proxy.mojis.dev/proxy/${ctx.unicode_version}.0/ucd/emoji/emoji-variation-sequences.txt`,
+          url: `https://unicode-proxy.ucdjs.dev/proxy/${ctx.unicode_version}.0/ucd/emoji/emoji-variation-sequences.txt`,
           cacheKey: `v${ctx.emoji_version}/variations`,
         };
       })

packages/adapters/test/handlers/source/metadata.test.ts

@@ -22,7 +22,7 @@ describe("metadata adapter handler", () => {
     });
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
     ]);
 
     const result = await runSourceAdapter(metadataHandler, mockContext);
@@ -143,7 +143,7 @@ describe("metadata adapter handler", () => {
 `;
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
     ]);
 
     const result = await runSourceAdapter(metadataHandler, mockContext);
@@ -166,7 +166,7 @@ describe("metadata adapter handler", () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text("")],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text("")],
     ]);
 
     const result = await runSourceAdapter(metadataHandler, mockContext);
@@ -179,7 +179,7 @@ describe("metadata adapter handler", () => {
   it("should handle network errors", async () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
-    mockFetch(`GET https://unicode-proxy.mojis.dev/proxy/emoji/${mockContext.emoji_version}/emoji-test.txt`, () => {
+    mockFetch(`GET https://unicode-proxy.ucdjs.dev/proxy/emoji/${mockContext.emoji_version}/emoji-test.txt`, () => {
       return HttpResponse.error();
     });
 
@@ -200,7 +200,7 @@ describe("metadata adapter handler", () => {
 
     let fetchCount = 0;
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => {
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => {
         fetchCount++;
         return HttpResponse.text(mockEmojiTest);
       }],
@@ -225,7 +225,7 @@ describe("metadata adapter handler", () => {
     });
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
     ]);
 
     await expect(runSourceAdapter(metadataHandler, mockContext))
@@ -242,7 +242,7 @@ describe("metadata adapter handler", () => {
 `;
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-test.txt", () => HttpResponse.text(mockEmojiTest)],
     ]);
 
     await expect(runSourceAdapter(metadataHandler, mockContext))

packages/adapters/test/handlers/source/sequences.test.ts

@@ -48,11 +48,11 @@ describe("sequences adapter handler", () => {
 
     mockFetch([
       [
-        "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-sequences.txt",
+        "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-sequences.txt",
         () => HttpResponse.text(mockedSequences),
       ],
       [
-        "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt",
+        "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt",
         () => HttpResponse.text(),
       ],
     ]);
@@ -128,11 +128,11 @@ describe("sequences adapter handler", () => {
 
     mockFetch([
       [
-        "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-sequences.txt",
+        "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-sequences.txt",
         () => HttpResponse.text(),
       ],
       [
-        "GET https://unicode-proxy.mojis.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt",
+        "GET https://unicode-proxy.ucdjs.dev/proxy/emoji/15.0/emoji-zwj-sequences.txt",
         () => HttpResponse.text(mockedSequences),
       ],
     ]);

packages/adapters/test/handlers/source/unicode-names.test.ts

@@ -15,7 +15,7 @@ describe("unicode-names adapter handler", () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600;GRINNING FACE")],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600;GRINNING FACE")],
     ]);
 
     const result = await runSourceAdapter(unicodeNamesHandler, mockContext);
@@ -28,7 +28,7 @@ describe("unicode-names adapter handler", () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt", () => HttpResponse.text("1F600;GRINNING FACE")],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/1.1-Update/UnicodeData-1.1.5.txt", () => HttpResponse.text("1F600;GRINNING FACE")],
     ]);
 
     const result = await runSourceAdapter(unicodeNamesHandler, { ...mockContext, emoji_version: "1.0" });
@@ -41,7 +41,7 @@ describe("unicode-names adapter handler", () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text(
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text(
         "1F600;GRINNING FACE\n"
         + "1F601;GRINNING FACE WITH SMILING EYES\n"
         + "1F602;FACE WITH TEARS OF JOY",
@@ -60,7 +60,7 @@ describe("unicode-names adapter handler", () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600")],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("1F600")],
     ]);
 
     await expect(runSourceAdapter(unicodeNamesHandler, mockContext))
@@ -72,7 +72,7 @@ describe("unicode-names adapter handler", () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("")],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => HttpResponse.text("")],
     ]);
 
     const result = await runSourceAdapter(unicodeNamesHandler, mockContext);
@@ -82,7 +82,7 @@ describe("unicode-names adapter handler", () => {
   it("should handle network errors", async () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
-    mockFetch(`GET https://unicode-proxy.mojis.dev/proxy/${mockContext.emoji_version}.0/ucd/UnicodeData.txt`, () => {
+    mockFetch(`GET https://unicode-proxy.ucdjs.dev/proxy/${mockContext.emoji_version}.0/ucd/UnicodeData.txt`, () => {
       return HttpResponse.error();
     });
 
@@ -97,7 +97,7 @@ describe("unicode-names adapter handler", () => {
 
     let fetchCount = 0;
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => {
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/UnicodeData.txt", () => {
         fetchCount++;
         return HttpResponse.text("1F600;GRINNING FACE");
       }],

packages/adapters/test/handlers/source/variations.test.ts

@@ -22,7 +22,7 @@ describe("variations adapter handler", () => {
     });
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
     ]);
 
     const result = await runSourceAdapter(variationsHandler, mockContext);
@@ -66,7 +66,7 @@ FE0E ; text style     # VS-15
 `;
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/emoji/12.1/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/emoji/12.1/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
     ]);
 
     const result = await runSourceAdapter(variationsHandler, {
@@ -100,7 +100,7 @@ FE0E ; text style     # VS-15
     const { runSourceAdapter } = await setupAdapterTest();
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text("")],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text("")],
     ]);
 
     const result = await runSourceAdapter(variationsHandler, mockContext);
@@ -110,7 +110,7 @@ FE0E ; text style     # VS-15
   it("should handle network errors", async () => {
     const { runSourceAdapter } = await setupAdapterTest();
 
-    mockFetch(`GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt`, () => {
+    mockFetch(`GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt`, () => {
       return HttpResponse.error();
     });
 
@@ -125,7 +125,7 @@ FE0E ; text style     # VS-15
 
     let fetchCount = 0;
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => {
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => {
         fetchCount++;
         return HttpResponse.text("FE0E ; text style     # VS-15");
       }],
@@ -148,7 +148,7 @@ invalid-line
 `;
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
     ]);
 
     await expect(runSourceAdapter(variationsHandler, mockContext))
@@ -166,7 +166,7 @@ invalid-line
     });
 
     mockFetch([
-      ["GET https://unicode-proxy.mojis.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
+      ["GET https://unicode-proxy.ucdjs.dev/proxy/15.0.0/ucd/emoji/emoji-variation-sequences.txt", () => HttpResponse.text(mockVariations)],
     ]);
 
     await expect(runSourceAdapter(variationsHandler, mockContext))

packages/cli/src/cmd/validate.ts

@@ -1,5 +1,9 @@
 import type { CLIArguments } from "../cli-utils";
-import { green } from "farver/fast";
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { sourceHandlers } from "@mojis/adapters";
+import { arktypeParse } from "@mojis/internal-utils";
+import { green, red } from "farver/fast";
 import { printHelp } from "../cli-utils";
 
 export interface CLIValidateCmdOptions {
@@ -9,7 +13,7 @@
   versions: string[];
 }
 
-export async function runValidate({ versions: _providedVersions, flags }: CLIValidateCmdOptions) {
+export async function runValidate({ versions, flags }: CLIValidateCmdOptions) {
   if (flags?.help || flags?.h) {
     printHelp({
       headline: "Validate generated Emoji Data.",
@@ -25,6 +29,60 @@
     return;
   }
 
+  const inputDir = flags.inputDir ?? "data";
+  const versionsToValidate = versions.length > 0 ? versions : ["15.0"]; // Default to latest version
+
   // eslint-disable-next-line no-console
   console.log(`  ${green("validating emoji data...")}`);
+
+  let hasErrors = false;
+
+  for (const version of versionsToValidate) {
+    const errors = await validateVersion(version, inputDir);
+
+    if (errors.length > 0) {
+      hasErrors = true;
+      console.error(`\n  ${red(`✖ validation failed for version ${version}:`)}`);
+      for (const error of errors) {
+        console.error(`    ${red("•")} ${error}`);
+      }
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(`  ${green(`✓ version ${version} validated successfully`)}`);
+    }
+  }
+
+  if (hasErrors) {
+    // eslint-disable-next-line node/prefer-global/process
+    process.exit(1);
+  }
+}
+
+async function validateVersion(version: string, inputDir: string) {
+  const versionDir = join(inputDir, `v${version}`);
+  const errors: string[] = [];
+
+  for (const [, adapter] of Object.entries(sourceHandlers)) {
+    // For each schema defined in the adapter's persistence
+    for (const [key, schema] of Object.entries(adapter.persistence.schemas)) {
+      try {
+        // Handle files that could match the pattern
+        const files = await readdir(join(versionDir), { recursive: true });
+        for (const file of files) {
+          console.log(join(versionDir, file));
+          const content = await readFile(join(versionDir, file), "utf-8");
+          const data = JSON.parse(content);
+
+          const result = arktypeParse(data, schema.schema);
+          if (!result.success) {
+            errors.push(`${file}: ${result.errors.join(", ")}`);
+          }
+        }
+      } catch (err: any) {
+        errors.push(`Failed to validate ${key}: ${err.message}`);
+      }
+    }
+  }
+
+  return errors;
 }

packages/parsers/scripts/update-parser-fixtures.ts

@@ -11,7 +11,7 @@ interface Entry {
 }
 
 async function run() {
-  const rootResponse = await fetch("https://unicode-proxy.mojis.dev/proxy/emoji/");
+  const rootResponse = await fetch("https://unicode-proxy.ucdjs.dev/proxy/emoji/");
 
   if (!rootResponse.ok) {
     throw new Error("failed to fetch root entry");
@@ -21,7 +21,7 @@ async function run() {
   await mkdir(root, { recursive: true });
 
   async function processDirectory(entry: Entry, basePath: string) {
-    const dirResponse = await fetch(`https://unicode-proxy.mojis.dev/proxy${entry.path}`);
+    const dirResponse = await fetch(`https://unicode-proxy.ucdjs.dev/proxy${entry.path}`);
     const dirEntries: Entry[] = await dirResponse.json();
 
     const fileEntries = dirEntries.filter(
@@ -41,7 +41,7 @@ async function run() {
 
         await mkdir(new URL(type, root), { recursive: true });
 
-        const content = await fetch(`https://unicode-proxy.mojis.dev/proxy${fullPath}`).then((res) => res.text());
+        const content = await fetch(`https://unicode-proxy.ucdjs.dev/proxy${fullPath}`).then((res) => res.text());
         await writeFile(
           new URL(`${type}/v${version.toString()}${fileExt}`, root),
           content,