Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 103 additions & 2 deletions apps/api/src/lib/files.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,16 @@
/* eslint-disable no-console */
import type { Entry } from "apache-autoindex-parse";
import { trimLeadingSlash, trimTrailingSlash } from "@luxass/utils";
import { createGlobMatcher } from "@ucdjs-internal/shared";
import {
UCD_STAT_CHILDREN_DIRS_HEADER,
UCD_STAT_CHILDREN_FILES_HEADER,
UCD_STAT_CHILDREN_HEADER,
UCD_STAT_SIZE_HEADER,
UCD_STAT_TYPE_HEADER,
} from "@ucdjs/env";
import { parse } from "apache-autoindex-parse";
import { HTML_EXTENSIONS } from "../constants";

/**
* Parses an HTML directory listing from Unicode.org and extracts file/directory entries.
Expand Down Expand Up @@ -75,15 +84,13 @@ export function applyDirectoryFiltersAndSort(

// Apply query filter (prefix search, case-insensitive)
if (options.query) {
// eslint-disable-next-line no-console
console.info(`[v1_files]: applying query filter: ${options.query}`);
const queryLower = options.query.toLowerCase();
filtered = filtered.filter((entry) => entry.name.toLowerCase().startsWith(queryLower));
}

// Apply pattern filter if provided
if (options.pattern) {
// eslint-disable-next-line no-console
console.info(`[v1_files]: applying glob pattern filter: ${options.pattern}`);
const matcher = createGlobMatcher(options.pattern);
filtered = filtered.filter((entry) => matcher(entry.name));
Expand Down Expand Up @@ -123,3 +130,97 @@ export function applyDirectoryFiltersAndSort(

return filtered;
}

export function buildDirectoryHeaders(
files: Entry[],
baseHeaders: Record<string, string>,
): Record<string, string> {
return {
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "directory",
[UCD_STAT_CHILDREN_HEADER]: `${files.length}`,
[UCD_STAT_CHILDREN_FILES_HEADER]: `${files.filter((f) => f.type === "file").length}`,
[UCD_STAT_CHILDREN_DIRS_HEADER]: `${files.filter((f) => f.type === "directory").length}`,
};
}

export function buildFileHeaders(
contentType: string,
baseHeaders: Record<string, string>,
response: Response,
actualContentLength: number,
): Record<string, string> {
const headers: Record<string, string> = {
"Content-Type": contentType,
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "file",
[UCD_STAT_SIZE_HEADER]: `${actualContentLength}`,
"Content-Length": `${actualContentLength}`,
};

const cd = response.headers.get("Content-Disposition");
if (cd) headers["Content-Disposition"] = cd;

return headers;
}

export interface FileResponseOptions {
contentType: string;
baseHeaders: Record<string, string>;
response: Response;
isHeadRequest: boolean;
}

export async function handleFileResponse(
c: any,
Copy link

Copilot AI Jan 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The parameter c has type any which bypasses type checking. Consider using Context<HonoEnv> imported from "hono" for better type safety.

Copilot uses AI. Check for mistakes.
options: FileResponseOptions,
): Promise<Response> {
const { contentType, baseHeaders, response, isHeadRequest } = options;

if (isHeadRequest) {
const blob = await response.blob();
const actualSize = blob.size;
const headers = buildFileHeaders(contentType, baseHeaders, response, actualSize);
console.log(`[file-handler]: HEAD request, calculated size: ${actualSize}`);
return c.newResponse(null, 200, headers);
}

const headers: Record<string, string> = {
"Content-Type": contentType,
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "file",
};

const cd = response.headers.get("Content-Disposition");
if (cd) headers["Content-Disposition"] = cd;

console.log(`[file-handler]: binary file, streaming without buffering`);
Comment on lines +184 to +197
Copy link

Copilot AI Jan 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The log messages now use a generic [file-handler] prefix instead of route-specific prefixes like [v1_files] or [ucd-store]. This makes it harder to trace which route generated the log entry. Consider either passing a logger prefix/context as a parameter to these functions, or using the route-specific logging before calling these shared handlers.

Copilot uses AI. Check for mistakes.

return c.newResponse(response.body, 200, headers);
}

export interface DirectoryResponseOptions {
files: Entry[];
baseHeaders: Record<string, string>;
}

export function handleDirectoryResponse(
c: any,
Copy link

Copilot AI Jan 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The parameter c has type any which bypasses type checking. Consider using Context<HonoEnv> imported from "hono" for better type safety.

Copilot uses AI. Check for mistakes.
options: DirectoryResponseOptions,
): Response {
const { files, baseHeaders } = options;
const headers = buildDirectoryHeaders(files, baseHeaders);
return c.json(files, 200, headers);
}
Comment on lines +134 to +214
Copy link

Copilot AI Jan 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The newly added utility functions lack documentation. Consider adding JSDoc comments to describe their purpose, parameters, return values, and usage examples, especially for handleFileResponse, handleDirectoryResponse, buildDirectoryHeaders, and buildFileHeaders which are public exports.

Copilot uses AI. Check for mistakes.

export function determineFileExtension(leaf: string): string {
return leaf.includes(".") ? leaf.split(".").pop()!.toLowerCase() : "";
}

export function isHtmlFile(extName: string): boolean {
return HTML_EXTENSIONS.includes(`.${extName}`);
}

export function isDirectoryListing(contentType: string, extName: string): boolean {
return contentType.includes("text/html") && !isHtmlFile(extName);
}
Comment on lines +134 to +226
Copy link

Copilot AI Jan 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The newly extracted functions handleFileResponse, handleDirectoryResponse, determineFileExtension, isHtmlFile, isDirectoryListing, buildDirectoryHeaders, and buildFileHeaders lack test coverage. Consider adding unit tests for these functions to ensure they behave correctly in isolation, especially for edge cases like empty file extensions, different content types, and header handling.

Copilot uses AI. Check for mistakes.
84 changes: 22 additions & 62 deletions apps/api/src/routes/v1_files/$wildcard.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
/* eslint-disable no-console */
import type { OpenAPIHono } from "@hono/zod-openapi";
import type { Entry } from "apache-autoindex-parse";
import type { HonoEnv } from "../../types";
import { createRoute, z } from "@hono/zod-openapi";
import { dedent } from "@luxass/utils";
Expand All @@ -15,9 +14,16 @@ import {
} from "@ucdjs/env";
import { FileEntryListSchema } from "@ucdjs/schemas";
import { cache } from "hono/cache";
import { HTML_EXTENSIONS, MAX_AGE_ONE_WEEK_SECONDS } from "../../constants";
import { MAX_AGE_ONE_WEEK_SECONDS } from "../../constants";
import { badGateway, badRequest, notFound } from "../../lib/errors";
import { applyDirectoryFiltersAndSort, parseUnicodeDirectory } from "../../lib/files";
import {
applyDirectoryFiltersAndSort,
determineFileExtension,
handleDirectoryResponse,
handleFileResponse,
isDirectoryListing,
parseUnicodeDirectory,
} from "../../lib/files";
import { generateReferences, OPENAPI_TAGS } from "../../openapi";
import {
ORDER_QUERY_PARAM,
Expand Down Expand Up @@ -267,36 +273,6 @@ export const METADATA_WILDCARD_ROUTE = createRoute({
},
});

function buildDirectoryHeaders(files: Entry[], baseHeaders: Record<string, string>): Record<string, string> {
return {
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "directory",
[UCD_STAT_CHILDREN_HEADER]: `${files.length}`,
[UCD_STAT_CHILDREN_FILES_HEADER]: `${files.filter((f) => f.type === "file").length}`,
[UCD_STAT_CHILDREN_DIRS_HEADER]: `${files.filter((f) => f.type === "directory").length}`,
};
}

function buildFileHeaders(
contentType: string,
baseHeaders: Record<string, string>,
response: Response,
actualContentLength: number,
): Record<string, string> {
const headers: Record<string, string> = {
"Content-Type": contentType,
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "file",
[UCD_STAT_SIZE_HEADER]: `${actualContentLength}`,
"Content-Length": `${actualContentLength}`,
};

const cd = response.headers.get("Content-Disposition");
if (cd) headers["Content-Disposition"] = cd;

return headers;
}

export function registerWildcardRoute(router: OpenAPIHono<HonoEnv>) {
router.openAPIRegistry.registerPath(WILDCARD_ROUTE);
router.openAPIRegistry.registerPath(METADATA_WILDCARD_ROUTE);
Expand Down Expand Up @@ -347,14 +323,11 @@ export function registerWildcardRoute(router: OpenAPIHono<HonoEnv>) {
if (lastModified) baseHeaders["Last-Modified"] = lastModified;

const leaf = normalizedPath.split("/").pop() ?? "";
const extName = leaf.includes(".") ? leaf.split(".").pop()!.toLowerCase() : "";
const isHtmlFile = HTML_EXTENSIONS.includes(`.${extName}`);

// check if this is a directory listing (HTML response for non-HTML files)
const isDirectoryListing = contentType.includes("text/html") && !isHtmlFile;
const extName = determineFileExtension(leaf);
const isDir = isDirectoryListing(contentType, extName);

console.info(`[v1_files]: fetched content type: ${contentType} for .${extName} file`);
if (isDirectoryListing) {
if (isDir) {
const html = await response.text();
const parsedFiles = await parseUnicodeDirectory(html, normalizedPath || "/");

Expand Down Expand Up @@ -382,8 +355,10 @@ export function registerWildcardRoute(router: OpenAPIHono<HonoEnv>) {
order: c.req.query("order"),
});

const headers = buildDirectoryHeaders(files, baseHeaders);
return c.json(files, 200, headers);
return handleDirectoryResponse(c, {
files,
baseHeaders,
});
}

// Handle file response
Expand All @@ -393,27 +368,12 @@ export function registerWildcardRoute(router: OpenAPIHono<HonoEnv>) {

const isHeadRequest = c.req.method === "HEAD";

// For HEAD requests, buffer to calculate accurate size
if (isHeadRequest) {
const blob = await response.blob();
const actualSize = blob.size;
const headers = buildFileHeaders(contentType, baseHeaders, response, actualSize);
console.log(`[v1_files]: HEAD request, calculated size: ${actualSize}`);
return c.newResponse(null, 200, headers);
}

const headers: Record<string, string> = {
"Content-Type": contentType,
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "file",
};

const cd = response.headers.get("Content-Disposition");
if (cd) headers["Content-Disposition"] = cd;

console.log(`[v1_files]: binary file, streaming without buffering`);

return c.newResponse(response.body, 200, headers);
return await handleFileResponse(c, {
contentType,
baseHeaders,
response,
isHeadRequest,
});
},
);
}
55 changes: 20 additions & 35 deletions apps/api/src/ucd-store/routes/files.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,19 @@
/* eslint-disable no-console */
import type { OpenAPIHono } from "@hono/zod-openapi";
import type { HonoEnv } from "../../types";
import {
DEFAULT_USER_AGENT,
UCD_STAT_CHILDREN_DIRS_HEADER,
UCD_STAT_CHILDREN_FILES_HEADER,
UCD_STAT_CHILDREN_HEADER,
UCD_STAT_SIZE_HEADER,
UCD_STAT_TYPE_HEADER,
} from "@ucdjs/env";
import { cache } from "hono/cache";
import { HTML_EXTENSIONS, MAX_AGE_ONE_WEEK_SECONDS } from "../../constants";
import { MAX_AGE_ONE_WEEK_SECONDS } from "../../constants";
import { badGateway, badRequest, notFound } from "../../lib/errors";
import { applyDirectoryFiltersAndSort, parseUnicodeDirectory } from "../../lib/files";
import {
applyDirectoryFiltersAndSort,
determineFileExtension,
handleDirectoryResponse,
handleFileResponse,
isDirectoryListing,
parseUnicodeDirectory,
} from "../../lib/files";
import { determineContentTypeFromExtension, isInvalidPath } from "../../routes/v1_files/utils";
import { stripUCDPrefix, transformPathForUnicodeOrg } from "../lib/path-utils";

Expand All @@ -38,8 +39,6 @@ export function registerFilesRoute(router: OpenAPIHono<HonoEnv>) {
const unicodeOrgPath = transformPathForUnicodeOrg(version, filepath);
const url = `https://unicode.org/Public/${unicodeOrgPath}?F=2`;

console.info(`[ucd-store]: fetching ${url}`);

const response = await fetch(url, {
method: "GET",
headers: { "User-Agent": DEFAULT_USER_AGENT },
Expand All @@ -58,11 +57,10 @@ export function registerFilesRoute(router: OpenAPIHono<HonoEnv>) {
if (lastModified) baseHeaders["Last-Modified"] = lastModified;

const leaf = filepath.split("/").pop() ?? "";
const extName = leaf.includes(".") ? leaf.split(".").pop()!.toLowerCase() : "";
const isHtmlFile = HTML_EXTENSIONS.includes(`.${extName}`);
const isDirectoryListing = contentType.includes("text/html") && !isHtmlFile;
const extName = determineFileExtension(leaf);
const isDir = isDirectoryListing(contentType, extName);

if (isDirectoryListing) {
if (isDir) {
const html = await response.text();
const parsedFiles = await parseUnicodeDirectory(html, `/${version}/${filepath}`);

Expand All @@ -81,34 +79,21 @@ export function registerFilesRoute(router: OpenAPIHono<HonoEnv>) {
order: c.req.query("order"),
});

return c.json(files, 200, {
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "directory",
[UCD_STAT_CHILDREN_HEADER]: `${files.length}`,
[UCD_STAT_CHILDREN_FILES_HEADER]: `${files.filter((f) => f.type === "file").length}`,
[UCD_STAT_CHILDREN_DIRS_HEADER]: `${files.filter((f) => f.type === "directory").length}`,
return handleDirectoryResponse(c, {
files,
baseHeaders,
});
}

// Handle file response
contentType ||= determineContentTypeFromExtension(extName);

const isHeadRequest = c.req.method === "HEAD";
if (isHeadRequest) {
const blob = await response.blob();
return c.newResponse(null, 200, {
"Content-Type": contentType,
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "file",
[UCD_STAT_SIZE_HEADER]: `${blob.size}`,
"Content-Length": `${blob.size}`,
});
}

return c.newResponse(response.body, 200, {
"Content-Type": contentType,
...baseHeaders,
[UCD_STAT_TYPE_HEADER]: "file",
return await handleFileResponse(c, {
contentType,
baseHeaders,
response,
isHeadRequest,
});
},
);
Expand Down
Loading