feat(llms-txt): add llms.txt generation for npm packages

nuxt.config.ts

@@ -112,6 +112,7 @@ export default defineNuxtConfig({
     '/api/registry/file/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } },
     '/api/registry/provenance/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } },
     '/api/registry/files/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } },
+    '/package/**/llms.txt': { isr: 3600 },
     '/api/registry/package-meta/**': { isr: 300 },
     '/:pkg/.well-known/skills/**': { isr: 3600 },
     '/:scope/:pkg/.well-known/skills/**': { isr: 3600 },

server/routes/package/[name]/llms.txt.get.ts

@@ -0,0 +1,54 @@
+import * as v from 'valibot'
+import { PackageRouteParamsSchema } from '#shared/schemas/package'
+import { CACHE_MAX_AGE_ONE_HOUR } from '#shared/utils/constants'
+import { handleApiError } from '#server/utils/error-handler'
+import { handleLlmsTxt } from '#server/utils/llms-txt'
+
+/**
+ * Serves llms.txt for an npm package.
+ *
+ * Handles all URL shapes via re-exports:
+ * - /package/:name/llms.txt
+ * - /package/:org/:name/llms.txt
+ * - /package/:name/v/:version/llms.txt
+ * - /package/:org/:name/v/:version/llms.txt
+ */
+export default defineCachedEventHandler(
+  async event => {
+    const org = getRouterParam(event, 'org')
+    const name = getRouterParam(event, 'name')
+    const rawVersion = getRouterParam(event, 'version')
+    if (!name) {
+      throw createError({ statusCode: 404, message: 'Package name is required.' })
+    }
+
+    const rawPackageName = org ? `${org}/${name}` : name
+
+    try {
+      const { packageName, version } = v.parse(PackageRouteParamsSchema, {
+        packageName: rawPackageName,
+        version: rawVersion,
+      })
+
+      const content = await handleLlmsTxt(packageName, version)
+      setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
+      return content
+    } catch (error: unknown) {
+      handleApiError(error, {
+        statusCode: 502,
+        message: 'Failed to generate llms.txt.',
+      })
+    }
+  },
+  {
+    maxAge: CACHE_MAX_AGE_ONE_HOUR,
+    swr: true,
+    getKey: event => {
+      const org = getRouterParam(event, 'org')
+      const name = getRouterParam(event, 'name')
+      const version = getRouterParam(event, 'version')
+      const pkg = org ? `${org}/${name}` : name
+      return version ? `llms-txt:${pkg}@${version}` : `llms-txt:${pkg}`
+    },
+  },
+)

server/routes/package/[name]/v/[version]/llms.txt.get.ts

@@ -0,0 +1 @@
+export { default } from '../../llms.txt.get'

server/routes/package/[org]/[name]/llms.txt.get.ts

@@ -0,0 +1 @@
+export { default } from '../../[name]/llms.txt.get'

server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts

@@ -0,0 +1 @@
+export { default } from '../../../../[name]/llms.txt.get'

server/utils/llms-txt.ts

@@ -0,0 +1,246 @@
+import type { Packument } from '@npm/types'
+import type { JsDelivrFileNode, AgentFile, LlmsTxtResult } from '#shared/types'
+import { NPM_MISSING_README_SENTINEL } from '#shared/utils/constants'
+
+/** Well-known agent instruction files at the package root */
+const ROOT_AGENT_FILES: Record<string, string> = {
+  'CLAUDE.md': 'Claude Code',
+  'AGENTS.md': 'Agent Instructions',
+  'AGENT.md': 'Agent Instructions',
+  '.cursorrules': 'Cursor Rules',
+  '.windsurfrules': 'Windsurf Rules',
+  '.clinerules': 'Cline Rules',
+}
+
+/** Well-known agent files inside specific directories */
+const DIRECTORY_AGENT_FILES: Record<string, string> = {
+  '.github/copilot-instructions.md': 'GitHub Copilot',
+}
+
+/** Directories containing rule files (match *.md inside) */
+const RULE_DIRECTORIES: Record<string, string> = {
+  '.cursor/rules': 'Cursor Rules',
+  '.windsurf/rules': 'Windsurf Rules',
+}
+
+/**
+ * Discover agent instruction file paths from a jsDelivr file tree.
+ * Scans root-level files, known subdirectory files, and rule directories.
+ */
+export function discoverAgentFiles(files: JsDelivrFileNode[]): string[] {
+  const discovered: string[] = []
+
+  for (const node of files) {
+    // Root-level well-known files
+    if (node.type === 'file' && node.name in ROOT_AGENT_FILES) {
+      discovered.push(node.name)
+    }
+
+    // Directory-based files
+    if (node.type === 'directory') {
+      // .github/copilot-instructions.md
+      if (node.name === '.github' && node.files) {
+        for (const child of node.files) {
+          const fullPath = `.github/${child.name}`
+          if (child.type === 'file' && fullPath in DIRECTORY_AGENT_FILES) {
+            discovered.push(fullPath)
+          }
+        }
+      }
+
+      // .cursor/rules/*.md and .windsurf/rules/*.md
+      for (const dirPath of Object.keys(RULE_DIRECTORIES)) {
+        const [topDir, subDir] = dirPath.split('/')
+        if (node.name === topDir && node.files) {
+          const rulesDir = node.files.find(f => f.type === 'directory' && f.name === subDir)
+          if (rulesDir?.files) {
+            for (const ruleFile of rulesDir.files) {
+              if (ruleFile.type === 'file' && ruleFile.name.endsWith('.md')) {
+                discovered.push(`${dirPath}/${ruleFile.name}`)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return discovered
+}
+
+/**
+ * Get the display name for an agent file path.
+ */
+function getDisplayName(filePath: string): string {
+  if (filePath in ROOT_AGENT_FILES) return ROOT_AGENT_FILES[filePath]
+  if (filePath in DIRECTORY_AGENT_FILES) return DIRECTORY_AGENT_FILES[filePath]
+
+  for (const [dirPath, displayName] of Object.entries(RULE_DIRECTORIES)) {
+    if (filePath.startsWith(`${dirPath}/`)) return `${displayName}: ${filePath.split('/').pop()}`
+  }
+
+  return filePath
+}
+
+/**
+ * Fetch agent instruction files from jsDelivr CDN.
+ * Fetches in parallel, gracefully skipping failures.
+ */
+export async function fetchAgentFiles(
+  packageName: string,
+  version: string,
+  filePaths: string[],
+): Promise<AgentFile[]> {
+  const results = await Promise.all(
+    filePaths.map(async (path): Promise<AgentFile | null> => {
+      try {
+        const url = `https://cdn.jsdelivr.net/npm/${packageName}@${version}/${path}`
+        const response = await fetch(url)
+        if (!response.ok) return null
+        const content = await response.text()
+        return { path, content, displayName: getDisplayName(path) }
+      } catch {
+        return null
+      }
+    }),
+  )
+
+  return results.filter((r): r is AgentFile => r !== null)
+}
+
+/**
+ * Generate llms.txt markdown content per the llmstxt.org spec.
+ *
+ * Structure:
+ * - H1 title with package name and version
+ * - Blockquote description (if available)
+ * - Metadata list (homepage, repository, npm)
+ * - README section
+ * - Agent Instructions section (one sub-heading per file)
+ */
+export function generateLlmsTxt(result: LlmsTxtResult): string {
+  const lines: string[] = []
+
+  // Title
+  lines.push(`# ${result.packageName}@${result.version}`)
+  lines.push('')
+
+  // Description blockquote
+  if (result.description) {
+    lines.push(`> ${result.description}`)
+    lines.push('')
+  }
+
+  // Metadata
+  const meta: string[] = []
+  if (result.homepage) meta.push(`- Homepage: ${result.homepage}`)
+  if (result.repositoryUrl) meta.push(`- Repository: ${result.repositoryUrl}`)
+  meta.push(`- npm: https://www.npmjs.com/package/${result.packageName}/v/${result.version}`)
+  lines.push(...meta)
+  lines.push('')
+
+  // README
+  if (result.readme) {
+    lines.push('## README')
+    lines.push('')
+    lines.push(result.readme)
+    lines.push('')
+  }
+
+  // Agent instructions
+  if (result.agentFiles.length > 0) {
+    lines.push('## Agent Instructions')
+    lines.push('')
+
+    for (const file of result.agentFiles) {
+      lines.push(`### ${file.displayName} (\`${file.path}\`)`)
+      lines.push('')
+      lines.push(file.content)
+      lines.push('')
+    }
+  }
+
+  return lines.join('\n').trimEnd() + '\n'
+}
+
+/** Standard README filenames to try from jsDelivr CDN */
+const README_FILENAMES = ['README.md', 'readme.md', 'Readme.md']
+
+/** Fetch README from jsDelivr CDN as fallback */
+async function fetchReadmeFromCdn(packageName: string, version: string): Promise<string | null> {
+  for (const filename of README_FILENAMES) {
+    try {
+      const url = `https://cdn.jsdelivr.net/npm/${packageName}@${version}/${filename}`
+      const response = await fetch(url)
+      if (response.ok) return await response.text()
+    } catch {
+      // Try next
+    }
+  }
+  return null
+}
+
+/** Extract README from packument data */
+function getReadmeFromPackument(packageData: Packument, requestedVersion?: string): string | null {
+  const readme = requestedVersion
+    ? packageData.versions[requestedVersion]?.readme
+    : packageData.readme
+
+  if (readme && readme !== NPM_MISSING_README_SENTINEL) {
+    return readme
+  }
+  return null
+}
+
+/** Extract a clean repository URL from packument repository field */
+function parseRepoUrl(
+  repository?: { type?: string; url?: string; directory?: string } | string,
+): string | undefined {
+  if (!repository) return undefined
+  const url = typeof repository === 'string' ? repository : repository.url
+  if (!url) return undefined
+  return url.replace(/^git\+/, '').replace(/\.git$/, '')
+}
+
+/**
+ * Orchestrates fetching all data and generating llms.txt for a package.
+ * Shared by both versioned and unversioned route handlers.
+ */
+export async function handleLlmsTxt(
+  packageName: string,
+  requestedVersion?: string,
+): Promise<string> {
+  const packageData = await fetchNpmPackage(packageName)
+  const resolvedVersion = requestedVersion ?? packageData['dist-tags']?.latest
+
+  if (!resolvedVersion) {
+    throw createError({ statusCode: 404, message: 'Could not resolve package version.' })
+  }
+
+  // Extract README from packument (sync)
+  const readmeFromPackument = getReadmeFromPackument(packageData, requestedVersion)
+
+  // Fetch file tree (and README from CDN if packument didn't have one)
+  const [fileTreeData, cdnReadme] = await Promise.all([
+    fetchFileTree(packageName, resolvedVersion),
+    readmeFromPackument ? null : fetchReadmeFromCdn(packageName, resolvedVersion),
+  ])
+
+  const readme = readmeFromPackument ?? cdnReadme ?? undefined
+
+  // Discover and fetch agent files
+  const agentFilePaths = discoverAgentFiles(fileTreeData.files)
+  const agentFiles = await fetchAgentFiles(packageName, resolvedVersion, agentFilePaths)
+
+  const result: LlmsTxtResult = {
+    packageName,
+    version: resolvedVersion,
+    description: packageData.description,
+    homepage: packageData.homepage,
+    repositoryUrl: parseRepoUrl(packageData.repository),
+    readme,
+    agentFiles,
+  }
+
+  return generateLlmsTxt(result)
+}

shared/types/index.ts

@@ -9,3 +9,4 @@ export * from './i18n-status'
 export * from './comparison'
 export * from './skills'
 export * from './version-downloads'
+export * from './llms-txt'

shared/types/llms-txt.ts

@@ -0,0 +1,31 @@
+/**
+ * Agent instruction file discovered in a package
+ */
+export interface AgentFile {
+  /** Relative path within the package (e.g., "CLAUDE.md", ".github/copilot-instructions.md") */
+  path: string
+  /** File content */
+  content: string
+  /** Human-readable display name (e.g., "Claude Code", "GitHub Copilot") */
+  displayName: string
+}
+
+/**
+ * Result of gathering all data needed to generate llms.txt
+ */
+export interface LlmsTxtResult {
+  /** Package name (e.g., "nuxt" or "@nuxt/kit") */
+  packageName: string
+  /** Resolved version (e.g., "3.12.0") */
+  version: string
+  /** Package description from packument */
+  description?: string
+  /** Homepage URL */
+  homepage?: string
+  /** Repository URL */
+  repositoryUrl?: string
+  /** README content (raw markdown) */
+  readme?: string
+  /** Discovered agent instruction files */
+  agentFiles: AgentFile[]
+}