Deploy: worker improvements, markdown serving, and tests

.github/workflows/deploy-docs.yml

@@ -86,6 +86,13 @@ jobs:
             echo "changed=false" >> "$GITHUB_OUTPUT"
           fi
 
+      - name: Test Worker
+        if: steps.worker.outputs.changed == 'true'
+        working-directory: worker
+        run: |
+          yarn install --frozen-lockfile
+          yarn test
+
       - name: Deploy Worker
         if: steps.worker.outputs.changed == 'true'
         uses: cloudflare/wrangler-action@v3

.gitignore

@@ -26,4 +26,5 @@ yarn-error.log*
 .env
 
 static/robots.txt
+worker/node_modules
 

docusaurus.config.js

@@ -93,8 +93,8 @@ const config = {
         sidebarPath: require.resolve("./sidebars.js"),
 
         editUrl: ({ docPath }) => {
-          const branch = process.env.PROD == "true" ? "main" : "dev";
-          return `https://raw.githubusercontent.com/MarketDataApp/documentation/${branch}/api/${docPath}`;
+          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
+          return `https://www.marketdata.app${base}/api/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
       },
     ],
@@ -106,8 +106,8 @@ const config = {
         path: "sdk",
         routeBasePath: "sdk",
         editUrl: ({ docPath }) => {
-          const branch = process.env.PROD == "true" ? "main" : "dev";
-          return `https://raw.githubusercontent.com/MarketDataApp/documentation/${branch}/sdk/${docPath}`;
+          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
+          return `https://www.marketdata.app${base}/sdk/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
         sidebarPath: require.resolve("./sidebars.js"),
       },
@@ -120,8 +120,8 @@ const config = {
         path: "sheets",
         routeBasePath: "sheets",
         editUrl: ({ docPath }) => {
-          const branch = process.env.PROD == "true" ? "main" : "dev";
-          return `https://raw.githubusercontent.com/MarketDataApp/documentation/${branch}/sheets/${docPath}`;
+          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
+          return `https://www.marketdata.app${base}/sheets/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
         sidebarPath: require.resolve("./sidebars.js"),
       },
@@ -134,8 +134,8 @@ const config = {
         path: "account",
         routeBasePath: "account",
         editUrl: ({ docPath }) => {
-          const branch = process.env.PROD == "true" ? "main" : "dev";
-          return `https://raw.githubusercontent.com/MarketDataApp/documentation/${branch}/account/${docPath}`;
+          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
+          return `https://www.marketdata.app${base}/account/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
         sidebarPath: require.resolve("./sidebars.js"),
       },

sdk/go/markets/index.mdx

@@ -1,6 +1,6 @@
 ---
 title: Markets
-slug: /markets
+slug: /go/markets
 sidebar_position: 9
 ---
 

sdk/go/options/index.mdx

@@ -1,6 +1,6 @@
 ---
 title: Options
-slug: /options
+slug: /go/options
 sidebar_position: 11
 ---
 

sdk/go/stocks/index.mdx

@@ -1,6 +1,6 @@
 ---
 title: Stocks
-slug: /stocks
+slug: /go/stocks
 sidebar_position: 10
 ---
 

worker/handler.integration.test.js

@@ -0,0 +1,78 @@
+/**
+ * Integration test: fetches the live sitemap and verifies every doc URL
+ * returns valid markdown via Accept: text/markdown header.
+ *
+ * Run with: yarn test:integration
+ * Requires network access to www.marketdata.app
+ */
+import { describe, it, expect } from 'vitest';
+
+const SITEMAP_URL = 'https://www.marketdata.app/docs/sitemap.xml';
+
+// Pages generated by Docusaurus that have no markdown source
+const SKIP_PATTERNS = [
+  '/docs/search',
+  '/docs/docs/',        // root landing page (auto-generated)
+  '/tags',              // tag listing pages
+];
+
+// Pages with no markdown source — auto-generated or mapped to unexpected paths
+// See: https://github.com/MarketDataApp/documentation/issues/127
+const SKIP_EXACT = [
+  '/docs/',             // root page source is at docs/index.md, outside doc plugin paths
+  '/docs/sdk/stocks',   // broken category URLs (should be /sdk/go/stocks, etc.)
+  '/docs/sdk/markets',
+  '/docs/sdk/options',
+];
+
+function shouldSkip(url) {
+  const path = new URL(url).pathname;
+  if (SKIP_EXACT.includes(path)) return true;
+  return SKIP_PATTERNS.some((pattern) => url.includes(pattern));
+}
+
+async function fetchSitemapUrls() {
+  const res = await fetch(SITEMAP_URL);
+  const xml = await res.text();
+  const urls = [];
+  for (const match of xml.matchAll(/<loc>([^<]+)<\/loc>/g)) {
+    urls.push(match[1]);
+  }
+  return urls;
+}
+
+describe('markdown serving (live sitemap)', async () => {
+  const allUrls = await fetchSitemapUrls();
+  const docUrls = allUrls.filter((url) => !shouldSkip(url));
+
+  it(`sitemap has URLs to test`, () => {
+    expect(docUrls.length).toBeGreaterThan(50);
+  });
+
+  for (const url of docUrls) {
+    const path = new URL(url).pathname;
+
+    it(`${path} returns markdown`, async () => {
+      const res = await fetch(url, {
+        headers: { Accept: 'text/markdown' },
+        redirect: 'follow',
+      });
+
+      expect(res.status, `${path} returned ${res.status}`).toBe(200);
+      expect(res.headers.get('content-type')).toContain('text/markdown');
+
+      const text = await res.text();
+      expect(text.length, `${path} returned empty body`).toBeGreaterThan(0);
+
+      // Should not contain raw frontmatter
+      expect(text.startsWith('---\n'), `${path} still has frontmatter`).toBe(false);
+
+      // Should not contain MDX import statements
+      expect(text, `${path} still has import statements`).not.toMatch(/^import\s+\w+\s+from\s+"/m);
+
+      // Should not contain raw JSX tags from Docusaurus components
+      expect(text, `${path} still has <Tabs>`).not.toContain('<Tabs>');
+      expect(text, `${path} still has <TabItem`).not.toMatch(/<TabItem\s/);
+    });
+  }
+});

worker/handler.js

@@ -0,0 +1,150 @@
+/**
+ * Cloudflare Worker to proxy /docs/ and /docs-staging/ paths
+ * from www.marketdata.app to Cloudflare Pages deployments.
+ *
+ * Last deployed: 2026-02-24
+ *
+ * Routing:
+ * - www.marketdata.app/docs/*         → marketdata-docs.pages.dev/docs/*
+ * - www.marketdata.app/docs-staging/* → marketdata-docs-staging.pages.dev/docs-staging/*
+ *
+ * The path is preserved as-is (including the /docs/ or /docs-staging/ prefix).
+ * Cloudflare Pages serves files from a matching directory structure in the
+ * build output, so no path rewriting is needed.
+ *
+ * Requests that don't match any route (e.g. the main WordPress site)
+ * are passed through unchanged.
+ */
+
+const ORIGINAL_HOSTNAME = 'www.marketdata.app';
+
+/**
+ * Route definitions mapping URL path prefixes to Cloudflare Pages targets.
+ * Order matters: /docs-staging/ must come before /docs/ to avoid
+ * /docs-staging/* being matched by the /docs/ prefix.
+ */
+const ROUTES = [
+  {
+    prefix: '/docs-staging/',
+    target: 'marketdata-docs-staging.pages.dev',
+  },
+  {
+    prefix: '/docs/',
+    target: 'marketdata-docs.pages.dev',
+  },
+];
+
+/**
+ * Checks whether a URL path matches a route, handling both
+ * trailing-slash (/docs-staging/) and no-trailing-slash (/docs-staging) forms.
+ *
+ * @param {string} pathname - The URL path to check (e.g. "/docs-staging/api")
+ * @param {string} prefix - The route prefix to match (e.g. "/docs-staging/")
+ * @returns {boolean}
+ */
+function matchesRoute(pathname, prefix) {
+  return pathname.startsWith(prefix) || pathname === prefix.slice(0, -1);
+}
+
+/**
+ * Converts raw MDX/markdown source into clean markdown by stripping
+ * frontmatter, imports, and converting Docusaurus components to headings.
+ *
+ * @param {string} text - Raw file contents
+ * @returns {string}
+ */
+function cleanMarkdown(text) {
+  // Strip frontmatter
+  text = text.replace(/^---\n[\s\S]*?\n---\n*/, '');
+  // Strip import statements
+  text = text.replace(/^import\s+.*;\s*\n/gm, '');
+  // Convert <TabItem> to ### headings, strip <Tabs> wrappers
+  text = text.replace(/<Tabs>\n?/g, '');
+  text = text.replace(/<\/Tabs>\n?/g, '');
+  text = text.replace(/<TabItem\s+(?:[^>"]*(?:"[^"]*")?)*?label="([^"]*)"(?:[^>"]*(?:"[^"]*")?)*?>\n?/g, '### $1\n\n');
+  text = text.replace(/<\/TabItem>\n?/g, '');
+  // Clean up excess blank lines
+  text = text.replace(/\n{3,}/g, '\n\n').trim() + '\n';
+  return text;
+}
+
+/**
+ * Routes incoming requests to the appropriate Cloudflare Pages deployment
+ * based on the URL path. Non-matching requests pass through to the
+ * default origin (WordPress on cPanel).
+ *
+ * @param {Request} request - The incoming request
+ * @returns {Promise<Response>}
+ */
+async function handleRequest(request) {
+  const url = new URL(request.url);
+
+  if (url.hostname !== ORIGINAL_HOSTNAME) {
+    return fetch(request);
+  }
+
+  // Redirect legacy SDK PHP docs to GitHub Pages
+  const sdkPhpPrefix = '/docs/sdk-php/';
+  if (url.pathname.startsWith(sdkPhpPrefix) || url.pathname === '/docs/sdk-php') {
+    let subpath = url.pathname.slice(sdkPhpPrefix.length);
+    // Fix doubled directory names (e.g. classes/classes/ → classes/)
+    subpath = subpath.replace(/^(\w+)\/\1\//, '$1/');
+    return Response.redirect(`https://marketdataapp.github.io/sdk-php/${subpath}`, 301);
+  }
+
+  // Serve raw markdown for .md URLs or Accept: text/markdown header
+  const docsPrefix = url.pathname.startsWith('/docs-staging/') ? '/docs-staging/'
+    : url.pathname.startsWith('/docs/') ? '/docs/' : null;
+
+  if (docsPrefix) {
+    const wantsMd = url.pathname.endsWith('.md');
+    const acceptsMd = (request.headers.get('accept') || '').includes('text/markdown');
+
+    if (wantsMd || acceptsMd) {
+      const stem = wantsMd
+        ? url.pathname.slice(docsPrefix.length, -3)
+        : url.pathname.replace(/\/$/, '').slice(docsPrefix.length);
+      const branch = docsPrefix === '/docs-staging/' ? 'staging' : 'main';
+      const base = `https://raw.githubusercontent.com/MarketDataApp/documentation/${branch}`;
+      const candidates = [
+        `${base}/${stem}.md`,
+        `${base}/${stem}.mdx`,
+        `${base}/${stem}/index.md`,
+        `${base}/${stem}/index.mdx`,
+      ];
+      for (const candidate of candidates) {
+        const res = await fetch(candidate);
+        if (res.ok) {
+          const text = cleanMarkdown(await res.text());
+          return new Response(text, {
+            headers: { 'content-type': 'text/markdown; charset=utf-8' },
+          });
+        }
+      }
+    }
+  }
+
+  for (const route of ROUTES) {
+    if (matchesRoute(url.pathname, route.prefix)) {
+      // Docs sites don't serve robots.txt; block stale cached copies
+      if (url.pathname.endsWith('/robots.txt')) {
+        return new Response('', { status: 404 });
+      }
+
+      url.hostname = route.target;
+      const response = await fetch(new Request(url, request), { cf: { cacheEverything: true } });
+
+      if (response.status === 404) {
+        const pathname = new URL(request.url).pathname;
+        const referer = request.headers.get('referer');
+        console.log({ level: '404', message: pathname, referer: referer || '' });
+      }
+
+      return response;
+    }
+  }
+
+  return fetch(request);
+}
+
+module.exports = { handleRequest, matchesRoute, cleanMarkdown, ROUTES, ORIGINAL_HOSTNAME };