Migrate to unified CF Pages deployment via orchestrator

.github/workflows/deploy-docs.yml

@@ -1,4 +1,4 @@
-name: Deploy Docs to Cloudflare Pages
+name: Deploy Docs
 
 on:
   push:
@@ -9,7 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-      deployments: write
     steps:
       - uses: actions/checkout@v4
         with:
@@ -27,15 +26,12 @@ jobs:
         env:
           GIT_REF: ${{ github.ref }}
         run: |
+          echo "base_url=docs" >> "$GITHUB_OUTPUT"
           if [ "$GIT_REF" = "refs/heads/main" ]; then
             echo "prod=true" >> "$GITHUB_OUTPUT"
-            echo "base_url=docs" >> "$GITHUB_OUTPUT"
-            echo "project=marketdata-docs" >> "$GITHUB_OUTPUT"
             echo "environment=production" >> "$GITHUB_OUTPUT"
           else
             echo "prod=" >> "$GITHUB_OUTPUT"
-            echo "base_url=docs-staging" >> "$GITHUB_OUTPUT"
-            echo "project=marketdata-docs-staging" >> "$GITHUB_OUTPUT"
             echo "environment=staging" >> "$GITHUB_OUTPUT"
           fi
 
@@ -75,34 +71,25 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           AWS_ENDPOINT_URL: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
           AWS_REGION: auto
+          ENVIRONMENT: ${{ steps.env.outputs.environment }}
         run: |
           aws s3 sync build/ \
-            "s3://www-marketdata-app-builds/${{ steps.env.outputs.environment }}/" \
+            "s3://www-marketdata-app-builds/${ENVIRONMENT}/sources/docs/" \
             --delete
 
-      # - name: Notify orchestrator
-      #   if: success()
-      #   uses: peter-evans/repository-dispatch@v3
-      #   with:
-      #     token: ${{ secrets.ORCHESTRATOR_PAT }}
-      #     repository: MarketDataApp/www-marketdata-app
-      #     event-type: site-built
-      #     client-payload: >-
-      #       {
-      #         "source": "docs",
-      #         "environment": "${{ steps.env.outputs.environment }}",
-      #         "commit_sha": "${{ github.sha }}"
-      #       }
-
-      - name: Deploy to Cloudflare Pages
-        uses: cloudflare/wrangler-action@v3
+      - name: Notify orchestrator
+        if: success()
+        uses: peter-evans/repository-dispatch@v3
         with:
-          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
-          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
-          command: >-
-            pages deploy build
-            --project-name=${{ steps.env.outputs.project }}
-            --commit-dirty=true
+          token: ${{ secrets.ORCHESTRATOR_PAT }}
+          repository: MarketDataApp/www-marketdata-app
+          event-type: site-built
+          client-payload: >-
+            {
+              "source": "docs",
+              "environment": "${{ steps.env.outputs.environment }}",
+              "commit_sha": "${{ github.sha }}"
+            }
 
       - name: Check if Worker changed
         id: worker
@@ -128,34 +115,3 @@ jobs:
           accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           command: deploy
           workingDirectory: worker
-
-  post-deploy-tests:
-    needs: deploy
-    if: github.ref == 'refs/heads/main'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-          cache: yarn
-
-      - run: yarn install --frozen-lockfile
-
-      - name: Install worker test dependencies
-        working-directory: worker
-        run: yarn install --frozen-lockfile
-
-      - name: Run integration tests against production
-        working-directory: worker
-        env:
-          TEST_ENV: production
-        run: yarn test:integration
-
-      - run: npx playwright install chromium
-
-      - name: Run e2e tests against production
-        env:
-          TEST_ENV: production
-        run: yarn test:e2e

.github/workflows/post-deploy-tests.yml

@@ -0,0 +1,37 @@
+name: Post-Deploy Tests
+
+on:
+  repository_dispatch:
+    types: [deploy-complete]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.client_payload.commit_sha }}
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: yarn
+
+      - run: yarn install --frozen-lockfile
+
+      - name: Install worker test dependencies
+        working-directory: worker
+        run: yarn install --frozen-lockfile
+
+      - name: Run integration tests
+        working-directory: worker
+        env:
+          TEST_ENV: ${{ github.event.client_payload.environment }}
+        run: yarn test:integration
+
+      - run: npx playwright install chromium
+
+      - name: Run e2e tests
+        env:
+          TEST_ENV: ${{ github.event.client_payload.environment }}
+        run: yarn test:e2e

CLAUDE.md

@@ -3,15 +3,66 @@
 ## Hosting & URLs
 
 - Docs site is hosted on **Cloudflare Pages** with a **Cloudflare Worker** reverse proxy
-- Worker (`worker/`) proxies `www.marketdata.app/docs/*` → `marketdata-docs.pages.dev`
-- Staging: `www.marketdata.app/docs-staging/*` → `marketdata-docs-staging.pages.dev`
-- CI/CD: GitHub Actions (`.github/workflows/deploy-docs.yml`) builds Docusaurus, restructures output to match URL paths, and deploys to Cloudflare Pages + Worker via Wrangler
-- Build output is restructured in CI to nest under `docs/` or `docs-staging/` so static files serve from correct paths without rewrite rules
-- Edge caching enabled on Worker subrequests; `_headers` file generated in CI for asset cache control
+- Both environments use the same `/docs/` base path — routing is by hostname, not path prefix
+
+| Environment | URL | Pages Project | Branch |
+|-------------|-----|---------------|--------|
+| Production | `www.marketdata.app/docs/` | `www-marketdata-app` | `main` |
+| Staging | `www-staging.marketdata.app/docs/` | `www-staging-marketdata-app` | `staging` |
+
+## Architecture
+
+### Request flow
+
+1. DNS resolves the hostname (both are proxied CNAMEs in Cloudflare)
+2. Cloudflare routes `/docs` and `/docs/*` to the Worker (via `wrangler.toml` route patterns)
+3. Worker (`worker/handler.js`) looks up the hostname in the `TARGETS` map to find the Pages target
+4. Worker rewrites the hostname and fetches from the Pages project (e.g. `www-marketdata-app.pages.dev/docs/api/stocks`)
+5. Pages serves the file from its `docs/` directory (built and nested there by CI)
+6. Worker returns the response to the client — path stays the same throughout
+
+### Worker features (`worker/`)
+
+- **Hostname-based routing**: `TARGETS` map in `handler.js` maps each hostname to its Cloudflare Pages deployment
+- **Markdown serving**: Requests with `.md` extension or `Accept: text/markdown` header fetch raw source from GitHub and return cleaned markdown (frontmatter stripped, JSX components converted)
+- **SDK PHP redirect**: `/docs/sdk-php/*` → `marketdataapp.github.io/sdk-php/*` (301)
+- **robots.txt blocking**: Returns 404 for `/docs/robots.txt` to prevent stale cached copies
+- **Edge caching**: Passes `cf.cacheEverything` on subrequests
+- **404 logging**: Logs pathname and referer for 404 responses
+- Non-docs paths pass through to the origin (WordPress)
+
+### CI/CD pipeline
+
+**Docs repo** (`.github/workflows/deploy-docs.yml`):
+
+1. Builds Docusaurus (`yarn build`)
+2. Restructures build output to nest under `build/docs/`
+3. Generates `_headers` file for asset cache control
+4. Uploads build to R2 (`www-marketdata-app-builds` bucket) at `{env}/sources/docs/`
+5. Triggers orchestrator via `repository_dispatch`
+6. If `worker/` files changed: runs worker tests, then deploys the worker
+
+**Orchestrator** (`MarketDataApp/www-marketdata-app`, `.github/workflows/deploy-site.yml`):
+
+1. Downloads all sources from R2 (`{env}/sources/`)
+2. Merges into unified `build/` directory
+3. Deploys to Cloudflare Pages (`www-marketdata-app` or `www-staging-marketdata-app`)
+4. Notifies source repo via `deploy-complete` dispatch
+
+**Post-deploy tests** (`.github/workflows/post-deploy-tests.yml`):
+
+1. Triggered by `deploy-complete` from the orchestrator
+2. Checks out the docs repo at the deployed commit SHA
+3. Runs integration tests and e2e tests against the deployed environment
+
+### DNS
+
+- `www-staging.marketdata.app` → CNAME to `www-staging-marketdata-app.pages.dev` (proxied)
+- `www.marketdata.app` → existing DNS (proxied)
 
 ## Workflow
 
-- Work on the **staging** branch, verify changes at `www.marketdata.app/docs-staging/`
+- Work on the **staging** branch, verify changes at `www-staging.marketdata.app/docs/`
 - Once verified, open a PR from `staging` → `main` and merge to deploy to production
 
 ## Package Manager
@@ -29,3 +80,10 @@
 - Badges (New, Premium, Beta, High Usage) are configured via `sidebar_custom_props: { badge: n/p/b/h }` in page frontmatter
 - Rendered by `src/theme/RenderTag.js`, styled in `src/css/custom.css`
 - Supported in sidebar links, sidebar categories, and page titles via swizzled theme components
+
+## Testing
+
+- **Unit tests**: `cd worker && yarn test` — tests worker routing, markdown serving, robots.txt, 404 logging
+- **Integration tests**: `cd worker && TEST_ENV=staging yarn test:integration` — fetches live sitemap and verifies markdown serving for every doc URL
+- **Redirect tests**: `cd worker && TEST_ENV=staging yarn test:integration` — verifies client-side redirects from `docusaurus.config.js`
+- **E2E tests**: `TEST_ENV=staging yarn test:e2e` — Playwright tests for Context7 widget rendering

README.md

@@ -3,7 +3,7 @@
 The official documentation for [Market Data](https://www.marketdata.app/) — covering the REST API, SDKs, and Google Sheets Add-On. Built with [Docusaurus 3](https://docusaurus.io/).
 
 **Production:** [www.marketdata.app/docs/](https://www.marketdata.app/docs/)
-**Staging:** [www.marketdata.app/docs-staging/](https://www.marketdata.app/docs-staging/)
+**Staging:** [www-staging.marketdata.app/docs/](https://www-staging.marketdata.app/docs/)
 
 ## Documentation Sections
 
@@ -22,15 +22,70 @@ yarn start    # Start dev server at localhost:3000
 yarn build    # Production build
 ```
 
+## Architecture
+
+The site is hosted on **Cloudflare Pages** with a **Cloudflare Worker** reverse proxy. Both production and staging use the same `/docs/` base path — routing is determined by hostname, not path prefix. Deployment is handled by a separate orchestrator repo (`MarketDataApp/www-marketdata-app`) that merges build artifacts from R2 and deploys to unified Pages projects.
+
+### Request flow
+
+```
+Browser → Cloudflare DNS → Worker (hostname lookup) → Cloudflare Pages → Response
+```
+
+1. DNS resolves the hostname (both are proxied CNAMEs in Cloudflare)
+2. Cloudflare routes `/docs` and `/docs/*` to the Worker (via `wrangler.toml` route patterns)
+3. Worker looks up the hostname in a `TARGETS` map to find the Pages deployment target
+4. Worker rewrites the hostname and fetches from Pages (e.g. `www-marketdata-app.pages.dev/docs/api/stocks`)
+5. Pages serves the file from its `docs/` directory (built and nested there by CI)
+6. Worker returns the response — the URL path stays the same throughout
+
+### Environments
+
+| Environment | Hostname | Pages Project | Git Branch |
+|-------------|----------|---------------|------------|
+| Production | `www.marketdata.app` | `www-marketdata-app` | `main` |
+| Staging | `www-staging.marketdata.app` | `www-staging-marketdata-app` | `staging` |
+
+### Worker features
+
+The Worker (`worker/handler.js`) handles more than just proxying:
+
+- **Markdown serving** — Requests with `.md` extension or `Accept: text/markdown` header return cleaned markdown fetched from the raw GitHub source (frontmatter and JSX stripped)
+- **SDK PHP redirect** — `/docs/sdk-php/*` redirects to GitHub Pages (301)
+- **Edge caching** — Subrequests use `cf.cacheEverything`
+- **404 logging** — Logs pathname and referer for missing pages
+- Non-docs paths pass through to the origin (WordPress)
+
 ## Deployment
 
-The site is hosted on **Cloudflare Pages** with a **Cloudflare Worker** reverse proxy. Deployment is fully automated via GitHub Actions.
+Deployment is fully automated via GitHub Actions across two repos:
+
+1. **This repo** (`.github/workflows/deploy-docs.yml`) — builds Docusaurus, uploads to R2, triggers orchestrator
+2. **Orchestrator** (`MarketDataApp/www-marketdata-app`) — downloads all sources from R2, merges into unified build, deploys to CF Pages, runs post-deploy tests
+
+```
+Push to staging/main → Build → Upload to R2 → Trigger orchestrator → Deploy to CF Pages → Tests
+```
 
-1. Push to `staging` — deploys to the staging site
-2. Verify changes at the staging URL
+Workflow:
+1. Push to `staging` — builds and deploys to staging
+2. Verify changes at `www-staging.marketdata.app/docs/`
 3. Open a PR from `staging` → `main` and merge — deploys to production
 
-The CI pipeline (`.github/workflows/deploy-docs.yml`) builds the Docusaurus site, restructures the output to match the `/docs/` and `/docs-staging/` URL paths, generates cache headers, and deploys via Wrangler.
+If files in `worker/` changed, the docs CI also runs worker tests and deploys the Worker.
+
+## Testing
+
+```bash
+# Worker unit tests
+cd worker && yarn test
+
+# Integration tests (markdown serving against live site)
+cd worker && TEST_ENV=staging yarn test:integration
+
+# E2E tests (Playwright — Context7 widget)
+TEST_ENV=staging yarn test:e2e
+```
 
 ## Project Structure
 
@@ -43,6 +98,7 @@ src/
   theme/          # Swizzled Docusaurus theme components
   css/            # Custom styles
 worker/           # Cloudflare Worker reverse proxy
+e2e/              # Playwright end-to-end tests
 .github/workflows # CI/CD pipeline
 ```
 

docusaurus.config.js

@@ -15,9 +15,9 @@ const config = {
   url:
     process.env.PROD == "true"
       ? "https://www.marketdata.app/"
-      : "https://www.marketdata.app/",
+      : "https://www-staging.marketdata.app/",
 
-  baseUrl: process.env.PROD == "true" ? "/docs/" : "/docs-staging/",
+  baseUrl: "/docs/",
   noIndex: process.env.PROD !== "true",
   onBrokenLinks: "ignore",
   onBrokenMarkdownLinks: "warn",
@@ -121,8 +121,8 @@ const config = {
         sidebarPath: require.resolve("./sidebars.js"),
 
         editUrl: ({ docPath }) => {
-          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
-          return `https://www.marketdata.app${base}/api/${docPath.replace(/\.mdx?$/, '.md')}`;
+          const host = process.env.PROD == "true" ? "www.marketdata.app" : "www-staging.marketdata.app";
+          return `https://${host}/docs/api/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
       },
     ],
@@ -134,8 +134,8 @@ const config = {
         path: "sdk",
         routeBasePath: "sdk",
         editUrl: ({ docPath }) => {
-          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
-          return `https://www.marketdata.app${base}/sdk/${docPath.replace(/\.mdx?$/, '.md')}`;
+          const host = process.env.PROD == "true" ? "www.marketdata.app" : "www-staging.marketdata.app";
+          return `https://${host}/docs/sdk/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
         sidebarPath: require.resolve("./sidebars.js"),
       },
@@ -148,8 +148,8 @@ const config = {
         path: "sheets",
         routeBasePath: "sheets",
         editUrl: ({ docPath }) => {
-          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
-          return `https://www.marketdata.app${base}/sheets/${docPath.replace(/\.mdx?$/, '.md')}`;
+          const host = process.env.PROD == "true" ? "www.marketdata.app" : "www-staging.marketdata.app";
+          return `https://${host}/docs/sheets/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
         sidebarPath: require.resolve("./sidebars.js"),
       },
@@ -162,8 +162,8 @@ const config = {
         path: "account",
         routeBasePath: "account",
         editUrl: ({ docPath }) => {
-          const base = process.env.PROD == "true" ? "/docs" : "/docs-staging";
-          return `https://www.marketdata.app${base}/account/${docPath.replace(/\.mdx?$/, '.md')}`;
+          const host = process.env.PROD == "true" ? "www.marketdata.app" : "www-staging.marketdata.app";
+          return `https://${host}/docs/account/${docPath.replace(/\.mdx?$/, '.md')}`;
         },
         sidebarPath: require.resolve("./sidebars.js"),
       },
@@ -177,12 +177,6 @@ const config = {
         appId: "IUHZFO750H",
         apiKey: "c29b76b827a4fa1a0ac3abe15f69ec5c",
         indexName: "Market Data Documentation",
-        ...(process.env.PROD !== "true" && {
-          replaceSearchResultPathname: {
-            from: "/docs/",
-            to: "/docs-staging/",
-          },
-        }),
       },
 
       navbar: {