diff --git a/skills/dev-browser/SKILL.md b/skills/dev-browser/SKILL.md index 78119e2..ff2771f 100644 --- a/skills/dev-browser/SKILL.md +++ b/skills/dev-browser/SKILL.md @@ -153,6 +153,10 @@ await client.disconnect(); // Disconnect (pages persist) // ARIA Snapshot methods for element discovery and interaction const snapshot = await client.getAISnapshot("name"); // Get ARIA accessibility tree const element = await client.selectSnapshotRef("name", "e5"); // Get element by ref + +// Frame-aware helpers for embedded widgets (Stripe, PayPal, etc.) +const result = await client.findInFrames("name", "input[name='card']"); // Find in any frame +const formResult = await client.fillForm("name", { "Card Number": "4242..." }); // Smart form fill ``` The `page` object is a standard Playwright Page—use normal Playwright methods. @@ -279,12 +283,85 @@ await client.disconnect(); EOF ``` +## Working with Iframes (Stripe, PayPal, etc.) + +Payment forms and embedded widgets often use iframes that are invisible to normal selectors. Use `findInFrames()` and `fillForm()` to work with these. + +### Finding Elements in Iframes + +`findInFrames()` searches all frames (main + nested) for an element: + +```bash +cd skills/dev-browser && bun x tsx <<'EOF' +import { connect, waitForPageLoad } from "@/client.js"; + +const client = await connect(); +const page = await client.page("checkout"); + +await page.goto("https://example.com/checkout"); +await waitForPageLoad(page); + +// Find card input in Stripe iframe +const result = await client.findInFrames("checkout", 'input[name="cardnumber"]'); +if (result.element) { + console.log("Found in:", result.frameInfo); // e.g., "(unnamed) [Stripe]: https://js.stripe.com/..." + await result.element.fill("4242424242424242"); +} else { + console.log("Not found:", result.frameInfo); +} + +await client.disconnect(); +EOF +``` + +### Smart Form Filling + +`fillForm()` finds fields by label, name, placeholder, or aria-label—across all frames: + +```bash +cd skills/dev-browser && bun x tsx <<'EOF' +import { connect, waitForPageLoad } from "@/client.js"; + +const client = await connect(); +const page = await client.page("checkout"); + +await page.goto("https://example.com/checkout"); +await waitForPageLoad(page); + +// Fill form using field labels - works across frames +const result = await client.fillForm("checkout", { + "Card Number": "4242424242424242", + "Expiration Date": "12/25", + "CVC": "123", + "Name on Card": "Test User" +}, { submit: true }); + +console.log("Filled:", result.filled); +console.log("Not found:", result.notFound); +console.log("Submitted:", result.submitted); + +await client.disconnect(); +EOF +``` + +### Options + +**findInFrames options:** +- `timeout` - Max wait time in ms (default: 5000) +- `includeMainFrame` - Search main frame too (default: true) + +**fillForm options:** +- `timeout` - Max wait per field in ms (default: 5000) +- `submit` - Click submit button after filling (default: false) +- `clear` - Clear fields before filling (default: true) + ## Debugging Tips 1. **Use getAISnapshot** to see what elements are available and their refs 2. **Take screenshots** when you need visual context 3. **Use waitForSelector** before interacting with dynamic content 4. **Check page.url()** to confirm navigation worked +5. **Use findInFrames** when selectors work in DevTools but not in scripts (likely in iframe) ## Error Recovery diff --git a/skills/dev-browser/src/client.ts b/skills/dev-browser/src/client.ts index 9c95408..3d3d998 100644 --- a/skills/dev-browser/src/client.ts +++ b/skills/dev-browser/src/client.ts @@ -1,4 +1,4 @@ -import { chromium, type Browser, type Page, type ElementHandle } from "playwright"; +import { chromium, type Browser, type Page, type ElementHandle, type Frame } from "playwright"; import type { GetPageRequest, GetPageResponse, @@ -7,6 +7,52 @@ import type { } from "./types"; import { getSnapshotScript } from "./snapshot/browser-script"; +/** + * Options for finding elements in frames + */ +export interface FindInFramesOptions { + /** Maximum time to wait for element in ms (default: 5000) */ + timeout?: number; + /** Include main frame in search (default: true) */ + includeMainFrame?: boolean; +} + +/** + * Result of finding an element in frames + */ +export interface FindInFramesResult { + /** The element handle if found */ + element: ElementHandle | null; + /** The frame containing the element */ + frame: Frame | null; + /** Frame name or src for debugging */ + frameInfo: string; +} + +/** + * Options for filling forms + */ +export interface FillFormOptions { + /** Maximum time to wait for elements in ms (default: 5000) */ + timeout?: number; + /** Submit form after filling (default: false) */ + submit?: boolean; + /** Clear fields before filling (default: true) */ + clear?: boolean; +} + +/** + * Result of filling a form + */ +export interface FillFormResult { + /** Fields that were successfully filled */ + filled: string[]; + /** Fields that could not be found */ + notFound: string[]; + /** Whether form was submitted (if requested) */ + submitted: boolean; +} + /** * Options for waiting for page load */ @@ -222,6 +268,26 @@ export interface DevBrowserClient { * Refs persist across Playwright connections. */ selectSnapshotRef: (name: string, ref: string) => Promise; + /** + * Find an element across all frames (including iframes like Stripe, PayPal). + * Searches main frame and all nested iframes for the selector. + * Useful for payment forms and embedded widgets. + */ + findInFrames: ( + name: string, + selector: string, + options?: FindInFramesOptions + ) => Promise; + /** + * Smart form filling using field labels, names, or placeholders. + * Automatically finds fields by matching labels, aria-labels, names, or placeholders. + * Works across frames (including Stripe iframes). + */ + fillForm: ( + name: string, + fields: Record, + options?: FillFormOptions + ) => Promise; } export async function connect(serverUrl = "http://localhost:9222"): Promise { @@ -399,5 +465,166 @@ export async function connect(serverUrl = "http://localhost:9222"): Promise { + const { timeout = 5000, includeMainFrame = true } = options; + const page = await getPage(name); + + // Get all frames (including nested) + const allFrames = page.frames(); + + // Try each frame + for (const frame of allFrames) { + // Skip main frame if not wanted + if (!includeMainFrame && frame === page.mainFrame()) { + continue; + } + + try { + // Wait briefly for element in this frame + const element = await frame.waitForSelector(selector, { + timeout: Math.min(timeout / allFrames.length, 1000), + state: "attached", + }); + + if (element) { + // Build frame info for debugging + const frameName = frame.name() || "(unnamed)"; + const frameUrl = frame.url(); + const isStripe = frameUrl.includes("stripe"); + const isPaypal = frameUrl.includes("paypal"); + const badge = isStripe ? " [Stripe]" : isPaypal ? " [PayPal]" : ""; + const frameInfo = `${frameName}${badge}: ${frameUrl.substring(0, 60)}`; + + return { + element, + frame, + frameInfo, + }; + } + } catch { + // Element not in this frame, continue + } + } + + // Not found in any frame + return { + element: null, + frame: null, + frameInfo: "Element not found in any frame", + }; + }, + + async fillForm( + name: string, + fields: Record, + options: FillFormOptions = {} + ): Promise { + const { timeout = 5000, submit = false, clear = true } = options; + const page = await getPage(name); + const allFrames = page.frames(); + + const filled: string[] = []; + const notFound: string[] = []; + + for (const [fieldLabel, value] of Object.entries(fields)) { + let found = false; + + // Build selectors to try - from most specific to least + const normalizedLabel = fieldLabel.toLowerCase().trim(); + const selectors = [ + // Exact matches + `input[name="${fieldLabel}"]`, + `input[name="${normalizedLabel}"]`, + `input[id="${fieldLabel}"]`, + `input[id="${normalizedLabel}"]`, + `select[name="${fieldLabel}"]`, + `select[name="${normalizedLabel}"]`, + `textarea[name="${fieldLabel}"]`, + // Placeholder matches + `input[placeholder*="${fieldLabel}" i]`, + `input[placeholder*="${normalizedLabel}" i]`, + // Aria-label matches + `input[aria-label*="${fieldLabel}" i]`, + `[aria-label*="${fieldLabel}" i]`, + // Data attribute matches (common in Stripe) + `[data-elements-stable-field-name="${normalizedLabel}"]`, + // Label association + `label:has-text("${fieldLabel}") + input`, + `label:has-text("${fieldLabel}") input`, + ]; + + // Try each frame + for (const frame of allFrames) { + if (found) break; + + for (const selector of selectors) { + try { + const element = await frame.waitForSelector(selector, { + timeout: Math.min(timeout / (allFrames.length * selectors.length), 200), + state: "attached", + }); + + if (element) { + // Clear if requested + if (clear) { + await element.click({ clickCount: 3 }); // Select all + await page.keyboard.press("Backspace"); + } + + // Fill the field + await element.fill(value); + filled.push(fieldLabel); + found = true; + break; + } + } catch { + // Selector not found in this frame, continue + } + } + } + + if (!found) { + notFound.push(fieldLabel); + } + } + + // Submit if requested and we filled at least one field + let submitted = false; + if (submit && filled.length > 0) { + try { + // Try common submit patterns + const submitSelectors = [ + 'button[type="submit"]', + 'input[type="submit"]', + 'button:has-text("Submit")', + 'button:has-text("Pay")', + 'button:has-text("Continue")', + 'button:has-text("Place Order")', + ]; + + for (const selector of submitSelectors) { + try { + const btn = await page.waitForSelector(selector, { timeout: 500 }); + if (btn) { + await btn.click(); + submitted = true; + break; + } + } catch { + // Continue trying + } + } + } catch { + // Submit failed + } + } + + return { filled, notFound, submitted }; + }, }; }