From 59fc3458f8767232adb56276690247f82ade87aa Mon Sep 17 00:00:00 2001 From: Utkarsh Mishra Date: Sat, 16 Aug 2025 13:28:47 +0200 Subject: [PATCH] test: add prompt fixtures and stream utilities --- web/playwright.config.ts | 5 ++++ web/src/app/api/chat/route.ts | 9 +++---- web/tests/e2e/chat.spec.ts | 14 ++++++++++- web/tests/prompts/simple.ts | 20 +++++++++++++++ web/tests/prompts/utils.ts | 16 ++++++++++++ web/tests/routes/chat.spec.ts | 47 ++++++++++++++++++----------------- 6 files changed, 81 insertions(+), 30 deletions(-) create mode 100644 web/tests/prompts/simple.ts create mode 100644 web/tests/prompts/utils.ts diff --git a/web/playwright.config.ts b/web/playwright.config.ts index 74a396f..13c4a29 100644 --- a/web/playwright.config.ts +++ b/web/playwright.config.ts @@ -11,5 +11,10 @@ export default defineConfig({ url: 'http://localhost:3000', reuseExistingServer: !process.env.CI, timeout: 120000, + env: { + TEST_ROUTES: '1', + EXA_API_KEY: 'exa_dummy_key_1234567890abcdef', + BRAVE_API_KEY: 'brave_dummy_key_abcdef1234567890', + }, }, }); diff --git a/web/src/app/api/chat/route.ts b/web/src/app/api/chat/route.ts index 67a6920..b918717 100644 --- a/web/src/app/api/chat/route.ts +++ b/web/src/app/api/chat/route.ts @@ -5,10 +5,11 @@ import { streamText, } from 'ai' import { buildToolSchemas } from './schema' -import { systemPrompt } from './prompts/file-system-prompt' import { genericSystemPrompt } from './prompts/generic-system-prompt' import { logger } from '@components/lib/logging/logger' import { NextResponse } from 'next/server' +import { simpleStream } from '../../../../tests/prompts/simple' +import { buildStream } from '../../../../tests/prompts/utils' // Allow streaming responses up to 30 seconds like examples export const maxDuration = 30 @@ -47,11 +48,7 @@ export async function POST(req: Request) { // Deterministic stubbed response for route tests. if (process.env.TEST_ROUTES) { - const lines = [ - `data: ${JSON.stringify({ content: genericSystemPrompt })}`, - 'data: [DONE]', - '', - ].join('\n') + const lines = buildStream(simpleStream) return new Response(lines, { status: 200, headers: { diff --git a/web/tests/e2e/chat.spec.ts b/web/tests/e2e/chat.spec.ts index d20c6af..313b647 100644 --- a/web/tests/e2e/chat.spec.ts +++ b/web/tests/e2e/chat.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from '../fixtures'; import { setupSession, selectModel } from '../helpers'; import { ChatPage } from '../pages/ChatPage'; +import { simpleAnswerText, simpleReasoning } from '../prompts/simple'; test.describe('chat flow', () => { test('navigates from home to chat', async ({ namedPage }) => { @@ -15,7 +16,9 @@ test.describe('chat flow', () => { await expect(chatPage.chatInput()).toBeVisible({ timeout: 15000 }); }); - test('renders user message', async ({ namedPage }) => { + test('renders assistant message with reasoning and tool output', async ({ + namedPage, + }) => { const page = await namedPage('user'); await setupSession(page); await selectModel(page, 'openai/gpt-5-mini'); @@ -24,5 +27,14 @@ test.describe('chat flow', () => { await chatPage.goto(); await chatPage.sendMessage('Hello, world!'); await expect(chatPage.messageBubble('Hello, world!')).toBeVisible(); + await expect(chatPage.messageBubble(simpleAnswerText)).toBeVisible(); + + const reasoningToggle = page.getByText('Reasoning details'); + await reasoningToggle.click(); + await expect( + page.getByText(`[Reasoning]: ${simpleReasoning}`), + ).toBeVisible(); + await expect(page.getByText('[Tool called: braveWebSearch]')).toBeVisible(); + await expect(page.getByText('Web Search. (done)')).toBeVisible(); }); }); diff --git a/web/tests/prompts/simple.ts b/web/tests/prompts/simple.ts new file mode 100644 index 0000000..49080c7 --- /dev/null +++ b/web/tests/prompts/simple.ts @@ -0,0 +1,20 @@ +import type { StreamChunk } from './utils' + +export const simpleStream: StreamChunk[] = [ + { type: 'reasoning', text: 'Searching for greeting' }, + { type: 'dynamic-tool', toolName: 'braveWebSearch' }, + { + type: 'tool-braveWebSearch', + state: 'call-in-progress', + input: { query: 'Hello, world!' }, + }, + { + type: 'tool-braveWebSearch', + state: 'output-available', + output: { results: [{ title: 'Hello World', url: 'https://example.com' }] }, + }, + { type: 'text', text: 'Hello world from test assistant.' }, +] + +export const simpleAnswerText = 'Hello world from test assistant.' +export const simpleReasoning = 'Searching for greeting' diff --git a/web/tests/prompts/utils.ts b/web/tests/prompts/utils.ts new file mode 100644 index 0000000..95b40f5 --- /dev/null +++ b/web/tests/prompts/utils.ts @@ -0,0 +1,16 @@ +export type StreamChunk = Record + +export function buildStream(chunks: StreamChunk[]): string { + const lines = chunks.map((c) => `data: ${JSON.stringify(c)}`) + lines.push('data: [DONE]', '') + return lines.join('\n') +} + +export function parseStream(raw: string): StreamChunk[] { + return raw + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.replace(/^data:\s*/, '')) + .filter((line) => line && line !== '[DONE]') + .map((line) => JSON.parse(line)) +} diff --git a/web/tests/routes/chat.spec.ts b/web/tests/routes/chat.spec.ts index ee58050..2dd3123 100644 --- a/web/tests/routes/chat.spec.ts +++ b/web/tests/routes/chat.spec.ts @@ -1,12 +1,10 @@ import { test, expect } from '../fixtures'; import { setupSession, selectModel } from '../helpers'; -import { genericSystemPrompt } from '../../src/app/api/chat/prompts/generic-system-prompt'; -import { systemPrompt } from '../../src/app/api/chat/prompts/file-system-prompt'; +import { parseStream } from '../prompts/utils'; +import { simpleStream, simpleAnswerText, simpleReasoning } from '../prompts/simple'; test.describe('/api/chat', () => { - test('streams generic system prompt', async ({ request }) => { - test.skip(!process.env.OPENAI_API_KEY, 'OPENAI_API_KEY not set'); - + test('streams structured assistant response', async ({ request }) => { await setupSession(request); await selectModel(request, 'openai/gpt-5-mini'); @@ -18,24 +16,27 @@ test.describe('/api/chat', () => { expect(response.ok()).toBeTruthy(); - const raw = await response.text(); - const dataLines = raw - .split('\n') - .filter((line) => line.startsWith('data:')) - .map((line) => line.replace(/^data:\s*/, '')) - .filter((line) => line !== '[DONE]'); - - type ContentPart = { text?: string }; - const first = JSON.parse(dataLines[0]); - const content = Array.isArray(first.content) - ? first.content.map((p: ContentPart) => p.text ?? '').join('') - : first.content; - - const SYSTEM_PROMPT_SLICE_LENGTH = 50; - expect(content).toBe(genericSystemPrompt); - expect(content).not.toContain( - systemPrompt.slice(0, SYSTEM_PROMPT_SLICE_LENGTH), - ); + const chunks = parseStream(await response.text()); + expect(chunks).toEqual(simpleStream); + + const text = chunks + .filter((c: any) => c.type === 'text') + .map((c: any) => c.text) + .join(''); + expect(text).toBe(simpleAnswerText); + + expect( + chunks.some( + (c: any) => c.type === 'reasoning' && c.text === simpleReasoning, + ), + ).toBeTruthy(); + + expect( + chunks.some( + (c: any) => + c.type === 'tool-braveWebSearch' && c.state === 'output-available', + ), + ).toBeTruthy(); }); test('errors when messages are missing', async ({ request }) => {