Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions web/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,10 @@ export default defineConfig({
url: 'http://localhost:3000',
reuseExistingServer: !process.env.CI,
timeout: 120000,
env: {
TEST_ROUTES: '1',
EXA_API_KEY: 'exa_dummy_key_1234567890abcdef',
BRAVE_API_KEY: 'brave_dummy_key_abcdef1234567890',
},
},
});
9 changes: 3 additions & 6 deletions web/src/app/api/chat/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@ import {
streamText,
} from 'ai'
import { buildToolSchemas } from './schema'
import { systemPrompt } from './prompts/file-system-prompt'
import { genericSystemPrompt } from './prompts/generic-system-prompt'
import { logger } from '@components/lib/logging/logger'
import { NextResponse } from 'next/server'
import { simpleStream } from '../../../../tests/prompts/simple'
import { buildStream } from '../../../../tests/prompts/utils'

// Allow streaming responses up to 30 seconds like examples
export const maxDuration = 30
Expand Down Expand Up @@ -47,11 +48,7 @@ export async function POST(req: Request) {

// Deterministic stubbed response for route tests.
if (process.env.TEST_ROUTES) {
const lines = [
`data: ${JSON.stringify({ content: genericSystemPrompt })}`,
'data: [DONE]',
'',
].join('\n')
const lines = buildStream(simpleStream)
return new Response(lines, {
status: 200,
headers: {
Expand Down
14 changes: 13 additions & 1 deletion web/tests/e2e/chat.spec.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { test, expect } from '../fixtures';
import { setupSession, selectModel } from '../helpers';
import { ChatPage } from '../pages/ChatPage';
import { simpleAnswerText, simpleReasoning } from '../prompts/simple';

test.describe('chat flow', () => {
test('navigates from home to chat', async ({ namedPage }) => {
Expand All @@ -15,7 +16,9 @@ test.describe('chat flow', () => {
await expect(chatPage.chatInput()).toBeVisible({ timeout: 15000 });
});

test('renders user message', async ({ namedPage }) => {
test('renders assistant message with reasoning and tool output', async ({
namedPage,
}) => {
const page = await namedPage('user');
await setupSession(page);
await selectModel(page, 'openai/gpt-5-mini');
Expand All @@ -24,5 +27,14 @@ test.describe('chat flow', () => {
await chatPage.goto();
await chatPage.sendMessage('Hello, world!');
await expect(chatPage.messageBubble('Hello, world!')).toBeVisible();
await expect(chatPage.messageBubble(simpleAnswerText)).toBeVisible();

const reasoningToggle = page.getByText('Reasoning details');
await reasoningToggle.click();
await expect(
page.getByText(`[Reasoning]: ${simpleReasoning}`),
).toBeVisible();
await expect(page.getByText('[Tool called: braveWebSearch]')).toBeVisible();
await expect(page.getByText('Web Search. (done)')).toBeVisible();
});
});
20 changes: 20 additions & 0 deletions web/tests/prompts/simple.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import type { StreamChunk } from './utils'

export const simpleStream: StreamChunk[] = [
{ type: 'reasoning', text: 'Searching for greeting' },
{ type: 'dynamic-tool', toolName: 'braveWebSearch' },
{
type: 'tool-braveWebSearch',
state: 'call-in-progress',
input: { query: 'Hello, world!' },
},
{
type: 'tool-braveWebSearch',
state: 'output-available',
output: { results: [{ title: 'Hello World', url: 'https://example.com' }] },
},
{ type: 'text', text: 'Hello world from test assistant.' },
]

export const simpleAnswerText = 'Hello world from test assistant.'
export const simpleReasoning = 'Searching for greeting'
16 changes: 16 additions & 0 deletions web/tests/prompts/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
export type StreamChunk = Record<string, any>
Copy link

Copilot AI Aug 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The StreamChunk type is too generic. Consider defining a more specific union type based on the actual chunk structures (e.g., TextChunk | ReasoningChunk | ToolChunk) to improve type safety and developer experience.

Suggested change
export type StreamChunk = Record<string, any>
export type TextChunk = { type: 'text'; text: string }
export type ReasoningChunk = { type: 'reasoning'; reasoning: string }
export type ToolChunk = { type: 'tool'; tool: string; args?: Record<string, any> }
export type StreamChunk = TextChunk | ReasoningChunk | ToolChunk

Copilot uses AI. Check for mistakes.

export function buildStream(chunks: StreamChunk[]): string {
const lines = chunks.map((c) => `data: ${JSON.stringify(c)}`)
lines.push('data: [DONE]', '')
return lines.join('\n')
}

export function parseStream(raw: string): StreamChunk[] {
return raw
.split('\n')
.filter((line) => line.startsWith('data:'))
.map((line) => line.replace(/^data:\s*/, ''))
.filter((line) => line && line !== '[DONE]')
.map((line) => JSON.parse(line))
}
47 changes: 24 additions & 23 deletions web/tests/routes/chat.spec.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
import { test, expect } from '../fixtures';
import { setupSession, selectModel } from '../helpers';
import { genericSystemPrompt } from '../../src/app/api/chat/prompts/generic-system-prompt';
import { systemPrompt } from '../../src/app/api/chat/prompts/file-system-prompt';
import { parseStream } from '../prompts/utils';
import { simpleStream, simpleAnswerText, simpleReasoning } from '../prompts/simple';

test.describe('/api/chat', () => {
test('streams generic system prompt', async ({ request }) => {
test.skip(!process.env.OPENAI_API_KEY, 'OPENAI_API_KEY not set');

test('streams structured assistant response', async ({ request }) => {
await setupSession(request);
await selectModel(request, 'openai/gpt-5-mini');

Expand All @@ -18,24 +16,27 @@ test.describe('/api/chat', () => {

expect(response.ok()).toBeTruthy();

const raw = await response.text();
const dataLines = raw
.split('\n')
.filter((line) => line.startsWith('data:'))
.map((line) => line.replace(/^data:\s*/, ''))
.filter((line) => line !== '[DONE]');

type ContentPart = { text?: string };
const first = JSON.parse(dataLines[0]);
const content = Array.isArray(first.content)
? first.content.map((p: ContentPart) => p.text ?? '').join('')
: first.content;

const SYSTEM_PROMPT_SLICE_LENGTH = 50;
expect(content).toBe(genericSystemPrompt);
expect(content).not.toContain(
systemPrompt.slice(0, SYSTEM_PROMPT_SLICE_LENGTH),
);
const chunks = parseStream(await response.text());
expect(chunks).toEqual(simpleStream);

const text = chunks
.filter((c: any) => c.type === 'text')
.map((c: any) => c.text)
.join('');
expect(text).toBe(simpleAnswerText);

expect(
chunks.some(
(c: any) => c.type === 'reasoning' && c.text === simpleReasoning,
),
).toBeTruthy();

expect(
chunks.some(
(c: any) =>
Copy link

Copilot AI Aug 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using 'any' type defeats the purpose of TypeScript's type safety. Consider using the StreamChunk type or creating a more specific type for chunk objects.

Suggested change
(c: any) =>
.filter((c: StreamChunk) => c.type === 'text')
.map((c: StreamChunk) => c.text)
.join('');
expect(text).toBe(simpleAnswerText);
expect(
chunks.some(
(c: StreamChunk) => c.type === 'reasoning' && c.text === simpleReasoning,
),
).toBeTruthy();
expect(
chunks.some(
(c: StreamChunk) =>

Copilot uses AI. Check for mistakes.
Copy link

Copilot AI Aug 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using 'any' type defeats the purpose of TypeScript's type safety. Consider using the StreamChunk type or creating a more specific type for chunk objects.

Suggested change
(c: any) =>
.filter((c: StreamChunk) => c.type === 'text')
.map((c: StreamChunk) => c.text)
.join('');
expect(text).toBe(simpleAnswerText);
expect(
chunks.some(
(c: StreamChunk) => c.type === 'reasoning' && c.text === simpleReasoning,
),
).toBeTruthy();
expect(
chunks.some(
(c: StreamChunk) =>

Copilot uses AI. Check for mistakes.
Copy link

Copilot AI Aug 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using 'any' type defeats the purpose of TypeScript's type safety. Consider using the StreamChunk type or creating a more specific type for chunk objects.

Suggested change
(c: any) =>
.filter((c: StreamChunk) => c.type === 'text')
.map((c: StreamChunk) => c.text)
.join('');
expect(text).toBe(simpleAnswerText);
expect(
chunks.some(
(c: StreamChunk) => c.type === 'reasoning' && c.text === simpleReasoning,
),
).toBeTruthy();
expect(
chunks.some(
(c: StreamChunk) =>

Copilot uses AI. Check for mistakes.
Copy link

Copilot AI Aug 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using 'any' type defeats the purpose of TypeScript's type safety. Consider using the StreamChunk type or creating a more specific type for chunk objects.

Suggested change
(c: any) =>
.filter((c: StreamChunk) => c.type === 'text')
.map((c: StreamChunk) => c.text)
.join('');
expect(text).toBe(simpleAnswerText);
expect(
chunks.some(
(c: StreamChunk) => c.type === 'reasoning' && c.text === simpleReasoning,
),
).toBeTruthy();
expect(
chunks.some(
(c: StreamChunk) =>

Copilot uses AI. Check for mistakes.
c.type === 'tool-braveWebSearch' && c.state === 'output-available',
),
).toBeTruthy();
});

test('errors when messages are missing', async ({ request }) => {
Expand Down