Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,5 @@ fastlane/test_output

claude-agent-sdk-demos
xcuserdata

apps/pipeline/app.log
89 changes: 77 additions & 12 deletions apps/pipeline-ui/src/pages/StandardEbooksPage.tsx
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { useState, useEffect, useCallback, useRef, useLayoutEffect } from "react";
import { useNavigate } from "react-router-dom";
import { trpc } from "../trpc";
import { Loader2, ArrowLeft, BookOpen } from "lucide-react";
import { Loader2, ArrowLeft, BookOpen, Search, X } from "lucide-react";
import { BookCard, type CollectionBook } from "../components/BookCard";
import { BookModal } from "../components/BookModal";
import { Button } from "@/components/ui/button";
Expand Down Expand Up @@ -136,15 +136,18 @@ function AuthorLetterRow({
export function StandardEbooksPage() {
const navigate = useNavigate();
const [groupedBooks, setGroupedBooks] = useState<Record<string, SEBook[]>>({});
const [allBooks, setAllBooks] = useState<SEBook[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [totalBooks, setTotalBooks] = useState(0);
const [modalBook, setModalBook] = useState<CollectionBook | null>(null);
const [searchQuery, setSearchQuery] = useState("");

useEffect(() => {
const loadData = async () => {
try {
const data = await trpc.getStandardEbooksIndex.query();
setGroupedBooks(data.groupedByAuthorLetter);
setAllBooks(data.books);
setTotalBooks(data.books.length);
} catch (e) {
console.error("Failed to load Standard Ebooks index:", e);
Expand Down Expand Up @@ -178,7 +181,33 @@ export function StandardEbooksPage() {
);
}

const sortedLetters = Object.keys(groupedBooks).sort();
const normalizedQuery = searchQuery.trim().toLowerCase();
const filteredBooks = normalizedQuery
? allBooks.filter((book) => {
const haystack = [
book.title,
book.author,
book.authorFileAs,
book.description,
book.subjects.join(" "),
]
.join(" ")
.toLowerCase();
return haystack.includes(normalizedQuery);
})
: allBooks;

const visibleGroupedBooks = normalizedQuery
? filteredBooks.reduce<Record<string, SEBook[]>>((acc, book) => {
const firstLetter = (book.authorFileAs || book.author).charAt(0).toUpperCase();
if (!acc[firstLetter]) acc[firstLetter] = [];
acc[firstLetter].push(book);
return acc;
}, {})
: groupedBooks;

const sortedLetters = Object.keys(visibleGroupedBooks).sort();
const visibleCount = normalizedQuery ? filteredBooks.length : totalBooks;

return (
<div className="pb-24 page-enter">
Expand All @@ -189,7 +218,7 @@ export function StandardEbooksPage() {
Standard Ebooks
</h2>
<p className="text-lg text-muted-foreground mt-2">
{totalBooks} professionally formatted public domain books
{visibleCount} professionally formatted public domain books
</p>
</div>
<Button variant="ghost" size="default" onClick={() => navigate("/")} className="gap-2">
Expand All @@ -198,17 +227,53 @@ export function StandardEbooksPage() {
</Button>
</div>

<div className="flex flex-col">
{sortedLetters.map((letter, idx) => (
<div key={letter} className={`animate-fade-in stagger-${Math.min(idx + 1, 6)}`}>
<AuthorLetterRow
letter={letter}
books={groupedBooks[letter]}
onSelectBook={handleBookSelect}
onOpenModal={handleOpenModal}
<div className="px-4 md:px-8 xl:px-48 mb-10">
<div className="flex flex-col gap-3 md:flex-row md:items-center md:justify-between">
<div className="relative w-full md:max-w-2xl">
<Search className="w-4 h-4 text-muted-foreground absolute left-3 top-1/2 -translate-y-1/2" />
<input
type="text"
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
placeholder="Search by title, author, or subject..."
className="w-full pl-10 pr-10 py-2 rounded-md bg-background border border-border text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-primary/50"
/>
{searchQuery.trim() && (
<button
type="button"
onClick={() => setSearchQuery("")}
className="absolute right-2 top-1/2 -translate-y-1/2 p-1 rounded hover:bg-muted"
aria-label="Clear search"
>
<X className="w-4 h-4 text-muted-foreground" />
</button>
)}
</div>
))}
<div className="text-sm text-muted-foreground">
{normalizedQuery
? `Showing ${visibleCount} of ${totalBooks} books`
: `${totalBooks} books`}
</div>
</div>
</div>

<div className="flex flex-col">
{sortedLetters.length === 0 ? (
<div className="px-4 md:px-8 xl:px-48 py-10 text-muted-foreground">
No books match your search.
</div>
) : (
sortedLetters.map((letter, idx) => (
<div key={letter} className={`animate-fade-in stagger-${Math.min(idx + 1, 6)}`}>
<AuthorLetterRow
letter={letter}
books={visibleGroupedBooks[letter]}
onSelectBook={handleBookSelect}
onOpenModal={handleOpenModal}
/>
</div>
))
)}
</div>

<BookModal book={modalBook} onClose={handleCloseModal} onSelect={handleBookSelect} />
Expand Down
1 change: 1 addition & 0 deletions apps/pipeline/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ bun src/continue-pipeline-cli.ts books-data/my-book --status
| ----------------------------- | ------------------------- | -------------------------------- |
| `import_epub` | EPUB → FB2 → rich.xml | `input/rich.xml` |
| `create_settings` | Detect language, metadata | `bookSettings.json` |
| `upload_figures` | Upload SE figures | Convex `books/*/figures` |
| `generate_reference_cards` | Character summaries | `single-summary-per-person.json` |
| `rewrite_paragraphs` | Inject character tags | `rewritten-paragraphs-*.xml` |
| `generate_graphical_style` | Visual style JSON | `graphicalStyle.json` |
Expand Down
1 change: 1 addition & 0 deletions apps/pipeline/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
},
"dependencies": {
"@ai-sdk/anthropic": "2.0.38",
"@ai-sdk/azure": "2.0.91",
"@ai-sdk/cerebras": "^1.0.20",
"@ai-sdk/google": "^2.0.14",
"@ai-sdk/groq": "^2.0.21",
Expand Down
6 changes: 5 additions & 1 deletion apps/pipeline/src/callClaude.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,11 @@ export const callClaude = async <T = string>(
};

const doIt = async () => {
const result = await callClaude("Identify all named book characters (people) in this page.\n");
const result = (await callGeminiWrapper(
"Identify all named book characters (people) in this page.\n",
undefined,
1,
)) as string;
logger.info(result);
};
// Execute only if this file is being run directly (not imported)
Expand Down
47 changes: 12 additions & 35 deletions apps/pipeline/src/callFastGemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,44 +76,21 @@ Based on the book text answer the user's question, using quotes from the wider b
};

export const callGeminiWithThinking = async (prompt: string) => {
const ai = new GoogleGenAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY });
const config = {
responseMimeType: "text/plain",
httpOptions: {
timeout: 15 * 60 * 1000, // 15 minutes in milliseconds
},
};
const model = "gemini-3-flash-preview";
// const model = "gemini-3-pro-preview";

const contents = [{ role: "user", parts: [{ text: prompt }] }];
const safetySettings = [
{ category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_NONE },
{ category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_NONE },
{
category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
threshold: HarmBlockThreshold.BLOCK_NONE,
},
{
category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
threshold: HarmBlockThreshold.BLOCK_NONE,
},
{
category: HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY,
threshold: HarmBlockThreshold.BLOCK_NONE,
},
{ category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.OFF },
{ category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.OFF },
{ category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, threshold: HarmBlockThreshold.OFF },
{ category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: HarmBlockThreshold.OFF },
{ category: HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY, threshold: HarmBlockThreshold.OFF },
];

console.log("before response", model);
const response = await ai.models.generateContent({
model,
config: { ...config, safetySettings },
contents,
console.log("CALLING GEMINI WITH THINKING");
const { text } = await generateText({
model: google("gemini-3-flash-preview"),
prompt,
experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true },
providerOptions: { google: { safetySettings } },
});

console.log("after response");

return response?.text;
return text;
};
Comment on lines 78 to 94
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Missing thinkingConfig despite function name implying thinking is enabled.

The function callGeminiWithThinking no longer configures thinking in providerOptions. Compare with callFastGemini (lines 20-25) which includes thinkingConfig: { thinkingBudget: 256, includeThoughts: true }. The function name is now misleading—either add the thinking configuration or rename the function.

Additionally, this function uses HarmBlockThreshold.OFF while all other functions in this file use HarmBlockThreshold.BLOCK_NONE. This inconsistency may cause unexpected behavior differences between Gemini call variants.

Proposed fix to add thinking configuration
   const { text } = await generateText({
     model: google("gemini-3-flash-preview"),
     prompt,
     experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true },
-    providerOptions: { google: { safetySettings } },
+    providerOptions: {
+      google: {
+        safetySettings,
+        thinkingConfig: { thinkingBudget: 1024, includeThoughts: true },
+      },
+    },
   });
🤖 Prompt for AI Agents
In `@apps/pipeline/src/callFastGemini.ts` around lines 78 - 94, The function
callGeminiWithThinking is missing the thinkingConfig and uses an inconsistent
HarmBlockThreshold; update the providerOptions passed to generateText inside
callGeminiWithThinking to include thinkingConfig: { thinkingBudget: 256,
includeThoughts: true } (matching callFastGemini) and change all safetySettings
thresholds from HarmBlockThreshold.OFF to HarmBlockThreshold.BLOCK_NONE so
behavior matches other Gemini variants; ensure you only modify the
providerOptions block in callGeminiWithThinking and keep the rest of the call
(model, prompt, experimental_telemetry) unchanged.


export const callGeminiWithThinkingAndSchema = async <T>(
Expand Down
50 changes: 50 additions & 0 deletions apps/pipeline/src/callGrokAzure.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import OpenAI from "openai";
import { type z } from "zod";

const endpoint = "https://bookgenius.services.ai.azure.com/openai/v1/";
const model = "grok-4-fast-reasoning";
const api_key = process.env.AZURE_GROK_KEY;

const client = new OpenAI({ baseURL: endpoint, apiKey: api_key });
Comment on lines +6 to +8
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The api_key is read from process.env.AZURE_GROK_KEY but there's no check to ensure it's defined. If the environment variable is missing, api_key will be undefined, and the OpenAI client will be initialized with an invalid key. This will lead to runtime errors later on. It's better to fail fast with a clear error message if the key is not configured.

const api_key = process.env.AZURE_GROK_KEY;

if (!api_key) {
  throw new Error("Missing environment variable: AZURE_GROK_KEY");
}

const client = new OpenAI({ baseURL: endpoint, apiKey: api_key });

Comment on lines +6 to +8
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Missing API key validation before client initialization.

If AZURE_GROK_KEY is undefined, the OpenAI client will be created with an undefined API key, which will fail at runtime during the first API call rather than at module load time. Consider adding validation.

🛡️ Proposed fix
 const api_key = process.env.AZURE_GROK_KEY;
+
+if (!api_key) {
+  throw new Error("Missing AZURE_GROK_KEY environment variable");
+}

 const client = new OpenAI({ baseURL: endpoint, apiKey: api_key });
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
const api_key = process.env.AZURE_GROK_KEY;
const client = new OpenAI({ baseURL: endpoint, apiKey: api_key });
const api_key = process.env.AZURE_GROK_KEY;
if (!api_key) {
throw new Error("Missing AZURE_GROK_KEY environment variable");
}
const client = new OpenAI({ baseURL: endpoint, apiKey: api_key });
🤖 Prompt for AI Agents
In `@apps/pipeline/src/callGrokAzure.ts` around lines 6 - 8, Ensure AZURE_GROK_KEY
is validated before creating the OpenAI client: check process.env.AZURE_GROK_KEY
(the api_key variable) and if it's missing/empty throw a clear Error or log and
exit, then only instantiate new OpenAI({ baseURL: endpoint, apiKey: api_key });
that way OpenAI client creation (the client variable) never occurs with an
undefined key and failure happens at startup instead of at first API call.


export const callGrokAzure = async (prompt: string) => {
const completion = await client.chat.completions.create({
messages: [{ role: "user", content: prompt }],
model,
});

return completion.choices[0].message.content;
};
Comment on lines +10 to +17
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Return type may be null; consider handling.

completion.choices[0].message.content can be null when the model returns a refusal or empty response. The current signature implies it always returns a string.

🛡️ Proposed fix to handle null content
-export const callGrokAzure = async (prompt: string) => {
+export const callGrokAzure = async (prompt: string): Promise<string> => {
   const completion = await client.chat.completions.create({
     messages: [{ role: "user", content: prompt }],
     model,
   });

-  return completion.choices[0].message.content;
+  const content = completion.choices[0].message.content;
+  if (content === null) {
+    throw new Error("Model returned null content");
+  }
+  return content;
 };
🤖 Prompt for AI Agents
In `@apps/pipeline/src/callGrokAzure.ts` around lines 10 - 17, callGrokAzure
currently returns completion.choices[0].message.content which can be null;
update callGrokAzure to handle that case by checking completion and
completion.choices[0].message.content and either (a) return a safe fallback
string (e.g., empty string) or (b) throw a clear error, and update the function
signature accordingly to reflect string | null or throw; reference the symbols
callGrokAzure, completion, choices, and message.content when making the change
so the null-check covers completion.choices[0] and message.content before
returning.


export const callGrokAzureWithSchema = async <T>(prompt: string, zodSchema: z.ZodSchema<T>) => {
const completion = await client.chat.completions.create({
messages: [{ role: "user", content: prompt }],
model,
response_format: {
type: "json_schema",
json_schema: {
name: "response",
strict: true,
// @ts-expect-error(zod typing)
schema: zodSchema.shape,
Comment on lines +28 to +29
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Using zodSchema.shape to generate the JSON schema is not robust. This property only exists on z.object schemas and doesn't correctly represent the full schema structure expected by the OpenAI API, especially for nested objects, arrays, or non-object types. This can lead to runtime errors or incorrect schema validation.

Consider using a library like zod-to-json-schema for a more reliable conversion. For example:

import { zodToJsonSchema } from 'zod-to-json-schema';

// ... inside callGrokAzureWithSchema
const jsonSchema = zodToJsonSchema(zodSchema);

const completion = await client.chat.completions.create({
  // ...
  response_format: {
    type: "json_schema",
    json_schema: {
      name: "response",
      strict: true,
      schema: jsonSchema,
    },
  },
});

This will require adding zod-to-json-schema as a dependency. The current implementation with @ts-expect-error is a strong indicator of a potential issue.

},
},
});
let result: T;
try {
result = JSON.parse(completion.choices[0].message.content as string) as T;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The API response completion.choices[0].message.content can be null. Attempting to JSON.parse(null) will result in null, which might be acceptable, but if the model returns an empty response for some reason, it's better to handle this case explicitly to avoid unexpected behavior or provide a clearer error message.

    const content = completion.choices[0].message.content;
    if (content === null) {
      throw new Error("Received null content from the model.");
    }
    result = JSON.parse(content) as T;

} catch (e) {
console.error("Error parsing JSON", e);
throw e;
}
return result;
};

// if (require.main === module) {
// const schema = z.object({ name: z.string(), age: z.number() });
// const prompt = "What is my name? My name is John Doe and I'm 30";
// const result = await callGrokAzureWithSchema(prompt, schema);
// console.log(result);
// console.log(result.name);
// console.log(result.age);
// }
3 changes: 1 addition & 2 deletions apps/pipeline/src/callO3.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ export const callO3WithSchema = async <T>(
model: openai(model),
schema: zodSchema,
prompt,
// providerOptions: { google: { thinkingConfig: { thinkingBudget: 0, includeThoughts: true } } },
experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true },
});

Expand All @@ -36,7 +35,7 @@ export const callGpt5 = async <T = string>(
) => {
const chatCompletion = await client.chat.completions.create({
messages: [{ role: "user", content: prompt }],
model: "gpt-5.1",
model: "gpt-5.2",
reasoning_effort: "medium",
Comment on lines 36 to 39
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

🧩 Analysis chain

🌐 Web query:

OpenAI gpt-5.2 model availability chat completions API 2026

💡 Result:

As of February 5, 2026, GPT‑5.2 is available to all developers in the OpenAI API, including the Chat Completions API.

  • Chat Completions API model IDs
    • gpt-5.2 (GPT‑5.2 Thinking) — usable in Chat Completions (/v1/chat/completions) and also in Responses. (openai.com)
    • gpt-5.2-chat-latest (GPT‑5.2 Instant) — intended for chat-style usage. (openai.com)
  • Pinned snapshot available: gpt-5.2-2025-12-11 (plus alias gpt-5.2). (platform.openai.com)
  • Docs / pricing / capabilities page: confirms modalities (text I/O, image input) and that it supports Chat Completions. (platform.openai.com)

Citations:


🏁 Script executed:

cat -n apps/pipeline/src/callO3.ts

Repository: TheBrainFamily/BookgeniusPlayer

Length of output: 1671


Remove the review concern: gpt-5.2 is available in the Chat Completions API.

The model is officially supported for all developers as of February 2026, including the direct OpenAI SDK path used here.

However, the callGpt5 function does not implement the required callModel<T> interface specified in the coding guidelines for apps/pipeline/src/call*.ts. The function accepts schema and maxRetries parameters but does not use them. Implement structured output handling via the schema parameter and retry logic to comply with the shared interface, or remove the unused parameters.

🤖 Prompt for AI Agents
In `@apps/pipeline/src/callO3.ts` around lines 36 - 39, The callGpt5 function
currently calls client.chat.completions.create (chatCompletion) but does not
implement the callModel<T> interface: it accepts schema and maxRetries but never
uses them; update callGpt5 to either (A) implement structured output handling by
using the provided schema to request and validate structured output (e.g.,
include a system/instruction to output JSON matching schema, parse the model
response into type T and validate against schema) and add retry logic around
client.chat.completions.create honoring the maxRetries parameter with
exponential backoff and proper error propagation, or (B) remove the unused
schema and maxRetries parameters and update the signature to match the intended
simple call; reference symbols: callGpt5, callModel<T>, schema, maxRetries, and
client.chat.completions.create (chatCompletion) so reviewers can locate and
verify the changes.

});
return chatCompletion.choices[0].message.content as string;
Expand Down
8 changes: 8 additions & 0 deletions apps/pipeline/src/helpers/logError.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
export function logError(contextMessage: string, err: unknown) {
if (err instanceof Error) {
console.error(`${contextMessage} ${err.message}`);
console.error(err.stack);
return;
}
console.error(`${contextMessage} ${String(err)}`);
}
16 changes: 16 additions & 0 deletions apps/pipeline/src/lib/domParser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import { JSDOM } from "jsdom";

let initialized = false;

export function ensureDomParser(): void {
if (typeof (globalThis as { DOMParser?: unknown }).DOMParser !== "undefined") {
return;
}
if (initialized) {
return;
}

const { window } = new JSDOM("");
(globalThis as { DOMParser: typeof window.DOMParser }).DOMParser = window.DOMParser;
initialized = true;
}
10 changes: 10 additions & 0 deletions apps/pipeline/src/lib/paragraphCount.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import {
countParagraphsFromChapterHtml,
type ParagraphCountOptions,
} from "@player/services/htmlNormalizer";
import { ensureDomParser } from "./domParser";

export function computeParagraphCount(html: string, options?: ParagraphCountOptions): number {
ensureDomParser();
return countParagraphsFromChapterHtml(html, options);
}
Loading
Loading