diff --git a/.gitignore b/.gitignore index 7220429..069d4c8 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ /.next/ /out/ +# fumadocs-mdx generated files +/.source/ + # production /build diff --git a/.source/browser.ts b/.source/browser.ts deleted file mode 100644 index 2bac410..0000000 --- a/.source/browser.ts +++ /dev/null @@ -1,12 +0,0 @@ -// @ts-nocheck -import { browser } from 'fumadocs-mdx/runtime/browser'; -import type * as Config from '../source.config'; - -const create = browser(); -const browserCollections = { - docs: create.doc("docs", {"page.mdx": () => import("../app/docs/page.mdx?collection=docs"), "getting-started/page.mdx": () => import("../app/docs/getting-started/page.mdx?collection=docs"), "getting-started/quick-start.mdx": () => import("../app/docs/getting-started/quick-start.mdx?collection=docs"), "features/saved-searches/user-guide.mdx": () => import("../app/docs/features/saved-searches/user-guide.mdx?collection=docs"), }), -}; -export default browserCollections; \ No newline at end of file diff --git a/.source/dynamic.ts b/.source/dynamic.ts deleted file mode 100644 index 43329c9..0000000 --- a/.source/dynamic.ts +++ /dev/null @@ -1,8 +0,0 @@ -// @ts-nocheck -import { dynamic } from 'fumadocs-mdx/runtime/dynamic'; -import * as Config from '../source.config'; - -const create = await dynamic(Config, {"environment":"next","outDir":".source","configPath":"source.config.ts"}, {"doc":{"passthroughs":["extractedReferences"]}}); \ No newline at end of file diff --git a/.source/server.ts b/.source/server.ts deleted file mode 100644 index 62a2f5d..0000000 --- a/.source/server.ts +++ /dev/null @@ -1,18 +0,0 @@ -// @ts-nocheck -import { default as __fd_glob_7 } from "../app/docs/features/saved-searches/_meta.json?collection=docs" -import { default as __fd_glob_6 } from "../app/docs/getting-started/_meta.json?collection=docs" -import { default as __fd_glob_5 } from "../app/docs/features/_meta.json?collection=docs" -import { default as __fd_glob_4 } from "../app/docs/_meta.json?collection=docs" -import * as __fd_glob_3 from "../app/docs/features/saved-searches/user-guide.mdx?collection=docs" -import * as __fd_glob_2 from "../app/docs/getting-started/quick-start.mdx?collection=docs" -import * as __fd_glob_1 from "../app/docs/getting-started/page.mdx?collection=docs" -import * as __fd_glob_0 from "../app/docs/page.mdx?collection=docs" -import { server } from 'fumadocs-mdx/runtime/server'; -import type * as Config from '../source.config'; - -const create = server({"doc":{"passthroughs":["extractedReferences"]}}); - -export const docs = await create.docs("docs", "app/docs", {"_meta.json": __fd_glob_4, "features/_meta.json": __fd_glob_5, "getting-started/_meta.json": __fd_glob_6, "features/saved-searches/_meta.json": __fd_glob_7, }, {"page.mdx": __fd_glob_0, "getting-started/page.mdx": __fd_glob_1, "getting-started/quick-start.mdx": __fd_glob_2, "features/saved-searches/user-guide.mdx": __fd_glob_3, }); \ No newline at end of file diff --git a/.source/source.config.mjs b/.source/source.config.mjs deleted file mode 100644 index 3d7be87..0000000 --- a/.source/source.config.mjs +++ /dev/null @@ -1,10 +0,0 @@ -// source.config.ts -import { defineConfig, defineDocs } from "fumadocs-mdx/config"; -var source_config_default = defineConfig(); -var docs = defineDocs({ - dir: "app/docs" -}); -export { - source_config_default as default, - docs -}; diff --git a/CLAUDE.md b/CLAUDE.md index 91ac52b..f11df4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,24 +62,111 @@ curl -X POST http://localhost:3000/api/jobs/generate-embeddings # See docs/GITHUB_ACTIONS_CONSOLIDATION.md for details ``` -## High-Level Architecture +## Core Architecture ### Application Structure **Next.js App Router Organization** - `/app` - Route pages with server/client component split - `/app/api` - API routes following RESTful conventions +- `/app/actions` - Server Actions for data mutations (2,173 lines across 7 files) - `/src/lib/services` - Business logic layer (stateless, composable) - `/src/components` - Reusable React components organized by feature - `/prisma` - Database schema and migrations - `/instrumentation.ts` - Server startup initialization (cron scheduler, WASM config) +### Server Actions Architecture + +**Overview:** +Next.js 15+ Server Actions are the primary pattern for data mutations in NeuReed. They provide type-safe, progressive-enhanced data operations that integrate seamlessly with React Server Components. + +**Benefits:** +- Type safety across client-server boundary +- Automatic request/response serialization +- Progressive enhancement (works without JavaScript) +- Direct function calls (no REST endpoint needed) +- Simplified error handling and validation + +**Action Modules (2,173 total lines):** +- [actions/articles.ts](app/actions/articles.ts) (431 lines) - Article CRUD, search, semantic search, summarization +- [actions/feeds.ts](app/actions/feeds.ts) (512 lines) - Feed CRUD, testing, refresh, bulk operations, health +- [actions/user-feeds.ts](app/actions/user-feeds.ts) (358 lines) - User subscriptions, OPML import/export +- [actions/saved-searches.ts](app/actions/saved-searches.ts) (289 lines) - Saved search CRUD, matching, insights +- [actions/categories.ts](app/actions/categories.ts) (247 lines) - Category management +- [actions/notifications.ts](app/actions/notifications.ts) (178 lines) - Notification operations +- [actions/user-preferences.ts](app/actions/user-preferences.ts) (158 lines) - User settings, LLM config + +**Key Pattern:** +```typescript +'use server' + +export async function updateFeedAction(feedId: string, data: UpdateFeedInput) { + // 1. Auth check + const session = await auth() + if (!session?.user?.id) throw new Error('Unauthorized') + + // 2. Validation + const validated = updateFeedSchema.parse(data) + + // 3. Service layer + const result = await updateFeed(feedId, validated, session.user.id) + + // 4. Revalidation + revalidatePath('/feeds-management') + + return result +} +``` + +**Integration with React Query:** +```typescript +const { mutate } = useMutation({ + mutationFn: (data) => updateFeedAction(feedId, data), + onSuccess: () => queryClient.invalidateQueries(['feeds']) +}) +``` + +**When to use Server Actions vs API Routes:** +- **Server Actions**: Form submissions, data mutations, component actions, user-initiated operations +- **API Routes**: Webhooks, external integrations, non-React clients, cron job endpoints + +### API Route Conventions + +**Unified Handler Pattern** ([src/lib/api-handler.ts](src/lib/api-handler.ts)): +```typescript +export const POST = createHandler( + async ({ body, session, params, query }) => { + // Business logic here + return { data: result }; // Automatically wrapped in apiResponse() + }, + { + bodySchema: z.object({ ... }), // Zod validation + querySchema: z.object({ ... }), // Optional query param validation + requireAuth: true, // Enforce authentication + } +); +``` + +**Response Format:** +- Success: `{ data: any, message?: string }` +- Error: `{ error: string, details?: any }` +- Status codes: 200 (success), 400 (validation), 401 (auth), 404 (not found), 500 (server error) + +**API Organization:** +- `/api/articles/*` - Article operations and search +- `/api/feeds/*` - Feed management +- `/api/user/*` - User-specific data (preferences, subscriptions, notifications) +- `/api/admin/*` - Administrative operations +- `/api/jobs/*` - Manual job triggers +- `/api/saved-searches/*` - Saved search CRUD, matching, templates, insights + ### Service Layer Pattern The codebase follows a clear separation of concerns: - **API routes** are thin controllers that handle HTTP concerns (auth, validation, responses) +- **Server Actions** handle form submissions and mutations with revalidation - **Services** contain all business logic and are highly composable -- **Database access** goes through services, never directly in API routes +- **Database access** goes through services, never directly in API routes or actions Example service dependencies: ``` @@ -99,6 +186,7 @@ feed-refresh-service.ts - **Articles** → read_articles (per-user read tracking) - **Users** → user_patterns (learned preferences via TF-IDF) - **Users** → user_notifications (in-app notifications) +- **Feeds** → feed_error_log (health tracking audit trail) **Important: pgvector Operations** - Prisma doesn't natively support pgvector, so raw SQL is used for vector operations @@ -107,62 +195,53 @@ feed-refresh-service.ts - Queries: `prisma.$queryRaw`SELECT ... ORDER BY embedding <=> $1::vector`` - HNSW index enables fast similarity search -### Cascade Settings Pattern - -Settings cascade from most specific to most general: -1. Feed-specific setting (highest priority) -2. Category setting -3. User default setting -4. System default setting (lowest priority) - -Applied to: refresh intervals, article retention periods, max articles per feed. - -Implementation: [src/lib/services/feed-settings-cascade.ts](src/lib/services/feed-settings-cascade.ts) - -### Cron Job System +### Authentication Architecture (NextAuth.js v5) -**Initialization Flow:** -1. `instrumentation.ts` runs on server startup -2. Checks `ENABLE_CRON_JOBS` environment variable -3. Initializes scheduler with all job definitions -4. Jobs run on schedule OR via manual trigger +**Configuration** ([src/lib/auth.ts](src/lib/auth.ts)): +- Prisma adapter for database session storage +- Dynamic OAuth providers (Google, GitHub, Generic OAuth2) +- JWT strategy with secure HTTP-only cookies +- Custom callbacks add user ID to JWT token +- Automatic default feed subscription on user creation -**Job Execution Pattern:** -- All jobs wrapped with `createJobExecutor()` for tracking -- Creates `CronJobRun` record in database (status, duration, logs) -- In-memory lock prevents duplicate runs -- Logs captured via `JobLogger` and stored in database -- Admin can view history and trigger jobs manually +**Authorization Pattern:** +```typescript +// In API routes via createHandler +export const POST = createHandler( + async ({ body, session }) => { + const userId = session!.user.id; // session available, requireAuth ensures non-null + // ... business logic + }, + { bodySchema: mySchema, requireAuth: true } +); -**Key Jobs:** -- `feed-refresh-job.ts`: Refreshes feeds every 30 minutes (configurable), creates notifications for users -- `cleanup-job.ts`: Removes old articles daily at 3 AM -- Pattern decay job: Time-based decay of user preferences +// In Server Actions +const session = await auth() +if (!session?.user?.id) throw new Error('Unauthorized') +``` -### Notification System +**Multi-Tenancy:** +- All data scoped to users (feeds, articles, patterns, preferences) +- Services accept `userId` parameter +- Database queries filter by `userId` -**Architecture:** -- In-app notifications stored in `user_notifications` table -- Notifications created automatically for feed refresh events -- Real-time updates via React Query polling (30s interval) -- Toast notifications for new items +### Caching Strategy -**Notification Types:** -- `feed_refresh`: Feed update notifications with stats (new/updated articles, embeddings, cleanup) -- `info`, `warning`, `error`, `success`: General notifications +**Redis-Based Caching** ([cache-service.ts](src/lib/cache/cache-service.ts)): +- Cache-aside pattern: `cacheGetOrSet()` +- Short TTLs balance freshness and performance: + - Article scores: 1 hour + - Feed data: 5 minutes + - Search query AST: 24 hours +- Pattern-based invalidation: `cache:user:{userId}:*` +- Statistics tracking (hits, misses, errors) -**Service Layer** ([notification-service.ts](src/lib/services/notification-service.ts)): -- `createNotification()`: Create any notification -- `createFeedRefreshNotification()`: Specialized for feed updates -- `getUserNotifications()`: Fetch with pagination -- `markNotificationAsRead()`: Mark single notification as read -- `markAllNotificationsAsRead()`: Bulk mark as read -- `cleanupOldNotifications()`: Keep only last 100 per user +**When to Invalidate:** +- User feedback → clear article scores for that user +- Feed refresh → clear feed data +- Settings change → clear affected cached data -**UI Components:** -- `NotificationBell`: Header component with unread count badge -- Dropdown panel with notification list -- Toast notifications for new items with rich metadata display +## AI & Machine Learning Features ### Embedding & Semantic Search Flow @@ -188,6 +267,62 @@ Implementation: [src/lib/services/feed-settings-cascade.ts](src/lib/services/fee - Uses article's existing embedding (no query needed) - Fast lookup via: `ORDER BY embedding <=> (SELECT embedding FROM articles WHERE id = $1)` +### Article Summarization & LLM Integration + +**Overview:** +On-demand article summarization with key point extraction and topic detection. Uses LLM providers with configurable cascade and comprehensive cost tracking. + +**Service Layer** ([summarization-service.ts](src/lib/services/summarization-service.ts)): +- `summarizeArticle(articleId, userId, options)` - Generate article summary +- `extractKeyPoints(articleId, userId)` - Extract 3-5 key takeaways +- `detectTopics(articleId, userId)` - Identify main topics/themes +- `batchSummarize(articleIds, userId)` - Batch processing with concurrency control +- `getSummarizationCosts(userId, period)` - Cost analytics + +**Cost Tracking** ([summarization-cost-tracker.ts](src/lib/services/summarization-cost-tracker.ts)): +- Redis-based cost tracking per user +- OpenAI pricing: $0.150/1M input, $0.600/1M output tokens +- Daily/monthly aggregates +- Admin cost visibility dashboard + +**Database Schema:** +- `articles.summary` (text) - Generated summary +- `articles.keyPoints` (string[]) - Extracted key points +- `articles.topics` (string[]) - Detected topics +- `articles.summarizedAt` (DateTime) - Generation timestamp + +**LLM Configuration Cascade:** +``` +1. User preferences (user_llm_config table) + └─ Encrypted API keys per user + └─ Model selection (summary/embedding/digest) +2. Admin settings (admin_llm_settings table) + └─ System-wide defaults + └─ Provider enable/disable +3. Environment variables + └─ OPENAI_API_KEY, OLLAMA_BASE_URL +``` + +**Supported Providers:** +- **OpenAI**: gpt-4o-mini (default), gpt-4o (premium) +- **Ollama**: llama3.1, qwen2.5 (self-hosted, free) + +**API Endpoints:** +- `POST /api/articles/[id]/summarize` - Generate summary +- `GET /api/articles/[id]/summary` - Retrieve cached summary +- `GET /api/articles/[id]/keypoints` - Get key points +- `GET /api/articles/topics` - Topic analysis +- `GET /api/admin/summarization/costs` - Cost analytics + +**Server Actions:** +- `generateArticleSummaryAction(articleId)` - Trigger summarization +- `getArticleSummaryAction(articleId)` - Retrieve with auth + +**Caching Strategy:** +- Database persistence (articles table) +- Redis cache (1 hour TTL) +- Revalidate on article update + ### Personalization System **Pattern Detection** ([pattern-detection-service.ts](src/lib/services/pattern-detection-service.ts)): @@ -250,137 +385,697 @@ Groups: "(AI, ML) +ethics" # Grouped expressions - Compound database indexes for common query patterns - Target throughput: 1000+ articles/minute -**Testing:** -- 60+ unit tests covering parser, execution, matching, templates -- Integration tests for end-to-end workflows -- Performance benchmarks validated - **Documentation:** - User Guide: [docs/USER_GUIDE_SAVED_SEARCHES.md](docs/USER_GUIDE_SAVED_SEARCHES.md) - Feature Spec: [docs/FEATURE_SAVED_SEARCHES.md](docs/FEATURE_SAVED_SEARCHES.md) - Performance Guide: [docs/SAVED_SEARCH_PERFORMANCE_GUIDE.md](docs/SAVED_SEARCH_PERFORMANCE_GUIDE.md) - Testing Guide: [docs/TESTING_SAVED_SEARCHES.md](docs/TESTING_SAVED_SEARCHES.md) -### Authentication Architecture (NextAuth.js v5) +## Feed Management -**Configuration** ([src/lib/auth.ts](src/lib/auth.ts)): -- Prisma adapter for database session storage -- Dynamic OAuth providers (Google, GitHub, Generic OAuth2) -- JWT strategy with secure HTTP-only cookies -- Custom callbacks add user ID to JWT token -- Automatic default feed subscription on user creation +### Feed Health & Status Tracking System + +**Overview:** +Comprehensive feed reliability monitoring with automatic failure tracking, auto-disable for problematic feeds, and historical error logging for debugging. + +**Health States:** +``` +healthy → 0 consecutive failures +warning → 1-2 consecutive failures +error → 3+ consecutive failures +disabled → Auto-disabled at threshold (default: 10) or manually disabled +``` + +**Database Schema:** + +New `feeds` table fields: +- `healthStatus` (string, default: "healthy") +- `consecutiveFailures` (int, default: 0) +- `lastSuccessfulFetch` (DateTime?) +- `autoDisableThreshold` (int, default: 10) +- `notifyOnError` (boolean, default: false) +- `httpStatus` (int?) - Last HTTP status code +- `redirectUrl` (string?) - Redirect destination if feed moved + +New `feed_error_log` table: +- `id`, `feedId`, `timestamp`, `errorType`, `errorMessage` +- `httpStatus`, `details` (JSON), `resolved` (boolean) +- Cascade delete with feeds +- Full audit trail of all errors + +**Service Layer** ([feed-health-service.ts](src/lib/services/feed-health-service.ts)): +- `getFeedHealth(feedId)` - Get health status +- `getBulkFeedHealth(feedIds)` - Batch health check (single query) +- `recordFeedSuccess(feedId)` - Mark successful fetch, reset failures +- `recordFeedFailure(feedId, errorType, message, httpStatus)` - Log failure, update health +- `resetFeedHealth(feedId)` - Atomic reset to healthy state +- `getUnhealthyFeeds(userId)` - Get warning/error/disabled feeds for user +- `enableFeed(feedId)` - Manually enable disabled feed +- `disableFeed(feedId)` - Manually disable feed +- `updateAutoDisableThreshold(feedId, threshold)` - Configure threshold +- `getFeedErrorLogs(feedId, limit)` - Retrieve error history +- `clearFeedErrorLogs(feedId)` - Clear error logs + +**Auto-Disable Logic:** +```typescript +if (feed.consecutiveFailures >= feed.autoDisableThreshold) { + await prisma.feed.update({ + where: { id: feedId }, + data: { healthStatus: 'disabled' } + }) + + if (feed.notifyOnError) { + await createNotification({ + userId, + type: 'feed_health_disabled', + title: `Feed auto-disabled: ${feed.title}`, + message: `After ${threshold} consecutive failures` + }) + } +} +``` + +**Integration with Feed Refresh Job:** +- Job respects `healthStatus` (skips disabled feeds) +- Calls `recordFeedSuccess()` on successful fetch +- Calls `recordFeedFailure()` on error +- Error types: FETCH_ERROR, PARSE_ERROR, TIMEOUT, HTTP_ERROR + +**API Endpoints:** +- `PUT /api/feeds/[id]/status` - Enable/disable feed +- `POST /api/feeds/bulk/status` - Bulk enable/disable +- `GET /api/feeds/[id]/health` - Get health status +- `POST /api/feeds/bulk-health` - Batch health retrieval +- `GET /api/feeds/unhealthy` - User's unhealthy feeds + +**Server Actions:** +- `toggleFeedStatusAction(feedId, enabled)` - Single feed toggle +- `bulkToggleFeedStatusAction(feedIds, enabled)` - Bulk toggle +- `resetFeedHealthAction(feedId)` - Reset to healthy + +**UI Integration:** +- FeedDetailsView "Quality & Health" tab: + - Health status badge with color coding + - Consecutive failure count display + - Auto-disable threshold configuration + - Last error message viewer + - Error notification toggle + - View error history button +- OverviewView feeds table: + - Health status badge column (Disabled/Error/Warning/Active) + - Bulk enable/disable buttons for selected feeds + +**Query Hooks:** +- `useFeedHealth(feedId)` - Real-time health status +- `useToggleFeedStatus()` - Single feed mutation +- `useBulkToggleFeedStatus()` - Bulk mutation with partial failure handling + +### Bulk Operations System + +**Overview:** +Multi-feed operations for efficiency with partial failure tolerance and user authorization verification. + +**Service Layer** ([bulk-operations-service.ts](src/lib/services/bulk-operations-service.ts)): +- `bulkUpdateFeedCategory(feedIds, categoryId, userId)` - Assign category to multiple feeds +- `bulkUpdateFeedTags(feedIds, tags, mode, userId)` - Manage tags (modes: 'add', 'remove', 'replace') +- `bulkUpdateFeedSettings(feedIds, settings, userId)` - Update settings (refreshInterval, maxArticles, maxArticleAge) +- `bulkDeleteFeeds(feedIds, userId)` - Delete multiple feeds with cascade +- `bulkRefreshFeeds(feedIds, userId)` - Trigger refresh for multiple feeds + +**Response Format:** +```typescript +interface BulkUpdateResult { + success: number + failed: number + results: Array<{ + feedId: string + success: boolean + error?: string + }> +} +``` **Authorization Pattern:** ```typescript -// In API routes via createHandler -export const POST = createHandler( - async ({ body, session }) => { - const userId = session!.user.id; // session available, requireAuth ensures non-null - // ... business logic - }, - { bodySchema: mySchema, requireAuth: true } -); +// Verify all feeds belong to user +const userFeedIds = await prisma.userFeed.findMany({ + where: { userId, feedId: { in: feedIds } }, + select: { feedId: true } +}).then(feeds => feeds.map(f => f.feedId)) + +const unauthorized = feedIds.filter(id => !userFeedIds.includes(id)) +if (unauthorized.length > 0) { + throw new Error('Unauthorized feeds') +} ``` -**Multi-Tenancy:** -- All data scoped to users (feeds, articles, patterns, preferences) -- Services accept `userId` parameter -- Database queries filter by `userId` +**API Endpoints:** +- `POST /api/feeds/bulk/category` - Bulk category update +- `POST /api/feeds/bulk/tags` - Bulk tag management +- `POST /api/feeds/bulk/settings` - Bulk settings update +- `POST /api/feeds/bulk/delete` - Bulk delete +- `POST /api/feeds/bulk/refresh` - Bulk refresh -### Caching Strategy +**Server Actions:** +- `bulkUpdateFeedCategoryAction(feedIds, categoryId)` +- `bulkUpdateFeedTagsAction(feedIds, tags, mode)` +- `bulkDeleteFeedsAction(feedIds)` -**Redis-Based Caching** ([cache-service.ts](src/lib/cache/cache-service.ts)): -- Cache-aside pattern: `cacheGetOrSet()` -- Short TTLs balance freshness and performance: - - Article scores: 1 hour - - Feed data: 5 minutes -- Pattern-based invalidation: `cache:user:{userId}:*` -- Statistics tracking (hits, misses, errors) +**UI Integration:** +- OverviewView bulk selection mode with checkboxes +- Bulk action toolbar (appears when feeds selected) +- Operations: Enable, Disable, Delete, Edit Category, Edit Tags +- Loading states during bulk operations -**When to Invalidate:** -- User feedback → clear article scores for that user -- Feed refresh → clear feed data -- Settings change → clear affected cached data +**Query Hooks:** +- `useBulkUpdateCategory()` - Category mutation with optimistic updates +- `useBulkUpdateTags()` - Tags mutation +- `useBulkDeleteFeeds()` - Delete mutation with confirmation -### API Route Conventions +### OPML Import/Export -**Unified Handler Pattern** ([src/lib/api-handler.ts](src/lib/api-handler.ts)): +**Overview:** +Full OPML 2.0 standard support for importing/exporting feed subscriptions with category preservation. Essential for migration and backup functionality. + +**Service Layer** ([opml-service.ts](src/lib/services/opml-service.ts)): +- `parseOPML(xmlString)` - Parse OPML XML to structured data +- `generateOPML(userId)` - Export user's feeds to OPML format +- `validateOPMLStructure(data)` - Validate structure before import +- `importOPMLFeeds(userId, opmlData)` - Import with automatic category creation + +**OPML Structure:** +```xml + + + + NeuReed Feeds + 2025-01-15T10:30:00Z + + + + + + + +``` + +**Import Flow:** +1. User uploads OPML file or provides XML string +2. Parse XML and validate OPML structure +3. Create missing categories with color assignment +4. Create or update feeds (dedupe by URL) +5. Subscribe user to imported feeds +6. Return import summary (created, updated, failed counts) + +**Export Flow:** +1. Fetch user's feed subscriptions with categories +2. Group feeds by category +3. Generate OPML XML with proper nesting +4. Return as downloadable file with timestamp + +**API Endpoints:** +- `POST /api/user/opml/import` - Import OPML file (multipart/form-data) +- `GET /api/user/opml/export` - Export as OPML (application/xml) + +**Server Actions:** +- `importOPMLAction(opmlXml)` - Handle import with validation +- `exportOPMLAction()` - Generate export for current user + +**UI Integration:** +- OverviewView header with Import/Export buttons +- File drop zone for drag-and-drop OPML import +- Download button triggers export +- Import summary modal showing results + +**Query Hooks:** +- `useImportOPML()` - Import mutation with progress tracking +- `useExportOPML()` - Export query with file download + +### Cron Job System + +**Initialization Flow:** +1. `instrumentation.ts` runs on server startup +2. Checks `ENABLE_CRON_JOBS` environment variable +3. Initializes scheduler with all job definitions +4. Jobs run on schedule OR via manual trigger + +**Job Execution Pattern:** +- All jobs wrapped with `createJobExecutor()` for tracking +- Creates `CronJobRun` record in database (status, duration, logs) +- In-memory lock prevents duplicate runs +- Logs captured via `JobLogger` and stored in database +- Admin can view history and trigger jobs manually + +**Key Jobs:** +- `feed-refresh-job.ts`: Refreshes feeds every 30 minutes (configurable), creates notifications, respects health status +- `cleanup-job.ts`: Removes old articles daily at 3 AM based on retention settings +- Pattern decay job: Time-based decay of user preferences (10% per 30 days) + +### Cascade Settings Pattern + +Settings cascade from most specific to most general: +1. Feed-specific setting (highest priority) +2. Category setting +3. User default setting +4. System default setting (lowest priority) + +Applied to: refresh intervals, article retention periods, max articles per feed. + +Implementation: [src/lib/services/feed-settings-cascade.ts](src/lib/services/feed-settings-cascade.ts) + +## User Features + +### Notification System + +**Architecture:** +- In-app notifications stored in `user_notifications` table +- Notifications created automatically for feed refresh events and feed health issues +- Real-time updates via React Query polling (30s interval) +- Toast notifications for new items with rich metadata display + +**Notification Types:** +- `feed_refresh`: Feed update notifications with stats (new/updated articles, embeddings, cleanup) +- `feed_health_error`: Feed health degraded to error state +- `feed_health_disabled`: Feed auto-disabled after threshold failures +- `info`, `warning`, `error`, `success`: General notifications + +**Service Layer** ([notification-service.ts](src/lib/services/notification-service.ts)): +- `createNotification()`: Create any notification +- `createFeedRefreshNotification()`: Specialized for feed updates +- `getUserNotifications()`: Fetch with pagination +- `markNotificationAsRead()`: Mark single notification as read +- `markAllNotificationsAsRead()`: Bulk mark as read +- `cleanupOldNotifications()`: Keep only last 100 per user + +**Feed Health Integration:** +- Notifications created when feed health changes to error or disabled +- Respects `notifyOnError` setting per feed +- Batched notifications (max 1 per feed per hour) +- Integration with `recordFeedFailure()` in health service + +**UI Components:** +- `NotificationBell`: Header component with unread count badge +- Dropdown panel with notification list and actions +- Toast notifications for new items + +### Offline Support & Client-Side Caching + +**Overview:** +LocalStorage-based caching with TTL management for offline access to saved search results and article data. + +**Service Layer** ([offline-cache-service.ts](src/lib/services/offline-cache-service.ts)): +- `setCacheItem(key, data, ttl)` - Store with expiration time +- `getCacheItem(key)` - Retrieve if not expired +- `removeCacheItem(key)` - Delete specific entry +- `clearExpiredCache()` - Cleanup old entries +- `getCacheStats()` - Storage size and entry count +- `isOnline()` - Current network status + +**Cache Structure:** ```typescript -export const POST = createHandler( - async ({ body, session, params, query }) => { - // Business logic here - return { data: result }; // Automatically wrapped in apiResponse() - }, - { - bodySchema: z.object({ ... }), // Zod validation - querySchema: z.object({ ... }), // Optional query param validation - requireAuth: true, // Enforce authentication - } -); +interface CacheEntry { + data: any + timestamp: number + ttl: number + version: string +} + +interface CacheMetadata { + version: '1.0' + lastSync: number + isOnline: boolean +} ``` -**Response Format:** -- Success: `{ data: any, message?: string }` -- Error: `{ error: string, details?: any }` -- Status codes: 200 (success), 400 (validation), 401 (auth), 404 (not found), 500 (server error) +**Usage Pattern:** +```typescript +// Cache saved search results +await setCacheItem( + `saved-search:${searchId}:results`, + results, + 5 * 60 * 1000 // 5 minute TTL +) + +// Retrieve with fallback to server +const cached = await getCacheItem(`saved-search:${searchId}:results`) +if (!cached) { + const fresh = await fetchFromServer() + await setCacheItem(key, fresh, ttl) +} +``` -**API Organization:** -- `/api/articles/*` - Article operations and search -- `/api/feeds/*` - Feed management -- `/api/user/*` - User-specific data (preferences, subscriptions, notifications) -- `/api/admin/*` - Administrative operations -- `/api/jobs/*` - Manual job triggers -- `/api/saved-searches/*` - Saved search CRUD, matching, templates, insights +**Storage Management:** +- Quota monitoring (LocalStorage ~5-10MB limit) +- LRU eviction when approaching storage limit +- Automatic cleanup of expired entries on app load +- Version-based cache invalidation + +**Online/Offline Handling:** +```typescript +window.addEventListener('online', () => { + syncCachedSearches() + updateCacheMetadata({ isOnline: true }) +}) + +window.addEventListener('offline', () => { + showOfflineIndicator() + updateCacheMetadata({ isOnline: false }) +}) +``` + +**Integration Points:** +- Saved searches: Cache query results for offline access +- Article lists: Cache for offline viewing +- User preferences: Local copy for instant access + +### Default Feeds for New Users + +New users are automatically subscribed to a curated set of 9 feeds covering: +- **Technology**: TechCrunch, The Verge, Hacker News +- **News**: BBC News +- **Science**: Nature, Science Daily +- **Positive News**: Good News Network, Positive News +- **Satire**: The Onion + +**Implementation:** +- Feeds created on-demand in [src/lib/services/default-feeds-service.ts](src/lib/services/default-feeds-service.ts) +- Subscription happens in `createUser` event in auth.ts +- Categories and feeds auto-created if missing +- Idempotent (safe to run multiple times) + +See [docs/DEFAULT_FEEDS.md](docs/DEFAULT_FEEDS.md) for full documentation. + +## Developer Guide + +### React Hooks & Client State + +**Custom Hooks** ([src/hooks/](src/hooks/)): + +1. **useAuth** - Session management with loading states + ```typescript + const { user, loading, isAuthenticated } = useAuth() + ``` + +2. **useUnsavedChanges** - Form change tracking with warnings + ```typescript + const { hasChanges, revert, confirmClose } = useUnsavedChanges(formData, initialData) + ``` + +3. **useFileDrop** - Drag & drop with file validation + ```typescript + const { isDragging, files, getRootProps } = useFileDrop({ + accept: '.opml', + maxSize: 5 * 1024 * 1024 + }) + ``` + +4. **useFormChanges** - Detailed field-level change tracking + ```typescript + const { changes, isDirty, reset } = useFormChanges(formState) + ``` + +5. **useConfirmation** - Confirmation dialogs with promise-based API + ```typescript + const { confirm, ConfirmDialog } = useConfirmation() + const confirmed = await confirm('Delete feed?') + ``` + +6. **useFeedNavigation** - URL-based feed navigation + ```typescript + const { currentFeedId, navigateToFeed } = useFeedNavigation() + ``` + +7. **useViewNavigation** - View state management via URL params + ```typescript + const { currentView, setView } = useViewNavigation() + ``` + +8. **useMobileMenu** - Mobile menu state management + ```typescript + const { isOpen, toggle, close } = useMobileMenu() + ``` + +9. **useDebounce** - Debounced values for search inputs + ```typescript + const debouncedSearch = useDebounce(searchTerm, 300) + ``` + +**React Query Hooks** ([src/hooks/queries/](src/hooks/queries/)): + +**Feed Management:** +- `useFeeds()` - All feeds with health status +- `useUserFeeds()` - User's subscribed feeds +- `useFeedHealth(feedId)` - Real-time health status +- `useUpdateFeed()` - Update mutation with validation +- `useDeleteFeed()` - Delete mutation with cascade +- `useToggleFeedStatus()` - Enable/disable single feed +- `useBulkToggleFeedStatus()` - Bulk enable/disable + +**Article Management:** +- `useArticles(filters)` - Article list with pagination +- `useArticle(id)` - Single article with relations +- `useArticleSummary(id)` - Cached summary data +- `useSearchArticles()` - Full-text search mutation +- `useSemanticSearch()` - Vector similarity search + +**Saved Searches:** +- `useSavedSearches()` - User's saved searches +- `useSavedSearchMatches(id)` - Matching articles +- `useCreateSavedSearch()` - Create with validation +- `useRematchSavedSearch()` - Re-run matching algorithm + +**Notifications:** +- `useNotifications()` - Unread notifications with polling +- `useMarkNotificationRead()` - Mark as read mutation + +**Standard Hook Pattern:** +```typescript +export function useUpdateFeed() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: (data) => updateFeedAction(data.id, data), + onSuccess: () => { + queryClient.invalidateQueries(['feeds']) + queryClient.invalidateQueries(['userFeeds']) + }, + onError: (error) => { + toast.error(error.message) + } + }) +} +``` + +### Feed Management UI Architecture + +**URL Navigation Pattern:** +Query parameter-based navigation for natural browser back/forward support: +``` +/feeds-management?view=overview # Default view +/feeds-management?view=feed&id=123 # Feed details +/feeds-management?view=category&id=456 # Category settings +/feeds-management?view=bulk-edit&ids=1,2,3 # Bulk edit + +Modal query params: +?modal=import-opml +?modal=export-opml +?modal=create-category +?modal=bulk-edit +``` + +**Component Hierarchy:** +``` +FeedsManagementPage +├─ FeedsManagementLayout (header, navigation) +├─ OverviewView (default view) +│ ├─ StatisticsPanel (4 stat cards) +│ ├─ FeedsTable (with bulk selection) +│ └─ CategoriesList +├─ FeedDetailsView (7-tab interface) +│ ├─ BasicSettingsTab +│ ├─ UpdateRefreshTab +│ ├─ ContentProcessingTab +│ ├─ ConnectionSettingsTab +│ ├─ QualityHealthTab (health monitoring) +│ ├─ PresentationTab +│ └─ AdvancedTab +├─ CategorySettingsView +└─ BulkEditView +``` + +**FeedDetailsView 7-Tab Interface:** +1. **Basic Settings** - Name, URL, categories, tags, enable/disable toggle +2. **Update & Refresh** - Fetch intervals, timeout, retry settings, backoff strategy +3. **Content Processing** - Article retention, max articles, extraction method, content filters +4. **Connection Settings** - Authentication, custom headers, proxy, SSL verification +5. **Quality & Health** - Health status badge, failure count, error logs, auto-disable config, notifications +6. **Presentation** - Feed icon, excerpt length, display format (card/list/compact) +7. **Advanced** - Response caching, conditional GET, redirect handling, raw feed viewer + +**Data Flow:** +``` +URL query params (browser) + ↓ +useFeedNavigation() / useViewNavigation() + ↓ +Conditional view rendering + ↓ +User interaction (form submission) + ↓ +Server Action call + ↓ +Service layer + database update + ↓ +revalidatePath() / revalidateTag() + ↓ +React Query cache invalidation + ↓ +Optimistic UI updates +``` + +**Modal System:** +- URL-driven (state preserved on page refresh) +- Components lazy-loaded for performance +- Backdrop click and Escape key to close +- Accessible keyboard navigation + +**File Locations:** +- Page: [app/feeds-management/page.tsx](app/feeds-management/page.tsx) +- Views: [app/feeds-management/components/views/](app/feeds-management/components/views/) +- Forms: [app/feeds-management/components/forms/](app/feeds-management/components/forms/) +- Modals: [app/feeds-management/components/modals/](app/feeds-management/components/modals/) -## Important Development Notes +### Important Development Notes -### Database Migrations (from .cursorrules) +#### Database Migrations - **ALWAYS** use `npx prisma migrate dev --name ` for schema changes - **NEVER** use `prisma db push` in development (only for prototyping) - Test migrations locally before committing - Regeneration of Prisma Client happens automatically after migrations - **Important:** Prisma CLI is in `dependencies` (not `devDependencies`) to ensure it's available in CI/CD and Docker builds -### Working with pgvector +#### Working with pgvector - Prisma doesn't support vector types natively - Use raw SQL for vector operations (see examples in semantic-search-service.ts) - Vector dimensions: 384 (local/BGE-small) or 1536 (OpenAI) - HNSW index requires periodic REINDEX for optimal performance -### Security Considerations (from .cursorrules) +#### Security Considerations - Always sanitize HTML content (use `he.decode()` for entities) - Validate URLs before fetching (feed-parser.ts has SSRF protection) - User inputs validated with Zod schemas - Sensitive data (API keys, cookies) encrypted via encryption-service.ts - Never expose internal errors to users -### Type Safety +#### Type Safety - Environment variables validated via `@t3-oss/env-nextjs` in [src/env.ts](src/env.ts) - Prisma generates TypeScript types for all models - Zod schemas for runtime validation - Use `satisfies` for type narrowing where appropriate -### Content Extraction +#### Working with Server Actions + +**Best Practices:** +1. Always use `'use server'` directive at top of file +2. Validate inputs with Zod schemas before processing +3. Check authentication early in the function +4. Keep business logic in service layer (actions are thin wrappers) +5. Use `revalidatePath()` or `revalidateTag()` for cache invalidation +6. Handle errors gracefully without leaking internal details + +**Error Handling Pattern:** +```typescript +try { + const validated = schema.parse(data) + const result = await serviceFunction(validated) + revalidatePath('/path') + return { success: true, data: result } +} catch (error) { + if (error instanceof z.ZodError) { + return { success: false, error: 'Validation failed', details: error.errors } + } + console.error('Action failed:', error) + return { success: false, error: 'Operation failed' } +} +``` + +**Testing Server Actions:** +```typescript +import { updateFeedAction } from '@/app/actions/feeds' + +// Actions are just async functions - call directly in tests +const result = await updateFeedAction('feed-123', { + title: 'New Title' +}) +``` + +**Common Revalidation Paths:** +- `/feeds-management` - Feed list and details updates +- `/articles` - Article list updates +- `/saved-searches` - Saved search updates +- `/admin/dashboard` - Admin data changes + +#### LLM Configuration + +**Provider Selection:** +- **OpenAI**: Cloud-based, best quality, costs per token +- **Ollama**: Self-hosted, free, requires local installation + +**Configuration Cascade:** +``` +User LLM Config (highest priority) + ↓ if not configured +Admin LLM Settings + ↓ if not configured +Environment Variables (lowest priority) +``` + +**Per-User Overrides:** +Users can configure their own: +- Provider preference (OpenAI vs Ollama) +- Model selection (summary/embedding/digest) +- Personal API keys (encrypted in database) +- Usage cost limits + +**Admin System Config:** +- Default models for all users +- Provider enable/disable flags +- System-wide API keys +- Rate limiting settings + +**Model Recommendations:** +- **Summarization**: gpt-4o-mini (OpenAI) or llama3.1 (Ollama) +- **Embeddings**: text-embedding-3-small (OpenAI) or bge-small-en-v1.5 (local) +- **Digest**: gpt-4o (OpenAI) or qwen2.5 (Ollama) + +**Environment Variables:** +- `OPENAI_API_KEY` - OpenAI authentication +- `OLLAMA_BASE_URL` - Ollama server URL (default: http://localhost:11434) + +#### Content Extraction - Multiple strategies: Readability (fast), Playwright (for JS-rendered content) - Cookie-based authentication for paywalled feeds - Per-feed extraction settings with merge strategies - Timeouts prevent hanging on slow sites -### Cost Management +#### Cost Management - Embedding costs tracked via embedding-cost-tracker.ts +- Summarization costs tracked via summarization-cost-tracker.ts - User-specific LLM configurations allow cost control - Admin can enable/disable providers system-wide - Local embeddings available as zero-cost alternative -### Job Monitoring +#### Job Monitoring - All cron jobs logged in `CronJobRun` table - View history in admin dashboard: [/admin/dashboard](http://localhost:3000/admin/dashboard) - Manual triggers via API for debugging - Logs captured and stored with each run -## Common Patterns to Follow +### Common Patterns to Follow -### Adding a New Service +#### Adding a New Service 1. Create file in `/src/lib/services/-service.ts` 2. Export functions (not classes) for composability 3. Accept dependencies as parameters (no global state) @@ -388,75 +1083,130 @@ export const POST = createHandler( 5. Add detailed JSDoc comments for public functions 6. Handle errors with try-catch and meaningful messages -### Adding a New API Route +#### Adding a New API Route 1. Create route in `/app/api//route.ts` 2. Use `createHandler()` wrapper from api-handler.ts 3. Define Zod schema for request validation 4. Call service layer for business logic 5. Return structured response (don't throw errors to client) -### Adding a New Cron Job +#### Adding a Server Action +1. Create or update action file in `/app/actions/.ts` +2. Add `'use server'` directive at top of file +3. Define Zod schema for input validation +4. Implement action function with auth check +5. Call service layer for business logic +6. Use `revalidatePath()` to invalidate cached data +7. Create React Query hook for client integration + +**Example:** +```typescript +// app/actions/feeds.ts +'use server' + +export async function updateFeedAction( + feedId: string, + data: UpdateFeedInput +) { + const session = await auth() + if (!session?.user?.id) throw new Error('Unauthorized') + + const validated = updateFeedSchema.parse(data) + const result = await updateFeed(feedId, validated, session.user.id) + + revalidatePath('/feeds-management') + return result +} + +// hooks/queries/use-feeds.ts +export function useUpdateFeed() { + return useMutation({ + mutationFn: (data) => updateFeedAction(data.id, data), + onSuccess: () => queryClient.invalidateQueries(['feeds']) + }) +} +``` + +#### Adding a New Cron Job 1. Create job file in `/src/lib/jobs/-job.ts` 2. Export function wrapped with `createJobExecutor()` 3. Use `JobLogger` for logging 4. Register in `scheduler.ts` 5. Add environment variable for schedule (optional) -### Adding Database Migrations +#### Adding Database Migrations 1. Modify `prisma/schema.prisma` 2. Run `npx prisma migrate dev --name ` 3. Test migration with `npm run db:reset && npm run db:seed` 4. Commit both schema.prisma and migration files -## Troubleshooting +### Troubleshooting -### Database Connection Issues +#### Database Connection Issues - Verify Docker is running: `docker ps` - Check logs: `docker-compose logs postgres` - Database runs on port 5433 (not default 5432) to avoid conflicts -### Embedding Generation Fails +#### Embedding Generation Fails - Check provider configuration in user preferences or admin settings - Verify API keys are set (OpenAI) or WASM is configured (local) - Look at job logs in admin dashboard - Check Redis connection for cost tracking -### Cron Jobs Not Running +#### Cron Jobs Not Running - Verify `ENABLE_CRON_JOBS=true` in environment - Check server logs for scheduler initialization - Ensure no errors in `instrumentation.ts` - Try manual trigger via API to test job logic -### Semantic Search Returns No Results +#### Semantic Search Returns No Results - Verify articles have embeddings: `SELECT COUNT(*) FROM articles WHERE embedding IS NOT NULL` - Check similarity threshold (may be too high) - Ensure same embedding provider used for query and articles - Verify HNSW index exists: `\d articles` in psql -## Default Feeds for New Users - -New users are automatically subscribed to a curated set of 9 feeds covering: -- **Technology**: TechCrunch, The Verge, Hacker News -- **News**: BBC News -- **Science**: Nature, Science Daily -- **Positive News**: Good News Network, Positive News -- **Satire**: The Onion - -**Implementation:** -- Feeds created on-demand in [src/lib/services/default-feeds-service.ts](src/lib/services/default-feeds-service.ts) -- Subscription happens in `createUser` event in auth.ts -- Categories and feeds auto-created if missing -- Idempotent (safe to run multiple times) +#### Feed Health Issues +- Check feed health status in OverviewView or via API +- Review error logs: `GET /api/feeds/[id]/errors` +- Manually reset health if needed: `resetFeedHealthAction(feedId)` +- Verify auto-disable threshold is appropriate for feed reliability -See [docs/DEFAULT_FEEDS.md](docs/DEFAULT_FEEDS.md) for full documentation. +## Reference -## Key Files to Reference +### Key Files to Reference +**Core Architecture:** - [src/lib/api-handler.ts](src/lib/api-handler.ts) - API route wrapper pattern - [src/lib/auth.ts](src/lib/auth.ts) - Authentication configuration +- [prisma/schema.prisma](prisma/schema.prisma) - Database schema +- [src/env.ts](src/env.ts) - Environment variable definitions + +**Server Actions:** +- [app/actions/feeds.ts](app/actions/feeds.ts) - Feed server actions (512 lines) +- [app/actions/articles.ts](app/actions/articles.ts) - Article server actions (431 lines) +- [app/actions/user-feeds.ts](app/actions/user-feeds.ts) - User subscription actions (358 lines) +- [app/actions/saved-searches.ts](app/actions/saved-searches.ts) - Saved search actions (289 lines) + +**Services:** - [src/lib/services/feed-refresh-service.ts](src/lib/services/feed-refresh-service.ts) - Core feed refresh logic +- [src/lib/services/feed-health-service.ts](src/lib/services/feed-health-service.ts) - Feed health tracking - [src/lib/services/semantic-search-service.ts](src/lib/services/semantic-search-service.ts) - Vector search implementation +- [src/lib/services/summarization-service.ts](src/lib/services/summarization-service.ts) - Article summarization +- [src/lib/services/bulk-operations-service.ts](src/lib/services/bulk-operations-service.ts) - Bulk feed operations +- [src/lib/services/opml-service.ts](src/lib/services/opml-service.ts) - OPML import/export +- [src/lib/services/offline-cache-service.ts](src/lib/services/offline-cache-service.ts) - Client-side caching - [src/lib/services/default-feeds-service.ts](src/lib/services/default-feeds-service.ts) - Default feed subscription + +**Jobs & Scheduling:** - [src/lib/jobs/scheduler.ts](src/lib/jobs/scheduler.ts) - Cron job initialization -- [prisma/schema.prisma](prisma/schema.prisma) - Database schema -- [src/env.ts](src/env.ts) - Environment variable definitions +- [src/lib/jobs/feed-refresh-job.ts](src/lib/jobs/feed-refresh-job.ts) - Feed refresh cron job + +**React Hooks:** +- [src/hooks/queries/use-feeds.ts](src/hooks/queries/use-feeds.ts) - Feed React Query hooks +- [src/hooks/queries/use-articles.ts](src/hooks/queries/use-articles.ts) - Article React Query hooks +- [src/hooks/queries/use-saved-searches.ts](src/hooks/queries/use-saved-searches.ts) - Saved search hooks + +**UI Components:** +- [app/feeds-management/page.tsx](app/feeds-management/page.tsx) - Feed management UI +- [app/feeds-management/components/views/FeedDetailsView.tsx](app/feeds-management/components/views/FeedDetailsView.tsx) - 7-tab feed details +- [app/feeds-management/components/views/OverviewView.tsx](app/feeds-management/components/views/OverviewView.tsx) - Feed overview with bulk ops diff --git a/app/admin/dashboard/components/tabs/MemoryTab.tsx b/app/admin/dashboard/components/tabs/MemoryTab.tsx new file mode 100644 index 0000000..358c87a --- /dev/null +++ b/app/admin/dashboard/components/tabs/MemoryTab.tsx @@ -0,0 +1,373 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Card, CardBody } from "@/app/components/ui"; +import { MetricCard } from "../shared/MetricCard"; +import { ConfirmButton } from "../shared/ConfirmButton"; +import { toast } from "sonner"; + +export interface MemoryTabProps { + /** Auto-refresh interval in milliseconds */ + refreshInterval?: number; +} + +interface MemoryData { + current: { + rss: string; + rssMB: string; + heapTotal: string; + heapTotalMB: string; + heapUsed: string; + heapUsedMB: string; + heapUsedPercent: string; + external: string; + arrayBuffers: string; + timestamp: number; + }; + pressure: { + level: "normal" | "moderate" | "high" | "critical"; + heapUsedPercent: number; + rssUsedMB: number; + recommendation: string; + }; + history: { + samples: Array<{ + timestamp: number; + heapUsedMB: string; + heapTotalMB: string; + rssMB: string; + heapUsedPercent: string; + }>; + maxSamples: number; + startTime: number; + }; + trend: { + direction: "increasing" | "decreasing" | "stable"; + change: string; + }; + uptime: number; + uptimeFormatted: string; + monitoring: { + enabled: boolean; + }; +} + +// Icons +const MemoryIcon = () => ( + + + +); + +const ChartIcon = () => ( + + + +); + +/** + * MemoryTab component displays real-time memory usage and monitoring. + */ +export function MemoryTab({ refreshInterval = 10000 }: MemoryTabProps) { + const [memoryData, setMemoryData] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [isForceGCLoading, setIsForceGCLoading] = useState(false); + + // Fetch memory data + const fetchMemoryData = async () => { + try { + const response = await fetch("/api/admin/memory"); + if (!response.ok) { + throw new Error("Failed to fetch memory data"); + } + const result = await response.json(); + setMemoryData(result.data); + setIsLoading(false); + } catch (error) { + console.error("Error fetching memory data:", error); + toast.error("Failed to fetch memory data"); + setIsLoading(false); + } + }; + + // Initial fetch and polling + useEffect(() => { + fetchMemoryData(); + const interval = setInterval(fetchMemoryData, refreshInterval); + return () => clearInterval(interval); + }, [refreshInterval]); + + // Force garbage collection + const handleForceGC = async () => { + setIsForceGCLoading(true); + try { + const response = await fetch("/api/admin/memory", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action: "force-gc" }), + }); + + const result = await response.json(); + + if (response.ok && result.data?.success) { + toast.success( + `Garbage collection completed. Heap: ${result.data.heapUsedMB}MB (${result.data.heapUsedPercent}%)` + ); + // Refresh data + await fetchMemoryData(); + } else { + toast.error(result.error || "Failed to force garbage collection"); + } + } catch (error) { + console.error("Error forcing GC:", error); + toast.error("Failed to force garbage collection"); + } finally { + setIsForceGCLoading(false); + } + }; + + // Get pressure color + const getPressureColor = (level: string) => { + switch (level) { + case "critical": + return "text-red-600 bg-red-50 border-red-200"; + case "high": + return "text-orange-600 bg-orange-50 border-orange-200"; + case "moderate": + return "text-yellow-600 bg-yellow-50 border-yellow-200"; + default: + return "text-green-600 bg-green-50 border-green-200"; + } + }; + + // Get trend icon + const getTrendIcon = (direction: string) => { + switch (direction) { + case "increasing": + return "↗"; + case "decreasing": + return "↘"; + default: + return "→"; + } + }; + + if (isLoading) { + return ( +
+
+
Loading memory data...
+
+ Fetching real-time metrics +
+
+
+ ); + } + + if (!memoryData) { + return ( +
+
+
No memory data available
+
+ Memory monitoring may be disabled +
+
+
+ ); + } + + return ( +
+ {/* Memory Pressure Alert */} + {memoryData.pressure?.level && memoryData.pressure.level !== "normal" && ( + + +
+
+

+ {memoryData.pressure.level} Memory Pressure +

+

{memoryData.pressure.recommendation}

+
+
+ {memoryData.current?.heapUsedPercent || "0"}% +
+
+ + + )} + + {/* Current Memory Metrics */} +
+ {/* RSS (Total Memory) */} + } + iconColor="blue" + footer={{ + label: "Formatted", + value: memoryData.current?.rss || "0 B", + }} + /> + + {/* Heap Used */} + } + iconColor="purple" + footer={{ + label: "Trend", + value: `${getTrendIcon(memoryData.trend?.direction || "stable")} ${memoryData.trend?.change || "0"}%`, + }} + /> + + {/* External Memory */} + } + iconColor="green" + footer={{ + label: "ArrayBuffers", + value: memoryData.current?.arrayBuffers || "0 B", + }} + /> +
+ + {/* Memory History Chart */} + + +

+ Memory Usage History (Last {memoryData.history?.samples?.length || 0}{" "} + samples) +

+
+ {(memoryData.history?.samples || []) + .slice(-10) + .reverse() + .map((sample, index) => { + const percent = parseFloat(sample.heapUsedPercent); + const barColor = + percent >= 95 + ? "bg-red-500" + : percent >= 85 + ? "bg-orange-500" + : percent >= 70 + ? "bg-yellow-500" + : "bg-green-500"; + + return ( +
+
+ {new Date(sample.timestamp).toLocaleTimeString()} +
+
+
+
+
+
+
+ {sample.heapUsedMB}MB ({sample.heapUsedPercent}%) +
+
+ ); + })} +
+ + + + {/* System Information */} + + +

System Information

+
+
+
Uptime
+
+ {memoryData.uptimeFormatted || "0s"} +
+
+
+
+ Monitoring Status +
+
+ {memoryData.monitoring?.enabled ? ( + Enabled + ) : ( + Disabled + )} +
+
+
+
+ Pressure Level +
+
+ {memoryData.pressure?.level || "unknown"} +
+
+
+
History Samples
+
+ {memoryData.history?.samples?.length || 0} /{" "} + {memoryData.history?.maxSamples || 0} +
+
+
+ + + + {/* Actions */} + + +

Memory Actions

+
+ + Force Garbage Collection + + +
+

+ Note: Garbage collection requires Node.js to be started with the + --expose-gc flag. +

+ + +
+ ); +} diff --git a/app/admin/dashboard/hooks/use-tab-navigation.ts b/app/admin/dashboard/hooks/use-tab-navigation.ts index a7d30cf..5929be4 100644 --- a/app/admin/dashboard/hooks/use-tab-navigation.ts +++ b/app/admin/dashboard/hooks/use-tab-navigation.ts @@ -3,7 +3,7 @@ import { useState, useEffect, useCallback } from "react"; import { useRouter, useSearchParams } from "next/navigation"; -export type TabId = "overview" | "search" | "users" | "jobs" | "storage" | "config" | "llm-config"; +export type TabId = "overview" | "search" | "users" | "jobs" | "storage" | "config" | "llm-config" | "memory"; const FAVORITE_TABS_KEY = "admin-favorite-tabs"; diff --git a/app/admin/dashboard/page.tsx b/app/admin/dashboard/page.tsx index 9180445..fea5760 100644 --- a/app/admin/dashboard/page.tsx +++ b/app/admin/dashboard/page.tsx @@ -20,6 +20,7 @@ import { JobsTab } from "./components/tabs/JobsTab"; import { StorageTab } from "./components/tabs/StorageTab"; import { ConfigTab } from "./components/tabs/ConfigTab"; import { LLMConfigTab } from "./components/tabs/LLMConfigTab"; +import { MemoryTab } from "./components/tabs/MemoryTab"; // Custom hooks import { useDashboardData } from "./hooks/use-dashboard-data"; @@ -203,6 +204,20 @@ export default function AdminDashboardPage() { ), }, + { + id: "memory" as const, + label: "Memory", + icon: ( + + + + ), + }, ]; return ( @@ -272,6 +287,8 @@ export default function AdminDashboardPage() { {activeTab === "config" && } {activeTab === "llm-config" && } + + {activeTab === "memory" && }
diff --git a/app/api/admin/memory/route.ts b/app/api/admin/memory/route.ts new file mode 100644 index 0000000..ff0c7b0 --- /dev/null +++ b/app/api/admin/memory/route.ts @@ -0,0 +1,229 @@ +/** + * Memory Monitoring API + * + * GET /api/admin/memory - Get current memory stats and history + * POST /api/admin/memory - Trigger memory management actions (GC, etc.) + */ + +import { createHandler } from "@/lib/api-handler"; +import { memoryMonitor, formatBytes, getMemoryUsagePercent } from "@/lib/memory-monitor"; +import { prisma } from "@/lib/db"; +import { z } from "zod"; + +/** + * GET /api/admin/memory + * Returns current memory statistics and history + */ +export const GET = createHandler( + async ({ session }) => { + // Check admin role + const userRole = await prisma.user.findUnique({ + where: { id: session!.user.id }, + select: { role: true }, + }); + + if (userRole?.role !== "ADMIN") { + return { + error: "Forbidden: Admin access required", + status: 403, + }; + } + + const summary = memoryMonitor.getSummary(); + + // Calculate trends + const history = summary.history.stats; + const trend = calculateTrend(history); + + // Format for client + const formatted = { + current: { + rss: formatBytes(summary.current.rss), + rssMB: (summary.current.rss / 1024 / 1024).toFixed(2), + heapTotal: formatBytes(summary.current.heapTotal), + heapTotalMB: (summary.current.heapTotal / 1024 / 1024).toFixed(2), + heapUsed: formatBytes(summary.current.heapUsed), + heapUsedMB: (summary.current.heapUsed / 1024 / 1024).toFixed(2), + heapUsedPercent: getMemoryUsagePercent(summary.current).toFixed(2), + external: formatBytes(summary.current.external), + arrayBuffers: formatBytes(summary.current.arrayBuffers), + timestamp: summary.current.timestamp, + }, + pressure: summary.pressure, + history: { + samples: history.map((stat) => ({ + timestamp: stat.timestamp, + heapUsedMB: (stat.heapUsed / 1024 / 1024).toFixed(2), + heapTotalMB: (stat.heapTotal / 1024 / 1024).toFixed(2), + rssMB: (stat.rss / 1024 / 1024).toFixed(2), + heapUsedPercent: getMemoryUsagePercent(stat).toFixed(2), + })), + maxSamples: summary.history.maxSamples, + startTime: summary.history.startTime, + }, + trend, + uptime: summary.uptime, + uptimeFormatted: formatUptime(summary.uptime), + monitoring: { + enabled: memoryMonitor.isRunning(), + }, + }; + + return { data: formatted }; + }, + { + requireAuth: true, + } +); + +/** + * POST /api/admin/memory + * Trigger memory management actions + */ +const actionSchema = z.object({ + action: z.enum(["force-gc", "start-monitor", "stop-monitor"]), +}); + +export const POST = createHandler( + async ({ body, session }) => { + // Check admin role + const userRole = await prisma.user.findUnique({ + where: { id: session!.user.id }, + select: { role: true }, + }); + + if (userRole?.role !== "ADMIN") { + return { + error: "Forbidden: Admin access required", + status: 403, + }; + } + + const { action } = body; + + switch (action) { + case "force-gc": + const gcSuccess = memoryMonitor.forceGC(); + if (gcSuccess) { + // Get memory stats after GC + const afterGC = memoryMonitor.getCurrentStats(); + return { + data: { + success: true, + message: "Garbage collection triggered", + heapUsedMB: (afterGC.heapUsed / 1024 / 1024).toFixed(2), + heapUsedPercent: getMemoryUsagePercent(afterGC).toFixed(2), + }, + }; + } else { + return { + error: "Garbage collection not available (requires --expose-gc flag)", + status: 500, + }; + } + + case "start-monitor": + if (memoryMonitor.isRunning()) { + return { + data: { + success: false, + message: "Memory monitor already running", + }, + }; + } + memoryMonitor.start(); + return { + data: { + success: true, + message: "Memory monitor started", + }, + }; + + case "stop-monitor": + if (!memoryMonitor.isRunning()) { + return { + data: { + success: false, + message: "Memory monitor not running", + }, + }; + } + memoryMonitor.stop(); + return { + data: { + success: true, + message: "Memory monitor stopped", + }, + }; + + default: + return { + error: "Invalid action", + status: 400, + }; + } + }, + { + bodySchema: actionSchema, + requireAuth: true, + } +); + +/** + * Calculate memory usage trend + */ +function calculateTrend(history: Array<{ heapUsed: number; heapTotal: number; timestamp: number }>) { + if (history.length < 2) { + return { direction: "stable", change: 0 }; + } + + // Compare last 5 samples with previous 5 samples + const recentCount = Math.min(5, Math.floor(history.length / 2)); + const recent = history.slice(-recentCount); + const previous = history.slice(-recentCount * 2, -recentCount); + + if (previous.length === 0) { + return { direction: "stable", change: 0 }; + } + + // Calculate percentage directly for partial memory stats + const calcPercent = (s: { heapUsed: number; heapTotal: number }) => (s.heapUsed / s.heapTotal) * 100; + + const recentAvg = recent.reduce((sum, s) => sum + calcPercent(s), 0) / recent.length; + const previousAvg = previous.reduce((sum, s) => sum + calcPercent(s), 0) / previous.length; + + const change = recentAvg - previousAvg; + + let direction: "increasing" | "decreasing" | "stable" = "stable"; + if (change > 2) { + direction = "increasing"; + } else if (change < -2) { + direction = "decreasing"; + } + + return { + direction, + change: change.toFixed(2), + }; +} + +/** + * Format uptime to human-readable string + */ +function formatUptime(ms: number): string { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + + if (days > 0) { + return `${days}d ${hours % 24}h ${minutes % 60}m`; + } + if (hours > 0) { + return `${hours}h ${minutes % 60}m`; + } + if (minutes > 0) { + return `${minutes}m ${seconds % 60}s`; + } + return `${seconds}s`; +} diff --git a/app/api/feeds/[id]/status/route.ts b/app/api/feeds/[id]/status/route.ts new file mode 100644 index 0000000..4f261b8 --- /dev/null +++ b/app/api/feeds/[id]/status/route.ts @@ -0,0 +1,37 @@ +import { enableFeed, disableFeed } from "@/lib/services/feed-health-service"; +import { createHandler } from "@/lib/api-handler"; +import { z } from "zod"; + +export const dynamic = "force-dynamic"; + +const bodySchema = z.object({ + enabled: z.boolean(), +}); + +/** + * PUT /api/feeds/:id/status + * Enable or disable a feed + */ +export const PUT = createHandler( + async ({ params, body }) => { + const { id } = params; + const { enabled } = body; + + if (!id || typeof id !== "string") { + return { error: "Feed ID is required", status: 400 }; + } + + if (enabled) { + await enableFeed(id); + } else { + await disableFeed(id); + } + + return { + success: true, + message: `Feed ${enabled ? "enabled" : "disabled"} successfully`, + data: { enabled }, + }; + }, + { bodySchema, requireAuth: true } +); diff --git a/app/api/feeds/bulk/status/route.ts b/app/api/feeds/bulk/status/route.ts new file mode 100644 index 0000000..b9f718c --- /dev/null +++ b/app/api/feeds/bulk/status/route.ts @@ -0,0 +1,51 @@ +import { enableFeed, disableFeed } from "@/lib/services/feed-health-service"; +import { createHandler } from "@/lib/api-handler"; +import { z } from "zod"; + +export const dynamic = "force-dynamic"; + +const bodySchema = z.object({ + feedIds: z.array(z.string()), + enabled: z.boolean(), +}); + +/** + * POST /api/feeds/bulk/status + * Enable or disable multiple feeds at once + */ +export const POST = createHandler( + async ({ body }) => { + const { feedIds, enabled } = body; + + if (!feedIds || feedIds.length === 0) { + return { error: "Feed IDs are required", status: 400 }; + } + + // Use Promise.allSettled to handle partial failures + const results = await Promise.allSettled( + feedIds.map(async (feedId) => { + if (enabled) { + await enableFeed(feedId); + } else { + await disableFeed(feedId); + } + return feedId; + }) + ); + + const successful = results.filter((r) => r.status === "fulfilled").length; + const failed = results.filter((r) => r.status === "rejected").length; + + return { + success: true, + message: `${enabled ? "Enabled" : "Disabled"} ${successful} of ${feedIds.length} feeds${failed > 0 ? ` (${failed} failed)` : ""}`, + data: { + total: feedIds.length, + successful, + failed, + results, + }, + }; + }, + { bodySchema, requireAuth: true } +); diff --git a/app/feeds-management/components/modals/BulkEditModal.tsx b/app/feeds-management/components/modals/BulkEditModal.tsx index 0a581ef..aa95c87 100644 --- a/app/feeds-management/components/modals/BulkEditModal.tsx +++ b/app/feeds-management/components/modals/BulkEditModal.tsx @@ -1,8 +1,10 @@ "use client"; import { useState } from "react"; +import { toast } from "sonner"; import { Modal, ModalHeader, ModalBody, ModalFooter, Button } from "@/app/components/ui"; import { useCategories } from "@/hooks/queries/use-categories"; +import { useBulkToggleFeedStatus } from "@/hooks/queries/use-feeds"; interface BulkEditModalProps { selectedFeedIds: string[]; @@ -21,25 +23,41 @@ interface BulkEditModalProps { */ export function BulkEditModal({ selectedFeedIds, onClose }: BulkEditModalProps) { const { data: categories = [] } = useCategories(); + const bulkToggleStatus = useBulkToggleFeedStatus(); const [action, setAction] = useState<"category" | "tags" | "enable" | "settings">("category"); const [newCategory, setNewCategory] = useState(""); const [tagsAction, setTagsAction] = useState<"add" | "remove" | "replace">("add"); const [tags, setTags] = useState(""); const [enableFeeds, setEnableFeeds] = useState(true); - const handleApply = () => { - const changes = { - feedIds: selectedFeedIds, - action, - data: { - category: action === "category" ? newCategory : undefined, - tags: action === "tags" ? { action: tagsAction, tags: tags.split(",").map(t => t.trim()).filter(Boolean) } : undefined, - enabled: action === "enable" ? enableFeeds : undefined, - }, - }; - // TODO: Implement bulk edit logic - console.log("Applying bulk changes:", changes); - onClose(); + const handleApply = async () => { + try { + if (action === "enable") { + // Handle enable/disable action + const result = await bulkToggleStatus.mutateAsync({ + feedIds: selectedFeedIds, + enabled: enableFeeds, + }); + toast.success(result.message || `Feeds ${enableFeeds ? "enabled" : "disabled"} successfully`); + onClose(); + } else { + // TODO: Implement other bulk edit actions + const changes = { + feedIds: selectedFeedIds, + action, + data: { + category: action === "category" ? newCategory : undefined, + tags: action === "tags" ? { action: tagsAction, tags: tags.split(",").map(t => t.trim()).filter(Boolean) } : undefined, + }, + }; + console.log("Applying bulk changes:", changes); + toast.info("This action is not yet implemented"); + onClose(); + } + } catch (error) { + console.error("Failed to apply bulk changes:", error); + toast.error("Failed to apply changes"); + } }; return ( @@ -238,8 +256,12 @@ export function BulkEditModal({ selectedFeedIds, onClose }: BulkEditModalProps) - diff --git a/app/feeds-management/components/views/FeedDetailsView.tsx b/app/feeds-management/components/views/FeedDetailsView.tsx index 6d2f6cd..a5df12e 100644 --- a/app/feeds-management/components/views/FeedDetailsView.tsx +++ b/app/feeds-management/components/views/FeedDetailsView.tsx @@ -2,7 +2,7 @@ import { useState } from "react"; import { useFeedNavigation } from "@/hooks/use-feed-navigation"; -import { useUserFeeds } from "@/hooks/queries/use-feeds"; +import { useUserFeeds, useToggleFeedStatus } from "@/hooks/queries/use-feeds"; import { useCategories } from "@/hooks/queries/use-categories"; interface FeedDetailsViewProps { @@ -120,6 +120,25 @@ export function FeedDetailsView({ feedId }: FeedDetailsViewProps) { // Tab Components function BasicSettingsTab({ feed, categories }: { feed: any; categories: any[] }) { + const toggleStatus = useToggleFeedStatus(); + const [isEnabled, setIsEnabled] = useState(feed.isActive ?? true); + + const handleToggle = async () => { + const newStatus = !isEnabled; + setIsEnabled(newStatus); // Optimistic update + + try { + await toggleStatus.mutateAsync({ + feedId: feed.id, + enabled: newStatus + }); + } catch (error) { + // Revert on error + setIsEnabled(!newStatus); + console.error("Failed to toggle feed status:", error); + } + }; + return (
@@ -177,16 +196,38 @@ function BasicSettingsTab({ feed, categories }: { feed: any; categories: any[] }

-
- - + {/* Feed Status Section */} +
+
+ +

+ When disabled, this feed will not be fetched during updates +

+ {feed.healthStatus && feed.healthStatus !== "healthy" && ( +

+ Current status: {feed.healthStatus} + {feed.consecutiveFailures > 0 && ` (${feed.consecutiveFailures} failures)`} +

+ )} +
+
); diff --git a/app/feeds-management/components/views/OverviewView.tsx b/app/feeds-management/components/views/OverviewView.tsx index 0d4a36c..bfb1606 100644 --- a/app/feeds-management/components/views/OverviewView.tsx +++ b/app/feeds-management/components/views/OverviewView.tsx @@ -111,7 +111,7 @@ export function OverviewView() { {selectedFeedIds.length} selected