diff --git a/backend/src/mainloop/api.py b/backend/src/mainloop/api.py index d5279d9..a7e2d42 100644 --- a/backend/src/mainloop/api.py +++ b/backend/src/mainloop/api.py @@ -19,18 +19,14 @@ from mainloop.services.github_pr import ( CommitSummary, ProjectPRSummary, - add_issue_comment, get_repo_metadata, list_open_prs, list_recent_commits, - update_github_issue, ) from mainloop.sse import ( create_sse_response, event_stream, notify_inbox_updated, - notify_task_updated, - task_log_stream, ) # Import DBOS config to initialize DBOS before defining workflows @@ -40,7 +36,6 @@ from mainloop.workflows.main_thread import ( get_or_start_main_thread, ) -from mainloop.workflows.worker import worker_task_workflow # noqa: F401 from pydantic import BaseModel from models import ( @@ -48,13 +43,10 @@ Project, QueueItem, QueueItemResponse, - QueueItemType, Session, SessionCreate, SessionNotification, SessionStatus, - TaskStatus, - WorkerTask, ) logger = logging.getLogger(__name__) @@ -177,34 +169,6 @@ async def sse_events( return create_sse_response(event_stream(user_id, request)) -@app.get("/tasks/{task_id}/logs/stream") -async def sse_task_logs( - task_id: str, - request: Request, - user_id: str = Header(alias="X-User-ID", default=None), -): - """SSE endpoint for streaming task logs. - - Streams events for: - - log - new log lines from the K8s pod - - status - task status changes - - end - stream is ending (task completed/failed) - - Polls K8s logs every 2 seconds and streams new content. - """ - if not user_id: - user_id = get_user_id_from_cf_header() - - # Verify task exists and belongs to user - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - if task.user_id != user_id: - raise HTTPException(status_code=403, detail="Not your task") - - return create_sse_response(task_log_stream(task_id, user_id, request)) - - # ============= Main Thread Endpoints ============= @@ -533,7 +497,7 @@ class ProjectDetail(BaseModel): project: Project open_prs: list[ProjectPRSummary] recent_commits: list[CommitSummary] - tasks: list[WorkerTask] + sessions: list[Session] @app.get("/projects/{project_id}/detail", response_model=ProjectDetail) @@ -541,7 +505,7 @@ async def get_project_detail( project_id: str, user_id: str = Header(alias="X-User-ID", default=None), ): - """Get project with GitHub data (PRs, commits, tasks).""" + """Get project with GitHub data (PRs, commits, sessions).""" if not user_id: user_id = get_user_id_from_cf_header() @@ -555,14 +519,14 @@ async def get_project_detail( project.html_url, branch=project.default_branch, limit=10 ) - # Fetch tasks for this project - tasks = await db.list_worker_tasks(user_id=user_id, project_id=project_id, limit=50) + # Fetch sessions for this project + sessions = await db.list_sessions(user_id=user_id, project_id=project_id, limit=50) return ProjectDetail( project=project, open_prs=open_prs, recent_commits=recent_commits, - tasks=tasks, + sessions=sessions, ) @@ -586,353 +550,6 @@ async def refresh_project_metadata(project_id: str): return {"status": "ok", "message": "Project metadata refreshed"} -# ============= Task Endpoints ============= - - -@app.get("/tasks", response_model=list[WorkerTask]) -async def list_tasks( - user_id: str = Header(alias="X-User-ID", default=None), - status: str | None = None, - project_id: str | None = None, -): - """List worker tasks for the user, optionally filtered by project.""" - if not user_id: - user_id = get_user_id_from_cf_header() - - tasks = await db.list_worker_tasks( - user_id=user_id, status=status, project_id=project_id - ) - return tasks - - -class TaskContext(BaseModel): - """Full task context for pull-based retrieval.""" - - task: WorkerTask - queue_items: list[QueueItem] - - -@app.get("/tasks/{task_id}", response_model=WorkerTask) -async def get_task(task_id: str): - """Get a specific worker task.""" - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - return task - - -@app.get("/tasks/{task_id}/context", response_model=TaskContext) -async def get_task_context( - task_id: str, - user_id: str = Header(alias="X-User-ID", default=None), -): - """Get full task context including queue items. - - This endpoint is for pull-based context retrieval when the main thread - needs to know what's happening with a specific task. - """ - if not user_id: - user_id = get_user_id_from_cf_header() - - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - if task.user_id != user_id: - raise HTTPException(status_code=403, detail="Not your task") - - # Get all queue items for this task - queue_items = await db.list_queue_items(user_id=user_id, task_id=task_id) - - return TaskContext(task=task, queue_items=queue_items) - - -class TaskLogsResponse(BaseModel): - """Response for task logs endpoint.""" - - logs: str - source: str # "k8s" or "none" - task_status: str - - -@app.get("/tasks/{task_id}/logs", response_model=TaskLogsResponse) -async def get_task_logs( - task_id: str, - tail: int = 100, - user_id: str = Header(alias="X-User-ID", default=None), -): - """Get logs for a worker task from its K8s pod. - - Args: - task_id: The task ID - tail: Number of lines to return from the end (default 100) - user_id: User ID from X-User-ID header - - """ - if not user_id: - user_id = get_user_id_from_cf_header() - - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - if task.user_id != user_id: - raise HTTPException(status_code=403, detail="Not your task") - - # Try to get live logs from K8s - import logging - - from mainloop.services.k8s_jobs import get_job_logs - - logger = logging.getLogger(__name__) - namespace = f"task-{task_id[:8]}" - - logs = None - source = "k8s" - - try: - logs = await get_job_logs(task_id, namespace) - except Exception as e: - logger.warning(f"Failed to get K8s logs for task {task_id}: {e}") - - if not logs: - logs = "" - source = "none" - else: - # Tail the logs to requested number of lines - lines = logs.split("\n") - if len(lines) > tail: - lines = lines[-tail:] - logs = "\n".join(lines) - - return TaskLogsResponse( - logs=logs, - source=source, - task_status=task.status.value, - ) - - -@app.post("/tasks/{task_id}/cancel") -async def cancel_task( - task_id: str, - user_id: str = Header(alias="X-User-ID", default=None), -): - """Cancel a running task and close associated GitHub issue/PR.""" - if not user_id: - user_id = get_user_id_from_cf_header() - - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - if task.user_id != user_id: - raise HTTPException(status_code=403, detail="Not your task") - - # Cancel the DBOS workflow - DBOS.cancel_workflow(task_id) - - await db.update_worker_task(task_id, status=TaskStatus.CANCELLED) - - # Close GitHub issue if exists - if task.repo_url and task.issue_number: - await add_issue_comment( - task.repo_url, task.issue_number, "❌ Task cancelled by user." - ) - await update_github_issue(task.repo_url, task.issue_number, state="closed") - - # Close GitHub PR if exists (PRs are also issues in GitHub API) - if task.repo_url and task.pr_number and task.pr_number != task.issue_number: - await add_issue_comment( - task.repo_url, task.pr_number, "❌ Task cancelled by user." - ) - await update_github_issue(task.repo_url, task.pr_number, state="closed") - - # Notify SSE clients - await notify_task_updated(user_id, task_id, "cancelled") - - return {"status": "cancelled"} - - -class AnswerQuestionsRequest(BaseModel): - """Request to answer task questions.""" - - answers: dict[str, str] # question_id -> answer text - action: str = "answer" # "answer" or "cancel" - - -@app.post("/tasks/{task_id}/answer-questions") -async def answer_task_questions( - task_id: str, - body: AnswerQuestionsRequest, - user_id: str = Header(alias="X-User-ID", default=None), -): - """Answer questions asked by an agent during planning. - - The agent can ask questions via AskUserQuestion tool during plan mode. - These questions are stored on the task and surfaced in the UI. - This endpoint sends the user's answers back to the waiting worker workflow. - """ - if not user_id: - user_id = get_user_id_from_cf_header() - - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - if task.user_id != user_id: - raise HTTPException(status_code=403, detail="Not your task") - - if task.status != TaskStatus.WAITING_QUESTIONS: - raise HTTPException( - status_code=400, - detail=f"Task is not waiting for questions (status: {task.status})", - ) - - # Send answers to the worker workflow - from dbos import error as dbos_error - from mainloop.workflows.worker import TOPIC_QUESTION_RESPONSE - - # Try to send to worker workflow, but continue if workflow doesn't exist (e.g., in tests) - try: - DBOS.send( - task_id, # Worker workflow ID is the task ID - { - "action": body.action, - "answers": body.answers, - }, - topic=TOPIC_QUESTION_RESPONSE, - ) - except dbos_error.DBOSNonExistentWorkflowError: - # Workflow doesn't exist (e.g., test environment) - that's okay, continue with status update - pass - - # Update task status immediately so frontend sees correct state on refetch - # The workflow will also update this, but we do it here to prevent race conditions - if body.action == "cancel": - await db.update_worker_task( - task_id, status=TaskStatus.CANCELLED, pending_questions=[] - ) - # Close GitHub issue if exists - if task.repo_url and task.issue_number: - await add_issue_comment( - task.repo_url, task.issue_number, "❌ Task cancelled by user." - ) - await update_github_issue(task.repo_url, task.issue_number, state="closed") - await notify_task_updated(user_id, task_id, "cancelled") - else: - await db.update_worker_task( - task_id, status=TaskStatus.PLANNING, pending_questions=[] - ) - await notify_task_updated(user_id, task_id, "planning") - - return {"status": "ok", "message": f"Sent {len(body.answers)} answer(s) to task"} - - -@app.post("/tasks/{task_id}/approve-plan") -async def approve_task_plan( - task_id: str, - action: str = "approve", - revision_text: str | None = None, - user_id: str = Header(alias="X-User-ID", default=None), -): - """Approve or revise a plan shown in the task UI. - - This replaces the queue-item-based plan approval for the embedded task UI flow. - """ - if not user_id: - user_id = get_user_id_from_cf_header() - - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - if task.user_id != user_id: - raise HTTPException(status_code=403, detail="Not your task") - - if task.status != TaskStatus.WAITING_PLAN_REVIEW: - raise HTTPException( - status_code=400, - detail=f"Task is not waiting for plan review (status: {task.status})", - ) - - # Send response to the worker workflow - from mainloop.workflows.worker import TOPIC_PLAN_RESPONSE - - DBOS.send( - task_id, - { - "action": action, # "approve", "cancel", or revision text - "text": revision_text or "", - }, - topic=TOPIC_PLAN_RESPONSE, - ) - - # Update task status immediately so frontend sees correct state on refetch - if action == "cancel": - await db.update_worker_task( - task_id, status=TaskStatus.CANCELLED, plan_text=None - ) - # Close GitHub issue if exists - if task.repo_url and task.issue_number: - await add_issue_comment( - task.repo_url, task.issue_number, "❌ Task cancelled by user." - ) - await update_github_issue(task.repo_url, task.issue_number, state="closed") - await notify_task_updated(user_id, task_id, "cancelled") - elif action == "approve": - await db.update_worker_task(task_id, status=TaskStatus.READY_TO_IMPLEMENT) - await notify_task_updated(user_id, task_id, "ready_to_implement") - else: - # Revision - back to planning - await db.update_worker_task(task_id, status=TaskStatus.PLANNING) - await notify_task_updated(user_id, task_id, "planning") - - return {"status": "ok", "action": action} - - -@app.post("/tasks/{task_id}/start-implementation") -async def start_task_implementation( - task_id: str, - user_id: str = Header(alias="X-User-ID", default=None), -): - """Start implementation of an approved plan. - - This triggers the worker to proceed from ready_to_implement to implementing. - """ - if not user_id: - user_id = get_user_id_from_cf_header() - - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - if task.user_id != user_id: - raise HTTPException(status_code=403, detail="Not your task") - - if task.status != TaskStatus.READY_TO_IMPLEMENT: - raise HTTPException( - status_code=400, - detail=f"Task is not ready for implementation (status: {task.status})", - ) - - # Send implementation trigger to the worker workflow - from mainloop.workflows.worker import TOPIC_START_IMPLEMENTATION - - DBOS.send( - task_id, - {"action": "start"}, - topic=TOPIC_START_IMPLEMENTATION, - ) - - # Update task status immediately so frontend sees correct state on refetch - await db.update_worker_task(task_id, status=TaskStatus.IMPLEMENTING) - - # Notify SSE clients - await notify_task_updated(user_id, task_id, "implementing") - - return {"status": "ok", "message": "Implementation started"} - - # ============= Session Endpoints ============= @@ -1185,79 +802,34 @@ async def dismiss_notification( # ============= Internal Endpoints (for K8s Jobs) ============= -class TaskResult(BaseModel): - """Result from a worker Job.""" +class SessionResult(BaseModel): + """Result from a session Job.""" - task_id: str + session_id: str status: str # "completed" or "failed" result: dict[str, Any] | None = None error: str | None = None completed_at: str | None = None -@app.post("/internal/tasks/{task_id}/complete") -async def task_complete(task_id: str, result: TaskResult): - """Handle K8s Job completion callbacks. +@app.post("/internal/sessions/{session_id}/complete") +async def session_complete(session_id: str, result: SessionResult): + """Handle K8s Job completion callbacks for sessions. - This is called by the job_runner when a worker Job finishes. - It updates the task status and notifies the main thread workflow. + This is called by the job_runner when a session Job finishes. + It notifies the session workflow to add the response and continue. """ - # Verify task exists - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - # Update task with job result - # NOTE: "completed" here means the K8s job finished, NOT that the task is done. - # The task is only truly completed when the PR is merged (handled by workflow). - # We just store the URL/number here without changing status. - if result.status == "completed": - # Handle both issue URLs (plan phase) and PR URLs (implement phase) - issue_url = result.result.get("issue_url") if result.result else None - pr_url = result.result.get("pr_url") if result.result else None - issue_number = None - pr_number = None - - if issue_url: - # Extract issue number from URL (e.g., https://github.com/owner/repo/issues/123) - try: - issue_number = int(issue_url.split("/")[-1]) - except (ValueError, IndexError): - pass - - if pr_url: - # Extract PR number from URL (e.g., https://github.com/owner/repo/pull/123) - try: - pr_number = int(pr_url.split("/")[-1]) - except (ValueError, IndexError): - pass - - # Update task with URLs - don't mark COMPLETED, workflow manages status - if issue_url or pr_url: - await db.update_worker_task( - task_id, - result=result.result, - issue_url=issue_url, - issue_number=issue_number, - pr_url=pr_url, - pr_number=pr_number, - ) - elif result.status == "failed": - await db.update_worker_task( - task_id, - status=TaskStatus.FAILED, - error=result.error, - ) + # Verify session exists + session = await db.get_session(session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") - # Send result to the worker workflow via DBOS.send() - # The worker workflow is waiting on TOPIC_JOB_RESULT - from mainloop.workflows.worker import TOPIC_JOB_RESULT + # Send result to the session workflow via DBOS.send() + # The session workflow is waiting on TOPIC_JOB_RESULT + from mainloop.workflows.session_worker import TOPIC_JOB_RESULT - # Send to the worker workflow (which uses task_id as workflow ID via the queue) - # Actually, we need to send to the workflow that's waiting - # The worker_task_workflow is running with the task_id as part of its workflow context - # We use DBOS.send with the workflow_id to target it - workflow_id = task_id # The worker workflow uses task_id for idempotency + # The session workflow uses session_id as workflow ID + workflow_id = session_id DBOS.send( workflow_id, @@ -1269,257 +841,12 @@ async def task_complete(task_id: str, result: TaskResult): topic=TOPIC_JOB_RESULT, ) - return {"status": "ok", "task_id": task_id} - - -# ============= Debug Endpoints ============= - - -class DebugTaskInfo(BaseModel): - """Debug info for a worker task including workflow state.""" - - task: WorkerTask - workflow_status: str | None = None - workflow_error: str | None = None - workflow_created_at: datetime | None = None - workflow_updated_at: datetime | None = None - namespace_exists: bool = False - k8s_jobs: list[str] = [] - - -@app.get("/debug/tasks", response_model=list[DebugTaskInfo]) -async def debug_list_tasks( - limit: int = 10, -): - """List all tasks with debug info (no auth required for debugging).""" - from datetime import timezone - - from mainloop.services.k8s_namespace import get_k8s_client, namespace_exists - - # Get all tasks (bypass user filter for debugging) - async with db.connection() as conn: - task_rows = await conn.fetch( - """ - SELECT * FROM worker_tasks - ORDER BY created_at DESC - LIMIT $1 - """, - limit, - ) - - # Get workflow status for each task - results = [] - for task_row in task_rows: - task = db._row_to_worker_task(task_row) - - # Get DBOS workflow status - workflow_row = await conn.fetchrow( - """ - SELECT status, error, created_at, updated_at - FROM dbos.workflow_status - WHERE workflow_uuid = $1 - """, - task.id, - ) - - workflow_status = None - workflow_error = None - workflow_created_at = None - workflow_updated_at = None - - if workflow_row: - workflow_status = workflow_row["status"] - # Just show raw error string (it's base64-encoded pickle, but we show it raw) - if workflow_row["error"]: - workflow_error = f"[encoded] {workflow_row['error'][:200]}..." - workflow_created_at = datetime.fromtimestamp( - workflow_row["created_at"] / 1000, tz=timezone.utc - ) - workflow_updated_at = datetime.fromtimestamp( - workflow_row["updated_at"] / 1000, tz=timezone.utc - ) - - # Check if namespace exists - ns_exists = False - k8s_jobs = [] - try: - ns_exists = await namespace_exists(task.id) - if ns_exists: - _, batch_v1 = get_k8s_client() - namespace_name = f"task-{task.id[:8]}" - jobs = batch_v1.list_namespaced_job(namespace=namespace_name) - k8s_jobs = [j.metadata.name for j in jobs.items] - except Exception: - pass # nosec B110 - - results.append( - DebugTaskInfo( - task=task, - workflow_status=workflow_status, - workflow_error=workflow_error, - workflow_created_at=workflow_created_at, - workflow_updated_at=workflow_updated_at, - namespace_exists=ns_exists, - k8s_jobs=k8s_jobs, - ) - ) - - return results - - -@app.post("/tasks/{task_id}/retry") -async def retry_task(task_id: str): - """Retry a failed task by resetting its status and re-enqueueing.""" - from dbos import SetWorkflowID - from mainloop.workflows.dbos_config import worker_queue - - task = await db.get_worker_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - - # Reset task status to pending - await db.update_worker_task(task_id, status=TaskStatus.PENDING, error=None) - - # Delete the failed workflow from DBOS - async with db.connection() as conn: - await conn.execute( - "DELETE FROM dbos.workflow_status WHERE workflow_uuid = $1", - task_id, - ) - - # Re-enqueue via DBOS worker queue - with SetWorkflowID(task_id): - worker_queue.enqueue(worker_task_workflow, task_id) - - return {"status": "retried", "task_id": task_id} - - -@app.delete("/debug/tasks/{task_id}/namespace") -async def debug_delete_namespace(task_id: str): - """Force delete a task namespace.""" - from mainloop.services.k8s_namespace import delete_task_namespace - - await delete_task_namespace(task_id) - return {"status": "deleted", "namespace": f"task-{task_id[:8]}"} + return {"status": "ok", "session_id": session_id} # ============= Test Helpers (E2E only) ============= -class SeedTaskRequest(BaseModel): - """Request to seed a task for testing.""" - - status: TaskStatus - task_type: str = "feature" - description: str = "Test task" - repo_url: str | None = None - plan: str | None = None - questions: list[dict] | None = None # For waiting_questions status - - -@app.post("/internal/test/seed-task") -async def seed_task_for_testing( - request: SeedTaskRequest, - user_id: str = Header(alias="X-User-ID", default=None), -): - """Create a task in a specific state for E2E testing. - - WARNING: Only available in test environments. Do not use in production. - """ - if not settings.is_test_env: - raise HTTPException( - status_code=403, detail="Only available in test environment" - ) - - from uuid import uuid4 - - # Get or create a test main thread - if not user_id: - user_id = get_user_id_from_cf_header() - thread = await db.get_main_thread_by_user(user_id) - if not thread: - # Create test thread directly (no workflow needed for tests) - thread = MainThread(user_id=user_id, workflow_run_id="test-workflow") - thread = await db.create_main_thread(thread) - thread_id = thread.id - - # Create task - # Convert questions dict to TaskQuestion models if provided - pending_questions = None - if request.questions: - from models.workflow import QuestionOption, TaskQuestion - - pending_questions = [ - TaskQuestion( - id=q["id"], - header=q.get( - "header", q["question"][:30] - ), # Default header from question - question=q["question"], - options=[ - QuestionOption( - label=opt["label"], description=opt.get("description") - ) - for opt in q.get("options", []) - ], - multi_select=q.get("multi_select", False), - response=None, - ) - for q in request.questions - ] - print(f"[DEBUG] Created {len(pending_questions)} pending_questions") - print(f"[DEBUG] pending_questions: {pending_questions}") - - task = WorkerTask( - id=str(uuid4()), - main_thread_id=thread_id, - user_id=user_id, - task_type=request.task_type, - description=request.description, - prompt=f"Implement: {request.description}", - repo_url=request.repo_url, - status=request.status, - plan_text=request.plan, # Store plan on task for UI display - pending_questions=pending_questions, # Store questions on task for UI - created_at=datetime.now(), - updated_at=datetime.now(), - ) - - await db.create_worker_task(task) - - # If task needs plan review, create a queue item - if request.status == TaskStatus.WAITING_PLAN_REVIEW and request.plan: - queue_item = QueueItem( - id=str(uuid4()), - main_thread_id=thread_id, - task_id=task.id, - user_id=user_id, - item_type=QueueItemType.PLAN_REVIEW, - title="Review Plan", - content=request.plan, - created_at=datetime.now(), - ) - await db.create_queue_item(queue_item) - - # If task has questions, create a queue item - if request.status == TaskStatus.WAITING_QUESTIONS and request.questions: - import json - - queue_item = QueueItem( - id=str(uuid4()), - main_thread_id=thread_id, - task_id=task.id, - user_id=user_id, - item_type=QueueItemType.QUESTION, - title="Answer Questions", - content=json.dumps(request.questions), # Serialize questions to string - created_at=datetime.now(), - ) - await db.create_queue_item(queue_item) - - return {"task_id": task.id, "status": task.status} - - class SeedSessionRequest(BaseModel): """Request to seed a session for testing.""" @@ -1637,7 +964,7 @@ async def reset_test_data(all: bool = False): """ TRUNCATE TABLE queue_items, messages, session_notifications, sessions, - worker_tasks, projects, conversations, main_threads + projects, conversations, main_threads CASCADE """ ) @@ -1677,11 +1004,11 @@ async def reset_test_data(all: bool = False): # Test-only reset async with db.connection() as conn: - # Get task IDs and workflow IDs for test users before deleting - test_tasks = await conn.fetch( - "SELECT id FROM worker_tasks WHERE user_id LIKE 'test-%'" + # Get session IDs and workflow IDs for test users before deleting + test_sessions = await conn.fetch( + "SELECT id FROM sessions WHERE user_id LIKE 'test-%'" ) - test_task_ids = [row["id"] for row in test_tasks] + test_session_ids = [row["id"] for row in test_sessions] test_workflow_ids = await conn.fetch( """ @@ -1701,7 +1028,6 @@ async def reset_test_data(all: bool = False): await conn.execute( "DELETE FROM messages WHERE conversation_id IN (SELECT id FROM conversations WHERE user_id LIKE 'test-%')" ) - await conn.execute("DELETE FROM worker_tasks WHERE user_id LIKE 'test-%'") await conn.execute("DELETE FROM projects WHERE user_id LIKE 'test-%'") await conn.execute("DELETE FROM conversations WHERE user_id LIKE 'test-%'") await conn.execute("DELETE FROM main_threads WHERE user_id LIKE 'test-%'") @@ -1730,8 +1056,8 @@ async def reset_test_data(all: bool = False): workflow_ids, ) - # Delete K8s namespaces for test tasks - if test_task_ids: + # Delete K8s namespaces for test sessions + if test_session_ids: try: from kubernetes.client.rest import ApiException from mainloop.services.k8s_namespace import get_k8s_client @@ -1744,8 +1070,8 @@ async def reset_test_data(all: bool = False): ) for ns in namespaces.items: - task_id = ns.metadata.labels.get("mainloop.dev/task-id", "") - if task_id in test_task_ids: + session_id = ns.metadata.labels.get("mainloop.dev/session-id", "") + if session_id in test_session_ids: try: core_v1.delete_namespace(name=ns.metadata.name) deleted_namespaces.append(ns.metadata.name) diff --git a/backend/src/mainloop/db/postgres.py b/backend/src/mainloop/db/postgres.py index 1a14182..a33fbda 100644 --- a/backend/src/mainloop/db/postgres.py +++ b/backend/src/mainloop/db/postgres.py @@ -19,8 +19,6 @@ Session, SessionNotification, SessionStatus, - TaskStatus, - WorkerTask, ) @@ -53,47 +51,11 @@ def _parse_json_field(value: Any) -> list | dict | None: ); CREATE INDEX IF NOT EXISTS idx_main_threads_user_id ON main_threads(user_id); --- Worker tasks -CREATE TABLE IF NOT EXISTS worker_tasks ( - id TEXT PRIMARY KEY, - main_thread_id TEXT NOT NULL REFERENCES main_threads(id), - user_id TEXT NOT NULL, - task_type TEXT NOT NULL, - description TEXT NOT NULL, - prompt TEXT NOT NULL, - model TEXT, - repo_url TEXT, - branch_name TEXT, - base_branch TEXT DEFAULT 'main', - status TEXT NOT NULL DEFAULT 'pending', - workflow_run_id TEXT, - worker_pod_name TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - started_at TIMESTAMPTZ, - completed_at TIMESTAMPTZ, - result JSONB, - error TEXT, - pr_url TEXT, - pr_number INTEGER, - commit_sha TEXT, - -- Conversation linking for routing - conversation_id TEXT, - message_id TEXT, - keywords TEXT[] DEFAULT '{}', - skip_plan BOOLEAN DEFAULT FALSE, - -- Interactive planning state - pending_questions JSONB, - plan_text TEXT -); -CREATE INDEX IF NOT EXISTS idx_worker_tasks_main_thread ON worker_tasks(main_thread_id); -CREATE INDEX IF NOT EXISTS idx_worker_tasks_user_id ON worker_tasks(user_id); -CREATE INDEX IF NOT EXISTS idx_worker_tasks_status ON worker_tasks(status); - -- Queue items (human-in-the-loop / inbox) CREATE TABLE IF NOT EXISTS queue_items ( id TEXT PRIMARY KEY, main_thread_id TEXT NOT NULL REFERENCES main_threads(id), - task_id TEXT REFERENCES worker_tasks(id), + task_id TEXT, user_id TEXT NOT NULL, item_type TEXT NOT NULL, priority TEXT NOT NULL DEFAULT 'normal', @@ -192,13 +154,6 @@ def _parse_json_field(value: Any) -> list | dict | None: -- Inline thread anchoring anchor_message_id TEXT REFERENCES messages(id), color VARCHAR(20), - -- Routing and task metadata - keywords TEXT[] DEFAULT '{}', - skip_plan BOOLEAN DEFAULT FALSE, - -- Interactive planning state - pending_questions JSONB, - plan_text TEXT, - -- Additional result data result JSONB ); CREATE INDEX IF NOT EXISTS idx_sessions_user_id ON sessions(user_id); @@ -224,54 +179,9 @@ def _parse_json_field(value: Any) -> list | dict | None: # Migration SQL for adding new columns to existing tables MIGRATION_SQL = """ --- Add new columns to worker_tasks if they don't exist +-- Add queue_items and conversation migrations DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='model') THEN - ALTER TABLE worker_tasks ADD COLUMN model TEXT; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='pr_number') THEN - ALTER TABLE worker_tasks ADD COLUMN pr_number INTEGER; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='conversation_id') THEN - ALTER TABLE worker_tasks ADD COLUMN conversation_id TEXT; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='message_id') THEN - ALTER TABLE worker_tasks ADD COLUMN message_id TEXT; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='keywords') THEN - ALTER TABLE worker_tasks ADD COLUMN keywords TEXT[] DEFAULT '{}'; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='skip_plan') THEN - ALTER TABLE worker_tasks ADD COLUMN skip_plan BOOLEAN DEFAULT FALSE; - END IF; - -- Issue tracking columns (plan phase) - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='issue_url') THEN - ALTER TABLE worker_tasks ADD COLUMN issue_url TEXT; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='issue_number') THEN - ALTER TABLE worker_tasks ADD COLUMN issue_number INTEGER; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='issue_etag') THEN - ALTER TABLE worker_tasks ADD COLUMN issue_etag TEXT; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='issue_last_modified') THEN - ALTER TABLE worker_tasks ADD COLUMN issue_last_modified TIMESTAMPTZ; - END IF; - -- ETag columns for PR polling - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='pr_etag') THEN - ALTER TABLE worker_tasks ADD COLUMN pr_etag TEXT; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='pr_last_modified') THEN - ALTER TABLE worker_tasks ADD COLUMN pr_last_modified TIMESTAMPTZ; - END IF; - -- Interactive planning columns - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='pending_questions') THEN - ALTER TABLE worker_tasks ADD COLUMN pending_questions JSONB; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='plan_text') THEN - ALTER TABLE worker_tasks ADD COLUMN plan_text TEXT; - END IF; -- Add read_at to queue_items IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='queue_items' AND column_name='read_at') THEN ALTER TABLE queue_items ADD COLUMN read_at TIMESTAMPTZ; @@ -290,70 +200,9 @@ def _parse_json_field(value: Any) -> list | dict | None: IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='conversations' AND column_name='claude_session_id') THEN ALTER TABLE conversations DROP COLUMN claude_session_id; END IF; - -- Add project_id to worker_tasks - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker_tasks' AND column_name='project_id') THEN - ALTER TABLE worker_tasks ADD COLUMN project_id TEXT REFERENCES projects(id); - END IF; -END $$; - --- Migrate existing repo URLs to projects and link tasks -DO $$ -DECLARE - task_record RECORD; - v_project_id TEXT; - repo_owner TEXT; - repo_name TEXT; - v_full_name TEXT; -BEGIN - -- Only run if projects table is empty (first migration) - IF NOT EXISTS (SELECT 1 FROM projects LIMIT 1) THEN - FOR task_record IN - SELECT DISTINCT user_id, repo_url - FROM worker_tasks - WHERE repo_url IS NOT NULL - LOOP - -- Parse owner/name from URL (handles both https://github.com/owner/repo and https://github.com/owner/repo.git) - repo_owner := split_part( - replace(replace(task_record.repo_url, 'https://github.com/', ''), '.git', ''), - '/', 1 - ); - repo_name := split_part( - replace(replace(task_record.repo_url, 'https://github.com/', ''), '.git', ''), - '/', 2 - ); - v_full_name := repo_owner || '/' || repo_name; - - -- Check if project already exists for this user - SELECT id INTO v_project_id - FROM projects - WHERE projects.user_id = task_record.user_id AND projects.full_name = v_full_name; - - IF v_project_id IS NULL THEN - v_project_id := gen_random_uuid()::TEXT; - INSERT INTO projects (id, user_id, owner, name, full_name, html_url, created_at, last_used_at) - VALUES ( - v_project_id, - task_record.user_id, - repo_owner, - repo_name, - v_full_name, - task_record.repo_url, - NOW(), - NOW() - ); - END IF; - - -- Update tasks to reference the project - UPDATE worker_tasks - SET project_id = v_project_id - WHERE worker_tasks.repo_url = task_record.repo_url AND worker_tasks.user_id = task_record.user_id; - END LOOP; - END IF; END $$; -- Create indexes if they don't exist -CREATE INDEX IF NOT EXISTS idx_worker_tasks_keywords ON worker_tasks USING GIN(keywords); -CREATE INDEX IF NOT EXISTS idx_worker_tasks_project ON worker_tasks(project_id); CREATE INDEX IF NOT EXISTS idx_queue_items_read_at ON queue_items(read_at); -- Add new columns to sessions for unified model @@ -404,19 +253,6 @@ def _parse_json_field(value: Any) -> list | dict | None: IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='sessions' AND column_name='commit_sha') THEN ALTER TABLE sessions ADD COLUMN commit_sha TEXT; END IF; - -- Routing and planning fields - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='sessions' AND column_name='keywords') THEN - ALTER TABLE sessions ADD COLUMN keywords TEXT[] DEFAULT '{}'; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='sessions' AND column_name='skip_plan') THEN - ALTER TABLE sessions ADD COLUMN skip_plan BOOLEAN DEFAULT FALSE; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='sessions' AND column_name='pending_questions') THEN - ALTER TABLE sessions ADD COLUMN pending_questions JSONB; - END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='sessions' AND column_name='plan_text') THEN - ALTER TABLE sessions ADD COLUMN plan_text TEXT; - END IF; IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='sessions' AND column_name='result') THEN ALTER TABLE sessions ADD COLUMN result JSONB; END IF; @@ -432,7 +268,6 @@ def _parse_json_field(value: Any) -> list | dict | None: -- Create session indexes CREATE INDEX IF NOT EXISTS idx_sessions_repo_url ON sessions(repo_url); CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_id); -CREATE INDEX IF NOT EXISTS idx_sessions_keywords ON sessions USING GIN(keywords); CREATE INDEX IF NOT EXISTS idx_sessions_anchor ON sessions(anchor_message_id); """ @@ -678,266 +513,6 @@ def _row_to_main_thread(self, row: asyncpg.Record) -> MainThread: ), ) - # ============= Worker Task Operations ============= - - async def create_worker_task(self, task: WorkerTask) -> WorkerTask: - """Create a new worker task.""" - if not self._pool: - return task - async with self.connection() as conn: - # Serialize pending_questions to JSON for storage - import json - - pending_questions_json = ( - json.dumps([q.model_dump() for q in task.pending_questions]) - if task.pending_questions - else None - ) - - await conn.execute( - """ - INSERT INTO worker_tasks - (id, main_thread_id, user_id, task_type, description, prompt, model, - repo_url, branch_name, base_branch, status, created_at, - conversation_id, message_id, keywords, skip_plan, plan_text, pending_questions) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) - """, - task.id, - task.main_thread_id, - task.user_id, - task.task_type, - task.description, - task.prompt, - task.model, - task.repo_url, - task.branch_name, - task.base_branch, - task.status.value, - task.created_at, - task.conversation_id, - task.message_id, - task.keywords, - task.skip_plan, - task.plan_text, - pending_questions_json, - ) - return task - - async def get_worker_task(self, task_id: str) -> WorkerTask | None: - """Get a worker task by ID.""" - if not self._pool: - return None - async with self.connection() as conn: - row = await conn.fetchrow( - "SELECT * FROM worker_tasks WHERE id = $1", task_id - ) - if not row: - return None - return self._row_to_worker_task(row) - - async def list_worker_tasks( - self, - user_id: str, - status: str | None = None, - project_id: str | None = None, - limit: int = 50, - ) -> list[WorkerTask]: - """List worker tasks for a user.""" - if not self._pool: - return [] - - query = "SELECT * FROM worker_tasks WHERE user_id = $1" - params: list[Any] = [user_id] - - if status: - query += f" AND status = ${len(params) + 1}" - params.append(status) - - if project_id: - query += f" AND project_id = ${len(params) + 1}" - params.append(project_id) - - query += f" ORDER BY created_at DESC LIMIT ${len(params) + 1}" - params.append(limit) - - async with self.connection() as conn: - rows = await conn.fetch(query, *params) - return [self._row_to_worker_task(row) for row in rows] - - async def update_worker_task( - self, - task_id: str, - status: TaskStatus | None = None, - workflow_run_id: str | None = None, - worker_pod_name: str | None = None, - started_at: datetime | None = None, - completed_at: datetime | None = None, - result: dict | None = None, - error: str | None = None, - project_id: str | None = None, - # Issue fields (plan phase) - issue_url: str | None = None, - issue_number: int | None = None, - issue_etag: str | None = None, - issue_last_modified: datetime | None = None, - # PR fields (implementation phase) - pr_url: str | None = None, - pr_number: int | None = None, - pr_etag: str | None = None, - pr_last_modified: datetime | None = None, - commit_sha: str | None = None, - branch_name: str | None = None, - # Interactive planning fields - pending_questions: list[dict] | None = None, - plan_text: str | None = None, - ): - """Update worker task fields.""" - if not self._pool: - return - updates = [] - params = [] - param_idx = 1 - - if status is not None: - updates.append(f"status = ${param_idx}") - params.append(status.value if isinstance(status, TaskStatus) else status) - param_idx += 1 - if workflow_run_id is not None: - updates.append(f"workflow_run_id = ${param_idx}") - params.append(workflow_run_id) - param_idx += 1 - if worker_pod_name is not None: - updates.append(f"worker_pod_name = ${param_idx}") - params.append(worker_pod_name) - param_idx += 1 - if started_at is not None: - updates.append(f"started_at = ${param_idx}") - params.append(started_at) - param_idx += 1 - if completed_at is not None: - updates.append(f"completed_at = ${param_idx}") - params.append(completed_at) - param_idx += 1 - if result is not None: - updates.append(f"result = ${param_idx}") - params.append(json.dumps(result)) - param_idx += 1 - if error is not None: - updates.append(f"error = ${param_idx}") - params.append(error) - param_idx += 1 - if project_id is not None: - updates.append(f"project_id = ${param_idx}") - params.append(project_id) - param_idx += 1 - if pr_url is not None: - updates.append(f"pr_url = ${param_idx}") - params.append(pr_url) - param_idx += 1 - if pr_number is not None: - updates.append(f"pr_number = ${param_idx}") - params.append(pr_number) - param_idx += 1 - if commit_sha is not None: - updates.append(f"commit_sha = ${param_idx}") - params.append(commit_sha) - param_idx += 1 - if branch_name is not None: - updates.append(f"branch_name = ${param_idx}") - params.append(branch_name) - param_idx += 1 - if issue_url is not None: - updates.append(f"issue_url = ${param_idx}") - params.append(issue_url) - param_idx += 1 - if issue_number is not None: - updates.append(f"issue_number = ${param_idx}") - params.append(issue_number) - param_idx += 1 - if issue_etag is not None: - updates.append(f"issue_etag = ${param_idx}") - params.append(issue_etag) - param_idx += 1 - if issue_last_modified is not None: - updates.append(f"issue_last_modified = ${param_idx}") - params.append(issue_last_modified) - param_idx += 1 - if pr_etag is not None: - updates.append(f"pr_etag = ${param_idx}") - params.append(pr_etag) - param_idx += 1 - if pr_last_modified is not None: - updates.append(f"pr_last_modified = ${param_idx}") - params.append(pr_last_modified) - param_idx += 1 - # pending_questions: empty list [] means clear, list with items means set - # None means don't update (standard pattern) - if pending_questions is not None: - updates.append(f"pending_questions = ${param_idx}") - # Empty list clears (stores null), non-empty stores as JSON string - # asyncpg requires explicit JSON serialization for JSONB columns - params.append(json.dumps(pending_questions) if pending_questions else None) - param_idx += 1 - if plan_text is not None: - updates.append(f"plan_text = ${param_idx}") - params.append(plan_text) - param_idx += 1 - - params.append(task_id) - - if updates: - async with self.connection() as conn: - await conn.execute( - f"UPDATE worker_tasks SET {', '.join(updates)} WHERE id = ${param_idx}", - *params, - ) - - def _row_to_worker_task(self, row: asyncpg.Record) -> WorkerTask: - return WorkerTask( - id=row["id"], - main_thread_id=row["main_thread_id"], - user_id=row["user_id"], - task_type=row["task_type"], - description=row["description"], - prompt=row["prompt"], - model=row.get("model"), - repo_url=row["repo_url"], - project_id=row.get("project_id"), - branch_name=row["branch_name"], - base_branch=row["base_branch"], - status=TaskStatus(row["status"]), - workflow_run_id=row["workflow_run_id"], - worker_pod_name=row["worker_pod_name"], - created_at=row["created_at"], - started_at=row["started_at"], - completed_at=row["completed_at"], - result=( - row["result"] - if isinstance(row["result"], dict) - else (json.loads(row["result"]) if row["result"] else None) - ), - error=row["error"], - # Issue fields (plan phase) - issue_url=row.get("issue_url"), - issue_number=row.get("issue_number"), - issue_etag=row.get("issue_etag"), - issue_last_modified=row.get("issue_last_modified"), - # PR fields (implementation phase) - pr_url=row["pr_url"], - pr_number=row.get("pr_number"), - pr_etag=row.get("pr_etag"), - pr_last_modified=row.get("pr_last_modified"), - commit_sha=row["commit_sha"], - conversation_id=row.get("conversation_id"), - message_id=row.get("message_id"), - keywords=list(row["keywords"]) if row.get("keywords") else [], - skip_plan=row.get("skip_plan", False), - # Interactive planning state - # Handle both JSONB (returns list) and legacy string data - pending_questions=_parse_json_field(row.get("pending_questions")), - plan_text=row.get("plan_text"), - ) - # ============= Project Operations ============= async def create_project(self, project: Project) -> Project: @@ -1650,11 +1225,10 @@ async def create_session(self, session: Session) -> Session: repo_url, project_id, branch_name, base_branch, model, issue_url, issue_number, issue_etag, issue_last_modified, pr_url, pr_number, pr_etag, pr_last_modified, commit_sha, - anchor_message_id, color, - keywords, skip_plan, pending_questions, plan_text, result) + anchor_message_id, color, result) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, - $29, $30, $31, $32, $33, $34, $35) + $29, $30, $31) """, session.id, session.user_id, @@ -1690,15 +1264,6 @@ async def create_session(self, session: Session) -> Session: # Inline thread anchoring session.anchor_message_id, session.color, - # Routing and planning fields - session.keywords, - session.skip_plan, - ( - json.dumps([q.model_dump() for q in session.pending_questions]) - if session.pending_questions - else None - ), - session.plan_text, json.dumps(session.result) if session.result else None, ) return session @@ -1766,9 +1331,6 @@ async def update_session( # Inline thread anchoring anchor_message_id: str | None = None, color: str | None = None, - # Planning fields - pending_questions: list | None = None, - plan_text: str | None = None, result: dict | None = None, ): """Update session fields.""" @@ -1862,24 +1424,6 @@ async def update_session( updates.append(f"color = ${param_idx}") params.append(color) param_idx += 1 - # Planning fields - if pending_questions is not None: - updates.append(f"pending_questions = ${param_idx}") - params.append( - json.dumps( - [ - q.model_dump() if hasattr(q, "model_dump") else q - for q in pending_questions - ] - ) - if pending_questions - else None - ) - param_idx += 1 - if plan_text is not None: - updates.append(f"plan_text = ${param_idx}") - params.append(plan_text) - param_idx += 1 if result is not None: updates.append(f"result = ${param_idx}") params.append(json.dumps(result)) @@ -1895,17 +1439,6 @@ async def update_session( ) def _row_to_session(self, row: asyncpg.Record) -> Session: - from models import SessionQuestion - - # Parse pending_questions JSON - pending_questions = None - raw_questions = row.get("pending_questions") - if raw_questions: - parsed = _parse_json_field(raw_questions) - if parsed and isinstance(parsed, list): - pending_questions = [SessionQuestion(**q) for q in parsed] - - # Parse result JSON result = _parse_json_field(row.get("result")) return Session( @@ -1943,11 +1476,6 @@ def _row_to_session(self, row: asyncpg.Record) -> Session: # Inline thread anchoring anchor_message_id=row.get("anchor_message_id"), color=row.get("color"), - # Routing and planning fields - keywords=row.get("keywords", []), - skip_plan=row.get("skip_plan", False), - pending_questions=pending_questions, - plan_text=row.get("plan_text"), result=result, ) diff --git a/backend/src/mainloop/services/chat_handler.py b/backend/src/mainloop/services/chat_handler.py index 060d09d..2fb7133 100644 --- a/backend/src/mainloop/services/chat_handler.py +++ b/backend/src/mainloop/services/chat_handler.py @@ -17,7 +17,6 @@ from dbos import SetWorkflowID from mainloop.config import settings from mainloop.db import db -from mainloop.services.task_router import extract_keywords from mainloop.workflows.dbos_config import worker_queue from models import ( @@ -97,7 +96,6 @@ async def spawn_session_impl(args: dict[str, Any]) -> dict[str, Any]: print(f"[SESSION] spawn_session_impl called with args: {args}") title = args.get("title", "") repo_url = args.get("repo_url") # Optional - if provided, this is code work - skip_plan = args.get("skip_plan", False) request_message_id = args.get( "request_message_id" ) # ID of the user's original request @@ -169,9 +167,6 @@ async def spawn_session_impl(args: dict[str, Any]) -> dict[str, Any]: conv = await db.create_conversation(user_id, title=title) print(f"[SESSION] Created conversation: {conv.id}") - # Extract keywords for code work - keywords = extract_keywords(prompt) if is_code_work else [] - # Create project from repo URL if provided (so it shows in sidebar) project_id = None if repo_url: @@ -201,8 +196,6 @@ async def spawn_session_impl(args: dict[str, Any]) -> dict[str, Any]: # Code work fields (optional) repo_url=repo_url, project_id=project_id, - skip_plan=skip_plan, - keywords=keywords, ) session = await db.create_session(session) print(f"[SESSION] Session saved to DB: {session.id}") @@ -286,7 +279,6 @@ def create_spawn_session_tool( "title": str, # Short title for the session (e.g., "Add quickstart to README") "request_message_id": str, # ID from conversation [ID: ...] with the user's request "repo_url": str, # Optional - if provided, enables code work with GitHub - "skip_plan": bool, # Optional - skip planning phase for code work }, ) async def spawn_session(args: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/src/mainloop/services/github_pr.py b/backend/src/mainloop/services/github_pr.py index 05a7663..2de6a7f 100644 --- a/backend/src/mainloop/services/github_pr.py +++ b/backend/src/mainloop/services/github_pr.py @@ -1078,293 +1078,6 @@ async def get_comment_reactions( return [] -def format_plan_for_issue(plan_text: str) -> str: - """Format implementation plan as a GitHub issue comment with approval instructions. - - Args: - plan_text: The plan markdown content - - Returns: - Markdown-formatted comment body with approval instructions - - """ - lines = [ - "## 📋 Implementation Plan", - "", - plan_text, - "", - "---", - "", - "**To approve:** React with 👍 on this comment, or reply `lgtm`", - "", - "**To request changes:** Reply with your feedback", - "", - "_You can also review in the [Mainloop app](https://mainloop.olds.network)._", - ] - return "\n".join(lines) - - -def format_questions_for_issue(questions: list[dict]) -> str: - """Format task questions as a GitHub issue comment. - - Args: - questions: List of question dicts with id, header, question, options, multi_select - - Returns: - Markdown-formatted comment body - - """ - lines = [ - "## ❓ Questions", - "", - "I have some questions before I can create the implementation plan:", - "", - ] - - for i, q in enumerate(questions, 1): - header = q.get("header", f"Question {i}") - question_text = q.get("question", "") - options = q.get("options", []) - multi = q.get("multi_select", False) - - lines.append(f"### {i}. [{header}]") - lines.append(f"**{question_text}**") - if multi: - lines.append("_(Select all that apply)_") - lines.append("") - - for j, opt in enumerate(options): - letter = chr(ord("A") + j) - label = opt.get("label", "") - desc = opt.get("description", "") - if desc: - lines.append(f"- **{letter})** {label} - {desc}") - else: - lines.append(f"- **{letter})** {label}") - - lines.append("") - - # Add instructions for responding - lines.extend( - [ - "---", - "", - "**To answer**, reply to this issue with your choices:", - "```", - ] - ) - for i in range(1, len(questions) + 1): - lines.append(f"{i}. A") - lines.extend( - [ - "```", - "Or answer using the option label: `1. JWT tokens`", - "", - "_You can also answer in the [Mainloop app](https://mainloop.olds.network)._", - ] - ) - - return "\n".join(lines) - - -def parse_plan_approval_from_comment(comment_body: str) -> str | None: - """Parse plan approval or revision request from an issue comment. - - Returns: - - "approve" if the comment approves the plan - - The revision text if requesting changes - - None if no approval/revision detected - - """ - body = comment_body.strip() - body_lower = body.lower() - - # Skip bot comments - if ( - body_lower.startswith("🤖") - or body_lower.startswith("##") - or body_lower.startswith("❓") - or body_lower.startswith("📋") - ): - return None - - # Approval keywords (case-insensitive) - approval_patterns = [ - "lgtm", - "looks good", - "approve", - "/approve", - "ship it", - "go ahead", - "proceed", - "approved", - ] - - # Check for approval - must be short or start with approval keyword - for pattern in approval_patterns: - if body_lower == pattern or body_lower.startswith(pattern): - return "approve" - - # Check for emoji-only approval - if body.strip() in ["👍", "✅", "🚀", "💯"]: - return "approve" - - # If the comment is substantial (>20 chars) and not an approval, - # treat it as revision feedback - if len(body) > 20: - return body # Return as revision text - - return None - - -def parse_question_answers_from_comment( - comment_body: str, - questions: list[dict], -) -> dict[str, str] | None: - """Parse answers to questions from an issue comment. - - Supports formats: - - "1. A" or "1: A" or "1) A" - - "1. JWT tokens" (by label) - - "Auth method: JWT tokens" (by header) - - Multi-select: "1. A, B" or "1. A and B" - - Args: - comment_body: The comment text to parse - questions: List of question dicts to match against - - Returns: - Dict mapping question ID to answer string, or None if no valid answers found - - """ - import re - - body = comment_body.strip() - answers: dict[str, str] = {} - - # Skip if this looks like a bot comment or system message - if body.startswith("🤖") or body.startswith("##") or body.startswith("❓"): - return None - - # Build lookup maps for each question - for i, q in enumerate(questions, 1): - q_id = q.get("id", str(i)) - header = q.get("header", "").lower().strip() - options = q.get("options", []) - - # Build option lookup: letter -> label, label_lower -> label - option_by_letter: dict[str, str] = {} - option_by_label: dict[str, str] = {} - for j, opt in enumerate(options): - letter = chr(ord("A") + j).upper() - label = opt.get("label", "") - option_by_letter[letter] = label - option_by_label[label.lower().strip()] = label - - # Try to find answer by number: "1. A" or "1: JWT" or "1) A, B" - patterns = [ - rf"^{i}[\.\:\)]\s*(.+)$", # "1. answer" or "1: answer" or "1) answer" - rf"^\#{i}[\.\:\)]?\s*(.+)$", # "#1. answer" - ] - - for pattern in patterns: - match = re.search(pattern, body, re.MULTILINE | re.IGNORECASE) - if match: - answer_text = match.group(1).strip() - resolved = _resolve_answer( - answer_text, option_by_letter, option_by_label - ) - if resolved: - answers[q_id] = resolved - break - - # Also try by header: "Auth method: JWT tokens" - if q_id not in answers and header: - header_pattern = rf"{re.escape(header)}[\:\s]+(.+)" - match = re.search(header_pattern, body, re.MULTILINE | re.IGNORECASE) - if match: - answer_text = match.group(1).strip() - resolved = _resolve_answer( - answer_text, option_by_letter, option_by_label - ) - if resolved: - answers[q_id] = resolved - - return answers if answers else None - - -def _resolve_answer( - answer_text: str, - option_by_letter: dict[str, str], - option_by_label: dict[str, str], -) -> str | None: - """Resolve an answer text to an option label. - - Args: - answer_text: Raw answer like "A" or "JWT tokens" or "A, B" - option_by_letter: Map of letter -> label - option_by_label: Map of lowercase label -> label - - Returns: - The resolved label(s) or None - - """ - import re - - answer_text = answer_text.strip() - - # First, try exact match on the whole string - if answer_text.upper() in option_by_letter: - return option_by_letter[answer_text.upper()] - if answer_text.lower() in option_by_label: - return option_by_label[answer_text.lower()] - - # Try fuzzy match on whole string first (e.g., "JWT" matches "JWT tokens") - for label_lower, label in option_by_label.items(): - if ( - label_lower.startswith(answer_text.lower()) - or answer_text.lower() in label_lower - ): - return label - - # Handle multi-select: "A, B" or "A and B" - # Only split on comma or "and" - not spaces (to preserve "JWT tokens") - parts = re.split(r"\s*[,&]\s*|\s+and\s+", answer_text) - if len(parts) == 1: - return None # Single part already tried above - - resolved_parts = [] - seen = set() # Avoid duplicates - - for part in parts: - part = part.strip().rstrip(".,;") - if not part: - continue - - resolved = None - - # Check if it's a letter (A, B, C, etc.) - if len(part) == 1 and part.upper() in option_by_letter: - resolved = option_by_letter[part.upper()] - # Check if it matches an option label - elif part.lower() in option_by_label: - resolved = option_by_label[part.lower()] - # Fuzzy match - else: - for label_lower, label in option_by_label.items(): - if label_lower.startswith(part.lower()) or part.lower() in label_lower: - resolved = label - break - - if resolved and resolved not in seen: - resolved_parts.append(resolved) - seen.add(resolved) - - if resolved_parts: - return ", ".join(resolved_parts) - return None - - # ============= Project/Repo Metadata Functions ============= diff --git a/backend/src/mainloop/services/k8s_jobs.py b/backend/src/mainloop/services/k8s_jobs.py index 4fd862a..b97d433 100644 --- a/backend/src/mainloop/services/k8s_jobs.py +++ b/backend/src/mainloop/services/k8s_jobs.py @@ -1,7 +1,6 @@ -"""Kubernetes Job management for worker tasks.""" +"""Kubernetes Job management for session tasks.""" import logging -from typing import Literal from kubernetes import client from kubernetes.client.rest import ApiException @@ -14,34 +13,22 @@ JOB_TTL_SECONDS = 3600 # Keep completed jobs for 1 hour -async def create_worker_job( - task_id: str, +async def create_session_job( + session_id: str, namespace: str, prompt: str, - mode: Literal["plan", "implement", "feedback", "fix"], callback_url: str, model: str | None = None, - repo_url: str | None = None, - pr_number: int | None = None, - issue_number: int | None = None, - branch_name: str | None = None, - feedback_context: str | None = None, iteration: int = 0, ) -> str: - """Create a worker Job in the task namespace. + """Create a session Job in the session namespace. Args: - task_id: The task ID + session_id: The session ID namespace: Target namespace for the Job - prompt: The task prompt/description - mode: "plan" for issue with plan, "implement" for code, "feedback" for comments, "fix" for CI + prompt: The full prompt with conversation context callback_url: URL to POST results to - model: Claude model to use (defaults to settings.claude_worker_model) - repo_url: Repository URL to clone - pr_number: PR number (for implement, feedback, fix modes) - issue_number: Issue number (for implement mode - references plan issue) - branch_name: Branch name to use (for implement, feedback, fix modes) - feedback_context: PR comments/feedback to address + model: Claude model to use (defaults to settings.claude_model) iteration: Iteration number for jobs (ensures unique names) Returns: @@ -50,11 +37,11 @@ async def create_worker_job( """ _, batch_v1 = get_k8s_client() - # Include iteration in job name to ensure uniqueness across feedback rounds + # Include iteration in job name to ensure uniqueness across rounds if iteration > 0: - job_name = f"worker-{task_id[:8]}-{mode[:3]}-{iteration}" + job_name = f"session-{session_id[:8]}-{iteration}" else: - job_name = f"worker-{task_id[:8]}-{mode[:3]}" + job_name = f"session-{session_id[:8]}" # Check if job already exists and is completed - delete it to allow retry try: @@ -78,14 +65,14 @@ async def create_worker_job( if e.status != 404: raise # Job doesn't exist, continue with creation - model = model or settings.claude_worker_model + + model = model or settings.claude_model # Environment variables for the job env_vars = [ - client.V1EnvVar(name="TASK_ID", value=task_id), + client.V1EnvVar(name="SESSION_ID", value=session_id), client.V1EnvVar(name="TASK_PROMPT", value=prompt), client.V1EnvVar(name="CALLBACK_URL", value=callback_url), - client.V1EnvVar(name="MODE", value=mode), client.V1EnvVar(name="CLAUDE_MODEL", value=model), # Claude credentials from secret client.V1EnvVar( @@ -97,7 +84,7 @@ async def create_worker_job( ) ), ), - # GitHub token from secret + # GitHub token from secret (optional, for gh CLI access) client.V1EnvVar( name="GH_TOKEN", value_from=client.V1EnvVarSource( @@ -110,20 +97,6 @@ async def create_worker_job( ), ] - # Add optional env vars - if repo_url: - env_vars.append(client.V1EnvVar(name="REPO_URL", value=repo_url)) - if pr_number: - env_vars.append(client.V1EnvVar(name="PR_NUMBER", value=str(pr_number))) - if issue_number: - env_vars.append(client.V1EnvVar(name="ISSUE_NUMBER", value=str(issue_number))) - if branch_name: - env_vars.append(client.V1EnvVar(name="BRANCH_NAME", value=branch_name)) - if feedback_context: - env_vars.append( - client.V1EnvVar(name="FEEDBACK_CONTEXT", value=feedback_context) - ) - # Job spec job = client.V1Job( metadata=client.V1ObjectMeta( @@ -131,8 +104,7 @@ async def create_worker_job( namespace=namespace, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, - "mainloop.dev/mode": mode, + "mainloop.dev/session-id": session_id, }, ), spec=client.V1JobSpec( @@ -142,7 +114,7 @@ async def create_worker_job( metadata=client.V1ObjectMeta( labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, }, ), spec=client.V1PodSpec( @@ -196,11 +168,11 @@ async def create_worker_job( return job_name -async def get_job_status(task_id: str, namespace: str) -> dict | None: - """Get the status of a worker Job. +async def get_job_status(session_id: str, namespace: str) -> dict | None: + """Get the status of a session Job. Args: - task_id: The task ID + session_id: The session ID namespace: Namespace where the Job is running Returns: @@ -212,7 +184,7 @@ async def get_job_status(task_id: str, namespace: str) -> dict | None: try: jobs = batch_v1.list_namespaced_job( namespace=namespace, - label_selector=f"mainloop.dev/task-id={task_id}", + label_selector=f"mainloop.dev/session-id={session_id}", ) if not jobs.items: @@ -238,11 +210,11 @@ async def get_job_status(task_id: str, namespace: str) -> dict | None: raise -async def delete_job(task_id: str, namespace: str) -> None: - """Delete a worker Job. +async def delete_job(session_id: str, namespace: str) -> None: + """Delete a session Job. Args: - task_id: The task ID + session_id: The session ID namespace: Namespace where the Job is running """ @@ -251,7 +223,7 @@ async def delete_job(task_id: str, namespace: str) -> None: try: jobs = batch_v1.list_namespaced_job( namespace=namespace, - label_selector=f"mainloop.dev/task-id={task_id}", + label_selector=f"mainloop.dev/session-id={session_id}", ) for job in jobs.items: @@ -266,16 +238,16 @@ async def delete_job(task_id: str, namespace: str) -> None: except ApiException as e: if e.status == 404: - logger.info(f"No jobs found for task {task_id} in {namespace}") + logger.info(f"No jobs found for session {session_id} in {namespace}") else: raise -async def get_job_logs(task_id: str, namespace: str) -> str | None: - """Get logs from a worker Job's pod. +async def get_job_logs(session_id: str, namespace: str) -> str | None: + """Get logs from a session Job's pod. Args: - task_id: The task ID + session_id: The session ID namespace: Namespace where the Job is running Returns: @@ -287,7 +259,7 @@ async def get_job_logs(task_id: str, namespace: str) -> str | None: try: pods = core_v1.list_namespaced_pod( namespace=namespace, - label_selector=f"mainloop.dev/task-id={task_id}", + label_selector=f"mainloop.dev/session-id={session_id}", ) if not pods.items: diff --git a/backend/src/mainloop/services/k8s_namespace.py b/backend/src/mainloop/services/k8s_namespace.py index 05909cf..990f80b 100644 --- a/backend/src/mainloop/services/k8s_namespace.py +++ b/backend/src/mainloop/services/k8s_namespace.py @@ -7,8 +7,8 @@ logger = logging.getLogger(__name__) -# Namespace prefix for task namespaces -TASK_NAMESPACE_PREFIX = "task-" +# Namespace prefix for session namespaces +SESSION_NAMESPACE_PREFIX = "mainloop-session-" # Secrets to copy from mainloop namespace to task namespaces DEFAULT_SECRETS_TO_COPY = [ @@ -61,25 +61,25 @@ def get_networking_client() -> client.NetworkingV1Api: return client.NetworkingV1Api() -async def create_task_namespace(task_id: str) -> str: - """Create an isolated namespace for a task. +async def create_session_namespace(session_id: str) -> str: + """Create an isolated namespace for a session. Args: - task_id: The task ID (will be used in namespace name) + session_id: The session ID (will be used in namespace name) Returns: The namespace name that was created """ core_v1, _ = get_k8s_client() - namespace_name = f"{TASK_NAMESPACE_PREFIX}{task_id[:8]}" + namespace_name = f"{SESSION_NAMESPACE_PREFIX}{session_id[:8]}" namespace = client.V1Namespace( metadata=client.V1ObjectMeta( name=namespace_name, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, }, ) ) @@ -97,14 +97,14 @@ async def create_task_namespace(task_id: str) -> str: async def copy_secrets_to_namespace( - task_id: str, + session_id: str, namespace: str, secrets: list[str] | None = None, ) -> None: - """Copy secrets from mainloop namespace to task namespace. + """Copy secrets from mainloop namespace to session namespace. Args: - task_id: The task ID + session_id: The session ID namespace: Target namespace to copy secrets to secrets: List of secret names to copy (defaults to DEFAULT_SECRETS_TO_COPY) @@ -127,7 +127,7 @@ async def copy_secrets_to_namespace( namespace=namespace, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, "mainloop.dev/copied-from": SOURCE_NAMESPACE, }, ), @@ -153,13 +153,13 @@ async def copy_secrets_to_namespace( raise -async def setup_worker_rbac(task_id: str, namespace: str) -> None: - """Create ServiceAccount and RoleBinding for worker in task namespace. +async def setup_session_rbac(session_id: str, namespace: str) -> None: + """Create ServiceAccount and RoleBinding for worker in session namespace. This gives the worker pod permissions to deploy resources within its namespace. Args: - task_id: The task ID + session_id: The session ID namespace: Target namespace """ @@ -173,7 +173,7 @@ async def setup_worker_rbac(task_id: str, namespace: str) -> None: namespace=namespace, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, }, ), ) @@ -198,7 +198,7 @@ async def setup_worker_rbac(task_id: str, namespace: str) -> None: namespace=namespace, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, }, ), subjects=[ @@ -225,19 +225,21 @@ async def setup_worker_rbac(task_id: str, namespace: str) -> None: raise -async def apply_task_namespace_network_policies(task_id: str, namespace: str) -> None: - """Apply network policies to task namespace for security isolation. +async def apply_session_namespace_network_policies( + session_id: str, namespace: str +) -> None: + """Apply network policies to session namespace for security isolation. Applies strict network policies: - Default deny-all ingress and egress - Allow DNS (required for internet access) - Allow egress to internet ONLY (blocks all cluster internal communication) - This ensures worker agents can only communicate with external services, + This ensures session agents can only communicate with external services, not with other cluster resources or each other. Args: - task_id: The task ID + session_id: The session ID namespace: Target namespace """ @@ -250,7 +252,7 @@ async def apply_task_namespace_network_policies(task_id: str, namespace: str) -> namespace=namespace, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, }, ), spec=client.V1NetworkPolicySpec( @@ -266,7 +268,7 @@ async def apply_task_namespace_network_policies(task_id: str, namespace: str) -> namespace=namespace, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, }, ), spec=client.V1NetworkPolicySpec( @@ -296,7 +298,7 @@ async def apply_task_namespace_network_policies(task_id: str, namespace: str) -> namespace=namespace, labels={ "app.kubernetes.io/managed-by": "mainloop", - "mainloop.dev/task-id": task_id, + "mainloop.dev/session-id": session_id, }, ), spec=client.V1NetworkPolicySpec( @@ -323,8 +325,40 @@ async def apply_task_namespace_network_policies(task_id: str, namespace: str) -> ), ) + # Allow egress to mainloop namespace (for callback to backend) + allow_mainloop_policy = client.V1NetworkPolicy( + metadata=client.V1ObjectMeta( + name="allow-mainloop-callback", + namespace=namespace, + labels={ + "app.kubernetes.io/managed-by": "mainloop", + "mainloop.dev/session-id": session_id, + }, + ), + spec=client.V1NetworkPolicySpec( + pod_selector=client.V1LabelSelector(), + policy_types=["Egress"], + egress=[ + client.V1NetworkPolicyEgressRule( + to=[ + client.V1NetworkPolicyPeer( + namespace_selector=client.V1LabelSelector( + match_labels={"kubernetes.io/metadata.name": "mainloop"} + ) + ) + ], + ) + ], + ), + ) + # Apply policies - for policy in [deny_all_policy, allow_dns_policy, allow_internet_policy]: + for policy in [ + deny_all_policy, + allow_dns_policy, + allow_internet_policy, + allow_mainloop_policy, + ]: try: networking_v1.create_namespaced_network_policy( namespace=namespace, body=policy @@ -339,15 +373,15 @@ async def apply_task_namespace_network_policies(task_id: str, namespace: str) -> raise -async def delete_task_namespace(task_id: str) -> None: - """Delete a task namespace and all its resources. +async def delete_session_namespace(session_id: str) -> None: + """Delete a session namespace and all its resources. Args: - task_id: The task ID (used to construct namespace name) + session_id: The session ID (used to construct namespace name) """ core_v1, _ = get_k8s_client() - namespace_name = f"{TASK_NAMESPACE_PREFIX}{task_id[:8]}" + namespace_name = f"{SESSION_NAMESPACE_PREFIX}{session_id[:8]}" try: core_v1.delete_namespace( @@ -364,18 +398,18 @@ async def delete_task_namespace(task_id: str) -> None: raise -async def namespace_exists(task_id: str) -> bool: - """Check if a task namespace exists. +async def session_namespace_exists(session_id: str) -> bool: + """Check if a session namespace exists. Args: - task_id: The task ID + session_id: The session ID Returns: True if namespace exists, False otherwise """ core_v1, _ = get_k8s_client() - namespace_name = f"{TASK_NAMESPACE_PREFIX}{task_id[:8]}" + namespace_name = f"{SESSION_NAMESPACE_PREFIX}{session_id[:8]}" try: core_v1.read_namespace(name=namespace_name) @@ -386,8 +420,8 @@ async def namespace_exists(task_id: str) -> bool: raise -async def list_task_namespaces() -> list[str]: - """List all task namespaces managed by mainloop. +async def list_session_namespaces() -> list[str]: + """List all session namespaces managed by mainloop. Returns: List of namespace names diff --git a/backend/src/mainloop/services/task_router.py b/backend/src/mainloop/services/task_router.py deleted file mode 100644 index 52ffab9..0000000 --- a/backend/src/mainloop/services/task_router.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Task routing service - matches user messages to active tasks.""" - -import logging -import re -from dataclasses import dataclass - -from mainloop.db import db - -from models import TaskStatus, WorkerTask - -logger = logging.getLogger(__name__) - - -@dataclass -class RouteMatch: - """A potential task match for routing.""" - - task: WorkerTask - confidence: float # 0.0 to 1.0 - match_reasons: list[str] - - -def extract_keywords(message: str) -> list[str]: - """Extract routing keywords from a user message. - - Looks for: - - Domain names (understanding.news, example.com) - - Repository names (owner/repo) - - Technical terms (background, header, button, etc.) - - Color terms - """ - keywords: list[str] = [] - message_lower = message.lower() - - # Extract domain-like patterns - domain_pattern = r"\b([a-z0-9-]+\.(?:com|org|net|io|dev|news|app|co))\b" - keywords.extend(re.findall(domain_pattern, message_lower)) - - # Extract GitHub repo patterns (owner/repo) - repo_pattern = r"\b([a-z0-9_-]+/[a-z0-9_-]+)\b" - keywords.extend(re.findall(repo_pattern, message_lower)) - - # Extract common UI/code terms - ui_terms = [ - "background", - "header", - "footer", - "button", - "color", - "style", - "layout", - "font", - "image", - "icon", - "nav", - "navbar", - "sidebar", - "menu", - "form", - "input", - "modal", - "dialog", - "card", - "table", - "list", - "api", - "endpoint", - "route", - "auth", - "login", - "signup", - "database", - "schema", - "test", - "bug", - "fix", - "feature", - ] - for term in ui_terms: - if term in message_lower: - keywords.append(term) - - # Extract color terms - colors = [ - "red", - "blue", - "green", - "yellow", - "pink", - "grey", - "gray", - "white", - "black", - "purple", - "orange", - "cyan", - "magenta", - "brown", - "dark", - "light", - ] - for color in colors: - if color in message_lower: - keywords.append(color) - - # Deduplicate - return list(set(keywords)) - - -async def find_matching_tasks( - user_id: str, - message: str, - min_confidence: float = 0.3, -) -> list[RouteMatch]: - """Find active tasks that might match a user message. - - Matching criteria: - 1. Exact repo_url match in message - 2. Keywords overlap (description, keywords array) - 3. Only considers PLANNING, WAITING_PLAN_REVIEW, IMPLEMENTING, or UNDER_REVIEW tasks - """ - # Get active tasks - active_statuses = [ - TaskStatus.PLANNING.value, - TaskStatus.WAITING_PLAN_REVIEW.value, - TaskStatus.IMPLEMENTING.value, - TaskStatus.UNDER_REVIEW.value, - ] - - all_tasks: list[WorkerTask] = [] - for status in active_statuses: - tasks = await db.list_worker_tasks(user_id=user_id, status=status) - all_tasks.extend(tasks) - - if not all_tasks: - return [] - - # Extract keywords from incoming message - message_keywords = set(extract_keywords(message)) - message_lower = message.lower() - - matches: list[RouteMatch] = [] - - for task in all_tasks: - confidence = 0.0 - reasons: list[str] = [] - - # Check repo URL match - if task.repo_url: - # Extract repo name from URL - repo_name = task.repo_url.rstrip("/").split("/")[-1].replace(".git", "") - if repo_name.lower() in message_lower: - confidence += 0.4 - reasons.append(f"Repo '{repo_name}' mentioned") - - # Also check for domain if it's in the repo URL - # e.g., "understanding.news" from github.com/user/understanding.news - if "." in repo_name and repo_name.lower() in message_lower: - confidence += 0.2 - reasons.append(f"Domain '{repo_name}' mentioned") - - # Check keyword overlap - task_keywords = set(task.keywords or []) - # Also extract keywords from task description - task_keywords.update(extract_keywords(task.description)) - - overlap = message_keywords & task_keywords - if overlap: - # Score based on how many keywords overlap - overlap_score = len(overlap) / max(len(message_keywords), 1) - confidence += 0.4 * overlap_score - reasons.append(f"Keywords: {', '.join(sorted(overlap))}") - - # Check PR URL in message - if task.pr_url: - pr_num_str = str(task.pr_number) if task.pr_number else "" - if pr_num_str and f"#{pr_num_str}" in message: - confidence += 0.3 - reasons.append(f"PR #{pr_num_str} mentioned") - - # Only include if above minimum confidence - if confidence >= min_confidence: - matches.append( - RouteMatch( - task=task, - confidence=min(confidence, 1.0), - match_reasons=reasons, - ) - ) - - # Sort by confidence descending - matches.sort(key=lambda m: m.confidence, reverse=True) - - logger.info( - f"Found {len(matches)} matching tasks for message: {message[:50]}..." - if len(message) > 50 - else f"Found {len(matches)} matching tasks for message: {message}" - ) - - return matches - - -def should_skip_plan(message: str) -> bool: - """Check if user wants to skip plan phase. - - Detected phrases: "just do it", "skip plan", "no plan needed", "go ahead" - """ - message_lower = message.lower() - skip_phrases = [ - "just do it", - "skip plan", - "no plan", - "go ahead", - "don't plan", - "dont plan", - "skip planning", - "no planning", - "straight to", - "directly", - ] - return any(phrase in message_lower for phrase in skip_phrases) diff --git a/backend/src/mainloop/sse.py b/backend/src/mainloop/sse.py index 95fc6dd..4098773 100644 --- a/backend/src/mainloop/sse.py +++ b/backend/src/mainloop/sse.py @@ -19,19 +19,12 @@ class EventType(str, Enum): """SSE event types.""" - # Global events - TASK_UPDATED = "task:updated" INBOX_UPDATED = "inbox:updated" SESSION_UPDATED = "session:updated" SESSION_NEEDS_INPUT = "session:needs_input" SESSION_MESSAGE = "session:message" HEARTBEAT = "heartbeat" - # Task log events - LOG = "log" - STATUS = "status" - END = "end" - @dataclass class SSEEvent: @@ -61,8 +54,6 @@ class EventBus: def __init__(self): # user_id -> list of queues self._user_queues: dict[str, list[asyncio.Queue]] = defaultdict(list) - # task_id -> list of queues (for log streaming) - self._task_queues: dict[str, list[asyncio.Queue]] = defaultdict(list) self._lock = asyncio.Lock() async def subscribe_user(self, user_id: str) -> asyncio.Queue: @@ -85,26 +76,6 @@ async def unsubscribe_user(self, user_id: str, queue: asyncio.Queue): pass logger.info(f"User {user_id} unsubscribed from events") - async def subscribe_task(self, task_id: str) -> asyncio.Queue: - """Subscribe to events for a task (log streaming).""" - queue: asyncio.Queue = asyncio.Queue() - async with self._lock: - self._task_queues[task_id].append(queue) - logger.info(f"Subscribed to task {task_id} logs") - return queue - - async def unsubscribe_task(self, task_id: str, queue: asyncio.Queue): - """Unsubscribe from task events.""" - async with self._lock: - if task_id in self._task_queues: - try: - self._task_queues[task_id].remove(queue) - if not self._task_queues[task_id]: - del self._task_queues[task_id] - except ValueError: - pass - logger.info(f"Unsubscribed from task {task_id} logs") - async def publish_to_user(self, user_id: str, event: SSEEvent): """Publish an event to all subscribers for a user.""" async with self._lock: @@ -115,16 +86,6 @@ async def publish_to_user(self, user_id: str, event: SSEEvent): except asyncio.QueueFull: logger.warning(f"Queue full for user {user_id}, dropping event") - async def publish_to_task(self, task_id: str, event: SSEEvent): - """Publish an event to all subscribers for a task.""" - async with self._lock: - queues = self._task_queues.get(task_id, []) - for queue in queues: - try: - queue.put_nowait(event) - except asyncio.QueueFull: - logger.warning(f"Queue full for task {task_id}, dropping event") - # Global event bus instance event_bus = EventBus() @@ -173,80 +134,6 @@ async def event_stream( await event_bus.unsubscribe_user(user_id, queue) -async def task_log_stream( - task_id: str, - user_id: str, - request: Request, - poll_interval: int = 2, -) -> AsyncGenerator[str, None]: - """Generate SSE events for task logs. - - Polls K8s logs and streams them to the client. - Also listens for task status changes. - """ - from mainloop.db import db - from mainloop.services.k8s_jobs import get_job_logs - - queue = await event_bus.subscribe_task(task_id) - last_log_length = 0 - namespace = f"task-{task_id[:8]}" - - try: - # Send initial status - task = await db.get_worker_task(task_id) - if task: - yield SSEEvent( - event=EventType.STATUS, - data={"status": task.status.value, "task_id": task_id}, - ).encode() - - while True: - if await request.is_disconnected(): - break - - # Check for queued events first (status changes from event bus) - try: - event = queue.get_nowait() - yield event.encode() - if event.event == EventType.END: - break - except asyncio.QueueEmpty: - pass - - # Poll for new logs - try: - logs = await get_job_logs(task_id, namespace) - if logs and len(logs) > last_log_length: - new_logs = logs[last_log_length:] - last_log_length = len(logs) - yield SSEEvent( - event=EventType.LOG, - data={"logs": new_logs, "task_id": task_id}, - ).encode() - except Exception as e: - logger.debug(f"Failed to get logs for task {task_id}: {e}") - - # Check task status - task = await db.get_worker_task(task_id) - if task and task.status.value in ("completed", "failed", "cancelled"): - yield SSEEvent( - event=EventType.STATUS, - data={"status": task.status.value, "task_id": task_id}, - ).encode() - yield SSEEvent( - event=EventType.END, - data={"task_id": task_id}, - ).encode() - break - - await asyncio.sleep(poll_interval) - except asyncio.CancelledError: - # Graceful shutdown - don't propagate as error - pass - finally: - await event_bus.unsubscribe_task(task_id, queue) - - def create_sse_response(generator: AsyncGenerator[str, None]) -> StreamingResponse: """Create an SSE StreamingResponse.""" return StreamingResponse( @@ -263,25 +150,6 @@ def create_sse_response(generator: AsyncGenerator[str, None]) -> StreamingRespon # Helper functions to publish events from other parts of the app -async def notify_task_updated(user_id: str, task_id: str, status: str, **extra): - """Notify user that a task was updated.""" - await event_bus.publish_to_user( - user_id, - SSEEvent( - event=EventType.TASK_UPDATED, - data={"task_id": task_id, "status": status, **extra}, - ), - ) - # Also notify task log subscribers - await event_bus.publish_to_task( - task_id, - SSEEvent( - event=EventType.STATUS, - data={"task_id": task_id, "status": status, **extra}, - ), - ) - - async def notify_inbox_updated( user_id: str, item_id: str | None = None, unread_count: int | None = None ): diff --git a/backend/src/mainloop/workflows/dbos_config.py b/backend/src/mainloop/workflows/dbos_config.py index ec0bd47..9ac7848 100644 --- a/backend/src/mainloop/workflows/dbos_config.py +++ b/backend/src/mainloop/workflows/dbos_config.py @@ -8,7 +8,7 @@ # DBOS configuration # application_version prevents recovery of old workflows after code changes # Bump this when workflow step order/logic changes to avoid DBOSUnexpectedStepError -WORKFLOW_VERSION = "10" # v10: API saves user messages directly (not workflow) +WORKFLOW_VERSION = "11" # v11: Sessions use K8s job isolation, removed worker tasks dbos_config: DBOSConfig = { "name": "mainloop", @@ -21,11 +21,11 @@ # Initialize DBOS - must be done before defining workflows DBOS(config=dbos_config) -# Queue for worker tasks with concurrency limit -# This ensures we don't overwhelm resources with too many concurrent workers +# Queue for session workflows with concurrency limit +# This ensures we don't overwhelm resources with too many concurrent sessions worker_queue = Queue( - "worker_tasks", - concurrency=3, # Max 3 workers running at once globally + "worker_tasks", # Queue name kept for backwards compatibility + concurrency=3, # Max 3 sessions running at once globally ) # Queue for user main threads - one at a time per partition (user) diff --git a/backend/src/mainloop/workflows/main_thread.py b/backend/src/mainloop/workflows/main_thread.py index 4d37c60..20f2542 100644 --- a/backend/src/mainloop/workflows/main_thread.py +++ b/backend/src/mainloop/workflows/main_thread.py @@ -3,15 +3,11 @@ import logging from dbos import DBOS, SetWorkflowID -from mainloop.workflows.dbos_config import worker_queue from mainloop.workflows.transactions import ( get_main_thread_by_user, - save_assistant_message, save_main_thread, save_queue_item, - save_worker_task, update_queue_item_response, - update_task_status, ) from models import ( @@ -19,8 +15,6 @@ QueueItem, QueueItemPriority, QueueItemType, - TaskStatus, - WorkerTask, ) logger = logging.getLogger(__name__) @@ -28,16 +22,14 @@ # Message topics for workflow communication TOPIC_USER_MESSAGE = "user_message" TOPIC_QUEUE_RESPONSE = "queue_response" -TOPIC_WORKER_RESULT = "worker_result" -@DBOS.workflow() # v2: Added plan_review handling +@DBOS.workflow() async def main_thread_workflow(user_id: str) -> None: """Run the main thread workflow for a user. - This workflow runs as long as needed, processing user messages - and coordinating worker agents. It uses DBOS.recv() to wait for - messages from the user or from workers. + This workflow runs as long as needed, processing queue responses. + Chat messages are handled directly by chat_handler with Claude Agent SDK. The workflow is started per-user and identified by user_id. """ @@ -53,7 +45,7 @@ async def main_thread_workflow(user_id: str) -> None: # Main event loop - wait for messages while True: - # Wait for any message (user input, queue response, or worker result) + # Wait for any message (queue responses) # Timeout after 1 hour - workflow will be recovered and continue message = await DBOS.recv_async(timeout_seconds=3600) @@ -66,12 +58,8 @@ async def main_thread_workflow(user_id: str) -> None: msg_type = message.get("type") payload = message.get("payload", {}) - if msg_type == TOPIC_USER_MESSAGE: - await handle_user_message(thread, payload) - elif msg_type == TOPIC_QUEUE_RESPONSE: + if msg_type == TOPIC_QUEUE_RESPONSE: await handle_queue_response(thread, payload) - elif msg_type == TOPIC_WORKER_RESULT: - await handle_worker_result(thread, payload) else: logger.warning(f"Unknown message type: {msg_type}") @@ -87,465 +75,16 @@ async def main_thread_workflow(user_id: str) -> None: ) -async def handle_user_message(thread: MainThread, payload: dict) -> None: - """Process a user message with smart routing.""" - message = payload.get("message", "") - conversation_id = payload.get("conversation_id") - message_id = payload.get("message_id") - skip_routing = payload.get("skip_routing", False) - - logger.info(f"User message: {message[:100]}...") - - # Import routing service - from mainloop.services.task_router import ( - extract_keywords, - find_matching_tasks, - should_skip_plan, - ) - - # Check for routing to existing tasks (unless we've already processed this) - if not skip_routing: - matches = await find_matching_tasks(thread.user_id, message) - - if matches: - best_match = matches[0] - - if best_match.confidence >= 0.7: - # High confidence - suggest routing with confirmation - logger.info( - f"High confidence match ({best_match.confidence:.2f}): {best_match.task.id}" - ) - - # Save response to conversation for chat UI - response_content = f"This looks related to an existing task: {best_match.task.description[:100]}. Check your inbox to confirm routing." - if conversation_id: - save_assistant_message(conversation_id, response_content) - - add_to_queue( - thread, - item_type=QueueItemType.ROUTING_SUGGESTION, - title="Route to existing task?", - content=f"This looks related to: {best_match.task.description[:100]}", - task_id=best_match.task.id, - priority=QueueItemPriority.HIGH, - options=["Route to this task", "Create new task"], - context={ - "suggested_task_id": best_match.task.id, - "confidence": best_match.confidence, - "match_reasons": best_match.match_reasons, - "original_message": message, - "conversation_id": conversation_id, - "message_id": message_id, - }, - ) - return - elif len(matches) > 1 and matches[0].confidence >= 0.4: - # Multiple possible matches - ask user to choose - logger.info("Multiple matches found, asking user to choose") - task_options = [f"{m.task.description[:50]}..." for m in matches[:3]] - task_options.append("Create new task") - - # Save response to conversation for chat UI - response_content = ( - "Multiple active tasks might match. Check your inbox to choose one." - ) - if conversation_id: - save_assistant_message(conversation_id, response_content) - - add_to_queue( - thread, - item_type=QueueItemType.ROUTING_SUGGESTION, - title="Which task?", - content=f"Multiple active tasks might match: {message[:100]}", - priority=QueueItemPriority.HIGH, - options=task_options, - context={ - "matches": [ - {"task_id": m.task.id, "confidence": m.confidence} - for m in matches[:3] - ], - "original_message": message, - "conversation_id": conversation_id, - "message_id": message_id, - }, - ) - return - - # No match or low confidence - check if message needs worker - needs_worker = any( - keyword in message.lower() - for keyword in [ - "build", - "fix", - "create", - "update", - "implement", - "add", - "remove", - "change", - ] - ) - - if needs_worker: - # Extract keywords for future routing - keywords = extract_keywords(message) - skip_plan = should_skip_plan(message) - - task = WorkerTask( - main_thread_id=thread.id, - user_id=thread.user_id, - task_type="feature", - description=message, - prompt=message, - status=TaskStatus.PENDING, - conversation_id=conversation_id, - message_id=message_id, - keywords=keywords, - skip_plan=skip_plan, - ) - task = save_worker_task(task) - - # Enqueue the worker task - from mainloop.workflows.worker import worker_task_workflow - - with SetWorkflowID(task.id): - worker_queue.enqueue(worker_task_workflow, task.id) - - logger.info(f"Spawned worker task: {task.id} (skip_plan={skip_plan})") - - # Create response message - response_content = f"I'm working on: {task.description[:100]}. I'll update you when I have progress." - - # Save assistant message to conversation for chat UI - if conversation_id: - save_assistant_message(conversation_id, response_content) - - # Add acknowledgment to queue (for inbox) - add_to_queue( - thread, - task_id=task.id, - item_type=QueueItemType.NOTIFICATION, - title="Task started", - content=response_content, - priority=QueueItemPriority.LOW, - ) - else: - # Direct response (will use Claude for real responses) - response_content = f"I received your message: {message}. For now I can only spawn workers for tasks that involve building, fixing, or implementing something." - - # Save assistant message to conversation for chat UI - if conversation_id: - save_assistant_message(conversation_id, response_content) - - add_to_queue( - thread, - item_type=QueueItemType.NOTIFICATION, - title="Response", - content=response_content, - priority=QueueItemPriority.NORMAL, - ) - - async def handle_queue_response(thread: MainThread, payload: dict) -> None: """Handle a human response to a queue item.""" queue_item_id = payload.get("queue_item_id") response = payload.get("response") - task_id = payload.get("task_id") - item_context = payload.get("context", {}) - item_type = payload.get("item_type") logger.info(f"Queue response for {queue_item_id}: {response}") # Update the queue item update_queue_item_response(queue_item_id, response) - # Handle plan review responses - route back to worker - if item_type == QueueItemType.PLAN_REVIEW.value: - if not task_id: - logger.error("Plan review response without task_id") - return - - # Determine action and text from response - # Response could be an option click ("Approve", "Option A") or custom text - if response.lower() in ["approve", "lgtm", "looks good"]: - action = "approve" - text = "" - elif response.lower() == "cancel": - action = "cancel" - text = "" - else: - # Custom feedback or selected option that needs revision - action = "revise" - text = response - - # Send response to worker on plan_response topic - DBOS.send( - task_id, - {"action": action, "text": text}, - topic="plan_response", - ) - logger.info(f"Sent plan response to worker {task_id}: action={action}") - return - - # Handle routing suggestions - if item_type == QueueItemType.ROUTING_SUGGESTION.value: - original_message = item_context.get("original_message", "") - conversation_id = item_context.get("conversation_id") - message_id = item_context.get("message_id") - - if response == "Route to this task": - # User chose to route to existing task - target_task_id = item_context.get("suggested_task_id") - if target_task_id: - # Send the message as additional context to the worker - DBOS.send( - target_task_id, - { - "type": "additional_context", - "message": original_message, - "conversation_id": conversation_id, - }, - ) - - add_to_queue( - thread, - task_id=target_task_id, - item_type=QueueItemType.NOTIFICATION, - title="Message routed", - content=f"Added context to task: {original_message[:50]}...", - priority=QueueItemPriority.LOW, - ) - elif response == "Create new task": - # User wants a new task - process as new message - await handle_user_message( - thread, - { - "message": original_message, - "conversation_id": conversation_id, - "message_id": message_id, - "skip_routing": True, # Prevent infinite loop - }, - ) - elif item_context.get("matches"): - # User selected from multiple matches - try: - # Response might be the task description or an index - matches = item_context["matches"] - for i, match in enumerate(matches): - if response.startswith(f"{i+1}.") or response == match.get( - "task_id" - ): - target_task_id = match["task_id"] - DBOS.send( - target_task_id, - { - "type": "additional_context", - "message": original_message, - }, - ) - add_to_queue( - thread, - task_id=target_task_id, - item_type=QueueItemType.NOTIFICATION, - title="Message routed", - content="Added context to selected task", - priority=QueueItemPriority.LOW, - ) - break - except Exception as e: - logger.error(f"Error routing to selected task: {e}") - return - - # If this was for a worker task, forward the response - if task_id: - # Send message to the worker workflow - DBOS.send(task_id, {"type": "human_response", "response": response}) - - -async def handle_worker_result(thread: MainThread, payload: dict) -> None: - """Handle a result from a worker task.""" - task_id = payload.get("task_id") - status = payload.get("status") - result = payload.get("result", {}) - error = payload.get("error") - - logger.info(f"Worker result for {task_id}: {status}") - - if status == "plan_review": - # Interactive plan review in inbox (new flow) - plan_text = result.get("plan_text", "") - suggested_options = result.get("suggested_options", []) - - # Use suggested options directly (already includes "Approve" from job_runner) - options = ( - list(suggested_options) - if suggested_options - else ["Approve", "Request changes"] - ) - - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.PLAN_REVIEW, - title="Review implementation plan", - content=plan_text, - priority=QueueItemPriority.HIGH, - options=options, - context={ - "plan_text": plan_text, - "suggested_options": suggested_options, - }, - ) - - elif status == "plan_ready": - # Legacy: Plan draft PR is ready for review - pr_url = result.get("pr_url") - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.PLAN_READY, - title="Plan ready for review", - content=result.get("message", f"Draft PR created: {pr_url}"), - priority=QueueItemPriority.HIGH, - context={"pr_url": pr_url}, - ) - - elif status == "plan_approved": - # Plan was approved and GitHub issue created - issue_url = result.get("issue_url") - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.NOTIFICATION, - title="Plan approved", - content=f"Implementation starting. Plan issue: {issue_url}", - priority=QueueItemPriority.NORMAL, - context={"issue_url": issue_url}, - ) - - elif status == "plan_updated": - # Plan was revised based on feedback - pr_url = result.get("pr_url") - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.NOTIFICATION, - title="Plan updated", - content=f"Plan revised based on your feedback: {pr_url}", - priority=QueueItemPriority.NORMAL, - context={"pr_url": pr_url}, - ) - - elif status == "code_ready": - # Code is ready for review - pr_url = result.get("pr_url") - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.CODE_READY, - title="Code ready for review", - content=result.get("message", f"Implementation complete: {pr_url}"), - priority=QueueItemPriority.HIGH, - context={"pr_url": pr_url}, - ) - - elif status == "feedback_addressed": - # Worker addressed PR feedback - pr_url = result.get("pr_url") - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.FEEDBACK_ADDRESSED, - title="Feedback addressed", - content=f"Changes pushed to address your feedback: {pr_url}", - priority=QueueItemPriority.NORMAL, - context={"pr_url": pr_url}, - ) - - elif status == "completed": - update_task_status( - task_id, - TaskStatus.COMPLETED, - result=result, - pr_url=result.get("pr_url"), - ) - - # Add completion notification - pr_url = result.get("pr_url") - merged = result.get("merged", False) - if pr_url and merged: - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.NOTIFICATION, - title="PR merged", - content=f"Pull request merged: {pr_url}", - priority=QueueItemPriority.NORMAL, - context={"pr_url": pr_url, "merged": True}, - ) - elif pr_url: - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.NOTIFICATION, - title="Task completed", - content=f"Task completed: {pr_url}", - priority=QueueItemPriority.NORMAL, - context={"pr_url": pr_url}, - ) - else: - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.NOTIFICATION, - title="Task completed", - content=result.get("summary", "Task completed successfully"), - priority=QueueItemPriority.NORMAL, - ) - - elif status == "cancelled": - update_task_status(task_id, TaskStatus.CANCELLED) - - pr_url = result.get("pr_url") - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.NOTIFICATION, - title="Task cancelled", - content=f"PR was closed: {pr_url}" if pr_url else "Task was cancelled", - priority=QueueItemPriority.NORMAL, - context={"pr_url": pr_url} if pr_url else {}, - ) - - elif status == "failed": - update_task_status(task_id, TaskStatus.FAILED, error=error) - - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.ERROR, - title="Task failed", - content=f"Error: {error}", - priority=QueueItemPriority.URGENT, - options=["Retry", "Cancel"], - ) - - elif status == "needs_input": - update_task_status(task_id, TaskStatus.UNDER_REVIEW) - - question = result.get("question", "The worker needs your input.") - options = result.get("options") - - add_to_queue( - thread, - task_id=task_id, - item_type=QueueItemType.QUESTION, - title="Worker needs input", - content=question, - priority=QueueItemPriority.HIGH, - options=options, - ) - def add_to_queue( thread: MainThread, @@ -584,23 +123,6 @@ def get_or_start_main_thread(user_id: str) -> str: return handle.get_workflow_id() -def send_user_message( - user_id: str, message: str, conversation_id: str | None = None -) -> None: - """Send a user message to the main thread workflow.""" - workflow_id = f"main-thread-{user_id}" - DBOS.send( - workflow_id, - { - "type": TOPIC_USER_MESSAGE, - "payload": { - "message": message, - "conversation_id": conversation_id, - }, - }, - ) - - def send_queue_response( user_id: str, queue_item_id: str, diff --git a/backend/src/mainloop/workflows/session_worker.py b/backend/src/mainloop/workflows/session_worker.py index 52ab9da..3783336 100644 --- a/backend/src/mainloop/workflows/session_worker.py +++ b/backend/src/mainloop/workflows/session_worker.py @@ -1,25 +1,28 @@ -"""Session worker workflow - direct conversation with agent SDK. +"""Session worker workflow - runs Claude in isolated K8s Jobs. -Simple model: +Sessions run in their own K8s namespace with full isolation: 1. User sends message -2. Agent SDK responds -3. Wait for next user message -4. Repeat +2. K8s Job spawned with prompt +3. Job POSTs result back via callback +4. Result added to conversation +5. Wait for next user message +6. Repeat """ import logging from datetime import datetime, timezone from typing import Any -from claude_agent_sdk import ( - AssistantMessage, - ClaudeAgentOptions, - TextBlock, - query, -) from dbos import DBOS from mainloop.config import settings -from mainloop.db import db +from mainloop.services.k8s_jobs import create_session_job +from mainloop.services.k8s_namespace import ( + apply_session_namespace_network_policies, + copy_secrets_to_namespace, + create_session_namespace, + delete_session_namespace, + setup_session_rbac, +) from mainloop.sse import notify_session_updated from mainloop.workflows.transactions import ( add_message_to_conversation, @@ -33,19 +36,61 @@ # Topics for DBOS messaging TOPIC_USER_MESSAGE = "user_message" +TOPIC_JOB_RESULT = "job_result" # Timeouts USER_INPUT_TIMEOUT = 86400 # 24 hours +JOB_TIMEOUT = 3600 # 1 hour per job SESSION_SYSTEM_PROMPT = """You are an AI assistant working in a background session. Respond directly to the user's request.""" @DBOS.step() -async def get_claude_response(conversation_id: str, repo_url: str | None = None) -> str: - """Get Claude's response for the session conversation.""" - messages = await db.get_messages(conversation_id) +async def setup_namespace(session_id: str) -> str: + """Create namespace, copy secrets, set up RBAC, and apply network policies.""" + namespace = await create_session_namespace(session_id) + await copy_secrets_to_namespace(session_id, namespace) + await setup_session_rbac(session_id, namespace) + await apply_session_namespace_network_policies(session_id, namespace) + return namespace + + +@DBOS.step() +async def cleanup_namespace(session_id: str) -> None: + """Delete the session namespace.""" + await delete_session_namespace(session_id) + - # Build conversation context +@DBOS.step() +async def spawn_session_job( + session_id: str, + namespace: str, + prompt: str, + model: str | None = None, + iteration: int = 0, +) -> str: + """Spawn a K8s Job to run Claude with the given prompt.""" + callback_url = ( + f"{settings.backend_internal_url}/internal/sessions/{session_id}/complete" + ) + + job_name = await create_session_job( + session_id=session_id, + namespace=namespace, + prompt=prompt, + callback_url=callback_url, + model=model, + iteration=iteration, + ) + + return job_name + + +def build_conversation_prompt( + messages: list, + repo_url: str | None = None, +) -> str: + """Build prompt from conversation history.""" history_parts = [] for msg in messages: role = "User" if msg.role == "user" else "Assistant" @@ -54,28 +99,14 @@ async def get_claude_response(conversation_id: str, repo_url: str | None = None) context = "\n\n".join(history_parts) repo_context = f"\n\nRepository: {repo_url}" if repo_url else "" - prompt_text = f"""Continue this conversation:{repo_context} + return f"""{SESSION_SYSTEM_PROMPT} + +Continue this conversation:{repo_context} {context} Respond to the user's latest message.""" - model = settings.claude_model - options = ClaudeAgentOptions( - model=model, - permission_mode="bypassPermissions", - system_prompt=SESSION_SYSTEM_PROMPT, - ) - - collected_text = [] - async for msg in query(prompt=prompt_text, options=options): - if isinstance(msg, AssistantMessage): - for block in msg.content: - if isinstance(block, TextBlock): - collected_text.append(block.text) - - return "\n".join(collected_text) if collected_text else "No response generated." - async def notify_status(user_id: str, session_id: str, status: str): """Send SSE notification about session status change.""" @@ -84,12 +115,16 @@ async def notify_status(user_id: str, session_id: str, status: str): @DBOS.workflow() async def session_worker_workflow(session_id: str) -> dict[str, Any]: - """Run session workflow with direct conversation via agent SDK. - - 1. Process initial prompt - 2. Wait for user message - 3. Process user message - 4. Repeat + """Run session workflow with Claude running in isolated K8s Jobs. + + 1. Set up isolated K8s namespace + 2. Spawn job for initial prompt + 3. Wait for job result (via callback) + 4. Add response to conversation + 5. Wait for user message + 6. Spawn job for response + 7. Repeat steps 3-6 + 8. Clean up namespace on completion """ logger.info(f"Starting session workflow: {session_id}") @@ -97,7 +132,14 @@ async def session_worker_workflow(session_id: str) -> dict[str, Any]: if not session: return {"status": "failed", "error": "Session not found"} + namespace = None + iteration = 0 + try: + # Set up isolated namespace + logger.info(f"Setting up namespace for session: {session_id}") + namespace = await setup_namespace(session_id) + # Mark session as active update_session_status( session_id, @@ -113,11 +155,35 @@ async def session_worker_workflow(session_id: str) -> dict[str, Any]: session.prompt, ) - # Get agent response to initial prompt - response = await get_claude_response( - session.conversation_id, - repo_url=session.repo_url, + # Build initial prompt and spawn job + from mainloop.db import db + + messages = await db.get_messages(session.conversation_id) + prompt = build_conversation_prompt(messages, session.repo_url) + + logger.info(f"Spawning initial job for session: {session_id}") + await spawn_session_job( + session_id, + namespace, + prompt, + model=session.model, + iteration=iteration, ) + + # Wait for job result + result = await DBOS.recv_async( + topic=TOPIC_JOB_RESULT, + timeout_seconds=JOB_TIMEOUT, + ) + + if result is None: + raise RuntimeError("Job timed out waiting for response") + + if result.get("status") == "failed": + raise RuntimeError(result.get("error", "Job failed")) + + # Add response to conversation + response = result.get("result", {}).get("output", "No response generated.") add_message_to_conversation( session.conversation_id, "assistant", @@ -126,6 +192,8 @@ async def session_worker_workflow(session_id: str) -> dict[str, Any]: # Now wait for user messages in a loop while True: + iteration += 1 + # Wait for user input update_session_status(session_id, SessionStatus.WAITING_ON_USER) await notify_status(session.user_id, session_id, "waiting_on_user") @@ -146,14 +214,39 @@ async def session_worker_workflow(session_id: str) -> dict[str, Any]: return {"status": "completed", "reason": "timeout"} # Got notification that user sent a message (already saved by API) - # Mark as active and get response + # Mark as active and spawn job for response update_session_status(session_id, SessionStatus.ACTIVE) await notify_status(session.user_id, session_id, "active") - response = await get_claude_response( - session.conversation_id, - repo_url=session.repo_url, + # Get updated conversation and spawn job + messages = await db.get_messages(session.conversation_id) + prompt = build_conversation_prompt(messages, session.repo_url) + + logger.info( + f"Spawning job for session: {session_id} (iteration {iteration})" + ) + await spawn_session_job( + session_id, + namespace, + prompt, + model=session.model, + iteration=iteration, + ) + + # Wait for job result + result = await DBOS.recv_async( + topic=TOPIC_JOB_RESULT, + timeout_seconds=JOB_TIMEOUT, ) + + if result is None: + raise RuntimeError("Job timed out waiting for response") + + if result.get("status") == "failed": + raise RuntimeError(result.get("error", "Job failed")) + + # Add response to conversation + response = result.get("result", {}).get("output", "No response generated.") add_message_to_conversation( session.conversation_id, "assistant", @@ -172,3 +265,12 @@ async def session_worker_workflow(session_id: str) -> dict[str, Any]: ) await notify_status(session.user_id, session_id, "failed") return {"status": "failed", "error": str(e)} + + finally: + # Clean up namespace + if namespace: + logger.info(f"Cleaning up namespace for session: {session_id}") + try: + await cleanup_namespace(session_id) + except Exception as e: + logger.warning(f"Failed to cleanup namespace: {e}") diff --git a/backend/src/mainloop/workflows/transactions.py b/backend/src/mainloop/workflows/transactions.py index e3a7bb9..b5b384f 100644 --- a/backend/src/mainloop/workflows/transactions.py +++ b/backend/src/mainloop/workflows/transactions.py @@ -19,10 +19,7 @@ MainThread, QueueItem, Session, - SessionQuestion, SessionStatus, - TaskStatus, - WorkerTask, ) @@ -219,243 +216,6 @@ def add_message_to_conversation(conversation_id: str, role: str, content: str) - return message_id -# ============= Worker Task Transactions ============= - - -@DBOS.transaction() -def save_worker_task(task: WorkerTask) -> WorkerTask: - """Save worker task to database.""" - pending_questions_json = ( - json.dumps([q.model_dump() for q in task.pending_questions]) - if task.pending_questions - else None - ) - - DBOS.sql_session.execute( - text( - """ - INSERT INTO worker_tasks - (id, main_thread_id, user_id, task_type, description, prompt, model, - repo_url, branch_name, base_branch, status, created_at, - conversation_id, message_id, keywords, skip_plan, plan_text, pending_questions) - VALUES (:id, :main_thread_id, :user_id, :task_type, :description, :prompt, :model, - :repo_url, :branch_name, :base_branch, :status, :created_at, - :conversation_id, :message_id, :keywords, :skip_plan, :plan_text, :pending_questions) - """ - ), - { - "id": task.id, - "main_thread_id": task.main_thread_id, - "user_id": task.user_id, - "task_type": task.task_type, - "description": task.description, - "prompt": task.prompt, - "model": task.model, - "repo_url": task.repo_url, - "branch_name": task.branch_name, - "base_branch": task.base_branch, - "status": task.status.value, - "created_at": task.created_at, - "conversation_id": task.conversation_id, - "message_id": task.message_id, - "keywords": task.keywords, - "skip_plan": task.skip_plan, - "plan_text": task.plan_text, - "pending_questions": pending_questions_json, - }, - ) - return task - - -@DBOS.transaction() -def load_worker_task(task_id: str) -> WorkerTask | None: - """Load worker task from database.""" - result = DBOS.sql_session.execute( - text("SELECT * FROM worker_tasks WHERE id = :id"), - {"id": task_id}, - ) - row = result.mappings().first() - if not row: - return None - - return WorkerTask( - id=row["id"], - main_thread_id=row["main_thread_id"], - user_id=row["user_id"], - task_type=row["task_type"], - description=row["description"], - prompt=row["prompt"], - model=row.get("model"), - repo_url=row["repo_url"], - project_id=row.get("project_id"), - branch_name=row["branch_name"], - base_branch=row["base_branch"], - status=TaskStatus(row["status"]), - workflow_run_id=row["workflow_run_id"], - worker_pod_name=row["worker_pod_name"], - created_at=row["created_at"], - started_at=row["started_at"], - completed_at=row["completed_at"], - result=_parse_json_field(row["result"]), - error=row["error"], - issue_url=row.get("issue_url"), - issue_number=row.get("issue_number"), - issue_etag=row.get("issue_etag"), - issue_last_modified=row.get("issue_last_modified"), - pr_url=row["pr_url"], - pr_number=row.get("pr_number"), - pr_etag=row.get("pr_etag"), - pr_last_modified=row.get("pr_last_modified"), - commit_sha=row["commit_sha"], - conversation_id=row.get("conversation_id"), - message_id=row.get("message_id"), - keywords=list(row["keywords"]) if row.get("keywords") else [], - skip_plan=row.get("skip_plan", False), - pending_questions=_parse_json_field(row.get("pending_questions")), - plan_text=row.get("plan_text"), - ) - - -@DBOS.transaction() -def update_task_status( - task_id: str, - status: TaskStatus, - result: dict | None = None, - error: str | None = None, - pr_url: str | None = None, -) -> None: - """Update worker task status.""" - params: dict = {"id": task_id, "status": status.value} - set_clauses = ["status = :status"] - - if result is not None: - set_clauses.append("result = :result") - params["result"] = json.dumps(result) - if error is not None: - set_clauses.append("error = :error") - params["error"] = error - if pr_url is not None: - set_clauses.append("pr_url = :pr_url") - params["pr_url"] = pr_url - - DBOS.sql_session.execute( - text( - f"UPDATE worker_tasks SET {', '.join(set_clauses)} WHERE id = :id" # nosec B608 - ), - params, - ) - - -@DBOS.transaction() -def update_worker_task_status( - task_id: str, - status: TaskStatus | None = None, - issue_url: str | None = None, - issue_number: int | None = None, - pr_url: str | None = None, - pr_number: int | None = None, - branch_name: str | None = None, - error: str | None = None, - pending_questions: list | None = None, - plan_text: str | None = None, - started_at: datetime | None = None, - completed_at: datetime | None = None, - result: dict | None = None, - project_id: str | None = None, - commit_sha: str | None = None, - issue_etag: str | None = None, - pr_etag: str | None = None, -) -> None: - """Update worker task fields.""" - params: dict = {"id": task_id} - set_clauses = [] - - if status is not None: - set_clauses.append("status = :status") - params["status"] = status.value - if issue_url is not None: - set_clauses.append("issue_url = :issue_url") - params["issue_url"] = issue_url - if issue_number is not None: - set_clauses.append("issue_number = :issue_number") - params["issue_number"] = issue_number - if pr_url is not None: - set_clauses.append("pr_url = :pr_url") - params["pr_url"] = pr_url - if pr_number is not None: - set_clauses.append("pr_number = :pr_number") - params["pr_number"] = pr_number - if branch_name is not None: - set_clauses.append("branch_name = :branch_name") - params["branch_name"] = branch_name - if error is not None: - set_clauses.append("error = :error") - params["error"] = error - if pending_questions is not None: - set_clauses.append("pending_questions = :pending_questions") - params["pending_questions"] = ( - json.dumps(pending_questions) if pending_questions else None - ) - if plan_text is not None: - set_clauses.append("plan_text = :plan_text") - params["plan_text"] = plan_text - if started_at is not None: - set_clauses.append("started_at = :started_at") - params["started_at"] = started_at - if completed_at is not None: - set_clauses.append("completed_at = :completed_at") - params["completed_at"] = completed_at - if result is not None: - set_clauses.append("result = :result") - params["result"] = json.dumps(result) - if project_id is not None: - set_clauses.append("project_id = :project_id") - params["project_id"] = project_id - if commit_sha is not None: - set_clauses.append("commit_sha = :commit_sha") - params["commit_sha"] = commit_sha - if issue_etag is not None: - set_clauses.append("issue_etag = :issue_etag") - params["issue_etag"] = issue_etag - if pr_etag is not None: - set_clauses.append("pr_etag = :pr_etag") - params["pr_etag"] = pr_etag - - if set_clauses: - DBOS.sql_session.execute( - text( - f"UPDATE worker_tasks SET {', '.join(set_clauses)} WHERE id = :id" # nosec B608 - ), - params, - ) - - -@DBOS.transaction() -def update_task_etag( - task_id: str, - issue_etag: str | None = None, - pr_etag: str | None = None, -) -> None: - """Update task etag fields.""" - params: dict = {"id": task_id} - set_clauses = [] - - if issue_etag is not None: - set_clauses.append("issue_etag = :issue_etag") - params["issue_etag"] = issue_etag - if pr_etag is not None: - set_clauses.append("pr_etag = :pr_etag") - params["pr_etag"] = pr_etag - - if set_clauses: - DBOS.sql_session.execute( - text( - f"UPDATE worker_tasks SET {', '.join(set_clauses)} WHERE id = :id" # nosec B608 - ), - params, - ) - - # ============= Session Transactions ============= @@ -470,14 +230,6 @@ def load_session(session_id: str) -> Session | None: if not row: return None - # Parse pending_questions JSON - pending_questions = None - raw_questions = row.get("pending_questions") - if raw_questions: - parsed = _parse_json_field(raw_questions) - if parsed and isinstance(parsed, list): - pending_questions = [SessionQuestion(**q) for q in parsed] - return Session( id=row["id"], user_id=row["user_id"], @@ -509,10 +261,6 @@ def load_session(session_id: str) -> Session | None: commit_sha=row.get("commit_sha"), anchor_message_id=row.get("anchor_message_id"), color=row.get("color"), - keywords=list(row["keywords"]) if row.get("keywords") else [], - skip_plan=row.get("skip_plan", False), - pending_questions=pending_questions, - plan_text=row.get("plan_text"), result=_parse_json_field(row.get("result")), ) diff --git a/backend/src/mainloop/workflows/worker.py b/backend/src/mainloop/workflows/worker.py deleted file mode 100644 index 38d7154..0000000 --- a/backend/src/mainloop/workflows/worker.py +++ /dev/null @@ -1,1582 +0,0 @@ -"""Worker task workflow - executes tasks in isolated K8s namespaces with PR feedback loop.""" - -import logging -from datetime import datetime, timezone -from typing import Any - -from dbos import DBOS -from mainloop.config import settings -from mainloop.services import github_pr # Import module for mockability -from mainloop.services.github_pr import ( # Non-mocked utilities - format_feedback_for_agent, - format_issue_feedback_for_agent, - format_plan_for_issue, - format_questions_for_issue, - generate_branch_name, - parse_plan_approval_from_comment, - parse_question_answers_from_comment, -) -from mainloop.services.k8s_jobs import create_worker_job -from mainloop.services.k8s_namespace import ( - apply_task_namespace_network_policies, - copy_secrets_to_namespace, - create_task_namespace, - delete_task_namespace, - setup_worker_rbac, -) -from mainloop.workflows.transactions import load_worker_task, update_worker_task_status - -from models import TaskStatus, WorkerTask - -logger = logging.getLogger(__name__) - -# Topics for DBOS messaging -TOPIC_JOB_RESULT = "job_result" # Results from K8s Jobs -TOPIC_PLAN_RESPONSE = "plan_response" # User approval/revision of plan from inbox -TOPIC_QUESTION_RESPONSE = "question_response" # User answers to agent questions -TOPIC_START_IMPLEMENTATION = ( - "start_implementation" # User triggers implementation after plan approval -) - -# Polling intervals (seconds) -ISSUE_POLL_INTERVAL = 60 # Plan issues - less urgent, rate-limit friendly -PR_POLL_INTERVAL = 30 # Implementation PRs - more urgent during CI - -# Plan review timeout (wait for user to approve/revise plan) -PLAN_REVIEW_TIMEOUT_SECONDS = 86400 # 24 hours - -# Retry configuration -MAX_JOB_RETRIES = 5 # Max retry attempts for failed jobs -JOB_TIMEOUT_SECONDS = 3600 # 1 hour timeout per job attempt - - -def _build_issue_body( - original_prompt: str, - task_id: str, - requirements: dict[str, str] | None = None, - plan_text: str | None = None, - status: str = "Planning", -) -> str: - """Build the GitHub issue body with evolving sections. - - The issue body is structured with clear sections that get filled in - as the planning process progresses. - """ - sections = [] - - # Original request section - sections.append( - f"""## Original Request - -> {original_prompt} -""" - ) - - # Requirements section (filled in after questions are answered) - if requirements: - req_lines = "\n".join([f"- **{k}**: {v}" for k, v in requirements.items()]) - sections.append( - f"""## Requirements - -{req_lines} -""" - ) - - # Plan section (filled in when plan is ready) - if plan_text: - sections.append( - f"""## Implementation Plan - -{plan_text} -""" - ) - - # Footer - sections.append( - f"""--- - -_Task ID: `{task_id}`_ | _Status: {status}_ -_Managed by [Mainloop](https://github.com/oldsj/mainloop)_ -""" - ) - - return "\n".join(sections) - - -def _generate_issue_title(description: str, max_length: int = 70) -> str: - """Generate a brief, intelligent issue title from a task description. - - Extracts the core intent and truncates at word boundaries. - """ - # Take first line only - first_line = description.split("\n")[0].strip() - - # Remove common prefixes that add noise - prefixes_to_remove = [ - "please ", - "can you ", - "could you ", - "i want to ", - "i need to ", - "help me ", - "i'd like to ", - "let's ", - "we should ", - "we need to ", - ] - lower = first_line.lower() - for prefix in prefixes_to_remove: - if lower.startswith(prefix): - first_line = first_line[len(prefix) :] - break - - # Capitalize first letter - if first_line: - first_line = first_line[0].upper() + first_line[1:] - - # If already short enough, return it - if len(first_line) <= max_length: - return first_line - - # Truncate at word boundary - truncated = first_line[:max_length] - last_space = truncated.rfind(" ") - if last_space > max_length // 2: - truncated = truncated[:last_space] - - return truncated.rstrip(".,;:") + "..." - - -@DBOS.step() -async def setup_namespace(task_id: str) -> str: - """Create namespace, copy secrets, set up worker RBAC, and apply network policies.""" - namespace = await create_task_namespace(task_id) - await copy_secrets_to_namespace(task_id, namespace) - await setup_worker_rbac(task_id, namespace) - await apply_task_namespace_network_policies(task_id, namespace) - return namespace - - -@DBOS.step() -async def spawn_plan_job( - task_id: str, - namespace: str, - task: WorkerTask, - feedback: str | None = None, - iteration: int = 1, -) -> str: - """Spawn a Job to create the implementation plan (GitHub issue).""" - callback_url = f"{settings.backend_internal_url}/internal/tasks/{task_id}/complete" - - job_name = await create_worker_job( - task_id=task_id, - namespace=namespace, - prompt=task.description, - mode="plan", - callback_url=callback_url, - model=task.model, - repo_url=task.repo_url, - feedback_context=feedback, - iteration=iteration, - ) - - return job_name - - -@DBOS.step() -async def spawn_implement_job( - task_id: str, - namespace: str, - task: WorkerTask, - issue_number: int | None = None, - branch_name: str | None = None, -) -> str: - """Spawn a Job to implement the approved plan.""" - callback_url = f"{settings.backend_internal_url}/internal/tasks/{task_id}/complete" - - job_name = await create_worker_job( - task_id=task_id, - namespace=namespace, - prompt=task.description, - mode="implement", - callback_url=callback_url, - model=task.model, - repo_url=task.repo_url, - issue_number=issue_number, - branch_name=branch_name, - ) - - return job_name - - -@DBOS.step() -async def spawn_feedback_job( - task_id: str, - namespace: str, - task: WorkerTask, - pr_number: int, - feedback: str, - branch_name: str | None = None, - iteration: int = 1, -) -> str: - """Spawn a Job to address PR feedback.""" - callback_url = f"{settings.backend_internal_url}/internal/tasks/{task_id}/complete" - - job_name = await create_worker_job( - task_id=task_id, - namespace=namespace, - prompt=task.description, - mode="feedback", - callback_url=callback_url, - model=task.model, - repo_url=task.repo_url, - pr_number=pr_number, - branch_name=branch_name, - feedback_context=feedback, - iteration=iteration, - ) - - return job_name - - -@DBOS.step() -async def check_pr_status(repo_url: str, pr_number: int) -> dict: - """Check the current status of a PR.""" - status = await github_pr.get_pr_status(repo_url, pr_number) - if not status: - return {"state": "not_found"} - - return { - "state": status.state, - "merged": status.merged, - "updated_at": status.updated_at.isoformat(), - } - - -@DBOS.step() -async def check_issue_status_step( - repo_url: str, - issue_number: int, - etag: str | None = None, -) -> dict: - """Check the current status of an issue with conditional request support.""" - response = await github_pr.get_issue_status(repo_url, issue_number, etag=etag) - - if response.not_modified: - return {"not_modified": True} - - if not response.data or response.data.get("state") == "not_found": - return {"state": "not_found"} - - return { - "not_modified": False, - "state": response.data["state"], - "title": response.data["title"], - "updated_at": response.data["updated_at"], - "etag": response.etag, - } - - -@DBOS.step() -async def check_for_new_issue_comments( - repo_url: str, - issue_number: int, - since: datetime, - etag: str | None = None, -) -> dict: - """Check for new comments on an issue with conditional request support.""" - response = await github_pr.get_issue_comments( - repo_url, issue_number, since=since, etag=etag - ) - - if response.not_modified: - return {"not_modified": True, "comments": []} - - return { - "not_modified": False, - "comments": response.data or [], - "etag": response.etag, - } - - -@DBOS.step() -async def get_issue_feedback_context( - repo_url: str, - issue_number: int, - since: datetime, -) -> str: - """Get formatted issue feedback for the agent.""" - return await format_issue_feedback_for_agent(repo_url, issue_number, since=since) - - -@DBOS.step() -async def generate_branch_name_step( - issue_number: int, - title: str, - task_type: str, -) -> str: - """Generate an intelligent branch name from issue metadata.""" - return generate_branch_name(issue_number, title, task_type) - - -@DBOS.step() -async def update_github_issue_step( - repo_url: str, - issue_number: int, - title: str | None = None, - body: str | None = None, - state: str | None = None, - labels: list[str] | None = None, -) -> bool: - """Update a GitHub issue.""" - return await github_pr.update_github_issue( - repo_url, issue_number, title, body, state, labels - ) - - -@DBOS.step() -async def add_issue_comment_step( - repo_url: str, - issue_number: int, - body: str, -) -> bool: - """Add a comment to a GitHub issue.""" - return await github_pr.add_issue_comment(repo_url, issue_number, body) - - -@DBOS.step() -async def post_questions_to_issue( - repo_url: str, - issue_number: int, - questions: list[dict], -) -> bool: - """Format and post questions as a GitHub issue comment.""" - body = format_questions_for_issue(questions) - return await github_pr.add_issue_comment(repo_url, issue_number, body) - - -@DBOS.step() -async def check_issue_for_question_answers( - repo_url: str, - issue_number: int, - questions: list[dict], - since: datetime, -) -> dict[str, str] | None: - """Check issue comments for answers to questions. - - Returns: - Dict mapping question ID to answer, or None if no answers found. - - """ - response = await github_pr.get_issue_comments(repo_url, issue_number, since=since) - - if response.not_modified or not response.data: - return None - - # Check each comment for answers (newest first) - sorted_comments = sorted( - response.data, - key=lambda c: c.get("created_at", ""), - reverse=True, - ) - - for comment in sorted_comments: - body = comment.get("body", "") - answers = parse_question_answers_from_comment(body, questions) - if answers: - logger.info(f"Found answers in issue comment: {answers}") - return answers - - return None - - -@DBOS.step() -async def post_plan_to_issue( - repo_url: str, - issue_number: int, - plan_text: str, -) -> int: - """Format and post plan as a GitHub issue comment with approval instructions. - - Returns: - The comment ID (for checking reactions later), or 0 on failure. - - """ - body = format_plan_for_issue(plan_text) - return await github_pr.add_issue_comment( - repo_url, issue_number, body, return_id=True - ) - - -@DBOS.step() -async def check_plan_comment_reactions( - repo_url: str, - comment_id: int, -) -> bool: - """Check if the plan comment has approval reactions (+1, rocket, heart, hooray). - - Returns: - True if approval reaction found, False otherwise. - - """ - if not comment_id: - return False - - reactions = await github_pr.get_comment_reactions(repo_url, comment_id) - - # Approval reactions - approval_reactions = {"+1", "rocket", "heart", "hooray"} - for reaction in reactions: - if reaction in approval_reactions: - logger.info(f"Found approval reaction on plan comment: {reaction}") - return True - - return False - - -@DBOS.step() -async def check_issue_for_plan_approval( - repo_url: str, - issue_number: int, - since: datetime, -) -> tuple[str, str | None] | None: - """Check issue comments for plan approval or revision request. - - Returns: - Tuple of (action, text) where action is "approve" or "revise", - or None if no relevant comments found. - - """ - response = await github_pr.get_issue_comments(repo_url, issue_number, since=since) - - if response.not_modified or not response.data: - return None - - # Check each comment (newest first) - sorted_comments = sorted( - response.data, - key=lambda c: c.get("created_at", ""), - reverse=True, - ) - - for comment in sorted_comments: - body = comment.get("body", "") - result = parse_plan_approval_from_comment(body) - if result: - if result == "approve": - logger.info("Found plan approval in issue comment") - return ("approve", None) - else: - logger.info( - f"Found revision request in issue comment: {result[:50]}..." - ) - return ("revise", result) - - return None - - -@DBOS.step() -async def create_github_issue_step( - repo_url: str, - title: str, - body: str, - labels: list[str] | None = None, -) -> dict | None: - """Create a GitHub issue from the approved plan. - - Args: - repo_url: GitHub repository URL - title: Issue title (derived from task description) - body: Issue body (the approved plan) - labels: Optional labels to apply - - Returns: - Dict with issue number, url, and title, or None if failed - - """ - result = await github_pr.create_github_issue(repo_url, title, body, labels) - if result: - return { - "number": result.number, - "url": result.url, - "title": result.title, - } - return None - - -@DBOS.step() -async def check_for_new_comments( - repo_url: str, - pr_number: int, - since: datetime, -) -> list[dict]: - """Check for new comments on a PR.""" - comments = await github_pr.get_pr_comments(repo_url, pr_number, since=since) - return [ - { - "id": c.id, - "user": c.user, - "body": c.body, - "created_at": c.created_at.isoformat(), - } - for c in comments - ] - - -@DBOS.step() -async def get_feedback_context( - repo_url: str, - pr_number: int, - since: datetime, -) -> str: - """Get formatted feedback for the agent and acknowledge comments.""" - # Get comments first to acknowledge them - from mainloop.services.github_pr import _should_agent_act_on_comment - - comments = await github_pr.get_pr_comments(repo_url, pr_number, since=since) - actionable_comments = [c for c in comments if _should_agent_act_on_comment(c)] - - # Add 👀 reaction to acknowledge we've seen the comments - if actionable_comments: - await github_pr.acknowledge_comments(repo_url, actionable_comments) - - return await format_feedback_for_agent(repo_url, pr_number, since=since) - - -@DBOS.step() -async def cleanup_namespace(task_id: str) -> None: - """Delete the task namespace.""" - await delete_task_namespace(task_id) - - -@DBOS.step() -async def get_check_status_step(repo_url: str, pr_number: int) -> dict: - """Get combined status of GitHub Actions checks for a PR.""" - status = await github_pr.get_check_status(repo_url, pr_number) - return { - "status": status.status, - "total_count": status.total_count, - "failed_count": len(status.failed_runs), - "failed_names": [r.name for r in status.failed_runs], - } - - -@DBOS.step() -async def get_check_failure_logs_step(repo_url: str, pr_number: int) -> str: - """Get formatted failure logs from failed check runs.""" - return await github_pr.get_check_failure_logs(repo_url, pr_number) - - -@DBOS.step() -async def spawn_fix_job( - task_id: str, - namespace: str, - task: WorkerTask, - pr_number: int, - failure_logs: str, - branch_name: str | None = None, - iteration: int = 1, -) -> str: - """Spawn a Job to fix CI failures.""" - callback_url = f"{settings.backend_internal_url}/internal/tasks/{task_id}/complete" - - job_name = await create_worker_job( - task_id=task_id, - namespace=namespace, - prompt=task.description, - mode="fix", - callback_url=callback_url, - model=task.model, - repo_url=task.repo_url, - pr_number=pr_number, - branch_name=branch_name, - feedback_context=failure_logs, - iteration=iteration, - ) - - return job_name - - -def notify_main_thread( - user_id: str, - task_id: str, - message_type: str, - payload: dict, -) -> None: - """Send notification to main thread workflow. Must be called from within a workflow.""" - main_thread_workflow_id = f"main-thread-{user_id}" - DBOS.send( - main_thread_workflow_id, - { - "type": message_type, - "payload": {"task_id": task_id, **payload}, - }, - ) - - -async def run_job_with_retry( - spawn_fn, - job_name: str, - max_retries: int = MAX_JOB_RETRIES, -) -> dict: - """Run a job with exponential backoff retry on failure. - - Args: - spawn_fn: Async function that spawns the job (already bound with args) - job_name: Human-readable job name for logging - max_retries: Maximum number of retry attempts - - Returns: - The successful job result - - Raises: - RuntimeError: If all retries are exhausted - - """ - last_error = None - - for attempt in range(1, max_retries + 1): - logger.info(f"Spawning {job_name} (attempt {attempt}/{max_retries})...") - await spawn_fn() - - # Wait for job result - result = await DBOS.recv_async( - topic=TOPIC_JOB_RESULT, timeout_seconds=JOB_TIMEOUT_SECONDS - ) - - if result is None: - last_error = f"Timed out waiting for {job_name} to complete" - logger.warning(f"{last_error} (attempt {attempt}/{max_retries})") - elif result.get("status") == "failed": - last_error = result.get("error", f"{job_name} failed") - logger.warning( - f"{job_name} failed: {last_error} (attempt {attempt}/{max_retries})" - ) - else: - # Success! - return result - - # If we have more retries, wait with exponential backoff - if attempt < max_retries: - backoff_seconds = 2**attempt # 2, 4, 8, 16, 32 seconds - logger.info(f"Retrying {job_name} in {backoff_seconds}s...") - await DBOS.sleep_async(backoff_seconds) - - # All retries exhausted - raise RuntimeError(f"{job_name} failed after {max_retries} attempts: {last_error}") - - -async def _run_code_review_loop( - task_id: str, - task: Any, - namespace: str, - pr_url: str, - pr_number: int, - since: datetime | None = None, -) -> dict[str, Any]: - """Run the code review loop for an existing PR. - - This is used both by the main workflow and when resuming a task - that already has a PR. - - Args: - task_id: The unique identifier of the task. - task: The worker task object. - namespace: The Kubernetes namespace for the job. - pr_url: The URL of the pull request. - pr_number: The pull request number. - since: Check for comments from this time. If None, uses task.created_at - for resumed tasks, or now for new PRs. - - """ - try: - logger.info(f"Starting code review loop for PR #{pr_number}") - # For resumed tasks, check from task creation time to catch missed comments - last_check = since or task.created_at.replace(tzinfo=timezone.utc) - feedback_iteration = 0 - - while True: - await DBOS.sleep_async(PR_POLL_INTERVAL) - - # Check PR status - pr_status = await check_pr_status(task.repo_url, pr_number) - - if pr_status["state"] == "not_found": - logger.warning(f"PR #{pr_number} not found") - break - - if pr_status["merged"]: - logger.info(f"PR #{pr_number} has been merged") - update_worker_task_status(task_id, TaskStatus.COMPLETED) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "completed", - "result": {"pr_url": pr_url, "merged": True}, - }, - ) - return {"status": "completed", "pr_url": pr_url, "merged": True} - - if pr_status["state"] == "closed": - logger.info(f"PR #{pr_number} was closed without merge") - update_worker_task_status(task_id, TaskStatus.CANCELLED) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "cancelled", - "result": {"pr_url": pr_url, "closed": True}, - }, - ) - return {"status": "cancelled", "pr_url": pr_url} - - # Check for new comments - new_comments = await check_for_new_comments( - task.repo_url, pr_number, last_check - ) - - if new_comments: - logger.info( - f"Found {len(new_comments)} new comments on PR #{pr_number}" - ) - feedback = await get_feedback_context( - task.repo_url, pr_number, last_check - ) - - if feedback: - feedback_iteration += 1 - - # Set status to implementing while Claude works - update_worker_task_status(task_id, TaskStatus.IMPLEMENTING) - - await run_job_with_retry( - lambda fb=feedback, fi=feedback_iteration: spawn_feedback_job( - task_id, - namespace, - task, - pr_number, - fb, - task.branch_name, - fi, - ), - f"feedback job (iteration {feedback_iteration})", - ) - - # Set status back to under_review after job completes - update_worker_task_status( - task_id, - TaskStatus.UNDER_REVIEW, - pr_url=pr_url, - pr_number=pr_number, - ) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - {"status": "feedback_addressed", "result": {"pr_url": pr_url}}, - ) - - last_check = datetime.now(timezone.utc) - - return {"status": "completed", "pr_url": pr_url} - - except Exception as e: - logger.error(f"Code review loop failed: {e}") - update_worker_task_status(task_id, TaskStatus.FAILED, error=str(e)) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - {"status": "failed", "error": str(e)}, - ) - return {"status": "failed", "error": str(e)} - - finally: - # Cleanup namespace - logger.info(f"Cleaning up namespace: {namespace}") - try: - await cleanup_namespace(task_id) - except Exception as e: - logger.warning(f"Cleanup failed: {e}") - - -@DBOS.workflow() # v2: Interactive plan review in inbox -async def worker_task_workflow(task_id: str) -> dict[str, Any]: - """ - Worker workflow that executes a task in an isolated K8s namespace. - - This workflow follows a plan-first approach: - 1. Creates an isolated namespace for the task - 2. PLAN PHASE: Spawns a Job to create a GitHub issue with implementation plan - 3. Polls for plan approval (user comments "looks good", "lgtm", etc.) - 4. IMPLEMENT PHASE: Spawns a Job to implement the approved plan, creates PR - 5. REVIEW PHASE: Polls for code review feedback, addresses comments - 6. Waits for human to merge the PR - 7. Cleans up the namespace - - The workflow uses DBOS for durable execution - it survives restarts - and will resume from the last completed step. - """ - print(f"[WORKFLOW] Starting worker_task_workflow for task: {task_id}") - logger.info(f"Starting worker for task: {task_id}") - - # Load the task - print(f"[WORKFLOW] Loading task from DB: {task_id}") - task = load_worker_task(task_id) - print(f"[WORKFLOW] Task loaded: {task}") - if not task: - return {"status": "failed", "error": "Task not found"} - - # Check if task already has a PR - if so, skip to code review - if task.pr_url and task.pr_number: - logger.info(f"Task already has PR #{task.pr_number}, skipping to code review") - pr_url = task.pr_url - pr_number = task.pr_number - - # Setup namespace for any feedback jobs - namespace = await setup_namespace(task_id) - - try: - # Jump directly to code review loop - update_worker_task_status( - task_id, TaskStatus.UNDER_REVIEW, pr_url=pr_url, pr_number=pr_number - ) - except Exception as e: - logger.error(f"Failed to resume task: {e}") - await cleanup_namespace(task_id) - raise - - # Go to code review loop (PHASE 3) - return await _run_code_review_loop(task_id, task, namespace, pr_url, pr_number) - - # Setup namespace - print(f"[WORKFLOW] Creating namespace for task: {task_id}") - logger.info(f"Creating namespace for task: {task_id}") - namespace = await setup_namespace(task_id) - print(f"[WORKFLOW] Namespace created: {namespace}") - - try: - issue_url: str | None = None - issue_number: int | None = None - branch_name: str | None = None - pr_url: str | None = None - pr_number: int | None = None - plan_iteration = 0 - - # ============================================================ - # PHASE 1: PLANNING (unless skip_plan is True) - # Creates GitHub issue immediately, updates as plan evolves - # Pauses after plan approval, waiting for user to trigger implementation - # ============================================================ - if not task.skip_plan: - print(f"[WORKFLOW] Setting task status to PLANNING: {task_id}") - update_worker_task_status(task_id, TaskStatus.PLANNING) - print("[WORKFLOW] Task status updated to PLANNING") - plan_iteration = 0 - plan_text: str | None = None - suggested_options: list[str] = [] - user_answers: dict[str, str] = {} # Accumulated answers to questions - - # Create GitHub issue immediately with original prompt - issue_title = _generate_issue_title(task.description) - initial_issue_body = _build_issue_body( - original_prompt=task.description, - task_id=task_id, - ) - - issue_result = await create_github_issue_step( - task.repo_url, - issue_title, - initial_issue_body, - labels=["mainloop-plan", "planning"], - ) - - if not issue_result: - raise RuntimeError("Failed to create GitHub issue") - - issue_url = issue_result["url"] - issue_number = issue_result["number"] - logger.info(f"Created GitHub issue #{issue_number}: {issue_url}") - - # Store issue info immediately - update_worker_task_status( - task_id, - TaskStatus.PLANNING, - issue_url=issue_url, - issue_number=issue_number, - ) - - # Add comment that we're starting to explore - await add_issue_comment_step( - task.repo_url, - issue_number, - "🤖 Starting to explore the codebase and gather requirements...", - ) - - while True: - plan_iteration += 1 - - # Build feedback context from previous iteration or user answers - feedback_context = None - if plan_iteration > 1 and plan_text: - feedback_context = plan_text - if user_answers: - # Format answers as context for the agent - answers_text = "\n".join( - [f"- {k}: {v}" for k, v in user_answers.items()] - ) - if feedback_context: - feedback_context = f"{feedback_context}\n\nUser answered your questions:\n{answers_text}" - else: - feedback_context = ( - f"User answered your questions:\n{answers_text}" - ) - - # Run plan job with retry - returns plan content and any questions - result = await run_job_with_retry( - lambda iteration=plan_iteration, fc=feedback_context: spawn_plan_job( - task_id, - namespace, - task, - feedback=fc, - iteration=iteration, - ), - f"plan job (iteration {plan_iteration})", - ) - - # Extract plan content and questions from result - plan_result = result.get("result", {}) - plan_text = plan_result.get("plan_text") - questions = plan_result.get("questions", []) - suggested_options = plan_result.get("suggested_options", []) - - if not plan_text: - raise RuntimeError("Plan job did not return plan content") - - logger.info( - f"Plan ready with {len(questions)} questions, {len(suggested_options)} options" - ) - - # PHASE 1a: If agent asked questions, get user answers first - if questions: - logger.info( - f"Agent asked {len(questions)} questions - waiting for user answers" - ) - - # Update task with questions - update_worker_task_status( - task_id, - TaskStatus.WAITING_QUESTIONS, - pending_questions=questions, - plan_text=plan_text, - ) - - # Post formatted questions to GitHub issue - await post_questions_to_issue( - task.repo_url, issue_number, questions - ) - - # Notify main thread about questions (task UI will show them) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "questions", - "result": { - "questions": questions, - "plan_text": plan_text, - "issue_url": issue_url, - "message": "Please answer these questions to continue", - }, - }, - ) - - # Poll for answers with exponential backoff - # Check both: 1) DBOS message from UI, 2) GitHub issue comments - # Start at 10s, max out at 5 minutes - poll_interval = 10 # seconds - max_poll_interval = 300 # 5 minutes - questions_posted_at = datetime.now(timezone.utc) - total_wait = 0 - response = None - answer_source = None - - while total_wait < PLAN_REVIEW_TIMEOUT_SECONDS: - # First, check for DBOS message (non-blocking with short timeout) - response = await DBOS.recv_async( - topic=TOPIC_QUESTION_RESPONSE, - timeout_seconds=poll_interval, - ) - - if response is not None: - answer_source = "ui" - logger.info("Received answers via UI") - break - - # No UI response - check GitHub issue comments - gh_answers = await check_issue_for_question_answers( - task.repo_url, issue_number, questions, questions_posted_at - ) - - if gh_answers: - # Found answers in GitHub comments! - response = {"answers": gh_answers, "action": "answer"} - answer_source = "github" - logger.info( - f"Found answers in GitHub issue comment: {gh_answers}" - ) - break - - # No answers yet - increase poll interval (exponential backoff) - total_wait += poll_interval - poll_interval = min(poll_interval * 1.5, max_poll_interval) - logger.debug( - f"No answers yet, next poll in {poll_interval:.0f}s (total wait: {total_wait}s)" - ) - - if response is None: - raise RuntimeError("Question response timed out after 24 hours") - - if response.get("action") == "cancel": - logger.info("Task cancelled by user during questions") - update_worker_task_status(task_id, TaskStatus.CANCELLED) - await add_issue_comment_step( - task.repo_url, issue_number, "❌ Task cancelled by user." - ) - await update_github_issue_step( - task.repo_url, issue_number, state="closed" - ) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "cancelled", - "result": {"message": "Cancelled by user"}, - }, - ) - return {"status": "cancelled", "message": "Cancelled by user"} - - # Store answers and update issue with Q&A - user_answers = response.get("answers", {}) - logger.info( - f"User answered {len(user_answers)} questions via {answer_source}, re-running plan..." - ) - - # Update issue body with requirements - updated_body = _build_issue_body( - original_prompt=task.description, - requirements=user_answers, - task_id=task_id, - ) - await update_github_issue_step( - task.repo_url, issue_number, body=updated_body - ) - - # Post confirmation comment (mention source if from GitHub) - if answer_source == "github": - await add_issue_comment_step( - task.repo_url, - issue_number, - "✅ Got your answers! Generating implementation plan...", - ) - else: - await add_issue_comment_step( - task.repo_url, - issue_number, - "✅ Requirements gathered. Generating implementation plan...", - ) - - # Clear questions from task since they've been answered - update_worker_task_status( - task_id, - TaskStatus.PLANNING, - pending_questions=[], # Empty list clears questions - ) - continue # Re-run plan job with answers - - # PHASE 1b: No questions - proceed to plan review - # Update issue body with the plan - updated_body = _build_issue_body( - original_prompt=task.description, - requirements=user_answers, - plan_text=plan_text, - task_id=task_id, - ) - await update_github_issue_step( - task.repo_url, issue_number, body=updated_body - ) - - # Post plan as comment with approval instructions (get comment ID for reactions) - plan_comment_id = await post_plan_to_issue( - task.repo_url, issue_number, plan_text - ) - - # Update status to waiting for plan review - update_worker_task_status( - task_id, - TaskStatus.WAITING_PLAN_REVIEW, - plan_text=plan_text, - ) - - # Notify main thread - task UI will show plan for review - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "plan_review", - "result": { - "plan_text": plan_text, - "suggested_options": suggested_options, - "issue_url": issue_url, - "message": "Plan ready for review", - }, - }, - ) - - # Poll for approval with exponential backoff - # Check: 1) DBOS message from UI, 2) GitHub comments, 3) Reactions on plan comment - poll_interval = 10 # seconds - max_poll_interval = 300 # 5 minutes - plan_posted_at = datetime.now(timezone.utc) - total_wait = 0 - response = None - approval_source = None - - logger.info( - "Waiting for plan approval via UI, GitHub comment, or reaction..." - ) - - while total_wait < PLAN_REVIEW_TIMEOUT_SECONDS: - # First, check for DBOS message (non-blocking with short timeout) - response = await DBOS.recv_async( - topic=TOPIC_PLAN_RESPONSE, - timeout_seconds=poll_interval, - ) - - if response is not None: - approval_source = "ui" - logger.info("Received plan response via UI") - break - - # Check for approval reaction (👍, 🚀, etc.) on plan comment - if plan_comment_id: - has_approval_reaction = await check_plan_comment_reactions( - task.repo_url, plan_comment_id - ) - if has_approval_reaction: - response = {"action": "approve", "text": ""} - approval_source = "github_reaction" - logger.info("Found approval reaction on plan comment") - break - - # No UI response - check GitHub issue comments - gh_response = await check_issue_for_plan_approval( - task.repo_url, issue_number, plan_posted_at - ) - - if gh_response: - action, text = gh_response - response = {"action": action, "text": text or ""} - approval_source = "github" - logger.info(f"Found plan response in GitHub comment: {action}") - break - - # No response yet - increase poll interval (exponential backoff) - total_wait += poll_interval - poll_interval = min(poll_interval * 1.5, max_poll_interval) - logger.debug( - f"No plan response yet, next poll in {poll_interval:.0f}s" - ) - - if response is None: - raise RuntimeError("Plan review timed out after 24 hours") - - response_action = response.get("action", "") - response_text = response.get("text", "") - - if response_action == "approve" or response_action.lower() in [ - "approve", - "lgtm", - "looks good", - ]: - logger.info( - f"Plan approved via {approval_source}! Waiting for user to start implementation..." - ) - if approval_source in ("github", "github_reaction"): - await add_issue_comment_step( - task.repo_url, issue_number, "✅ Got it! Plan approved." - ) - break - elif response_action == "cancel": - logger.info("Plan cancelled by user") - update_worker_task_status(task_id, TaskStatus.CANCELLED) - await add_issue_comment_step( - task.repo_url, issue_number, "❌ Plan cancelled by user." - ) - await update_github_issue_step( - task.repo_url, issue_number, state="closed" - ) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "cancelled", - "result": {"message": "Plan cancelled by user"}, - }, - ) - return {"status": "cancelled", "message": "Plan cancelled by user"} - else: - # User provided feedback - re-run plan with feedback - feedback = response_text or response_action - logger.info( - f"User requested plan revision via {approval_source}: {feedback}" - ) - if approval_source == "github": - await add_issue_comment_step( - task.repo_url, - issue_number, - "📝 Got your feedback. Regenerating plan...", - ) - else: - await add_issue_comment_step( - task.repo_url, - issue_number, - f"📝 **Revision requested:**\n> {feedback}\n\nRegenerating plan...", - ) - plan_text = feedback # Pass feedback to next iteration - user_answers = {} # Clear answers for new iteration - continue - - # Plan approved - update issue and pause for implementation trigger - await add_issue_comment_step( - task.repo_url, - issue_number, - "✅ **Plan approved!** Ready to start implementation when triggered.", - ) - - # Update issue labels - await update_github_issue_step( - task.repo_url, - issue_number, - labels=["mainloop-plan", "approved"], - ) - - # Generate intelligent branch name - branch_name = await generate_branch_name_step( - issue_number, task.description[:50], task.task_type - ) - logger.info(f"Generated branch name: {branch_name}") - - # Set status to ready_to_implement and PAUSE - update_worker_task_status( - task_id, - TaskStatus.READY_TO_IMPLEMENT, - issue_url=issue_url, - issue_number=issue_number, - branch_name=branch_name, - plan_text=plan_text, - ) - - # Notify that plan is approved and waiting for implementation trigger - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "ready_to_implement", - "result": { - "issue_url": issue_url, - "plan_text": plan_text, - "message": "Plan approved. Click 'Start Implementation' when ready.", - }, - }, - ) - - # Wait for user to trigger implementation (24 hour timeout) - logger.info("Waiting for user to trigger implementation...") - impl_response = await DBOS.recv_async( - topic=TOPIC_START_IMPLEMENTATION, - timeout_seconds=PLAN_REVIEW_TIMEOUT_SECONDS, - ) - - if impl_response is None: - raise RuntimeError("Implementation trigger timed out after 24 hours") - - if impl_response.get("action") == "cancel": - logger.info("Implementation cancelled by user") - update_worker_task_status(task_id, TaskStatus.CANCELLED) - await add_issue_comment_step( - task.repo_url, issue_number, "❌ Implementation cancelled by user." - ) - await update_github_issue_step( - task.repo_url, issue_number, state="closed" - ) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - {"status": "cancelled", "result": {"message": "Cancelled by user"}}, - ) - return {"status": "cancelled", "message": "Cancelled by user"} - - logger.info("Implementation triggered! Proceeding to Phase 2...") - await add_issue_comment_step( - task.repo_url, issue_number, "🚀 **Starting implementation...**" - ) - - # ============================================================ - # PHASE 2: IMPLEMENTATION - # Creates a PR that references the plan issue - # ============================================================ - update_worker_task_status(task_id, TaskStatus.IMPLEMENTING) - - if task.skip_plan: - # Skip plan mode: create PR directly with implementation - impl_result = await run_job_with_retry( - lambda: spawn_implement_job(task_id, namespace, task), - "implement job (skip plan)", - ) - else: - # Normal mode: implement the approved plan - impl_result = await run_job_with_retry( - lambda: spawn_implement_job( - task_id, - namespace, - task, - issue_number=issue_number, - branch_name=branch_name, - ), - f"implement job (issue #{issue_number})", - ) - - # Extract PR URL from implementation result - pr_url = impl_result.get("result", {}).get("pr_url") - if not pr_url: - # No PR created - task is done (only expected in skip_plan mode) - logger.info("Task completed without PR") - update_worker_task_status(task_id, TaskStatus.COMPLETED) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - {"status": "completed", "result": impl_result.get("result")}, - ) - return {"status": "completed", "result": impl_result.get("result")} - pr_number = int(pr_url.split("/")[-1]) - logger.info(f"Implementation created PR #{pr_number}: {pr_url}") - - # ============================================================ - # PHASE 2.5: CI VERIFICATION LOOP - # ============================================================ - # Wait for GitHub Actions to pass before entering code review - logger.info(f"Waiting for GitHub Actions to complete on PR #{pr_number}") - ci_iteration = 0 - MAX_CI_ITERATIONS = 5 # Limit fix attempts - - while ci_iteration < MAX_CI_ITERATIONS: - await DBOS.sleep_async(PR_POLL_INTERVAL) - - check_status = await get_check_status_step(task.repo_url, pr_number) - - if check_status["status"] == "success": - logger.info("All CI checks passed! Continuing to code review...") - break - elif check_status["status"] == "failure": - ci_iteration += 1 - failed_names = check_status.get("failed_names", []) - logger.info( - f"CI checks failed ({', '.join(failed_names)}), " - f"spawning fix job (iteration {ci_iteration})..." - ) - - # Notify human about CI failure - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "ci_failed", - "result": { - "pr_url": pr_url, - "failed_checks": failed_names, - "iteration": ci_iteration, - }, - }, - ) - - failure_logs = await get_check_failure_logs_step( - task.repo_url, pr_number - ) - await run_job_with_retry( - lambda fl=failure_logs, ci=ci_iteration: spawn_fix_job( - task_id, - namespace, - task, - pr_number, - fl, - branch_name, - ci, - ), - f"fix job (CI iteration {ci_iteration})", - ) - - # After fix, wait a bit for new checks to start - await DBOS.sleep_async(30) - else: - # Pending - continue waiting - continue - - if ci_iteration >= MAX_CI_ITERATIONS: - raise RuntimeError( - f"CI checks still failing after {MAX_CI_ITERATIONS} fix attempts" - ) - - update_worker_task_status( - task_id, TaskStatus.UNDER_REVIEW, pr_url=pr_url, pr_number=pr_number - ) - - # Notify human that code is ready for review - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "code_ready", - "result": {"pr_url": pr_url, "message": "Code ready for review"}, - }, - ) - - # ============================================================ - # PHASE 3: CODE REVIEW LOOP - # ============================================================ - logger.info(f"Starting code review loop for PR #{pr_number}") - last_check = datetime.now(timezone.utc) - feedback_iteration = 0 - - while True: - await DBOS.sleep_async(PR_POLL_INTERVAL) - - # Check PR status - pr_status = await check_pr_status(task.repo_url, pr_number) - - if pr_status["state"] == "not_found": - logger.warning(f"PR #{pr_number} not found") - break - - if pr_status["merged"]: - logger.info(f"PR #{pr_number} has been merged") - update_worker_task_status(task_id, TaskStatus.COMPLETED) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "completed", - "result": {"pr_url": pr_url, "merged": True}, - }, - ) - break - - if pr_status["state"] == "closed": - logger.info(f"PR #{pr_number} was closed without merge") - update_worker_task_status(task_id, TaskStatus.CANCELLED) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - { - "status": "cancelled", - "result": {"pr_url": pr_url, "closed": True}, - }, - ) - break - - # Check for new comments - new_comments = await check_for_new_comments( - task.repo_url, pr_number, last_check - ) - - if new_comments: - logger.info( - f"Found {len(new_comments)} new comments on PR #{pr_number}" - ) - feedback = await get_feedback_context( - task.repo_url, pr_number, last_check - ) - - if feedback: - feedback_iteration += 1 - - # Set status to implementing while Claude works - update_worker_task_status(task_id, TaskStatus.IMPLEMENTING) - - await run_job_with_retry( - lambda fb=feedback, fi=feedback_iteration: spawn_feedback_job( - task_id, - namespace, - task, - pr_number, - fb, - branch_name, - fi, - ), - f"feedback job (iteration {feedback_iteration})", - ) - - # Set status back to under_review after job completes - update_worker_task_status( - task_id, - TaskStatus.UNDER_REVIEW, - pr_url=pr_url, - pr_number=pr_number, - ) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - {"status": "feedback_addressed", "result": {"pr_url": pr_url}}, - ) - - last_check = datetime.now(timezone.utc) - - return {"status": "completed", "pr_url": pr_url} - - except Exception as e: - logger.error(f"Worker task failed: {e}") - update_worker_task_status(task_id, TaskStatus.FAILED, error=str(e)) - notify_main_thread( - task.user_id, - task_id, - "worker_result", - {"status": "failed", "error": str(e)}, - ) - return {"status": "failed", "error": str(e)} - - finally: - # Cleanup namespace (always run) - retry up to 3 times - logger.info(f"Cleaning up namespace: {namespace}") - cleanup_attempts = 3 - for attempt in range(1, cleanup_attempts + 1): - try: - await cleanup_namespace(task_id) - logger.info(f"Successfully cleaned up namespace: {namespace}") - break - except Exception as e: - if attempt < cleanup_attempts: - logger.warning( - f"Cleanup attempt {attempt} failed: {e}, retrying..." - ) - import asyncio - - await asyncio.sleep(2**attempt) - else: - logger.error( - f"Failed to cleanup namespace after {cleanup_attempts} attempts: {e}" - ) diff --git a/claude-agent/job_runner.py b/claude-agent/job_runner.py index 91b7232..94f0cbe 100644 --- a/claude-agent/job_runner.py +++ b/claude-agent/job_runner.py @@ -1,21 +1,17 @@ #!/usr/bin/env python3 """ -Job runner for K8s worker Jobs. +Job runner for K8s session Jobs. This is the entry point when the claude-agent container runs as a K8s Job. It reads configuration from environment variables, executes the task using Claude Agent SDK, and POSTs the result back to the backend. -Modes: - - plan: Clone repo, analyze task, return implementation plan (no GitHub issue - reviewed in inbox) - - implement: Checkout existing branch, implement code per approved plan, create PR - - feedback: Address PR comments, push new commits +A session IS a job - no modes, just run Claude with a prompt and return the result. """ import asyncio import os import sys -import uuid from datetime import datetime, timezone from pathlib import Path @@ -25,627 +21,54 @@ ClaudeAgentOptions, ResultMessage, TextBlock, - ToolUseBlock, query, ) # Environment variables -TASK_ID = os.environ.get("TASK_ID", "") +SESSION_ID = os.environ.get("SESSION_ID", "") TASK_PROMPT = os.environ.get("TASK_PROMPT", "") CALLBACK_URL = os.environ.get("CALLBACK_URL", "") -MODE = os.environ.get("MODE", "plan") CLAUDE_MODEL = os.environ.get("CLAUDE_MODEL", "sonnet") -REPO_URL = os.environ.get("REPO_URL", "") -PR_NUMBER = os.environ.get("PR_NUMBER", "") -ISSUE_NUMBER = os.environ.get("ISSUE_NUMBER", "") -BRANCH_NAME = os.environ.get("BRANCH_NAME", "") -FEEDBACK_CONTEXT = os.environ.get("FEEDBACK_CONTEXT", "") WORKSPACE = "/workspace" -CLAUDE_PLANS_DIR = Path.home() / ".claude" / "plans" -def find_plan_file() -> Path | None: - """Find the most recently modified plan file in Claude's plans directory.""" - if not CLAUDE_PLANS_DIR.exists(): - print(f"[job_runner] Plans directory not found: {CLAUDE_PLANS_DIR}") - return None - - plan_files = list(CLAUDE_PLANS_DIR.glob("*.md")) - if not plan_files: - print(f"[job_runner] No plan files found in {CLAUDE_PLANS_DIR}") - return None - - # Return the most recently modified plan file - latest = max(plan_files, key=lambda p: p.stat().st_mtime) - print(f"[job_runner] Found plan file: {latest}") - return latest - - -def read_plan_file(path: Path) -> str: - """Read the content of a plan file.""" - try: - content = path.read_text() - return content - except Exception as e: - print(f"[job_runner] Error reading plan file {path}: {e}") - return "" - - -def pre_clone_repo() -> str | None: - """Pre-clone the repo before Claude starts. - - Returns the path to the cloned repo, or None if no repo URL. - Uses shallow clone (depth=1) for plan mode for speed. - """ - from urllib.parse import urlparse - - import git - - if not REPO_URL: - return None - - # Parse repo URL to get repo name for target dir - parsed = urlparse(REPO_URL) - path_parts = parsed.path.strip("/").split("/") - if len(path_parts) >= 2: - repo_name = path_parts[-1].replace(".git", "") - else: - repo_name = "repo" - - target_dir = f"{WORKSPACE}/{repo_name}" - - # Build authenticated URL if GH_TOKEN is available - clone_url = REPO_URL - gh_token = os.environ.get("GH_TOKEN") - if gh_token and "github.com" in REPO_URL: - clone_url = REPO_URL.replace( - "https://github.com", f"https://x-access-token:{gh_token}@github.com" - ) - print("[job_runner] Using authenticated clone URL") - - try: - # Clone - shallow for plan mode, full for others - if MODE == "plan": - print("[job_runner] Shallow cloning for plan mode...") - repo = git.Repo.clone_from(clone_url, target_dir, depth=1) - else: - print(f"[job_runner] Full cloning for {MODE} mode...") - repo = git.Repo.clone_from(clone_url, target_dir) - # Fetch all refs for branch checkout - repo.remotes.origin.fetch() - - print(f"[job_runner] Repo cloned to {target_dir}") - return target_dir - - except git.GitCommandError as e: - print(f"[job_runner] Clone failed: {e}") - return None - except Exception as e: - print(f"[job_runner] Clone error: {e}") - return None - - -def build_prompt() -> str: - """Build the prompt based on mode and context.""" - if MODE == "plan": - return build_plan_prompt() - elif MODE == "implement": - return build_implement_prompt() - elif MODE == "feedback": - return build_feedback_prompt() - elif MODE == "fix": - return build_fix_prompt() - else: - raise ValueError(f"Unknown mode: {MODE}") - - -def build_plan_prompt() -> str: - """Build prompt for planning phase - explores codebase and outputs a plan.""" - parts = [ - f"Task ID: {TASK_ID[:8]}", - f"Task: {TASK_PROMPT}", - "", - ] - - if REPO_URL: - parts.append(f"Repository: {REPO_URL}") - parts.append("") - parts.extend( - [ - "The repository is already cloned and you are in its directory.", - "Explore the codebase to understand the structure and patterns used.", - "", - "Create a detailed implementation plan that includes:", - "- **Summary**: Brief overview of the approach", - "- **Files to modify**: List each file and describe the changes", - "- **Files to create**: Any new files needed and their purpose", - "- **Considerations**: Risks, edge cases, or decisions needing confirmation", - "", - "If there are multiple valid approaches, present them as options:", - "- **Option A**: [approach name] - [brief description]", - "- **Option B**: [approach name] - [brief description]", - "And recommend which option you prefer.", - "", - "IMPORTANT: Your final output should be ONLY the plan in clean markdown.", - "The plan will be shown to the user for approval before implementation.", - "", - ] - ) - else: - parts.extend( - [ - "Create an implementation plan for this task.", - "If there are multiple approaches, present them as options.", - "Output your plan as clean markdown.", - "", - ] - ) - - # Add feedback context if this is a plan revision - if FEEDBACK_CONTEXT: - parts.extend( - [ - "Feedback on your previous plan:", - "---", - FEEDBACK_CONTEXT, - "---", - "", - "Update your plan to address this feedback.", - ] - ) - - return "\n".join(parts) - - -def build_implement_prompt() -> str: - """Build prompt for implementation phase (write code, create PR).""" - # Use provided branch name or fall back to default - branch_name = BRANCH_NAME or f"mainloop/{TASK_ID[:8]}" - - parts = [ - f"Task ID: {TASK_ID[:8]}", - f"Original Task: {TASK_PROMPT}", - "", - ] - - if REPO_URL: - parts.extend( - [ - f"Repository: {REPO_URL}", - f"Branch to create: {branch_name}", - ] - ) - if ISSUE_NUMBER: - parts.append(f"Plan issue: #{ISSUE_NUMBER}") - parts.extend( - [ - "", - "Your implementation plan has been approved. Now implement it:", - "", - "Instructions:", - "1. The repository is already cloned and you are in its directory", - f"2. Create and checkout a new branch: `git checkout -b {branch_name}`", - ] - ) - if ISSUE_NUMBER: - parts.append(f"3. Read your approved plan from issue #{ISSUE_NUMBER}") - parts.extend( - [ - "4. Implement the code according to your approved plan", - "5. Commit your changes with clear commit messages", - "6. Push the branch", - ] - ) - if ISSUE_NUMBER: - parts.extend( - [ - "7. Create a pull request that links to and auto-closes the plan issue:", - f" The PR body MUST start with 'Closes #{ISSUE_NUMBER}' - this auto-closes the issue on merge.", - f' Example: gh pr create --title "..." --body "Closes #{ISSUE_NUMBER} - "', - ] - ) - else: - parts.extend( - [ - '7. Create a pull request: `gh pr create --title "..." --body "..."`', - ] - ) - parts.extend( - [ - "", - "Follow your plan carefully. If you discover issues during implementation,", - "add a comment to the PR explaining any deviations from the plan.", - "", - ] - ) - else: - parts.extend( - [ - "Instructions:", - "1. Implement the task as planned", - "2. Create any necessary files in /workspace", - "3. Summarize what you accomplished", - "", - ] - ) - - return "\n".join(parts) - - -def build_feedback_prompt() -> str: - """Build prompt for addressing PR feedback.""" - branch_name = BRANCH_NAME or f"mainloop/{TASK_ID[:8]}" - - parts = [ - f"Task ID: {TASK_ID[:8]}", - f"Original Task: {TASK_PROMPT}", - "", - f"You are addressing feedback on PR #{PR_NUMBER}", - "", - ] - - if REPO_URL: - parts.extend( - [ - f"Repository: {REPO_URL}", - f"Branch: {branch_name}", - "", - ] - ) - - if FEEDBACK_CONTEXT: - parts.extend( - [ - "Feedback to address:", - "---", - FEEDBACK_CONTEXT, - "---", - "", - ] - ) - - parts.extend( - [ - "Instructions:", - f"1. The repository is already cloned. Checkout the branch: `git checkout {branch_name}`", - "2. Review the feedback above", - "3. Make the necessary changes to address the feedback", - "4. Commit your changes with a clear message referencing the feedback", - "5. Push the updated branch", - "", - "Be thorough in addressing all points raised in the feedback.", - ] - ) - - return "\n".join(parts) - - -def build_fix_prompt() -> str: - """Build prompt for fixing CI failures.""" - branch_name = BRANCH_NAME or f"mainloop/{TASK_ID[:8]}" - - parts = [ - f"Task ID: {TASK_ID[:8]}", - f"Original Task: {TASK_PROMPT}", - "", - f"You are fixing CI failures on PR #{PR_NUMBER}", - "", - ] - - if REPO_URL: - parts.extend( - [ - f"Repository: {REPO_URL}", - f"Branch: {branch_name}", - "", - ] - ) - - parts.extend( - [ - "GitHub Actions checks have FAILED. You must fix them.", - "", - ] - ) - - if FEEDBACK_CONTEXT: - parts.extend( - [ - "Failed checks and logs:", - "---", - FEEDBACK_CONTEXT, - "---", - "", - ] - ) - - parts.extend( - [ - "Instructions:", - f"1. The repository is already cloned. Checkout the branch: `git checkout {branch_name}`", - "2. Analyze the failure logs above carefully", - "3. Identify the root cause of each failure", - "4. Fix the issues (lint errors, test failures, type errors, build errors)", - "5. Run `trunk check` locally if available to verify before pushing", - "6. Commit and push your fix with a clear message", - "", - "IMPORTANT:", - "- Focus ONLY on fixing the specific failures shown above", - "- Do NOT refactor or change unrelated code", - "- Do NOT mark the PR ready - the workflow will re-check Actions after you push", - ] - ) - - return "\n".join(parts) - - -async def execute_task(working_dir: str | None = None) -> dict: - """Execute the task using Claude Agent SDK. - - Args: - working_dir: Directory where Claude should work. If provided (repo was cloned), - Claude works in the repo. Otherwise uses WORKSPACE. - - """ - prompt = build_prompt() - print(f"[job_runner] Mode: {MODE}") +async def execute_task() -> dict: + """Execute the task using Claude Agent SDK.""" print(f"[job_runner] Model: {CLAUDE_MODEL}") - print(f"[job_runner] Prompt:\n{prompt[:500]}...") - - # Use native plan mode for planning (allows reads, blocks writes) - # Use bypassPermissions for implement/feedback/fix (needs full access) - perm_mode = "plan" if MODE == "plan" else "bypassPermissions" - print(f"[job_runner] Permission mode: {perm_mode}") - - # Use the cloned repo directory if available, otherwise use WORKSPACE - cwd = working_dir or WORKSPACE - print(f"[job_runner] Claude working directory: {cwd}") + print(f"[job_runner] Prompt:\n{TASK_PROMPT[:500]}...") options = ClaudeAgentOptions( model=CLAUDE_MODEL, - permission_mode=perm_mode, - cwd=cwd, + permission_mode="bypassPermissions", + cwd=WORKSPACE, ) collected_text: list[str] = [] - collected_questions: list[dict] = [] # Questions from AskUserQuestion tool - plan_content: str | None = None session_id: str | None = None cost_usd: float | None = None - should_stop_for_questions = False # Flag to break cleanly - async for message in query(prompt=prompt, options=options): + async for message in query(prompt=TASK_PROMPT, options=options): if isinstance(message, AssistantMessage): for block in message.content: if isinstance(block, TextBlock): print(f"[claude] {block.text[:200]}...") collected_text.append(block.text) - elif isinstance(block, ToolUseBlock): - # Capture plan from ExitPlanMode tool call - if block.name == "ExitPlanMode": - # ExitPlanMode doesn't pass content - find and read the plan file - plan_file_path = find_plan_file() - if plan_file_path: - plan_content = read_plan_file(plan_file_path) - print( - f"[claude] ExitPlanMode - read plan from {plan_file_path} ({len(plan_content)} chars)" - ) - else: - print("[claude] ExitPlanMode called but no plan file found") - - # Capture questions from AskUserQuestion tool call - elif block.name == "AskUserQuestion": - questions = block.input.get("questions", []) - print( - f"[claude] AskUserQuestion called with {len(questions)} question(s)" - ) - # Deduplicate by header to avoid repeats from multiple AskUserQuestion calls - existing_headers = {q["header"] for q in collected_questions} - for q in questions: - header = q.get("header", "") - if header and header in existing_headers: - print(f"[claude] Skipping duplicate question: {header}") - continue - existing_headers.add(header) - collected_questions.append( - { - "id": str(uuid.uuid4()), - "header": header, - "question": q.get("question", ""), - "options": [ - { - "label": opt.get("label", ""), - "description": opt.get("description"), - } - for opt in q.get("options", []) - ], - "multi_select": q.get("multiSelect", False), - } - ) - # In plan mode, flag to stop after this message - if MODE == "plan" and collected_questions: - print( - f"[job_runner] Will stop after this message to get user answers for {len(collected_questions)} question(s)" - ) - should_stop_for_questions = True elif isinstance(message, ResultMessage): session_id = message.session_id cost_usd = message.total_cost_usd if message.is_error: raise RuntimeError(message.result or "Claude execution failed") - # If we collected questions in plan mode, return early with questions - if should_stop_for_questions and collected_questions: - print( - f"[job_runner] Returning {len(collected_questions)} question(s) for user answers" - ) - plan_text = ( - "\n".join(collected_text) - if collected_text - else "Plan in progress - answering questions first" - ) - return { - "output": plan_text, - "plan_text": plan_text, - "questions": collected_questions, - "suggested_options": [], - "session_id": session_id, - "cost_usd": cost_usd, - } - - # Use plan content if available (from plan mode), otherwise use collected text - output = plan_content if plan_content else "\n".join(collected_text) - - # Try to extract URLs from output - pr_url = extract_pr_url(output) - issue_url = extract_issue_url(output) - - # For plan mode: return plan content and questions for inbox review - if MODE == "plan" and output: - # Prefer plan_content from ExitPlanMode (actual plan file), fall back to collected text - if plan_content: - plan_text = plan_content - print(f"[job_runner] Using plan from ExitPlanMode ({len(plan_text)} chars)") - else: - # Fall back to last substantial text block - plan_text = None - for text in reversed(collected_text): - if len(text) > 200: # Substantial text block - plan_text = text - break - plan_text = plan_text or output - print(f"[job_runner] Using collected text as plan ({len(plan_text)} chars)") - print(f"[job_runner] Collected {len(collected_questions)} question(s)") - - # Extract suggested options from the plan - suggested_options = extract_plan_options(plan_text) - print(f"[job_runner] Extracted {len(suggested_options)} options from plan") - - return { - "output": output, - "plan_text": plan_text, - "questions": collected_questions, # Questions from AskUserQuestion - "suggested_options": suggested_options, - "session_id": session_id, - "cost_usd": cost_usd, - } + output = "\n".join(collected_text) if collected_text else "No response generated." return { "output": output, "session_id": session_id, "cost_usd": cost_usd, - "pr_url": pr_url, - "issue_url": issue_url, } -def extract_pr_url(output: str) -> str | None: - """Try to extract a GitHub PR URL from the output.""" - import re - - # Match GitHub PR URLs - pr_pattern = r"https://github\.com/[^/]+/[^/]+/pull/\d+" - match = re.search(pr_pattern, output) - return match.group(0) if match else None - - -def extract_issue_url(output: str) -> str | None: - """Try to extract a GitHub issue URL from the output.""" - import re - - # Match GitHub issue URLs - issue_pattern = r"https://github\.com/[^/]+/[^/]+/issues/\d+" - match = re.search(issue_pattern, output) - return match.group(0) if match else None - - -def extract_plan_options(plan_text: str) -> list[str]: - """Extract suggested options from a plan for user selection. - - Looks for patterns like: - - **Option A**: ... - - **Option 1**: ... - - **Approach A**: ... - - Bullet points starting with "Option" - """ - import re - - options = ["Approve"] # Always include approve option - - # Match patterns like "**Option A**:", "**Approach 1**:", "- Option A:" - option_patterns = [ - r"\*\*Option\s+([A-Za-z0-9]+)\*\*[:\s]+([^\n]+)", # **Option A**: description - r"\*\*Approach\s+([A-Za-z0-9]+)\*\*[:\s]+([^\n]+)", # **Approach 1**: description - r"[-*]\s*Option\s+([A-Za-z0-9]+)[:\s]+([^\n]+)", # - Option A: description - r"[-*]\s*Approach\s+([A-Za-z0-9]+)[:\s]+([^\n]+)", # - Approach 1: description - ] - - for pattern in option_patterns: - matches = re.findall(pattern, plan_text, re.IGNORECASE) - for match in matches: - label, desc = match - # Create a concise option label - option = f"Use {label.strip()}" - if option not in options: - options.append(option) - - # If no options found, add some defaults - if len(options) == 1: - options.append("Request changes") - - return options - - -def create_github_issue_from_plan(plan_content: str) -> str | None: - """Create a GitHub issue from plan content using githubkit.""" - from githubkit import GitHub - - if not REPO_URL: - print("[job_runner] No REPO_URL, cannot create issue") - return None - - gh_token = os.environ.get("GH_TOKEN") - if not gh_token: - print("[job_runner] No GH_TOKEN, cannot create issue") - return None - - # Parse owner/repo from URL - parts = REPO_URL.rstrip("/").replace(".git", "").split("/") - owner, repo_name = parts[-2], parts[-1] - - title = ( - f"Plan: {TASK_PROMPT[:80]}" if len(TASK_PROMPT) > 80 else f"Plan: {TASK_PROMPT}" - ) - - body = f"""{plan_content} - ---- - -## Commands -Reply with: -- `/implement` - Approve this plan and start implementation -- `/revise ` - Request changes to the plan -""" - - try: - gh = GitHub(gh_token) - # Create issue - response = gh.rest.issues.create( - owner=owner, - repo=repo_name, - title=title, - body=body, - labels=["mainloop-plan"], - ) - issue_url = response.parsed_data.html_url - print(f"[job_runner] Created issue: {issue_url}") - return issue_url - except Exception as e: - print(f"[job_runner] Error creating issue: {e}") - return None - - async def send_result( status: str, result: dict | None = None, error: str | None = None ): @@ -655,7 +78,7 @@ async def send_result( return payload = { - "task_id": TASK_ID, + "session_id": SESSION_ID, "status": status, "result": result, "error": error, @@ -687,12 +110,12 @@ async def send_result( async def main(): """Execute the job runner workflow.""" - print(f"[job_runner] Starting job for task {TASK_ID}") + print(f"[job_runner] Starting job for session {SESSION_ID}") print(f"[job_runner] Working directory: {WORKSPACE}") # Validate required env vars - if not TASK_ID: - print("[job_runner] ERROR: TASK_ID not set") + if not SESSION_ID: + print("[job_runner] ERROR: SESSION_ID not set") sys.exit(1) if not TASK_PROMPT: print("[job_runner] ERROR: TASK_PROMPT not set") @@ -702,21 +125,8 @@ async def main(): Path(WORKSPACE).mkdir(parents=True, exist_ok=True) os.chdir(WORKSPACE) - # Pre-clone repo if URL provided - repo_dir = pre_clone_repo() - if REPO_URL and not repo_dir: - # Clone was required but failed - abort the job - error_msg = "Failed to clone repository - check GH_TOKEN and repo URL" - print(f"[job_runner] ERROR: {error_msg}") - await send_result(status="failed", error=error_msg) - sys.exit(1) - if repo_dir: - # Change to cloned repo directory so Claude can work on it - os.chdir(repo_dir) - print(f"[job_runner] Working in repo: {repo_dir}") - try: - result = await execute_task(working_dir=repo_dir) + result = await execute_task() await send_result( status="completed", result=result, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 3d21817..71f9199 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -63,67 +63,6 @@ export interface QueueItem { expires_at: string | null; } -export interface QuestionOption { - label: string; - description: string | null; -} - -export interface TaskQuestion { - id: string; - header: string; - question: string; - options: QuestionOption[]; - multi_select: boolean; - response: string | null; -} - -export interface WorkerTask { - id: string; - main_thread_id: string; - user_id: string; - task_type: string; - description: string; - prompt: string; - model: string | null; - repo_url: string | null; - project_id: string | null; - branch_name: string | null; - base_branch: string; - status: string; - workflow_run_id: string | null; - worker_pod_name: string | null; - created_at: string; - started_at: string | null; - completed_at: string | null; - result: Record | null; - error: string | null; - // Plan phase (issue) - issue_url: string | null; - issue_number: number | null; - // Implementation phase (PR) - pr_url: string | null; - pr_number: number | null; - commit_sha: string | null; - conversation_id: string | null; - message_id: string | null; - keywords: string[]; - skip_plan: boolean; - // Interactive planning state - pending_questions: TaskQuestion[] | null; - plan_text: string | null; -} - -export interface TaskContext { - task: WorkerTask; - queue_items: QueueItem[]; -} - -export interface TaskLogsResponse { - logs: string; - source: 'k8s' | 'none'; - task_status: string; -} - export interface Project { id: string; user_id: string; @@ -164,7 +103,7 @@ export interface ProjectDetail { project: Project; open_prs: ProjectPRSummary[]; recent_commits: CommitSummary[]; - tasks: WorkerTask[]; + sessions: Session[]; } // Session types @@ -175,22 +114,9 @@ export type SessionStatus = | 'failed' | 'cancelled' | 'waiting_on_user' - | 'waiting_questions' - | 'waiting_plan_review' - | 'ready_to_implement' - | 'planning' | 'implementing' | 'under_review'; -export interface SessionQuestion { - id: string; - header: string; - question: string; - options: QuestionOption[]; - multi_select: boolean; - response: string | null; -} - export interface Session { id: string; user_id: string; @@ -222,11 +148,6 @@ export interface Session { // Inline thread anchoring anchor_message_id: string | null; color: string | null; - // Planning fields - keywords: string[]; - skip_plan: boolean; - pending_questions: SessionQuestion[] | null; - plan_text: string | null; result: Record | null; } @@ -235,7 +156,6 @@ export interface SessionCreate { description: string; prompt: string; repo_url?: string; - skip_plan?: boolean; anchor_message_id?: string; } @@ -358,93 +278,6 @@ export const api = { if (!response.ok) throw new Error('Failed to refresh project'); }, - // Task endpoints - async listTasks(options?: { status?: string; projectId?: string }): Promise { - const params = new URLSearchParams(); - if (options?.status) params.set('status', options.status); - if (options?.projectId) params.set('project_id', options.projectId); - const url = params.toString() ? `${API_URL}/tasks?${params}` : `${API_URL}/tasks`; - const response = await fetch(url); - if (!response.ok) throw new Error('Failed to list tasks'); - return response.json(); - }, - - async getTask(taskId: string): Promise { - const response = await fetch(`${API_URL}/tasks/${taskId}`); - if (!response.ok) throw new Error('Failed to get task'); - return response.json(); - }, - - async getTaskContext(taskId: string): Promise { - const response = await fetch(`${API_URL}/tasks/${taskId}/context`); - if (!response.ok) throw new Error('Failed to get task context'); - return response.json(); - }, - - async cancelTask(taskId: string): Promise { - const response = await fetch(`${API_URL}/tasks/${taskId}/cancel`, { - method: 'POST' - }); - if (!response.ok) throw new Error('Failed to cancel task'); - }, - - async retryTask(taskId: string): Promise { - const response = await fetch(`${API_URL}/tasks/${taskId}/retry`, { - method: 'POST' - }); - if (!response.ok) throw new Error('Failed to retry task'); - }, - - async getTaskLogs(taskId: string, tail: number = 100): Promise { - const response = await fetch(`${API_URL}/tasks/${taskId}/logs?tail=${tail}`); - if (!response.ok) throw new Error('Failed to get task logs'); - return response.json(); - }, - - async answerTaskQuestions( - taskId: string, - answers: Record, - action: 'answer' | 'cancel' = 'answer' - ): Promise { - const response = await fetch(`${API_URL}/tasks/${taskId}/answer-questions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ answers, action }) - }); - if (!response.ok) throw new Error('Failed to answer questions'); - }, - - async approveTaskPlan( - taskId: string, - action: 'approve' | 'cancel' | 'revise' = 'approve', - revisionText?: string - ): Promise { - const params = new URLSearchParams({ action }); - if (revisionText) params.set('revision_text', revisionText); - - const response = await fetch(`${API_URL}/tasks/${taskId}/approve-plan?${params}`, { - method: 'POST' - }); - if (!response.ok) throw new Error('Failed to approve plan'); - }, - - async startImplementation(taskId: string): Promise { - const response = await fetch(`${API_URL}/tasks/${taskId}/start-implementation`, { - method: 'POST' - }); - if (!response.ok) throw new Error('Failed to start implementation'); - }, - - /** - * Get the SSE endpoint URL for streaming task logs. - * Use with EventSource or the SSE client. - */ - getTaskLogsStreamUrl(taskId: string): string { - return `${API_URL}/tasks/${taskId}/logs/stream`; - }, - /** * Get the SSE endpoint URL for the global event stream. */ diff --git a/frontend/src/lib/components/ConversationView.svelte b/frontend/src/lib/components/ConversationView.svelte index f81bcc8..fa66de1 100644 --- a/frontend/src/lib/components/ConversationView.svelte +++ b/frontend/src/lib/components/ConversationView.svelte @@ -214,7 +214,7 @@ {#if $currentSession} {@const sessionColor = $currentSession.color} - {#if ['pending', 'active', 'planning', 'implementing'].includes($currentSession.status)} + {#if ['pending', 'active', 'implementing'].includes($currentSession.status)}
{ fetchLogs(); - const activeStatuses = ['planning', 'implementing', 'pending']; + const activeStatuses = ['implementing', 'pending']; if (activeStatuses.includes(taskStatus)) { pollInterval = setInterval(fetchLogs, 3000); } @@ -220,15 +216,6 @@ Copying credentials...
- {:else if taskStatus === 'planning'} -
- - - - - - Claude is exploring the codebase... -
{:else if taskStatus === 'implementing'}
@@ -237,15 +224,6 @@ Claude is writing code...
- {:else if taskStatus === 'waiting_plan_review'} -
- - - - - - Waiting for your review in inbox -
{:else}

Waiting for activity...

{/if} diff --git a/frontend/src/lib/components/SessionBlock.svelte b/frontend/src/lib/components/SessionBlock.svelte index c55e313..a452c9f 100644 --- a/frontend/src/lib/components/SessionBlock.svelte +++ b/frontend/src/lib/components/SessionBlock.svelte @@ -20,10 +20,6 @@ pending: 'PENDING', active: 'ACTIVE', waiting_on_user: 'NEEDS INPUT', - waiting_questions: 'NEEDS INPUT', - waiting_plan_review: 'REVIEW PLAN', - ready_to_implement: 'READY', - planning: 'PLANNING', implementing: 'IMPLEMENTING', under_review: 'IN REVIEW', completed: 'DONE', @@ -32,12 +28,10 @@ }; const isRunning = $derived( - ['pending', 'active', 'planning', 'implementing'].includes(session.status) + ['pending', 'active', 'implementing'].includes(session.status) ); - const needsAttention = $derived( - ['waiting_on_user', 'waiting_questions', 'waiting_plan_review'].includes(session.status) - ); + const needsAttention = $derived(session.status === 'waiting_on_user'); function handleClick() { if (onSelect) { diff --git a/frontend/src/lib/components/SessionListItem.svelte b/frontend/src/lib/components/SessionListItem.svelte index 8b7651e..8987120 100644 --- a/frontend/src/lib/components/SessionListItem.svelte +++ b/frontend/src/lib/components/SessionListItem.svelte @@ -13,10 +13,6 @@ pending: 'text-term-yellow', active: 'text-term-cyan', waiting_on_user: 'text-term-magenta', - waiting_questions: 'text-term-magenta', - waiting_plan_review: 'text-term-magenta', - ready_to_implement: 'text-term-yellow', - planning: 'text-term-cyan', implementing: 'text-term-cyan', under_review: 'text-term-yellow', completed: 'text-term-green', @@ -28,10 +24,6 @@ pending: 'PENDING', active: 'ACTIVE', waiting_on_user: 'NEEDS INPUT', - waiting_questions: 'NEEDS INPUT', - waiting_plan_review: 'REVIEW PLAN', - ready_to_implement: 'READY', - planning: 'PLANNING', implementing: 'IMPLEMENTING', under_review: 'IN REVIEW', completed: 'DONE', @@ -40,14 +32,10 @@ }; // Check if session needs user attention - const needsAttention = $derived( - ['waiting_on_user', 'waiting_questions', 'waiting_plan_review'].includes(session.status) - ); + const needsAttention = $derived(session.status === 'waiting_on_user'); // Check if session is actively running - const isActive = $derived( - ['active', 'planning', 'implementing'].includes(session.status) - ); + const isActive = $derived(['active', 'implementing'].includes(session.status)); // Extract repo name from URL function getRepoName(repoUrl: string): string { diff --git a/frontend/src/lib/components/SessionPicker.svelte b/frontend/src/lib/components/SessionPicker.svelte index 2fca44d..1762efe 100644 --- a/frontend/src/lib/components/SessionPicker.svelte +++ b/frontend/src/lib/components/SessionPicker.svelte @@ -19,10 +19,7 @@ // Status styling const statusColors: Record = { waiting_on_user: 'text-term-magenta', - waiting_questions: 'text-term-magenta', - waiting_plan_review: 'text-term-magenta', active: 'text-term-cyan', - planning: 'text-term-cyan', implementing: 'text-term-cyan', pending: 'text-term-yellow', completed: 'text-term-green', @@ -32,10 +29,7 @@ const statusLabels: Record = { waiting_on_user: 'NEEDS INPUT', - waiting_questions: 'NEEDS INPUT', - waiting_plan_review: 'REVIEW PLAN', active: 'ACTIVE', - planning: 'PLANNING', implementing: 'IMPLEMENTING', pending: 'PENDING', completed: 'DONE', @@ -148,7 +142,7 @@ {#each sortedSessions as session, i (session.id)} {@const itemIndex = i + 1} {@const isSelected = selectedIndex === itemIndex} - {@const needsAttention = ['waiting_on_user', 'waiting_questions', 'waiting_plan_review'].includes(session.status)} + {@const needsAttention = session.status === 'waiting_on_user'}