diff --git a/experimental/aitools/AGENT_TEMPLATE_ARCHITECTURE.md b/experimental/aitools/AGENT_TEMPLATE_ARCHITECTURE.md new file mode 100644 index 0000000000..fc4df56079 --- /dev/null +++ b/experimental/aitools/AGENT_TEMPLATE_ARCHITECTURE.md @@ -0,0 +1,281 @@ +# Agent Template Architecture: Design Decisions + +## Overview + +This document explains the architectural decisions around the agent template structure and its relationship to other app templates in the experimental AI tools. + +## Current Architecture + +### Template Structure + +The CLI provides several template types under `experimental aitools tools init-template`: + +- **app** - UI-focused web applications using AppKit (TypeScript/React/tRPC) +- **agent** - Conversational AI agents using OpenAI Agents SDK (Python) +- **job** - Python notebooks with wheel packages +- **pipeline** - Lakeflow Declarative Pipelines (SQL or Python) +- **empty** - Minimal bundle for custom resources + +### Template Locations + +- **Experimental templates** (app, agent): `experimental/aitools/templates/` + - Fetched from GitHub by default + - Support local override via environment variables + - Active development, evolving rapidly + +- **Stable templates** (job, pipeline, empty): `libs/template/templates/` + - Bundled with CLI binary + - Production-ready, stable APIs + - Standard template system + +## Should Agent Be a Sub-template of App? + +### Context + +Both "app" and "agent" templates ultimately deploy as Databricks Apps, but serve different purposes: + +- **App (appkit)**: TypeScript/React frontend, tRPC API layer, SQL queries, UI-focused +- **Agent**: Python backend, OpenAI Agents SDK, MLflow tracing, conversational AI (no UI by default) + +### Option 1: Keep Separate (Current Implementation) ✅ + +**Pros:** +- **Clear mental model**: "app" = UI apps, "agent" = AI agents +- **Independent evolution**: Can iterate on agent features without affecting UI apps +- **Simpler schemas**: Each template has focused, easy-to-understand configuration +- **No tech stack interference**: TypeScript/React vs Python/Agents SDK don't mix +- **Clearer documentation**: Each template has dedicated docs for its specific use case +- **Easier user experience**: Users know exactly what they're getting +- **Faster scaffolding**: No need to process complex conditionals + +**Cons:** +- **Some duplication**: App deployment configuration repeated across templates +- **Multiple commands**: Two separate entry points for app-related templates +- **Inconsistent patterns**: Each template may evolve deployment patterns differently + +### Option 2: Merge as Sub-templates + +**Approach:** Use conditional logic like `default-python` template: + +```json +{ + "properties": { + "app_type": { + "type": "string", + "enum": ["ui", "agent", "streamlit", "gradio"], + "description": "Type of app to create" + } + } +} +``` + +With `__preamble.tmpl`: +```go +{{$is_ui_app := eq .app_type "ui"}} +{{$is_agent := eq .app_type "agent"}} + +{{if not $is_ui_app}} + {{skip "{{.project_name}}/src/frontend"}} +{{end}} + +{{if not $is_agent}} + {{skip "{{.project_name}}/agent_server"}} +{{end}} +``` + +**Pros:** +- **Single entry point**: One "app" command for all app types +- **Shared deployment patterns**: Consistent app configuration across types +- **Extensible**: Easy to add new app types (Streamlit, Gradio, Flask) +- **Category clarity**: Everything under "app" is an app + +**Cons:** +- **Complex schema**: Many conditional options confuse users +- **Divergent tech stacks**: TypeScript/React + Python/Agents in one template is messy +- **Maintenance burden**: Changes to one app type risk breaking others +- **"Mega-template" anti-pattern**: Single template trying to do too much +- **Slower scaffolding**: Must process many conditionals even for simple cases +- **Documentation complexity**: Docs must explain all app types and their differences +- **Testing complexity**: Must test all combinations of conditionals + +## Recommendation: Keep Separate + Future Category System + +### Current (Recommended) + +Keep templates separate as they are now: + +```bash +databricks experimental aitools tools init-template app --name my-ui-app +databricks experimental aitools tools init-template agent --name my-agent +``` + +**Why:** +- Templates have fundamentally different tech stacks and purposes +- Each template can evolve independently +- Clear, focused user experience +- Simpler maintenance and testing + +### Future Enhancement: Category-Based Discovery + +If we need better organization, introduce a category system for discovery without merging templates: + +```bash +# Discovery +databricks experimental aitools tools init-template list --category apps +# Shows: app (ui), agent (ai), streamlit (dashboards), gradio (ml-demos) + +# Scaffolding (same as today) +databricks experimental aitools tools init-template app --name my-ui-app +databricks experimental aitools tools init-template agent --name my-agent +databricks experimental aitools tools init-template streamlit --name my-dashboard +``` + +This provides: +- ✅ Discoverability via categories +- ✅ Consistent app deployment experience +- ✅ Independent template maintenance +- ✅ Clear user experience +- ✅ Extensibility for new app types + +## How Standard Templates Handle Multiple Use Cases + +The `default-python` template provides a reference implementation: + +### Schema-Based Feature Flags + +```json +{ + "include_job": {"enum": ["yes", "no"]}, + "include_pipeline": {"enum": ["yes", "no"]}, + "include_python": {"enum": ["yes", "no"]}, + "language": {"enum": ["python", "sql"]} +} +``` + +### Conditional File Inclusion + +```go +{{$pipeline := eq .include_pipeline "yes"}} +{{$notebook_job := eq .include_job "yes"}} + +{{if not $notebook_job}} + {{skip "{{.project_name}}/src/sample_notebook.ipynb"}} +{{end}} + +{{if not $pipeline}} + {{skip "{{.project_name}}/resources/{{.project_name}}_etl.pipeline.yml"}} +{{end}} +``` + +### When to Use This Pattern + +✅ **Good for:** +- Templates with similar tech stacks (all Python, or all TypeScript) +- Variations that share most files (80%+ overlap) +- Features that can be mixed and matched +- Projects where users commonly want multiple features together + +❌ **Bad for:** +- Templates with different tech stacks (Python + TypeScript) +- Variations that share few files (<50% overlap) +- Mutually exclusive project types +- Complex conditional logic that confuses users + +## Decision Matrix + +| Factor | Keep Separate | Merge with Conditionals | +|--------|--------------|------------------------| +| Tech stack similarity | Different (TS vs Python) ✅ | Same/similar stack ✅ | +| File overlap | <30% | >80% ✅ | +| User mental model | Clear ✅ | Complex | +| Maintenance complexity | Low ✅ | High | +| Testing complexity | Low ✅ | High | +| Extensibility | Medium | High ✅ | +| Documentation clarity | High ✅ | Medium | + +**Conclusion:** For agent + app templates, "Keep Separate" is the right choice based on: +- Different tech stacks (TypeScript/React vs Python) +- Minimal file overlap (~20%) +- Clear mental models (UI app vs AI agent) + +## Implementation Notes + +### Environment Variable Overrides + +Both experimental templates support local development overrides: + +```bash +# App template +export DATABRICKS_APPKIT_TEMPLATE_PATH="/path/to/local/appkit" +databricks experimental aitools tools init-template app --name my-app + +# Agent template +export DATABRICKS_AGENT_TEMPLATE_PATH="/path/to/local/agent" +databricks experimental aitools tools init-template agent --name my-agent +``` + +This pattern is **only for experimental templates** - stable templates (job, pipeline, empty) are bundled with the CLI and don't need it. + +### Template Constants + +Each template command defines: + +```go +const ( + templateRepo = "https://github.com/databricks/cli" + templateDir = "experimental/aitools/templates/agent-openai-agents-sdk" + branch = "main" + pathEnvVar = "DATABRICKS_AGENT_TEMPLATE_PATH" +) +``` + +This allows: +- Default: Fetch from GitHub main branch +- Override: Use local path via environment variable + +## Future Considerations + +### Adding New App Types + +When adding new app types (Streamlit, Gradio, Flask): + +1. **Evaluate tech stack**: Similar to existing? → Consider sub-template. Different? → New template. +2. **Check file overlap**: >80%? → Consider sub-template. <50%? → New template. +3. **Assess user needs**: Do users commonly want both together? → Sub-template. Separate use cases? → New template. + +### Example: Streamlit Dashboard Template + +**Tech stack:** Python (similar to agent, different from app) +**File overlap:** ~30% with agent (databricks.yml, app.yaml, deployment) +**User needs:** Usually separate from conversational agents + +**Decision:** Create separate `streamlit` template + +```bash +databricks experimental aitools tools init-template streamlit --name my-dashboard +``` + +### Example: Agent with UI + +**Tech stack:** Python + TypeScript (hybrid) +**File overlap:** Needs both agent and app files +**User needs:** Common use case - deploy agent with chat UI + +**Decision:** This is a good candidate for a new combined template + +```bash +databricks experimental aitools tools init-template agent-ui --name my-agent-app +``` + +This would: +- Include agent backend (Python/OpenAI SDK) +- Include chat frontend (TypeScript/React) +- Use existing agent + app templates as reference +- Be a new template, not a sub-template of either + +## References + +- Standard template system: `libs/template/` +- Experimental templates: `experimental/aitools/templates/` +- Template schema reference: `databricks_template_schema.json` in each template +- Conditional logic examples: `libs/template/templates/default/template/__preamble.tmpl` diff --git a/experimental/aitools/AGENT_TEMPLATE_DISCOVERY_FIX.md b/experimental/aitools/AGENT_TEMPLATE_DISCOVERY_FIX.md new file mode 100644 index 0000000000..6cc300cd51 --- /dev/null +++ b/experimental/aitools/AGENT_TEMPLATE_DISCOVERY_FIX.md @@ -0,0 +1,192 @@ +# Agent Template Discovery Fix + +## Issue Summary + +When Claude Code users asked to create an agent, Claude would incorrectly use the `app` template instead of the `agent` template, resulting in errors like: + +``` +Error: validation for template input parameters failed. +no value provided for required property sql_warehouse_id +``` + +## Root Cause + +The `databricks_discover` MCP tool output did not include the `agent` template in its scaffolding examples. The output showed: + +``` +For apps: +invoke_databricks_cli 'experimental aitools tools init-template app --name my-app' + +For jobs (Python notebooks with wheel package): +invoke_databricks_cli 'experimental aitools tools init-template job --name my_job' + +For pipelines (Lakeflow Declarative Pipelines): +invoke_databricks_cli 'experimental aitools tools init-template pipeline --name my_pipeline' + +For custom resources (dashboards, alerts, etc.): +invoke_databricks_cli 'experimental aitools tools init-template empty --name my_project' +``` + +**Missing:** The agent template example! + +When Claude saw a request like "create an agent" or "scaffold an agent project", it would: +1. Look at the databricks_discover examples +2. Not find an `agent` template example +3. Default to using the `app` template (closest match) +4. Hit the SQL warehouse validation error since apps require a warehouse but the user didn't provide one + +## Why SQL Warehouse Was Required for Apps + +The `app` template (appkit) requires a SQL warehouse because: + +1. **Template schema defines it**: `experimental/aitools/templates/appkit/databricks_template_schema.json` + ```json + { + "properties": { + "sql_warehouse_id": { + "type": "string", + "description": "SQL Warehouse ID", + "order": 2 + } + } + } + ``` + +2. **Validation enforces all properties**: `libs/template/config.go` + ```go + func (c *config) validate() error { + // For final validation, all properties in the JSON schema + // should have a value defined. + c.schema.Required = maps.Keys(c.schema.Properties) + if err := c.schema.ValidateInstance(c.values); err != nil { + return fmt.Errorf("validation for template input parameters failed. %w", err) + } + return nil + } + ``` + +3. **AppKit needs SQL access**: The appkit template creates TypeScript/React apps that query SQL warehouses via tRPC, so a warehouse is a core requirement. + +## Why Agent Template Doesn't Need SQL Warehouse + +The `agent` template (OpenAI Agents SDK) has different requirements: + +1. **Template schema is minimal**: `experimental/aitools/templates/agent-openai-agents-sdk/databricks_template_schema.json` + ```json + { + "properties": { + "project_name": { + "type": "string", + "description": "Project Name", + "order": 1 + }, + "mlflow_experiment_name": { + "type": "string", + "description": "MLflow Experiment Name (Optional)", + "default": "", + "order": 2 + } + } + } + ``` + +2. **Agents use MCP, not SQL directly**: Agents access data through MCP servers (Unity Catalog functions, code interpreter), not by directly querying SQL warehouses. + +3. **Different architecture**: Agents are Python-based conversational AI, while apps are TypeScript/React data dashboards. + +## Solution + +Added the agent template to the `databricks_discover` output in `experimental/aitools/lib/prompts/flow.tmpl`: + +```diff + For apps: + invoke_databricks_cli 'experimental aitools tools init-template app --name my-app --description "My app description"' + ++For agents (Conversational AI with OpenAI Agents SDK): ++invoke_databricks_cli 'experimental aitools tools init-template agent --name my_agent' ++Note: Creates Python agent with OpenAI Agents SDK, Claude model, MLflow tracing, and MCP integration ++ + For jobs (Python notebooks with wheel package): + invoke_databricks_cli 'experimental aitools tools init-template job --name my_job' +``` + +Also updated the naming conventions note: +```diff + Notes: + - App name must be ≤26 characters (dev- prefix adds 4 chars, max total 30) +-- Job/pipeline/project names: letters, numbers, underscores only ++- Agent/job/pipeline/project names: letters, numbers, underscores only +``` + +## Impact + +After this fix: +- ✅ Claude Code sees the agent template option in databricks_discover +- ✅ When users ask to "create an agent", Claude uses the correct template +- ✅ No SQL warehouse validation errors for agent projects +- ✅ Clear distinction between app (UI + SQL) and agent (AI + MCP) templates + +## Testing + +To verify the fix works: + +1. **Restart Claude Code** to pick up the new CLI build +2. **Ask Claude**: "Create a new agent called test_agent" +3. **Expected behavior**: Claude uses `experimental aitools tools init-template agent --name test_agent` +4. **Expected output**: Agent project scaffolded without SQL warehouse errors + +### Before Fix + +``` +User: Create a new agent +Claude: I'll create an app for you +→ invoke_databricks_cli 'experimental aitools tools init-template app --name test-agent' +→ Error: no value provided for required property sql_warehouse_id +``` + +### After Fix + +``` +User: Create a new agent +Claude: I'll create an agent for you +→ invoke_databricks_cli 'experimental aitools tools init-template agent --name test_agent' +→ ✅ Success: Your new agent project has been created +``` + +## Related Files + +- **Discovery template**: `experimental/aitools/lib/prompts/flow.tmpl` (updated) +- **Agent template schema**: `experimental/aitools/templates/agent-openai-agents-sdk/databricks_template_schema.json` +- **App template schema**: `experimental/aitools/templates/appkit/databricks_template_schema.json` +- **Discovery implementation**: `experimental/aitools/lib/providers/clitools/discover.go` +- **Template validation**: `libs/template/config.go` + +## Key Learnings + +1. **Discovery is critical**: The `databricks_discover` output is the primary way Claude Code learns about available templates. Missing examples = missing functionality in Claude's eyes. + +2. **Template requirements vary**: Different templates have different required properties. Apps need SQL warehouses, agents don't. Make sure examples reflect these differences. + +3. **Validation is strict**: The template system marks ALL schema properties as required during validation. Properties without defaults must be explicitly provided. + +4. **Template vs MCP requirements**: Template scaffolding requirements (from schema) are separate from MCP server startup requirements (--warehouse-id flag). The MCP server can have a warehouse for SQL queries without requiring all templates to use it. + +## Future Improvements + +Consider adding more context to databricks_discover about when to use each template: + +``` +For apps (UI-focused data dashboards): +- Use when: Building interactive dashboards with SQL queries +- Tech stack: TypeScript/React/tRPC +- Requires: SQL warehouse for data access +invoke_databricks_cli 'experimental aitools tools init-template app --name my-app' + +For agents (Conversational AI): +- Use when: Building AI assistants, chatbots, or agentic workflows +- Tech stack: Python/OpenAI Agents SDK/MLflow +- Requires: No SQL warehouse (uses MCP for data access) +invoke_databricks_cli 'experimental aitools tools init-template agent --name my_agent' +``` + +This would help Claude Code make better decisions about which template to use based on user intent. diff --git a/experimental/aitools/APP_DEPLOY_FINDINGS.md b/experimental/aitools/APP_DEPLOY_FINDINGS.md new file mode 100644 index 0000000000..2466d62b6c --- /dev/null +++ b/experimental/aitools/APP_DEPLOY_FINDINGS.md @@ -0,0 +1,259 @@ + +| Issue | Description | Details | Status | +| :---- | :---- | :---- | :---- | +| MLflow experiment resources not in CLI | Required MLflow experiment resources for apps are not yet available in the released Databricks CLI | Must build the CLI from source to access experimental MLflow experiment support | No action needed, will be addressed with next Databricks CLI/SDK release. | +| Need to manually create experiment | \- | databricks.yml in the agent template doesn’t actually create an experiment; it requires passing an experiment ID as a variable. It would be better to create the experiment in the bundle itself (by default) and pass the ID of that experiment to the app as an app resource | TODO | +| Introducing an “agent” template confuses AI (and probably humans too) | The distinction between a new “agent” template and the existing app template is unclear | The concept of an “Agent” template is confusing to Claude (and probably to humans too :P) \- it doesn’t know when to use the “Agent” template or when to use the “app” template. If I ask it to build a “chatbot”, it tries to use the “app” template, which is pretty sensible but unfortunately not the right one | Backlog \- don’t work on this yet, needs discussion with app devex | +| Running from dev requires template override | Running the agent from a local dev environment needs an explicit template path override | Must set `DATABRICKS_AGENT_TEMPLATE_PATH="$HOME/cli/experimental/aitools/templates/agent-openai-agents-sdk"` to install from source instead of GitHub | Expected behavior | +| Validation fails due to Node.js assumptions | Claude runs the validation tool which assumes a Node.js frontend | Validation fails because the agent is pure Python and has no `package.json`; validation checks incorrectly run npm install | Backlog \- don’t work on this yet, needs discussion with app devex | +| Validation tool incompatible with Python agent | The validation process expects a Node.js-based app | The agent is Python-only and already deployable, but validation blocks deployment | Backlog \- don’t work on this yet, needs discussion with app devex | +| Quickstart script requires interactivity | The quickstart script in the agent app template fails in non-interactive execution | Script requires selecting a Databricks profile interactively; workaround is manually configuring `.env.local` | TODO. Address this by providing a flag-based or non-interactive option | +| FMAPI strict parameter validation error | Default agent template triggers strict FMAPI validation errors | Error: `tools.0.custom.strict: Extra inputs are not permitted`; Claude resolved by removing MCP tools from the agent | | +| Repo not initialized as Git repository | Generated project directory is not a Git repository | Causes `fatal: not a git repository` warnings during startup and logging | TODO. Address this by updating AGENT.md | +| Port 8000 already in use / Port flag ignored by start script | Backend ignores specified port and attempts to bind to 8000\. CLI `--port` argument is not respected | `uv run start-app --port 8001` still binds to port 8000, contradicting README | TODO. Investigate and address this | +| Difficulty awaiting app readiness | Claude struggles to wait for app creation and readiness before querying | Repeated deploy/get attempts while deployment is still in progress | | +| Required bundle variable not persisted | `mlflow_experiment_id` not remembered between bundle commands | Must repeatedly pass `--var mlflow_experiment_id=...` to avoid errors | Should address by updating bundle to create the experiment | +| App stuck in deployment-in-progress state | Attempts to redeploy fail due to active deployment | CLI reports “active deployment in progress” even when app status is unavailable | | +| Logs unavailable during deployment | App logs cannot be retrieved during certain deployment states | `apps logs` fails with HTTP 503 / bad websocket handshake | | + +Challenges: + +* MLflow experiment resources for apps not yet released in the CLI, need to build from source +* The concept of an “Agent” template is confusing to Claude (and probably to humans too :P) \- it doesn’t know when to use the “Agent” template or when to use the “app” template. If I ask it to build a “chatbot”, it tries to use the “app” template, which is pretty sensible but unfortunately not the right one +* Note/prerequisite for running from dev: need to override template path to install from source instead of from GitHub + +``` +export DATABRICKS_AGENT_TEMPLATE_PATH="$HOME/cli/experimental/aitools/templates/agent-openai-agents-sdk" +``` + +* Claude tries to run a validation command which fails against this Python-based app template + +``` +databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["experimental","aitools","tools","validate","./"]) + ⎿ Error: install failed (duration: 0.4s) + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + 🔍 Databricks AI Tools MCP server: Validating your app + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Validation Progress: + 🔄 Starting Node.js validation: build + typecheck + tests + ⏳ Step 1/6: Running Install... + ❌ Install failed (0.4s) + ❌ Failed to install dependencies + Exit code: 254 + Stdout: + Stderr: + (node:61716) ExperimentalWarning: CommonJS module /opt/homebrew/lib/node_modules/npm/node_modules/debug/src/node.js is loading + ES Module /opt/homebrew/lib/node_modules/npm/node_modules/supports-color/index.js using require(). + Support for loading ES Module in require() is an experimental feature and might change at any time + (Use `node --trace-warnings ...` to show where the warning was created) + npm error code ENOENT + npm error syscall open + npm error path /Users/sid.murching/customer_support_agent/package.json + npm error errno -2 + npm error enoent Could not read package.json: Error: ENOENT: no such file or directory, open + '/Users/sid.murching/customer_support_agent/package.json' + npm error enoent This is related to npm not being able to find a file. + npm error enoent + npm error A complete log of this run can be found in: /Users/sid.murching/.npm/_logs/2026-01-12T22_13_40_682Z-debug-0.log + Error: validation failed + Command failed with error: exit status 1 +⏺ The validation tool is checking for a Node.js frontend which this pure Python agent doesn02:13 PM Tclaude-sonnet-4-5-20250929on-based agent. The agent code is ready to deploy. +``` + +* Claude tries to run the quickstart script but it fails since it requires interactive input. Thankfully it finds a workaround (updating .env.local): + +``` +⏺ Bash(cd /Users/sid.murching/customer_support_agent && ./scripts/quickstart.sh) + ⎿ Error: Exit code 1 + Downloading https://nodejs.org/dist/v20.19.6/node-v20.19.6-darwin-arm64.tar.xz... +###### 0.0% +####################### +############################# +###################################################### +######################################################################## 100.0% + Computing checksum with sha256sum + Checksums matched! + =================================================================== + Agent on Apps - Quickstart Setup + =================================================================== + … +80 lines (ctrl+o to see all) +⏺ The quickstart script needs interactive input to select a Databricks profile. Let me configure it automatically using the DEFAULT profile. + +``` + +* Got the agent running locally, but the default agent template code is running into issues due to some FMAPI `strict` parameter validation. + +``` + +⏺ Bash(curl -X POST http://localhost:8001/invocations \ timeout: 1m 0s + -H "Content-Type: application/json" \…) + ⎿ { + "detail": "Error code: 400 - {'error_code': 'BAD_REQUEST', 'message': '{\"message\":\"tools.0.custom.strict: Extra inputs ar + e not permitted\"}'}" + … +5 lines (ctrl+o to expand) + +``` + +Claude got around it^ by removing MCP tools from the agent implementation. + +* Got two other errors; the repo that was created is not a .git repo and port 8000 was already in use + +``` +Last 50 lines of backend.log: +---------------------------------------- +fatal: not a git repository (or any of the parent directories): .git +2026/01/12 14:26:57 INFO mlflow.tracking.fluent: Active model is set to the logged model with ID: m-1010b5487fed49b6a208f7bf8e3c68e3 +2026/01/12 14:26:57 INFO mlflow.tracking.fluent: Use `mlflow.set_active_model` to set the active model to a different one if needed. +2026/01/12 14:26:57 INFO mlflow.genai.agent_server.utils: Active LoggedModel: 'local-no-git', Model ID: 'm-1010b5487fed49b6a208f7bf8e3c68e3' +INFO: Started server process [70415] +INFO: Waiting for application startup. +INFO: Application startup complete. +ERROR: [Errno 48] error while attempting to bind on address ('0.0.0.0', 8000): [errno 48] address already in use +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +---------------------------------------- +Last 50 lines of frontend.log: +---------------------------------------- +---------------------------------------- +========================================== +Shutting down both processes... +========================================== + +``` + +I can reproduce by running the server manually \- even if I try to use port 8001 following the README, it seems to be ignored.: + +``` +~/customer_support_agent +base ❯ uv run start-app --port 8001 [01/12/26 | 2:27:07] +Starting backend... +Running npm install... +Running npm build... +[backend] fatal: not a git repository (or any of the parent directories): .git +[backend] 2026/01/12 14:28:38 INFO mlflow.tracking.fluent: Active model is set to the logged model with ID: m-1010b5487fed49b6a208f7bf8e3c68e3 +[backend] 2026/01/12 14:28:38 INFO mlflow.tracking.fluent: Use `mlflow.set_active_model` to set the active model to a different one if needed. +[backend] 2026/01/12 14:28:38 INFO mlflow.genai.agent_server.utils: Active LoggedModel: 'local-no-git', Model ID: 'm-1010b5487fed49b6a208f7bf8e3c68e3' +[backend] INFO: Started server process [70949] +✓ Backend is ready! +[backend] INFO: Waiting for application startup. +[backend] INFO: Application startup complete. +[backend] ERROR: [Errno 48] error while attempting to bind on address ('0.0.0.0', 8000): [errno 48] address already in use +[backend] INFO: Waiting for application shutdown. +[backend] INFO: Application shutdown complete. +Starting frontend... +``` + +I manually killed the process on port 8000 to unblock making progress + +* Claude seems to struggle a bit with understanding how to await creation of the app and query it: + +``` +⏺ Update(customer_support_agent/databricks.yml) + ⎿ Added 1 line, removed 1 line + 9 resources: + 10 apps: + 11 customer_support_agent: + 12 - name: "${bundle.target}-customer_support_agent" + 12 + name: "${bundle.target}-customer-support-agent" + 13 description: "OpenAI Agents SDK agent application" + 14 source_code_path: ./ + 15 +⏺ Update(customer_support_agent/databricks.yml) + ⎿ Added 1 line, removed 1 line + 34 resources: + 35 apps: + 36 customer_support_agent: + 37 - name: customer_support_agent + 37 + name: customer-support-agent +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: ["bundle","deploy + ","--profile","DEFAULT","--var","mlflow_experiment_id=4043694633333313"]) + ⎿ Uploading bundle files to /Workspace/Users/sid.murching@databricks.com/.bundle/customer_support_agent/dev/files... + Deploying resources... + Updating deployment state... + Deployment complete! +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["bundle","summary","--profile","DEFAULT"]) + ⎿ Error: no value assigned to required variable mlflow_experiment_id. Variables are usually assigned in databricks.yml, and they + can be overridden using "--var", the BUNDLE_VAR_mlflow_experiment_id environment variable, or .databricks/bundle//var + iable-overrides.json + … +4 lines (ctrl+o to expand) +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: ["bundle","summar + y","--profile","DEFAULT","--var","mlflow_experiment_id=4043694633333313"]) + ⎿ Name: customer_support_agent + Target: dev + Workspace: + … +8 lines (ctrl+o to expand) +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","get","dev-customer-support-agent"]) + ⎿ { + "app_status": { + "message":"App status is unavailable.", + … +37 lines (ctrl+o to expand) +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","deploy","dev-customer-support-agent","--source-code-path", + "/Workspace/Users/sid.murching@databricks.com/.bundle/customer_support_agent/dev/files"]) + ⎿ Error: Cannot deploy app dev-customer-support-agent as there is an active deployment in progress. + Command failed with error: exit status 1 +⏺ Bash(sleep 30 && databricks --profile DEFAULT apps get dev-customer-support-agent --output json | python3 -c "import timeout: 1m + sys, json; data = json.load(sys.stdin); print(f\…) 0s + ⎿ App Status: UNAVAILABLE + Compute Status: STOPPED + URL: https://dev-customer-support-agent-3888667486068890.aws.databricksapps.com +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","list-deployments","dev-customer-support-agent"]) + ⎿ DeploymentId State CreatedAt +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","deploy","dev-customer-support-agent","--source-code-path", + "/Workspace/Users/sid.murching@databricks.com/.bundle/customer_support_agent/dev/files"]) + ⎿ Error: Cannot deploy app dev-customer-support-agent as there is an active deployment in progress. + Command failed with error: exit status 1 +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","get","dev-customer-support-agent","--output","json"]) + ⎿ { + "app_status": { + "message":"App status is unavailable.", + … +37 lines (ctrl+o to expand) +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","experimental","aitools","tools","deploy","--skip-validation"]) + ⎿ Error: unknown flag: --skip-validation + Usage: + … +15 lines (ctrl+o to expand) +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","experimental","aitools","tools","deploy","--var","mlflow_experime + nt_id=4043694633333313"]) + ⎿ Error: cannot deploy: project not validated (run validate first) + Command failed with error: exit status 1 +⏺ Let me check the app logs to see what's happening with the deployment. +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","logs","dev-customer-support-agent","--tail-lines","50"]) + ⎿ Error: websocket: bad handshake (HTTP 503 Service Unavailable) + Command failed with error: exit status 1 +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","start","dev-customer-support-agent"]) + ⎿ { + "app_status": { + "message":"App has status: App has not been deployed yet. Run your app by deploying source code", + … +37 lines (ctrl+o to expand) +⏺ Bash(sleep 5) + ⎿ (No content) +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","deploy","dev-customer-support-agent","--source-code-path", + "/Workspace/Users/sid.murching@databricks.com/.bundle/customer_support_agent/dev/files"]) + ⎿ { + "create_time":"2026-01-12T22:42:02Z", + "creator":"sid.murching@databricks.com", + … +12 lines (ctrl+o to expand) +⏺ databricks-mcp - invoke_databricks_cli (MCP)(working_directory: "/Users/sid.murching/customer_support_agent", args: + ["--profile","DEFAULT","apps","get","dev-customer-support-agent","--output","json"]) + ⎿ { + "active_deployment": { + "create_time":"2026-01-12T22:42:02Z", + … +52 lines (ctrl+o to expand) +⏺ Perfect! Your customer support agent has been successfully deployed to Databricks! 🎉 + Deployment Summary + App Name: dev-customer-support-agent + Status: ✅ RUNNING + URL: https://dev-customer-support-agent-3888667486068890.aws.databricksapps.com + +``` diff --git a/experimental/aitools/CLAUDE_AGENT_SETUP.md b/experimental/aitools/CLAUDE_AGENT_SETUP.md new file mode 100644 index 0000000000..88a615fbc4 --- /dev/null +++ b/experimental/aitools/CLAUDE_AGENT_SETUP.md @@ -0,0 +1,406 @@ +# Using the Agent Template with Claude Code + +This guide explains how to configure Claude Code to use the Databricks CLI from source and scaffold agent projects with the experimental agent template. + +## Quick Start + +### Prerequisites + +1. **Databricks CLI built from source** at `~/cli/cli` +2. **Claude Code** (or compatible MCP-enabled IDE) +3. **Databricks workspace** with authentication configured + +### Installation Steps + +#### 1. Configure Claude Code MCP Server + +Edit your Claude Code configuration file: + +**Location:** `~/Library/Application Support/Claude/claude_desktop_config.json` + +**Add the Databricks MCP server:** + +```json +{ + "mcpServers": { + "databricks-mcp": { + "command": "/Users/YOUR_USERNAME/cli/cli", + "args": [ + "experimental", + "aitools", + "--warehouse-id", + "YOUR_WAREHOUSE_ID" + ] + } + } +} +``` + +**Important:** +- Replace `YOUR_USERNAME` with your actual username +- Replace `YOUR_WAREHOUSE_ID` with your SQL warehouse ID (find in workspace or use default from profile) + +**Why this works:** The MCP server uses `os.Args[0]` to determine which CLI binary to invoke for all subsequent commands. By starting the MCP server with `~/cli/cli`, all CLI operations automatically use the source version. + +#### 2. Restart Claude Code + +After updating the configuration, **completely restart Claude Code** for changes to take effect. + +#### 3. Verify Installation + +In a new Claude Code session: + +1. Ask Claude to call `databricks_discover` with your working directory +2. Verify the output shows available commands including: + ``` + experimental aitools tools init-template agent --name my-agent + ``` + +#### 4. Authenticate with Workspace + +Before creating projects, authenticate: + +```bash +~/cli/cli auth login --profile dogfood +``` + +Or ask Claude to use the `databricks_configure_auth` MCP tool with your profile and host. + +## Creating Agent Projects + +### Using Local Template (Development) + +When testing local template changes: + +```bash +export DATABRICKS_AGENT_TEMPLATE_PATH="/Users/YOUR_USERNAME/cli/experimental/aitools/templates/agent-openai-agents-sdk" +~/cli/cli --profile dogfood experimental aitools tools init-template agent \ + --name my_agent \ + --output-dir ~/my-agent-app +``` + +### Using Remote Template (Production) + +Once the template is merged to main: + +```bash +~/cli/cli --profile dogfood experimental aitools tools init-template agent \ + --name my_agent \ + --output-dir ~/my-agent-app +``` + +The template will be automatically fetched from GitHub. + +### Via Claude Code + +Ask Claude: + +``` +Create a new Databricks agent app called "my_agent" using the agent template +``` + +Claude will: +1. Use the `invoke_databricks_cli` MCP tool +2. Set `DATABRICKS_AGENT_TEMPLATE_PATH` if testing local changes +3. Run the init-template command +4. Scaffold the project with all necessary files + +## What You Get + +The agent template creates a complete project with: + +``` +my-agent-app/ +├── agent_server/ +│ ├── agent.py # Agent logic, model config, MCP servers +│ ├── start_server.py # FastAPI server with MLflow tracing +│ ├── evaluate_agent.py # Agent evaluation framework +│ └── utils.py # Helper functions +├── scripts/ +│ ├── quickstart.sh # First-time setup script +│ └── start_app.py # Development server manager +├── databricks.yml # Bundle and app configuration +├── app.yaml # App environment variables +├── pyproject.toml # Python dependencies (uv) +├── AGENTS.md # Comprehensive development guide +├── CLAUDE.md # Claude Code integration +└── README.md # Quick start guide +``` + +### Key Features + +- **OpenAI Agents SDK** - Build conversational agents +- **Claude 3.7 Sonnet** - Pre-configured Databricks-hosted model +- **MCP integration** - Access to Unity Catalog functions (code interpreter) +- **MLflow tracing** - Automatic trace capture with `@invoke()` and `@stream()` decorators +- **Streaming support** - Both streaming and non-streaming endpoints +- **Evaluation framework** - Built-in MLflow scorers for agent quality + +## Next Steps After Scaffolding + +### Local Development + +```bash +cd my-agent-app +./scripts/quickstart.sh # Setup: auth, MLflow experiment, dependencies +uv run start-app # Start server at http://localhost:8000 +``` + +### Test Locally + +```bash +# Streaming +curl -X POST http://localhost:8000/invocations \ + -H "Content-Type: application/json" \ + -d '{"input": [{"role": "user", "content": "hi"}], "stream": true}' + +# Non-streaming +curl -X POST http://localhost:8000/invocations \ + -H "Content-Type: application/json" \ + -d '{"input": [{"role": "user", "content": "hi"}]}' +``` + +### Deploy to Databricks + +```bash +# Create MLflow experiment +~/cli/cli --profile dogfood experiments create-experiment \ + /Users/$USER/my-agent-app + +# Deploy bundle with experiment ID +~/cli/cli --profile dogfood bundle deploy \ + --var mlflow_experiment_id= + +# Get app URL +~/cli/cli --profile dogfood bundle summary +``` + +## Troubleshooting + +### MCP Server Not Found + +**Symptom:** Claude can't find Databricks tools + +**Solution:** +1. Verify `claude_desktop_config.json` has correct path to `~/cli/cli` +2. Check that `~/cli/cli` exists and is executable +3. Restart Claude Code completely + +### Template Not Available + +**Symptom:** Command says "unknown template" or can't find agent template + +**Solution:** +1. Ensure CLI was built with template: `cd ~/cli && make build` +2. Verify template exists: `ls ~/cli/experimental/aitools/templates/agent-openai-agents-sdk` +3. Check `databricks_template_schema.json` exists in template directory +4. Use `DATABRICKS_AGENT_TEMPLATE_PATH` environment variable for local template + +### Authentication Issues + +**Symptom:** "Invalid token" or "Not authenticated" + +**Solution:** +1. Run: `~/cli/cli auth login --profile ` +2. Or use Claude's `databricks_configure_auth` tool with profile and host +3. Verify: `~/cli/cli current-user me --profile ` + +### App Name Validation Errors + +**Symptom:** "App name must contain only lowercase letters, numbers, and dashes" + +**Solution:** +- Use only lowercase letters, numbers, and dashes in app names +- ✅ Good: `my-agent-app` +- ❌ Bad: `my_agent_app` (underscores not allowed) +- ❌ Bad: `MyAgentApp` (uppercase not allowed) + +## Understanding the MCP Server Architecture + +### How CLI Path Resolution Works + +```go +// In experimental/aitools/lib/common/common.go +func GetCLIPath() string { + return os.Args[0] // Returns the binary that started the process +} +``` + +When you configure Claude Code with: +```json +{ + "command": "/Users/YOUR_USERNAME/cli/cli", + "args": ["experimental", "aitools", "--warehouse-id", "..."] +} +``` + +The MCP server starts with `os.Args[0] = /Users/YOUR_USERNAME/cli/cli` + +Then every CLI invocation uses: +```go +cmd := exec.CommandContext(ctx, cliPath, args...) // cliPath = os.Args[0] +``` + +This ensures all CLI commands run from source automatically! + +### Why This Matters for Development + +**Without source CLI:** +- Changes to templates require CLI release +- Can't test experimental features +- Limited to stable, released functionality + +**With source CLI:** +- Immediate access to template changes +- Test experimental features before release +- Iterate quickly on template improvements +- Override templates with local versions via environment variables + +## Environment Variable Overrides + +### Agent Template + +```bash +export DATABRICKS_AGENT_TEMPLATE_PATH="/path/to/local/agent-template" +``` + +This tells the agent command to use your local template instead of fetching from GitHub. + +### App Template + +```bash +export DATABRICKS_APPKIT_TEMPLATE_PATH="/path/to/local/appkit-template" +``` + +This tells the app command to use your local template instead of fetching from GitHub. + +### When to Use + +**Local development:** +- Testing template changes before committing +- Iterating on template structure +- Debugging template issues +- Adding new features to templates + +**Not needed when:** +- Using released/merged templates +- Template is already in your CLI build +- Working with stable templates (job, pipeline, empty) + +## Advanced: Multiple CLI Versions + +If you need to work with multiple CLI versions: + +### Option 1: Multiple MCP Servers + +```json +{ + "mcpServers": { + "databricks-mcp-source": { + "command": "/Users/YOUR_USERNAME/cli/cli", + "args": ["experimental", "aitools", "--warehouse-id", "..."] + }, + "databricks-mcp-stable": { + "command": "/usr/local/bin/databricks", + "args": ["experimental", "aitools", "--warehouse-id", "..."] + } + } +} +``` + +Claude will have access to both and can use either based on context. + +### Option 2: Swap Configuration + +Keep different config files and swap them: + +```bash +# Use source CLI +cp ~/.config/claude-source.json ~/Library/Application\ Support/Claude/claude_desktop_config.json + +# Use stable CLI +cp ~/.config/claude-stable.json ~/Library/Application\ Support/Claude/claude_desktop_config.json +``` + +Restart Claude Code after swapping. + +## Template Development Workflow + +### 1. Make Template Changes + +Edit files in: +``` +~/cli/experimental/aitools/templates/agent-openai-agents-sdk/ +``` + +### 2. Build CLI (if code changes) + +```bash +cd ~/cli +make build +``` + +**Note:** Template file changes don't require rebuild - they're read at runtime. + +### 3. Test with Local Override + +```bash +export DATABRICKS_AGENT_TEMPLATE_PATH="$HOME/cli/experimental/aitools/templates/agent-openai-agents-sdk" +~/cli/cli experimental aitools tools init-template agent --name test-agent --output-dir /tmp/test +``` + +### 4. Verify Scaffolded Project + +```bash +cd /tmp/test/test-agent +ls -la +cat databricks.yml +``` + +### 5. Test Local Development + +```bash +./scripts/quickstart.sh +uv run start-app +# Test at http://localhost:8000 +``` + +### 6. Commit and Push + +```bash +cd ~/cli +git add experimental/aitools/templates/agent-openai-agents-sdk/ +git commit -m "Update agent template: [description]" +git push +``` + +### 7. Test Without Override + +After merging to main, test without environment variable: + +```bash +unset DATABRICKS_AGENT_TEMPLATE_PATH +~/cli/cli experimental aitools tools init-template agent --name test-agent2 --output-dir /tmp/test2 +``` + +This verifies the template works when fetched from GitHub. + +## Comparison: Experimental vs Stable Templates + +| Feature | Experimental (agent, app) | Stable (job, pipeline, empty) | +|---------|--------------------------|------------------------------| +| Location | `experimental/aitools/templates/` | `libs/template/templates/` | +| Source | GitHub (fetched) | CLI binary (bundled) | +| Override | Environment variable | Not needed | +| Evolution | Rapid, experimental | Stable, production-ready | +| CLI flag | `experimental aitools tools` | `bundle init` | +| Use case | Development, testing | Production, stable features | + +## References + +- **MCP Server Implementation:** `experimental/aitools/lib/providers/clitools/` +- **CLI Path Resolution:** `experimental/aitools/lib/common/common.go` +- **Agent Template Command:** `experimental/aitools/cmd/init_template/agent.go` +- **Agent Template Files:** `experimental/aitools/templates/agent-openai-agents-sdk/` +- **Template Schema:** `databricks_template_schema.json` in each template +- **Architecture Decisions:** `experimental/aitools/AGENT_TEMPLATE_ARCHITECTURE.md` diff --git a/experimental/aitools/cmd/init_template/agent.go b/experimental/aitools/cmd/init_template/agent.go new file mode 100644 index 0000000000..e7c90f5cf0 --- /dev/null +++ b/experimental/aitools/cmd/init_template/agent.go @@ -0,0 +1,84 @@ +package init_template + +import ( + "errors" + "os" + + "github.com/databricks/cli/cmd/root" + "github.com/spf13/cobra" +) + +const ( + agentTemplateRepo = "https://github.com/databricks/cli" + agentTemplateDir = "experimental/aitools/templates/agent-openai-agents-sdk" + agentBranch = "main" + agentPathEnvVar = "DATABRICKS_AGENT_TEMPLATE_PATH" +) + +// newAgentCmd creates the agent subcommand for init-template. +func newAgentCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "agent", + Short: "Initialize an OpenAI Agents SDK project", + Args: cobra.NoArgs, + Long: `Initialize an OpenAI Agents SDK project for building conversational agents. + +This creates a project with: +- OpenAI Agents SDK integration for building conversational agents +- Built-in chat UI and API endpoint for invoking the agent +- MLflow tracing and evaluation setup +- Access to Databricks built-in tools (code interpreter, etc.) +- Example agent implementation with MCP server support + +Examples: + experimental aitools tools init-template agent --name my-agent + experimental aitools tools init-template agent --name my-agent --output-dir ./projects + +Environment variables: + DATABRICKS_AGENT_TEMPLATE_PATH Override template source with local path (for development) + +After initialization: + cd + ./scripts/quickstart.sh # Set up environment and start server + uv run start-app # Start agent server at http://localhost:8000 +`, + } + + var name string + var outputDir string + + cmd.Flags().StringVar(&name, "name", "", "Project name (required)") + cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the initialized template to") + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if name == "" { + return errors.New("--name is required. Example: init-template agent --name my-agent") + } + + configMap := map[string]any{ + "project_name": name, + } + + // Resolve template source: env var override or default remote + templatePathOrUrl := os.Getenv(agentPathEnvVar) + templateDir := "" + branch := "" + + if templatePathOrUrl == "" { + templatePathOrUrl = agentTemplateRepo + templateDir = agentTemplateDir + branch = agentBranch + } + + return MaterializeTemplate(ctx, TemplateConfig{ + TemplatePath: templatePathOrUrl, + TemplateName: "agent-openai-agents-sdk", + TemplateDir: templateDir, + Branch: branch, + }, configMap, name, outputDir) + } + return cmd +} diff --git a/experimental/aitools/cmd/init_template/init_template.go b/experimental/aitools/cmd/init_template/init_template.go index 3e6adc2228..b8a4565852 100644 --- a/experimental/aitools/cmd/init_template/init_template.go +++ b/experimental/aitools/cmd/init_template/init_template.go @@ -13,11 +13,13 @@ func NewInitTemplateCommand() *cobra.Command { Subcommands: app Initialize a Databricks App using the appkit template + agent Initialize an OpenAI Agents SDK project job Initialize a job project using the default-python template pipeline Initialize a Lakeflow pipeline project empty Initialize an empty bundle for custom resources (dashboards, alerts, etc.)`, } cmd.AddCommand(newAppCmd()) + cmd.AddCommand(newAgentCmd()) cmd.AddCommand(newJobCmd()) cmd.AddCommand(newPipelineCmd()) cmd.AddCommand(newEmptyCmd()) diff --git a/experimental/aitools/lib/detector/bundle_detector.go b/experimental/aitools/lib/detector/bundle_detector.go index 1b05600446..43313b5fb7 100644 --- a/experimental/aitools/lib/detector/bundle_detector.go +++ b/experimental/aitools/lib/detector/bundle_detector.go @@ -43,18 +43,24 @@ func (d *BundleDetector) Detect(ctx context.Context, workDir string, detected *D // Detect all resource types present in the bundle hasApps := false + hasExperiments := false for _, group := range b.Config.Resources.AllResources() { if len(group.Resources) > 0 { detected.TargetTypes = append(detected.TargetTypes, group.Description.PluralName) if group.Description.PluralName == "apps" { hasApps = true } + if group.Description.PluralName == "experiments" { + hasExperiments = true + } } } - // Determine if this is an app-only project (only app resources, nothing else). + // Determine if this is an app-only project. This includes: + // - Only app resources + // - Apps + experiments (experiments are typically for agent tracing) // App-only projects get focused app guidance; others get general bundle guidance. - isAppOnly := hasApps && len(detected.TargetTypes) == 1 + isAppOnly := hasApps && (len(detected.TargetTypes) == 1 || (len(detected.TargetTypes) == 2 && hasExperiments)) detected.IsAppOnly = isAppOnly diff --git a/experimental/aitools/lib/prompts/flow.tmpl b/experimental/aitools/lib/prompts/flow.tmpl index 3a71c91e02..1c21dfccc7 100644 --- a/experimental/aitools/lib/prompts/flow.tmpl +++ b/experimental/aitools/lib/prompts/flow.tmpl @@ -33,6 +33,10 @@ The init-template commands create agent-friendly projects with AGENTS.md/CLAUDE. For apps: invoke_databricks_cli 'experimental aitools tools init-template app --name my-app --description "My app description"' +For agents (Conversational AI with OpenAI Agents SDK): +invoke_databricks_cli 'experimental aitools tools init-template agent --name my_agent' +Note: Creates Python agent with OpenAI Agents SDK, Claude model, MLflow tracing, and MCP integration + For jobs (Python notebooks with wheel package): invoke_databricks_cli 'experimental aitools tools init-template job --name my_job' invoke_databricks_cli 'experimental aitools tools init-template job --name my_job --catalog my_catalog' @@ -50,7 +54,7 @@ Note: Use this for resources OTHER than apps, jobs, or pipelines Notes: - App name must be ≤26 characters (dev- prefix adds 4 chars, max total 30) -- Job/pipeline/project names: letters, numbers, underscores only +- Agent/job/pipeline/project names: letters, numbers, underscores only - --catalog defaults to workspace default catalog{{if .DefaultCatalog}} (currently '{{.DefaultCatalog}}'){{end}} ### Custom SQL Queries diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/databricks_template_schema.json b/experimental/aitools/templates/agent-openai-agents-sdk/databricks_template_schema.json new file mode 100644 index 0000000000..0180650cc3 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/databricks_template_schema.json @@ -0,0 +1,18 @@ +{ + "properties": { + "project_name": { + "type": "string", + "description": "Project Name", + "order": 1, + "pattern": "^[A-Za-z0-9_-]+$", + "pattern_match_failure_message": "Name must consist of letters, numbers, underscores, or hyphens." + }, + "mlflow_experiment_name": { + "type": "string", + "description": "MLflow Experiment Name (Optional - will be created if not exists)", + "default": "", + "order": 2 + } + }, + "success_message": "\nYour new agent project has been created in the '{{.project_name}}' directory!\n\nNext steps:\n1. cd {{.project_name}}\n2. Run ./scripts/quickstart.sh to set up your environment\n3. Run 'uv run start-app' to start the agent server\n" +} diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/.env.example b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/.env.example new file mode 100644 index 0000000000..d6b5a70939 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/.env.example @@ -0,0 +1,18 @@ +# Make a copy of this to set environment variables for local development +# cp .env.example .env.local + +# TODO: Fill in auth related env vars +DATABRICKS_CONFIG_PROFILE=DEFAULT +# DATABRICKS_HOST=https://.databricks.com +# DATABRICKS_TOKEN=dapi.... + +# TODO: Update with the MLflow experiment you want to log traces and models to +MLFLOW_EXPERIMENT_ID= + +# TODO: Update the route to query agent if you used a different port to deploy your agent +API_PROXY=http://localhost:8000/invocations + +CHAT_APP_PORT=3000 +CHAT_PROXY_TIMEOUT_SECONDS=300 +MLFLOW_TRACKING_URI="databricks" +MLFLOW_REGISTRY_URI="databricks-uc" diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/.gitignore b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/.gitignore new file mode 100644 index 0000000000..63c3b98c9c --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/.gitignore @@ -0,0 +1,208 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +databricks.yml + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# VS Code +.vscode/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python + +# Created by https://www.toptal.com/developers/gitignore/api/react +# Edit at https://www.toptal.com/developers/gitignore?templates=react + +### react ### +.DS_* +*.log +logs +**/*.backup.* +**/*.back.* + +node_modules +bower_components + +*.sublime* + +psd +thumb +sketch + +# End of https://www.toptal.com/developers/gitignore/api/react + +**/uv.lock +**/mlruns/ +**/.vite/ +**/.databricks +**/.claude +**/.env.local diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/AGENTS.md b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/AGENTS.md new file mode 100644 index 0000000000..df30fb33ab --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/AGENTS.md @@ -0,0 +1,591 @@ +# Agent OpenAI Agents SDK Development Guide + +## For AI Agents: MANDATORY First Action + +**BEFORE any other action, call `mcp__databricks-mcp__databricks_discover` with this project's path.** + +This is required because the discover tool: +- Detects project state (new, deployed, running) and provides relevant guidance +- Shows available CLI commands and recommended next steps +- Configures workspace authentication + +Without this context, you cannot effectively help with this project. + +If the tool is unavailable, the user needs to set up the Databricks CLI MCP server: +1. Install CLI: https://docs.databricks.com/dev-tools/cli/install +2. Run: `databricks experimental aitools install` + +--- + +## Running the App + +**Prerequisites:** uv, nvm (Node 20), Databricks CLI + +**Quick Start:** + +```bash +# Initialize git repository (recommended for version control) +git init + +# First-time setup (auth, MLflow experiment, env) +./scripts/quickstart.sh + +# Or run non-interactively with a profile +./scripts/quickstart.sh --profile DEFAULT + +# Or with a host URL for initial setup +./scripts/quickstart.sh --host https://your-workspace.cloud.databricks.com + +# Start app at http://localhost:8000 +uv run start-app +``` + +**Note:** The template does not automatically initialize a git repository. Run `git init` if you want version control for your project. + +**Quickstart Options:** +- `--profile NAME`: Use specified Databricks profile (non-interactive) +- `--host URL`: Databricks workspace URL (for initial setup) +- `-h, --help`: Show help message + +**Advanced Server Options:** + +```bash +uv run start-server --reload # Hot-reload on code changes during development +uv run start-server --port 8001 +uv run start-server --workers 4 +``` + +**Test API:** + +```bash +# Streaming request +curl -X POST http://localhost:8000/invocations \ + -H "Content-Type: application/json" \ + -d '{ "input": [{ "role": "user", "content": "hi" }], "stream": true }' + +# Non-streaming request +curl -X POST http://localhost:8000/invocations \ + -H "Content-Type: application/json" \ + -d '{ "input": [{ "role": "user", "content": "hi" }] }' +``` + +## Testing the Agent + +**Run evaluation:** + +```bash +uv run agent-evaluate # Uses MLflow scorers (RelevanceToQuery, Safety) +``` + +**Run unit tests:** + +```bash +pytest [path] # Standard pytest execution +``` + +--- + +## Modifying the Agent + +Anytime the user wants to modify the agent, look through the following resources to help them accomplish their goal: + +If the user wants to convert their agent input and output format into MLflow's Responses API-compatible interface, refer to https://mlflow.org/docs/latest/genai/serving/responses-agent/. + +1. Look through existing databricks-openai APIs to see if they can use one of these to accomplish their goal. +2. Look through the folders in https://github.com/bbqiu/agent-on-app-prototype to see if there's an existing example similar to what they're looking to do. +3. Reference the documentation available under https://docs.databricks.com/aws/en/generative-ai/agent-framework/ and its subpages. +4. For adding tools and capabilities, refer to: https://docs.databricks.com/aws/en/generative-ai/agent-framework/agent-tool +5. Reference the OpenAI Agents SDK documentation: https://platform.openai.com/docs/guides/agents-sdk + + +### Discovering Available Tools and Data Sources + +When building your agent, you can discover what tools, MCP servers, and data sources are available in your workspace. + +**Run the discovery script:** + +```bash +# Discover all available resources +uv run discover-tools + +# Limit to a specific catalog +uv run discover-tools --catalog my_catalog + +# Limit to a specific schema +uv run discover-tools --catalog my_catalog --schema my_schema + +# Output as JSON +uv run discover-tools --format json --output tools.json + +# Save markdown report +uv run discover-tools --output tools.md +``` + +**What gets discovered:** + +1. **Unity Catalog Functions** - SQL UDFs that can be used as agent tools via MCP servers + - Example: `catalog.schema.get_customer_info(customer_id STRING)` + - Can be accessed via the UC functions MCP server: `{workspace_host}/api/2.0/mcp/functions/{catalog}/{schema}` (all functions in a schema) or `{workspace_host}/api/2.0/mcp/functions/{catalog}/{schema}/{function_name}` (to include a single function as a tool) + +2. **Unity Catalog Tables** - Structured data that agents can query + - Can be queried via UC SQL functions + +3. **Vector Search Indexes** - For RAG applications with unstructured data + - Can be accessed via the vector search MCP server at `{workspace_host}/api/2.0/mcp/vector-search/{catalog}/{schema}` or + `{workspace_host}/api/2.0/mcp/vector-search/{catalog}/{schema}/{index_name}` + +4. **Genie Spaces** - Conversational data access + - Natural language interface to data + - Can be accessed via Genie MCP server at `{workspace_host}/api/2.0/mcp/genie/{space_id}` + +5. **Custom MCP Servers** - Your own MCP servers deployed as Databricks Apps + - MCP servers running on Databricks (app names starting with `mcp-`) + - Access via app URL: `{app_url}/mcp` + - Example: Deploy your custom MCP server as an app named `mcp-my-tools` + +6. **External MCP Servers** - Third-party MCP servers via Unity Catalog connections + - External MCP services configured as UC connections with `is_mcp_connection=true` + - Provides secure access to external APIs and services through UC governance + - Connection credentials managed by Unity Catalog + - Use the Databricks-provided proxy URL to connect to the server: `{workspace_host}/api/2.0/mcp/external/{connection_name}` + +` + +**Using discovered tools in your agent:** + +After discovering tools, configure your agent to use them (e.g. for UC functions): + +```python +async def init_mcp_server(): + return McpServer( + url=f"{host}/api/2.0/mcp/functions/{catalog}/{schema}", + name="your custom tools mcp server", + ) +``` + +See the [MCP documentation](https://docs.databricks.com/aws/en/generative-ai/mcp/) for more details. + +--- + +**Main file to modify:** `agent_server/agent.py` + +--- + +## databricks-openai SDK Overview + +**SDK Location:** `https://github.com/databricks/databricks-ai-bridge/tree/main/integrations/openai` + +**Development Workflow:** + +```bash +uv add databricks-openai +``` + +Before making any changes, ensure that the APIs actually exist in the SDK. If something is missing from the documentation here, feel free to look in the venv's `site-packages` directory for the `databricks_openai` package. If it's not installed, run `uv sync` in this folder to create the .venv and install the package. + +--- + +### Key Components + +#### 1. MCP Servers - Tool Integration + +Connect to MCP (Model Context Protocol) servers to get tools for your agent. + +**Basic MCP Server:** + +```python +from databricks_openai.agents import McpServer + +async def init_mcp_server(): + return McpServer( + url=f"{host}/api/2.0/mcp/functions/system/ai", + name="system.ai uc function mcp server", + ) + +# Use in agent +agent = Agent( + name="code execution agent", + instructions="You are a code execution agent.", + model="databricks-claude-3-7-sonnet", + mcp_servers=[mcp_server], +) +``` + +#### 2. Sync and Async Databricks OpenAI Clients + +Query Databricks-hosted, OpenAI-compatible serving endpoints: + +```python +from databricks_openai import AsyncDatabricksOpenAI, DatabricksOpenAI +from agents import set_default_openai_api, set_default_openai_client + +# Async client (recommended for agent servers) +set_default_openai_client(AsyncDatabricksOpenAI()) +set_default_openai_api("chat_completions") + +# Sync client +client = DatabricksOpenAI() +``` + +**Note:** This works for all Databricks foundation models except GPT-OSS, which uses a slightly different API. + +--- + +## Agent Development Patterns + +### Creating and Running Agents + +```python +from agents import Agent, Runner +from databricks_openai.agents import McpServer + +# Create agent with MCP servers +agent = Agent( + name="code execution agent", + instructions="You are a code execution agent.", + model="databricks-claude-3-7-sonnet", + mcp_servers=[mcp_server], +) + +# Run agent (non-streaming) +messages = [{"role": "user", "content": "hi"}] +result = await Runner.run(agent, messages) + +# Run agent (streaming) +result = Runner.run_streamed(agent, input=messages) +async for event in result.stream_events(): + # Process stream events + pass +``` + +### MLflow Tracing with Decorators + +The template uses MLflow's `@invoke()` and `@stream()` decorators for automatic tracing: + +```python +from mlflow.genai.agent_server import invoke, stream +from mlflow.types.responses import ( + ResponsesAgentRequest, + ResponsesAgentResponse, + ResponsesAgentStreamEvent, +) + +@invoke() +async def invoke(request: ResponsesAgentRequest) -> ResponsesAgentResponse: + # Agent logic here + pass + +@stream() +async def stream(request: dict) -> AsyncGenerator[ResponsesAgentStreamEvent, None]: + # Streaming agent logic here + pass +``` + +### Adding Custom Tracing + +Beyond built-in tracing, you can add additional instrumentation: + +```python +import mlflow + +mlflow.openai.autolog() # Auto-trace OpenAI calls + +# For more granular tracing, see: +# https://docs.databricks.com/aws/en/mlflow3/genai/tracing/app-instrumentation/ +``` + +--- + +## Authentication Setup + +**Option 1: OAuth (Recommended)** + +```bash +databricks auth login +``` + +Set in `.env.local`: + +```bash +DATABRICKS_CONFIG_PROFILE=DEFAULT +``` + +**Option 2: Personal Access Token** + +Set in `.env.local`: + +```bash +DATABRICKS_HOST="https://host.databricks.com" +DATABRICKS_TOKEN="dapi_token" +``` + +--- + +## MLflow Experiment Setup + +Create and link an MLflow experiment: + +```bash +DATABRICKS_USERNAME=$(databricks current-user me | jq -r .userName) +databricks experiments create-experiment /Users/$DATABRICKS_USERNAME/agents-on-apps +``` + +Add the experiment ID to `.env.local`: + +```bash +MLFLOW_EXPERIMENT_ID= +``` + +--- + +## Key Files + +| File | Purpose | +| -------------------------------- | --------------------------------------------- | +| `agent_server/agent.py` | Agent logic, model, instructions, MCP servers | +| `agent_server/start_server.py` | FastAPI server + MLflow setup | +| `agent_server/evaluate_agent.py` | Agent evaluation with MLflow scorers | +| `agent_server/utils.py` | Databricks auth helpers, stream processing | +| `scripts/start_app.py` | Manages backend+frontend startup | + +--- + +## Deploying to Databricks Apps + +**Note:** This deployment approach requires the upcoming Databricks CLI release with experiment resource support. The examples use `databricks` as the CLI command, which will work once the feature is released. + +### Prerequisites + +1. **Databricks CLI** with OAuth authentication configured +2. **Workspace Configuration** in `databricks.yml` + +### Initial Setup + +**1. Configure OAuth Authentication:** + +```bash +# Login to your workspace with OAuth (required for apps) +databricks auth login --host https://your-workspace.cloud.databricks.com --profile + +# Verify authentication +databricks current-user me --profile +``` + +**2. Configure `databricks.yml`:** + +By default, `databricks.yml` defines a new MLflow experiment and app. You can specify an experiment ID +manually if you want to use an existing experiment: + +```yaml +bundle: + name: my-agent + +resources: + # MLflow experiment for agent tracing - bundle will create this + experiments: + agent-experiment: + name: /Users/${workspace.current_user.userName}/${bundle.name}-${bundle.target} + + # Databricks App + apps: + my-agent: + name: "${bundle.target}-my-agent" + description: "OpenAI Agents SDK agent application" + source_code_path: ./ + + # Grant app access to the experiment + resources: + - name: 'experiment' + experiment: + experiment_id: "${resources.experiments.agent-experiment.id}" + permission: 'CAN_MANAGE' + +targets: + dev: + mode: development + default: true + workspace: + host: https://your-workspace.cloud.databricks.com +``` + +**3. Configure `app.yaml`:** + +Reference the experiment resource: + +```yaml +env: + - name: MLFLOW_EXPERIMENT_ID + valueFrom: "experiment" + - name: MLFLOW_TRACKING_URI + value: "databricks" + - name: MLFLOW_REGISTRY_URI + value: "databricks-uc" +``` + +**4. Configure `.env.local` for Local Development:** + +```bash +DATABRICKS_CONFIG_PROFILE= +API_PROXY=http://localhost:8000/invocations +CHAT_APP_PORT=3000 +CHAT_PROXY_TIMEOUT_SECONDS=300 +MLFLOW_TRACKING_URI="databricks" +MLFLOW_REGISTRY_URI="databricks-uc" +``` + +### Deployment Workflow + +**Deploy the bundle:** + +```bash +# Deploy bundle - creates experiment and grants app permissions automatically +DATABRICKS_CONFIG_PROFILE= databricks bundle deploy --target dev + +# Deploy the source code +DATABRICKS_USERNAME=$(databricks current-user me --profile | jq -r .userName) +DATABRICKS_CONFIG_PROFILE= databricks apps deploy \ + --source-code-path /Workspace/Users/$DATABRICKS_USERNAME/.bundle//dev/files +``` + +**Get app URL:** + +```bash +DATABRICKS_CONFIG_PROFILE= databricks apps get --output json | jq -r '.url' +``` + +**What happens during deployment:** +- ✅ Bundle creates the MLflow experiment at `/Users//my-agent-dev` +- ✅ App is created with automatic access to the experiment +- ✅ App service principal receives CAN_MANAGE permission on the experiment +- ✅ No manual permission configuration required + +**Note:** You may see warnings about "unknown field" during deployment - these can be ignored and will be resolved in the CLI release. + +### Granting your app service principal permission to use resources + +Beyond MLflow experiments, you can grant apps access to other workspace resources by adding them to the `resources` array in databricks.yml; the app service principal will be granted permission to the resource automatically: + +**Supported resource types:** +- MLflow experiments (experiment tracking) +- SQL warehouses (data querying) +- Model serving endpoints (model inference) +- Unity Catalog volumes (file storage) +- Secrets (secure credential storage) +- Vector search indexes (semantic search) + +See: https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources + +### Testing the Deployed App + +**1. Generate OAuth Token:** + +```bash +# Generate a fresh OAuth token (expires after 1 hour) +DATABRICKS_CONFIG_PROFILE= databricks auth token --host https://your-workspace.cloud.databricks.com +``` + +Save the `access_token` from the JSON output. + +**2. Send Test Request:** + +```bash +# Non-streaming request +curl -X POST /invocations \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"input": [{"role": "user", "content": "hi"}]}' + +# Streaming request +curl -X POST /invocations \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"input": [{"role": "user", "content": "hi"}], "stream": true}' +``` + +**Note:** OAuth tokens are required for Databricks Apps - PATs are not supported. + +### Debugging Deployed Apps + +**View app logs:** + +```bash +# View recent logs (last 200 lines) +DATABRICKS_CONFIG_PROFILE= databricks apps logs + +# Follow logs in real-time +DATABRICKS_CONFIG_PROFILE= databricks apps logs --follow + +# Search logs for specific terms +DATABRICKS_CONFIG_PROFILE= databricks apps logs --search "ERROR" + +# Filter by source (APP or SYSTEM) +DATABRICKS_CONFIG_PROFILE= databricks apps logs --source APP +``` + +**Check app status:** + +```bash +# Get detailed app status +DATABRICKS_CONFIG_PROFILE= databricks apps get --output json | jq '{app_status, compute_status}' + +# List all deployments +DATABRICKS_CONFIG_PROFILE= databricks apps list-deployments +``` + +**Common issues:** + +1. **502 Bad Gateway** - App container not running or startup failed + - Check logs: `databricks apps logs ` + - Verify app status: `databricks apps get ` + +2. **Authentication Failed** - OAuth token expired or invalid + - Regenerate token: `databricks auth token` + - Verify profile is configured with OAuth: `databricks auth login` + +3. **App crashed on startup** - Check logs for errors + - View recent logs: `databricks apps logs --tail-lines 100` + - Look for Python errors, missing dependencies, or configuration issues + +### Development Loop + +For rapid iteration: + +1. **Make code changes locally** +2. **Test locally:** `uv run start-app` +3. **Query locally-deployed app** + curl -X POST http://localhost:8000/invocations -d '{"input": [{"role": "user", "content": "test"}]}' +3. **Deploy to Databricks:** + ```bash + DATABRICKS_CONFIG_PROFILE= databricks bundle deploy --target dev + DATABRICKS_CONFIG_PROFILE= databricks apps deploy --source-code-path /Workspace/Users/$DATABRICKS_USERNAME/.bundle//dev/files + ``` +4. **Test deployed app:** + ```bash + TOKEN=$(DATABRICKS_CONFIG_PROFILE= databricks auth token --host | jq -r '.access_token') + curl -X POST /invocations -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d '{"input": [{"role": "user", "content": "test"}]}' + ``` +5. **Check logs:** `databricks apps logs --follow` +6. **Iterate** + +--- + +## Agent Framework Capabilities + +Reference: https://docs.databricks.com/aws/en/generative-ai/agent-framework/ + +### Tool Types + +1. **Unity Catalog Function Tools** - SQL UDFs managed in UC with built-in governance +2. **Agent Code Tools** - Defined directly in agent code for REST APIs and low-latency operations +3. **MCP Tools** - Interoperable tools via Model Context Protocol (Databricks-managed, external, or self-hosted) + +### Built-in Tools + +- **system.ai.python_exec** - Execute Python code dynamically within agent queries (code interpreter) + +### Common Patterns + +- **Structured data retrieval** - Query SQL tables/databases +- **Unstructured data retrieval** - Document search and RAG via Vector Search +- **Code interpreter** - Python execution for analysis via system.ai.python_exec +- **External connections** - Integrate services like Slack via HTTP connections diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/CLAUDE.md b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/CLAUDE.md new file mode 100644 index 0000000000..4202b8f067 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/CLAUDE.md @@ -0,0 +1,3 @@ +@AGENTS.md + + diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/README.md b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/README.md new file mode 100644 index 0000000000..fbff317663 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/README.md @@ -0,0 +1,237 @@ +# Responses API Agent + +This template defines a conversational agent app. The app comes with a built-in chat UI, but also exposes an API endpoint for invoking the agent so that you can serve your UI elsewhere (e.g. on your website or in a mobile app). + +The agent in this template implements the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) interface. It has access to a single tool; the [built-in code interpreter tool](https://docs.databricks.com/aws/en/generative-ai/agent-framework/code-interpreter-tools#built-in-python-executor-tool) (`system.ai.python_exec`) on Databricks. You can customize agent code and test it via the API or UI. + +The agent input and output format are defined by MLflow's ResponsesAgent interface, which closely follows the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) interface. See [the MLflow docs](https://mlflow.org/docs/latest/genai/flavors/responses-agent-intro/) for input and output formats for streaming and non-streaming requests, tracing requirements, and other agent authoring details. + +## Quick start + +Run the `./scripts/quickstart.sh` script to quickly set up your local environment and start the agent server. At any step, if there are issues, refer to the manual local development loop setup below. + +This script will: + +1. Verify uv, nvm, and Databricks CLI installations +2. Configure Databricks authentication +3. Configure agent tracing, by creating and linking an MLflow experiment to your app +4. Start the agent server and chat app + +```bash +./scripts/quickstart.sh +``` + +After the setup is complete, you can start the agent server and the chat app locally with: + +```bash +uv run start-app +``` + +This will start the agent server and the chat app at http://localhost:8000. + +**Next steps**: see [modifying your agent](#modifying-your-agent) to customize and iterate on the agent code. + +## Manual local development loop setup + +1. **Set up your local environment** + Install `uv` (python package manager), `nvm` (node version manager), and the Databricks CLI: + + - [`uv` installation docs](https://docs.astral.sh/uv/getting-started/installation/) + - [`nvm` installation](https://github.com/nvm-sh/nvm?tab=readme-ov-file#installing-and-updating) + - Run the following to use Node 20 LTS: + ```bash + nvm use 20 + ``` + - [`databricks CLI` installation](https://docs.databricks.com/aws/en/dev-tools/cli/install) + +2. **Set up local authentication to Databricks** + + In order to access Databricks resources from your local machine while developing your agent, you need to authenticate with Databricks. Choose one of the following options: + + **Option 1: OAuth via Databricks CLI (Recommended)** + + Authenticate with Databricks using the CLI. See the [CLI OAuth documentation](https://docs.databricks.com/aws/en/dev-tools/cli/authentication#oauth-user-to-machine-u2m-authentication). + + ```bash + databricks auth login + ``` + + Set the `DATABRICKS_CONFIG_PROFILE` environment variable in your .env.local file to the profile you used to authenticate: + + ```bash + DATABRICKS_CONFIG_PROFILE="DEFAULT" # change to the profile name you chose + ``` + + **Option 2: Personal Access Token (PAT)** + + See the [PAT documentation](https://docs.databricks.com/aws/en/dev-tools/auth/pat#databricks-personal-access-tokens-for-workspace-users). + + ```bash + # Add these to your .env.local file + DATABRICKS_HOST="https://host.databricks.com" + DATABRICKS_TOKEN="dapi_token" + ``` + + See the [Databricks SDK authentication docs](https://docs.databricks.com/aws/en/dev-tools/sdk-python#authenticate-the-databricks-sdk-for-python-with-your-databricks-account-or-workspace). + +3. **Create and link an MLflow experiment to your app** + + Create an MLflow experiment to enable tracing and version tracking. This is automatically done by the `./scripts/quickstart.sh` script. + + Create the MLflow experiment via the CLI: + + ```bash + DATABRICKS_USERNAME=$(databricks current-user me | jq -r .userName) + databricks experiments create-experiment /Users/$DATABRICKS_USERNAME/agents-on-apps + ``` + + Make a copy of `.env.example` to `.env.local` and update the `MLFLOW_EXPERIMENT_ID` in your `.env.local` file with the experiment ID you created. The `.env.local` file will be automatically loaded when starting the server. + + ```bash + cp .env.example .env.local + # Edit .env.local and fill in your experiment ID + ``` + + See the [MLflow experiments documentation](https://docs.databricks.com/aws/en/mlflow/experiments#create-experiment-from-the-workspace). + +4. **Test your agent locally** + + Start up the agent server and chat UI locally: + + ```bash + uv run start-app + ``` + + Query your agent via the UI (http://localhost:8000) or REST API: + + **Advanced server options:** + + ```bash + uv run start-server --reload # hot-reload the server on code changes + uv run start-server --port 8001 # change the port the server listens on + uv run start-server --workers 4 # run the server with multiple workers + ``` + + - Example streaming request: + ```bash + curl -X POST http://localhost:8000/invocations \ + -H "Content-Type: application/json" \ + -d '{ "input": [{ "role": "user", "content": "hi" }], "stream": true }' + ``` + - Example non-streaming request: + ```bash + curl -X POST http://localhost:8000/invocations \ + -H "Content-Type: application/json" \ + -d '{ "input": [{ "role": "user", "content": "hi" }] }' + ``` + +## Modifying your agent + +See the [OpenAI Agents SDK documentation](https://platform.openai.com/docs/guides/agents-sdk) for more information on how to edit your own agent. + +Required files for hosting with MLflow `AgentServer`: + +- `agent.py`: Contains your agent logic. Modify this file to create your custom agent. For example, you can [add agent tools](https://docs.databricks.com/aws/en/generative-ai/agent-framework/agent-tool) to give your agent additional capabilities +- `start_server.py`: Initializes and runs the MLflow `AgentServer` with agent_type="ResponsesAgent". You don't have to modify this file for most common use cases, but can add additional server routes (e.g. a `/metrics` endpoint) here + +**Common customization questions:** + +**Q: Can I add additional files or folders to my agent?** +Yes. Add additional files or folders as needed. Ensure the script within `pyproject.toml` runs the correct script that starts the server and sets up MLflow tracing. + +**Q: How do I add dependencies to my agent?** +Run `uv add ` (e.g., `uv add "mlflow-skinny[databricks]"`). See the [python pyproject.toml guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#dependencies-and-requirements). + +**Q: Can I add custom tracing beyond the built-in tracing?** +Yes. This template uses MLflow's agent server, which comes with automatic tracing for agent logic decorated with `@invoke()` and `@stream()`. It also uses [MLflow autologging APIs](https://mlflow.org/docs/latest/genai/tracing/#one-line-auto-tracing-integrations) to capture traces from LLM invocations. However, you can add additional instrumentation to capture more granular trace information when your agent runs. See the [MLflow tracing documentation](https://docs.databricks.com/aws/en/mlflow3/genai/tracing/app-instrumentation/). + +**Q: How can I extend this example with additional tools and capabilities?** +This template can be extended by integrating additional MCP servers, Vector Search Indexes, UC Functions, and other Databricks tools. See the ["Agent Framework Tools Documentation"](https://docs.databricks.com/aws/en/generative-ai/agent-framework/agent-tool). + +## Evaluating your agent + +Evaluate your agent by calling the invoke function you defined for the agent locally. + +- Update your `evaluate_agent.py` file with the preferred evaluation dataset and scorers. + +Run the evaluation using the evaluation script: + +```bash +uv run agent-evaluate +``` + +After it completes, open the MLflow UI link for your experiment to inspect results. + +## Deploying to Databricks Apps + +0. **Create a Databricks App**: + Ensure you have the [Databricks CLI](https://docs.databricks.com/aws/en/dev-tools/cli/tutorial) installed and configured. + + ```bash + databricks apps create agent-openai-agents-sdk + ``` + +1. **Set up authentication to Databricks resources** + + For this example, you need to add an MLflow Experiment as a resource to your app. Grant the App's Service Principal (SP) permission to edit the experiment by clicking `edit` on your app home page. See the [Databricks Apps MLflow experiment documentation](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/mlflow) for more information. + + To grant access to other resources like serving endpoints, genie spaces, UC Functions, and Vector Search Indexes, click `edit` on your app home page to grant the App's SP permission. See the [Databricks Apps resources documentation](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources). + + For resources that are not supported yet, see the [Agent Framework authentication documentation](https://docs.databricks.com/aws/en/generative-ai/agent-framework/deploy-agent#automatic-authentication-passthrough) for the correct permission level to grant to your app SP. + + **On-behalf-of (OBO) User Authentication**: Use `get_user_workspace_client()` from `agent_server.utils` to authenticate as the requesting user instead of the app service principal. See the [OBO authentication documentation](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/auth?language=Streamlit#retrieve-user-authorization-credentials). + +2. **Sync local files to your workspace** + + See the [Databricks Apps deploy documentation](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/deploy?language=Databricks+CLI#deploy-the-app). + + ```bash + DATABRICKS_USERNAME=$(databricks current-user me | jq -r .userName) + databricks sync . "/Users/$DATABRICKS_USERNAME/agent-openai-agents-sdk" + ``` + +3. **Deploy your Databricks App** + + See the [Databricks Apps deploy documentation](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/deploy?language=Databricks+CLI#deploy-the-app). + + ```bash + databricks apps deploy agent-openai-agents-sdk --source-code-path /Workspace/Users/$DATABRICKS_USERNAME/agent-openai-agents-sdk + ``` + +4. **Query your agent hosted on Databricks Apps** + + Databricks Apps are _only_ queryable via OAuth token. You cannot use a PAT to query your agent. Generate an [OAuth token with your credentials using the Databricks CLI](https://docs.databricks.com/aws/en/dev-tools/cli/authentication#u2m-auth): + + ```bash + databricks auth login --host + databricks auth token + ``` + + Send a request to the `/invocations` endpoint: + + - Example streaming request: + + ```bash + curl -X POST /invocations \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ "input": [{ "role": "user", "content": "hi" }], "stream": true }' + ``` + + - Example non-streaming request: + + ```bash + curl -X POST /invocations \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ "input": [{ "role": "user", "content": "hi" }] }' + ``` + +For future updates to the agent, sync and redeploy your agent. + +### FAQ + +- For a streaming response, I see a 200 OK in the logs, but an error in the actual stream. What's going on? + - This is expected behavior. The initial 200 OK confirms stream setup; streaming errors don't affect this status. +- When querying my agent, I get a 302 error. What's going on? + - Use an OAuth token. PATs are not supported for querying agents. diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/__init__.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/agent.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/agent.py new file mode 100644 index 0000000000..2bd44f19a2 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/agent.py @@ -0,0 +1,65 @@ +from typing import AsyncGenerator + +import mlflow +from agents import Agent, Runner, set_default_openai_api, set_default_openai_client +from agents.tracing import set_trace_processors +from databricks_openai import AsyncDatabricksOpenAI +from databricks_openai.agents import McpServer +from mlflow.genai.agent_server import invoke, stream +from mlflow.types.responses import ( + ResponsesAgentRequest, + ResponsesAgentResponse, + ResponsesAgentStreamEvent, +) + +from agent_server.utils import ( + get_databricks_host_from_env, + get_user_workspace_client, + process_agent_stream_events, +) + +# NOTE: this will work for all databricks models OTHER than GPT-OSS, which uses a slightly different API +set_default_openai_client(AsyncDatabricksOpenAI()) +set_default_openai_api("chat_completions") +set_trace_processors([]) # only use mlflow for trace processing +mlflow.openai.autolog() + + +async def init_mcp_server(): + return McpServer( + url=f"{get_databricks_host_from_env()}/api/2.0/mcp/functions/system/ai", + name="system.ai uc function mcp server", + ) + + +def create_coding_agent(mcp_server: McpServer) -> Agent: + return Agent( + name="code execution agent", + instructions="You are a code execution agent. You can execute code and return the results.", + model="databricks-claude-3-7-sonnet", + mcp_servers=[mcp_server], + ) + + +@invoke() +async def invoke(request: ResponsesAgentRequest) -> ResponsesAgentResponse: + # Optionally use the user's workspace client for on-behalf-of authentication + # user_workspace_client = get_user_workspace_client() + async with await init_mcp_server() as mcp_server: + agent = create_coding_agent(mcp_server) + messages = [i.model_dump() for i in request.input] + result = await Runner.run(agent, messages) + return ResponsesAgentResponse(output=[item.to_input_item() for item in result.new_items]) + + +@stream() +async def stream(request: dict) -> AsyncGenerator[ResponsesAgentStreamEvent, None]: + # Optionally use the user's workspace client for on-behalf-of authentication + # user_workspace_client = get_user_workspace_client() + async with await init_mcp_server() as mcp_server: + agent = create_coding_agent(mcp_server) + messages = [i.model_dump() for i in request.input] + result = Runner.run_streamed(agent, input=messages) + + async for event in process_agent_stream_events(result.stream_events()): + yield event diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/evaluate_agent.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/evaluate_agent.py new file mode 100644 index 0000000000..1035b2938e --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/evaluate_agent.py @@ -0,0 +1,53 @@ +import asyncio + +import mlflow +from dotenv import load_dotenv +from mlflow.genai.agent_server import get_invoke_function +from mlflow.genai.scorers import RelevanceToQuery, Safety +from mlflow.types.responses import ResponsesAgentRequest, ResponsesAgentResponse + +# Load environment variables from .env.local if it exists +load_dotenv(dotenv_path=".env.local", override=True) + +# Import agent for our @invoke function to be found +from agent_server import agent # noqa: F401 + +# Create your evaluation dataset +# Refer to documentation for evaluations: +# Scorers: https://docs.databricks.com/aws/en/mlflow3/genai/eval-monitor/concepts/scorers +# Predefined LLM scorers: https://mlflow.org/docs/latest/genai/eval-monitor/scorers/llm-judge/predefined +# Defining custom scorers: https://docs.databricks.com/aws/en/mlflow3/genai/eval-monitor/custom-scorers +eval_dataset = [ + { + "inputs": { + "request": { + "input": [{"role": "user", "content": "Calculate the 15th Fibonacci number"}] + } + }, + "expected_response": "The 15th Fibonacci number is 610.", + } +] + +# Get the invoke function that was registered via @invoke decorator in your agent +invoke_fn = get_invoke_function() +assert invoke_fn is not None, ( + "No function registered with the `@invoke` decorator found." + "Ensure you have a function decorated with `@invoke()`." +) + +# if invoke function is async, then we need to wrap it in a sync function +if asyncio.iscoroutinefunction(invoke_fn): + + def sync_invoke_fn(request: dict) -> ResponsesAgentResponse: + req = ResponsesAgentRequest(**request) + return asyncio.run(invoke_fn(req)) +else: + sync_invoke_fn = invoke_fn + + +def evaluate(): + mlflow.genai.evaluate( + data=eval_dataset, + predict_fn=sync_invoke_fn, + scorers=[RelevanceToQuery(), Safety()], + ) diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/start_server.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/start_server.py new file mode 100644 index 0000000000..5138884272 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/start_server.py @@ -0,0 +1,17 @@ +from dotenv import load_dotenv +from mlflow.genai.agent_server import AgentServer, setup_mlflow_git_based_version_tracking + +# Load env vars from .env.local before importing the agent for proper auth +load_dotenv(dotenv_path=".env.local", override=True) + +# Need to import the agent to register the functions with the server +import agent_server.agent # noqa: E402 + +agent_server = AgentServer("ResponsesAgent", enable_chat_proxy=True) +# Define the app as a module level variable to enable multiple workers +app = agent_server.app # noqa: F841 +setup_mlflow_git_based_version_tracking() + + +def main(): + agent_server.run(app_import_string="agent_server.start_server:app") diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/utils.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/utils.py new file mode 100644 index 0000000000..03e52e9e31 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/agent_server/utils.py @@ -0,0 +1,44 @@ +import logging +from typing import AsyncGenerator, AsyncIterator, Optional +from uuid import uuid4 + +from agents.result import StreamEvent +from databricks.sdk import WorkspaceClient +from mlflow.genai.agent_server import get_request_headers +from mlflow.types.responses import ResponsesAgentStreamEvent + + +def get_databricks_host_from_env() -> Optional[str]: + try: + w = WorkspaceClient() + return w.config.host + except Exception as e: + logging.exception(f"Error getting databricks host from env: {e}") + return None + + +def get_user_workspace_client() -> WorkspaceClient: + token = get_request_headers().get("x-forwarded-access-token") + return WorkspaceClient(token=token, auth_type="pat") + + +async def process_agent_stream_events( + async_stream: AsyncIterator[StreamEvent], +) -> AsyncGenerator[ResponsesAgentStreamEvent, None]: + curr_item_id = str(uuid4()) + async for event in async_stream: + if event.type == "raw_response_event": + event_data = event.data.model_dump() + if event_data["type"] == "response.output_item.added": + curr_item_id = str(uuid4()) + event_data["item"]["id"] = curr_item_id + elif event_data.get("item") is not None and event_data["item"].get("id") is not None: + event_data["item"]["id"] = curr_item_id + elif event_data.get("item_id") is not None: + event_data["item_id"] = curr_item_id + yield event_data + elif event.type == "run_item_stream_event" and event.item.type == "tool_call_output_item": + yield ResponsesAgentStreamEvent( + type="response.output_item.done", + item=event.item.to_input_item(), + ) diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/app.yaml b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/app.yaml new file mode 100644 index 0000000000..34465373f6 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/app.yaml @@ -0,0 +1,16 @@ +command: ["uv", "run", "start-app"] +# databricks apps listen by default on port 8000 + +env: + - name: MLFLOW_TRACKING_URI + value: "databricks" + - name: MLFLOW_REGISTRY_URI + value: "databricks-uc" + - name: API_PROXY + value: "http://localhost:8000/invocations" + - name: CHAT_APP_PORT + value: "3000" + - name: CHAT_PROXY_TIMEOUT_SECONDS + value: "300" + - name: MLFLOW_EXPERIMENT_ID + valueFrom: "experiment" diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/databricks.yml.tmpl b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/databricks.yml.tmpl new file mode 100644 index 0000000000..6feb50d3a2 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/databricks.yml.tmpl @@ -0,0 +1,37 @@ +bundle: + name: {{.project_name}} + +resources: + # MLflow experiment for agent tracing - automatically created by bundle + experiments: + {{.project_name}}_experiment: + name: {{if .mlflow_experiment_name}}{{.mlflow_experiment_name}}{{else}}/Users/${workspace.current_user.userName}/${bundle.name}-${bundle.target}{{end}} + + apps: + {{.project_name}}: + name: "${bundle.target}-{{.project_name}}" + description: "OpenAI Agents SDK agent application" + source_code_path: ./ + + # Resources which this app has access to + resources: + - name: 'experiment' + experiment: + experiment_id: "${resources.experiments.{{.project_name}}_experiment.id}" + permission: 'CAN_MANAGE' + +targets: + dev: + mode: development + default: true + workspace: + host: {{workspace_host}} + + prod: + mode: production + # workspace: + # host: https://... + resources: + apps: + {{.project_name}}: + name: {{.project_name}} diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/pyproject.toml b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/pyproject.toml new file mode 100644 index 0000000000..acd62d5172 --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/pyproject.toml @@ -0,0 +1,37 @@ +[project] +name = "agent-server" +version = "0.1.0" +description = "MLflow-compatible agent server with FastAPI" +readme = "README.md" +authors = [ + { name = "Agent Developer", email = "developer@example.com" } +] +requires-python = ">=3.11" +dependencies = [ + "fastapi>=0.115.12", + "uvicorn>=0.34.2", + "databricks-openai>=0.8.0", + "mlflow>=3.8.0rc0", + "openai-agents>=0.4.1", + "python-dotenv", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.uv] +prerelease = "allow" + +[dependency-groups] +dev = [ + "hatchling>=1.27.0", + "pytest>=7.0.0", +] + + +[project.scripts] +start-app = "scripts.start_app:main" +start-server = "agent_server.start_server:main" +agent-evaluate = "agent_server.evaluate_agent:evaluate" +discover-tools = "scripts.discover_tools:main" diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/requirements.txt b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/requirements.txt new file mode 100644 index 0000000000..60cc5e6a1a --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/requirements.txt @@ -0,0 +1 @@ +uv diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/__init__.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/discover_tools.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/discover_tools.py new file mode 100755 index 0000000000..2a7d0f365f --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/discover_tools.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +""" +Discover available tools and data sources for Databricks agents. + +This script scans for: +- Unity Catalog functions (potential tools) +- Unity Catalog tables (data sources) +- Vector search indexes (RAG data sources) +- Genie spaces (conversational data access) +- Custom MCP servers (mcp-* packages) +""" + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict, List + +from databricks.sdk import WorkspaceClient + +TOOL_LIST_PAGE_SIZE = 100 + +def run_databricks_cli(args: List[str]) -> str: + """Run databricks CLI command and return output.""" + try: + result = subprocess.run( + ["databricks"] + args, + capture_output=True, + text=True, + check=True, + ) + return result.stdout + except subprocess.CalledProcessError as e: + print(f"Error running databricks CLI: {e.stderr}", file=sys.stderr) + return "" + + +def discover_uc_functions(w: WorkspaceClient, catalog: str = None) -> List[Dict[str, Any]]: + """Discover Unity Catalog functions that could be used as tools.""" + functions = [] + + try: + catalogs = [catalog] if catalog else [c.name for c in w.catalogs.list()] + + for cat in catalogs: + try: + schemas = list(w.schemas.list(catalog_name=cat)) + for schema in schemas: + schema_name = f"{cat}.{schema.name}" + try: + funcs = list(w.functions.list(catalog_name=cat, schema_name=schema.name)) + for func in funcs: + functions.append({ + "type": "uc_function", + "name": func.full_name, + "catalog": cat, + "schema": schema.name, + "function_name": func.name, + "comment": func.comment, + "routine_definition": getattr(func, "routine_definition", None), + }) + except Exception as e: + # Skip schemas we can't access + continue + except Exception as e: + # Skip catalogs we can't access + continue + + except Exception as e: + print(f"Error discovering UC functions: {e}", file=sys.stderr) + + return functions + + +def discover_uc_tables(w: WorkspaceClient, catalog: str = None, schema: str = None) -> List[Dict[str, Any]]: + """Discover Unity Catalog tables that could be queried.""" + tables = [] + + try: + catalogs = [catalog] if catalog else [c.name for c in w.catalogs.list()] + + for cat in catalogs: + if cat in ["__databricks_internal", "system"]: + continue + + try: + schemas = [schema] if schema else [s.name for s in w.schemas.list(catalog_name=cat)] + for sch in schemas: + if sch == "information_schema": + continue + + try: + tbls = list(w.tables.list(catalog_name=cat, schema_name=sch)) + for tbl in tbls: + # Get column info + columns = [] + if hasattr(tbl, "columns") and tbl.columns: + columns = [ + {"name": col.name, "type": col.type_name.value if hasattr(col.type_name, "value") else str(col.type_name)} + for col in tbl.columns + ] + + tables.append({ + "type": "uc_table", + "name": tbl.full_name, + "catalog": cat, + "schema": sch, + "table_name": tbl.name, + "table_type": tbl.table_type.value if tbl.table_type else None, + "comment": tbl.comment, + "columns": columns, + }) + except Exception as e: + # Skip schemas we can't access + continue + except Exception as e: + # Skip catalogs we can't access + continue + + except Exception as e: + print(f"Error discovering UC tables: {e}", file=sys.stderr) + + return tables + + +def discover_vector_search_indexes(w: WorkspaceClient) -> List[Dict[str, Any]]: + """Discover Vector Search indexes for RAG applications.""" + indexes = [] + + try: + # List all vector search endpoints + endpoints = list(w.vector_search_endpoints.list_endpoints()) + + for endpoint in endpoints: + try: + # List indexes for each endpoint + endpoint_indexes = list(w.vector_search_indexes.list_indexes(endpoint_name=endpoint.name)) + for idx in endpoint_indexes: + indexes.append({ + "type": "vector_search_index", + "name": idx.name, + "endpoint": endpoint.name, + "primary_key": idx.primary_key, + "index_type": idx.index_type.value if idx.index_type else None, + "status": idx.status.state.value if idx.status and idx.status.state else None, + }) + except Exception as e: + # Skip endpoints we can't access + continue + + except Exception as e: + print(f"Error discovering vector search indexes: {e}", file=sys.stderr) + + return indexes + + +def discover_genie_spaces(w: WorkspaceClient) -> List[Dict[str, Any]]: + """Discover Genie spaces for conversational data access.""" + spaces = [] + + try: + # Use SDK to list genie spaces + response = w.genie.list_spaces() + genie_spaces = response.spaces if hasattr(response, "spaces") else [] + for space in genie_spaces: + spaces.append({ + "type": "genie_space", + "id": space.space_id, + "name": space.title, + "description": space.description, + }) + except Exception as e: + print(f"Error discovering Genie spaces: {e}", file=sys.stderr) + + return spaces + + +def discover_mcp_servers() -> List[Dict[str, Any]]: + """Discover custom MCP servers (Python packages starting with mcp-).""" + mcp_servers = [] + + try: + # Check if uv is available + result = subprocess.run( + ["uv", "pip", "list", "--format", "json"], + capture_output=True, + text=True, + ) + + if result.returncode == 0: + packages = json.loads(result.stdout) + for pkg in packages: + name = pkg.get("name", "") + if name.startswith("mcp-") or "mcp" in name.lower(): + mcp_servers.append({ + "type": "mcp_server_package", + "package": name, + "version": pkg.get("version"), + }) + except Exception as e: + print(f"Error discovering MCP servers: {e}", file=sys.stderr) + + return mcp_servers + + +def discover_custom_mcp_servers(w: WorkspaceClient) -> List[Dict[str, Any]]: + """Discover custom MCP servers deployed as Databricks apps.""" + custom_servers = [] + + try: + # List all apps and filter for those starting with mcp- + apps = w.apps.list() + for app in apps: + if app.name and app.name.startswith("mcp-"): + custom_servers.append({ + "type": "custom_mcp_server", + "name": app.name, + "url": app.url, + "status": app.app_status.state.value if app.app_status and app.app_status.state else None, + "description": app.description, + }) + except Exception as e: + print(f"Error discovering custom MCP servers: {e}", file=sys.stderr) + + return custom_servers + + +def discover_external_mcp_servers(w: WorkspaceClient) -> List[Dict[str, Any]]: + """Discover external MCP servers configured via Unity Catalog connections.""" + external_servers = [] + + try: + # List all connections and filter for MCP connections + connections = w.connections.list() + for conn in connections: + # Check if this is an MCP connection + if conn.options and conn.options.get("is_mcp_connection") == "true": + external_servers.append({ + "type": "external_mcp_server", + "name": conn.name, + "connection_type": conn.connection_type.value if hasattr(conn.connection_type, "value") else str(conn.connection_type), + "comment": conn.comment, + "full_name": conn.full_name, + }) + except Exception as e: + print(f"Error discovering external MCP servers: {e}", file=sys.stderr) + + return external_servers + + +def format_output_markdown(results: Dict[str, List[Dict[str, Any]]]) -> str: + """Format discovery results as markdown.""" + lines = ["# Agent Tools and Data Sources Discovery\n"] + + # UC Functions + functions = results.get("uc_functions", []) + if functions: + lines.append(f"## Unity Catalog Functions ({len(functions)})\n") + lines.append("These can be used as agent tools via MCP servers.\n") + for func in functions[:10]: # Show first 10 + lines.append(f"- `{func['name']}`") + if func.get("comment"): + lines.append(f" - {func['comment']}") + if len(functions) > 10: + lines.append(f"\n*...and {len(functions) - 10} more*\n") + lines.append("") + + # UC Tables + tables = results.get("uc_tables", []) + if tables: + lines.append(f"## Unity Catalog Tables ({len(tables)})\n") + lines.append("These can be queried by agents for structured data.\n") + for table in tables[:10]: # Show first 10 + lines.append(f"- `{table['name']}` ({table['table_type']})") + if table.get("comment"): + lines.append(f" - {table['comment']}") + if table.get("columns"): + col_names = [c["name"] for c in table["columns"][:5]] + lines.append(f" - Columns: {', '.join(col_names)}") + if len(tables) > 10: + lines.append(f"\n*...and {len(tables) - 10} more*\n") + lines.append("") + + # Vector Search Indexes + indexes = results.get("vector_search_indexes", []) + if indexes: + lines.append(f"## Vector Search Indexes ({len(indexes)})\n") + lines.append("These can be used for RAG applications.\n") + for idx in indexes: + lines.append(f"- `{idx['name']}`") + lines.append(f" - Endpoint: {idx['endpoint']}") + lines.append(f" - Status: {idx['status']}") + lines.append("") + + # Genie Spaces + spaces = results.get("genie_spaces", []) + if spaces: + lines.append(f"## Genie Spaces ({len(spaces)})\n") + lines.append("These provide conversational data access.\n") + for space in spaces: + lines.append(f"- `{space['name']}` (ID: {space['id']})") + if space.get("description"): + lines.append(f" - {space['description']}") + lines.append("") + + # Custom MCP Servers (Databricks Apps) + custom_servers = results.get("custom_mcp_servers", []) + if custom_servers: + lines.append(f"## Custom MCP Servers ({len(custom_servers)})\n") + lines.append("MCP servers deployed as Databricks Apps (names starting with mcp-).\n") + for server in custom_servers: + lines.append(f"- `{server['name']}`") + if server.get("url"): + lines.append(f" - URL: {server['url']}") + if server.get("status"): + lines.append(f" - Status: {server['status']}") + if server.get("description"): + lines.append(f" - {server['description']}") + lines.append("") + + # External MCP Servers (UC Connections) + external_servers = results.get("external_mcp_servers", []) + if external_servers: + lines.append(f"## External MCP Servers ({len(external_servers)})\n") + lines.append("External MCP servers configured via Unity Catalog connections.\n") + for server in external_servers: + lines.append(f"- `{server['name']}`") + if server.get("full_name"): + lines.append(f" - Full name: {server['full_name']}") + if server.get("comment"): + lines.append(f" - {server['comment']}") + lines.append("") + + return "\n".join(lines) + + +def main(): + """Main discovery function.""" + import argparse + + parser = argparse.ArgumentParser(description="Discover available agent tools and data sources") + parser.add_argument("--catalog", help="Limit discovery to specific catalog") + parser.add_argument("--schema", help="Limit discovery to specific schema (requires --catalog)") + parser.add_argument("--format", choices=["json", "markdown"], default="markdown", help="Output format") + parser.add_argument("--output", help="Output file (default: stdout)") + + args = parser.parse_args() + + if args.schema and not args.catalog: + print("Error: --schema requires --catalog", file=sys.stderr) + sys.exit(1) + + print("Discovering available tools and data sources...", file=sys.stderr) + + # Initialize Databricks workspace client + w = WorkspaceClient() + + results = {} + + # Discover each type (limit to first 20 of each) + print("- UC Functions...", file=sys.stderr) + results["uc_functions"] = discover_uc_functions(w, catalog=args.catalog)[:TOOL_LIST_PAGE_SIZE] + + print("- UC Tables...", file=sys.stderr) + results["uc_tables"] = discover_uc_tables(w, catalog=args.catalog, schema=args.schema)[:TOOL_LIST_PAGE_SIZE] + + print("- Vector Search Indexes...", file=sys.stderr) + results["vector_search_indexes"] = discover_vector_search_indexes(w)[:TOOL_LIST_PAGE_SIZE] + + print("- Genie Spaces...", file=sys.stderr) + results["genie_spaces"] = discover_genie_spaces(w)[:TOOL_LIST_PAGE_SIZE] + + print("- Custom MCP Servers (Apps)...", file=sys.stderr) + results["custom_mcp_servers"] = discover_custom_mcp_servers(w)[:TOOL_LIST_PAGE_SIZE] + + print("- External MCP Servers (Connections)...", file=sys.stderr) + results["external_mcp_servers"] = discover_external_mcp_servers(w)[:TOOL_LIST_PAGE_SIZE] + + # Format output + if args.format == "json": + output = json.dumps(results, indent=2) + else: + output = format_output_markdown(results) + + # Write output + if args.output: + Path(args.output).write_text(output) + print(f"\nResults written to {args.output}", file=sys.stderr) + else: + print("\n" + output) + + # Print summary + print("\n=== Discovery Summary ===", file=sys.stderr) + print(f"UC Functions: {len(results['uc_functions'])}", file=sys.stderr) + print(f"UC Tables: {len(results['uc_tables'])}", file=sys.stderr) + print(f"Vector Search Indexes: {len(results['vector_search_indexes'])}", file=sys.stderr) + print(f"Genie Spaces: {len(results['genie_spaces'])}", file=sys.stderr) + print(f"Custom MCP Servers: {len(results['custom_mcp_servers'])}", file=sys.stderr) + print(f"External MCP Servers: {len(results['external_mcp_servers'])}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/quickstart.sh b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/quickstart.sh new file mode 100755 index 0000000000..51f00c1b4f --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/quickstart.sh @@ -0,0 +1,385 @@ +#!/bin/bash +set -e + +# Parse command line arguments +PROFILE_ARG="" +HOST_ARG="" + +while [[ $# -gt 0 ]]; do + case $1 in + --profile) + PROFILE_ARG="$2" + shift 2 + ;; + --host) + HOST_ARG="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --profile NAME Use specified Databricks profile (non-interactive)" + echo " --host URL Databricks workspace URL (for initial setup)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Error: Unknown option $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Helper function to check if a command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Helper function to check if Homebrew is available +has_brew() { + command_exists brew +} + +echo "===================================================================" +echo "Agent on Apps - Quickstart Setup" +echo "===================================================================" +echo + +# =================================================================== +# Section 1: Prerequisites Installation +# =================================================================== + +echo "Checking and installing prerequisites..." +echo + +# Check and install UV +if command_exists uv; then + echo "✓ UV is already installed" + uv --version +else + echo "Installing UV..." + if has_brew; then + echo "Using Homebrew to install UV..." + brew install uv + else + echo "Using curl to install UV..." + curl -LsSf https://astral.sh/uv/install.sh | sh + # Add UV to PATH for current session + export PATH="$HOME/.cargo/bin:$PATH" + fi + echo "✓ UV installed successfully" +fi + +# Check and install nvm +if [ -s "$HOME/.nvm/nvm.sh" ]; then + echo "✓ nvm is already installed" + # Load nvm for current session + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +else + echo "Installing nvm..." + if has_brew; then + echo "Using Homebrew to install nvm..." + brew install nvm + # Create nvm directory + mkdir -p ~/.nvm + # Add nvm to current session + export NVM_DIR="$HOME/.nvm" + [ -s "/opt/homebrew/opt/nvm/nvm.sh" ] && \. "/opt/homebrew/opt/nvm/nvm.sh" + [ -s "/usr/local/opt/nvm/nvm.sh" ] && \. "/usr/local/opt/nvm/nvm.sh" + else + echo "Using curl to install nvm..." + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash + # Load nvm for current session + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + fi + echo "✓ nvm installed successfully" +fi + +# Use Node 20 +echo "Setting up Node.js 20..." +nvm install 20 +nvm use 20 +echo "✓ Node.js 20 is now active" +node --version +npm --version +echo + +# Check and install Databricks CLI +if command_exists databricks; then + echo "✓ Databricks CLI is already installed" + databricks --version +else + echo "Installing Databricks CLI..." + if has_brew; then + echo "Using Homebrew to install Databricks CLI..." + brew tap databricks/tap + brew install databricks + else + echo "Using curl to install Databricks CLI..." + if curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh; then + echo "✓ Databricks CLI installed successfully" + else + echo "Installation failed, trying with sudo..." + curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh + fi + fi + echo "✓ Databricks CLI installed successfully" +fi +echo + +# =================================================================== +# Section 2: Configuration Files Setup +# =================================================================== +echo "Setting up configuration files..." + +# Copy .env.example to .env.local if it doesn't exist +if [ ! -f ".env.local" ]; then + echo "Copying .env.example to .env.local..." + cp .env.example .env.local + echo +else + echo ".env.local already exists, skipping copy..." +fi +echo + +# =================================================================== +# Section 3: Databricks Authentication +# =================================================================== + +echo "Setting up Databricks authentication..." + +# If --profile flag was provided, use that directly +if [ -n "$PROFILE_ARG" ]; then + PROFILE_NAME="$PROFILE_ARG" + echo "Using specified profile: $PROFILE_NAME" +else + # Check if there are existing profiles + set +e + EXISTING_PROFILES=$(databricks auth profiles 2>/dev/null) + PROFILES_EXIT_CODE=$? + set -e + + if [ $PROFILES_EXIT_CODE -eq 0 ] && [ -n "$EXISTING_PROFILES" ]; then + # Profiles exist - let user select one + echo "Found existing Databricks profiles:" + echo + + # Parse profiles into an array (compatible with older bash) + # Skip the first line (header row) + PROFILE_ARRAY=() + PROFILE_NAMES=() + LINE_NUM=0 + while IFS= read -r line; do + if [ -n "$line" ]; then + if [ $LINE_NUM -eq 0 ]; then + # Print header without number + echo "$line" + else + # Add full line to display array + PROFILE_ARRAY+=("$line") + # Extract just the profile name (first column) for selection + PROFILE_NAME_ONLY=$(echo "$line" | awk '{print $1}') + PROFILE_NAMES+=("$PROFILE_NAME_ONLY") + fi + LINE_NUM=$((LINE_NUM + 1)) + fi + done <<< "$EXISTING_PROFILES" + echo + + # Display numbered list + for i in "${!PROFILE_ARRAY[@]}"; do + echo "$((i+1))) ${PROFILE_ARRAY[$i]}" + done + echo + + echo "Enter the number of the profile you want to use:" + read -r PROFILE_CHOICE + + if [ -z "$PROFILE_CHOICE" ]; then + echo "Error: Profile selection is required" + exit 1 + fi + + # Validate the choice is a number + if ! [[ "$PROFILE_CHOICE" =~ ^[0-9]+$ ]]; then + echo "Error: Please enter a valid number" + exit 1 + fi + + # Convert to array index (subtract 1) + PROFILE_INDEX=$((PROFILE_CHOICE - 1)) + + # Check if the index is valid + if [ $PROFILE_INDEX -lt 0 ] || [ $PROFILE_INDEX -ge ${#PROFILE_NAMES[@]} ]; then + echo "Error: Invalid selection. Please choose a number between 1 and ${#PROFILE_NAMES[@]}" + exit 1 + fi + + # Get the selected profile name (just the name, not the full line) + PROFILE_NAME="${PROFILE_NAMES[$PROFILE_INDEX]}" + echo "Selected profile: $PROFILE_NAME" + fi +fi + +# Validate the profile if it was specified +if [ -n "$PROFILE_NAME" ]; then + + # Test if the profile works + set +e + DATABRICKS_CONFIG_PROFILE="$PROFILE_NAME" databricks current-user me >/dev/null 2>&1 + PROFILE_TEST=$? + set -e + + if [ $PROFILE_TEST -eq 0 ]; then + echo "✓ Successfully validated profile '$PROFILE_NAME'" + else + # Profile exists but isn't authenticated - prompt to authenticate + echo "Profile '$PROFILE_NAME' is not authenticated." + echo "Authenticating profile '$PROFILE_NAME'..." + echo "You will be prompted to log in to Databricks in your browser." + echo + + # Temporarily disable exit on error for the auth command + set +e + + # Run auth login with the profile name and capture output while still showing it to the user + AUTH_LOG=$(mktemp) + databricks auth login --profile "$PROFILE_NAME" 2>&1 | tee "$AUTH_LOG" + AUTH_EXIT_CODE=$? + + set -e + + if [ $AUTH_EXIT_CODE -eq 0 ]; then + echo "✓ Successfully authenticated profile '$PROFILE_NAME'" + # Clean up temp file + rm -f "$AUTH_LOG" + else + # Clean up temp file + rm -f "$AUTH_LOG" + echo "Error: Profile '$PROFILE_NAME' authentication failed" + exit 1 + fi + fi + + # Update .env.local with the profile name + if grep -q "DATABRICKS_CONFIG_PROFILE=" .env.local; then + sed -i '' "s/DATABRICKS_CONFIG_PROFILE=.*/DATABRICKS_CONFIG_PROFILE=$PROFILE_NAME/" .env.local + else + echo "DATABRICKS_CONFIG_PROFILE=$PROFILE_NAME" >> .env.local + fi + echo "✓ Databricks profile '$PROFILE_NAME' saved to .env.local" +else + # No profiles exist - create default one + echo "No existing profiles found. Setting up Databricks authentication..." + + # Use --host flag if provided, otherwise prompt + if [ -n "$HOST_ARG" ]; then + DATABRICKS_HOST="$HOST_ARG" + echo "Using specified host: $DATABRICKS_HOST" + else + echo "Please enter your Databricks host URL (e.g., https://your-workspace.cloud.databricks.com):" + read -r DATABRICKS_HOST + + if [ -z "$DATABRICKS_HOST" ]; then + echo "Error: Databricks host is required" + exit 1 + fi + fi + + echo "Authenticating with Databricks..." + echo "You will be prompted to log in to Databricks in your browser." + echo + + # Temporarily disable exit on error for the auth command + set +e + + # Run auth login with host parameter and capture output while still showing it to the user + AUTH_LOG=$(mktemp) + databricks auth login --host "$DATABRICKS_HOST" 2>&1 | tee "$AUTH_LOG" + AUTH_EXIT_CODE=$? + + set -e + + if [ $AUTH_EXIT_CODE -eq 0 ]; then + echo "✓ Successfully authenticated with Databricks" + + # Extract profile name from the captured output + # Expected format: "Profile DEFAULT was successfully saved" + PROFILE_NAME=$(grep -i "Profile .* was successfully saved" "$AUTH_LOG" | sed -E 's/.*Profile ([^ ]+) was successfully saved.*/\1/' | head -1) + + # Clean up temp file + rm -f "$AUTH_LOG" + + # If we couldn't extract the profile name, default to "DEFAULT" + if [ -z "$PROFILE_NAME" ]; then + PROFILE_NAME="DEFAULT" + echo "Note: Could not detect profile name, using 'DEFAULT'" + fi + + # Update .env.local with the profile name + if grep -q "DATABRICKS_CONFIG_PROFILE=" .env.local; then + sed -i '' "s/DATABRICKS_CONFIG_PROFILE=.*/DATABRICKS_CONFIG_PROFILE=$PROFILE_NAME/" .env.local + else + echo "DATABRICKS_CONFIG_PROFILE=$PROFILE_NAME" >> .env.local + fi + + echo "✓ Databricks profile '$PROFILE_NAME' saved to .env.local" + else + # Clean up temp file + rm -f "$AUTH_LOG" + echo "Databricks authentication was cancelled or failed." + echo "Please run this script again when you're ready to authenticate." + exit 1 + fi +fi +echo + +# =================================================================== +# Section 4: MLflow Experiment Setup +# =================================================================== + + +# Get current Databricks username +echo "Getting Databricks username..." +DATABRICKS_USERNAME=$(databricks -p $PROFILE_NAME current-user me | jq -r .userName) +echo "Username: $DATABRICKS_USERNAME" +echo + +# Create MLflow experiment and capture the experiment ID +echo "Creating MLflow experiment..." +EXPERIMENT_NAME="/Users/$DATABRICKS_USERNAME/agents-on-apps" + +# Try to create the experiment with the default name first +if EXPERIMENT_RESPONSE=$(databricks -p $PROFILE_NAME experiments create-experiment $EXPERIMENT_NAME 2>/dev/null); then + EXPERIMENT_ID=$(echo $EXPERIMENT_RESPONSE | jq -r .experiment_id) + echo "Created experiment '$EXPERIMENT_NAME' with ID: $EXPERIMENT_ID" +else + echo "Experiment name already exists, creating with random suffix..." + RANDOM_SUFFIX=$(openssl rand -hex 4) + EXPERIMENT_NAME="/Users/$DATABRICKS_USERNAME/agents-on-apps-$RANDOM_SUFFIX" + EXPERIMENT_RESPONSE=$(databricks -p $PROFILE_NAME experiments create-experiment $EXPERIMENT_NAME) + EXPERIMENT_ID=$(echo $EXPERIMENT_RESPONSE | jq -r .experiment_id) + echo "Created experiment '$EXPERIMENT_NAME' with ID: $EXPERIMENT_ID" +fi +echo + +# Update .env.local with the experiment ID +echo "Updating .env.local with experiment ID..." +sed -i '' "s/MLFLOW_EXPERIMENT_ID=.*/MLFLOW_EXPERIMENT_ID=$EXPERIMENT_ID/" .env.local +echo + +echo "===================================================================" +echo "Setup Complete!" +echo "===================================================================" +echo "✓ Prerequisites installed (UV, nvm, Databricks CLI)" +echo "✓ Databricks authenticated with profile: $PROFILE_NAME" +echo "✓ Configuration files created (.env.local)" +echo "✓ MLflow experiment created: $EXPERIMENT_NAME" +echo "✓ Experiment ID: $EXPERIMENT_ID" +echo "✓ Configuration updated in .env.local" +echo "===================================================================" +echo diff --git a/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/start_app.py b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/start_app.py new file mode 100644 index 0000000000..487a84199a --- /dev/null +++ b/experimental/aitools/templates/agent-openai-agents-sdk/template/{{.project_name}}/scripts/start_app.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +""" +Start script for running frontend and backend processes concurrently. + +Requirements: +1. Not reporting ready until BOTH frontend and backend processes are ready +2. Exiting as soon as EITHER process fails +3. Printing error logs if either process fails + +Usage: + start-app [--port PORT] [--host HOST] [--workers N] [--reload] +""" + +import argparse +import re +import shutil +import subprocess +import sys +import threading +import time +from pathlib import Path + +from dotenv import load_dotenv + +# Readiness patterns +BACKEND_READY = [r"Uvicorn running on", r"Application startup complete", r"Started server process"] +FRONTEND_READY = [r"Server is running on http://localhost"] + + +class ProcessManager: + def __init__(self): + self.backend_process = None + self.frontend_process = None + self.backend_ready = False + self.frontend_ready = False + self.failed = threading.Event() + self.backend_log = None + self.frontend_log = None + + def monitor_process(self, process, name, log_file, patterns): + is_ready = False + try: + for line in iter(process.stdout.readline, ""): + if not line: + break + + line = line.rstrip() + log_file.write(line + "\n") + print(f"[{name}] {line}") + + # Check readiness + if not is_ready and any(re.search(p, line, re.IGNORECASE) for p in patterns): + is_ready = True + if name == "backend": + self.backend_ready = True + else: + self.frontend_ready = True + print(f"✓ {name.capitalize()} is ready!") + + if self.backend_ready and self.frontend_ready: + print("\n" + "=" * 50) + print("✓ Both frontend and backend are ready!") + print("✓ Open the frontend at http://localhost:8000") + print("=" * 50 + "\n") + + process.wait() + if process.returncode != 0: + self.failed.set() + + except Exception as e: + print(f"Error monitoring {name}: {e}") + self.failed.set() + + def clone_frontend_if_needed(self): + if Path("e2e-chatbot-app-next").exists(): + return True + + print("Cloning e2e-chatbot-app-next...") + for url in [ + "https://github.com/databricks/app-templates.git", + "git@github.com:databricks/app-templates.git", + ]: + try: + subprocess.run( + ["git", "clone", "--filter=blob:none", "--sparse", url, "temp-app-templates"], + check=True, + capture_output=True, + ) + break + except subprocess.CalledProcessError: + continue + else: + print("ERROR: Failed to clone repository.") + print( + "Manually download from: https://download-directory.github.io/?url=https://github.com/databricks/app-templates/tree/main/e2e-chatbot-app-next" + ) + return False + + subprocess.run( + ["git", "sparse-checkout", "set", "e2e-chatbot-app-next"], + cwd="temp-app-templates", + check=True, + ) + Path("temp-app-templates/e2e-chatbot-app-next").rename("e2e-chatbot-app-next") + shutil.rmtree("temp-app-templates", ignore_errors=True) + return True + + def start_process(self, cmd, name, log_file, patterns, cwd=None): + print(f"Starting {name}...") + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, cwd=cwd + ) + + thread = threading.Thread( + target=self.monitor_process, args=(process, name, log_file, patterns), daemon=True + ) + thread.start() + return process + + def print_logs(self, log_path): + print(f"\nLast 50 lines of {log_path}:") + print("-" * 40) + try: + lines = Path(log_path).read_text().splitlines() + print("\n".join(lines[-50:])) + except FileNotFoundError: + print(f"(no {log_path} found)") + print("-" * 40) + + def cleanup(self): + print("\n" + "=" * 42) + print("Shutting down both processes...") + print("=" * 42) + + for proc in [self.backend_process, self.frontend_process]: + if proc: + try: + proc.terminate() + proc.wait(timeout=5) + except (subprocess.TimeoutExpired, Exception): + proc.kill() + + if self.backend_log: + self.backend_log.close() + if self.frontend_log: + self.frontend_log.close() + + def run(self, args=None): + load_dotenv(dotenv_path=".env.local", override=True) + + if not self.clone_frontend_if_needed(): + return 1 + + # Open log files + self.backend_log = open("backend.log", "w", buffering=1) + self.frontend_log = open("frontend.log", "w", buffering=1) + + try: + # Build backend command with optional arguments + backend_cmd = ["uv", "run", "start-server"] + if args: + if args.port: + backend_cmd.extend(["--port", str(args.port)]) + if args.host: + backend_cmd.extend(["--host", args.host]) + if args.workers: + backend_cmd.extend(["--workers", str(args.workers)]) + if args.reload: + backend_cmd.append("--reload") + + # Start backend + self.backend_process = self.start_process( + backend_cmd, "backend", self.backend_log, BACKEND_READY + ) + + # Setup and start frontend + frontend_dir = Path("e2e-chatbot-app-next") + for cmd, desc in [("npm install", "install"), ("npm run build", "build")]: + print(f"Running npm {desc}...") + result = subprocess.run( + cmd.split(), cwd=frontend_dir, capture_output=True, text=True + ) + if result.returncode != 0: + print(f"npm {desc} failed: {result.stderr}") + return 1 + + self.frontend_process = self.start_process( + ["npm", "run", "start"], + "frontend", + self.frontend_log, + FRONTEND_READY, + cwd=frontend_dir, + ) + + print( + f"\nMonitoring processes (Backend PID: {self.backend_process.pid}, Frontend PID: {self.frontend_process.pid})\n" + ) + + # Wait for failure + while not self.failed.is_set(): + time.sleep(0.1) + for proc in [self.backend_process, self.frontend_process]: + if proc.poll() is not None: + self.failed.set() + break + + # Determine which failed + failed_name = "backend" if self.backend_process.poll() is not None else "frontend" + failed_proc = ( + self.backend_process if failed_name == "backend" else self.frontend_process + ) + exit_code = failed_proc.returncode if failed_proc else 1 + + print( + f"\n{'=' * 42}\nERROR: {failed_name} process exited with code {exit_code}\n{'=' * 42}" + ) + self.print_logs("backend.log") + self.print_logs("frontend.log") + return exit_code + + except KeyboardInterrupt: + print("\nInterrupted") + return 0 + + finally: + self.cleanup() + + +def main(): + parser = argparse.ArgumentParser(description="Start agent frontend and backend") + parser.add_argument("--port", type=int, help="Backend server port (default: 8000)") + parser.add_argument("--host", help="Backend server host (default: 0.0.0.0)") + parser.add_argument("--workers", type=int, help="Number of worker processes") + parser.add_argument("--reload", action="store_true", help="Enable auto-reload for development") + args = parser.parse_args() + + sys.exit(ProcessManager().run(args)) + + +if __name__ == "__main__": + main()