diff --git a/.github/workflows/llmobs.yml b/.github/workflows/llmobs.yml index 94ae65087a5..79fb192988d 100644 --- a/.github/workflows/llmobs.yml +++ b/.github/workflows/llmobs.yml @@ -220,6 +220,35 @@ jobs: api_key: ${{ secrets.DD_API_KEY }} service: dd-trace-js-tests + claude-agent-sdk: + runs-on: ubuntu-latest + env: + PLUGINS: claude-agent-sdk + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/oldest-maintenance-lts + - uses: ./.github/actions/install + - run: yarn test:plugins:ci + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: ./.github/actions/node/latest + - run: yarn test:plugins:ci + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: ./.github/actions/coverage + with: + flags: llmobs-${{ github.job }} + - if: always() + uses: ./.github/actions/testagent/logs + with: + suffix: llmobs-${{ github.job }} + - uses: DataDog/junit-upload-github-action@055560f63c405095e9228ba443eee7987e22bb94 # v2.1.1 + if: always() && github.actor != 'dependabot[bot]' + with: + api_key: ${{ secrets.DD_API_KEY }} + service: dd-trace-js-tests + google-genai: runs-on: ubuntu-latest env: diff --git a/benchmark/sirun/plugin-claude-agent-sdk/README.md b/benchmark/sirun/plugin-claude-agent-sdk/README.md new file mode 100644 index 00000000000..26df84abe1d --- /dev/null +++ b/benchmark/sirun/plugin-claude-agent-sdk/README.md @@ -0,0 +1,3 @@ +Simulates 500 agent sessions, each with 3 turns and 2 tool calls per turn +(22 hook invocations per session). Measures the overhead of the shimmer's +hook injection, diagnostics channel publishing, and span creation/finishing. diff --git a/benchmark/sirun/plugin-claude-agent-sdk/index.js b/benchmark/sirun/plugin-claude-agent-sdk/index.js new file mode 100644 index 00000000000..e7f43b09574 --- /dev/null +++ b/benchmark/sirun/plugin-claude-agent-sdk/index.js @@ -0,0 +1,63 @@ +'use strict' + +if (Number(process.env.USE_TRACER)) { + require('../../..').init() + + // Simulate SDK module load so the plugin manager activates channel + // subscriptions (same event that fires when the real ESM SDK is loaded). + const { channel } = require('dc-polyfill') + channel('dd-trace:instrumentation:load').publish({ name: '@anthropic-ai/claude-agent-sdk' }) +} + +const { wrapQuery } = require('../../../packages/datadog-instrumentations/src/claude-agent-sdk') + +const SESSIONS = 500 + +// Simulate the SDK calling registered hooks, matching the real SDK's +// matcher-array contract: each event is an array of matchers, each +// matcher has an array of hook functions. +function callHooks (matchers, input, extraArg) { + if (!matchers) return + for (const matcher of matchers) { + for (const hook of matcher.hooks) { + hook(input, extraArg) + } + } +} + +// Mock query() that simulates a realistic agent session: +// 3 turns, each with 2 tool calls = 22 hook invocations per call. +// This exercises the full shimmer path: mergeHooks, buildTracerHooks, +// runStores, and all 9 hook callbacks. +function mockQuery ({ prompt, options }) { + const hooks = options && options.hooks ? options.hooks : {} + + callHooks(hooks.SessionStart, { session_id: 'bench', source: 'api' }) + + for (let t = 0; t < 3; t++) { + callHooks(hooks.UserPromptSubmit, { session_id: 'bench', prompt: 'p' }) + + for (let u = 0; u < 2; u++) { + const id = 'tu-' + t + '-' + u + callHooks(hooks.PreToolUse, { + session_id: 'bench', tool_name: 'Read', + tool_input: { path: '/' }, tool_use_id: id + }, id) + callHooks(hooks.PostToolUse, { + session_id: 'bench', tool_response: 'ok', tool_use_id: id + }, id) + } + + callHooks(hooks.Stop, { stop_reason: 'end_turn' }) + } + + callHooks(hooks.SessionEnd, { reason: 'done' }) + + return { ok: true } +} + +const wrapped = wrapQuery(mockQuery) + +for (let i = 0; i < SESSIONS; i++) { + wrapped({ prompt: 'bench', options: { model: 'claude-opus-4-6' } }) +} diff --git a/benchmark/sirun/plugin-claude-agent-sdk/meta.json b/benchmark/sirun/plugin-claude-agent-sdk/meta.json new file mode 100644 index 00000000000..7565a199da1 --- /dev/null +++ b/benchmark/sirun/plugin-claude-agent-sdk/meta.json @@ -0,0 +1,17 @@ +{ + "name": "plugin-claude-agent-sdk", + "run": "node index.js", + "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"", + "cachegrind": false, + "iterations": 40, + "instructions": true, + "variants": { + "control": { + "env": { "USE_TRACER": "0" } + }, + "with-tracer": { + "baseline": "control", + "env": { "USE_TRACER": "1" } + } + } +} diff --git a/docs/API.md b/docs/API.md index 8a3512b7cdf..5585c601e77 100644 --- a/docs/API.md +++ b/docs/API.md @@ -28,6 +28,7 @@ tracer.use('pg', {
+ @@ -106,6 +107,7 @@ tracer.use('pg', { * [amqp10](./interfaces/export_.plugins.amqp10.html) * [amqplib](./interfaces/export_.plugins.amqplib.html) * [anthropic](./interfaces/export_.plugins.anthropic.html) +* [claude-agent-sdk](./interfaces/export_.plugins.claude_agent_sdk.html) * [apollo](./interfaces/export_.plugins.apollo.html) * [avsc](./interfaces/export_.plugins.avsc.html) * [aws-sdk](./interfaces/export_.plugins.aws_sdk.html) diff --git a/docs/test.ts b/docs/test.ts index c34b4329792..1317eba27e4 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -305,6 +305,7 @@ tracer.use('ai', true) tracer.use('amqp10'); tracer.use('amqplib'); tracer.use('anthropic'); +tracer.use('claude-agent-sdk'); tracer.use('avsc'); tracer.use('aws-sdk'); tracer.use('aws-sdk', awsSdkOptions); diff --git a/index.d.ts b/index.d.ts index 822016d22f1..9e131b7f20c 100644 --- a/index.d.ts +++ b/index.d.ts @@ -221,6 +221,7 @@ interface Plugins { "amqp10": tracer.plugins.amqp10; "amqplib": tracer.plugins.amqplib; "anthropic": tracer.plugins.anthropic; + "claude-agent-sdk": tracer.plugins.claude_agent_sdk; "apollo": tracer.plugins.apollo; "avsc": tracer.plugins.avsc; "aws-sdk": tracer.plugins.aws_sdk; @@ -1824,6 +1825,12 @@ declare namespace tracer { */ interface amqplib extends Instrumentation {} + /** + * This plugin automatically instruments the + * [@anthropic-ai/claude-agent-sdk](https://www.npmjs.com/package/@anthropic-ai/claude-agent-sdk) module. + */ + interface claude_agent_sdk extends Instrumentation {} + /** * This plugin automatically instruments the * [anthropic](https://www.npmjs.com/package/@anthropic-ai/sdk) module. diff --git a/packages/datadog-instrumentations/src/claude-agent-sdk.js b/packages/datadog-instrumentations/src/claude-agent-sdk.js new file mode 100644 index 00000000000..dad00b15217 --- /dev/null +++ b/packages/datadog-instrumentations/src/claude-agent-sdk.js @@ -0,0 +1,224 @@ +'use strict' + +// Shimmer is used here because we must merge user-provided Claude hook matchers +// into `options.hooks` at runtime before calling `query()`, which orchestrion +// cannot express without custom argument mutation logic. + +const { tracingChannel } = require('dc-polyfill') +const shimmer = require('../../datadog-shimmer') +const { addHook } = require('./helpers/instrument') + +const sessionCh = tracingChannel('apm:claude-agent-sdk:session') +const turnCh = tracingChannel('apm:claude-agent-sdk:turn') +const toolCh = tracingChannel('apm:claude-agent-sdk:tool') +const subagentCh = tracingChannel('apm:claude-agent-sdk:subagent') + +function mergeHooks (userHooks, tracerHooks) { + const merged = {} + + for (const event of Object.keys(tracerHooks)) { + const userMatchers = (userHooks && userHooks[event]) || [] + const tracerMatchers = tracerHooks[event] + merged[event] = [...userMatchers, ...tracerMatchers] + } + + // Preserve any user hooks for events we don't trace + if (userHooks) { + for (const event of Object.keys(userHooks)) { + if (!merged[event]) { + merged[event] = userHooks[event] + } + } + } + + return merged +} + +function buildTracerHooks (sessionCtx) { + return { + SessionStart: [{ + hooks: [function onSessionStart (input) { + sessionCtx.sessionId = input.session_id + sessionCtx.source = input.source + return {} + }], + }], + + SessionEnd: [{ + hooks: [function onSessionEnd (input) { + sessionCtx.endReason = input.reason + sessionCh.asyncEnd.publish(sessionCtx) + return {} + }], + }], + + UserPromptSubmit: [{ + hooks: [function onUserPromptSubmit (input) { + const turnCtx = { + sessionId: input.session_id, + prompt: input.prompt, + } + + sessionCtx.currentTurn = turnCtx + turnCh.start.runStores(turnCtx, () => { + turnCh.end.publish(turnCtx) + }) + return {} + }], + }], + + Stop: [{ + hooks: [function onStop (input) { + const turnCtx = sessionCtx.currentTurn + if (turnCtx) { + turnCtx.stopReason = input.stop_reason + turnCh.asyncEnd.publish(turnCtx) + sessionCtx.currentTurn = null + } + return {} + }], + }], + + PreToolUse: [{ + hooks: [function onPreToolUse (input, toolUseId) { + const id = toolUseId || input.tool_use_id + if (!id) return {} + + const toolCtx = { + sessionId: input.session_id, + toolName: input.tool_name, + toolInput: input.tool_input, + toolUseId: id, + } + + sessionCtx.pendingTools.set(id, toolCtx) + toolCh.start.runStores(toolCtx, () => { + toolCh.end.publish(toolCtx) + }) + return {} + }], + }], + + PostToolUse: [{ + hooks: [function onPostToolUse (input, toolUseId) { + const id = toolUseId || input.tool_use_id + const toolCtx = sessionCtx.pendingTools.get(id) + if (toolCtx) { + toolCtx.toolResponse = input.tool_response + sessionCtx.pendingTools.delete(id) + toolCh.asyncEnd.publish(toolCtx) + } + return {} + }], + }], + + PostToolUseFailure: [{ + hooks: [function onPostToolUseFailure (input, toolUseId) { + const id = toolUseId || input.tool_use_id + const toolCtx = sessionCtx.pendingTools.get(id) + if (toolCtx) { + toolCtx.error = input.error + sessionCtx.pendingTools.delete(id) + toolCh.error.publish(toolCtx) + toolCh.asyncEnd.publish(toolCtx) + } + return {} + }], + }], + + SubagentStart: [{ + hooks: [function onSubagentStart (input) { + const agentId = input.agent_id + if (!agentId) return {} + + const subagentCtx = { + sessionId: input.session_id, + agentId, + agentType: input.agent_type, + } + + sessionCtx.pendingSubagents.set(agentId, subagentCtx) + subagentCh.start.runStores(subagentCtx, () => { + subagentCh.end.publish(subagentCtx) + }) + return {} + }], + }], + + SubagentStop: [{ + hooks: [function onSubagentStop (input) { + const agentId = input.agent_id + const subagentCtx = sessionCtx.pendingSubagents.get(agentId) + if (subagentCtx) { + subagentCtx.transcriptPath = input.agent_transcript_path + sessionCtx.pendingSubagents.delete(agentId) + subagentCh.asyncEnd.publish(subagentCtx) + } + return {} + }], + }], + } +} + +function wrapQuery (originalQuery) { + return function wrappedQuery ({ prompt, options }) { + if (!sessionCh.start.hasSubscribers) { + return originalQuery.call(this, { prompt, options }) + } + + const resolvedOptions = options || {} + const sessionCtx = { + prompt: typeof prompt === 'string' ? prompt : '[async iterable]', + model: resolvedOptions.model, + resume: resolvedOptions.resume, + maxTurns: resolvedOptions.maxTurns, + permissionMode: resolvedOptions.permissionMode, + currentTurn: null, + pendingTools: new Map(), + pendingSubagents: new Map(), + } + + const tracerHooks = buildTracerHooks(sessionCtx) + const mergedOptions = { + ...resolvedOptions, + hooks: mergeHooks(resolvedOptions.hooks, tracerHooks), + } + + return sessionCh.start.runStores(sessionCtx, () => { + let result + try { + result = originalQuery.call(this, { prompt, options: mergedOptions }) + } catch (err) { + sessionCtx.error = err + sessionCh.error.publish(sessionCtx) + sessionCh.asyncEnd.publish(sessionCtx) + throw err + } + + sessionCh.end.publish(sessionCtx) + + // If query() returns a promise/thenable, catch rejections + if (result && typeof result.then === 'function') { + result.then(null, (err) => { + sessionCtx.error = err + sessionCh.error.publish(sessionCtx) + sessionCh.asyncEnd.publish(sessionCtx) + }) + } + + return result + }) + } +} + +addHook({ + name: '@anthropic-ai/claude-agent-sdk', + file: 'sdk.mjs', + versions: ['>=0.2.0'], +}, (exports) => { + shimmer.wrap(exports, 'query', wrapQuery) + return exports +}) + +// Exported for unit testing +module.exports = { mergeHooks, buildTracerHooks, wrapQuery } diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 24cebb591ec..9600e6f6fe7 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -1,6 +1,7 @@ 'use strict' module.exports = { + '@anthropic-ai/claude-agent-sdk': { esmFirst: true, fn: () => require('../claude-agent-sdk') }, '@anthropic-ai/sdk': { esmFirst: true, fn: () => require('../anthropic') }, '@apollo/server': () => require('../apollo-server'), '@apollo/gateway': () => require('../apollo'), diff --git a/packages/datadog-plugin-claude-agent-sdk/src/index.js b/packages/datadog-plugin-claude-agent-sdk/src/index.js new file mode 100644 index 00000000000..7e8c95a5848 --- /dev/null +++ b/packages/datadog-plugin-claude-agent-sdk/src/index.js @@ -0,0 +1,29 @@ +'use strict' + +const CompositePlugin = require('../../dd-trace/src/plugins/composite') +const ClaudeAgentSdkLLMObsPlugins = require('../../dd-trace/src/llmobs/plugins/claude-agent-sdk') +const ClaudeAgentSdkTracingPlugins = require('./tracing') + +class ClaudeAgentSdkPlugin extends CompositePlugin { + static id = 'claude-agent-sdk' + + static get plugins () { + const plugins = {} + + // LLM Obs plugins must be registered before tracing plugins so that + // annotations are added to the span before it finishes. + // The tracing plugin uses `bindStart` vs the LLM Obs plugin's `start`, + // so the span is created in the tracing plugin before the LLM Obs one runs. + for (const Plugin of ClaudeAgentSdkLLMObsPlugins) { + plugins[Plugin.id] = Plugin + } + + for (const Plugin of ClaudeAgentSdkTracingPlugins) { + plugins[Plugin.id] = Plugin + } + + return plugins + } +} + +module.exports = ClaudeAgentSdkPlugin diff --git a/packages/datadog-plugin-claude-agent-sdk/src/tracing.js b/packages/datadog-plugin-claude-agent-sdk/src/tracing.js new file mode 100644 index 00000000000..2c7bf448aca --- /dev/null +++ b/packages/datadog-plugin-claude-agent-sdk/src/tracing.js @@ -0,0 +1,95 @@ +'use strict' + +const TracingPlugin = require('../../dd-trace/src/plugins/tracing') + +class BaseClaudeAgentSdkTracingPlugin extends TracingPlugin { + static system = 'claude-agent-sdk' + + bindStart (ctx) { + const tags = this.constructor.extractTags(ctx) + + this.startSpan(this.constructor.spanName, { + meta: { + 'resource.name': ctx.resource || this.constructor.spanName, + ...tags, + }, + }, ctx) + + return ctx.currentStore + } + + asyncEnd (ctx) { + const span = ctx.currentStore?.span + span?.finish() + } + + static extractTags () { + return null + } +} + +class SessionTracingPlugin extends BaseClaudeAgentSdkTracingPlugin { + static id = 'claude_agent_sdk_session' + static operation = 'session' + static prefix = 'tracing:apm:claude-agent-sdk:session' + static spanName = 'claude-agent-sdk.session' + + static extractTags (ctx) { + const tags = {} + if (ctx.sessionId) tags['claude-agent-sdk.session.id'] = ctx.sessionId + if (ctx.model) tags['claude-agent-sdk.session.model'] = ctx.model + if (ctx.resume) tags['claude-agent-sdk.session.parent_session_id'] = ctx.resume + if (ctx.permissionMode) tags['claude-agent-sdk.session.permission_mode'] = ctx.permissionMode + return tags + } +} + +class TurnTracingPlugin extends BaseClaudeAgentSdkTracingPlugin { + static id = 'claude_agent_sdk_turn' + static operation = 'turn' + static prefix = 'tracing:apm:claude-agent-sdk:turn' + static spanName = 'claude-agent-sdk.turn' + + static extractTags (ctx) { + const tags = {} + if (ctx.sessionId) tags['claude-agent-sdk.session.id'] = ctx.sessionId + return tags + } +} + +class ToolTracingPlugin extends BaseClaudeAgentSdkTracingPlugin { + static id = 'claude_agent_sdk_tool' + static operation = 'tool' + static prefix = 'tracing:apm:claude-agent-sdk:tool' + static spanName = 'claude-agent-sdk.tool' + + static extractTags (ctx) { + const tags = {} + if (ctx.toolName) tags['claude-agent-sdk.tool.name'] = ctx.toolName + if (ctx.toolUseId) tags['claude-agent-sdk.tool.use_id'] = ctx.toolUseId + if (ctx.sessionId) tags['claude-agent-sdk.session.id'] = ctx.sessionId + return tags + } +} + +class SubagentTracingPlugin extends BaseClaudeAgentSdkTracingPlugin { + static id = 'claude_agent_sdk_subagent' + static operation = 'subagent' + static prefix = 'tracing:apm:claude-agent-sdk:subagent' + static spanName = 'claude-agent-sdk.subagent' + + static extractTags (ctx) { + const tags = {} + if (ctx.agentId) tags['claude-agent-sdk.subagent.id'] = ctx.agentId + if (ctx.agentType) tags['claude-agent-sdk.subagent.type'] = ctx.agentType + if (ctx.sessionId) tags['claude-agent-sdk.session.id'] = ctx.sessionId + return tags + } +} + +module.exports = [ + SessionTracingPlugin, + TurnTracingPlugin, + ToolTracingPlugin, + SubagentTracingPlugin, +] diff --git a/packages/datadog-plugin-claude-agent-sdk/test/index.spec.js b/packages/datadog-plugin-claude-agent-sdk/test/index.spec.js new file mode 100644 index 00000000000..3e777bedca9 --- /dev/null +++ b/packages/datadog-plugin-claude-agent-sdk/test/index.spec.js @@ -0,0 +1,775 @@ +'use strict' + +const assert = require('node:assert') +const { describe, before, after, it } = require('mocha') +const { tracingChannel, channel } = require('dc-polyfill') +const agent = require('../../dd-trace/test/plugins/agent') + +// Use tracingChannel to match the shimmer's channel contract. +// tracingChannel('apm:X') creates channels at tracing:apm:X:{start,end,asyncEnd,error}, +// which the TracingPlugin prefix 'tracing:apm:X' subscribes to. +const sessionCh = tracingChannel('apm:claude-agent-sdk:session') +const turnCh = tracingChannel('apm:claude-agent-sdk:turn') +const toolCh = tracingChannel('apm:claude-agent-sdk:tool') +const subagentCh = tracingChannel('apm:claude-agent-sdk:subagent') + +describe('Plugin', () => { + describe('claude-agent-sdk', () => { + // Shimmer unit tests — run without agent.load so channel publishing is a no-op. + // These test the pure logic of mergeHooks and buildTracerHooks. + describe('shimmer', () => { + const { + mergeHooks, + buildTracerHooks, + wrapQuery, + } = require('../../datadog-instrumentations/src/claude-agent-sdk') + + describe('mergeHooks', () => { + it('merges tracer hooks with null user hooks', () => { + const tracerHooks = { + SessionStart: [{ hooks: [() => ({})] }], + Stop: [{ hooks: [() => ({})] }], + } + + const merged = mergeHooks(null, tracerHooks) + + assert.equal(merged.SessionStart.length, 1) + assert.equal(merged.Stop.length, 1) + }) + + it('merges tracer hooks with undefined user hooks', () => { + const tracerHooks = { + SessionStart: [{ hooks: [() => ({})] }], + } + + const merged = mergeHooks(undefined, tracerHooks) + + assert.equal(merged.SessionStart.length, 1) + }) + + it('merges user hooks and tracer hooks for the same event', () => { + const userHooks = { + SessionStart: [{ hooks: [() => ({ decision: 'allow' })] }], + } + const tracerHooks = { + SessionStart: [{ hooks: [() => ({})] }], + Stop: [{ hooks: [() => ({})] }], + } + + const merged = mergeHooks(userHooks, tracerHooks) + + assert.equal(merged.SessionStart.length, 2) + assert.equal(merged.Stop.length, 1) + }) + + it('preserves user hooks for events the tracer does not trace', () => { + const userHooks = { + CustomEvent: [{ hooks: [() => ({ allowed: true })] }], + SessionStart: [{ hooks: [() => ({})] }], + } + const tracerHooks = { + SessionStart: [{ hooks: [() => ({})] }], + } + + const merged = mergeHooks(userHooks, tracerHooks) + + assert.equal(merged.SessionStart.length, 2) + assert.equal(merged.CustomEvent.length, 1) + }) + + it('places user hooks before tracer hooks in the array', () => { + const userMatcher = { hooks: [() => 'user'] } + const tracerMatcher = { hooks: [() => 'tracer'] } + + const merged = mergeHooks( + { SessionStart: [userMatcher] }, + { SessionStart: [tracerMatcher] } + ) + + assert.strictEqual(merged.SessionStart[0], userMatcher) + assert.strictEqual(merged.SessionStart[1], tracerMatcher) + }) + }) + + describe('buildTracerHooks', () => { + it('creates hooks for all expected events', () => { + const sessionCtx = { + pendingTools: new Map(), + pendingSubagents: new Map(), + currentTurn: null, + } + + const hooks = buildTracerHooks(sessionCtx) + + const expectedEvents = [ + 'SessionStart', 'SessionEnd', 'UserPromptSubmit', 'Stop', + 'PreToolUse', 'PostToolUse', 'PostToolUseFailure', + 'SubagentStart', 'SubagentStop', + ] + + for (const event of expectedEvents) { + assert.ok(hooks[event], `should have ${event} hook`) + assert.equal(hooks[event].length, 1, `${event} should have one matcher`) + assert.equal(hooks[event][0].hooks.length, 1, `${event} matcher should have one hook`) + } + }) + + it('SessionStart sets sessionId and source', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + const result = hooks.SessionStart[0].hooks[0]({ + session_id: 'sess-123', + source: 'startup', + }) + + assert.equal(sessionCtx.sessionId, 'sess-123') + assert.equal(sessionCtx.source, 'startup') + assert.deepStrictEqual(result, {}) + }) + + it('SessionEnd sets endReason', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + hooks.SessionEnd[0].hooks[0]({ reason: 'completed' }) + + assert.equal(sessionCtx.endReason, 'completed') + }) + + it('UserPromptSubmit creates a turnCtx on sessionCtx', () => { + const sessionCtx = { + pendingTools: new Map(), + pendingSubagents: new Map(), + currentTurn: null, + } + const hooks = buildTracerHooks(sessionCtx) + + hooks.UserPromptSubmit[0].hooks[0]({ + session_id: 'sess-123', + prompt: 'What files exist?', + }) + + assert.ok(sessionCtx.currentTurn) + assert.equal(sessionCtx.currentTurn.sessionId, 'sess-123') + assert.equal(sessionCtx.currentTurn.prompt, 'What files exist?') + }) + + it('Stop sets stopReason and clears currentTurn', () => { + const sessionCtx = { + pendingTools: new Map(), + pendingSubagents: new Map(), + currentTurn: { sessionId: 'sess-123' }, + } + const hooks = buildTracerHooks(sessionCtx) + + hooks.Stop[0].hooks[0]({ stop_reason: 'end_turn' }) + + assert.equal(sessionCtx.currentTurn, null) + }) + + it('Stop is a no-op when there is no currentTurn', () => { + const sessionCtx = { + pendingTools: new Map(), + pendingSubagents: new Map(), + currentTurn: null, + } + const hooks = buildTracerHooks(sessionCtx) + + const result = hooks.Stop[0].hooks[0]({ stop_reason: 'end_turn' }) + + assert.deepStrictEqual(result, {}) + assert.equal(sessionCtx.currentTurn, null) + }) + + it('PreToolUse adds a tool context to pendingTools', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + hooks.PreToolUse[0].hooks[0]({ + session_id: 'sess-123', + tool_name: 'Read', + tool_input: { file_path: '/tmp/x' }, + tool_use_id: 'tu-1', + }, 'tu-1') + + assert.ok(sessionCtx.pendingTools.has('tu-1')) + const toolCtx = sessionCtx.pendingTools.get('tu-1') + assert.equal(toolCtx.toolName, 'Read') + assert.equal(toolCtx.toolUseId, 'tu-1') + }) + + it('PreToolUse falls back to input.tool_use_id when toolUseId param is missing', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + hooks.PreToolUse[0].hooks[0]({ + session_id: 'sess-123', + tool_name: 'Bash', + tool_input: { command: 'ls' }, + tool_use_id: 'tu-fallback', + }) + + assert.ok(sessionCtx.pendingTools.has('tu-fallback')) + }) + + it('PreToolUse is a no-op when neither toolUseId param nor input.tool_use_id exist', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + const result = hooks.PreToolUse[0].hooks[0]({ + session_id: 'sess-123', + tool_name: 'Read', + tool_input: {}, + }) + + assert.deepStrictEqual(result, {}) + assert.equal(sessionCtx.pendingTools.size, 0) + }) + + it('PostToolUse removes tool from pendingTools and sets toolResponse', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + sessionCtx.pendingTools.set('tu-1', { toolName: 'Read', toolUseId: 'tu-1' }) + + const hooks = buildTracerHooks(sessionCtx) + + hooks.PostToolUse[0].hooks[0]({ tool_response: 'file contents' }, 'tu-1') + + assert.equal(sessionCtx.pendingTools.size, 0) + }) + + it('PostToolUse is a no-op for unknown tool IDs', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + const result = hooks.PostToolUse[0].hooks[0]({ tool_response: 'data' }, 'unknown-id') + + assert.deepStrictEqual(result, {}) + }) + + it('PostToolUseFailure removes tool from pendingTools and sets error', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const toolCtx = { toolName: 'Bash', toolUseId: 'tu-err' } + sessionCtx.pendingTools.set('tu-err', toolCtx) + + const hooks = buildTracerHooks(sessionCtx) + const err = new Error('command failed') + + hooks.PostToolUseFailure[0].hooks[0]({ error: err }, 'tu-err') + + assert.equal(sessionCtx.pendingTools.size, 0) + assert.strictEqual(toolCtx.error, err) + }) + + it('PostToolUseFailure is a no-op for unknown tool IDs', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + const result = hooks.PostToolUseFailure[0].hooks[0]( + { error: new Error('fail') }, + 'unknown-id' + ) + + assert.deepStrictEqual(result, {}) + }) + + it('SubagentStart adds a subagent context to pendingSubagents', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + hooks.SubagentStart[0].hooks[0]({ + session_id: 'sess-123', + agent_id: 'agent-abc', + agent_type: 'code-reviewer', + }) + + assert.ok(sessionCtx.pendingSubagents.has('agent-abc')) + const ctx = sessionCtx.pendingSubagents.get('agent-abc') + assert.equal(ctx.agentType, 'code-reviewer') + }) + + it('SubagentStart is a no-op when agent_id is missing', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + const result = hooks.SubagentStart[0].hooks[0]({ + session_id: 'sess-123', + }) + + assert.deepStrictEqual(result, {}) + assert.equal(sessionCtx.pendingSubagents.size, 0) + }) + + it('SubagentStop removes subagent from pendingSubagents and sets transcriptPath', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const subCtx = { agentId: 'agent-abc', agentType: 'code-reviewer' } + sessionCtx.pendingSubagents.set('agent-abc', subCtx) + + const hooks = buildTracerHooks(sessionCtx) + + hooks.SubagentStop[0].hooks[0]({ + agent_id: 'agent-abc', + agent_transcript_path: '/tmp/transcript.json', + }) + + assert.equal(sessionCtx.pendingSubagents.size, 0) + assert.equal(subCtx.transcriptPath, '/tmp/transcript.json') + }) + + it('SubagentStop is a no-op for unknown agent IDs', () => { + const sessionCtx = { pendingTools: new Map(), pendingSubagents: new Map() } + const hooks = buildTracerHooks(sessionCtx) + + const result = hooks.SubagentStop[0].hooks[0]({ + agent_id: 'unknown-agent', + agent_transcript_path: '/tmp/t.json', + }) + + assert.deepStrictEqual(result, {}) + }) + }) + + describe('wrapQuery', () => { + it('passes through to original when no subscribers', () => { + let called = false + const originalQuery = function () { + called = true + return 'result' + } + + // wrapQuery returns a new function that wraps originalQuery + const wrapped = wrapQuery(originalQuery) + + // Without subscribers (no agent.load), hasSubscribers is false + // so it should call originalQuery directly + const result = wrapped({ prompt: 'test', options: {} }) + + assert.equal(called, true) + assert.equal(result, 'result') + }) + + it('returns the wrappedQuery function with the correct name', () => { + const wrapped = wrapQuery(function original () {}) + + assert.equal(wrapped.name, 'wrappedQuery') + }) + }) + }) + + // Channel-based span tests — verify that the TracingPlugin creates and finishes + // spans when diagnostics channels fire. These do NOT require the actual SDK. + // + // NOTE: The Claude Agent SDK is pure ESM ("type": "module", "main": "sdk.mjs") + // which cannot be loaded via CJS require() in the test harness. withVersions + // is not used because it requires the SDK to be CJS-importable for .get(). + // Full ESM integration testing (shimmer wrapping the real SDK) requires a + // subprocess with --import dd-trace/initialize.mjs (deferred to follow-up). + describe('tracing', () => { + before(async () => { + await agent.load('claude-agent-sdk') + + // The Claude Agent SDK is pure ESM and can't be CJS-required in tests. + // Simulate the module load event so the plugin manager activates the + // plugin's channel subscriptions (same event register.js publishes + // when a real SDK module is loaded and version-matched). + const loadCh = channel('dd-trace:instrumentation:load') + loadCh.publish({ name: '@anthropic-ai/claude-agent-sdk' }) + }) + + after(() => agent.close({ ritmReset: false })) + + describe('session span', () => { + it('creates an agent span for a session', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.session') + assert.equal(span.meta['claude-agent-sdk.session.id'], 'test-session-123') + assert.equal(span.meta['claude-agent-sdk.session.model'], 'claude-opus-4-6') + }) + + const ctx = { + prompt: 'Hello, world!', + model: 'claude-opus-4-6', + sessionId: 'test-session-123', + permissionMode: 'default', + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + + ctx.endReason = 'completed' + sessionCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('tags resumed sessions with parent_session_id', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.session') + assert.equal( + span.meta['claude-agent-sdk.session.parent_session_id'], + 'original-session-456' + ) + }) + + const ctx = { + prompt: 'Continue from before', + model: 'claude-opus-4-6', + sessionId: 'resumed-session-789', + resume: 'original-session-456', + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + sessionCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('creates a session span with minimal fields', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.session') + assert.equal(span.meta['claude-agent-sdk.session.model'], undefined) + assert.equal(span.meta['claude-agent-sdk.session.parent_session_id'], undefined) + assert.equal(span.meta['claude-agent-sdk.session.permission_mode'], undefined) + }) + + const ctx = { + prompt: 'Hello', + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + sessionCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('tags permission_mode on the session span', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.meta['claude-agent-sdk.session.permission_mode'], 'plan') + }) + + const ctx = { + prompt: 'Plan something', + model: 'claude-opus-4-6', + sessionId: 'perm-session', + permissionMode: 'plan', + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + sessionCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('handles session error via error channel', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.session') + assert.ok(span.error) + }) + + const ctx = { + prompt: 'This will fail', + model: 'claude-opus-4-6', + sessionId: 'error-session', + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + + ctx.error = new Error('session crashed') + sessionCh.error.publish(ctx) + sessionCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + }) + + describe('turn span', () => { + it('creates a workflow span for a turn', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.turn') + assert.equal(span.meta['claude-agent-sdk.session.id'], 'test-session-123') + }) + + const ctx = { + sessionId: 'test-session-123', + prompt: 'What files are in this directory?', + } + + turnCh.start.runStores(ctx, () => { + turnCh.end.publish(ctx) + }) + + ctx.stopReason = 'end_turn' + turnCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('creates a turn span with minimal fields', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.turn') + assert.equal(span.meta['claude-agent-sdk.session.id'], undefined) + }) + + const ctx = {} + + turnCh.start.runStores(ctx, () => { + turnCh.end.publish(ctx) + }) + turnCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + }) + + describe('tool span', () => { + it('creates a tool span for a tool call', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.tool') + assert.equal(span.meta['claude-agent-sdk.tool.name'], 'Read') + assert.equal(span.meta['claude-agent-sdk.tool.use_id'], 'tool-use-abc') + }) + + const ctx = { + sessionId: 'test-session-123', + toolName: 'Read', + toolInput: { file_path: '/tmp/test.txt' }, + toolUseId: 'tool-use-abc', + } + + toolCh.start.runStores(ctx, () => { + toolCh.end.publish(ctx) + }) + + ctx.toolResponse = 'file contents here' + toolCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('tags tool errors', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.tool') + assert.equal(span.meta['claude-agent-sdk.tool.name'], 'Bash') + assert.ok(span.error) + }) + + const ctx = { + sessionId: 'test-session-123', + toolName: 'Bash', + toolInput: { command: 'false' }, + toolUseId: 'tool-use-err', + } + + toolCh.start.runStores(ctx, () => { + toolCh.end.publish(ctx) + }) + + ctx.error = new Error('command failed with exit code 1') + toolCh.error.publish(ctx) + toolCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('creates a tool span with minimal fields', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.tool') + assert.equal(span.meta['claude-agent-sdk.tool.name'], undefined) + assert.equal(span.meta['claude-agent-sdk.tool.use_id'], undefined) + assert.equal(span.meta['claude-agent-sdk.session.id'], undefined) + }) + + const ctx = {} + + toolCh.start.runStores(ctx, () => { + toolCh.end.publish(ctx) + }) + toolCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + }) + + describe('subagent span', () => { + it('creates an agent span for a subagent', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.subagent') + assert.equal(span.meta['claude-agent-sdk.subagent.id'], 'agent-xyz') + assert.equal(span.meta['claude-agent-sdk.subagent.type'], 'code-reviewer') + }) + + const ctx = { + sessionId: 'test-session-123', + agentId: 'agent-xyz', + agentType: 'code-reviewer', + } + + subagentCh.start.runStores(ctx, () => { + subagentCh.end.publish(ctx) + }) + + ctx.transcriptPath = '/tmp/transcript.json' + subagentCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + + it('creates a subagent span with minimal fields', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const span = traces[0][0] + + assert.equal(span.name, 'claude-agent-sdk.subagent') + assert.equal(span.meta['claude-agent-sdk.subagent.id'], undefined) + assert.equal(span.meta['claude-agent-sdk.subagent.type'], undefined) + assert.equal(span.meta['claude-agent-sdk.session.id'], undefined) + }) + + const ctx = {} + + subagentCh.start.runStores(ctx, () => { + subagentCh.end.publish(ctx) + }) + subagentCh.asyncEnd.publish(ctx) + + await tracesPromise + }) + }) + + describe('span hierarchy', () => { + it('nests tool span inside session span', async () => { + const tracesPromise = agent.assertSomeTraces(traces => { + const spans = traces[0] + assert.ok(spans.length >= 2, 'should have at least 2 spans') + + const sessionSpan = spans.find(s => s.name === 'claude-agent-sdk.session') + const toolSpan = spans.find(s => s.name === 'claude-agent-sdk.tool') + + assert.ok(sessionSpan, 'should have a session span') + assert.ok(toolSpan, 'should have a tool span') + assert.equal( + toolSpan.parent_id.toString(), + sessionSpan.span_id.toString(), + 'tool span should be child of session span' + ) + }) + + const sessionCtx = { + prompt: 'Do something', + model: 'claude-opus-4-6', + sessionId: 'hierarchy-session', + } + + sessionCh.start.runStores(sessionCtx, () => { + sessionCh.end.publish(sessionCtx) + + // Tool inside session context + const toolCtx = { + sessionId: 'hierarchy-session', + toolName: 'Read', + toolUseId: 'tool-hierarchy', + } + toolCh.start.runStores(toolCtx, () => { + toolCh.end.publish(toolCtx) + }) + toolCh.asyncEnd.publish(toolCtx) + }) + + sessionCh.asyncEnd.publish(sessionCtx) + + await tracesPromise + }) + }) + + // wrapQuery integration tests — exercise the subscriber code path in the shimmer. + // The shimmer unit tests above only cover the no-subscriber path because agent.load + // isn't called there. Here, agent.load has activated the channel subscribers. + describe('wrapQuery with active subscribers', () => { + const { wrapQuery } = require('../../datadog-instrumentations/src/claude-agent-sdk') + + it('calls originalQuery through the tracing path', () => { + let queryCalled = false + const originalQuery = function ({ prompt, options }) { + queryCalled = true + assert.equal(prompt, 'Subscriber test') + assert.ok(options.hooks, 'options should have merged tracer hooks') + return 'sync-result' + } + + const wrapped = wrapQuery(originalQuery) + const result = wrapped({ prompt: 'Subscriber test', options: { model: 'test-model' } }) + + assert.equal(queryCalled, true) + assert.equal(result, 'sync-result') + }) + + it('handles undefined options gracefully', () => { + const originalQuery = function ({ prompt, options }) { + assert.ok(options.hooks, 'should have tracer hooks even with no user options') + return 'default-opts' + } + + const wrapped = wrapQuery(originalQuery) + const result = wrapped({ prompt: 'No options' }) + + assert.equal(result, 'default-opts') + }) + + it('publishes error and asyncEnd when originalQuery throws', () => { + const testErr = new Error('wrapQuery sync error') + const originalQuery = function () { throw testErr } + + const wrapped = wrapQuery(originalQuery) + + assert.throws(() => { + wrapped({ prompt: 'Will throw', options: {} }) + }, (err) => err === testErr) + }) + + it('attaches rejection handler when originalQuery returns a thenable', () => { + const testErr = new Error('wrapQuery async rejection') + const originalQuery = function () { + return Promise.reject(testErr) + } + + const wrapped = wrapQuery(originalQuery) + const result = wrapped({ prompt: 'Will reject', options: {} }) + + // wrapQuery attaches its own .then(null, handler) for error/asyncEnd publishing. + // Suppress the rejection in the test to avoid unhandled rejection warnings. + return result.catch(() => {}) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/src/config/supported-configurations.json b/packages/dd-trace/src/config/supported-configurations.json index bec954421d2..443f879dbe7 100644 --- a/packages/dd-trace/src/config/supported-configurations.json +++ b/packages/dd-trace/src/config/supported-configurations.json @@ -257,6 +257,7 @@ "DD_TRACE_BUNYAN_ENABLED": ["A"], "DD_TRACE_CASSANDRA_DRIVER_ENABLED": ["A"], "DD_TRACE_CHILD_PROCESS_ENABLED": ["A"], + "DD_TRACE_CLAUDE_AGENT_SDK_ENABLED": ["A"], "DD_TRACE_CLIENT_IP_ENABLED": ["A"], "DD_TRACE_CLIENT_IP_HEADER": ["A"], "DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH": ["A"], diff --git a/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js b/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js new file mode 100644 index 00000000000..cc0e87a181c --- /dev/null +++ b/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js @@ -0,0 +1,141 @@ +'use strict' + +const LLMObsPlugin = require('../base') + +function safeStringify (value) { + if (value == null) return '' + if (typeof value === 'string') return value + try { return JSON.stringify(value) } catch { return '[unserializable]' } +} + +class SessionLLMObsPlugin extends LLMObsPlugin { + static integration = 'claude-agent-sdk' + static id = 'llmobs_claude_agent_sdk_session' + static prefix = 'tracing:apm:claude-agent-sdk:session' + + getLLMObsSpanRegisterOptions (ctx) { + const opts = { + kind: 'agent', + modelProvider: 'anthropic', + name: 'claude-agent-sdk.session', + } + if (ctx.model) opts.modelName = ctx.model + return opts + } + + setLLMObsTags (ctx) { + const span = ctx.currentStore?.span + if (!span) return + + const input = ctx.prompt || '' + this._tagger.tagTextIO(span, input, '') + + const metadata = {} + if (ctx.sessionId) metadata.session_id = ctx.sessionId + if (ctx.model) metadata.model = ctx.model + if (ctx.resume) metadata.parent_session_id = ctx.resume + if (ctx.permissionMode) metadata.permission_mode = ctx.permissionMode + if (ctx.maxTurns) metadata.max_turns = ctx.maxTurns + if (ctx.source) metadata.source = ctx.source + if (ctx.endReason) metadata.end_reason = ctx.endReason + + this._tagger.tagMetadata(span, metadata) + } +} + +class TurnLLMObsPlugin extends LLMObsPlugin { + static integration = 'claude-agent-sdk' + static id = 'llmobs_claude_agent_sdk_turn' + static prefix = 'tracing:apm:claude-agent-sdk:turn' + + getLLMObsSpanRegisterOptions () { + return { + kind: 'workflow', + modelProvider: 'anthropic', + name: 'claude-agent-sdk.turn', + } + } + + setLLMObsTags (ctx) { + const span = ctx.currentStore?.span + if (!span) return + + const input = ctx.prompt || '' + const output = ctx.stopReason || '' + this._tagger.tagTextIO(span, input, output) + + const metadata = {} + if (ctx.sessionId) metadata.session_id = ctx.sessionId + if (ctx.stopReason) metadata.stop_reason = ctx.stopReason + + this._tagger.tagMetadata(span, metadata) + } +} + +class ToolLLMObsPlugin extends LLMObsPlugin { + static integration = 'claude-agent-sdk' + static id = 'llmobs_claude_agent_sdk_tool' + static prefix = 'tracing:apm:claude-agent-sdk:tool' + + getLLMObsSpanRegisterOptions (ctx) { + return { + kind: 'tool', + modelProvider: 'anthropic', + name: ctx.toolName || 'claude-agent-sdk.tool', + } + } + + setLLMObsTags (ctx) { + const span = ctx.currentStore?.span + if (!span) return + + const input = safeStringify(ctx.toolInput) + const output = safeStringify(ctx.toolResponse) + + this._tagger.tagTextIO(span, input, output) + + const metadata = {} + if (ctx.toolName) metadata.tool_name = ctx.toolName + if (ctx.toolUseId) metadata.tool_use_id = ctx.toolUseId + if (ctx.sessionId) metadata.session_id = ctx.sessionId + + this._tagger.tagMetadata(span, metadata) + } +} + +class SubagentLLMObsPlugin extends LLMObsPlugin { + static integration = 'claude-agent-sdk' + static id = 'llmobs_claude_agent_sdk_subagent' + static prefix = 'tracing:apm:claude-agent-sdk:subagent' + + getLLMObsSpanRegisterOptions () { + return { + kind: 'agent', + modelProvider: 'anthropic', + name: 'claude-agent-sdk.subagent', + } + } + + setLLMObsTags (ctx) { + const span = ctx.currentStore?.span + if (!span) return + + const input = ctx.agentType || ctx.agentId || '' + if (input) this._tagger.tagTextIO(span, input, '') + + const metadata = {} + if (ctx.agentId) metadata.agent_id = ctx.agentId + if (ctx.agentType) metadata.agent_type = ctx.agentType + if (ctx.sessionId) metadata.session_id = ctx.sessionId + if (ctx.transcriptPath) metadata.transcript_path = ctx.transcriptPath + + this._tagger.tagMetadata(span, metadata) + } +} + +module.exports = [ + SessionLLMObsPlugin, + TurnLLMObsPlugin, + ToolLLMObsPlugin, + SubagentLLMObsPlugin, +] diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index b470dc1b1f0..99395a44f96 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -1,6 +1,7 @@ 'use strict' const plugins = { + get '@anthropic-ai/claude-agent-sdk' () { return require('../../../datadog-plugin-claude-agent-sdk/src') }, get '@anthropic-ai/sdk' () { return require('../../../datadog-plugin-anthropic/src') }, get '@apollo/gateway' () { return require('../../../datadog-plugin-apollo/src') }, get '@aws-sdk/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, diff --git a/packages/dd-trace/test/llmobs/plugins/claude-agent-sdk/index.spec.js b/packages/dd-trace/test/llmobs/plugins/claude-agent-sdk/index.spec.js new file mode 100644 index 00000000000..d923849a452 --- /dev/null +++ b/packages/dd-trace/test/llmobs/plugins/claude-agent-sdk/index.spec.js @@ -0,0 +1,348 @@ +'use strict' + +const { describe, before, it } = require('mocha') +const { tracingChannel, channel } = require('dc-polyfill') + +const { + useLlmObs, + assertLlmObsSpanEvent, +} = require('../../util') + +// Use tracingChannel to match the shimmer's channel contract. +const sessionCh = tracingChannel('apm:claude-agent-sdk:session') +const turnCh = tracingChannel('apm:claude-agent-sdk:turn') +const toolCh = tracingChannel('apm:claude-agent-sdk:tool') +const subagentCh = tracingChannel('apm:claude-agent-sdk:subagent') + +describe('Plugin', () => { + const { getEvents } = useLlmObs({ plugin: 'claude-agent-sdk' }) + + // The Claude Agent SDK is pure ESM and can't be CJS-required in tests. + // Simulate the module load event so the plugin manager activates the + // plugin's channel subscriptions (same event register.js publishes + // when a real SDK module is loaded and version-matched). + before(() => { + const loadCh = channel('dd-trace:instrumentation:load') + loadCh.publish({ name: '@anthropic-ai/claude-agent-sdk' }) + }) + + describe('claude-agent-sdk', () => { + describe('session', () => { + it('creates an agent span with session metadata', async () => { + const ctx = { + prompt: 'Fix the bug in auth.py', + model: 'claude-opus-4-6', + sessionId: 'sess-001', + permissionMode: 'default', + maxTurns: 20, + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + + ctx.endReason = 'completed' + ctx.source = 'startup' + sessionCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'agent', + name: 'claude-agent-sdk.session', + inputValue: 'Fix the bug in auth.py', + outputValue: '', + metadata: { + session_id: 'sess-001', + model: 'claude-opus-4-6', + permission_mode: 'default', + max_turns: 20, + source: 'startup', + end_reason: 'completed', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + + it('tags resumed sessions with parent_session_id', async () => { + const ctx = { + prompt: 'Continue', + model: 'claude-opus-4-6', + sessionId: 'sess-002', + resume: 'sess-001', + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + sessionCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'agent', + name: 'claude-agent-sdk.session', + inputValue: 'Continue', + outputValue: '', + metadata: { + session_id: 'sess-002', + model: 'claude-opus-4-6', + parent_session_id: 'sess-001', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + + it('creates a session span without model', async () => { + const ctx = { + prompt: 'Hello', + sessionId: 'sess-no-model', + } + + sessionCh.start.runStores(ctx, () => { + sessionCh.end.publish(ctx) + }) + sessionCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'agent', + name: 'claude-agent-sdk.session', + inputValue: 'Hello', + outputValue: '', + metadata: { + session_id: 'sess-no-model', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + }) + + describe('turn', () => { + it('creates a workflow span', async () => { + const ctx = { + sessionId: 'sess-001', + prompt: 'List all files', + } + + turnCh.start.runStores(ctx, () => { + turnCh.end.publish(ctx) + }) + + ctx.stopReason = 'end_turn' + turnCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'workflow', + name: 'claude-agent-sdk.turn', + inputValue: 'List all files', + outputValue: 'end_turn', + metadata: { + session_id: 'sess-001', + stop_reason: 'end_turn', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + }) + + describe('tool', () => { + it('creates a tool span with input and output', async () => { + const ctx = { + sessionId: 'sess-001', + toolName: 'Read', + toolInput: { file_path: '/tmp/test.txt' }, + toolUseId: 'tu-001', + } + + toolCh.start.runStores(ctx, () => { + toolCh.end.publish(ctx) + }) + + ctx.toolResponse = 'Hello from the file' + toolCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'tool', + name: 'Read', + inputValue: '{"file_path":"/tmp/test.txt"}', + outputValue: 'Hello from the file', + metadata: { + tool_name: 'Read', + tool_use_id: 'tu-001', + session_id: 'sess-001', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + + it('creates a tool span with string input', async () => { + const ctx = { + sessionId: 'sess-001', + toolName: 'Bash', + toolInput: 'echo hello', + toolUseId: 'tu-string', + } + + toolCh.start.runStores(ctx, () => { + toolCh.end.publish(ctx) + }) + + ctx.toolResponse = 'hello' + toolCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'tool', + name: 'Bash', + inputValue: 'echo hello', + outputValue: 'hello', + metadata: { + tool_name: 'Bash', + tool_use_id: 'tu-string', + session_id: 'sess-001', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + + it('falls back to default name when toolName is missing', async () => { + const ctx = { + sessionId: 'sess-001', + toolUseId: 'tu-noname', + toolInput: { key: 'value' }, + } + + toolCh.start.runStores(ctx, () => { + toolCh.end.publish(ctx) + }) + + ctx.toolResponse = 'result' + toolCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'tool', + name: 'claude-agent-sdk.tool', + inputValue: '{"key":"value"}', + outputValue: 'result', + metadata: { + tool_use_id: 'tu-noname', + session_id: 'sess-001', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + + it('handles unserializable input via safeStringify', async () => { + const circular = {} + circular.self = circular + + const ctx = { + sessionId: 'sess-001', + toolName: 'Circular', + toolInput: circular, + toolUseId: 'tu-circular', + } + + toolCh.start.runStores(ctx, () => { + toolCh.end.publish(ctx) + }) + + ctx.toolResponse = 'done' + toolCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'tool', + name: 'Circular', + inputValue: '[unserializable]', + outputValue: 'done', + metadata: { + tool_name: 'Circular', + tool_use_id: 'tu-circular', + session_id: 'sess-001', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + }) + + describe('subagent', () => { + it('creates an agent span for subagent', async () => { + const ctx = { + sessionId: 'sess-001', + agentId: 'agent-abc', + agentType: 'code-reviewer', + } + + subagentCh.start.runStores(ctx, () => { + subagentCh.end.publish(ctx) + }) + + ctx.transcriptPath = '/tmp/agent-abc-transcript.json' + subagentCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'agent', + name: 'claude-agent-sdk.subagent', + inputValue: 'code-reviewer', + outputValue: '', + metadata: { + agent_id: 'agent-abc', + agent_type: 'code-reviewer', + session_id: 'sess-001', + transcript_path: '/tmp/agent-abc-transcript.json', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + + it('creates a subagent span with minimal fields', async () => { + const ctx = { + sessionId: 'sess-001', + agentId: 'agent-minimal', + } + + subagentCh.start.runStores(ctx, () => { + subagentCh.end.publish(ctx) + }) + subagentCh.asyncEnd.publish(ctx) + + const { apmSpans, llmobsSpans } = await getEvents() + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'agent', + name: 'claude-agent-sdk.subagent', + inputValue: 'agent-minimal', + outputValue: '', + metadata: { + agent_id: 'agent-minimal', + session_id: 'sess-001', + }, + tags: { ml_app: 'test', integration: 'claude-agent-sdk' }, + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index e617ace9948..4f0dd5ce500 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -5,6 +5,7 @@ "private": true, "dependencies": { "@ai-sdk/openai": "3.0.12", + "@anthropic-ai/claude-agent-sdk": "0.2.50", "@anthropic-ai/sdk": "0.73.0", "@apollo/gateway": "2.12.2", "@apollo/server": "5.2.0",