diff --git a/packages/datadog-instrumentations/src/ai.js b/packages/datadog-instrumentations/src/ai.js index 764850f35a0..d2afcebaa23 100644 --- a/packages/datadog-instrumentations/src/ai.js +++ b/packages/datadog-instrumentations/src/ai.js @@ -2,45 +2,11 @@ const { channel, tracingChannel } = require('dc-polyfill') const shimmer = require('../../datadog-shimmer') -const { addHook } = require('./helpers/instrument') - -const toolCreationChannel = channel('dd-trace:vercel-ai:tool') - -const TRACED_FUNCTIONS = { - generateText: wrapWithTracer, - streamText: wrapWithTracer, - generateObject: wrapWithTracer, - streamObject: wrapWithTracer, - embed: wrapWithTracer, - embedMany: wrapWithTracer, - tool: wrapTool, -} +const { addHook, getHooks } = require('./helpers/instrument') const vercelAiTracingChannel = tracingChannel('dd-trace:vercel-ai') const vercelAiSpanSetAttributesChannel = channel('dd-trace:vercel-ai:span:setAttributes') -const noopTracer = { - startActiveSpan () { - const fn = arguments[arguments.length - 1] - - const span = { - spanContext () { return { traceId: '', spanId: '', traceFlags: 0 } }, - setAttribute () { return this }, - setAttributes () { return this }, - addEvent () { return this }, - addLink () { return this }, - addLinks () { return this }, - setStatus () { return this }, - updateName () { return this }, - end () { return this }, - isRecording () { return false }, - recordException () { return this }, - } - - return fn(span) - }, -} - const tracers = new WeakSet() function wrapTracer (tracer) { @@ -63,21 +29,27 @@ function wrapTracer (tracer) { arguments[arguments.length - 1] = shimmer.wrapFunction(cb, function (originalCb) { return function (span) { - shimmer.wrap(span, 'end', function (spanEnd) { + // the below is necessary in the case that the span is vercel ai's noopSpan. + // while we don't want to patch the noopSpan more than once, we do want to treat each as a + // fresh instance. However, this is really not necessary for non-noop spans, but not sure + // how to differentiate. + const freshSpan = Object.create(span) // TODO: does this cause memory leaks? + + shimmer.wrap(freshSpan, 'end', function (spanEnd) { return function () { vercelAiTracingChannel.asyncEnd.publish(ctx) return spanEnd.apply(this, arguments) } }) - shimmer.wrap(span, 'setAttributes', function (setAttributes) { + shimmer.wrap(freshSpan, 'setAttributes', function (setAttributes) { return function (attributes) { vercelAiSpanSetAttributesChannel.publish({ ctx, attributes }) return setAttributes.apply(this, arguments) } }) - shimmer.wrap(span, 'recordException', function (recordException) { + shimmer.wrap(freshSpan, 'recordException', function (recordException) { return function (exception) { ctx.error = exception vercelAiTracingChannel.error.publish(ctx) @@ -85,7 +57,7 @@ function wrapTracer (tracer) { } }) - return originalCb.apply(this, arguments) + return originalCb.call(this, freshSpan) } }) @@ -98,58 +70,50 @@ function wrapTracer (tracer) { }) } -function wrapWithTracer (fn) { - return function () { - const options = arguments[0] - - const experimentalTelemetry = options.experimental_telemetry - if (experimentalTelemetry?.isEnabled === false) { - return fn.apply(this, arguments) - } - - if (experimentalTelemetry == null) { - options.experimental_telemetry = { isEnabled: true, tracer: noopTracer } - } else { - experimentalTelemetry.isEnabled = true - experimentalTelemetry.tracer ??= noopTracer - } - - wrapTracer(options.experimental_telemetry.tracer) - - return fn.apply(this, arguments) +for (const hook of getHooks('ai')) { + if (hook.file === 'dist/index.js') { + // if not removed, the below hook will never match correctly + // however, it is still needed in the orchestrion definition + hook.file = null } -} - -function wrapTool (tool) { - return function () { - const args = arguments[0] - toolCreationChannel.publish(args) - - return tool.apply(this, arguments) - } -} -// CJS exports -addHook({ - name: 'ai', - versions: ['>=4.0.0'], -}, exports => { - for (const [fnName, patchingFn] of Object.entries(TRACED_FUNCTIONS)) { - exports = shimmer.wrap(exports, fnName, patchingFn, { replaceGetter: true }) - } + addHook(hook, exports => { + const getTracerChannel = tracingChannel('orchestrion:ai:getTracer') + getTracerChannel.subscribe({ + end (ctx) { + const { arguments: args, result: tracer } = ctx + const { isEnabled } = args[0] ?? {} - return exports -}) - -// ESM exports -addHook({ - name: 'ai', - versions: ['>=4.0.0'], - file: 'dist/index.mjs', -}, exports => { - for (const [fnName, patchingFn] of Object.entries(TRACED_FUNCTIONS)) { - exports = shimmer.wrap(exports, fnName, patchingFn, { replaceGetter: true }) - } + if (isEnabled !== false) { + wrapTracer(tracer) + } + }, + }) + + /** + * We patch this function to ensure that the telemetry attributes/tags are set always, + * even when telemetry options are not specified. This is to ensure easy use of this integration. + * + * If it is explicitly disabled, however, we will not change the options. + */ + const selectTelemetryAttributesChannel = tracingChannel('orchestrion:ai:selectTelemetryAttributes') + selectTelemetryAttributesChannel.subscribe({ + start (ctx) { + const { arguments: args } = ctx + const options = args[0] + + if (options.telemetry?.isEnabled !== false) { + args[0] = { + ...options, + telemetry: { + ...options.telemetry, + isEnabled: true, + }, + } + } + }, + }) - return exports -}) + return exports + }) +} diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/index.js b/packages/datadog-instrumentations/src/helpers/rewriter/index.js index 663548515db..66b28db92ce 100644 --- a/packages/datadog-instrumentations/src/helpers/rewriter/index.js +++ b/packages/datadog-instrumentations/src/helpers/rewriter/index.js @@ -42,6 +42,7 @@ const instrumentations = require('./instrumentations') const NODE_OPTIONS = getEnvironmentVariable('NODE_OPTIONS') +/** @type {Record>} map of module base name to supported function query versions */ const supported = {} const disabled = new Set() @@ -104,19 +105,21 @@ function disable (instrumentation) { function satisfies (filename, filePath, versions) { const [basename] = filename.split(filePath) - if (supported[basename] === undefined) { + supported[basename] ??= new Set() + + if (!supported[basename].has(versions)) { try { const pkg = JSON.parse(readFileSync( join(basename, 'package.json'), 'utf8' )) - supported[basename] = semifies(pkg.version, versions) - } catch { - supported[basename] = false - } + if (semifies(pkg.version, versions)) { + supported[basename].add(versions) + } + } catch {} } - return supported[basename] + return supported[basename].has(versions) } // TODO: Support index diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/ai.js b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/ai.js new file mode 100644 index 00000000000..5af67668946 --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/ai.js @@ -0,0 +1,103 @@ +'use strict' + +module.exports = [ + // getTracer - for patching tracer + { + module: { + name: 'ai', + versionRange: '>=4.0.0', + filePath: 'dist/index.js', + }, + functionQuery: { + functionName: 'getTracer', + kind: 'Sync', + }, + channelName: 'getTracer', + }, + { + module: { + name: 'ai', + versionRange: '>=4.0.0', + filePath: 'dist/index.mjs', + }, + functionQuery: { + functionName: 'getTracer', + kind: 'Sync', + }, + channelName: 'getTracer', + }, + // selectTelemetryAttributes - makes sure we set isEnabled properly + { + module: { + name: 'ai', + versionRange: '>=4.0.0 <6.0.0', + filePath: 'dist/index.js', + }, + functionQuery: { + functionName: 'selectTelemetryAttributes', + kind: 'Sync', + }, + channelName: 'selectTelemetryAttributes', + }, + { + module: { + name: 'ai', + versionRange: '>=4.0.0 <6.0.0', + filePath: 'dist/index.mjs', + }, + functionQuery: { + functionName: 'selectTelemetryAttributes', + kind: 'Sync', + }, + channelName: 'selectTelemetryAttributes', + }, + { + module: { + name: 'ai', + versionRange: '>=6.0.0', + filePath: 'dist/index.js', + }, + functionQuery: { + functionName: 'selectTelemetryAttributes', + kind: 'Async', + }, + channelName: 'selectTelemetryAttributes', + }, + { + module: { + name: 'ai', + versionRange: '>=6.0.0', + filePath: 'dist/index.mjs', + }, + functionQuery: { + functionName: 'selectTelemetryAttributes', + kind: 'Async', + }, + channelName: 'selectTelemetryAttributes', + }, + // tool + { + module: { + name: 'ai', + versionRange: '>=4.0.0', + filePath: 'dist/index.js', + }, + functionQuery: { + functionName: 'tool', + kind: 'Sync', + }, + channelName: 'tool', + }, + { + module: { + name: 'ai', + versionRange: '>=4.0.0', + filePath: 'dist/index.mjs', + }, + functionQuery: { + functionName: 'tool', + kind: 'Sync', + }, + channelName: 'tool', + }, +] diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js index f19a67163b4..aa384785bd8 100644 --- a/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js +++ b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js @@ -1,6 +1,7 @@ 'use strict' module.exports = [ - ...require('./langchain'), + ...require('./ai'), ...require('./bullmq'), + ...require('./langchain'), ] diff --git a/packages/datadog-plugin-ai/test/index.spec.js b/packages/datadog-plugin-ai/test/index.spec.js index f47e7a97c17..5ed23552d73 100644 --- a/packages/datadog-plugin-ai/test/index.spec.js +++ b/packages/datadog-plugin-ai/test/index.spec.js @@ -12,7 +12,13 @@ const { NODE_MAJOR } = require('../../../version') const range = NODE_MAJOR < 22 ? '>=4.0.2' : '>=4.0.0' function getAiSdkOpenAiPackage (vercelAiVersion) { - return semifies(vercelAiVersion, '>=5.0.0') ? '@ai-sdk/openai' : '@ai-sdk/openai@1.3.23' + if (semifies(vercelAiVersion, '>=6.0.0')) { + return '@ai-sdk/openai' + } else if (semifies(vercelAiVersion, '>=5.0.0')) { + return '@ai-sdk/openai@2.0.0' + } else { + return '@ai-sdk/openai@1.3.23' + } } // making a different reference from the default no-op tracer in the instrumentation @@ -116,7 +122,6 @@ describe('Plugin', () => { }) assert.ok(result.text, 'Expected result to be truthy') - assert.ok(experimentalTelemetry.tracer != null, 'Tracer should be set when `isEnabled` is true') await checkTraces }) @@ -157,7 +162,6 @@ describe('Plugin', () => { }) assert.ok(result.text, 'Expected result to be truthy') - assert.ok(experimentalTelemetry.isEnabled, 'isEnabled should be set to true') assert.ok(experimentalTelemetry.tracer === myTracer, 'Tracer should be set when `isEnabled` is true') await checkTraces diff --git a/packages/dd-trace/src/llmobs/plugins/ai/index.js b/packages/dd-trace/src/llmobs/plugins/ai/index.js index 86abe1a74f2..55efcb0f929 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/index.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/index.js @@ -4,7 +4,7 @@ const { channel } = require('dc-polyfill') const BaseLLMObsPlugin = require('../base') const { getModelProvider } = require('../../../../../datadog-plugin-ai/src/utils') -const toolCreationCh = channel('dd-trace:vercel-ai:tool') +const toolCreationCh = channel('tracing:orchestrion:ai:tool:start') const setAttributesCh = channel('dd-trace:vercel-ai:span:setAttributes') const { MODEL_NAME, MODEL_PROVIDER, NAME } = require('../../constants/tags') @@ -94,8 +94,10 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin { this.#toolCallIdsToName = {} this.#availableTools = new Set() - toolCreationCh.subscribe(toolArgs => { - this.#availableTools.add(toolArgs) + toolCreationCh.subscribe(ctx => { + const toolArgs = ctx.arguments + const tool = toolArgs[0] ?? {} + this.#availableTools.add(tool) }) setAttributesCh.subscribe(({ ctx, attributes }) => { diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_2097b7a8.yaml b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_2097b7a8.yaml new file mode 100644 index 00000000000..9f884ccf692 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_2097b7a8.yaml @@ -0,0 +1,136 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","input":[{"role":"system","content":"You are a helpful + assistant"},{"role":"user","content":[{"type":"input_text","text":"What is the + weather in Tokyo?"}]}],"store":false,"tools":[{"type":"function","name":"weather","description":"Get + the weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The + location to get the weather for"}}}}],"tool_choice":"auto","stream":true}' + headers: + ? !!python/object/apply:multidict._multidict.istr + - Accept + : - '*/*' + ? !!python/object/apply:multidict._multidict.istr + - Accept-Encoding + : - gzip, deflate + ? !!python/object/apply:multidict._multidict.istr + - Accept-Language + : - '*' + ? !!python/object/apply:multidict._multidict.istr + - Connection + : - keep-alive + Content-Length: + - '455' + ? !!python/object/apply:multidict._multidict.istr + - Content-Type + : - application/json + ? !!python/object/apply:multidict._multidict.istr + - User-Agent + : - ai-sdk/openai/3.0.12 ai-sdk/provider-utils/4.0.8 runtime/node.js/22 + ? !!python/object/apply:multidict._multidict.istr + - sec-fetch-mode + : - cors + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_09306f386f8ee2650169961e90ebe881a3b3dbcbc2d92cf086","object":"response","created_at":1771445904,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Get + the weather in a given location","name":"weather","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The + location to get the weather for"}},"additionalProperties":false,"required":["location"]},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_09306f386f8ee2650169961e90ebe881a3b3dbcbc2d92cf086","object":"response","created_at":1771445904,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Get + the weather in a given location","name":"weather","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The + location to get the weather for"}},"additionalProperties":false,"required":["location"]},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","type":"function_call","status":"in_progress","arguments":"","call_id":"call_XGGwP1jrWvsIWpiBa8XikiVm","name":"weather"},"output_index":0,"sequence_number":2} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","obfuscation":"8QjEihq4cumLYh","output_index":0,"sequence_number":3} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"location","item_id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","obfuscation":"vPj7MjmD","output_index":0,"sequence_number":4} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","obfuscation":"ff2lpqDbA033y","output_index":0,"sequence_number":5} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"Tokyo","item_id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","obfuscation":"wFoDfJBjBga","output_index":0,"sequence_number":6} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","obfuscation":"t2Bc6HBFVkX3Ek","output_index":0,"sequence_number":7} + + + event: response.function_call_arguments.done + + data: {"type":"response.function_call_arguments.done","arguments":"{\"location\":\"Tokyo\"}","item_id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","output_index":0,"sequence_number":8} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","type":"function_call","status":"completed","arguments":"{\"location\":\"Tokyo\"}","call_id":"call_XGGwP1jrWvsIWpiBa8XikiVm","name":"weather"},"output_index":0,"sequence_number":9} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_09306f386f8ee2650169961e90ebe881a3b3dbcbc2d92cf086","object":"response","created_at":1771445904,"status":"completed","background":false,"completed_at":1771445905,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d","type":"function_call","status":"completed","arguments":"{\"location\":\"Tokyo\"}","call_id":"call_XGGwP1jrWvsIWpiBa8XikiVm","name":"weather"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Get + the weather in a given location","name":"weather","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The + location to get the weather for"}},"additionalProperties":false,"required":["location"]},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":63,"input_tokens_details":{"cached_tokens":0},"output_tokens":14,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":77},"user":null,"metadata":{}},"sequence_number":10} + + + ' + headers: + CF-RAY: + - 9d0036a87851a0fb-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 18 Feb 2026 20:18:25 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - datadog-staging + openai-processing-ms: + - '47' + openai-project: + - proj_gt6TQZPRbZfoY2J9AQlEJMpd + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=gSDiqWOnBKwjpgHgOX40DKxupXKcUv3bRecY7dfnNFk-1771445904.7133994-1.0.1.1-8p2yOgzsGAqpaBOrvdmZwh01ybSoMy8MeG.Pck24lAJAUaaLQjyL4mZizRCo_Zyo6eW00gGIKy5goScYRRqZrmp5aDcoY0tyjhyxtXDK0tdDYEmCD9KJB43PHuBEOsUq; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 18 Feb 2026 + 20:48:25 GMT + x-request-id: + - req_998a9c780ae34fd58c43821cb0d1dc3d + status: + code: 200 + message: OK +version: 1 diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_25afe662.yaml b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_25afe662.yaml new file mode 100644 index 00000000000..354a33aa835 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_25afe662.yaml @@ -0,0 +1,117 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","input":[{"role":"system","content":"You are a helpful + assistant"},{"role":"user","content":[{"type":"input_text","text":"What is the + weather in Tokyo?"}]},{"type":"function_call","call_id":"call_NvLCkOSGXSGU6TjvCG1wvtRI","name":"weather","arguments":"{\"location\":\"Tokyo\"}","id":"fc_0f98d1be00d90a980169961e351bd08192a4376b726f67ecd5"},{"type":"function_call_output","call_id":"call_NvLCkOSGXSGU6TjvCG1wvtRI","output":"{\"location\":\"Tokyo\",\"temperature\":72}"}],"store":false,"tools":[{"type":"function","name":"weather","description":"Get + the weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The + location to get the weather for"}}}}],"tool_choice":"auto"}' + headers: + ? !!python/object/apply:multidict._multidict.istr + - Accept + : - '*/*' + ? !!python/object/apply:multidict._multidict.istr + - Accept-Encoding + : - gzip, deflate + ? !!python/object/apply:multidict._multidict.istr + - Accept-Language + : - '*' + ? !!python/object/apply:multidict._multidict.istr + - Connection + : - keep-alive + Content-Length: + - '754' + ? !!python/object/apply:multidict._multidict.istr + - Content-Type + : - application/json + ? !!python/object/apply:multidict._multidict.istr + - User-Agent + : - ai/6.0.39 ai-sdk/provider-utils/4.0.8 runtime/node.js/22 + ? !!python/object/apply:multidict._multidict.istr + - sec-fetch-mode + : - cors + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0f98d1be00d90a980169961e3580ac819280313125aadb1624\",\n + \ \"object\": \"response\",\n \"created_at\": 1771445813,\n \"status\": + \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": + \"developer\"\n },\n \"completed_at\": 1771445813,\n \"error\": null,\n + \ \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4o-mini-2024-07-18\",\n \"output\": [\n {\n \"id\": \"msg_0f98d1be00d90a980169961e35d05c819287ec9300fe21fa00\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"The current temperature + in Tokyo is 72\\u00b0F.\"\n }\n ],\n \"role\": \"assistant\"\n + \ }\n ],\n \"parallel_tool_calls\": true,\n \"presence_penalty\": 0.0,\n + \ \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": + null,\n \"reasoning\": {\n \"effort\": null,\n \"summary\": null\n + \ },\n \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": + false,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": + \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": + \"auto\",\n \"tools\": [\n {\n \"type\": \"function\",\n \"description\": + \"Get the weather in a given location\",\n \"name\": \"weather\",\n \"parameters\": + {\n \"type\": \"object\",\n \"properties\": {\n \"location\": + {\n \"type\": \"string\",\n \"description\": \"The location + to get the weather for\"\n }\n },\n \"additionalProperties\": + false,\n \"required\": [\n \"location\"\n ]\n },\n + \ \"strict\": true\n }\n ],\n \"top_logprobs\": 0,\n \"top_p\": + 1.0,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": + 90,\n \"input_tokens_details\": {\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 12,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 102\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + CF-RAY: + - 9d00346e2a7b7d1e-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 18 Feb 2026 20:16:54 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - datadog-staging + openai-processing-ms: + - '452' + openai-project: + - proj_gt6TQZPRbZfoY2J9AQlEJMpd + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=wG2AqPW2a7LeXBBXltAYsrAOtwf3N62IF5aqRCknSmM-1771445813.4690728-1.0.1.1-TYDs8FW7X2TvKP5hb94ASi9Y4vxTMt7PFjhXBjqYWQwSuGClkLbrAsnn2mdrgZRfMxifwhfiKgNa8HOom4j9OvDhllnjd5Ntm2CMx9.oy0WZyVJQ01mP_pHJwMwgYYFY; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 18 Feb 2026 + 20:46:54 GMT + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999692' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_66869dd3f1d04f5583d5af32b5ae3e6e + status: + code: 200 + message: OK +version: 1 diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_e812b0c6.yaml b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_e812b0c6.yaml new file mode 100644 index 00000000000..3832e38fedd --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_e812b0c6.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","input":[{"role":"system","content":"You are a helpful + assistant"},{"role":"user","content":[{"type":"input_text","text":"What is the + weather in Tokyo?"}]},{"type":"function_call","call_id":"call_XGGwP1jrWvsIWpiBa8XikiVm","name":"weather","arguments":"{\"location\":\"Tokyo\"}","id":"fc_09306f386f8ee2650169961e91861481a3b87785843364299d"},{"type":"function_call_output","call_id":"call_XGGwP1jrWvsIWpiBa8XikiVm","output":"{\"location\":\"Tokyo\",\"temperature\":72}"}],"store":false,"tools":[{"type":"function","name":"weather","description":"Get + the weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The + location to get the weather for"}}}}],"tool_choice":"auto","stream":true}' + headers: + ? !!python/object/apply:multidict._multidict.istr + - Accept + : - '*/*' + ? !!python/object/apply:multidict._multidict.istr + - Accept-Encoding + : - gzip, deflate + ? !!python/object/apply:multidict._multidict.istr + - Accept-Language + : - '*' + ? !!python/object/apply:multidict._multidict.istr + - Connection + : - keep-alive + Content-Length: + - '768' + ? !!python/object/apply:multidict._multidict.istr + - Content-Type + : - application/json + ? !!python/object/apply:multidict._multidict.istr + - User-Agent + : - ai-sdk/openai/3.0.12 ai-sdk/provider-utils/4.0.8 runtime/node.js/22 + ? !!python/object/apply:multidict._multidict.istr + - sec-fetch-mode + : - cors + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_09306f386f8ee2650169961e91f56481a3bd6c6c30cab116ef\",\"object\":\"response\",\"created_at\":1771445905,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get + the weather in a given location\",\"name\":\"weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The + location to get the weather for\"}},\"additionalProperties\":false,\"required\":[\"location\"]},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: + response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_09306f386f8ee2650169961e91f56481a3bd6c6c30cab116ef\",\"object\":\"response\",\"created_at\":1771445905,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get + the weather in a given location\",\"name\":\"weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The + location to get the weather for\"}},\"additionalProperties\":false,\"required\":[\"location\"]},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: + response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: + response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"TFZl8hWtrOCuz\",\"output_index\":0,\"sequence_number\":4}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + current\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"Zf9vSEqF\",\"output_index\":0,\"sequence_number\":5}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + temperature\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"Ka74\",\"output_index\":0,\"sequence_number\":6}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + in\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"brVWrKOTi2DYh\",\"output_index\":0,\"sequence_number\":7}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + Tokyo\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"2c9pZtqzMW\",\"output_index\":0,\"sequence_number\":8}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + is\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"r81kobPcEC2tc\",\"output_index\":0,\"sequence_number\":9}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"8Oqi7A2GODxv8dk\",\"output_index\":0,\"sequence_number\":10}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"72\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"ikt2hOC96l94Ow\",\"output_index\":0,\"sequence_number\":11}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\xB0F\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"TXTgYs0FirjE5E\",\"output_index\":0,\"sequence_number\":12}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"9CYQpKDe3P8aHIH\",\"output_index\":0,\"sequence_number\":13}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + If\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"ez0BdI9LnUWtF\",\"output_index\":0,\"sequence_number\":14}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + you\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"DMPHxnKBiXPw\",\"output_index\":0,\"sequence_number\":15}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + need\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"PUAO0fsVRFi\",\"output_index\":0,\"sequence_number\":16}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + more\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"sy0enpp5As7\",\"output_index\":0,\"sequence_number\":17}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + specific\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"B0K8iPv\",\"output_index\":0,\"sequence_number\":18}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + details\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"jKZcAQPs\",\"output_index\":0,\"sequence_number\":19}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + about\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"SnyUI6EN9C\",\"output_index\":0,\"sequence_number\":20}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + the\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"uNYboxhF0ehG\",\"output_index\":0,\"sequence_number\":21}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + weather\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"abe003fo\",\"output_index\":0,\"sequence_number\":22}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"V2mKoKEy5jvDKYS\",\"output_index\":0,\"sequence_number\":23}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + feel\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"OS3hF3oHXva\",\"output_index\":0,\"sequence_number\":24}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + free\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"COBLH4C6TQt\",\"output_index\":0,\"sequence_number\":25}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + to\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"eQUSrkJUJ7Oyz\",\"output_index\":0,\"sequence_number\":26}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + ask\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"PWefp2pT6sZO\",\"output_index\":0,\"sequence_number\":27}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"!\",\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"obfuscation\":\"zway3P6i9iMjzXm\",\"output_index\":0,\"sequence_number\":28}\n\nevent: + response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":29,\"text\":\"The + current temperature in Tokyo is 72\xB0F. If you need more specific details + about the weather, feel free to ask!\"}\n\nevent: response.content_part.done\ndata: + {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + current temperature in Tokyo is 72\xB0F. If you need more specific details + about the weather, feel free to ask!\"},\"sequence_number\":30}\n\nevent: + response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + current temperature in Tokyo is 72\xB0F. If you need more specific details + about the weather, feel free to ask!\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":31}\n\nevent: + response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_09306f386f8ee2650169961e91f56481a3bd6c6c30cab116ef\",\"object\":\"response\",\"created_at\":1771445905,\"status\":\"completed\",\"background\":false,\"completed_at\":1771445906,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"msg_09306f386f8ee2650169961e924f1881a399d3a1aff5cec458\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + current temperature in Tokyo is 72\xB0F. If you need more specific details + about the weather, feel free to ask!\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get + the weather in a given location\",\"name\":\"weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The + location to get the weather for\"}},\"additionalProperties\":false,\"required\":[\"location\"]},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":90,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":27,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":117},\"user\":null,\"metadata\":{}},\"sequence_number\":32}\n\n" + headers: + CF-RAY: + - 9d0036b008e98110-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 18 Feb 2026 20:18:26 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - datadog-staging + openai-processing-ms: + - '57' + openai-project: + - proj_gt6TQZPRbZfoY2J9AQlEJMpd + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=2IsGHIF973._I5Sir_Tu_fzmpS6Q7t3Cojra28wcSkc-1771445905.9225588-1.0.1.1-5_Bkdgc7cHzqOWXp9w3YdVnBibeEcoGSovKbTShoKxaiTMT8S9mvAROozUSgnwwmf7uuZtgIY.HLm8HL0xhm8O_wgOn5Hab1EYeoJ0AsKvatwo5O8mxNXn5i16voulMc; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 18 Feb 2026 + 20:48:26 GMT + x-request-id: + - req_709538fd1fff445ca7e57ea353e6e742 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_f5f637a6.yaml b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_f5f637a6.yaml new file mode 100644 index 00000000000..4ed53f84ec2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_f5f637a6.yaml @@ -0,0 +1,115 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","input":[{"role":"system","content":"You are a helpful + assistant"},{"role":"user","content":[{"type":"input_text","text":"What is the + weather in Tokyo?"}]}],"store":false,"tools":[{"type":"function","name":"weather","description":"Get + the weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The + location to get the weather for"}}}}],"tool_choice":"auto"}' + headers: + ? !!python/object/apply:multidict._multidict.istr + - Accept + : - '*/*' + ? !!python/object/apply:multidict._multidict.istr + - Accept-Encoding + : - gzip, deflate + ? !!python/object/apply:multidict._multidict.istr + - Accept-Language + : - '*' + ? !!python/object/apply:multidict._multidict.istr + - Connection + : - keep-alive + Content-Length: + - '441' + ? !!python/object/apply:multidict._multidict.istr + - Content-Type + : - application/json + ? !!python/object/apply:multidict._multidict.istr + - User-Agent + : - ai/6.0.39 ai-sdk/provider-utils/4.0.8 runtime/node.js/22 + ? !!python/object/apply:multidict._multidict.istr + - sec-fetch-mode + : - cors + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0f98d1be00d90a980169961e34a8a08192a386d48964088873\",\n + \ \"object\": \"response\",\n \"created_at\": 1771445812,\n \"status\": + \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": + \"developer\"\n },\n \"completed_at\": 1771445813,\n \"error\": null,\n + \ \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4o-mini-2024-07-18\",\n \"output\": [\n {\n \"id\": \"fc_0f98d1be00d90a980169961e351bd08192a4376b726f67ecd5\",\n + \ \"type\": \"function_call\",\n \"status\": \"completed\",\n \"arguments\": + \"{\\\"location\\\":\\\"Tokyo\\\"}\",\n \"call_id\": \"call_NvLCkOSGXSGU6TjvCG1wvtRI\",\n + \ \"name\": \"weather\"\n }\n ],\n \"parallel_tool_calls\": true,\n + \ \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_retention\": null,\n \"reasoning\": {\n \"effort\": + null,\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": + \"default\",\n \"store\": false,\n \"temperature\": 1.0,\n \"text\": {\n + \ \"format\": {\n \"type\": \"text\"\n },\n \"verbosity\": \"medium\"\n + \ },\n \"tool_choice\": \"auto\",\n \"tools\": [\n {\n \"type\": + \"function\",\n \"description\": \"Get the weather in a given location\",\n + \ \"name\": \"weather\",\n \"parameters\": {\n \"type\": \"object\",\n + \ \"properties\": {\n \"location\": {\n \"type\": + \"string\",\n \"description\": \"The location to get the weather + for\"\n }\n },\n \"additionalProperties\": false,\n + \ \"required\": [\n \"location\"\n ]\n },\n \"strict\": + true\n }\n ],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": + \"disabled\",\n \"usage\": {\n \"input_tokens\": 63,\n \"input_tokens_details\": + {\n \"cached_tokens\": 0\n },\n \"output_tokens\": 14,\n \"output_tokens_details\": + {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 77\n },\n + \ \"user\": null,\n \"metadata\": {}\n}" + headers: + CF-RAY: + - 9d00346729d7a62e-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 18 Feb 2026 20:16:53 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - datadog-staging + openai-processing-ms: + - '568' + openai-project: + - proj_gt6TQZPRbZfoY2J9AQlEJMpd + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=_FtUDrpxnc9zfYnrBRLAWzlFqRkZjU0G0djDAO0Nrf4-1771445812.3412933-1.0.1.1-dmBwu0gl7184DnwMm27wxEZdAkxWSHJfO2Vs.KQPOh9aHYOEYS_0HYGU4HNdIiUVRJt7k53_xqGxSyQN45Ur4dSvRLjE3VYEiSCJYL34AT6KbGHX5W2FimoLdbHl1M5o; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 18 Feb 2026 + 20:46:53 GMT + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999720' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_f3e8a14a3fe24c39ab0efa1a20def486 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index bc3a4cb6c62..3c9ebefcc33 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -3,9 +3,12 @@ const semifies = require('semifies') const { useEnv } = require('../../../../../../integration-tests/helpers') const { withVersions } = require('../../../setup/mocha') +const iastFilter = require('../../../../src/appsec/iast/taint-tracking/filter') const { NODE_MAJOR } = require('../../../../../../version') +const isDdTrace = iastFilter.isDdTrace + const { assertLlmObsSpanEvent, MOCK_STRING, @@ -18,7 +21,13 @@ const { const range = NODE_MAJOR < 22 ? '>=4.0.2' : '>=4.0.0' function getAiSdkOpenAiPackage (vercelAiVersion) { - return semifies(vercelAiVersion, '>=5.0.0') ? '@ai-sdk/openai' : '@ai-sdk/openai@1.3.23' + if (semifies(vercelAiVersion, '>=6.0.0')) { + return '@ai-sdk/openai' + } else if (semifies(vercelAiVersion, '>=5.0.0')) { + return '@ai-sdk/openai@2.0.0' + } else { + return '@ai-sdk/openai@1.3.23' + } } describe('Plugin', () => { @@ -28,6 +37,19 @@ describe('Plugin', () => { const { getEvents } = useLlmObs({ plugin: 'ai' }) + before(async () => { + iastFilter.isDdTrace = file => { + if (file.includes('dd-trace-js/versions/')) { + return false + } + return isDdTrace(file) + } + }) + + after(() => { + iastFilter.isDdTrace = isDdTrace + }) + withVersions('ai', 'ai', range, (version, _, realVersion) => { let ai let openai @@ -711,5 +733,252 @@ describe('Plugin', () => { tags: { ml_app: 'test', integration: 'ai' }, }) }) + + describe('ToolLoopAgent', function () { + beforeEach(function () { + if (semifies(realVersion, '<6.0.0')) { + this.skip() + } + }) + + it('creates a text generation root span for ToolLoopAgent.generate', async () => { + const agent = new ai.ToolLoopAgent({ + model: openai('gpt-4o-mini'), + instructions: 'You are a helpful assistant', + providerOptions: { + openai: { + store: false, + }, + }, + tools: { + weather: ai.tool({ + description: 'Get the weather in a given location', + inputSchema: ai.jsonSchema({ + type: 'object', + properties: { + location: { type: 'string', description: 'The location to get the weather for' }, + }, + }), + execute: async ({ location }) => ({ + location, + temperature: 72, + }), + }), + }, + }) + + const result = await agent.generate({ + prompt: 'What is the weather in Tokyo?', + }) + + const toolCallId = result.steps[0].toolCalls[0].toolCallId + + const { apmSpans, llmobsSpans } = await getEvents(4) + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + name: 'generateText', + spanKind: 'workflow', + inputValue: 'What is the weather in Tokyo?', + outputValue: MOCK_STRING, + metadata: { + maxRetries: MOCK_NUMBER, + }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + + assertLlmObsSpanEvent(llmobsSpans[1], { + span: apmSpans[1], + parentId: llmobsSpans[0].span_id, + spanKind: 'llm', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', + name: 'doGenerate', + inputMessages: [ + { content: 'You are a helpful assistant', role: 'system' }, + { content: 'What is the weather in Tokyo?', role: 'user' }, + ], + outputMessages: [{ + role: 'assistant', + tool_calls: [{ + tool_id: toolCallId, + name: 'weather', + arguments: { + location: 'Tokyo', + }, + type: 'function', + }], + }], + metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + + assertLlmObsSpanEvent(llmobsSpans[2], { + span: apmSpans[2], + parentId: llmobsSpans[0].span_id, + name: 'weather', + spanKind: 'tool', + inputValue: JSON.stringify({ location: 'Tokyo' }), + outputValue: JSON.stringify({ location: 'Tokyo', temperature: 72 }), + tags: { ml_app: 'test', integration: 'ai' }, + }) + + assertLlmObsSpanEvent(llmobsSpans[3], { + span: apmSpans[3], + parentId: llmobsSpans[0].span_id, + spanKind: 'llm', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', + name: 'doGenerate', + inputMessages: [ + { content: 'You are a helpful assistant', role: 'system' }, + { content: 'What is the weather in Tokyo?', role: 'user' }, + { + content: '', + role: 'assistant', + tool_calls: [{ + tool_id: toolCallId, + name: 'weather', + arguments: { + location: 'Tokyo', + }, + type: 'function', + }], + }, + { + content: JSON.stringify({ location: 'Tokyo', temperature: 72 }), + role: 'tool', + tool_id: toolCallId, + }, + ], + outputMessages: [{ content: MOCK_STRING, role: 'assistant' }], + metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + }) + + it('creates a text generation root span for ToolLoopAgent.stream', async () => { + const agent = new ai.ToolLoopAgent({ + model: openai('gpt-4o-mini'), + instructions: 'You are a helpful assistant', + providerOptions: { + openai: { + store: false, + }, + }, + tools: { + weather: ai.tool({ + description: 'Get the weather in a given location', + inputSchema: ai.jsonSchema({ + type: 'object', + properties: { + location: { type: 'string', description: 'The location to get the weather for' }, + }, + }), + execute: async ({ location }) => ({ + location, + temperature: 72, + }), + }), + }, + }) + + const result = await agent.stream({ + prompt: 'What is the weather in Tokyo?', + }) + + const textStream = result.textStream + + for await (const part of textStream) {} // eslint-disable-line + + const stepsPromise = result._steps ?? result.stepsPromise + const steps = stepsPromise.status.value + const toolCallId = steps[0].toolCalls[0].toolCallId + + const { apmSpans, llmobsSpans } = await getEvents(4) + + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + name: 'streamText', + spanKind: 'workflow', + inputValue: 'What is the weather in Tokyo?', + outputValue: MOCK_STRING, + metadata: { + maxRetries: MOCK_NUMBER, + }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + + assertLlmObsSpanEvent(llmobsSpans[1], { + span: apmSpans[1], + parentId: llmobsSpans[0].span_id, + spanKind: 'llm', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', + name: 'doStream', + inputMessages: [ + { content: 'You are a helpful assistant', role: 'system' }, + { content: 'What is the weather in Tokyo?', role: 'user' }, + ], + outputMessages: [{ + role: 'assistant', + content: MOCK_STRING, + tool_calls: [{ + tool_id: toolCallId, + name: 'weather', + arguments: { + location: 'Tokyo', + }, + type: 'function', + }], + }], + metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + + assertLlmObsSpanEvent(llmobsSpans[2], { + span: apmSpans[2], + parentId: llmobsSpans[0].span_id, + name: 'weather', + spanKind: 'tool', + inputValue: JSON.stringify({ location: 'Tokyo' }), + outputValue: JSON.stringify({ location: 'Tokyo', temperature: 72 }), + tags: { ml_app: 'test', integration: 'ai' }, + }) + + assertLlmObsSpanEvent(llmobsSpans[3], { + span: apmSpans[3], + parentId: llmobsSpans[0].span_id, + spanKind: 'llm', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', + name: 'doStream', + inputMessages: [ + { content: 'You are a helpful assistant', role: 'system' }, + { content: 'What is the weather in Tokyo?', role: 'user' }, + { + content: '', + role: 'assistant', + tool_calls: [{ + tool_id: toolCallId, + name: 'weather', + arguments: { + location: 'Tokyo', + }, + type: 'function', + }], + }, + { + content: JSON.stringify({ location: 'Tokyo', temperature: 72 }), + role: 'tool', + tool_id: toolCallId, + }, + ], + outputMessages: [{ content: MOCK_STRING, role: 'assistant' }], + metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + }) + }) }) }) diff --git a/packages/dd-trace/test/plugins/externals.js b/packages/dd-trace/test/plugins/externals.js index e45b3ed099f..73441dc3871 100644 --- a/packages/dd-trace/test/plugins/externals.js +++ b/packages/dd-trace/test/plugins/externals.js @@ -8,7 +8,7 @@ module.exports = { }, { name: '@ai-sdk/openai', - versions: ['>=1.3.23'], + versions: ['>=1.3.23', '>=2.0.0'], }, { name: 'zod',