From cbd622682457282712ff382f2da38ca40419e695 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Feb 2026 10:30:30 +0100 Subject: [PATCH 01/39] llmobs: set model tag even when llmobs disabled --- .../openai_java/ChatCompletionDecorator.java | 18 +++++++------ .../openai_java/CompletionDecorator.java | 27 ++++++++++--------- .../openai_java/EmbeddingDecorator.java | 19 ++++++------- .../openai_java/ResponseDecorator.java | 23 +++++++--------- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 6c9e9cad9d9..74f14a9daf2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -31,11 +31,6 @@ public void withChatCompletionCreateParams( AgentSpan span, ChatCompletionCreateParams params, boolean stream) { span.setResourceName(CHAT_COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/chat/completions"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -45,6 +40,12 @@ public void withChatCompletionCreateParams( .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + span.setTag( CommonTags.INPUT, params.messages().stream() @@ -97,13 +98,14 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { } public void withChatCompletion(AgentSpan span, ChatCompletion completion) { - if (!llmObsEnabled) { - return; - } String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() .map(ChatCompletionDecorator::llmMessage) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 2291c860d00..51e7c12ef1c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -23,11 +23,6 @@ public class CompletionDecorator { public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams params) { span.setResourceName(COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/completions"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -37,6 +32,12 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa ._value() .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); params .prompt() .flatMap(p -> p.string()) @@ -61,14 +62,14 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa } public void withCompletion(AgentSpan span, Completion completion) { - if (!llmObsEnabled) { - return; - } - String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() .map(v -> LLMObs.LLMMessage.from(null, v.text())) @@ -86,10 +87,6 @@ public void withCompletion(AgentSpan span, Completion completion) { } public void withCompletions(AgentSpan span, List completions) { - if (!llmObsEnabled) { - return; - } - if (completions.isEmpty()) { return; } @@ -99,6 +96,10 @@ public void withCompletions(AgentSpan span, List completions) { span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + Map textByChoiceIndex = new HashMap<>(); for (Completion completion : completions) { completion diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 88098358ebf..02d4588358c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -25,11 +25,6 @@ public class EmbeddingDecorator { public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams params) { span.setResourceName(EMBEDDINGS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/embeddings"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); if (params == null) { return; } @@ -39,6 +34,12 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); + span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); Map metadata = new HashMap<>(); @@ -59,14 +60,14 @@ private List embeddingDocuments(EmbeddingCreateParams.Input inp } public void withCreateEmbeddingResponse(AgentSpan span, CreateEmbeddingResponse response) { - if (!llmObsEnabled) { - return; - } - String modelName = response.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + if (!response.data().isEmpty()) { int embeddingCount = response.data().size(); Embedding firstEmbedding = response.data().get(0); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 4bc95e32934..db2258785d4 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -37,11 +37,6 @@ public class ResponseDecorator { public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params) { span.setResourceName(RESPONSES_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/responses"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -51,6 +46,12 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params String modelName = extractResponseModel(params._model()); span.setTag(CommonTags.OPENAI_REQUEST_MODEL, modelName); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + List inputMessages = new ArrayList<>(); params @@ -369,10 +370,6 @@ public void withResponse(AgentSpan span, Response response) { } public void withResponseStreamEvents(AgentSpan span, List events) { - if (!llmObsEnabled) { - return; - } - for (ResponseStreamEvent event : events) { if (event.isCompleted()) { Response response = event.asCompleted().response(); @@ -388,14 +385,14 @@ public void withResponseStreamEvents(AgentSpan span, List e } private void withResponse(AgentSpan span, Response response, boolean stream) { - if (!llmObsEnabled) { - return; - } - String modelName = extractResponseModel(response._model()); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List outputMessages = extractResponseOutputMessages(response.output()); if (!outputMessages.isEmpty()) { span.setTag(CommonTags.OUTPUT, outputMessages); From 4f2767372d0c37dae170c3f964c317295ac93b14 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 13:30:21 +0100 Subject: [PATCH 02/39] Set metadata.stream tag no matter it's true or false --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 74f14a9daf2..eb918ef3886 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -56,9 +56,7 @@ public void withChatCompletionCreateParams( // maxTokens is deprecated but integration tests missing to provide maxCompletionTokens params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); params.temperature().ifPresent(v -> metadata.put("temperature", v)); - if (stream) { - metadata.put("stream", true); - } + metadata.put("stream", stream); params .streamOptions() .ifPresent( From d128d6baddaf647fa34ad6fa11c7be0672547b15 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 13:46:41 +0100 Subject: [PATCH 03/39] Set chat/completion CACHE_READ_INPUT_TOKENS tag --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index eb918ef3886..95c746880cf 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -117,6 +117,10 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); span.setTag(CommonTags.OUTPUT_TOKENS, usage.completionTokens()); span.setTag(CommonTags.TOTAL_TOKENS, usage.totalTokens()); + usage + .promptTokensDetails() + .flatMap(details -> details.cachedTokens()) + .ifPresent(v -> span.setTag(CommonTags.CACHE_READ_INPUT_TOKENS, v)); }); } From 3fc5cebce75f121245d4b850445069741ff23d52 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 18:31:44 +0100 Subject: [PATCH 04/39] Set error nad error_type tags --- .../trace/instrumentation/openai_java/CommonTags.java | 3 +++ .../trace/instrumentation/openai_java/OpenAiDecorator.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index f779662ac0d..a4335a7edfe 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -21,6 +21,9 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String VERSION = TAG_PREFIX + "version"; + String ERROR = TAG_PREFIX + "error"; + String ERROR_TYPE = TAG_PREFIX + "error_type"; + String ENV = TAG_PREFIX + "env"; String SERVICE = TAG_PREFIX + "service"; String PARENT_ID = TAG_PREFIX + "parent_id"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 331269bad83..2332b578ebf 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -3,6 +3,7 @@ import com.openai.core.ClientOptions; import com.openai.core.http.Headers; import datadog.trace.api.Config; +import datadog.trace.api.DDTags; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -12,6 +13,7 @@ import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; +import datadog.trace.core.CoreSpan; import java.util.List; public class OpenAiDecorator extends ClientDecorator { @@ -111,6 +113,9 @@ public AgentSpan afterStart(AgentSpan span) { @Override public AgentSpan beforeFinish(AgentSpan span) { if (llmObsEnabled) { + span.setTag(CommonTags.ERROR, span.isError() ? 1 : 0); + span.setTag(CommonTags.ERROR_TYPE, span.getTag(DDTags.ERROR_TYPE)); + Object spanKindTag = span.getTag(CommonTags.SPAN_KIND); if (spanKindTag != null) { String spanKind = spanKindTag.toString(); From 021a9d1c9bf9a04ac3951120cb1dea26c9c92df5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 22:37:22 +0100 Subject: [PATCH 05/39] Use "" instead of null for the role in CompletionDecorator to comply wthTestOpenAiLlmInteractions::test_completion --- .../instrumentation/openai_java/CompletionDecorator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 51e7c12ef1c..1b95491b64b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -45,7 +45,7 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa input -> span.setTag( CommonTags.INPUT, - Collections.singletonList(LLMObs.LLMMessage.from(null, input)))); + Collections.singletonList(LLMObs.LLMMessage.from("", input)))); Map metadata = new HashMap<>(); params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); @@ -72,7 +72,7 @@ public void withCompletion(AgentSpan span, Completion completion) { List output = completion.choices().stream() - .map(v -> LLMObs.LLMMessage.from(null, v.text())) + .map(v -> LLMObs.LLMMessage.from("", v.text())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); @@ -116,7 +116,7 @@ public void withCompletions(AgentSpan span, List completions) { List output = textByChoiceIndex.entrySet().stream() .sorted(Map.Entry.comparingByKey()) - .map(entry -> LLMObs.LLMMessage.from(null, entry.getValue().toString())) + .map(entry -> LLMObs.LLMMessage.from("", entry.getValue().toString())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); From 0637931c5bb710c3ea29581dcdec36ccbba7d514 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 23:12:24 +0100 Subject: [PATCH 06/39] Use "" instead of null for the content to comply with TestOpenAiLlmInteractions::test_chat_completion_tool_call --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 95c746880cf..c8151c00a2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -131,7 +131,7 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletion.Choice choice) { if (roleOpt.isPresent()) { role = String.valueOf(roleOpt.get()); } - String content = msg.content().orElse(null); + String content = msg.content().orElse(""); Optional> toolCallsOpt = msg.toolCalls(); if (toolCallsOpt.isPresent() && !toolCallsOpt.get().isEmpty()) { From 0cb41e1a9522f617afed58a4dcc2e1e5ef966388 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 13:34:57 +0100 Subject: [PATCH 07/39] Add missing metatadata.tool_choice --- .../openai_java/ChatCompletionDecorator.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index c8151c00a2a..618c7982410 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -71,6 +71,24 @@ public void withChatCompletionCreateParams( params.n().ifPresent(v -> metadata.put("n", v)); params.seed().ifPresent(v -> metadata.put("seed", v)); span.setTag(CommonTags.METADATA, metadata); + params + .toolChoice() + .ifPresent( + toolChoice -> { + String choice = null; + if (toolChoice.isAuto()) { + choice = "auto"; + } else if (toolChoice.isAllowedToolChoice()) { + choice = "allowed_tools"; + } else if (toolChoice.isNamedToolChoice()) { + choice = "function"; + } else if (toolChoice.isNamedToolChoiceCustom()) { + choice = "custom"; + } + if (choice != null) { + metadata.put("tool_choice", choice); + } + }); } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { From a42f8aa2e3a34c8bf14e031a502ca8bb9a468e6f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 21:09:03 +0100 Subject: [PATCH 08/39] Add missing tool_definitions --- .../openai_java/ChatCompletionDecorator.java | 108 ++++++++++++++++++ .../openai_java/CommonTags.java | 2 + .../openai_java/OpenAiDecorator.java | 1 - .../datadog/trace/api/llmobs/LLMObsTags.java | 1 + .../writer/ddintake/LLMObsSpanMapper.java | 1 + 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 618c7982410..e5def4f67bb 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -1,12 +1,16 @@ package datadog.trace.instrumentation.openai_java; +import com.openai.core.JsonValue; import com.openai.helpers.ChatCompletionAccumulator; +import com.openai.models.FunctionDefinition; import com.openai.models.chat.completions.ChatCompletion; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.chat.completions.ChatCompletionFunctionTool; import com.openai.models.chat.completions.ChatCompletionMessage; import com.openai.models.chat.completions.ChatCompletionMessageParam; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.chat.completions.ChatCompletionTool; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -89,6 +93,110 @@ public void withChatCompletionCreateParams( metadata.put("tool_choice", choice); } }); + + List tools = params.tools().orElse(Collections.emptyList()); + if (!tools.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, extractToolDefinitions(tools)); + } + } + + private List> extractToolDefinitions(List tools) { + List> toolDefinitions = new ArrayList<>(); + for (ChatCompletionTool tool : tools) { + if (tool.isFunction()) { + Map toolDef = extractFunctionToolDef(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + } + return toolDefinitions; + } + + private static Map extractFunctionToolDef(ChatCompletionFunctionTool funcTool) { + // Try typed access first (works when built programmatically) + Optional funcDefOpt = funcTool._function().asKnown(); + if (funcDefOpt.isPresent()) { + FunctionDefinition funcDef = funcDefOpt.get(); + Map toolDef = new HashMap<>(); + toolDef.put("name", funcDef.name()); + funcDef.description().ifPresent(desc -> toolDef.put("description", desc)); + funcDef + .parameters() + .ifPresent( + params -> + toolDef.put("schema", jsonValueMapToObject(params._additionalProperties()))); + return toolDef; + } + + // Fall back to raw JSON extraction (when deserialized from HTTP request) + Optional rawOpt = funcTool._function().asUnknown(); + if (!rawOpt.isPresent()) { + return null; + } + Optional> objOpt = rawOpt.get().asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + JsonValue nameValue = obj.get("name"); + if (nameValue == null) { + return null; + } + Optional nameOpt = nameValue.asString(); + if (!nameOpt.isPresent()) { + return null; + } + Map toolDef = new HashMap<>(); + toolDef.put("name", nameOpt.get()); + JsonValue descValue = obj.get("description"); + if (descValue != null) { + descValue.asString().ifPresent(desc -> toolDef.put("description", desc)); + } + JsonValue paramsValue = obj.get("parameters"); + if (paramsValue != null) { + Object schema = jsonValueToObject(paramsValue); + if (schema != null) { + toolDef.put("schema", schema); + } + } + return toolDef; + } + + private static Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + private static Object jsonValueToObject(JsonValue value) { + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index a4335a7edfe..ed5b689cd9b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -28,6 +28,8 @@ interface CommonTags { String SERVICE = TAG_PREFIX + "service"; String PARENT_ID = TAG_PREFIX + "parent_id"; + String TOOL_DEFINITIONS = TAG_PREFIX + "tool_definitions"; + String METRIC_PREFIX = "_ml_obs_metric."; String INPUT_TOKENS = METRIC_PREFIX + "input_tokens"; String OUTPUT_TOKENS = METRIC_PREFIX + "output_tokens"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 2332b578ebf..6380f981797 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -13,7 +13,6 @@ import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; -import datadog.trace.core.CoreSpan; import java.util.List; public class OpenAiDecorator extends ClientDecorator { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java index afa4f2b241e..130cf610dc0 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -12,4 +12,5 @@ public class LLMObsTags { public static final String MODEL_NAME = "model_name"; public static final String MODEL_VERSION = "model_version"; public static final String MODEL_PROVIDER = "model_provider"; + public static final String TOOL_DEFINITIONS = "tool_definitions"; } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7241d469006..e0cd7db3e02 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -230,6 +230,7 @@ private static final class MetaWriter implements MetadataConsumer { LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_NAME, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_VERSION, + LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS, LLMOBS_TAG_PREFIX + LLMObsTags.METADATA))); MetaWriter withWritable(Writable writable, Map errorInfo) { From 6e10255898af899792665ac5e0d84a84e204fe14 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 21:51:50 +0100 Subject: [PATCH 09/39] Add source:integration tag --- .../datadog/trace/instrumentation/openai_java/CommonTags.java | 1 + .../trace/instrumentation/openai_java/OpenAiDecorator.java | 1 + 2 files changed, 2 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index ed5b689cd9b..0e550437026 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -20,6 +20,7 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String VERSION = TAG_PREFIX + "version"; + String SOURCE = TAG_PREFIX + "source"; String ERROR = TAG_PREFIX + "error"; String ERROR_TYPE = TAG_PREFIX + "error_type"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 6380f981797..fae2880c082 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -98,6 +98,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.VERSION, wellKnownTags.getVersion()); span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); + span.setTag(CommonTags.SOURCE, "integration"); AgentSpanContext parent = LLMObsContext.current(); String parentSpanId = LLMObsContext.ROOT_SPAN_ID; From 34f3a07ec396815b6878bcf56024d9d1dac861e8 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 11:31:52 +0100 Subject: [PATCH 10/39] Add missing _dd attribute to the llmobs span event --- .../llmobs/writer/ddintake/LLMObsSpanMapper.java | 16 ++++++++++++++-- .../writer/ddintake/LLMObsSpanMapperTest.groovy | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index e0cd7db3e02..1dec4ad62e5 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -52,6 +52,8 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPAN_ID = "span_id".getBytes(StandardCharsets.UTF_8); private static final byte[] TRACE_ID = "trace_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] DD = "_dd".getBytes(StandardCharsets.UTF_8); + private static final byte[] APM_TRACE_ID = "apm_trace_id".getBytes(StandardCharsets.UTF_8); private static final byte[] PARENT_ID = "parent_id".getBytes(StandardCharsets.UTF_8); private static final byte[] NAME = "name".getBytes(StandardCharsets.UTF_8); private static final byte[] DURATION = "duration".getBytes(StandardCharsets.UTF_8); @@ -120,7 +122,7 @@ public void map(List> trace, Writable writable) { } for (CoreSpan span : llmobsSpans) { - writable.startMap(11); + writable.startMap(12); // 1 writable.writeUTF8(SPAN_ID); writable.writeString(String.valueOf(span.getSpanId()), null); @@ -156,7 +158,17 @@ public void map(List> trace, Writable writable) { writable.writeUTF8(STATUS); writable.writeString(errored ? "error" : "ok", null); - /* 9 (metrics), 10 (tags), 11 meta */ + // 9 + writable.writeUTF8(DD); + writable.startMap(3); + writable.writeUTF8(SPAN_ID); + writable.writeString(String.valueOf(span.getSpanId()), null); + writable.writeUTF8(TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + writable.writeUTF8(APM_TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + + /* 10 (metrics), 11 (tags), 12 meta */ span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 20656ba2a1e..74fb39fed90 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -102,6 +102,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("start_ns") spanData.containsKey("duration") spanData["error"] == 0 + spanData.containsKey("_dd") + spanData["_dd"]["span_id"] == spanData["span_id"] + spanData["_dd"]["trace_id"] == spanData["trace_id"] + spanData["_dd"]["apm_trace_id"] == spanData["trace_id"] spanData.containsKey("meta") spanData["meta"]["span.kind"] == "llm" From a0c1139404dcc77a654c3ed30ebd91c2808db995 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 11:49:52 +0100 Subject: [PATCH 11/39] Add missing error tags --- .../writer/ddintake/LLMObsSpanMapper.java | 37 +++++++++++++++---- .../ddintake/LLMObsSpanMapperTest.groovy | 11 +++++- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 1dec4ad62e5..f4c31e2c6ec 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -60,6 +60,9 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] START_NS = "start_ns".getBytes(StandardCharsets.UTF_8); private static final byte[] STATUS = "status".getBytes(StandardCharsets.UTF_8); private static final byte[] ERROR = "error".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_MESSAGE = "message".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_TYPE = "type".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_STACK = "stack".getBytes(StandardCharsets.UTF_8); private static final byte[] META = "meta".getBytes(StandardCharsets.UTF_8); private static final byte[] METADATA = "metadata".getBytes(StandardCharsets.UTF_8); @@ -215,15 +218,15 @@ private static Map getErrorsMap(CoreSpan span) { Map errors = new HashMap<>(); String errorMsg = span.getTag(DDTags.ERROR_MSG); if (errorMsg != null && !errorMsg.isEmpty()) { - errors.put(DDTags.ERROR_MSG, errorMsg); + errors.put("message", errorMsg); } String errorType = span.getTag(DDTags.ERROR_TYPE); if (errorType != null && !errorType.isEmpty()) { - errors.put(DDTags.ERROR_TYPE, errorType); + errors.put("type", errorType); } String errorStack = span.getTag(DDTags.ERROR_STACK); if (errorStack != null && !errorStack.isEmpty()) { - errors.put(DDTags.ERROR_STACK, errorStack); + errors.put("stack", errorStack); } return errors; } @@ -306,15 +309,35 @@ public void accept(Metadata metadata) { } // write meta (11) - int metaSize = tagsToRemapToMeta.size() + 1 + (null != errorInfo ? errorInfo.size() : 0); + int metaSize = + tagsToRemapToMeta.size() + + 1 + + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); writable.writeUTF8(META); writable.startMap(metaSize); writable.writeUTF8(SPAN_KIND); writable.writeString(spanKind, null); - for (Map.Entry error : errorInfo.entrySet()) { - writable.writeUTF8(error.getKey().getBytes()); - writable.writeString(error.getValue(), null); + if (null != errorInfo && !errorInfo.isEmpty()) { + writable.writeUTF8(ERROR); + writable.startMap(errorInfo.size()); + for (Map.Entry error : errorInfo.entrySet()) { + switch (error.getKey()) { + case "message": + writable.writeUTF8(ERROR_MESSAGE); + break; + case "type": + writable.writeUTF8(ERROR_TYPE); + break; + case "stack": + writable.writeUTF8(ERROR_STACK); + break; + default: + writable.writeString(error.getKey(), null); + break; + } + writable.writeString(error.getValue(), null); + } } for (Map.Entry tag : tagsToRemapToMeta.entrySet()) { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 74fb39fed90..1923c07470b 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import datadog.communication.serialization.ByteBufferConsumer import datadog.communication.serialization.FlushingBuffer import datadog.communication.serialization.msgpack.MsgPackWriter +import datadog.trace.api.DDTags import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes import datadog.trace.bootstrap.instrumentation.api.Tags @@ -44,6 +45,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { llmSpan.setTag("_ml_obs_tag.input", inputMessages) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) + llmSpan.setError(true) + llmSpan.setTag(DDTags.ERROR_MSG, "boom") + llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") + llmSpan.setTag(DDTags.ERROR_STACK, "stacktrace") llmSpan.finish() @@ -101,7 +106,7 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("trace_id") spanData.containsKey("start_ns") spanData.containsKey("duration") - spanData["error"] == 0 + spanData["error"] == 1 spanData.containsKey("_dd") spanData["_dd"]["span_id"] == spanData["span_id"] spanData["_dd"]["trace_id"] == spanData["trace_id"] @@ -109,6 +114,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("meta") spanData["meta"]["span.kind"] == "llm" + spanData["meta"].containsKey("error") + spanData["meta"]["error"]["message"] == "boom" + spanData["meta"]["error"]["type"] == "java.lang.IllegalStateException" + spanData["meta"]["error"]["stack"] == "stacktrace" spanData["meta"].containsKey("input") spanData["meta"]["input"].containsKey("messages") spanData["meta"]["input"]["messages"][0].containsKey("content") From effc34302592382744dfc0dc100b264c788923c0 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 12:24:40 +0100 Subject: [PATCH 12/39] Remove error from the llmobs span event. It must be part of meta block --- .../writer/ddintake/LLMObsSpanMapper.java | 18 +++++------------- .../ddintake/LLMObsSpanMapperTest.groovy | 1 - 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index f4c31e2c6ec..e0b5dce3550 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -125,7 +125,7 @@ public void map(List> trace, Writable writable) { } for (CoreSpan span : llmobsSpans) { - writable.startMap(12); + writable.startMap(11); // 1 writable.writeUTF8(SPAN_ID); writable.writeString(String.valueOf(span.getSpanId()), null); @@ -152,16 +152,10 @@ public void map(List> trace, Writable writable) { writable.writeFloat(span.getDurationNano()); // 7 - writable.writeUTF8(ERROR); - writable.writeInt(span.getError()); - - boolean errored = span.getError() == 1; - - // 8 writable.writeUTF8(STATUS); - writable.writeString(errored ? "error" : "ok", null); + writable.writeString(span.getError() == 0 ? "ok" : "error", null); - // 9 + // 8 writable.writeUTF8(DD); writable.startMap(3); writable.writeUTF8(SPAN_ID); @@ -171,7 +165,7 @@ public void map(List> trace, Writable writable) { writable.writeUTF8(APM_TRACE_ID); writable.writeString(span.getTraceId().toHexString(), null); - /* 10 (metrics), 11 (tags), 12 meta */ + /* 9 (metrics), 10 (tags), 11 meta */ span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); } @@ -310,9 +304,7 @@ public void accept(Metadata metadata) { // write meta (11) int metaSize = - tagsToRemapToMeta.size() - + 1 - + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); + tagsToRemapToMeta.size() + 1 + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); writable.writeUTF8(META); writable.startMap(metaSize); writable.writeUTF8(SPAN_KIND); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 1923c07470b..6140431b836 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -106,7 +106,6 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("trace_id") spanData.containsKey("start_ns") spanData.containsKey("duration") - spanData["error"] == 1 spanData.containsKey("_dd") spanData["_dd"]["span_id"] == spanData["span_id"] spanData["_dd"]["trace_id"] == spanData["trace_id"] From c0e38761a4f2515ec1ed43e7173c6be3096f25bd Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 13:06:00 +0100 Subject: [PATCH 13/39] Add missing meta.text.verbosity --- .../openai-java/openai-java-3.0/build.gradle | 2 +- .../openai_java/ResponseDecorator.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle index cd654d03334..ceda41c2804 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle @@ -1,6 +1,6 @@ apply from: "$rootDir/gradle/java.gradle" -def minVer = '3.0.0' +def minVer = '3.0.1' muzzle { pass { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index db2258785d4..e7be85b77c6 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -424,6 +424,7 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { (Response.Truncation t) -> metadata.put("truncation", t._value().asString().orElse(null))); + Map textMap = new HashMap<>(); response .text() .ifPresent( @@ -432,7 +433,6 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { .format() .ifPresent( format -> { - Map textMap = new HashMap<>(); Map formatMap = new HashMap<>(); if (format.isText()) { formatMap.put("type", "text"); @@ -442,9 +442,17 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { formatMap.put("type", "json_object"); } textMap.put("format", formatMap); - metadata.put("text", textMap); + }); + textConfig + .verbosity() + .ifPresent( + verbosity -> { + textMap.put("verbosity", verbosity.asString()); }); }); + if (!textMap.isEmpty()) { + metadata.put("text", textMap); + } if (stream) { metadata.put("stream", true); From b00077055aec4e578bab797a76bebd3c070ffb78 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 15:42:47 +0100 Subject: [PATCH 14/39] Add summaryText and encrypted_content --- .../openai_java/ResponseDecorator.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index e7be85b77c6..320d82da76f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -496,12 +496,30 @@ private List extractResponseOutputMessages(List writer.name("encrypted_content").value(v)); - writer.name("id").value(reasoning.id()); + + String id = reasoning.id(); + writer.name("id").value(id == null ? "" : id); + writer.endObject(); + messages.add(LLMObs.LLMMessage.from("reasoning", writer.toString())); } } From 53471a2cb6041f96d77c3f1e1ca86ae10fcaeead Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 16:24:38 +0100 Subject: [PATCH 15/39] Add missing tool_calls and tool_results for responses --- .../openai_java/ResponseDecorator.java | 18 ++++++++ .../openai_java/ToolCallExtractor.java | 42 +++++++++++++++++++ .../java/datadog/trace/api/llmobs/LLMObs.java | 5 +++ .../writer/ddintake/LLMObsSpanMapper.java | 10 +++-- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 320d82da76f..c61c7cc712f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -6,6 +6,7 @@ import com.openai.models.ResponsesModel; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; @@ -486,6 +487,23 @@ private List extractResponseOutputMessages(List toolCalls = Collections.singletonList(toolCall); messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls)); } + } else if (item.isCustomToolCall()) { + ResponseCustomToolCall customToolCall = item.asCustomToolCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(customToolCall); + if (toolCall != null) { + messages.add( + LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall))); + } + } else if (item.isMcpCall()) { + ResponseOutputItem.McpCall mcpCall = item.asMcpCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(mcpCall); + List toolCalls = + toolCall == null ? null : Collections.singletonList(toolCall); + String outputText = mcpCall.output().orElse(""); + LLMObs.ToolResult toolResult = + LLMObs.ToolResult.from(mcpCall.name(), "mcp_tool_result", mcpCall.id(), outputText); + List toolResults = Collections.singletonList(toolResult); + messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls, toolResults)); } else if (item.isMessage()) { ResponseOutputMessage message = item.asMessage(); String textContent = extractMessageContent(message); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java index 357c73de0aa..21d2bf6835d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java @@ -4,7 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; +import com.openai.models.responses.ResponseOutputItem.McpCall; import datadog.trace.api.llmobs.LLMObs; import java.util.Collections; import java.util.Map; @@ -64,6 +66,46 @@ public static LLMObs.ToolCall getToolCall(ResponseFunctionToolCall functionCall) return null; } + public static LLMObs.ToolCall getToolCall(ResponseCustomToolCall customToolCall) { + try { + String name = customToolCall.name(); + String callId = customToolCall.callId(); + String inputJson = customToolCall.input(); + + String type = "custom_tool_call"; + Optional typeOpt = customToolCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(inputJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract custom tool call information", e); + } + return null; + } + + public static LLMObs.ToolCall getToolCall(McpCall mcpCall) { + try { + String name = mcpCall.name(); + String callId = mcpCall.id(); + String argumentsJson = mcpCall.arguments(); + + String type = "mcp_call"; + Optional typeOpt = mcpCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(argumentsJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract MCP tool call information", e); + } + return null; + } + private static Map parseArguments(String argumentsJson) { try { return MAPPER.readValue(argumentsJson, MAP_TYPE_REF); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 512a3106ce6..629faa23f5a 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -217,6 +217,11 @@ public static LLMMessage from(String role, String content, List toolCa return new LLMMessage(role, content, toolCalls, null); } + public static LLMMessage from( + String role, String content, List toolCalls, List toolResults) { + return new LLMMessage(role, content, toolCalls, toolResults); + } + public static LLMMessage from(String role, String content) { return new LLMMessage(role, content, null, null); } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index e0b5dce3550..b4dba725406 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -354,14 +354,18 @@ public void accept(Metadata metadata) { List toolResults = message.getToolResults(); boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); - int mapSize = 2; // role and content + boolean hasContent = message.getContent() != null; + int mapSize = 1; // role + if (hasContent) mapSize++; if (hasToolCalls) mapSize++; if (hasToolResults) mapSize++; writable.startMap(mapSize); writable.writeUTF8(LLM_MESSAGE_ROLE); writable.writeString(message.getRole(), null); - writable.writeUTF8(LLM_MESSAGE_CONTENT); - writable.writeString(message.getContent(), null); + if (hasContent) { + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + } if (hasToolCalls) { writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); writable.startArray(toolCalls.size()); From 2207c46e77a2f1a33948f7a2fc60d566956bd266 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 16:55:53 +0100 Subject: [PATCH 16/39] Always set stream param to produce the same request body to be aligned with python openai instrumentation and system-tests --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index c61c7cc712f..dce941af568 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -455,9 +455,7 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { metadata.put("text", textMap); } - if (stream) { - metadata.put("stream", true); - } + metadata.put("stream", stream); span.setTag(CommonTags.METADATA, metadata); From ca6e2d13649115fe897a7c2cca65131e4c11f912 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 13:41:50 +0100 Subject: [PATCH 17/39] Add OpenAI prompt-tracking reconstruction for responses (input.prompt with variables + chat_template, longest-first overlap handling) and support map-based LLM input serialization (messages + prompt) in LLMObs mapper. Also filter empty instruction messages to match system-test expectations. --- .../openai_java/CommonTags.java | 1 + .../openai_java/ResponseDecorator.java | 365 +++++++++++++++++- .../writer/ddintake/LLMObsSpanMapper.java | 151 +++++--- 3 files changed, 451 insertions(+), 66 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index 0e550437026..228b3d52a81 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -39,4 +39,5 @@ interface CommonTags { String CACHE_READ_INPUT_TOKENS = METRIC_PREFIX + "cache_read_input_tokens"; String REQUEST_REASONING = "_ml_obs_request.reasoning"; + String REQUEST_PROMPT = "_ml_obs_request.prompt"; } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index dce941af568..3b7e4b50a2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -7,6 +7,7 @@ import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; +import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; @@ -15,6 +16,7 @@ import com.openai.models.responses.ResponseOutputText; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.ResponsePrompt; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -24,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,6 +35,8 @@ public class ResponseDecorator { public static final ResponseDecorator DECORATE = new ResponseDecorator(); private static final CharSequence RESPONSES_CREATE = UTF8BytesString.create("createResponse"); + private static final String IMAGE_FALLBACK_MARKER = "[image]"; + private static final String FILE_FALLBACK_MARKER = "[file]"; private final boolean llmObsEnabled = Config.get().isLlmObsEnabled(); @@ -111,13 +116,26 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractReasoningFromParams(params) .ifPresent(reasoningMap -> span.setTag(CommonTags.REQUEST_REASONING, reasoningMap)); + + extractPromptFromParams(params).ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { - if (item.isMessage()) { + if (item.isEasyInputMessage()) { + EasyInputMessage message = item.asEasyInputMessage(); + String role = message.role().asString(); + String content = extractEasyInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } else if (item.isMessage()) { ResponseInputItem.Message message = item.asMessage(); String role = message.role().asString(); String content = extractInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } return LLMObs.LLMMessage.from(role, content); } else if (item.isFunctionCall()) { // Function call is mapped to assistant message with tool_calls @@ -139,6 +157,26 @@ private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { return null; } + private String extractEasyInputMessageContent(EasyInputMessage message) { + if (message.content().isTextInput()) { + String content = message.content().asTextInput(); + return content == null || content.isEmpty() ? null : content; + } + + if (message.content().isResponseInputMessageContentList()) { + StringBuilder contentBuilder = new StringBuilder(); + for (ResponseInputContent content : message.content().asResponseInputMessageContentList()) { + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); + } + } + String result = contentBuilder.toString(); + return result.isEmpty() ? null : result; + } + return null; + } + private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { Optional> objOpt = jsonValue.asObject(); if (!objOpt.isPresent()) { @@ -324,14 +362,35 @@ private String removeQuotes(String str) { private String extractInputMessageContent(ResponseInputItem.Message message) { StringBuilder contentBuilder = new StringBuilder(); for (ResponseInputContent content : message.content()) { - if (content.isInputText()) { - contentBuilder.append(content.asInputText().text()); + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); } } String result = contentBuilder.toString(); return result.isEmpty() ? null : result; } + private String extractInputContentText(ResponseInputContent content) { + if (content.isInputText()) { + return content.asInputText().text(); + } + if (content.isInputImage()) { + return content.asInputImage().imageUrl().orElse(content.asInputImage().fileId().orElse("")); + } + if (content.isInputFile()) { + return content + .asInputFile() + .fileUrl() + .orElse( + content + .asInputFile() + .fileId() + .orElse(content.asInputFile().filename().orElse(FILE_FALLBACK_MARKER))); + } + return null; + } + private Optional> extractReasoningFromParams(ResponseCreateParams params) { JsonField reasoningField = params._reasoning(); if (reasoningField.isMissing()) { @@ -399,6 +458,8 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { span.setTag(CommonTags.OUTPUT, outputMessages); } + enrichInputWithPromptTracking(span, response); + Map metadata = new HashMap<>(); Object reasoningTag = span.getTag(CommonTags.REQUEST_REASONING); @@ -474,6 +535,304 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { }); } + private void enrichInputWithPromptTracking(AgentSpan span, Response response) { + Object promptTag = span.getTag(CommonTags.REQUEST_PROMPT); + if (!(promptTag instanceof Map)) { + return; + } + + Map prompt = new LinkedHashMap<>((Map) promptTag); + Map variables = Collections.emptyMap(); + Object variablesTag = prompt.get("variables"); + if (variablesTag instanceof Map) { + variables = (Map) variablesTag; + } + + Map inputMap = new LinkedHashMap<>(); + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof Map) { + inputMap.putAll((Map) inputTag); + } + + List inputMessages = extractInputMessagesForPromptTracking(span, response); + if (!inputMessages.isEmpty()) { + inputMap.put("messages", inputMessages); + } + + List> chatTemplate = extractChatTemplate(inputMessages, variables); + if (!chatTemplate.isEmpty()) { + prompt.put("chat_template", chatTemplate); + } + + inputMap.put("prompt", prompt); + + span.setTag(CommonTags.INPUT, inputMap); + } + + private List> extractChatTemplate( + List messages, Map variables) { + Map valueToPlaceholder = new LinkedHashMap<>(); + for (Map.Entry variable : variables.entrySet()) { + if (variable.getValue() == null) { + continue; + } + String valueStr = String.valueOf(variable.getValue()); + if (valueStr.isEmpty() + || IMAGE_FALLBACK_MARKER.equals(valueStr) + || FILE_FALLBACK_MARKER.equals(valueStr)) { + continue; + } + valueToPlaceholder.put(valueStr, "{{" + variable.getKey() + "}}"); + } + + List sortedValues = new ArrayList<>(valueToPlaceholder.keySet()); + sortedValues.sort((a, b) -> Integer.compare(b.length(), a.length())); + + List> chatTemplate = new ArrayList<>(); + for (LLMObs.LLMMessage message : messages) { + String role = message.getRole(); + String content = message.getContent(); + if (role == null || role.isEmpty() || content == null || content.isEmpty()) { + continue; + } + + String templateContent = content; + for (String value : sortedValues) { + templateContent = templateContent.replace(value, valueToPlaceholder.get(value)); + } + + Map messageMap = new LinkedHashMap<>(); + messageMap.put("role", role); + messageMap.put("content", templateContent); + chatTemplate.add(messageMap); + } + return chatTemplate; + } + + private List extractInputMessagesForPromptTracking( + AgentSpan span, Response response) { + List messages = new ArrayList<>(); + + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof List) { + for (Object messageObj : (List) inputTag) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } else if (inputTag instanceof Map) { + Object messagesObj = ((Map) inputTag).get("messages"); + if (messagesObj instanceof List) { + for (Object messageObj : (List) messagesObj) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } + } + + if (!messages.isEmpty()) { + return messages; + } + + response + .instructions() + .ifPresent( + instructions -> { + if (instructions.isInputItemList()) { + for (ResponseInputItem item : instructions.asInputItemList()) { + LLMObs.LLMMessage message = extractInputItemMessage(item); + if (message != null) { + messages.add(message); + } + } + } else if (instructions.isString()) { + String text = instructions.asString(); + if (text != null && !text.isEmpty()) { + messages.add(LLMObs.LLMMessage.from("user", text)); + } + } + }); + + if (!messages.isEmpty()) { + return messages; + } + + // Fallback for SDK union parsing mismatches: parse raw instructions payload. + Optional rawInstructions = response._instructions().asUnknown(); + if (rawInstructions.isPresent()) { + Optional> rawList = rawInstructions.get().asArray(); + if (rawList.isPresent()) { + for (JsonValue item : rawList.get()) { + LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); + if (message != null) { + messages.add(message); + } + } + } + } + + return messages; + } + + private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instructionValue) { + Optional> objOpt = instructionValue.asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + String role = getJsonString(obj.get("role")); + if (role == null || role.isEmpty()) { + return null; + } + + JsonValue contentValue = obj.get("content"); + if (contentValue == null) { + return null; + } + Optional> contentList = contentValue.asArray(); + if (!contentList.isPresent()) { + return null; + } + + StringBuilder contentBuilder = new StringBuilder(); + for (JsonValue contentItem : contentList.get()) { + Optional> contentObjOpt = contentItem.asObject(); + if (!contentObjOpt.isPresent()) { + continue; + } + Map contentObj = contentObjOpt.get(); + String type = getJsonString(contentObj.get("type")); + if ("input_text".equals(type)) { + String text = getJsonString(contentObj.get("text")); + if (text != null) { + contentBuilder.append(text); + } + } else if ("input_image".equals(type)) { + String imageUrl = getJsonString(contentObj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + contentBuilder.append(imageUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + contentBuilder.append(fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); + } + } else if ("input_file".equals(type)) { + String fileUrl = getJsonString(contentObj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + contentBuilder.append(fileUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + contentBuilder.append(fileId); + } else { + String filename = getJsonString(contentObj.get("filename")); + contentBuilder.append( + filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename); + } + } + } + } + + String content = contentBuilder.toString(); + if (content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } + + private Optional> extractPromptFromParams(ResponseCreateParams params) { + Optional promptOpt = params.prompt(); + if (!promptOpt.isPresent()) { + return Optional.empty(); + } + + ResponsePrompt prompt = promptOpt.get(); + Map promptMap = new LinkedHashMap<>(); + + String id = prompt.id(); + if (id != null && !id.isEmpty()) { + promptMap.put("id", id); + } + prompt.version().ifPresent(version -> promptMap.put("version", version)); + prompt + .variables() + .ifPresent( + variables -> { + Map normalized = normalizePromptVariables(variables); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + }); + + return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); + } + + private Map normalizePromptVariables(ResponsePrompt.Variables variables) { + Map normalized = new LinkedHashMap<>(); + for (Map.Entry entry : variables._additionalProperties().entrySet()) { + Object value = normalizePromptVariable(entry.getValue()); + if (value != null) { + normalized.put(entry.getKey(), value); + } + } + return normalized; + } + + private Object normalizePromptVariable(JsonValue value) { + if (value == null) { + return null; + } + + Optional asString = value.asString(); + if (asString.isPresent()) { + return asString.get(); + } + + Optional> asObject = value.asObject(); + if (!asObject.isPresent()) { + return value.toString(); + } + + Map obj = asObject.get(); + String type = getJsonString(obj.get("type")); + String text = getJsonString(obj.get("text")); + if (text != null && !text.isEmpty()) { + return text; + } + + if ("input_image".equals(type)) { + String imageUrl = getJsonString(obj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + return imageUrl; + } + String fileId = getJsonString(obj.get("file_id")); + return fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId; + } + + if ("input_file".equals(type)) { + String fileUrl = getJsonString(obj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + return fileUrl; + } + String fileId = getJsonString(obj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + return fileId; + } + String filename = getJsonString(obj.get("filename")); + return filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename; + } + + return value.toString(); + } + + private String getJsonString(JsonValue value) { + if (value == null) { + return null; + } + Optional asString = value.asString(); + return asString.orElse(null); + } + private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index b4dba725406..7ff16cbed4e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -337,74 +337,19 @@ public void accept(Metadata metadata) { Object val = tag.getValue(); if (key.equals(INPUT) || key.equals(OUTPUT)) { if (spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) { - if (!(val instanceof List)) { + writable.writeString(key, null); + if (val instanceof List) { + writable.startMap(1); + writable.writeString("messages", null); + writeLlmMessages((List) val); + } else if (key.equals(INPUT) && val instanceof Map) { + writeLlmInputMap((Map) val); + } else { LOGGER.warn( "unexpectedly found incorrect type for LLM span IO {}, expecting list", val.getClass().getName()); continue; } - writable.writeString(key, null); - writable.startMap(1); - // llm span kind must have llm objects - List messages = (List) val; - writable.writeString("messages", null); - writable.startArray(messages.size()); - for (LLMObs.LLMMessage message : messages) { - List toolCalls = message.getToolCalls(); - List toolResults = message.getToolResults(); - boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); - boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); - boolean hasContent = message.getContent() != null; - int mapSize = 1; // role - if (hasContent) mapSize++; - if (hasToolCalls) mapSize++; - if (hasToolResults) mapSize++; - writable.startMap(mapSize); - writable.writeUTF8(LLM_MESSAGE_ROLE); - writable.writeString(message.getRole(), null); - if (hasContent) { - writable.writeUTF8(LLM_MESSAGE_CONTENT); - writable.writeString(message.getContent(), null); - } - if (hasToolCalls) { - writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); - writable.startArray(toolCalls.size()); - for (LLMObs.ToolCall toolCall : toolCalls) { - Map arguments = toolCall.getArguments(); - boolean hasArguments = null != arguments && !arguments.isEmpty(); - writable.startMap(hasArguments ? 4 : 3); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolCall.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolCall.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolCall.getToolId(), null); - if (hasArguments) { - writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); - writable.startMap(arguments.size()); - for (Map.Entry argument : arguments.entrySet()) { - writable.writeString(argument.getKey(), null); - writable.writeObject(argument.getValue(), null); - } - } - } - } - if (hasToolResults) { - writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); - writable.startArray(toolResults.size()); - for (LLMObs.ToolResult toolResult : toolResults) { - writable.startMap(4); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolResult.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolResult.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolResult.getToolId(), null); - writable.writeUTF8(LLM_TOOL_RESULT_RESULT); - writable.writeString(toolResult.getResult(), null); - } - } - } } else if (spanKind.equals(Tags.LLMOBS_EMBEDDING_SPAN_KIND) && key.equals(INPUT)) { if (!(val instanceof List)) { LOGGER.warn( @@ -442,6 +387,86 @@ public void accept(Metadata metadata) { } } } + + private void writeLlmInputMap(Map inputMap) { + writable.startMap(inputMap.size()); + for (Map.Entry entry : inputMap.entrySet()) { + String inputKey = String.valueOf(entry.getKey()); + Object inputValue = entry.getValue(); + writable.writeString(inputKey, null); + if ("messages".equals(inputKey) && inputValue instanceof List) { + writeLlmMessages((List) inputValue); + } else { + writable.writeObject(inputValue, null); + } + } + } + + private void writeLlmMessages(List messages) { + writable.startArray(messages.size()); + for (Object messageObj : messages) { + if (!(messageObj instanceof LLMObs.LLMMessage)) { + writable.writeObject(messageObj, null); + continue; + } + + LLMObs.LLMMessage message = (LLMObs.LLMMessage) messageObj; + List toolCalls = message.getToolCalls(); + List toolResults = message.getToolResults(); + boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); + boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); + boolean hasContent = message.getContent() != null; + int mapSize = 1; + if (hasContent) mapSize++; + if (hasToolCalls) mapSize++; + if (hasToolResults) mapSize++; + writable.startMap(mapSize); + writable.writeUTF8(LLM_MESSAGE_ROLE); + writable.writeString(message.getRole(), null); + if (hasContent) { + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + } + if (hasToolCalls) { + writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); + writable.startArray(toolCalls.size()); + for (LLMObs.ToolCall toolCall : toolCalls) { + Map arguments = toolCall.getArguments(); + boolean hasArguments = null != arguments && !arguments.isEmpty(); + writable.startMap(hasArguments ? 4 : 3); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolCall.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolCall.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolCall.getToolId(), null); + if (hasArguments) { + writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); + writable.startMap(arguments.size()); + for (Map.Entry argument : arguments.entrySet()) { + writable.writeString(argument.getKey(), null); + writable.writeObject(argument.getValue(), null); + } + } + } + } + if (hasToolResults) { + writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); + writable.startArray(toolResults.size()); + for (LLMObs.ToolResult toolResult : toolResults) { + writable.startMap(4); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolResult.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolResult.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolResult.getToolId(), null); + writable.writeUTF8(LLM_TOOL_RESULT_RESULT); + writable.writeString(toolResult.getResult(), null); + } + } + } + } } private static class PayloadV1 extends Payload { From 7d683b6209d4c2e90e630cf9215a237b01fd7e62 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 14:03:03 +0100 Subject: [PATCH 18/39] Fix OpenAI Responses prompt tracking to use response instructions first and return [image] (not empty) when stripped input_image URLs are missing, aligning mixed-input chat_template output with expected behavior. --- .../openai_java/ResponseDecorator.java | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 3b7e4b50a2a..aa334485730 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -4,19 +4,19 @@ import com.openai.core.JsonValue; import com.openai.models.Reasoning; import com.openai.models.ResponsesModel; +import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; -import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; import com.openai.models.responses.ResponseOutputItem; import com.openai.models.responses.ResponseOutputMessage; import com.openai.models.responses.ResponseOutputText; +import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; -import com.openai.models.responses.ResponsePrompt; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -117,7 +117,8 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractReasoningFromParams(params) .ifPresent(reasoningMap -> span.setTag(CommonTags.REQUEST_REASONING, reasoningMap)); - extractPromptFromParams(params).ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); + extractPromptFromParams(params) + .ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { @@ -376,7 +377,10 @@ private String extractInputContentText(ResponseInputContent content) { return content.asInputText().text(); } if (content.isInputImage()) { - return content.asInputImage().imageUrl().orElse(content.asInputImage().fileId().orElse("")); + return content + .asInputImage() + .imageUrl() + .orElse(content.asInputImage().fileId().orElse(IMAGE_FALLBACK_MARKER)); } if (content.isInputFile()) { return content @@ -613,28 +617,6 @@ private List extractInputMessagesForPromptTracking( AgentSpan span, Response response) { List messages = new ArrayList<>(); - Object inputTag = span.getTag(CommonTags.INPUT); - if (inputTag instanceof List) { - for (Object messageObj : (List) inputTag) { - if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); - } - } - } else if (inputTag instanceof Map) { - Object messagesObj = ((Map) inputTag).get("messages"); - if (messagesObj instanceof List) { - for (Object messageObj : (List) messagesObj) { - if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); - } - } - } - } - - if (!messages.isEmpty()) { - return messages; - } - response .instructions() .ifPresent( @@ -672,6 +654,24 @@ private List extractInputMessagesForPromptTracking( } } + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof List) { + for (Object messageObj : (List) inputTag) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } else if (inputTag instanceof Map) { + Object messagesObj = ((Map) inputTag).get("messages"); + if (messagesObj instanceof List) { + for (Object messageObj : (List) messagesObj) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } + } + return messages; } @@ -714,7 +714,8 @@ private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instruction contentBuilder.append(imageUrl); } else { String fileId = getJsonString(contentObj.get("file_id")); - contentBuilder.append(fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); + contentBuilder.append( + fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); } } else if ("input_file".equals(type)) { String fileUrl = getJsonString(contentObj.get("file_url")); From 2c17ddc1a038595f7028142b214388a3d8092eeb Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 14:24:04 +0100 Subject: [PATCH 19/39] Set LLMObs error-path defaults in Java to always emit model_name and output.messages from request params so existing error-span tests pass. --- .../openai_java/ChatCompletionDecorator.java | 14 +++++++++----- .../openai_java/CompletionDecorator.java | 15 ++++++++++----- .../openai_java/EmbeddingDecorator.java | 10 +++++----- .../openai_java/ResponseDecorator.java | 7 +++++++ 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index e5def4f67bb..d5dfde5bd6b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -38,16 +38,20 @@ public void withChatCompletionCreateParams( if (params == null) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); span.setTag( diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 1b95491b64b..f0f29386582 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class CompletionDecorator { @@ -27,16 +28,20 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); params .prompt() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 02d4588358c..4e986603d93 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -28,16 +28,16 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para if (params == null) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name stable on error paths where no response is available. + modelName.ifPresent(str -> span.setTag(CommonTags.MODEL_NAME, str)); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index aa334485730..b91ccf6c954 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -56,6 +56,13 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params return; } + // Keep model_name/output/metadata shape stable on error paths where no response is available. + if (modelName != null && !modelName.isEmpty()) { + span.setTag(CommonTags.MODEL_NAME, modelName); + } + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + span.setTag(CommonTags.METADATA, new HashMap()); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); List inputMessages = new ArrayList<>(); From ad3b782f56e40eb3ae8e4a99e42e54d7b2aca510 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:05:00 +0100 Subject: [PATCH 20/39] Add OpenAI Responses tool definition extraction to populate LLMObs tool_definitions tags --- .../openai_java/ResponseDecorator.java | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index b91ccf6c954..3ca06d74f10 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -17,6 +17,8 @@ import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.FunctionTool; +import com.openai.models.responses.Tool; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -126,6 +128,123 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractPromptFromParams(params) .ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); + + List> toolDefinitions = extractToolDefinitionsFromParams(params); + if (!toolDefinitions.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, toolDefinitions); + } + } + + private List> extractToolDefinitionsFromParams(ResponseCreateParams params) { + try { + Optional> toolsOpt = params.tools(); + if (toolsOpt.isPresent()) { + List> toolDefinitions = new ArrayList<>(); + for (Tool tool : toolsOpt.get()) { + if (!tool.isFunction()) { + continue; + } + Map toolDef = extractFunctionToolDefinition(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + if (!toolDefinitions.isEmpty()) { + return toolDefinitions; + } + } + } catch (Throwable ignored) { + // fall back to raw JSON if typed extraction is unavailable or fails + } + + try { + Optional rawToolsOpt = params._tools().asUnknown(); + if (!rawToolsOpt.isPresent()) { + return Collections.emptyList(); + } + Optional> rawToolListOpt = rawToolsOpt.get().asArray(); + if (!rawToolListOpt.isPresent()) { + return Collections.emptyList(); + } + + List> toolDefinitions = new ArrayList<>(); + for (JsonValue rawTool : rawToolListOpt.get()) { + Map toolDef = extractFunctionToolDefinition(rawTool); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + return toolDefinitions; + } catch (Throwable ignored) { + return Collections.emptyList(); + } + } + + private Map extractFunctionToolDefinition(FunctionTool functionTool) { + String name = functionTool.name(); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + functionTool.description().ifPresent(desc -> toolDef.put("description", desc)); + functionTool + .parameters() + .ifPresent(parameters -> toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); + return toolDef; + } + + private Map extractFunctionToolDefinition(JsonValue rawTool) { + Optional> toolObjOpt = rawTool.asObject(); + if (!toolObjOpt.isPresent()) { + return null; + } + + Map toolObj = toolObjOpt.get(); + String type = getJsonString(toolObj.get("type")); + if (!"function".equals(type)) { + return null; + } + + JsonValue functionObjValue = toolObj.get("function"); + Map functionObj = null; + if (functionObjValue != null) { + Optional> nestedFunctionOpt = functionObjValue.asObject(); + if (nestedFunctionOpt.isPresent()) { + functionObj = nestedFunctionOpt.get(); + } + } + + String name = + functionObj == null + ? getJsonString(toolObj.get("name")) + : getJsonString(functionObj.get("name")); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + + String description = + functionObj == null + ? getJsonString(toolObj.get("description")) + : getJsonString(functionObj.get("description")); + if (description != null) { + toolDef.put("description", description); + } + + JsonValue parameters = + functionObj == null ? toolObj.get("parameters") : functionObj.get("parameters"); + if (parameters != null) { + Object schema = jsonValueToObject(parameters); + if (schema != null) { + toolDef.put("schema", schema); + } + } + + return toolDef; } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { @@ -841,6 +960,45 @@ private String getJsonString(JsonValue value) { return asString.orElse(null); } + private Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + private Object jsonValueToObject(JsonValue value) { + if (value == null) { + return null; + } + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; + } + private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); From 1810327aef053000f2a6ffefc54b2cbf39a8aee5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:13:58 +0100 Subject: [PATCH 21/39] Fix ChatCompletionServiceTest --- .../src/test/groovy/ChatCompletionServiceTest.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 22e7fbbb579..580c0ecd262 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -6,7 +6,6 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.chat.completions.ChatCompletion import com.openai.models.chat.completions.ChatCompletionChunk -import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags @@ -94,7 +93,7 @@ class ChatCompletionServiceTest extends OpenAiTest { } def "create async chat/completion test withRawResponse"() { - CompletableFuture> completionFuture = runUnderTrace("parent") { + CompletableFuture> completionFuture = runUnderTrace("parent") { openAiClient.async().chat().completions().withRawResponse().create(params) } From 46221e411d13675090941c7fb15c38d74401eda1 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:25:04 +0100 Subject: [PATCH 22/39] Extract JsonValueUtils --- .../openai_java/ChatCompletionDecorator.java | 39 ++------------ .../openai_java/ChatCompletionModule.java | 1 + .../openai_java/JsonValueUtils.java | 51 +++++++++++++++++++ .../openai_java/ResponseDecorator.java | 42 ++------------- .../openai_java/ResponseModule.java | 1 + 5 files changed, 59 insertions(+), 75 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index d5dfde5bd6b..f35e485c90e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -1,5 +1,8 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + import com.openai.core.JsonValue; import com.openai.helpers.ChatCompletionAccumulator; import com.openai.models.FunctionDefinition; @@ -167,42 +170,6 @@ private static Map extractFunctionToolDef(ChatCompletionFunction return toolDef; } - private static Map jsonValueMapToObject(Map map) { - Map result = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), jsonValueToObject(entry.getValue())); - } - return result; - } - - private static Object jsonValueToObject(JsonValue value) { - Optional str = value.asString(); - if (str.isPresent()) { - return str.get(); - } - Optional num = value.asNumber(); - if (num.isPresent()) { - return num.get(); - } - Optional bool = value.asBoolean(); - if (bool.isPresent()) { - return bool.get(); - } - Optional> obj = value.asObject(); - if (obj.isPresent()) { - return jsonValueMapToObject(obj.get()); - } - Optional> arr = value.asArray(); - if (arr.isPresent()) { - List list = new ArrayList<>(); - for (JsonValue item : arr.get()) { - list.add(jsonValueToObject(item)); - } - return list; - } - return null; - } - private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { String role = "unknown"; String content = null; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java index f30d02f9570..2c89bf339ad 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java @@ -18,6 +18,7 @@ public String[] helperClassNames() { packageName + ".CommonTags", packageName + ".ChatCompletionDecorator", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java new file mode 100644 index 00000000000..c251f5c17fd --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java @@ -0,0 +1,51 @@ +package datadog.trace.instrumentation.openai_java; + +import com.openai.core.JsonValue; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public final class JsonValueUtils { + private JsonValueUtils() {} + + public static Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + public static Object jsonValueToObject(JsonValue value) { + if (value == null) { + return null; + } + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; + } +} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 3ca06d74f10..631d5d213e1 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -1,5 +1,8 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + import com.openai.core.JsonField; import com.openai.core.JsonValue; import com.openai.models.Reasoning; @@ -960,45 +963,6 @@ private String getJsonString(JsonValue value) { return asString.orElse(null); } - private Map jsonValueMapToObject(Map map) { - Map result = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), jsonValueToObject(entry.getValue())); - } - return result; - } - - private Object jsonValueToObject(JsonValue value) { - if (value == null) { - return null; - } - Optional str = value.asString(); - if (str.isPresent()) { - return str.get(); - } - Optional num = value.asNumber(); - if (num.isPresent()) { - return num.get(); - } - Optional bool = value.asBoolean(); - if (bool.isPresent()) { - return bool.get(); - } - Optional> obj = value.asObject(); - if (obj.isPresent()) { - return jsonValueMapToObject(obj.get()); - } - Optional> arr = value.asArray(); - if (arr.isPresent()) { - List list = new ArrayList<>(); - for (JsonValue item : arr.get()) { - list.add(jsonValueToObject(item)); - } - return list; - } - return null; - } - private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java index 25266504f53..b87b3910490 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java @@ -19,6 +19,7 @@ public String[] helperClassNames() { packageName + ".ResponseDecorator", packageName + ".FunctionCallOutputExtractor", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", From 61ad6678b8fe9d52ad56990195f56b1c0bd1243a Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:48:39 +0100 Subject: [PATCH 23/39] Refactor OpenAI responses instrumentation to reuse ToolCallExtractor JSON argument parsing and remove duplicate manual parsing logic from ResponseDecorator. --- .../openai_java/ResponseDecorator.java | 77 +------------------ .../openai_java/ToolCallExtractor.java | 2 +- 2 files changed, 2 insertions(+), 77 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 631d5d213e1..810e8419176 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -352,7 +352,7 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { } if (callId != null && name != null && argumentsStr != null) { - Map arguments = parseJsonString(argumentsStr); + Map arguments = ToolCallExtractor.parseArguments(argumentsStr); LLMObs.ToolCall toolCall = LLMObs.ToolCall.from(name, "function_call", callId, arguments); return LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall)); @@ -414,81 +414,6 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { return null; } - private Map parseJsonString(String jsonStr) { - if (jsonStr == null || jsonStr.isEmpty()) { - return Collections.emptyMap(); - } - try { - jsonStr = jsonStr.trim(); - if (!jsonStr.startsWith("{") || !jsonStr.endsWith("}")) { - return Collections.emptyMap(); - } - - Map result = new HashMap<>(); - String content = jsonStr.substring(1, jsonStr.length() - 1).trim(); - - if (content.isEmpty()) { - return result; - } - - // Parse JSON manually, respecting quoted strings - List pairs = splitByCommaRespectingQuotes(content); - - for (String pair : pairs) { - int colonIdx = pair.indexOf(':'); - if (colonIdx > 0) { - String key = pair.substring(0, colonIdx).trim(); - String value = pair.substring(colonIdx + 1).trim(); - - // Remove quotes from key - key = removeQuotes(key); - // Remove quotes from value - value = removeQuotes(value); - - result.put(key, value); - } - } - - return result; - } catch (Exception e) { - return Collections.emptyMap(); - } - } - - private List splitByCommaRespectingQuotes(String str) { - List result = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - boolean inQuotes = false; - - for (int i = 0; i < str.length(); i++) { - char c = str.charAt(i); - - if (c == '"') { - inQuotes = !inQuotes; - current.append(c); - } else if (c == ',' && !inQuotes) { - result.add(current.toString()); - current = new StringBuilder(); - } else { - current.append(c); - } - } - - if (current.length() > 0) { - result.add(current.toString()); - } - - return result; - } - - private String removeQuotes(String str) { - str = str.trim(); - if (str.startsWith("\"") && str.endsWith("\"") && str.length() >= 2) { - return str.substring(1, str.length() - 1); - } - return str; - } - private String extractInputMessageContent(ResponseInputItem.Message message) { StringBuilder contentBuilder = new StringBuilder(); for (ResponseInputContent content : message.content()) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java index 21d2bf6835d..ffeca857a20 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java @@ -106,7 +106,7 @@ public static LLMObs.ToolCall getToolCall(McpCall mcpCall) { return null; } - private static Map parseArguments(String argumentsJson) { + static Map parseArguments(String argumentsJson) { try { return MAPPER.readValue(argumentsJson, MAP_TYPE_REF); } catch (Exception e) { From f0957b79844420163b73cf302fd1faa07afa6f53 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 19:04:37 +0100 Subject: [PATCH 24/39] Fix test assertions --- .../openai_java/ResponseDecorator.java | 6 ++++-- .../groovy/ChatCompletionServiceTest.groovy | 19 ++++++++++++++++--- .../test/groovy/CompletionServiceTest.groovy | 2 ++ .../test/groovy/EmbeddingServiceTest.groovy | 2 ++ .../test/groovy/ResponseServiceTest.groovy | 2 ++ 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 810e8419176..07359a9b326 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -8,6 +8,7 @@ import com.openai.models.Reasoning; import com.openai.models.ResponsesModel; import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.FunctionTool; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; @@ -20,7 +21,6 @@ import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; -import com.openai.models.responses.FunctionTool; import com.openai.models.responses.Tool; import datadog.json.JsonWriter; import datadog.trace.api.Config; @@ -194,7 +194,9 @@ private Map extractFunctionToolDefinition(FunctionTool functionT functionTool.description().ifPresent(desc -> toolDef.put("description", desc)); functionTool .parameters() - .ifPresent(parameters -> toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); + .ifPresent( + parameters -> + toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); return toolDef; } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 580c0ecd262..fad254f69c5 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -147,7 +147,7 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(false, outputTag, [:]) + assertChatCompletionTrace(false, outputTag, [:], true) and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -178,7 +178,7 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(true, outputTag, [stream: true]) + assertChatCompletionTrace(true, outputTag, [stream: true], true) and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -294,6 +294,13 @@ class ChatCompletionServiceTest extends OpenAiTest { } private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata) { + assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false) + } + + private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions) { + def expectedMetadata = new LinkedHashMap(metadata) + expectedMetadata.putIfAbsent("stream", isStreaming) + assertTraces(1) { trace(3) { sortSpansByStart() @@ -312,7 +319,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.span.kind" "llm" "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String - "_ml_obs_tag.metadata" metadata + "_ml_obs_tag.metadata" expectedMetadata "_ml_obs_tag.input" List "_ml_obs_tag.output" List def outputTags = tag("_ml_obs_tag.output") @@ -324,10 +331,16 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long + "_ml_obs_metric.cache_read_input_tokens" Long } "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + if (expectToolDefinitions) { + "$CommonTags.TOOL_DEFINITIONS" List + } + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/chat/completions" "openai.api_base" openAiBaseApi diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 43f5e247c55..72ffaee8052 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -165,6 +165,8 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/completions" "openai.api_base" openAiBaseApi diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index eb14f2999de..44999641fc2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -64,6 +64,8 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.total_tokens" Long "openai.request.method" "POST" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index f747005cff4..3b205d8496f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -220,6 +220,8 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 if (reasoning != null) { "_ml_obs_request.reasoning" reasoning } From f3f1f75ec26f06c157090a87f5a64f45df3b720e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 10:35:37 +0100 Subject: [PATCH 25/39] Add integration tag --- .../trace/instrumentation/openai_java/CommonTags.java | 1 + .../trace/instrumentation/openai_java/OpenAiDecorator.java | 1 + .../src/test/groovy/ChatCompletionServiceTest.groovy | 1 + .../src/test/groovy/CompletionServiceTest.groovy | 1 + .../src/test/groovy/EmbeddingServiceTest.groovy | 1 + .../src/test/groovy/ResponseServiceTest.groovy | 1 + .../trace/llmobs/writer/ddintake/LLMObsSpanMapper.java | 3 ++- .../llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy | 6 ++++-- 8 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index 228b3d52a81..c9917332e7e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -19,6 +19,7 @@ interface CommonTags { String MODEL_PROVIDER = TAG_PREFIX + LLMObsTags.MODEL_PROVIDER; String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; + String INTEGRATION = TAG_PREFIX + "integration"; String VERSION = TAG_PREFIX + "version"; String SOURCE = TAG_PREFIX + "source"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index fae2880c082..033381d60bb 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -99,6 +99,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); span.setTag(CommonTags.SOURCE, "integration"); + span.setTag(CommonTags.INTEGRATION, INTEGRATION); AgentSpanContext parent = LLMObsContext.current(); String parentSpanId = LLMObsContext.ROOT_SPAN_ID; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index fad254f69c5..c4a066d1895 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -340,6 +340,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "$CommonTags.TOOL_DEFINITIONS" List } "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/chat/completions" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 72ffaee8052..de9af838086 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -166,6 +166,7 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/completions" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 44999641fc2..112b649a856 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -65,6 +65,7 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.total_tokens" Long diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 3b205d8496f..030f61009ac 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -219,6 +219,7 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_metric.cache_read_input_tokens" Long "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String + "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" "$CommonTags.ERROR" 0 diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7ff16cbed4e..cb8ebd3d8a1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -70,6 +70,7 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPANS = "spans".getBytes(StandardCharsets.UTF_8); private static final byte[] METRICS = "metrics".getBytes(StandardCharsets.UTF_8); private static final byte[] TAGS = "tags".getBytes(StandardCharsets.UTF_8); + private static final String LLMOBS_LANGUAGE_TAG = "language:jvm"; private static final byte[] LLM_MESSAGE_ROLE = "role".getBytes(StandardCharsets.UTF_8); private static final byte[] LLM_MESSAGE_CONTENT = "content".getBytes(StandardCharsets.UTF_8); @@ -293,7 +294,7 @@ public void accept(Metadata metadata) { // write tags (10) writable.writeUTF8(TAGS); writable.startArray(tagsSize + 1); - writable.writeString("language:jvm", null); + writable.writeString(LLMOBS_LANGUAGE_TAG, null); for (Map.Entry tag : metadata.getTags().entrySet()) { String key = tag.getKey(); Object value = tag.getValue(); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 6140431b836..3cf52ae0150 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -160,7 +160,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace = [regularSpan1, regularSpan2] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace, mapper) @@ -204,7 +205,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace1 = [llmSpan1, llmSpan2] def trace2 = [llmSpan3] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace1, mapper) From 668e955e352ee8306d2f9ea1ce69570e15fe837f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 11:01:25 +0100 Subject: [PATCH 26/39] Add ddtrace.verion --- .../java/datadog/trace/llmobs/domain/DDLLMObsSpan.java | 3 +++ .../datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy | 9 +++++++++ .../trace/instrumentation/openai_java/CommonTags.java | 1 + .../instrumentation/openai_java/OpenAiDecorator.java | 2 ++ .../src/test/groovy/ChatCompletionServiceTest.groovy | 1 + .../src/test/groovy/CompletionServiceTest.groovy | 1 + .../src/test/groovy/EmbeddingServiceTest.groovy | 1 + .../src/test/groovy/ResponseServiceTest.groovy | 1 + 8 files changed, 19 insertions(+) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 56bc1f69c88..c7636f9cd7a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -2,6 +2,7 @@ import datadog.context.ContextScope; import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.DDTraceId; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObs; @@ -38,6 +39,7 @@ public class DDLLMObsSpan implements LLMObsSpan { private static final String SERVICE = LLMOBS_TAG_PREFIX + "service"; private static final String VERSION = LLMOBS_TAG_PREFIX + "version"; + private static final String DDTRACE_VERSION = LLMOBS_TAG_PREFIX + "ddtrace.version"; private static final String ENV = LLMOBS_TAG_PREFIX + "env"; private static final String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; @@ -74,6 +76,7 @@ public DDLLMObsSpan( this.span.setTag(ENV, wellKnownTags.getEnv()); this.span.setTag(SERVICE, wellKnownTags.getService()); this.span.setTag(VERSION, wellKnownTags.getVersion()); + this.span.setTag(DDTRACE_VERSION, DDTraceApiInfo.VERSION); this.span.setTag(SPAN_KIND, kind); this.spanKind = kind; diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index 7595b51e82a..87123f9f473 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.llmobs.domain import datadog.trace.agent.tooling.TracerInstaller import datadog.trace.api.DDTags +import datadog.trace.api.DDTraceApiInfo import datadog.trace.api.IdGenerationStrategy import datadog.trace.api.WellKnownTags import datadog.trace.api.llmobs.LLMObs @@ -131,6 +132,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test span with overwrites"() { @@ -216,6 +219,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span string input formatted to messages"() { @@ -267,6 +272,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span with messages"() { @@ -323,6 +330,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } private LLMObsSpan givenALLMObsSpan(String kind, name){ diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index c9917332e7e..a992c85400c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -21,6 +21,7 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String INTEGRATION = TAG_PREFIX + "integration"; String VERSION = TAG_PREFIX + "version"; + String DDTRACE_VERSION = TAG_PREFIX + "ddtrace.version"; String SOURCE = TAG_PREFIX + "source"; String ERROR = TAG_PREFIX + "error"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 033381d60bb..1bb3530fc93 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -4,6 +4,7 @@ import com.openai.core.http.Headers; import datadog.trace.api.Config; import datadog.trace.api.DDTags; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -96,6 +97,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.ENV, wellKnownTags.getEnv()); span.setTag(CommonTags.SERVICE, wellKnownTags.getService()); span.setTag(CommonTags.VERSION, wellKnownTags.getVersion()); + span.setTag(CommonTags.DDTRACE_VERSION, DDTraceApiInfo.VERSION); span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); span.setTag(CommonTags.SOURCE, "integration"); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index c4a066d1895..28430fcccac 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -336,6 +336,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { "$CommonTags.TOOL_DEFINITIONS" List } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index de9af838086..dcf537df854 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -165,6 +165,7 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 112b649a856..32468dd00df 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -64,6 +64,7 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 030f61009ac..62f64b04223 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -221,6 +221,7 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.ERROR" 0 if (reasoning != null) { From d57402ef743e1a5cd566cf654bc0d66b8023d548 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 12:08:53 +0100 Subject: [PATCH 27/39] Improve test assertions --- .../groovy/ChatCompletionServiceTest.groovy | 28 ++++++- .../test/groovy/CompletionServiceTest.groovy | 46 ++++++++--- .../test/groovy/EmbeddingServiceTest.groovy | 15 ++++ .../test/groovy/ResponseServiceTest.groovy | 76 ++++++++++++++++--- .../ddintake/LLMObsSpanMapperTest.groovy | 39 +++++++++- 5 files changed, 177 insertions(+), 27 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 28430fcccac..7376f73f92e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -147,7 +147,18 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(false, outputTag, [:], true) + List> toolDefinitions = [] + assertChatCompletionTrace(false, outputTag, [:], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" + toolDefinitions[0].schema.type == "object" + (toolDefinitions[0].schema.properties as Map).containsKey("name") + (toolDefinitions[0].schema.properties as Map).containsKey("major") + (toolDefinitions[0].schema.properties as Map).containsKey("school") + (toolDefinitions[0].schema.properties as Map).containsKey("grades") + (toolDefinitions[0].schema.properties as Map).containsKey("clubs") and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -178,7 +189,12 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(true, outputTag, [stream: true], true) + List> toolDefinitions = [] + assertChatCompletionTrace(true, outputTag, [stream: true], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -294,10 +310,10 @@ class ChatCompletionServiceTest extends OpenAiTest { } private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata) { - assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false) + assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false, null) } - private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions) { + private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions, List> toolDefinitionsOut) { def expectedMetadata = new LinkedHashMap(metadata) expectedMetadata.putIfAbsent("stream", isStreaming) @@ -339,6 +355,10 @@ class ChatCompletionServiceTest extends OpenAiTest { "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { "$CommonTags.TOOL_DEFINITIONS" List + def toolDefinitions = tag("$CommonTags.TOOL_DEFINITIONS") + if (toolDefinitionsOut != null && toolDefinitions != null) { + toolDefinitionsOut.addAll(toolDefinitions) + } } "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index dcf537df854..8ca98ca3677 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -8,6 +8,7 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes +import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags import java.util.concurrent.CompletableFuture import java.util.stream.Stream @@ -20,7 +21,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -35,7 +36,7 @@ class CompletionServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -52,7 +53,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -69,7 +70,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -83,7 +84,7 @@ class CompletionServiceTest extends OpenAiTest { completionFuture.get() expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -98,7 +99,7 @@ class CompletionServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -113,7 +114,7 @@ class CompletionServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -131,13 +132,17 @@ class CompletionServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] } - private void assertCompletionTrace() { + private void assertCompletionTrace(boolean streamRequest) { + List inputTagsOut = [] + List outputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -157,8 +162,20 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long @@ -193,5 +210,16 @@ class CompletionServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].role == "" + assert inputTagsOut[0].content == "Tell me a story about building the best SDK!" + assert outputTagsOut.size() >= 1 + assert outputTagsOut.every { it.role == "" } + if (streamRequest) { + assert metadataOut.stream_options == [include_usage: true] + } else { + assert !metadataOut.containsKey("stream_options") + } } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 32468dd00df..0a4f76ff47a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -40,6 +40,9 @@ class EmbeddingServiceTest extends OpenAiTest { } private void assertEmbeddingTrace() { + List inputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -59,7 +62,15 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" "text-embedding-ada-002-v2" "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.output" "[1 embedding(s) returned with size 1536]" "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String @@ -94,5 +105,9 @@ class EmbeddingServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].text == "hello world" + assert metadataOut == [encoding_format: "base64"] } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 62f64b04223..4d26098050c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -23,7 +23,10 @@ class ResponseServiceTest extends OpenAiTest { expect: resp != null and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -38,7 +41,10 @@ class ResponseServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -55,7 +61,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: scenario | params @@ -76,7 +85,11 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"]) + Map metadata = [:] + assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"], null, null, metadata) + and: + metadata.stream == true + metadata.reasoning == [effort: "medium", summary: "detailed"] where: responseCreateParams << [responseCreateParamsWithReasoning(false), responseCreateParamsWithReasoning(true)] @@ -93,7 +106,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -107,7 +123,10 @@ class ResponseServiceTest extends OpenAiTest { responseFuture.get() expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -122,7 +141,10 @@ class ResponseServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -137,7 +159,10 @@ class ResponseServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -155,7 +180,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -173,8 +201,17 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags) + Map metadata = [:] + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata) and: + metadata.stream == true + inputTags.size() == 3 + inputTags[1].toolCalls.size() == 1 + inputTags[1].toolCalls[0].name == "get_weather" + inputTags[1].toolCalls[0].type == "function_call" + inputTags[1].toolCalls[0].arguments == [location: "San Francisco, CA"] + inputTags[2].toolResults.size() == 1 + inputTags[2].toolResults[0].type == "function_call_output" !inputTags.isEmpty() inputTags[2].toolResults[0].result == '{"temperature": "72°F", "conditions": "sunny", "humidity": "65%"}' @@ -183,10 +220,17 @@ class ResponseServiceTest extends OpenAiTest { } private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning) { - assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null) + assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null, null, null) } - private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning, List inputTagsOut) { + private void assertResponseTrace( + boolean isStreaming, + String reqModel, + String respModel, + Map reasoning, + List inputTagsOut, + List outputTagsOut, + Map metadataOut) { assertTraces(1) { trace(3) { sortSpansByStart() @@ -206,12 +250,20 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadataOut != null && metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.input" List def inputTags = tag("_ml_obs_tag.input") if (inputTagsOut != null && inputTags != null) { inputTagsOut.addAll(inputTags) } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTagsOut != null && outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 3cf52ae0150..6df08aa39ce 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -40,11 +40,29 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { llmSpan.setSpanType(InternalSpanTypes.LLMOBS) - def inputMessages = [LLMObs.LLMMessage.from("user", "Hello, what's the weather like?")] + def toolCall = LLMObs.ToolCall.from("get_weather", "function_call", "call_123", [location: "San Francisco"]) + def toolResult = LLMObs.ToolResult.from("get_weather", "function_call_output", "call_123", '{"temperature":"72F"}') + def inputMessages = [ + LLMObs.LLMMessage.from("user", "Hello, what's the weather like?"), + LLMObs.LLMMessage.from("assistant", null, [toolCall], [toolResult]) + ] def outputMessages = [LLMObs.LLMMessage.from("assistant", "I'll help you check the weather.")] - llmSpan.setTag("_ml_obs_tag.input", inputMessages) + llmSpan.setTag("_ml_obs_tag.input", [ + messages: inputMessages, + prompt: [ + id: "prompt_123", + version: "1", + variables: [city: "San Francisco"], + chat_template: [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] + ] + ]) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) + llmSpan.setTag("_ml_obs_tag.tool_definitions", [[ + name: "get_weather", + description: "Get weather by city", + schema: [type: "object", properties: [city: [type: "string"]]] + ]]) llmSpan.setError(true) llmSpan.setTag(DDTags.ERROR_MSG, "boom") llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") @@ -123,12 +141,29 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData["meta"]["input"]["messages"][0]["content"] == "Hello, what's the weather like?" spanData["meta"]["input"]["messages"][0].containsKey("role") spanData["meta"]["input"]["messages"][0]["role"] == "user" + spanData["meta"]["input"]["messages"][1]["role"] == "assistant" + !spanData["meta"]["input"]["messages"][1].containsKey("content") + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["type"] == "function_call" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["arguments"] == [location: "San Francisco"] + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["type"] == "function_call_output" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["result"] == '{"temperature":"72F"}' + spanData["meta"]["input"]["prompt"]["id"] == "prompt_123" + spanData["meta"]["input"]["prompt"]["version"] == "1" + spanData["meta"]["input"]["prompt"]["variables"] == [city: "San Francisco"] + spanData["meta"]["input"]["prompt"]["chat_template"] == [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] spanData["meta"].containsKey("output") spanData["meta"]["output"].containsKey("messages") spanData["meta"]["output"]["messages"][0].containsKey("content") spanData["meta"]["output"]["messages"][0]["content"] == "I'll help you check the weather." spanData["meta"]["output"]["messages"][0].containsKey("role") spanData["meta"]["output"]["messages"][0]["role"] == "assistant" + spanData["meta"]["tool_definitions"][0]["name"] == "get_weather" + spanData["meta"]["tool_definitions"][0]["description"] == "Get weather by city" + spanData["meta"]["tool_definitions"][0]["schema"] == [type: "object", properties: [city: [type: "string"]]] spanData["meta"].containsKey("metadata") spanData.containsKey("metrics") From 0c879ba692386cd944b87734bcfce2285beaa37e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 12:53:59 +0100 Subject: [PATCH 28/39] Fix format --- .../openai_java/ResponseDecorator.java | 4 ++++ .../src/test/groovy/ResponseServiceTest.groovy | 14 +++++++------- .../writer/ddintake/LLMObsSpanMapperTest.groovy | 12 +++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 07359a9b326..9488ad0c865 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -710,6 +710,10 @@ private List extractInputMessagesForPromptTracking( } } + if (!messages.isEmpty()) { + return messages; + } + Object inputTag = span.getTag(CommonTags.INPUT); if (inputTag instanceof List) { for (Object messageObj : (List) inputTag) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 4d26098050c..02e28237750 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -224,13 +224,13 @@ class ResponseServiceTest extends OpenAiTest { } private void assertResponseTrace( - boolean isStreaming, - String reqModel, - String respModel, - Map reasoning, - List inputTagsOut, - List outputTagsOut, - Map metadataOut) { + boolean isStreaming, + String reqModel, + String respModel, + Map reasoning, + List inputTagsOut, + List outputTagsOut, + Map metadataOut) { assertTraces(1) { trace(3) { sortSpansByStart() diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 6df08aa39ce..7d7de1180a7 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -58,11 +58,13 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { ]) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) - llmSpan.setTag("_ml_obs_tag.tool_definitions", [[ - name: "get_weather", - description: "Get weather by city", - schema: [type: "object", properties: [city: [type: "string"]]] - ]]) + llmSpan.setTag("_ml_obs_tag.tool_definitions", [ + [ + name: "get_weather", + description: "Get weather by city", + schema: [type: "object", properties: [city: [type: "string"]]] + ] + ]) llmSpan.setError(true) llmSpan.setTag(DDTags.ERROR_MSG, "boom") llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") From f4e3a8b50462fd00f1122324a5b699fb715b1b4b Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 17 Mar 2026 17:35:20 -0700 Subject: [PATCH 29/39] Include input messages when instructions are present in prompt tracking --- .../openai_java/ResponseDecorator.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 9488ad0c865..1b66e11e098 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -692,33 +692,35 @@ private List extractInputMessagesForPromptTracking( } }); - if (!messages.isEmpty()) { - return messages; - } - // Fallback for SDK union parsing mismatches: parse raw instructions payload. - Optional rawInstructions = response._instructions().asUnknown(); - if (rawInstructions.isPresent()) { - Optional> rawList = rawInstructions.get().asArray(); - if (rawList.isPresent()) { - for (JsonValue item : rawList.get()) { - LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); - if (message != null) { - messages.add(message); + if (messages.isEmpty()) { + Optional rawInstructions = response._instructions().asUnknown(); + if (rawInstructions.isPresent()) { + Optional> rawList = rawInstructions.get().asArray(); + if (rawList.isPresent()) { + for (JsonValue item : rawList.get()) { + LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); + if (message != null) { + messages.add(message); + } } } } } - if (!messages.isEmpty()) { - return messages; - } + boolean hasInstructions = !messages.isEmpty(); + // Always include input messages from the span tag (set by withResponseCreateParams, which + // records both instructions as "system" and input as "user"). When instructions were already + // collected above, skip "system" messages here to avoid duplicating them. Object inputTag = span.getTag(CommonTags.INPUT); if (inputTag instanceof List) { for (Object messageObj : (List) inputTag) { if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); + LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; + if (!hasInstructions || !"system".equals(msg.getRole())) { + messages.add(msg); + } } } } else if (inputTag instanceof Map) { @@ -726,7 +728,10 @@ private List extractInputMessagesForPromptTracking( if (messagesObj instanceof List) { for (Object messageObj : (List) messagesObj) { if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); + LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; + if (!hasInstructions || !"system".equals(msg.getRole())) { + messages.add(msg); + } } } } From 028d64f1f809961e51dbccc99529eb9eca453c70 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 17 Mar 2026 18:45:42 -0700 Subject: [PATCH 30/39] Fix instructions role to system in prompt tracking --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 1b66e11e098..bba4136d768 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -687,7 +687,7 @@ private List extractInputMessagesForPromptTracking( } else if (instructions.isString()) { String text = instructions.asString(); if (text != null && !text.isEmpty()) { - messages.add(LLMObs.LLMMessage.from("user", text)); + messages.add(LLMObs.LLMMessage.from("system", text)); } } }); From 717a8f0ecf90bd61cb35fde83bde750ec5e14937 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 19 Mar 2026 17:21:13 -0700 Subject: [PATCH 31/39] fix LLMObsSpanMapperTest --- .../trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 7d7de1180a7..fc254458920 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -74,7 +74,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace = [llmSpan] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace, mapper) From 8420f0a1f42072d2cd8d1a013a699a11ba05aefc Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Mar 2026 15:15:30 -0700 Subject: [PATCH 32/39] Catch exception not throwable --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index bba4136d768..7018005488d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -156,7 +156,7 @@ private List> extractToolDefinitionsFromParams(ResponseCreat return toolDefinitions; } } - } catch (Throwable ignored) { + } catch (Exception ignored) { // fall back to raw JSON if typed extraction is unavailable or fails } From 91707faa176a9e6f370321b4eadeed31e83df206 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Mar 2026 16:00:31 -0700 Subject: [PATCH 33/39] Add JsonValueUtilsTest --- .../openai_java/JsonValueUtilsTest.java | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java new file mode 100644 index 00000000000..21713649956 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java @@ -0,0 +1,158 @@ +package datadog.trace.instrumentation.openai_java; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.openai.core.JsonValue; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class JsonValueUtilsTest { + + @Test + void testNullReturnsNull() { + assertNull(JsonValueUtils.jsonValueToObject(null)); + } + + @Test + void testString() { + assertEquals("hello", JsonValueUtils.jsonValueToObject(JsonValue.from("hello"))); + } + + @Test + void testInteger() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(42)); + assertInstanceOf(Number.class, result); + assertEquals(42, ((Number) result).intValue()); + } + + @Test + void testDouble() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(3.14)); + assertInstanceOf(Number.class, result); + assertEquals(3.14, ((Number) result).doubleValue(), 0.0001); + } + + @Test + void testBooleanTrue() { + assertEquals(true, JsonValueUtils.jsonValueToObject(JsonValue.from(true))); + } + + @Test + void testBooleanFalse() { + assertEquals(false, JsonValueUtils.jsonValueToObject(JsonValue.from(false))); + } + + @Test + void testFlatObject() { + Map input = new HashMap<>(); + input.put("key1", "value1"); + input.put("key2", 123); + + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(input)); + + assertInstanceOf(Map.class, result); + @SuppressWarnings("unchecked") + Map map = (Map) result; + assertEquals("value1", map.get("key1")); + assertEquals(123, ((Number) map.get("key2")).intValue()); + } + + @Test + void testNestedObject() { + Map inner = new HashMap<>(); + inner.put("x", "nested"); + Map outer = new HashMap<>(); + outer.put("inner", inner); + + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(outer)); + + assertInstanceOf(Map.class, result); + @SuppressWarnings("unchecked") + Map outerMap = (Map) result; + assertInstanceOf(Map.class, outerMap.get("inner")); + @SuppressWarnings("unchecked") + Map innerMap = (Map) outerMap.get("inner"); + assertEquals("nested", innerMap.get("x")); + } + + @Test + void testFlatArray() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList("a", "b", "c"))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List list = (List) result; + assertEquals(3, list.size()); + assertEquals("a", list.get(0)); + assertEquals("b", list.get(1)); + assertEquals("c", list.get(2)); + } + + @Test + void testNestedArray() { + Object result = + JsonValueUtils.jsonValueToObject( + JsonValue.from(Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3, 4)))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List outer = (List) result; + assertInstanceOf(List.class, outer.get(0)); + @SuppressWarnings("unchecked") + List inner = (List) outer.get(0); + assertEquals(1, ((Number) inner.get(0)).intValue()); + } + + @Test + void testMixedArray() { + Object result = + JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList("text", 42, true))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List list = (List) result; + assertEquals("text", list.get(0)); + assertEquals(42, ((Number) list.get(1)).intValue()); + assertEquals(true, list.get(2)); + } + + @Test + void testEmptyObject() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(new HashMap<>())); + assertInstanceOf(Map.class, result); + assertTrue(((Map) result).isEmpty()); + } + + @Test + void testEmptyArray() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList())); + assertInstanceOf(List.class, result); + assertTrue(((List) result).isEmpty()); + } + + @Test + void testJsonValueMapToObject() { + Map input = new HashMap<>(); + input.put("str", JsonValue.from("hello")); + input.put("num", JsonValue.from(7)); + input.put("bool", JsonValue.from(false)); + + Map result = JsonValueUtils.jsonValueMapToObject(input); + + assertEquals("hello", result.get("str")); + assertEquals(7, ((Number) result.get("num")).intValue()); + assertEquals(false, result.get("bool")); + } + + @Test + void testJsonValueMapToObjectEmpty() { + Map result = JsonValueUtils.jsonValueMapToObject(new HashMap<>()); + assertTrue(result.isEmpty()); + } +} From 3d12515bb74cd8b541658dfffc875f9375c622d5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Mar 2026 17:54:25 -0700 Subject: [PATCH 34/39] Test that on HTTP error, the OpenAI response span retains model_name and placeholder output set by withResponseCreateParams. --- .../test/groovy/ResponseServiceTest.groovy | 67 ++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 02e28237750..ff66d56cd20 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -1,11 +1,14 @@ import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace +import com.openai.client.okhttp.OpenAIOkHttpClient import com.openai.core.http.AsyncStreamResponse import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse +import com.openai.credential.BearerTokenCredential import com.openai.models.responses.Response import com.openai.models.responses.ResponseStreamEvent +import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.DDSpanTypes import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags @@ -219,10 +222,70 @@ class ResponseServiceTest extends OpenAiTest { responseCreateParams << [responseCreateParamsWithToolInput(false), responseCreateParamsWithToolInput(true)] } - private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning) { - assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null, null, null) + def "create response error sets model_name and placeholder output"() { + setup: + def errorBackend = TestHttpServer.httpServer { + handlers { + prefix("/v1/") { + response.status(500).send('{"error":{"message":"Internal server error","type":"server_error"}}') + } + } + } + def errorClient = OpenAIOkHttpClient.builder() + .baseUrl("${errorBackend.address.toURL()}/v1") + .credential(BearerTokenCredential.create("")) + .maxRetries(0) + .build() + + when: + runUnderTrace("parent") { + try { + errorClient.responses().create(responseCreateParams(false)) + } catch (Exception ignored) {} + } + + then: + List outputMessages = [] + assertTraces(1) { + trace(3) { + sortSpansByStart() + span(0) { + operationName "parent" + parent() + errored false + } + span(1) { + operationName "openai.request" + resourceName "createResponse" + childOf span(0) + errored true + spanType DDSpanTypes.LLMOBS + tags(false) { + "_ml_obs_tag.model_name" "gpt-3.5-turbo" + "_ml_obs_tag.output" List + def out = tag("_ml_obs_tag.output") + if (out instanceof List) outputMessages.addAll(out) + } + } + span(2) { + operationName "okhttp.request" + resourceName "POST /v1/responses" + childOf span(1) + errored false + spanType "http" + } + } + } + and: + outputMessages.size() == 1 + outputMessages[0].role == "" + outputMessages[0].content == "" + + cleanup: + errorBackend.close() } + private void assertResponseTrace( boolean isStreaming, String reqModel, From 576cec73abdbb5614fda2e5c06d4a1a45d2261cf Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 11:41:22 -0700 Subject: [PATCH 35/39] Add "create response with prompt tracking" test to improve coverage of enrichInputWithPromptTracking(), extractChatTemplate(), extractPromptFromParams(), and normalizePromptVariable() --- .../src/test/groovy/OpenAiTest.groovy | 27 ++- .../test/groovy/ResponseServiceTest.groovy | 53 +++++- ...25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec | 178 ++++++++++++++++++ 3 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index 542f9e27960..34661da8954 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -21,6 +21,7 @@ import com.openai.models.responses.ResponseCreateParams import com.openai.models.responses.ResponseFunctionToolCall import com.openai.models.responses.ResponseIncludable import com.openai.models.responses.ResponseInputItem +import com.openai.models.responses.ResponsePrompt import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.config.LlmObsConfig @@ -310,6 +311,31 @@ He hopes to pursue a career in software engineering after graduating.""") } } + ResponseCreateParams responseCreateParamsWithPromptTracking(boolean json) { + def variables = ResponsePrompt.Variables.builder() + .putAdditionalProperty("user_message", JsonValue.from([type: "input_text", text: "Analyze these images and document"])) + .putAdditionalProperty("user_image_1", JsonValue.from([type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"])) + .putAdditionalProperty("user_file", JsonValue.from([type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"])) + .putAdditionalProperty("user_image_2", JsonValue.from([type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"])) + .build() + + def prompt = ResponsePrompt.builder() + .id("pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0") + .version("2") + .variables(variables) + .build() + + if (json) { + ResponseCreateParams.builder() + .prompt(prompt) + .build() + } else { + ResponseCreateParams.builder() + .prompt(prompt) + .build() + } + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() @@ -328,4 +354,3 @@ He hopes to pursue a career in software engineering after graduating.""") } } } - diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index ff66d56cd20..2a5b246ff6e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -222,6 +222,29 @@ class ResponseServiceTest extends OpenAiTest { responseCreateParams << [responseCreateParamsWithToolInput(false), responseCreateParamsWithToolInput(true)] } + def "create response with prompt tracking"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + expect: + resp != null + and: + Map metadata = [:] + Map input = [:] + assertResponseTrace(false, null, String, null, input, null, metadata, true) + and: + metadata.stream == false + input.prompt instanceof Map + input.messages instanceof List + def prompt = input.prompt as Map + prompt.chat_template instanceof List + prompt.variables instanceof Map + + where: + params << [responseCreateParamsWithPromptTracking(false), responseCreateParamsWithPromptTracking(true)] + } + def "create response error sets model_name and placeholder output"() { setup: def errorBackend = TestHttpServer.httpServer { @@ -289,11 +312,12 @@ class ResponseServiceTest extends OpenAiTest { private void assertResponseTrace( boolean isStreaming, String reqModel, - String respModel, + Object respModel, Map reasoning, - List inputTagsOut, + Object inputTagsOut, List outputTagsOut, - Map metadataOut) { + Map metadataOut, + boolean expectPromptTag = false) { assertTraces(1) { trace(3) { sortSpansByStart() @@ -313,14 +337,25 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + if (expectPromptTag) { + "_ml_obs_request.prompt" Map + } def metadata = tag("_ml_obs_tag.metadata") if (metadataOut != null && metadata != null) { metadataOut.putAll(metadata) } - "_ml_obs_tag.input" List + if (inputTagsOut instanceof Map) { + "_ml_obs_tag.input" Map + } else { + "_ml_obs_tag.input" List + } def inputTags = tag("_ml_obs_tag.input") if (inputTagsOut != null && inputTags != null) { - inputTagsOut.addAll(inputTags) + if (inputTagsOut instanceof List) { + inputTagsOut.addAll(inputTags as List) + } else if (inputTagsOut instanceof Map) { + inputTagsOut.putAll(inputTags as Map) + } } "_ml_obs_tag.output" List def outputTags = tag("_ml_obs_tag.output") @@ -347,13 +382,15 @@ class ResponseServiceTest extends OpenAiTest { "openai.api_base" openAiBaseApi "$CommonTags.OPENAI_RESPONSE_MODEL" respModel if (!isStreaming) { - "openai.organization.ratelimit.requests.limit" 10000 + "openai.organization.ratelimit.requests.limit" Integer "openai.organization.ratelimit.requests.remaining" Integer - "openai.organization.ratelimit.tokens.limit" 50000000 + "openai.organization.ratelimit.tokens.limit" Integer "openai.organization.ratelimit.tokens.remaining" Integer } "$CommonTags.OPENAI_ORGANIZATION" "datadog-staging" - "$CommonTags.OPENAI_REQUEST_MODEL" reqModel + if (reqModel != null) { + "$CommonTags.OPENAI_REQUEST_MODEL" reqModel + } "$Tags.COMPONENT" "openai" "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT defaultTags() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec new file mode 100644 index 00000000000..a72f03b8a33 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec @@ -0,0 +1,178 @@ +method: POST +path: responses +-- begin request body -- +{"prompt":{"id":"pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0","variables":{"user_message":{"type":"input_text","text":"Analyze these images and document"},"user_image_1":{"type":"input_image","image_url":"https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"},"user_file":{"type":"input_file","file_url":"https://www.berkshirehathaway.com/letters/2024ltr.pdf"},"user_image_2":{"type":"input_image","file_id":"file-BCuhT1HQ24kmtsuuzF1mh2"}},"version":"2"}} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e1fdb67e8b3d84e-SEA +content-type: application/json +date: Wed, 25 Mar 2026 18:08:13 GMT +openai-organization: datadog-staging +openai-processing-ms: 27175 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=0omhm_EDPiOwCkTnRHMTcH2143C9foKJkpkM5V98XPA-1774462065.9075637-1.0.1.1-Dr2KjfEYzM5.5hH26NB1U9ok305D3x0N1Jj2oNi9HRpE49YL0u3dMPv7FNeawtk.1duphFGgfCFpcbqn_qCR7bW1MtVt6IISK3BKO_TDbjrSg3oVneqJuZ62wmSRDDb2; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026 18:38:13 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 30000 +x-ratelimit-limit-tokens: 150000000 +x-ratelimit-remaining-requests: 29999 +x-ratelimit-remaining-tokens: 149998470 +x-ratelimit-reset-requests: 2ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_55ed921a6d294700bce3397e0b75e7da +-- end response headers -- +-- begin response body -- +{ + "id": "resp_00cd8b3bfcd5108a0169c42472356c8190a2f61c5a5f3b32e7", + "object": "response", + "created_at": 1774462067, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774462093, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": [ + { + "type": "message", + "content": [], + "role": "developer" + }, + { + "type": "message", + "content": [ + { + "type": "input_text", + "text": "Analyze the following content from the user:\n\nText message: Analyze these images and document\nImage reference 1: " + }, + { + "type": "input_image", + "detail": "auto", + "file_id": null, + "image_url": null + }, + { + "type": "input_text", + "text": "\nDocument reference: " + }, + { + "type": "input_file", + "file_id": null, + "file_url": "https://www.berkshirehathaway.com/letters/2024ltr.pdf" + }, + { + "type": "input_text", + "text": "\nImage reference 2: " + }, + { + "type": "input_image", + "detail": "auto", + "file_id": "file-BCuhT1HQ24kmtsuuzF1mh2", + "image_url": null + }, + { + "type": "input_text", + "text": "\n\nPlease provide a comprehensive analysis." + } + ], + "role": "user" + } + ], + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "output": [ + { + "id": "rs_00cd8b3bfcd5108a0169c424754a808190a4090b0188003f86", + "type": "reasoning", + "summary": [] + }, + { + "id": "msg_00cd8b3bfcd5108a0169c42480774c819082158e702c85be03", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Here\u2019s a step-by-step breakdown of the three items you\u2019ve provided:\n\n1. Image 1 (the two-snake logo) \n \u2022 Identification: This is the official logo of the Python programming language. \n \u2022 Design elements: \n \u2013 Two interlocking \u201csnake\u201d shapes (one blue, one yellow) arranged to form a stylized \u201cP.\u201d \n \u2013 Rounded corners, minimal line-weight, flat color palette\u2014characteristics of modern \u201cflat\u201d logo design. \n \u2013 Created to evoke both the language\u2019s namesake (Monty Python aside, a snake) and its emphasis on readability and simplicity. \n \u2022 Historical/contextual notes: \n \u2013 Python was conceived by Guido van Rossum in the late 1980s and first released in 1991. \n \u2013 Over the years the logo has come to symbolize a massive ecosystem\u2014from web back-ends (Django, Flask) to data science (Pandas, NumPy) to machine learning (scikit-learn, TensorFlow). \n \u2013 Its clean, friendly shape mirrors Python\u2019s design philosophy: code that reads like plain English, easy onboarding for newcomers. \n\n2. The Berkshire Hathaway 2024 Shareholder Letter (Warren E. Buffett) \n A. Purpose & Tone \n \u2013 Beyond the mandatory financial \u201creport,\u201d Buffett uses the letter to provide candid commentary\u2014both on successes and on mistakes. \n \u2013 He stresses the obligation of a CEO to treat shareholders as he\u2019d want to be treated if the roles were reversed. \n \u2013 He repeatedly admits errors (16 uses of \u201cmistake\u201d or \u201cerror\u201d in the last five years), contrasting this transparency with what he sees as glib \u201cfeel-good\u201d letters from many large companies. \n\n B. Major Themes \n 1. The Importance of Admitting and Correcting Mistakes \n \u2013 Mistakes in acquisitions or in assessing management fidelity can be painful but must be addressed promptly (\u201cno thumb-sucking\u201d). \n \u2013 Single great decisions\u2014e.g., GEICO purchase, finding Charlie Munger\u2014can compound into decade-long outperformance. \n\n 2. Capital Allocation Philosophy \n \u2013 The core measure Buffett emphasizes is \u201coperating earnings,\u201d excluding unrealized securities gains or losses. \n \u2013 He reaffirms that Berkshire will remain overwhelmingly invested in equities (both controlled subsidiaries and minority stakes in giants like Apple, Coca-Cola, American Express). \n \u2013 He warns against over-reliance on cash or bonds, which carry currency\u2010value risk; instead, intrinsic ownership of businesses is the best inflation hedge. \n\n 3. Insurance Float as a Growth Engine \n \u2013 Property-casualty underwriting delivers \u201cfloat\u201d (premiums received before claims are paid). \n \u2013 If priced intelligently, float can be \u201ccostless capital\u201d that Buffett can then deploy elsewhere. \n \u2013 He lauds Ajit Jain for building Berkshire\u2019s P/C operation into a global leader, but stresses the business is perilously dependent on accurate long-term loss estimates. \n\n 4. Record U.S. Tax Payments \n \u2013 In 2024 Berkshire paid $26.8 billion in U.S. federal income taxes\u2014more than any other corporation in American history. \n \u2013 Buffett attributes this partly to his long-ago decision (and shareholders\u2019 endorsement) to forgo dividends, instead reinvesting all earnings back into operations. \n\n 5. Discipline in CEO and Manager Selection \n \u2013 Buffett never considers pedigree (school attended) when hiring CEOs\u2014innate talent often outweighs credentials. \n \u2013 He illustrates this with the story of Pete Liegl (founder of Forest River), whose self-set $100 k base salary plus performance bonus yielded multiples of that investment for Berkshire over 19 years. \n\n 6. International Ventures\u2014The Japan Five \n \u2013 Since 2019 Berkshire has taken minority stakes (below 10%) in five large Japanese trading houses (Itochu, Marubeni, Mitsubishi, Mitsui, Sumitomo). \n \u2013 He praises their capital\u2010allocation discipline, low executive pay, willingness to buy back shares. \n \u2013 Berkshire balances that foreign\u2010currency exposure by matching it with yen-denominated debt, achieving near currency neutrality. \n\n 7. Shareholder-Centered Culture & Annual Meeting \n \u2013 Buffett invites shareholders to the May 3 gathering in Omaha, emphasizing accessibility of management, Q&A format, volunteer-run merch booths, and a single themed book for sale. \n \u2013 He recounts lighthearted family anecdotes (his sister Bertie\u2019s cane, grandchildren) to underscore the meeting\u2019s informal, \u201cfamily reunion\u201d atmosphere. \n\n C. Performance Snapshot \n \u2013 2024 operating earnings: $47.4 billion vs. $37.4 billion in 2023. \n \u2013 Insurance underwriting profit: $9.0 billion; investment income: $13.7 billion. \n \u2013 Continued strength at BNSF and Berkshire Hathaway Energy. \n \u2013 Slide reinstates the 1965\u20132024 compound annual gain of 19.9% vs. 10.4% for the S&P 500. \n\n D. Stylistic & Structural Notes \n \u2013 Plain\u2010spoken, anecdote-rich, avoids corporate jargon. \n \u2013 Buffett interleaves big-picture economic observations (capitalism\u2019s triumphs and pitfalls) with very granular \u201cinside\u201d details (how he negotiated with Pete Liegl). \n \u2013 Occasional dry humor (e.g., comparing his and his sister\u2019s canes as \u201cmale-deterrents\u201d). \n\n3. Image 2 \n \u2022 On my end the image rendered as a blank/empty panel. There\u2019s no visible content to describe or analyze. \n \u2022 If this was meant to convey text, a chart, code or a photograph, could you please re-upload a higher-resolution copy or a different file format? \n\n\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013 \nSummary: \n\u2013 You\u2019ve supplied two corporate \u201cicons\u201d (the Python logo and the Berkshire shareholder letter), both of which epitomize transparency, community-oriented design, and long-term stewardship\u2014one in software, the other in capital allocation. \n\u2013 The Python logo signals a language built for clarity, extensibility and an open ecosystem. \n\u2013 Buffett\u2019s letter models \u201cflat,\u201d jargon-free communication, candidly reporting mistakes alongside successes, and reaffirming a shareholder-centric culture. \n\u2013 Please let me know about Image 2, and I\u2019ll be happy to complete the visual analysis." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt": { + "id": "pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0", + "variables": { + "user_message": { + "type": "input_text", + "text": "Analyze these images and document" + }, + "user_image_1": { + "type": "input_image", + "detail": "auto", + "file_id": null, + "image_url": null + }, + "user_file": { + "type": "input_file", + "file_id": null, + "file_url": "https://www.berkshirehathaway.com/letters/2024ltr.pdf" + }, + "user_image_2": { + "type": "input_image", + "detail": "auto", + "file_id": "file-BCuhT1HQ24kmtsuuzF1mh2", + "image_url": null + } + }, + "version": "2" + }, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 11236, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 2366, + "output_tokens_details": { + "reasoning_tokens": 960 + }, + "total_tokens": 13602 + }, + "user": null, + "metadata": {} +}� +-- end response body -- From ba0cb2702698f225a67b67f088eb289e48b2fb5f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 14:55:07 -0700 Subject: [PATCH 36/39] Add "create response with custom tool call" test to improve coverage of getToolCall --- .../src/test/groovy/OpenAiTest.groovy | 31 +++++ .../test/groovy/ResponseServiceTest.groovy | 23 +++- ...4919ef6198916e1a+cf7135b364dfeac5.POST.rec | 112 ++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index 34661da8954..c265c704b7e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -22,6 +22,8 @@ import com.openai.models.responses.ResponseFunctionToolCall import com.openai.models.responses.ResponseIncludable import com.openai.models.responses.ResponseInputItem import com.openai.models.responses.ResponsePrompt +import com.openai.models.responses.CustomTool +import com.openai.models.responses.ToolChoiceCustom import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.config.LlmObsConfig @@ -336,6 +338,35 @@ He hopes to pursue a career in software engineering after graduating.""") } } + ResponseCreateParams responseCreateParamsWithCustomToolCall(boolean json) { + def customTool = CustomTool.builder() + .name("custom_weather") + .description("Return weather for a location") + .formatText() + .build() + + def toolChoice = ToolChoiceCustom.builder() + .name("custom_weather") + .type(JsonValue.from("custom")) + .build() + + if (json) { + ResponseCreateParams.builder() + .model("gpt-5") + .input("Use the custom_weather tool to answer: What's the weather in Boston?") + .addTool(customTool) + .toolChoice(toolChoice) + .build() + } else { + ResponseCreateParams.builder() + .model(ChatModel.GPT_5) + .input("Use the custom_weather tool to answer: What's the weather in Boston?") + .addTool(customTool) + .toolChoice(toolChoice) + .build() + } + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 2a5b246ff6e..f704fd3cb55 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -205,7 +205,7 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] Map metadata = [:] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata) + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata, false, true) and: metadata.stream == true inputTags.size() == 3 @@ -245,6 +245,24 @@ class ResponseServiceTest extends OpenAiTest { params << [responseCreateParamsWithPromptTracking(false), responseCreateParamsWithPromptTracking(true)] } + def "create response with custom tool call"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + expect: + resp != null + and: + List outputTags = [] + Map metadata = [:] + assertResponseTrace(false, "gpt-5", String, null, null, outputTags, metadata) + and: + !outputTags.isEmpty() + + where: + params << [responseCreateParamsWithCustomToolCall(false), responseCreateParamsWithCustomToolCall(true)] + } + def "create response error sets model_name and placeholder output"() { setup: def errorBackend = TestHttpServer.httpServer { @@ -317,7 +335,8 @@ class ResponseServiceTest extends OpenAiTest { Object inputTagsOut, List outputTagsOut, Map metadataOut, - boolean expectPromptTag = false) { + boolean expectPromptTag = false, + boolean expectRateLimitTags = false) { assertTraces(1) { trace(3) { sortSpansByStart() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec new file mode 100644 index 00000000000..8b748b9b57a --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec @@ -0,0 +1,112 @@ +method: POST +path: responses +-- begin request body -- +{"input":"Use the custom_weather tool to answer: What's the weather in Boston?","model":"gpt-5","tool_choice":{"name":"custom_weather","type":"custom"},"tools":[{"name":"custom_weather","type":"custom","description":"Return weather for a location","format":{"type":"text"}}]} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e2122d2ae6712d4-SEA +content-type: application/json +date: Wed, 25 Mar 2026 21:51:20 GMT +openai-organization: datadog-staging +openai-processing-ms: 3360 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=SP753lo3ga6JHum4FOPTYPa6B.h7QhGpK2Tih8iqYOI-1774475476.9017334-1.0.1.1-G_WqthMsAoHTvAkAACMpAsVWmtzzeqOHTjGZ_eY4WSxZz_pND9T9O78.pCiPD52gquRIzVxrXPd9J06VIMqiuAA9z_GWrLKLGc.s2dzDvUsWxIek6wIPjN4EOBFNlgoB; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026 22:21:20 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 15000 +x-ratelimit-limit-tokens: 40000000 +x-ratelimit-remaining-requests: 14999 +x-ratelimit-remaining-tokens: 40000000 +x-ratelimit-reset-requests: 4ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_dcbdd1cbcd494ff793ee2e099de4554e +-- end response headers -- +-- begin response body -- +{ + "id": "resp_0f18ec8851ed77310169c458d536188190ae13c792509ad0c7", + "object": "response", + "created_at": 1774475477, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774475480, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5-2025-08-07", + "output": [ + { + "id": "rs_0f18ec8851ed77310169c458d5b5108190a0c49289af33d93b", + "type": "reasoning", + "summary": [] + }, + { + "id": "ctc_0f18ec8851ed77310169c458d84a7c8190b1cefc253127979c", + "type": "custom_tool_call", + "status": "completed", + "call_id": "call_sxqlRD4E0PihmL74BNt7GqVV", + "input": "Boston, MA, USA", + "name": "custom_weather" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "type": "custom", + "name": "custom_weather" + }, + "tools": [ + { + "type": "custom", + "description": "Return weather for a location", + "format": { + "type": "text" + }, + "name": "custom_weather" + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 67, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 238, + "output_tokens_details": { + "reasoning_tokens": 192 + }, + "total_tokens": 305 + }, + "user": null, + "metadata": {} +}� +-- end response body -- From 8be92d704db1ba783d68d7ffd49acf6892625bd7 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 20:28:05 -0700 Subject: [PATCH 37/39] Prevent NPE when tag value is null --- .../groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy index b12984ecbee..8c39baf2851 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy @@ -247,7 +247,7 @@ class TagsAssert { if (expected instanceof Pattern) { assert value =~ expected: "Tag \"$name\": \"${value.toString()}\" does not match pattern \"$expected\"" } else if (expected instanceof Class) { - assert ((Class) expected).isInstance(value): "Tag \"$name\": instance check $expected failed for \"${value.toString()}\" of class \"${value.class}\"" + assert ((Class) expected).isInstance(value): "Tag \"$name\": instance check $expected failed for \"${value.toString()}\" of class \"${value?.class}\"" } else if (expected instanceof Closure) { assert ((Closure) expected).call(value): "Tag \"$name\": closure call ${expected.toString()} failed with \"$value\"" } else if (expected instanceof CharSequence) { From 1036ed40ce5f1d0772f65dfccd6f2a0891f907ca Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 20:29:58 -0700 Subject: [PATCH 38/39] Replace catch Throwable with catch Exception --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 7018005488d..f46b93e819e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -178,7 +178,7 @@ private List> extractToolDefinitionsFromParams(ResponseCreat } } return toolDefinitions; - } catch (Throwable ignored) { + } catch (Exception ignored) { return Collections.emptyList(); } } From 9911c514e788cdf18c9d5f23818bf45ddb0db50b Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 20:58:17 -0700 Subject: [PATCH 39/39] responseCreateParamsWithPromptTracking support both known and unknown format. Test cover extractPromptFromParams and related methods --- .../openai_java/ResponseDecorator.java | 91 ++++++++++++++++--- .../src/test/groovy/OpenAiTest.groovy | 39 +++++--- .../test/groovy/ResponseServiceTest.groovy | 9 +- 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index f46b93e819e..14d2166f6e2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -807,35 +807,96 @@ private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instruction } private Optional> extractPromptFromParams(ResponseCreateParams params) { - Optional promptOpt = params.prompt(); - if (!promptOpt.isPresent()) { + Optional typedPromptOpt = params._prompt().asKnown(); + if (typedPromptOpt.isPresent()) { + Optional> extractedPrompt = extractPrompt(typedPromptOpt.get()); + if (extractedPrompt.isPresent()) { + return extractedPrompt; + } + } + + try { + Optional rawPromptOpt = params._prompt().asUnknown(); + if (!rawPromptOpt.isPresent()) { + return Optional.empty(); + } + + Optional> rawPromptObjOpt = rawPromptOpt.get().asObject(); + if (!rawPromptObjOpt.isPresent()) { + return Optional.empty(); + } + + return extractPrompt(rawPromptObjOpt.get()); + } catch (Exception ignored) { return Optional.empty(); } + } - ResponsePrompt prompt = promptOpt.get(); + private Optional> extractPrompt(ResponsePrompt prompt) { Map promptMap = new LinkedHashMap<>(); - String id = prompt.id(); + String id = prompt._id().asString().orElse(null); if (id != null && !id.isEmpty()) { promptMap.put("id", id); } - prompt.version().ifPresent(version -> promptMap.put("version", version)); - prompt - .variables() - .ifPresent( - variables -> { - Map normalized = normalizePromptVariables(variables); - if (!normalized.isEmpty()) { - promptMap.put("variables", normalized); - } - }); + prompt._version().asString().ifPresent(version -> promptMap.put("version", version)); + + Optional typedVariablesOpt = prompt._variables().asKnown(); + if (typedVariablesOpt.isPresent()) { + Map normalized = normalizePromptVariables(typedVariablesOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } else { + Optional rawVariablesOpt = prompt._variables().asUnknown(); + if (rawVariablesOpt.isPresent()) { + Optional> rawVariablesObjOpt = rawVariablesOpt.get().asObject(); + if (rawVariablesObjOpt.isPresent()) { + Map normalized = normalizePromptVariables(rawVariablesObjOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } + } + } + + return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); + } + + private Optional> extractPrompt(Map promptObj) { + Map promptMap = new LinkedHashMap<>(); + + String id = getJsonString(promptObj.get("id")); + if (id != null && !id.isEmpty()) { + promptMap.put("id", id); + } + + String version = getJsonString(promptObj.get("version")); + if (version != null && !version.isEmpty()) { + promptMap.put("version", version); + } + + JsonValue variablesValue = promptObj.get("variables"); + if (variablesValue != null) { + Optional> variablesObjOpt = variablesValue.asObject(); + if (variablesObjOpt.isPresent()) { + Map normalized = normalizePromptVariables(variablesObjOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } + } return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); } private Map normalizePromptVariables(ResponsePrompt.Variables variables) { + return normalizePromptVariables(variables._additionalProperties()); + } + + private Map normalizePromptVariables(Map variables) { Map normalized = new LinkedHashMap<>(); - for (Map.Entry entry : variables._additionalProperties().entrySet()) { + for (Map.Entry entry : variables.entrySet()) { Object value = normalizePromptVariable(entry.getValue()); if (value != null) { normalized.put(entry.getKey(), value); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index c265c704b7e..e1eef17eb47 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -314,24 +314,35 @@ He hopes to pursue a career in software engineering after graduating.""") } ResponseCreateParams responseCreateParamsWithPromptTracking(boolean json) { - def variables = ResponsePrompt.Variables.builder() - .putAdditionalProperty("user_message", JsonValue.from([type: "input_text", text: "Analyze these images and document"])) - .putAdditionalProperty("user_image_1", JsonValue.from([type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"])) - .putAdditionalProperty("user_file", JsonValue.from([type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"])) - .putAdditionalProperty("user_image_2", JsonValue.from([type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"])) - .build() - - def prompt = ResponsePrompt.builder() - .id("pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0") - .version("2") - .variables(variables) - .build() - if (json) { + def rawPrompt = JsonValue.from([ + id: "pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0", + variables: [ + user_message: [type: "input_text", text: "Analyze these images and document"], + user_image_1: [type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"], + user_file: [type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"], + user_image_2: [type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"], + ], + version: "2", + ]) + ResponseCreateParams.builder() - .prompt(prompt) + .prompt(rawPrompt) .build() } else { + def variables = ResponsePrompt.Variables.builder() + .putAdditionalProperty("user_message", JsonValue.from([type: "input_text", text: "Analyze these images and document"])) + .putAdditionalProperty("user_image_1", JsonValue.from([type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"])) + .putAdditionalProperty("user_file", JsonValue.from([type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"])) + .putAdditionalProperty("user_image_2", JsonValue.from([type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"])) + .build() + + def prompt = ResponsePrompt.builder() + .id("pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0") + .version("2") + .variables(variables) + .build() + ResponseCreateParams.builder() .prompt(prompt) .build() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index f704fd3cb55..8029ab0bdd0 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -205,7 +205,7 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] Map metadata = [:] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata, false, true) + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata, false) and: metadata.stream == true inputTags.size() == 3 @@ -305,7 +305,9 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_name" "gpt-3.5-turbo" "_ml_obs_tag.output" List def out = tag("_ml_obs_tag.output") - if (out instanceof List) outputMessages.addAll(out) + if (out instanceof List) { + outputMessages.addAll(out) + } } } span(2) { @@ -335,8 +337,7 @@ class ResponseServiceTest extends OpenAiTest { Object inputTagsOut, List outputTagsOut, Map metadataOut, - boolean expectPromptTag = false, - boolean expectRateLimitTags = false) { + boolean expectPromptTag = false) { assertTraces(1) { trace(3) { sortSpansByStart()