diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index ab337bd6923..46fa6617534 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -2,6 +2,7 @@ import datadog.context.ContextScope; import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.DDTraceId; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObs; @@ -39,6 +40,7 @@ public class DDLLMObsSpan implements LLMObsSpan { private static final String SERVICE = LLMOBS_TAG_PREFIX + "service"; private static final String VERSION = LLMOBS_TAG_PREFIX + "version"; + private static final String DDTRACE_VERSION = LLMOBS_TAG_PREFIX + "ddtrace.version"; private static final String ENV = LLMOBS_TAG_PREFIX + "env"; private static final String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; @@ -76,6 +78,7 @@ public DDLLMObsSpan( span.setTag(ENV, wellKnownTags.getEnv()); span.setTag(SERVICE, wellKnownTags.getService()); span.setTag(VERSION, wellKnownTags.getVersion()); + span.setTag(DDTRACE_VERSION, DDTraceApiInfo.VERSION); span.setTag(SPAN_KIND, kind); spanKind = kind; diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index e667e3599d6..476d8fc991f 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -4,6 +4,7 @@ import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import datadog.trace.agent.tooling.TracerInstaller import datadog.trace.api.DDTags +import datadog.trace.api.DDTraceApiInfo import datadog.trace.api.IdGenerationStrategy import datadog.trace.api.WellKnownTags import datadog.trace.api.telemetry.LLMObsMetricCollector @@ -134,6 +135,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test span with overwrites"() { @@ -219,6 +222,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span string input formatted to messages"() { @@ -270,6 +275,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span with messages"() { @@ -326,6 +333,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "finish records span.finished telemetry when LLMObs enabled"() { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle index cd654d03334..ceda41c2804 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle @@ -1,6 +1,6 @@ apply from: "$rootDir/gradle/java.gradle" -def minVer = '3.0.0' +def minVer = '3.0.1' muzzle { pass { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 6c9e9cad9d9..f35e485c90e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -1,12 +1,19 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + +import com.openai.core.JsonValue; import com.openai.helpers.ChatCompletionAccumulator; +import com.openai.models.FunctionDefinition; import com.openai.models.chat.completions.ChatCompletion; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.chat.completions.ChatCompletionFunctionTool; import com.openai.models.chat.completions.ChatCompletionMessage; import com.openai.models.chat.completions.ChatCompletionMessageParam; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.chat.completions.ChatCompletionTool; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -31,19 +38,24 @@ public void withChatCompletionCreateParams( AgentSpan span, ChatCompletionCreateParams params, boolean stream) { span.setResourceName(CHAT_COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/chat/completions"); - if (!llmObsEnabled) { + if (params == null) { return; } + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); - if (params == null) { + if (!llmObsEnabled) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); span.setTag( CommonTags.INPUT, @@ -55,9 +67,7 @@ public void withChatCompletionCreateParams( // maxTokens is deprecated but integration tests missing to provide maxCompletionTokens params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); params.temperature().ifPresent(v -> metadata.put("temperature", v)); - if (stream) { - metadata.put("stream", true); - } + metadata.put("stream", stream); params .streamOptions() .ifPresent( @@ -72,6 +82,92 @@ public void withChatCompletionCreateParams( params.n().ifPresent(v -> metadata.put("n", v)); params.seed().ifPresent(v -> metadata.put("seed", v)); span.setTag(CommonTags.METADATA, metadata); + params + .toolChoice() + .ifPresent( + toolChoice -> { + String choice = null; + if (toolChoice.isAuto()) { + choice = "auto"; + } else if (toolChoice.isAllowedToolChoice()) { + choice = "allowed_tools"; + } else if (toolChoice.isNamedToolChoice()) { + choice = "function"; + } else if (toolChoice.isNamedToolChoiceCustom()) { + choice = "custom"; + } + if (choice != null) { + metadata.put("tool_choice", choice); + } + }); + + List tools = params.tools().orElse(Collections.emptyList()); + if (!tools.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, extractToolDefinitions(tools)); + } + } + + private List> extractToolDefinitions(List tools) { + List> toolDefinitions = new ArrayList<>(); + for (ChatCompletionTool tool : tools) { + if (tool.isFunction()) { + Map toolDef = extractFunctionToolDef(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + } + return toolDefinitions; + } + + private static Map extractFunctionToolDef(ChatCompletionFunctionTool funcTool) { + // Try typed access first (works when built programmatically) + Optional funcDefOpt = funcTool._function().asKnown(); + if (funcDefOpt.isPresent()) { + FunctionDefinition funcDef = funcDefOpt.get(); + Map toolDef = new HashMap<>(); + toolDef.put("name", funcDef.name()); + funcDef.description().ifPresent(desc -> toolDef.put("description", desc)); + funcDef + .parameters() + .ifPresent( + params -> + toolDef.put("schema", jsonValueMapToObject(params._additionalProperties()))); + return toolDef; + } + + // Fall back to raw JSON extraction (when deserialized from HTTP request) + Optional rawOpt = funcTool._function().asUnknown(); + if (!rawOpt.isPresent()) { + return null; + } + Optional> objOpt = rawOpt.get().asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + JsonValue nameValue = obj.get("name"); + if (nameValue == null) { + return null; + } + Optional nameOpt = nameValue.asString(); + if (!nameOpt.isPresent()) { + return null; + } + Map toolDef = new HashMap<>(); + toolDef.put("name", nameOpt.get()); + JsonValue descValue = obj.get("description"); + if (descValue != null) { + descValue.asString().ifPresent(desc -> toolDef.put("description", desc)); + } + JsonValue paramsValue = obj.get("parameters"); + if (paramsValue != null) { + Object schema = jsonValueToObject(paramsValue); + if (schema != null) { + toolDef.put("schema", schema); + } + } + return toolDef; } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { @@ -97,13 +193,14 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { } public void withChatCompletion(AgentSpan span, ChatCompletion completion) { - if (!llmObsEnabled) { - return; - } String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() .map(ChatCompletionDecorator::llmMessage) @@ -117,6 +214,10 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); span.setTag(CommonTags.OUTPUT_TOKENS, usage.completionTokens()); span.setTag(CommonTags.TOTAL_TOKENS, usage.totalTokens()); + usage + .promptTokensDetails() + .flatMap(details -> details.cachedTokens()) + .ifPresent(v -> span.setTag(CommonTags.CACHE_READ_INPUT_TOKENS, v)); }); } @@ -127,7 +228,7 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletion.Choice choice) { if (roleOpt.isPresent()) { role = String.valueOf(roleOpt.get()); } - String content = msg.content().orElse(null); + String content = msg.content().orElse(""); Optional> toolCallsOpt = msg.toolCalls(); if (toolCallsOpt.isPresent() && !toolCallsOpt.get().isEmpty()) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java index f30d02f9570..2c89bf339ad 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java @@ -18,6 +18,7 @@ public String[] helperClassNames() { packageName + ".CommonTags", packageName + ".ChatCompletionDecorator", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index f779662ac0d..a992c85400c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -19,12 +19,20 @@ interface CommonTags { String MODEL_PROVIDER = TAG_PREFIX + LLMObsTags.MODEL_PROVIDER; String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; + String INTEGRATION = TAG_PREFIX + "integration"; String VERSION = TAG_PREFIX + "version"; + String DDTRACE_VERSION = TAG_PREFIX + "ddtrace.version"; + String SOURCE = TAG_PREFIX + "source"; + + String ERROR = TAG_PREFIX + "error"; + String ERROR_TYPE = TAG_PREFIX + "error_type"; String ENV = TAG_PREFIX + "env"; String SERVICE = TAG_PREFIX + "service"; String PARENT_ID = TAG_PREFIX + "parent_id"; + String TOOL_DEFINITIONS = TAG_PREFIX + "tool_definitions"; + String METRIC_PREFIX = "_ml_obs_metric."; String INPUT_TOKENS = METRIC_PREFIX + "input_tokens"; String OUTPUT_TOKENS = METRIC_PREFIX + "output_tokens"; @@ -33,4 +41,5 @@ interface CommonTags { String CACHE_READ_INPUT_TOKENS = METRIC_PREFIX + "cache_read_input_tokens"; String REQUEST_REASONING = "_ml_obs_request.reasoning"; + String REQUEST_PROMPT = "_ml_obs_request.prompt"; } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 2291c860d00..f0f29386582 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class CompletionDecorator { @@ -23,20 +24,25 @@ public class CompletionDecorator { public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams params) { span.setResourceName(COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/completions"); - if (!llmObsEnabled) { + if (params == null) { return; } - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); - if (params == null) { + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + + if (!llmObsEnabled) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); params .prompt() .flatMap(p -> p.string()) @@ -44,7 +50,7 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa input -> span.setTag( CommonTags.INPUT, - Collections.singletonList(LLMObs.LLMMessage.from(null, input)))); + Collections.singletonList(LLMObs.LLMMessage.from("", input)))); Map metadata = new HashMap<>(); params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); @@ -61,17 +67,17 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa } public void withCompletion(AgentSpan span, Completion completion) { - if (!llmObsEnabled) { - return; - } - String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() - .map(v -> LLMObs.LLMMessage.from(null, v.text())) + .map(v -> LLMObs.LLMMessage.from("", v.text())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); @@ -86,10 +92,6 @@ public void withCompletion(AgentSpan span, Completion completion) { } public void withCompletions(AgentSpan span, List completions) { - if (!llmObsEnabled) { - return; - } - if (completions.isEmpty()) { return; } @@ -99,6 +101,10 @@ public void withCompletions(AgentSpan span, List completions) { span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + Map textByChoiceIndex = new HashMap<>(); for (Completion completion : completions) { completion @@ -115,7 +121,7 @@ public void withCompletions(AgentSpan span, List completions) { List output = textByChoiceIndex.entrySet().stream() .sorted(Map.Entry.comparingByKey()) - .map(entry -> LLMObs.LLMMessage.from(null, entry.getValue().toString())) + .map(entry -> LLMObs.LLMMessage.from("", entry.getValue().toString())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 88098358ebf..4e986603d93 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -25,19 +25,20 @@ public class EmbeddingDecorator { public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams params) { span.setResourceName(EMBEDDINGS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/embeddings"); - if (!llmObsEnabled) { + if (params == null) { return; } + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); - if (params == null) { + if (!llmObsEnabled) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + + // Keep model_name stable on error paths where no response is available. + modelName.ifPresent(str -> span.setTag(CommonTags.MODEL_NAME, str)); + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); @@ -59,14 +60,14 @@ private List embeddingDocuments(EmbeddingCreateParams.Input inp } public void withCreateEmbeddingResponse(AgentSpan span, CreateEmbeddingResponse response) { - if (!llmObsEnabled) { - return; - } - String modelName = response.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + if (!response.data().isEmpty()) { int embeddingCount = response.data().size(); Embedding firstEmbedding = response.data().get(0); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java new file mode 100644 index 00000000000..c251f5c17fd --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java @@ -0,0 +1,51 @@ +package datadog.trace.instrumentation.openai_java; + +import com.openai.core.JsonValue; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public final class JsonValueUtils { + private JsonValueUtils() {} + + public static Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + public static Object jsonValueToObject(JsonValue value) { + if (value == null) { + return null; + } + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; + } +} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index e6fca7e8979..a68966e9327 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -3,6 +3,8 @@ import com.openai.core.ClientOptions; import com.openai.core.http.Headers; import datadog.trace.api.Config; +import datadog.trace.api.DDTags; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -95,8 +97,11 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.ENV, wellKnownTags.getEnv()); span.setTag(CommonTags.SERVICE, wellKnownTags.getService()); span.setTag(CommonTags.VERSION, wellKnownTags.getVersion()); + span.setTag(CommonTags.DDTRACE_VERSION, DDTraceApiInfo.VERSION); span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); + span.setTag(CommonTags.SOURCE, "integration"); + span.setTag(CommonTags.INTEGRATION, INTEGRATION); AgentSpanContext parent = LLMObsContext.current(); String parentSpanId = LLMObsContext.ROOT_SPAN_ID; @@ -111,6 +116,9 @@ public AgentSpan afterStart(AgentSpan span) { @Override public AgentSpan beforeFinish(AgentSpan span) { if (llmObsEnabled) { + span.setTag(CommonTags.ERROR, span.isError() ? 1 : 0); + span.setTag(CommonTags.ERROR_TYPE, span.getTag(DDTags.ERROR_TYPE)); + Object spanKindTag = span.getTag(CommonTags.SPAN_KIND); if (spanKindTag != null) { String spanKind = spanKindTag.toString(); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 4bc95e32934..14d2166f6e2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -1,19 +1,27 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + import com.openai.core.JsonField; import com.openai.core.JsonValue; import com.openai.models.Reasoning; import com.openai.models.ResponsesModel; +import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.FunctionTool; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; import com.openai.models.responses.ResponseOutputItem; import com.openai.models.responses.ResponseOutputMessage; import com.openai.models.responses.ResponseOutputText; +import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.Tool; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -23,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -31,17 +40,14 @@ public class ResponseDecorator { public static final ResponseDecorator DECORATE = new ResponseDecorator(); private static final CharSequence RESPONSES_CREATE = UTF8BytesString.create("createResponse"); + private static final String IMAGE_FALLBACK_MARKER = "[image]"; + private static final String FILE_FALLBACK_MARKER = "[file]"; private final boolean llmObsEnabled = Config.get().isLlmObsEnabled(); public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params) { span.setResourceName(RESPONSES_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/responses"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -51,6 +57,19 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params String modelName = extractResponseModel(params._model()); span.setTag(CommonTags.OPENAI_REQUEST_MODEL, modelName); + if (!llmObsEnabled) { + return; + } + + // Keep model_name/output/metadata shape stable on error paths where no response is available. + if (modelName != null && !modelName.isEmpty()) { + span.setTag(CommonTags.MODEL_NAME, modelName); + } + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + span.setTag(CommonTags.METADATA, new HashMap()); + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + List inputMessages = new ArrayList<>(); params @@ -109,13 +128,146 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractReasoningFromParams(params) .ifPresent(reasoningMap -> span.setTag(CommonTags.REQUEST_REASONING, reasoningMap)); + + extractPromptFromParams(params) + .ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); + + List> toolDefinitions = extractToolDefinitionsFromParams(params); + if (!toolDefinitions.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, toolDefinitions); + } + } + + private List> extractToolDefinitionsFromParams(ResponseCreateParams params) { + try { + Optional> toolsOpt = params.tools(); + if (toolsOpt.isPresent()) { + List> toolDefinitions = new ArrayList<>(); + for (Tool tool : toolsOpt.get()) { + if (!tool.isFunction()) { + continue; + } + Map toolDef = extractFunctionToolDefinition(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + if (!toolDefinitions.isEmpty()) { + return toolDefinitions; + } + } + } catch (Exception ignored) { + // fall back to raw JSON if typed extraction is unavailable or fails + } + + try { + Optional rawToolsOpt = params._tools().asUnknown(); + if (!rawToolsOpt.isPresent()) { + return Collections.emptyList(); + } + Optional> rawToolListOpt = rawToolsOpt.get().asArray(); + if (!rawToolListOpt.isPresent()) { + return Collections.emptyList(); + } + + List> toolDefinitions = new ArrayList<>(); + for (JsonValue rawTool : rawToolListOpt.get()) { + Map toolDef = extractFunctionToolDefinition(rawTool); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + return toolDefinitions; + } catch (Exception ignored) { + return Collections.emptyList(); + } + } + + private Map extractFunctionToolDefinition(FunctionTool functionTool) { + String name = functionTool.name(); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + functionTool.description().ifPresent(desc -> toolDef.put("description", desc)); + functionTool + .parameters() + .ifPresent( + parameters -> + toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); + return toolDef; + } + + private Map extractFunctionToolDefinition(JsonValue rawTool) { + Optional> toolObjOpt = rawTool.asObject(); + if (!toolObjOpt.isPresent()) { + return null; + } + + Map toolObj = toolObjOpt.get(); + String type = getJsonString(toolObj.get("type")); + if (!"function".equals(type)) { + return null; + } + + JsonValue functionObjValue = toolObj.get("function"); + Map functionObj = null; + if (functionObjValue != null) { + Optional> nestedFunctionOpt = functionObjValue.asObject(); + if (nestedFunctionOpt.isPresent()) { + functionObj = nestedFunctionOpt.get(); + } + } + + String name = + functionObj == null + ? getJsonString(toolObj.get("name")) + : getJsonString(functionObj.get("name")); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + + String description = + functionObj == null + ? getJsonString(toolObj.get("description")) + : getJsonString(functionObj.get("description")); + if (description != null) { + toolDef.put("description", description); + } + + JsonValue parameters = + functionObj == null ? toolObj.get("parameters") : functionObj.get("parameters"); + if (parameters != null) { + Object schema = jsonValueToObject(parameters); + if (schema != null) { + toolDef.put("schema", schema); + } + } + + return toolDef; } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { - if (item.isMessage()) { + if (item.isEasyInputMessage()) { + EasyInputMessage message = item.asEasyInputMessage(); + String role = message.role().asString(); + String content = extractEasyInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } else if (item.isMessage()) { ResponseInputItem.Message message = item.asMessage(); String role = message.role().asString(); String content = extractInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } return LLMObs.LLMMessage.from(role, content); } else if (item.isFunctionCall()) { // Function call is mapped to assistant message with tool_calls @@ -137,6 +289,26 @@ private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { return null; } + private String extractEasyInputMessageContent(EasyInputMessage message) { + if (message.content().isTextInput()) { + String content = message.content().asTextInput(); + return content == null || content.isEmpty() ? null : content; + } + + if (message.content().isResponseInputMessageContentList()) { + StringBuilder contentBuilder = new StringBuilder(); + for (ResponseInputContent content : message.content().asResponseInputMessageContentList()) { + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); + } + } + String result = contentBuilder.toString(); + return result.isEmpty() ? null : result; + } + return null; + } + private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { Optional> objOpt = jsonValue.asObject(); if (!objOpt.isPresent()) { @@ -182,7 +354,7 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { } if (callId != null && name != null && argumentsStr != null) { - Map arguments = parseJsonString(argumentsStr); + Map arguments = ToolCallExtractor.parseArguments(argumentsStr); LLMObs.ToolCall toolCall = LLMObs.ToolCall.from(name, "function_call", callId, arguments); return LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall)); @@ -244,92 +416,41 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { return null; } - private Map parseJsonString(String jsonStr) { - if (jsonStr == null || jsonStr.isEmpty()) { - return Collections.emptyMap(); - } - try { - jsonStr = jsonStr.trim(); - if (!jsonStr.startsWith("{") || !jsonStr.endsWith("}")) { - return Collections.emptyMap(); - } - - Map result = new HashMap<>(); - String content = jsonStr.substring(1, jsonStr.length() - 1).trim(); - - if (content.isEmpty()) { - return result; - } - - // Parse JSON manually, respecting quoted strings - List pairs = splitByCommaRespectingQuotes(content); - - for (String pair : pairs) { - int colonIdx = pair.indexOf(':'); - if (colonIdx > 0) { - String key = pair.substring(0, colonIdx).trim(); - String value = pair.substring(colonIdx + 1).trim(); - - // Remove quotes from key - key = removeQuotes(key); - // Remove quotes from value - value = removeQuotes(value); - - result.put(key, value); - } - } - - return result; - } catch (Exception e) { - return Collections.emptyMap(); - } - } - - private List splitByCommaRespectingQuotes(String str) { - List result = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - boolean inQuotes = false; - - for (int i = 0; i < str.length(); i++) { - char c = str.charAt(i); - - if (c == '"') { - inQuotes = !inQuotes; - current.append(c); - } else if (c == ',' && !inQuotes) { - result.add(current.toString()); - current = new StringBuilder(); - } else { - current.append(c); - } - } - - if (current.length() > 0) { - result.add(current.toString()); - } - - return result; - } - - private String removeQuotes(String str) { - str = str.trim(); - if (str.startsWith("\"") && str.endsWith("\"") && str.length() >= 2) { - return str.substring(1, str.length() - 1); - } - return str; - } - private String extractInputMessageContent(ResponseInputItem.Message message) { StringBuilder contentBuilder = new StringBuilder(); for (ResponseInputContent content : message.content()) { - if (content.isInputText()) { - contentBuilder.append(content.asInputText().text()); + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); } } String result = contentBuilder.toString(); return result.isEmpty() ? null : result; } + private String extractInputContentText(ResponseInputContent content) { + if (content.isInputText()) { + return content.asInputText().text(); + } + if (content.isInputImage()) { + return content + .asInputImage() + .imageUrl() + .orElse(content.asInputImage().fileId().orElse(IMAGE_FALLBACK_MARKER)); + } + if (content.isInputFile()) { + return content + .asInputFile() + .fileUrl() + .orElse( + content + .asInputFile() + .fileId() + .orElse(content.asInputFile().filename().orElse(FILE_FALLBACK_MARKER))); + } + return null; + } + private Optional> extractReasoningFromParams(ResponseCreateParams params) { JsonField reasoningField = params._reasoning(); if (reasoningField.isMissing()) { @@ -369,10 +490,6 @@ public void withResponse(AgentSpan span, Response response) { } public void withResponseStreamEvents(AgentSpan span, List events) { - if (!llmObsEnabled) { - return; - } - for (ResponseStreamEvent event : events) { if (event.isCompleted()) { Response response = event.asCompleted().response(); @@ -388,19 +505,21 @@ public void withResponseStreamEvents(AgentSpan span, List e } private void withResponse(AgentSpan span, Response response, boolean stream) { - if (!llmObsEnabled) { - return; - } - String modelName = extractResponseModel(response._model()); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List outputMessages = extractResponseOutputMessages(response.output()); if (!outputMessages.isEmpty()) { span.setTag(CommonTags.OUTPUT, outputMessages); } + enrichInputWithPromptTracking(span, response); + Map metadata = new HashMap<>(); Object reasoningTag = span.getTag(CommonTags.REQUEST_REASONING); @@ -427,6 +546,7 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { (Response.Truncation t) -> metadata.put("truncation", t._value().asString().orElse(null))); + Map textMap = new HashMap<>(); response .text() .ifPresent( @@ -435,7 +555,6 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { .format() .ifPresent( format -> { - Map textMap = new HashMap<>(); Map formatMap = new HashMap<>(); if (format.isText()) { formatMap.put("type", "text"); @@ -445,14 +564,20 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { formatMap.put("type", "json_object"); } textMap.put("format", formatMap); - metadata.put("text", textMap); + }); + textConfig + .verbosity() + .ifPresent( + verbosity -> { + textMap.put("verbosity", verbosity.asString()); }); }); - - if (stream) { - metadata.put("stream", true); + if (!textMap.isEmpty()) { + metadata.put("text", textMap); } + metadata.put("stream", stream); + span.setTag(CommonTags.METADATA, metadata); response @@ -470,6 +595,371 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { }); } + private void enrichInputWithPromptTracking(AgentSpan span, Response response) { + Object promptTag = span.getTag(CommonTags.REQUEST_PROMPT); + if (!(promptTag instanceof Map)) { + return; + } + + Map prompt = new LinkedHashMap<>((Map) promptTag); + Map variables = Collections.emptyMap(); + Object variablesTag = prompt.get("variables"); + if (variablesTag instanceof Map) { + variables = (Map) variablesTag; + } + + Map inputMap = new LinkedHashMap<>(); + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof Map) { + inputMap.putAll((Map) inputTag); + } + + List inputMessages = extractInputMessagesForPromptTracking(span, response); + if (!inputMessages.isEmpty()) { + inputMap.put("messages", inputMessages); + } + + List> chatTemplate = extractChatTemplate(inputMessages, variables); + if (!chatTemplate.isEmpty()) { + prompt.put("chat_template", chatTemplate); + } + + inputMap.put("prompt", prompt); + + span.setTag(CommonTags.INPUT, inputMap); + } + + private List> extractChatTemplate( + List messages, Map variables) { + Map valueToPlaceholder = new LinkedHashMap<>(); + for (Map.Entry variable : variables.entrySet()) { + if (variable.getValue() == null) { + continue; + } + String valueStr = String.valueOf(variable.getValue()); + if (valueStr.isEmpty() + || IMAGE_FALLBACK_MARKER.equals(valueStr) + || FILE_FALLBACK_MARKER.equals(valueStr)) { + continue; + } + valueToPlaceholder.put(valueStr, "{{" + variable.getKey() + "}}"); + } + + List sortedValues = new ArrayList<>(valueToPlaceholder.keySet()); + sortedValues.sort((a, b) -> Integer.compare(b.length(), a.length())); + + List> chatTemplate = new ArrayList<>(); + for (LLMObs.LLMMessage message : messages) { + String role = message.getRole(); + String content = message.getContent(); + if (role == null || role.isEmpty() || content == null || content.isEmpty()) { + continue; + } + + String templateContent = content; + for (String value : sortedValues) { + templateContent = templateContent.replace(value, valueToPlaceholder.get(value)); + } + + Map messageMap = new LinkedHashMap<>(); + messageMap.put("role", role); + messageMap.put("content", templateContent); + chatTemplate.add(messageMap); + } + return chatTemplate; + } + + private List extractInputMessagesForPromptTracking( + AgentSpan span, Response response) { + List messages = new ArrayList<>(); + + response + .instructions() + .ifPresent( + instructions -> { + if (instructions.isInputItemList()) { + for (ResponseInputItem item : instructions.asInputItemList()) { + LLMObs.LLMMessage message = extractInputItemMessage(item); + if (message != null) { + messages.add(message); + } + } + } else if (instructions.isString()) { + String text = instructions.asString(); + if (text != null && !text.isEmpty()) { + messages.add(LLMObs.LLMMessage.from("system", text)); + } + } + }); + + // Fallback for SDK union parsing mismatches: parse raw instructions payload. + if (messages.isEmpty()) { + Optional rawInstructions = response._instructions().asUnknown(); + if (rawInstructions.isPresent()) { + Optional> rawList = rawInstructions.get().asArray(); + if (rawList.isPresent()) { + for (JsonValue item : rawList.get()) { + LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); + if (message != null) { + messages.add(message); + } + } + } + } + } + + boolean hasInstructions = !messages.isEmpty(); + + // Always include input messages from the span tag (set by withResponseCreateParams, which + // records both instructions as "system" and input as "user"). When instructions were already + // collected above, skip "system" messages here to avoid duplicating them. + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof List) { + for (Object messageObj : (List) inputTag) { + if (messageObj instanceof LLMObs.LLMMessage) { + LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; + if (!hasInstructions || !"system".equals(msg.getRole())) { + messages.add(msg); + } + } + } + } else if (inputTag instanceof Map) { + Object messagesObj = ((Map) inputTag).get("messages"); + if (messagesObj instanceof List) { + for (Object messageObj : (List) messagesObj) { + if (messageObj instanceof LLMObs.LLMMessage) { + LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; + if (!hasInstructions || !"system".equals(msg.getRole())) { + messages.add(msg); + } + } + } + } + } + + return messages; + } + + private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instructionValue) { + Optional> objOpt = instructionValue.asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + String role = getJsonString(obj.get("role")); + if (role == null || role.isEmpty()) { + return null; + } + + JsonValue contentValue = obj.get("content"); + if (contentValue == null) { + return null; + } + Optional> contentList = contentValue.asArray(); + if (!contentList.isPresent()) { + return null; + } + + StringBuilder contentBuilder = new StringBuilder(); + for (JsonValue contentItem : contentList.get()) { + Optional> contentObjOpt = contentItem.asObject(); + if (!contentObjOpt.isPresent()) { + continue; + } + Map contentObj = contentObjOpt.get(); + String type = getJsonString(contentObj.get("type")); + if ("input_text".equals(type)) { + String text = getJsonString(contentObj.get("text")); + if (text != null) { + contentBuilder.append(text); + } + } else if ("input_image".equals(type)) { + String imageUrl = getJsonString(contentObj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + contentBuilder.append(imageUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + contentBuilder.append( + fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); + } + } else if ("input_file".equals(type)) { + String fileUrl = getJsonString(contentObj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + contentBuilder.append(fileUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + contentBuilder.append(fileId); + } else { + String filename = getJsonString(contentObj.get("filename")); + contentBuilder.append( + filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename); + } + } + } + } + + String content = contentBuilder.toString(); + if (content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } + + private Optional> extractPromptFromParams(ResponseCreateParams params) { + Optional typedPromptOpt = params._prompt().asKnown(); + if (typedPromptOpt.isPresent()) { + Optional> extractedPrompt = extractPrompt(typedPromptOpt.get()); + if (extractedPrompt.isPresent()) { + return extractedPrompt; + } + } + + try { + Optional rawPromptOpt = params._prompt().asUnknown(); + if (!rawPromptOpt.isPresent()) { + return Optional.empty(); + } + + Optional> rawPromptObjOpt = rawPromptOpt.get().asObject(); + if (!rawPromptObjOpt.isPresent()) { + return Optional.empty(); + } + + return extractPrompt(rawPromptObjOpt.get()); + } catch (Exception ignored) { + return Optional.empty(); + } + } + + private Optional> extractPrompt(ResponsePrompt prompt) { + Map promptMap = new LinkedHashMap<>(); + + String id = prompt._id().asString().orElse(null); + if (id != null && !id.isEmpty()) { + promptMap.put("id", id); + } + prompt._version().asString().ifPresent(version -> promptMap.put("version", version)); + + Optional typedVariablesOpt = prompt._variables().asKnown(); + if (typedVariablesOpt.isPresent()) { + Map normalized = normalizePromptVariables(typedVariablesOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } else { + Optional rawVariablesOpt = prompt._variables().asUnknown(); + if (rawVariablesOpt.isPresent()) { + Optional> rawVariablesObjOpt = rawVariablesOpt.get().asObject(); + if (rawVariablesObjOpt.isPresent()) { + Map normalized = normalizePromptVariables(rawVariablesObjOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } + } + } + + return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); + } + + private Optional> extractPrompt(Map promptObj) { + Map promptMap = new LinkedHashMap<>(); + + String id = getJsonString(promptObj.get("id")); + if (id != null && !id.isEmpty()) { + promptMap.put("id", id); + } + + String version = getJsonString(promptObj.get("version")); + if (version != null && !version.isEmpty()) { + promptMap.put("version", version); + } + + JsonValue variablesValue = promptObj.get("variables"); + if (variablesValue != null) { + Optional> variablesObjOpt = variablesValue.asObject(); + if (variablesObjOpt.isPresent()) { + Map normalized = normalizePromptVariables(variablesObjOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } + } + + return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); + } + + private Map normalizePromptVariables(ResponsePrompt.Variables variables) { + return normalizePromptVariables(variables._additionalProperties()); + } + + private Map normalizePromptVariables(Map variables) { + Map normalized = new LinkedHashMap<>(); + for (Map.Entry entry : variables.entrySet()) { + Object value = normalizePromptVariable(entry.getValue()); + if (value != null) { + normalized.put(entry.getKey(), value); + } + } + return normalized; + } + + private Object normalizePromptVariable(JsonValue value) { + if (value == null) { + return null; + } + + Optional asString = value.asString(); + if (asString.isPresent()) { + return asString.get(); + } + + Optional> asObject = value.asObject(); + if (!asObject.isPresent()) { + return value.toString(); + } + + Map obj = asObject.get(); + String type = getJsonString(obj.get("type")); + String text = getJsonString(obj.get("text")); + if (text != null && !text.isEmpty()) { + return text; + } + + if ("input_image".equals(type)) { + String imageUrl = getJsonString(obj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + return imageUrl; + } + String fileId = getJsonString(obj.get("file_id")); + return fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId; + } + + if ("input_file".equals(type)) { + String fileUrl = getJsonString(obj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + return fileUrl; + } + String fileId = getJsonString(obj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + return fileId; + } + String filename = getJsonString(obj.get("filename")); + return filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename; + } + + return value.toString(); + } + + private String getJsonString(JsonValue value) { + if (value == null) { + return null; + } + Optional asString = value.asString(); + return asString.orElse(null); + } + private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); @@ -481,6 +971,23 @@ private List extractResponseOutputMessages(List toolCalls = Collections.singletonList(toolCall); messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls)); } + } else if (item.isCustomToolCall()) { + ResponseCustomToolCall customToolCall = item.asCustomToolCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(customToolCall); + if (toolCall != null) { + messages.add( + LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall))); + } + } else if (item.isMcpCall()) { + ResponseOutputItem.McpCall mcpCall = item.asMcpCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(mcpCall); + List toolCalls = + toolCall == null ? null : Collections.singletonList(toolCall); + String outputText = mcpCall.output().orElse(""); + LLMObs.ToolResult toolResult = + LLMObs.ToolResult.from(mcpCall.name(), "mcp_tool_result", mcpCall.id(), outputText); + List toolResults = Collections.singletonList(toolResult); + messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls, toolResults)); } else if (item.isMessage()) { ResponseOutputMessage message = item.asMessage(); String textContent = extractMessageContent(message); @@ -491,12 +998,30 @@ private List extractResponseOutputMessages(List writer.name("encrypted_content").value(v)); - writer.name("id").value(reasoning.id()); + + String id = reasoning.id(); + writer.name("id").value(id == null ? "" : id); + writer.endObject(); + messages.add(LLMObs.LLMMessage.from("reasoning", writer.toString())); } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java index 25266504f53..b87b3910490 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java @@ -19,6 +19,7 @@ public String[] helperClassNames() { packageName + ".ResponseDecorator", packageName + ".FunctionCallOutputExtractor", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java index 357c73de0aa..ffeca857a20 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java @@ -4,7 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; +import com.openai.models.responses.ResponseOutputItem.McpCall; import datadog.trace.api.llmobs.LLMObs; import java.util.Collections; import java.util.Map; @@ -64,7 +66,47 @@ public static LLMObs.ToolCall getToolCall(ResponseFunctionToolCall functionCall) return null; } - private static Map parseArguments(String argumentsJson) { + public static LLMObs.ToolCall getToolCall(ResponseCustomToolCall customToolCall) { + try { + String name = customToolCall.name(); + String callId = customToolCall.callId(); + String inputJson = customToolCall.input(); + + String type = "custom_tool_call"; + Optional typeOpt = customToolCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(inputJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract custom tool call information", e); + } + return null; + } + + public static LLMObs.ToolCall getToolCall(McpCall mcpCall) { + try { + String name = mcpCall.name(); + String callId = mcpCall.id(); + String argumentsJson = mcpCall.arguments(); + + String type = "mcp_call"; + Optional typeOpt = mcpCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(argumentsJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract MCP tool call information", e); + } + return null; + } + + static Map parseArguments(String argumentsJson) { try { return MAPPER.readValue(argumentsJson, MAP_TYPE_REF); } catch (Exception e) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 22e7fbbb579..7376f73f92e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -6,7 +6,6 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.chat.completions.ChatCompletion import com.openai.models.chat.completions.ChatCompletionChunk -import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags @@ -94,7 +93,7 @@ class ChatCompletionServiceTest extends OpenAiTest { } def "create async chat/completion test withRawResponse"() { - CompletableFuture> completionFuture = runUnderTrace("parent") { + CompletableFuture> completionFuture = runUnderTrace("parent") { openAiClient.async().chat().completions().withRawResponse().create(params) } @@ -148,7 +147,18 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(false, outputTag, [:]) + List> toolDefinitions = [] + assertChatCompletionTrace(false, outputTag, [:], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" + toolDefinitions[0].schema.type == "object" + (toolDefinitions[0].schema.properties as Map).containsKey("name") + (toolDefinitions[0].schema.properties as Map).containsKey("major") + (toolDefinitions[0].schema.properties as Map).containsKey("school") + (toolDefinitions[0].schema.properties as Map).containsKey("grades") + (toolDefinitions[0].schema.properties as Map).containsKey("clubs") and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -179,7 +189,12 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(true, outputTag, [stream: true]) + List> toolDefinitions = [] + assertChatCompletionTrace(true, outputTag, [stream: true], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -295,6 +310,13 @@ class ChatCompletionServiceTest extends OpenAiTest { } private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata) { + assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false, null) + } + + private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions, List> toolDefinitionsOut) { + def expectedMetadata = new LinkedHashMap(metadata) + expectedMetadata.putIfAbsent("stream", isStreaming) + assertTraces(1) { trace(3) { sortSpansByStart() @@ -313,7 +335,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.span.kind" "llm" "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String - "_ml_obs_tag.metadata" metadata + "_ml_obs_tag.metadata" expectedMetadata "_ml_obs_tag.input" List "_ml_obs_tag.output" List def outputTags = tag("_ml_obs_tag.output") @@ -325,10 +347,22 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long + "_ml_obs_metric.cache_read_input_tokens" Long } "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String + if (expectToolDefinitions) { + "$CommonTags.TOOL_DEFINITIONS" List + def toolDefinitions = tag("$CommonTags.TOOL_DEFINITIONS") + if (toolDefinitionsOut != null && toolDefinitions != null) { + toolDefinitionsOut.addAll(toolDefinitions) + } + } + "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/chat/completions" "openai.api_base" openAiBaseApi diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 43f5e247c55..8ca98ca3677 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -8,6 +8,7 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes +import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags import java.util.concurrent.CompletableFuture import java.util.stream.Stream @@ -20,7 +21,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -35,7 +36,7 @@ class CompletionServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -52,7 +53,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -69,7 +70,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -83,7 +84,7 @@ class CompletionServiceTest extends OpenAiTest { completionFuture.get() expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -98,7 +99,7 @@ class CompletionServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -113,7 +114,7 @@ class CompletionServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -131,13 +132,17 @@ class CompletionServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] } - private void assertCompletionTrace() { + private void assertCompletionTrace(boolean streamRequest) { + List inputTagsOut = [] + List outputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -157,14 +162,30 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/completions" "openai.api_base" openAiBaseApi @@ -189,5 +210,16 @@ class CompletionServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].role == "" + assert inputTagsOut[0].content == "Tell me a story about building the best SDK!" + assert outputTagsOut.size() >= 1 + assert outputTagsOut.every { it.role == "" } + if (streamRequest) { + assert metadataOut.stream_options == [include_usage: true] + } else { + assert !metadataOut.containsKey("stream_options") + } } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index eb14f2999de..0a4f76ff47a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -40,6 +40,9 @@ class EmbeddingServiceTest extends OpenAiTest { } private void assertEmbeddingTrace() { + List inputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -59,11 +62,23 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" "text-embedding-ada-002-v2" "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.output" "[1 embedding(s) returned with size 1536]" "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" + "$CommonTags.ERROR" 0 "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.total_tokens" Long "openai.request.method" "POST" @@ -90,5 +105,9 @@ class EmbeddingServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].text == "hello world" + assert metadataOut == [encoding_format: "base64"] } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index 542f9e27960..e1eef17eb47 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -21,6 +21,9 @@ import com.openai.models.responses.ResponseCreateParams import com.openai.models.responses.ResponseFunctionToolCall import com.openai.models.responses.ResponseIncludable import com.openai.models.responses.ResponseInputItem +import com.openai.models.responses.ResponsePrompt +import com.openai.models.responses.CustomTool +import com.openai.models.responses.ToolChoiceCustom import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.config.LlmObsConfig @@ -310,6 +313,71 @@ He hopes to pursue a career in software engineering after graduating.""") } } + ResponseCreateParams responseCreateParamsWithPromptTracking(boolean json) { + if (json) { + def rawPrompt = JsonValue.from([ + id: "pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0", + variables: [ + user_message: [type: "input_text", text: "Analyze these images and document"], + user_image_1: [type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"], + user_file: [type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"], + user_image_2: [type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"], + ], + version: "2", + ]) + + ResponseCreateParams.builder() + .prompt(rawPrompt) + .build() + } else { + def variables = ResponsePrompt.Variables.builder() + .putAdditionalProperty("user_message", JsonValue.from([type: "input_text", text: "Analyze these images and document"])) + .putAdditionalProperty("user_image_1", JsonValue.from([type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"])) + .putAdditionalProperty("user_file", JsonValue.from([type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"])) + .putAdditionalProperty("user_image_2", JsonValue.from([type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"])) + .build() + + def prompt = ResponsePrompt.builder() + .id("pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0") + .version("2") + .variables(variables) + .build() + + ResponseCreateParams.builder() + .prompt(prompt) + .build() + } + } + + ResponseCreateParams responseCreateParamsWithCustomToolCall(boolean json) { + def customTool = CustomTool.builder() + .name("custom_weather") + .description("Return weather for a location") + .formatText() + .build() + + def toolChoice = ToolChoiceCustom.builder() + .name("custom_weather") + .type(JsonValue.from("custom")) + .build() + + if (json) { + ResponseCreateParams.builder() + .model("gpt-5") + .input("Use the custom_weather tool to answer: What's the weather in Boston?") + .addTool(customTool) + .toolChoice(toolChoice) + .build() + } else { + ResponseCreateParams.builder() + .model(ChatModel.GPT_5) + .input("Use the custom_weather tool to answer: What's the weather in Boston?") + .addTool(customTool) + .toolChoice(toolChoice) + .build() + } + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() @@ -328,4 +396,3 @@ He hopes to pursue a career in software engineering after graduating.""") } } } - diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index f747005cff4..8029ab0bdd0 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -1,11 +1,14 @@ import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace +import com.openai.client.okhttp.OpenAIOkHttpClient import com.openai.core.http.AsyncStreamResponse import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse +import com.openai.credential.BearerTokenCredential import com.openai.models.responses.Response import com.openai.models.responses.ResponseStreamEvent +import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.DDSpanTypes import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags @@ -23,7 +26,10 @@ class ResponseServiceTest extends OpenAiTest { expect: resp != null and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -38,7 +44,10 @@ class ResponseServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -55,7 +64,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: scenario | params @@ -76,7 +88,11 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"]) + Map metadata = [:] + assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"], null, null, metadata) + and: + metadata.stream == true + metadata.reasoning == [effort: "medium", summary: "detailed"] where: responseCreateParams << [responseCreateParamsWithReasoning(false), responseCreateParamsWithReasoning(true)] @@ -93,7 +109,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -107,7 +126,10 @@ class ResponseServiceTest extends OpenAiTest { responseFuture.get() expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -122,7 +144,10 @@ class ResponseServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -137,7 +162,10 @@ class ResponseServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -155,7 +183,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -173,8 +204,17 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags) + Map metadata = [:] + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata, false) and: + metadata.stream == true + inputTags.size() == 3 + inputTags[1].toolCalls.size() == 1 + inputTags[1].toolCalls[0].name == "get_weather" + inputTags[1].toolCalls[0].type == "function_call" + inputTags[1].toolCalls[0].arguments == [location: "San Francisco, CA"] + inputTags[2].toolResults.size() == 1 + inputTags[2].toolResults[0].type == "function_call_output" !inputTags.isEmpty() inputTags[2].toolResults[0].result == '{"temperature": "72°F", "conditions": "sunny", "humidity": "65%"}' @@ -182,11 +222,122 @@ class ResponseServiceTest extends OpenAiTest { responseCreateParams << [responseCreateParamsWithToolInput(false), responseCreateParamsWithToolInput(true)] } - private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning) { - assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null) + def "create response with prompt tracking"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + expect: + resp != null + and: + Map metadata = [:] + Map input = [:] + assertResponseTrace(false, null, String, null, input, null, metadata, true) + and: + metadata.stream == false + input.prompt instanceof Map + input.messages instanceof List + def prompt = input.prompt as Map + prompt.chat_template instanceof List + prompt.variables instanceof Map + + where: + params << [responseCreateParamsWithPromptTracking(false), responseCreateParamsWithPromptTracking(true)] + } + + def "create response with custom tool call"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + expect: + resp != null + and: + List outputTags = [] + Map metadata = [:] + assertResponseTrace(false, "gpt-5", String, null, null, outputTags, metadata) + and: + !outputTags.isEmpty() + + where: + params << [responseCreateParamsWithCustomToolCall(false), responseCreateParamsWithCustomToolCall(true)] + } + + def "create response error sets model_name and placeholder output"() { + setup: + def errorBackend = TestHttpServer.httpServer { + handlers { + prefix("/v1/") { + response.status(500).send('{"error":{"message":"Internal server error","type":"server_error"}}') + } + } + } + def errorClient = OpenAIOkHttpClient.builder() + .baseUrl("${errorBackend.address.toURL()}/v1") + .credential(BearerTokenCredential.create("")) + .maxRetries(0) + .build() + + when: + runUnderTrace("parent") { + try { + errorClient.responses().create(responseCreateParams(false)) + } catch (Exception ignored) {} + } + + then: + List outputMessages = [] + assertTraces(1) { + trace(3) { + sortSpansByStart() + span(0) { + operationName "parent" + parent() + errored false + } + span(1) { + operationName "openai.request" + resourceName "createResponse" + childOf span(0) + errored true + spanType DDSpanTypes.LLMOBS + tags(false) { + "_ml_obs_tag.model_name" "gpt-3.5-turbo" + "_ml_obs_tag.output" List + def out = tag("_ml_obs_tag.output") + if (out instanceof List) { + outputMessages.addAll(out) + } + } + } + span(2) { + operationName "okhttp.request" + resourceName "POST /v1/responses" + childOf span(1) + errored false + spanType "http" + } + } + } + and: + outputMessages.size() == 1 + outputMessages[0].role == "" + outputMessages[0].content == "" + + cleanup: + errorBackend.close() } - private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning, List inputTagsOut) { + + private void assertResponseTrace( + boolean isStreaming, + String reqModel, + Object respModel, + Map reasoning, + Object inputTagsOut, + List outputTagsOut, + Map metadataOut, + boolean expectPromptTag = false) { assertTraces(1) { trace(3) { sortSpansByStart() @@ -206,12 +357,31 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map - "_ml_obs_tag.input" List + if (expectPromptTag) { + "_ml_obs_request.prompt" Map + } + def metadata = tag("_ml_obs_tag.metadata") + if (metadataOut != null && metadata != null) { + metadataOut.putAll(metadata) + } + if (inputTagsOut instanceof Map) { + "_ml_obs_tag.input" Map + } else { + "_ml_obs_tag.input" List + } def inputTags = tag("_ml_obs_tag.input") if (inputTagsOut != null && inputTags != null) { - inputTagsOut.addAll(inputTags) + if (inputTagsOut instanceof List) { + inputTagsOut.addAll(inputTags as List) + } else if (inputTagsOut instanceof Map) { + inputTagsOut.putAll(inputTags as Map) + } } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTagsOut != null && outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long @@ -219,7 +389,11 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_metric.cache_read_input_tokens" Long "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String + "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 if (reasoning != null) { "_ml_obs_request.reasoning" reasoning } @@ -228,13 +402,15 @@ class ResponseServiceTest extends OpenAiTest { "openai.api_base" openAiBaseApi "$CommonTags.OPENAI_RESPONSE_MODEL" respModel if (!isStreaming) { - "openai.organization.ratelimit.requests.limit" 10000 + "openai.organization.ratelimit.requests.limit" Integer "openai.organization.ratelimit.requests.remaining" Integer - "openai.organization.ratelimit.tokens.limit" 50000000 + "openai.organization.ratelimit.tokens.limit" Integer "openai.organization.ratelimit.tokens.remaining" Integer } "$CommonTags.OPENAI_ORGANIZATION" "datadog-staging" - "$CommonTags.OPENAI_REQUEST_MODEL" reqModel + if (reqModel != null) { + "$CommonTags.OPENAI_REQUEST_MODEL" reqModel + } "$Tags.COMPONENT" "openai" "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT defaultTags() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java new file mode 100644 index 00000000000..21713649956 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java @@ -0,0 +1,158 @@ +package datadog.trace.instrumentation.openai_java; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.openai.core.JsonValue; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class JsonValueUtilsTest { + + @Test + void testNullReturnsNull() { + assertNull(JsonValueUtils.jsonValueToObject(null)); + } + + @Test + void testString() { + assertEquals("hello", JsonValueUtils.jsonValueToObject(JsonValue.from("hello"))); + } + + @Test + void testInteger() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(42)); + assertInstanceOf(Number.class, result); + assertEquals(42, ((Number) result).intValue()); + } + + @Test + void testDouble() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(3.14)); + assertInstanceOf(Number.class, result); + assertEquals(3.14, ((Number) result).doubleValue(), 0.0001); + } + + @Test + void testBooleanTrue() { + assertEquals(true, JsonValueUtils.jsonValueToObject(JsonValue.from(true))); + } + + @Test + void testBooleanFalse() { + assertEquals(false, JsonValueUtils.jsonValueToObject(JsonValue.from(false))); + } + + @Test + void testFlatObject() { + Map input = new HashMap<>(); + input.put("key1", "value1"); + input.put("key2", 123); + + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(input)); + + assertInstanceOf(Map.class, result); + @SuppressWarnings("unchecked") + Map map = (Map) result; + assertEquals("value1", map.get("key1")); + assertEquals(123, ((Number) map.get("key2")).intValue()); + } + + @Test + void testNestedObject() { + Map inner = new HashMap<>(); + inner.put("x", "nested"); + Map outer = new HashMap<>(); + outer.put("inner", inner); + + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(outer)); + + assertInstanceOf(Map.class, result); + @SuppressWarnings("unchecked") + Map outerMap = (Map) result; + assertInstanceOf(Map.class, outerMap.get("inner")); + @SuppressWarnings("unchecked") + Map innerMap = (Map) outerMap.get("inner"); + assertEquals("nested", innerMap.get("x")); + } + + @Test + void testFlatArray() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList("a", "b", "c"))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List list = (List) result; + assertEquals(3, list.size()); + assertEquals("a", list.get(0)); + assertEquals("b", list.get(1)); + assertEquals("c", list.get(2)); + } + + @Test + void testNestedArray() { + Object result = + JsonValueUtils.jsonValueToObject( + JsonValue.from(Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3, 4)))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List outer = (List) result; + assertInstanceOf(List.class, outer.get(0)); + @SuppressWarnings("unchecked") + List inner = (List) outer.get(0); + assertEquals(1, ((Number) inner.get(0)).intValue()); + } + + @Test + void testMixedArray() { + Object result = + JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList("text", 42, true))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List list = (List) result; + assertEquals("text", list.get(0)); + assertEquals(42, ((Number) list.get(1)).intValue()); + assertEquals(true, list.get(2)); + } + + @Test + void testEmptyObject() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(new HashMap<>())); + assertInstanceOf(Map.class, result); + assertTrue(((Map) result).isEmpty()); + } + + @Test + void testEmptyArray() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList())); + assertInstanceOf(List.class, result); + assertTrue(((List) result).isEmpty()); + } + + @Test + void testJsonValueMapToObject() { + Map input = new HashMap<>(); + input.put("str", JsonValue.from("hello")); + input.put("num", JsonValue.from(7)); + input.put("bool", JsonValue.from(false)); + + Map result = JsonValueUtils.jsonValueMapToObject(input); + + assertEquals("hello", result.get("str")); + assertEquals(7, ((Number) result.get("num")).intValue()); + assertEquals(false, result.get("bool")); + } + + @Test + void testJsonValueMapToObjectEmpty() { + Map result = JsonValueUtils.jsonValueMapToObject(new HashMap<>()); + assertTrue(result.isEmpty()); + } +} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec new file mode 100644 index 00000000000..a72f03b8a33 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec @@ -0,0 +1,178 @@ +method: POST +path: responses +-- begin request body -- +{"prompt":{"id":"pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0","variables":{"user_message":{"type":"input_text","text":"Analyze these images and document"},"user_image_1":{"type":"input_image","image_url":"https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"},"user_file":{"type":"input_file","file_url":"https://www.berkshirehathaway.com/letters/2024ltr.pdf"},"user_image_2":{"type":"input_image","file_id":"file-BCuhT1HQ24kmtsuuzF1mh2"}},"version":"2"}} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e1fdb67e8b3d84e-SEA +content-type: application/json +date: Wed, 25 Mar 2026 18:08:13 GMT +openai-organization: datadog-staging +openai-processing-ms: 27175 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=0omhm_EDPiOwCkTnRHMTcH2143C9foKJkpkM5V98XPA-1774462065.9075637-1.0.1.1-Dr2KjfEYzM5.5hH26NB1U9ok305D3x0N1Jj2oNi9HRpE49YL0u3dMPv7FNeawtk.1duphFGgfCFpcbqn_qCR7bW1MtVt6IISK3BKO_TDbjrSg3oVneqJuZ62wmSRDDb2; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026 18:38:13 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 30000 +x-ratelimit-limit-tokens: 150000000 +x-ratelimit-remaining-requests: 29999 +x-ratelimit-remaining-tokens: 149998470 +x-ratelimit-reset-requests: 2ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_55ed921a6d294700bce3397e0b75e7da +-- end response headers -- +-- begin response body -- +{ + "id": "resp_00cd8b3bfcd5108a0169c42472356c8190a2f61c5a5f3b32e7", + "object": "response", + "created_at": 1774462067, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774462093, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": [ + { + "type": "message", + "content": [], + "role": "developer" + }, + { + "type": "message", + "content": [ + { + "type": "input_text", + "text": "Analyze the following content from the user:\n\nText message: Analyze these images and document\nImage reference 1: " + }, + { + "type": "input_image", + "detail": "auto", + "file_id": null, + "image_url": null + }, + { + "type": "input_text", + "text": "\nDocument reference: " + }, + { + "type": "input_file", + "file_id": null, + "file_url": "https://www.berkshirehathaway.com/letters/2024ltr.pdf" + }, + { + "type": "input_text", + "text": "\nImage reference 2: " + }, + { + "type": "input_image", + "detail": "auto", + "file_id": "file-BCuhT1HQ24kmtsuuzF1mh2", + "image_url": null + }, + { + "type": "input_text", + "text": "\n\nPlease provide a comprehensive analysis." + } + ], + "role": "user" + } + ], + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "output": [ + { + "id": "rs_00cd8b3bfcd5108a0169c424754a808190a4090b0188003f86", + "type": "reasoning", + "summary": [] + }, + { + "id": "msg_00cd8b3bfcd5108a0169c42480774c819082158e702c85be03", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Here\u2019s a step-by-step breakdown of the three items you\u2019ve provided:\n\n1. Image 1 (the two-snake logo) \n \u2022 Identification: This is the official logo of the Python programming language. \n \u2022 Design elements: \n \u2013 Two interlocking \u201csnake\u201d shapes (one blue, one yellow) arranged to form a stylized \u201cP.\u201d \n \u2013 Rounded corners, minimal line-weight, flat color palette\u2014characteristics of modern \u201cflat\u201d logo design. \n \u2013 Created to evoke both the language\u2019s namesake (Monty Python aside, a snake) and its emphasis on readability and simplicity. \n \u2022 Historical/contextual notes: \n \u2013 Python was conceived by Guido van Rossum in the late 1980s and first released in 1991. \n \u2013 Over the years the logo has come to symbolize a massive ecosystem\u2014from web back-ends (Django, Flask) to data science (Pandas, NumPy) to machine learning (scikit-learn, TensorFlow). \n \u2013 Its clean, friendly shape mirrors Python\u2019s design philosophy: code that reads like plain English, easy onboarding for newcomers. \n\n2. The Berkshire Hathaway 2024 Shareholder Letter (Warren E. Buffett) \n A. Purpose & Tone \n \u2013 Beyond the mandatory financial \u201creport,\u201d Buffett uses the letter to provide candid commentary\u2014both on successes and on mistakes. \n \u2013 He stresses the obligation of a CEO to treat shareholders as he\u2019d want to be treated if the roles were reversed. \n \u2013 He repeatedly admits errors (16 uses of \u201cmistake\u201d or \u201cerror\u201d in the last five years), contrasting this transparency with what he sees as glib \u201cfeel-good\u201d letters from many large companies. \n\n B. Major Themes \n 1. The Importance of Admitting and Correcting Mistakes \n \u2013 Mistakes in acquisitions or in assessing management fidelity can be painful but must be addressed promptly (\u201cno thumb-sucking\u201d). \n \u2013 Single great decisions\u2014e.g., GEICO purchase, finding Charlie Munger\u2014can compound into decade-long outperformance. \n\n 2. Capital Allocation Philosophy \n \u2013 The core measure Buffett emphasizes is \u201coperating earnings,\u201d excluding unrealized securities gains or losses. \n \u2013 He reaffirms that Berkshire will remain overwhelmingly invested in equities (both controlled subsidiaries and minority stakes in giants like Apple, Coca-Cola, American Express). \n \u2013 He warns against over-reliance on cash or bonds, which carry currency\u2010value risk; instead, intrinsic ownership of businesses is the best inflation hedge. \n\n 3. Insurance Float as a Growth Engine \n \u2013 Property-casualty underwriting delivers \u201cfloat\u201d (premiums received before claims are paid). \n \u2013 If priced intelligently, float can be \u201ccostless capital\u201d that Buffett can then deploy elsewhere. \n \u2013 He lauds Ajit Jain for building Berkshire\u2019s P/C operation into a global leader, but stresses the business is perilously dependent on accurate long-term loss estimates. \n\n 4. Record U.S. Tax Payments \n \u2013 In 2024 Berkshire paid $26.8 billion in U.S. federal income taxes\u2014more than any other corporation in American history. \n \u2013 Buffett attributes this partly to his long-ago decision (and shareholders\u2019 endorsement) to forgo dividends, instead reinvesting all earnings back into operations. \n\n 5. Discipline in CEO and Manager Selection \n \u2013 Buffett never considers pedigree (school attended) when hiring CEOs\u2014innate talent often outweighs credentials. \n \u2013 He illustrates this with the story of Pete Liegl (founder of Forest River), whose self-set $100 k base salary plus performance bonus yielded multiples of that investment for Berkshire over 19 years. \n\n 6. International Ventures\u2014The Japan Five \n \u2013 Since 2019 Berkshire has taken minority stakes (below 10%) in five large Japanese trading houses (Itochu, Marubeni, Mitsubishi, Mitsui, Sumitomo). \n \u2013 He praises their capital\u2010allocation discipline, low executive pay, willingness to buy back shares. \n \u2013 Berkshire balances that foreign\u2010currency exposure by matching it with yen-denominated debt, achieving near currency neutrality. \n\n 7. Shareholder-Centered Culture & Annual Meeting \n \u2013 Buffett invites shareholders to the May 3 gathering in Omaha, emphasizing accessibility of management, Q&A format, volunteer-run merch booths, and a single themed book for sale. \n \u2013 He recounts lighthearted family anecdotes (his sister Bertie\u2019s cane, grandchildren) to underscore the meeting\u2019s informal, \u201cfamily reunion\u201d atmosphere. \n\n C. Performance Snapshot \n \u2013 2024 operating earnings: $47.4 billion vs. $37.4 billion in 2023. \n \u2013 Insurance underwriting profit: $9.0 billion; investment income: $13.7 billion. \n \u2013 Continued strength at BNSF and Berkshire Hathaway Energy. \n \u2013 Slide reinstates the 1965\u20132024 compound annual gain of 19.9% vs. 10.4% for the S&P 500. \n\n D. Stylistic & Structural Notes \n \u2013 Plain\u2010spoken, anecdote-rich, avoids corporate jargon. \n \u2013 Buffett interleaves big-picture economic observations (capitalism\u2019s triumphs and pitfalls) with very granular \u201cinside\u201d details (how he negotiated with Pete Liegl). \n \u2013 Occasional dry humor (e.g., comparing his and his sister\u2019s canes as \u201cmale-deterrents\u201d). \n\n3. Image 2 \n \u2022 On my end the image rendered as a blank/empty panel. There\u2019s no visible content to describe or analyze. \n \u2022 If this was meant to convey text, a chart, code or a photograph, could you please re-upload a higher-resolution copy or a different file format? \n\n\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013 \nSummary: \n\u2013 You\u2019ve supplied two corporate \u201cicons\u201d (the Python logo and the Berkshire shareholder letter), both of which epitomize transparency, community-oriented design, and long-term stewardship\u2014one in software, the other in capital allocation. \n\u2013 The Python logo signals a language built for clarity, extensibility and an open ecosystem. \n\u2013 Buffett\u2019s letter models \u201cflat,\u201d jargon-free communication, candidly reporting mistakes alongside successes, and reaffirming a shareholder-centric culture. \n\u2013 Please let me know about Image 2, and I\u2019ll be happy to complete the visual analysis." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt": { + "id": "pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0", + "variables": { + "user_message": { + "type": "input_text", + "text": "Analyze these images and document" + }, + "user_image_1": { + "type": "input_image", + "detail": "auto", + "file_id": null, + "image_url": null + }, + "user_file": { + "type": "input_file", + "file_id": null, + "file_url": "https://www.berkshirehathaway.com/letters/2024ltr.pdf" + }, + "user_image_2": { + "type": "input_image", + "detail": "auto", + "file_id": "file-BCuhT1HQ24kmtsuuzF1mh2", + "image_url": null + } + }, + "version": "2" + }, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 11236, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 2366, + "output_tokens_details": { + "reasoning_tokens": 960 + }, + "total_tokens": 13602 + }, + "user": null, + "metadata": {} +}� +-- end response body -- diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec new file mode 100644 index 00000000000..8b748b9b57a --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec @@ -0,0 +1,112 @@ +method: POST +path: responses +-- begin request body -- +{"input":"Use the custom_weather tool to answer: What's the weather in Boston?","model":"gpt-5","tool_choice":{"name":"custom_weather","type":"custom"},"tools":[{"name":"custom_weather","type":"custom","description":"Return weather for a location","format":{"type":"text"}}]} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e2122d2ae6712d4-SEA +content-type: application/json +date: Wed, 25 Mar 2026 21:51:20 GMT +openai-organization: datadog-staging +openai-processing-ms: 3360 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=SP753lo3ga6JHum4FOPTYPa6B.h7QhGpK2Tih8iqYOI-1774475476.9017334-1.0.1.1-G_WqthMsAoHTvAkAACMpAsVWmtzzeqOHTjGZ_eY4WSxZz_pND9T9O78.pCiPD52gquRIzVxrXPd9J06VIMqiuAA9z_GWrLKLGc.s2dzDvUsWxIek6wIPjN4EOBFNlgoB; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026 22:21:20 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 15000 +x-ratelimit-limit-tokens: 40000000 +x-ratelimit-remaining-requests: 14999 +x-ratelimit-remaining-tokens: 40000000 +x-ratelimit-reset-requests: 4ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_dcbdd1cbcd494ff793ee2e099de4554e +-- end response headers -- +-- begin response body -- +{ + "id": "resp_0f18ec8851ed77310169c458d536188190ae13c792509ad0c7", + "object": "response", + "created_at": 1774475477, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774475480, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5-2025-08-07", + "output": [ + { + "id": "rs_0f18ec8851ed77310169c458d5b5108190a0c49289af33d93b", + "type": "reasoning", + "summary": [] + }, + { + "id": "ctc_0f18ec8851ed77310169c458d84a7c8190b1cefc253127979c", + "type": "custom_tool_call", + "status": "completed", + "call_id": "call_sxqlRD4E0PihmL74BNt7GqVV", + "input": "Boston, MA, USA", + "name": "custom_weather" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "type": "custom", + "name": "custom_weather" + }, + "tools": [ + { + "type": "custom", + "description": "Return weather for a location", + "format": { + "type": "text" + }, + "name": "custom_weather" + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 67, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 238, + "output_tokens_details": { + "reasoning_tokens": 192 + }, + "total_tokens": 305 + }, + "user": null, + "metadata": {} +}� +-- end response body -- diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy index b12984ecbee..8c39baf2851 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy @@ -247,7 +247,7 @@ class TagsAssert { if (expected instanceof Pattern) { assert value =~ expected: "Tag \"$name\": \"${value.toString()}\" does not match pattern \"$expected\"" } else if (expected instanceof Class) { - assert ((Class) expected).isInstance(value): "Tag \"$name\": instance check $expected failed for \"${value.toString()}\" of class \"${value.class}\"" + assert ((Class) expected).isInstance(value): "Tag \"$name\": instance check $expected failed for \"${value.toString()}\" of class \"${value?.class}\"" } else if (expected instanceof Closure) { assert ((Closure) expected).call(value): "Tag \"$name\": closure call ${expected.toString()} failed with \"$value\"" } else if (expected instanceof CharSequence) { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 512a3106ce6..629faa23f5a 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -217,6 +217,11 @@ public static LLMMessage from(String role, String content, List toolCa return new LLMMessage(role, content, toolCalls, null); } + public static LLMMessage from( + String role, String content, List toolCalls, List toolResults) { + return new LLMMessage(role, content, toolCalls, toolResults); + } + public static LLMMessage from(String role, String content) { return new LLMMessage(role, content, null, null); } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java index afa4f2b241e..130cf610dc0 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -12,4 +12,5 @@ public class LLMObsTags { public static final String MODEL_NAME = "model_name"; public static final String MODEL_VERSION = "model_version"; public static final String MODEL_PROVIDER = "model_provider"; + public static final String TOOL_DEFINITIONS = "tool_definitions"; } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7241d469006..cb8ebd3d8a1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -52,12 +52,17 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPAN_ID = "span_id".getBytes(StandardCharsets.UTF_8); private static final byte[] TRACE_ID = "trace_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] DD = "_dd".getBytes(StandardCharsets.UTF_8); + private static final byte[] APM_TRACE_ID = "apm_trace_id".getBytes(StandardCharsets.UTF_8); private static final byte[] PARENT_ID = "parent_id".getBytes(StandardCharsets.UTF_8); private static final byte[] NAME = "name".getBytes(StandardCharsets.UTF_8); private static final byte[] DURATION = "duration".getBytes(StandardCharsets.UTF_8); private static final byte[] START_NS = "start_ns".getBytes(StandardCharsets.UTF_8); private static final byte[] STATUS = "status".getBytes(StandardCharsets.UTF_8); private static final byte[] ERROR = "error".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_MESSAGE = "message".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_TYPE = "type".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_STACK = "stack".getBytes(StandardCharsets.UTF_8); private static final byte[] META = "meta".getBytes(StandardCharsets.UTF_8); private static final byte[] METADATA = "metadata".getBytes(StandardCharsets.UTF_8); @@ -65,6 +70,7 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPANS = "spans".getBytes(StandardCharsets.UTF_8); private static final byte[] METRICS = "metrics".getBytes(StandardCharsets.UTF_8); private static final byte[] TAGS = "tags".getBytes(StandardCharsets.UTF_8); + private static final String LLMOBS_LANGUAGE_TAG = "language:jvm"; private static final byte[] LLM_MESSAGE_ROLE = "role".getBytes(StandardCharsets.UTF_8); private static final byte[] LLM_MESSAGE_CONTENT = "content".getBytes(StandardCharsets.UTF_8); @@ -147,14 +153,18 @@ public void map(List> trace, Writable writable) { writable.writeFloat(span.getDurationNano()); // 7 - writable.writeUTF8(ERROR); - writable.writeInt(span.getError()); - - boolean errored = span.getError() == 1; + writable.writeUTF8(STATUS); + writable.writeString(span.getError() == 0 ? "ok" : "error", null); // 8 - writable.writeUTF8(STATUS); - writable.writeString(errored ? "error" : "ok", null); + writable.writeUTF8(DD); + writable.startMap(3); + writable.writeUTF8(SPAN_ID); + writable.writeString(String.valueOf(span.getSpanId()), null); + writable.writeUTF8(TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + writable.writeUTF8(APM_TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); /* 9 (metrics), 10 (tags), 11 meta */ span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); @@ -203,15 +213,15 @@ private static Map getErrorsMap(CoreSpan span) { Map errors = new HashMap<>(); String errorMsg = span.getTag(DDTags.ERROR_MSG); if (errorMsg != null && !errorMsg.isEmpty()) { - errors.put(DDTags.ERROR_MSG, errorMsg); + errors.put("message", errorMsg); } String errorType = span.getTag(DDTags.ERROR_TYPE); if (errorType != null && !errorType.isEmpty()) { - errors.put(DDTags.ERROR_TYPE, errorType); + errors.put("type", errorType); } String errorStack = span.getTag(DDTags.ERROR_STACK); if (errorStack != null && !errorStack.isEmpty()) { - errors.put(DDTags.ERROR_STACK, errorStack); + errors.put("stack", errorStack); } return errors; } @@ -230,6 +240,7 @@ private static final class MetaWriter implements MetadataConsumer { LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_NAME, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_VERSION, + LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS, LLMOBS_TAG_PREFIX + LLMObsTags.METADATA))); MetaWriter withWritable(Writable writable, Map errorInfo) { @@ -283,7 +294,7 @@ public void accept(Metadata metadata) { // write tags (10) writable.writeUTF8(TAGS); writable.startArray(tagsSize + 1); - writable.writeString("language:jvm", null); + writable.writeString(LLMOBS_LANGUAGE_TAG, null); for (Map.Entry tag : metadata.getTags().entrySet()) { String key = tag.getKey(); Object value = tag.getValue(); @@ -293,15 +304,33 @@ public void accept(Metadata metadata) { } // write meta (11) - int metaSize = tagsToRemapToMeta.size() + 1 + (null != errorInfo ? errorInfo.size() : 0); + int metaSize = + tagsToRemapToMeta.size() + 1 + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); writable.writeUTF8(META); writable.startMap(metaSize); writable.writeUTF8(SPAN_KIND); writable.writeString(spanKind, null); - for (Map.Entry error : errorInfo.entrySet()) { - writable.writeUTF8(error.getKey().getBytes()); - writable.writeString(error.getValue(), null); + if (null != errorInfo && !errorInfo.isEmpty()) { + writable.writeUTF8(ERROR); + writable.startMap(errorInfo.size()); + for (Map.Entry error : errorInfo.entrySet()) { + switch (error.getKey()) { + case "message": + writable.writeUTF8(ERROR_MESSAGE); + break; + case "type": + writable.writeUTF8(ERROR_TYPE); + break; + case "stack": + writable.writeUTF8(ERROR_STACK); + break; + default: + writable.writeString(error.getKey(), null); + break; + } + writable.writeString(error.getValue(), null); + } } for (Map.Entry tag : tagsToRemapToMeta.entrySet()) { @@ -309,70 +338,19 @@ public void accept(Metadata metadata) { Object val = tag.getValue(); if (key.equals(INPUT) || key.equals(OUTPUT)) { if (spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) { - if (!(val instanceof List)) { + writable.writeString(key, null); + if (val instanceof List) { + writable.startMap(1); + writable.writeString("messages", null); + writeLlmMessages((List) val); + } else if (key.equals(INPUT) && val instanceof Map) { + writeLlmInputMap((Map) val); + } else { LOGGER.warn( "unexpectedly found incorrect type for LLM span IO {}, expecting list", val.getClass().getName()); continue; } - writable.writeString(key, null); - writable.startMap(1); - // llm span kind must have llm objects - List messages = (List) val; - writable.writeString("messages", null); - writable.startArray(messages.size()); - for (LLMObs.LLMMessage message : messages) { - List toolCalls = message.getToolCalls(); - List toolResults = message.getToolResults(); - boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); - boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); - int mapSize = 2; // role and content - if (hasToolCalls) mapSize++; - if (hasToolResults) mapSize++; - writable.startMap(mapSize); - writable.writeUTF8(LLM_MESSAGE_ROLE); - writable.writeString(message.getRole(), null); - writable.writeUTF8(LLM_MESSAGE_CONTENT); - writable.writeString(message.getContent(), null); - if (hasToolCalls) { - writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); - writable.startArray(toolCalls.size()); - for (LLMObs.ToolCall toolCall : toolCalls) { - Map arguments = toolCall.getArguments(); - boolean hasArguments = null != arguments && !arguments.isEmpty(); - writable.startMap(hasArguments ? 4 : 3); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolCall.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolCall.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolCall.getToolId(), null); - if (hasArguments) { - writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); - writable.startMap(arguments.size()); - for (Map.Entry argument : arguments.entrySet()) { - writable.writeString(argument.getKey(), null); - writable.writeObject(argument.getValue(), null); - } - } - } - } - if (hasToolResults) { - writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); - writable.startArray(toolResults.size()); - for (LLMObs.ToolResult toolResult : toolResults) { - writable.startMap(4); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolResult.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolResult.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolResult.getToolId(), null); - writable.writeUTF8(LLM_TOOL_RESULT_RESULT); - writable.writeString(toolResult.getResult(), null); - } - } - } } else if (spanKind.equals(Tags.LLMOBS_EMBEDDING_SPAN_KIND) && key.equals(INPUT)) { if (!(val instanceof List)) { LOGGER.warn( @@ -410,6 +388,86 @@ public void accept(Metadata metadata) { } } } + + private void writeLlmInputMap(Map inputMap) { + writable.startMap(inputMap.size()); + for (Map.Entry entry : inputMap.entrySet()) { + String inputKey = String.valueOf(entry.getKey()); + Object inputValue = entry.getValue(); + writable.writeString(inputKey, null); + if ("messages".equals(inputKey) && inputValue instanceof List) { + writeLlmMessages((List) inputValue); + } else { + writable.writeObject(inputValue, null); + } + } + } + + private void writeLlmMessages(List messages) { + writable.startArray(messages.size()); + for (Object messageObj : messages) { + if (!(messageObj instanceof LLMObs.LLMMessage)) { + writable.writeObject(messageObj, null); + continue; + } + + LLMObs.LLMMessage message = (LLMObs.LLMMessage) messageObj; + List toolCalls = message.getToolCalls(); + List toolResults = message.getToolResults(); + boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); + boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); + boolean hasContent = message.getContent() != null; + int mapSize = 1; + if (hasContent) mapSize++; + if (hasToolCalls) mapSize++; + if (hasToolResults) mapSize++; + writable.startMap(mapSize); + writable.writeUTF8(LLM_MESSAGE_ROLE); + writable.writeString(message.getRole(), null); + if (hasContent) { + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + } + if (hasToolCalls) { + writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); + writable.startArray(toolCalls.size()); + for (LLMObs.ToolCall toolCall : toolCalls) { + Map arguments = toolCall.getArguments(); + boolean hasArguments = null != arguments && !arguments.isEmpty(); + writable.startMap(hasArguments ? 4 : 3); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolCall.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolCall.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolCall.getToolId(), null); + if (hasArguments) { + writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); + writable.startMap(arguments.size()); + for (Map.Entry argument : arguments.entrySet()) { + writable.writeString(argument.getKey(), null); + writable.writeObject(argument.getValue(), null); + } + } + } + } + if (hasToolResults) { + writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); + writable.startArray(toolResults.size()); + for (LLMObs.ToolResult toolResult : toolResults) { + writable.startMap(4); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolResult.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolResult.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolResult.getToolId(), null); + writable.writeUTF8(LLM_TOOL_RESULT_RESULT); + writable.writeString(toolResult.getResult(), null); + } + } + } + } } private static class PayloadV1 extends Payload { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 20656ba2a1e..fc254458920 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import datadog.communication.serialization.ByteBufferConsumer import datadog.communication.serialization.FlushingBuffer import datadog.communication.serialization.msgpack.MsgPackWriter +import datadog.trace.api.DDTags import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes import datadog.trace.bootstrap.instrumentation.api.Tags @@ -39,17 +40,42 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { llmSpan.setSpanType(InternalSpanTypes.LLMOBS) - def inputMessages = [LLMObs.LLMMessage.from("user", "Hello, what's the weather like?")] + def toolCall = LLMObs.ToolCall.from("get_weather", "function_call", "call_123", [location: "San Francisco"]) + def toolResult = LLMObs.ToolResult.from("get_weather", "function_call_output", "call_123", '{"temperature":"72F"}') + def inputMessages = [ + LLMObs.LLMMessage.from("user", "Hello, what's the weather like?"), + LLMObs.LLMMessage.from("assistant", null, [toolCall], [toolResult]) + ] def outputMessages = [LLMObs.LLMMessage.from("assistant", "I'll help you check the weather.")] - llmSpan.setTag("_ml_obs_tag.input", inputMessages) + llmSpan.setTag("_ml_obs_tag.input", [ + messages: inputMessages, + prompt: [ + id: "prompt_123", + version: "1", + variables: [city: "San Francisco"], + chat_template: [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] + ] + ]) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) + llmSpan.setTag("_ml_obs_tag.tool_definitions", [ + [ + name: "get_weather", + description: "Get weather by city", + schema: [type: "object", properties: [city: [type: "string"]]] + ] + ]) + llmSpan.setError(true) + llmSpan.setTag(DDTags.ERROR_MSG, "boom") + llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") + llmSpan.setTag(DDTags.ERROR_STACK, "stacktrace") llmSpan.finish() def trace = [llmSpan] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace, mapper) @@ -101,22 +127,46 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("trace_id") spanData.containsKey("start_ns") spanData.containsKey("duration") - spanData["error"] == 0 + spanData.containsKey("_dd") + spanData["_dd"]["span_id"] == spanData["span_id"] + spanData["_dd"]["trace_id"] == spanData["trace_id"] + spanData["_dd"]["apm_trace_id"] == spanData["trace_id"] spanData.containsKey("meta") spanData["meta"]["span.kind"] == "llm" + spanData["meta"].containsKey("error") + spanData["meta"]["error"]["message"] == "boom" + spanData["meta"]["error"]["type"] == "java.lang.IllegalStateException" + spanData["meta"]["error"]["stack"] == "stacktrace" spanData["meta"].containsKey("input") spanData["meta"]["input"].containsKey("messages") spanData["meta"]["input"]["messages"][0].containsKey("content") spanData["meta"]["input"]["messages"][0]["content"] == "Hello, what's the weather like?" spanData["meta"]["input"]["messages"][0].containsKey("role") spanData["meta"]["input"]["messages"][0]["role"] == "user" + spanData["meta"]["input"]["messages"][1]["role"] == "assistant" + !spanData["meta"]["input"]["messages"][1].containsKey("content") + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["type"] == "function_call" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["arguments"] == [location: "San Francisco"] + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["type"] == "function_call_output" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["result"] == '{"temperature":"72F"}' + spanData["meta"]["input"]["prompt"]["id"] == "prompt_123" + spanData["meta"]["input"]["prompt"]["version"] == "1" + spanData["meta"]["input"]["prompt"]["variables"] == [city: "San Francisco"] + spanData["meta"]["input"]["prompt"]["chat_template"] == [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] spanData["meta"].containsKey("output") spanData["meta"]["output"].containsKey("messages") spanData["meta"]["output"]["messages"][0].containsKey("content") spanData["meta"]["output"]["messages"][0]["content"] == "I'll help you check the weather." spanData["meta"]["output"]["messages"][0].containsKey("role") spanData["meta"]["output"]["messages"][0]["role"] == "assistant" + spanData["meta"]["tool_definitions"][0]["name"] == "get_weather" + spanData["meta"]["tool_definitions"][0]["description"] == "Get weather by city" + spanData["meta"]["tool_definitions"][0]["schema"] == [type: "object", properties: [city: [type: "string"]]] spanData["meta"].containsKey("metadata") spanData.containsKey("metrics") @@ -148,7 +198,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace = [regularSpan1, regularSpan2] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace, mapper) @@ -192,7 +243,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace1 = [llmSpan1, llmSpan2] def trace2 = [llmSpan3] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace1, mapper)