From 8ab723c33d0aaa5a4cfbf549bc4d1c7e22a48e85 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 28 Apr 2026 18:19:21 -0500 Subject: [PATCH 1/4] feat: Add ManagedGraphResult, GraphMetricSummary, and AgentGraphRunnerResult types - Add GraphMetrics dataclass (runner-layer return type for graph runs) - Add GraphMetricSummary dataclass (managed-layer metrics, analogous to LDAIMetricSummary for single-model invocations) - Add ManagedGraphResult dataclass (managed-layer return type from ManagedAgentGraph) - Add AgentGraphRunnerResult dataclass (future runner return type, no evaluations field) - ManagedAgentGraph.run() now returns ManagedGraphResult with GraphMetricSummary built from the runner's AgentGraphResult metrics - Export all new types from ldai package Co-Authored-By: Claude Sonnet 4.6 --- packages/sdk/server-ai/src/ldai/__init__.py | 8 ++ .../server-ai/src/ldai/managed_agent_graph.py | 40 +++++++--- .../server-ai/src/ldai/providers/__init__.py | 8 ++ .../sdk/server-ai/src/ldai/providers/types.py | 76 ++++++++++++++++++- packages/sdk/server-ai/src/ldai/tracker.py | 1 + .../tests/test_managed_agent_graph.py | 8 +- 6 files changed, 128 insertions(+), 13 deletions(-) diff --git a/packages/sdk/server-ai/src/ldai/__init__.py b/packages/sdk/server-ai/src/ldai/__init__.py index f02cee30..56d780d3 100644 --- a/packages/sdk/server-ai/src/ldai/__init__.py +++ b/packages/sdk/server-ai/src/ldai/__init__.py @@ -34,8 +34,12 @@ from ldai.providers import ( AgentGraphResult, AgentGraphRunner, + AgentGraphRunnerResult, AgentResult, AgentRunner, + GraphMetrics, + GraphMetricSummary, + ManagedGraphResult, ManagedResult, Runner, RunnerResult, @@ -51,6 +55,10 @@ 'AgentGraphRunner', 'AgentResult', 'AgentGraphResult', + 'AgentGraphRunnerResult', + 'GraphMetrics', + 'GraphMetricSummary', + 'ManagedGraphResult', 'ManagedResult', 'Runner', 'RunnerResult', diff --git a/packages/sdk/server-ai/src/ldai/managed_agent_graph.py b/packages/sdk/server-ai/src/ldai/managed_agent_graph.py index a146e60e..50b3440e 100644 --- a/packages/sdk/server-ai/src/ldai/managed_agent_graph.py +++ b/packages/sdk/server-ai/src/ldai/managed_agent_graph.py @@ -1,17 +1,19 @@ """ManagedAgentGraph — LaunchDarkly managed wrapper for agent graph execution.""" -from typing import Any +import asyncio +from typing import Any, List from ldai.providers import AgentGraphResult, AgentGraphRunner +from ldai.providers.types import GraphMetricSummary, JudgeResult, ManagedGraphResult class ManagedAgentGraph: """ LaunchDarkly managed wrapper for AI agent graph execution. - Holds an AgentGraphRunner. Auto-tracking of path, - tool calls, handoffs, latency, and invocation success/failure is handled - by the runner implementation. + Holds an AgentGraphRunner. Wraps the runner result in a + :class:`~ldai.providers.types.ManagedGraphResult` and builds a + :class:`~ldai.providers.types.GraphMetricSummary` from the runner's metrics. Obtain an instance via ``LDAIClient.create_agent_graph()``. """ @@ -27,17 +29,37 @@ def __init__( """ self._runner = runner - async def run(self, input: Any) -> AgentGraphResult: + async def run(self, input: Any) -> ManagedGraphResult: """ Run the agent graph with the given input. - Delegates to the underlying AgentGraphRunner, which handles - execution and all auto-tracking internally. + Delegates to the underlying AgentGraphRunner, builds a + :class:`GraphMetricSummary` from the result, and wraps everything in a + :class:`ManagedGraphResult`. :param input: The input prompt or structured input for the graph - :return: AgentGraphResult containing the output, raw response, and metrics + :return: ManagedGraphResult containing the content, metric summary, raw response, + and an optional evaluations task (currently always ``None`` for graphs — + per-graph evaluations will be added in a future PR). """ - return await self._runner.run(input) + result: AgentGraphResult = await self._runner.run(input) + + # Build a GraphMetricSummary from the runner result's LDAIMetrics. + # path and node_metrics will be populated once graph runners are migrated + # to return AgentGraphRunnerResult with GraphMetrics (PR 11). + metrics = result.metrics + summary = GraphMetricSummary( + success=metrics.success, + usage=metrics.usage, + duration_ms=getattr(metrics, 'duration_ms', None), + ) + + return ManagedGraphResult( + content=result.output, + metrics=summary, + raw=result.raw, + evaluations=None, + ) def get_agent_graph_runner(self) -> AgentGraphRunner: """ diff --git a/packages/sdk/server-ai/src/ldai/providers/__init__.py b/packages/sdk/server-ai/src/ldai/providers/__init__.py index 6f472c69..22dce784 100644 --- a/packages/sdk/server-ai/src/ldai/providers/__init__.py +++ b/packages/sdk/server-ai/src/ldai/providers/__init__.py @@ -6,9 +6,13 @@ from ldai.providers.runner_factory import RunnerFactory from ldai.providers.types import ( AgentGraphResult, + AgentGraphRunnerResult, AgentResult, + GraphMetrics, + GraphMetricSummary, JudgeResult, LDAIMetrics, + ManagedGraphResult, ManagedResult, ModelResponse, RunnerResult, @@ -20,10 +24,14 @@ 'AIProvider', 'AgentGraphResult', 'AgentGraphRunner', + 'AgentGraphRunnerResult', 'AgentResult', 'AgentRunner', + 'GraphMetrics', + 'GraphMetricSummary', 'JudgeResult', 'LDAIMetrics', + 'ManagedGraphResult', 'ManagedResult', 'ModelResponse', 'ModelRunner', diff --git a/packages/sdk/server-ai/src/ldai/providers/types.py b/packages/sdk/server-ai/src/ldai/providers/types.py index f5224e0e..5bb8ce47 100644 --- a/packages/sdk/server-ai/src/ldai/providers/types.py +++ b/packages/sdk/server-ai/src/ldai/providers/types.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional from ldai.models import LDMessage @@ -114,6 +114,80 @@ class StructuredResponse: metrics: LDAIMetrics +@dataclass +class GraphMetrics: + """Contains raw metrics from a single agent graph run.""" + + success: bool + """Whether the graph run succeeded.""" + + path: List[str] = field(default_factory=list) + """Ordered list of node keys visited during the run.""" + + duration_ms: Optional[int] = None + """Wall-clock duration of the graph run in milliseconds.""" + + usage: Optional[TokenUsage] = None + """Optional aggregate token usage information across all nodes in the graph run.""" + + node_metrics: Dict[str, LDAIMetrics] = field(default_factory=dict) + """Per-node metrics keyed by node key.""" + + +@dataclass +class GraphMetricSummary: + """Contains a summary of metrics for an agent graph run.""" + + success: bool + """Whether the graph run succeeded.""" + + path: List[str] = field(default_factory=list) + """Ordered list of node keys visited during the run.""" + + duration_ms: Optional[int] = None + """Wall-clock duration of the graph run in milliseconds.""" + + usage: Optional[TokenUsage] = None + """Optional aggregate token usage information across all nodes in the graph run.""" + + node_metrics: Dict[str, LDAIMetrics] = field(default_factory=dict) + """Per-node metrics keyed by node key.""" + + resumption_token: Optional[str] = None + """Optional resumption token from the graph tracker for cross-process resumption.""" + + +@dataclass +class ManagedGraphResult: + """Contains the result of a managed agent graph run, including metrics and optional judge evaluations.""" + + content: str + """The graph's final output content.""" + + metrics: GraphMetricSummary + """Aggregated metric summary from the graph tracker for this run.""" + + raw: Optional[Any] = None + """Optional provider-native response object for advanced consumers.""" + + evaluations: Optional[asyncio.Task[List[JudgeResult]]] = None + """Optional asyncio Task that resolves to the list of :class:`JudgeResult` instances when awaited.""" + + +@dataclass +class AgentGraphRunnerResult: + """Contains the result of an agent graph runner invocation.""" + + content: str + """The graph's final output content.""" + + metrics: GraphMetrics + """Metrics from the graph run.""" + + raw: Optional[Any] = None + """Optional provider-native response object for advanced consumers.""" + + @dataclass class JudgeResult: """Contains the result of a single judge evaluation.""" diff --git a/packages/sdk/server-ai/src/ldai/tracker.py b/packages/sdk/server-ai/src/ldai/tracker.py index 43c836bf..31416649 100644 --- a/packages/sdk/server-ai/src/ldai/tracker.py +++ b/packages/sdk/server-ai/src/ldai/tracker.py @@ -442,6 +442,7 @@ def track_tool_calls(self, tool_calls: Iterable[str]) -> None: for tool_key in tool_calls_list: self.track_tool_call(tool_key) + def track_success(self) -> None: """ Track a successful AI generation. diff --git a/packages/sdk/server-ai/tests/test_managed_agent_graph.py b/packages/sdk/server-ai/tests/test_managed_agent_graph.py index 35be2766..9cdceaed 100644 --- a/packages/sdk/server-ai/tests/test_managed_agent_graph.py +++ b/packages/sdk/server-ai/tests/test_managed_agent_graph.py @@ -5,7 +5,7 @@ from ldclient import Config, Context, LDClient from ldclient.integrations.test_data import TestData -from ldai import LDAIClient, ManagedAgentGraph +from ldai import LDAIClient, ManagedAgentGraph, ManagedGraphResult from ldai.providers.types import LDAIMetrics from ldai.providers import AgentGraphResult, AgentGraphRunner, ToolRegistry @@ -31,7 +31,8 @@ async def test_managed_agent_graph_run_delegates_to_runner(): runner = StubAgentGraphRunner("hello world") managed = ManagedAgentGraph(runner) result = await managed.run("test input") - assert result.output == "hello world" + assert isinstance(result, ManagedGraphResult) + assert result.content == "hello world" assert result.metrics.success is True @@ -172,7 +173,8 @@ async def test_create_agent_graph_run_produces_result(ldai_client: LDAIClient): assert managed is not None result = await managed.run("find restaurants") - assert result.output == "final answer" + assert isinstance(result, ManagedGraphResult) + assert result.content == "final answer" assert result.metrics.success is True From 9720fa9e0810756d6b5ece5fe8948d88e7ea3db3 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 1 May 2026 13:04:41 -0500 Subject: [PATCH 2/4] docs: trim redundant sentence from ManagedAgentGraph.run docstring Co-Authored-By: Claude Sonnet 4.6 --- packages/sdk/server-ai/src/ldai/managed_agent_graph.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/sdk/server-ai/src/ldai/managed_agent_graph.py b/packages/sdk/server-ai/src/ldai/managed_agent_graph.py index 50b3440e..96cf5dcc 100644 --- a/packages/sdk/server-ai/src/ldai/managed_agent_graph.py +++ b/packages/sdk/server-ai/src/ldai/managed_agent_graph.py @@ -33,10 +33,6 @@ async def run(self, input: Any) -> ManagedGraphResult: """ Run the agent graph with the given input. - Delegates to the underlying AgentGraphRunner, builds a - :class:`GraphMetricSummary` from the result, and wraps everything in a - :class:`ManagedGraphResult`. - :param input: The input prompt or structured input for the graph :return: ManagedGraphResult containing the content, metric summary, raw response, and an optional evaluations task (currently always ``None`` for graphs — From 75172d8ec3369761df5cb53db6192681203d0aec Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 1 May 2026 16:11:01 -0500 Subject: [PATCH 3/4] fix: remove unused asyncio, List, JudgeResult imports from managed_agent_graph Co-Authored-By: Claude Sonnet 4.6 --- packages/sdk/server-ai/src/ldai/managed_agent_graph.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/sdk/server-ai/src/ldai/managed_agent_graph.py b/packages/sdk/server-ai/src/ldai/managed_agent_graph.py index 96cf5dcc..60c8a93b 100644 --- a/packages/sdk/server-ai/src/ldai/managed_agent_graph.py +++ b/packages/sdk/server-ai/src/ldai/managed_agent_graph.py @@ -1,10 +1,9 @@ """ManagedAgentGraph — LaunchDarkly managed wrapper for agent graph execution.""" -import asyncio -from typing import Any, List +from typing import Any from ldai.providers import AgentGraphResult, AgentGraphRunner -from ldai.providers.types import GraphMetricSummary, JudgeResult, ManagedGraphResult +from ldai.providers.types import GraphMetricSummary, ManagedGraphResult class ManagedAgentGraph: From af60d270c3d4d69b0e4488d1d6594de9dcb30e70 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 1 May 2026 17:02:07 -0500 Subject: [PATCH 4/4] fix: remove extra blank line in tracker.py (E303) Co-Authored-By: Claude Sonnet 4.6 --- packages/sdk/server-ai/src/ldai/tracker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/sdk/server-ai/src/ldai/tracker.py b/packages/sdk/server-ai/src/ldai/tracker.py index 31416649..43c836bf 100644 --- a/packages/sdk/server-ai/src/ldai/tracker.py +++ b/packages/sdk/server-ai/src/ldai/tracker.py @@ -442,7 +442,6 @@ def track_tool_calls(self, tool_calls: Iterable[str]) -> None: for tool_key in tool_calls_list: self.track_tool_call(tool_key) - def track_success(self) -> None: """ Track a successful AI generation.