feat: add thread-safe AIConfigTracker, metrics & resumption tokens (AIC-2664)

Implements the AITRACK surface on LDAIConfigTracker: per-run UUID runId and
track data, the full set of track methods (duration, time-to-first-token,
success/error, feedback, tokens, tool calls, judge result) plus trackDurationOf
and trackMetricsOf wrappers, and a metric summary.

Record-once metrics use atomic claim-before-emit guards so exactly one event is
produced under concurrency; tool-call and judge-result events are not once-only.
Negative durations and token counts are clamped, and a null judge score is
distinct from a legitimate 0.0.

Resumption tokens are URL-safe Base64 of canonical JSON in fixed key order
(runId, configKey, variationKey, version, graphKey); variationKey is always
emitted for cross-SDK parity and modelName/providerName are not carried. Decode
strictly type-validates each field and rejects malformed/oversized tokens.

LDAIClientImpl now wires createTracker() on the config types to the real tracker
and adds createTracker(token, context) to reconstruct a run across processes.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: ctawiah

lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIClient.java

@@ -81,4 +81,19 @@ AIJudgeConfig judgeConfig(
       LDContext context,
       AIJudgeConfigDefault defaultValue,
       Map<String, Object> variables);
+
+  /**
+   * Reconstructs a tracker from a resumption token produced by
+   * {@link LDAIConfigTracker#getResumptionToken()}.
+   * <p>
+   * The reconstructed tracker shares the original run's {@code runId}, so events it emits (for
+   * example deferred user feedback recorded in another process) correlate with the original AI run.
+   * Model and provider names are not carried in the token and are reported as empty strings.
+   *
+   * @param resumptionToken the token to reconstruct from
+   * @param context the context the tracker's events will be attributed to
+   * @return a tracker sharing the original run's identity
+   * @throws IllegalArgumentException if the token is malformed
+   */
+  LDAIConfigTracker createTracker(String resumptionToken, LDContext context);
 }

lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIClientImpl.java

@@ -8,20 +8,23 @@
 import com.launchdarkly.sdk.LDContext;
 import com.launchdarkly.sdk.LDValue;
 import com.launchdarkly.sdk.LDValueType;
-import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Mode;
 import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Message;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Mode;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Model;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Provider;
 import com.launchdarkly.sdk.server.ai.internal.AIConfigFlagValue;
 import com.launchdarkly.sdk.server.ai.internal.AIConfigParser;
 import com.launchdarkly.sdk.server.ai.internal.AISdkInfo;
 import com.launchdarkly.sdk.server.ai.internal.Interpolator;
-import com.launchdarkly.sdk.server.ai.internal.NoOpAIConfigTracker;
+import com.launchdarkly.sdk.server.ai.internal.LDAIConfigTrackerImpl;
 import com.launchdarkly.sdk.server.interfaces.LDClientInterface;
 
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.UUID;
 import java.util.function.Supplier;
 
 /**
@@ -51,9 +54,6 @@ public final class LDAIClientImpl implements LDAIClient {
       .anonymous(true)
       .build();
 
-  // Tracking is implemented in a later step; until then every config hands out the no-op tracker.
-  private static final Supplier<LDAIConfigTracker> TRACKER_FACTORY = () -> NoOpAIConfigTracker.INSTANCE;
-
   private final LDClientInterface client;
   private final LDLogger logger;
   private final Interpolator interpolator;
@@ -140,6 +140,11 @@ public AIJudgeConfig judgeConfig(
     return (AIJudgeConfig) evaluate(key, context, effectiveDefault, Mode.JUDGE, variables);
   }
 
+  @Override
+  public LDAIConfigTracker createTracker(String resumptionToken, LDContext context) {
+    return LDAIConfigTrackerImpl.fromResumptionToken(resumptionToken, client, context, logger);
+  }
+
   private AIAgentConfig evaluateAgent(
       String key, LDContext context, AIAgentConfigDefault defaultValue, Map<String, Object> variables) {
     AIAgentConfigDefault effectiveDefault =
@@ -186,6 +191,8 @@ private AIConfig buildConfig(
       AIConfigFlagValue parsed,
       LDContext context,
       Map<String, Object> variables) {
+    Supplier<LDAIConfigTracker> trackerFactory = trackerFactory(
+        key, parsed.getVariationKey(), parsed.getVersion(), parsed.getModel(), parsed.getProvider(), context);
     switch (mode) {
       case AGENT:
         return new AIAgentConfig(
@@ -196,7 +203,7 @@ private AIConfig buildConfig(
             interpolate(parsed.getInstructions(), variables, context),
             parsed.getJudgeConfiguration(),
             parsed.getTools(),
-            TRACKER_FACTORY);
+            trackerFactory);
       case JUDGE:
         return new AIJudgeConfig(
             key,
@@ -205,7 +212,7 @@ private AIConfig buildConfig(
             parsed.getProvider(),
             interpolateMessages(parsed.getMessages(), variables, context),
             parsed.getEvaluationMetricKey(),
-            TRACKER_FACTORY);
+            trackerFactory);
       case COMPLETION:
       default:
         return new AICompletionConfig(
@@ -216,7 +223,7 @@ private AIConfig buildConfig(
             interpolateMessages(parsed.getMessages(), variables, context),
             parsed.getJudgeConfiguration(),
             parsed.getTools(),
-            TRACKER_FACTORY);
+            trackerFactory);
     }
   }
 
@@ -241,7 +248,7 @@ private AIConfig buildConfigFromDefault(
             interpolate(agent.getInstructions(), variables, context),
             agent.getJudgeConfiguration(),
             agent.getTools(),
-            TRACKER_FACTORY);
+            trackerFactory(key, null, null, agent.getModel(), agent.getProvider(), context));
       }
       case JUDGE: {
         AIJudgeConfigDefault judge = (AIJudgeConfigDefault) defaultValue;
@@ -252,7 +259,7 @@ private AIConfig buildConfigFromDefault(
             judge.getProvider(),
             interpolateMessages(judge.getMessages(), variables, context),
             judge.getEvaluationMetricKey(),
-            TRACKER_FACTORY);
+            trackerFactory(key, null, null, judge.getModel(), judge.getProvider(), context));
       }
       case COMPLETION:
       default: {
@@ -265,11 +272,25 @@ private AIConfig buildConfigFromDefault(
             interpolateMessages(completion.getMessages(), variables, context),
             completion.getJudgeConfiguration(),
             completion.getTools(),
-            TRACKER_FACTORY);
+            trackerFactory(key, null, null, completion.getModel(), completion.getProvider(), context));
       }
     }
   }
 
+  /**
+   * Builds a factory that produces a fresh tracker, with a new {@code runId}, on each call. The
+   * factory captures the config's correlation data and the evaluation context.
+   */
+  private Supplier<LDAIConfigTracker> trackerFactory(
+      String key, String variationKey, Integer version, Model model, Provider provider, LDContext context) {
+    String varKey = variationKey == null ? "" : variationKey;
+    int ver = version == null ? 0 : version;
+    String modelName = model != null && model.getName() != null ? model.getName() : "";
+    String providerName = provider != null && provider.getName() != null ? provider.getName() : "";
+    return () -> new LDAIConfigTrackerImpl(
+        client, UUID.randomUUID().toString(), key, varKey, ver, modelName, providerName, context, null, logger);
+  }
+
   private List<Message> interpolateMessages(
       List<Message> messages, Map<String, Object> variables, LDContext context) {
     if (messages == null) {

lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIConfigTracker.java

@@ -1,16 +1,167 @@
 package com.launchdarkly.sdk.server.ai;
 
+import com.launchdarkly.sdk.server.ai.datamodel.LDAITrackingTypes.FeedbackKind;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAITrackingTypes.JudgeResult;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAITrackingTypes.Metrics;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAITrackingTypes.MetricSummary;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAITrackingTypes.TokenUsage;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAITrackingTypes.TrackData;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.function.Function;
+
 /**
- * Reports events related to a single AI run of an {@link AIConfig}.
+ * Reports metrics related to a single AI run of an {@link AIConfig}.
  * <p>
- * A tracker is obtained from a retrieved config via {@link AIConfig#createTracker()}. Each tracker
- * corresponds to one AI run and is used to record metrics such as model usage, duration, and
- * feedback against the AI Config it was created from.
+ * A tracker is obtained from a retrieved config via {@link AIConfig#createTracker()}, or
+ * reconstructed across process boundaries via
+ * {@link LDAIClient#createTracker(String, com.launchdarkly.sdk.LDContext)}. Each tracker corresponds
+ * to one AI run; every event it emits shares a {@code runId} (a UUIDv4) so LaunchDarkly can
+ * correlate them in metrics views. Start a new run by calling {@link AIConfig#createTracker()} again.
  * <p>
- * <strong>This interface is an intentional placeholder.</strong> The metric- and feedback-reporting
- * methods (and resumption-token support) are introduced in a later step of the AI SDK build-out; it
- * is defined here so that the public config types expose a stable {@code createTracker()} surface.
- * The only implementation in this release is an internal no-op.
+ * <strong>Thread-safety.</strong> Implementations are safe to share across threads. The
+ * "record-once" metrics ({@link #trackDuration}, {@link #trackTimeToFirstToken},
+ * {@link #trackSuccess}/{@link #trackError}, {@link #trackFeedback}, {@link #trackTokens}) each emit
+ * at most once per tracker even under concurrent calls; later calls are ignored and logged.
+ * {@link #trackToolCall}/{@link #trackToolCalls} and {@link #trackJudgeResult} may be called any
+ * number of times and emit on every call.
  */
 public interface LDAIConfigTracker {
+  /**
+   * Returns the correlation data attached to every event this tracker emits.
+   *
+   * @return the track data, never {@code null}
+   */
+  TrackData getTrackData();
+
+  /**
+   * Returns a URL-safe Base64 token that encodes this tracker's {@code runId}, {@code configKey},
+   * {@code variationKey}, and {@code version}.
+   * <p>
+   * Pass it to {@link LDAIClient#createTracker(String, com.launchdarkly.sdk.LDContext)} to
+   * reconstruct a tracker in another process so deferred events (for example user feedback) still
+   * correlate with the original run.
+   *
+   * @return the resumption token, never {@code null}
+   */
+  String getResumptionToken();
+
+  /**
+   * Records the duration of the generation.
+   * <p>
+   * Records at most once per tracker; later calls are ignored. Negative durations (for example from
+   * clock skew) are clamped to zero.
+   *
+   * @param duration the generation duration; must not be {@code null}
+   */
+  void trackDuration(Duration duration);
+
+  /**
+   * Runs the given operation, recording its duration even if it throws.
+   * <p>
+   * This does not record success or error; use {@link #trackMetricsOf} for that. Because
+   * {@link #trackDuration} records at most once, calling this twice on the same tracker re-runs the
+   * operation but emits no second duration event.
+   *
+   * @param operation the operation to time
+   * @param <T> the operation's result type
+   * @return the operation's result
+   * @throws Exception if the operation throws
+   */
+  <T> T trackDurationOf(Callable<T> operation) throws Exception;
+
+  /**
+   * Records the time to first token for a streaming generation.
+   * <p>
+   * Records at most once per tracker; later calls are ignored. Negative values are clamped to zero.
+   *
+   * @param duration the time to first token; must not be {@code null}
+   */
+  void trackTimeToFirstToken(Duration duration);
+
+  /**
+   * Records that the generation succeeded.
+   * <p>
+   * Success and error share state: only the first of {@link #trackSuccess}/{@link #trackError}
+   * recorded on a tracker takes effect; later calls are ignored.
+   */
+  void trackSuccess();
+
+  /**
+   * Records that the generation failed.
+   * <p>
+   * Success and error share state: only the first of {@link #trackSuccess}/{@link #trackError}
+   * recorded on a tracker takes effect; later calls are ignored.
+   */
+  void trackError();
+
+  /**
+   * Records end-user feedback about the generation.
+   * <p>
+   * Records at most once per tracker; later calls are ignored.
+   *
+   * @param kind the feedback sentiment; must not be {@code null}
+   */
+  void trackFeedback(FeedbackKind kind);
+
+  /**
+   * Records token usage for the generation.
+   * <p>
+   * Records at most once per tracker; later calls are ignored. Negative counts are clamped to zero,
+   * and an individual count is only emitted when it is greater than zero.
+   *
+   * @param tokens the token usage; must not be {@code null}
+   */
+  void trackTokens(TokenUsage tokens);
+
+  /**
+   * Records a single tool invocation. May be called any number of times.
+   *
+   * @param toolKey the identifier of the invoked tool; must not be {@code null}
+   */
+  void trackToolCall(String toolKey);
+
+  /**
+   * Records several tool invocations. May be called any number of times.
+   *
+   * @param toolKeys the identifiers of the invoked tools; must not be {@code null}
+   */
+  void trackToolCalls(List<String> toolKeys);
+
+  /**
+   * Records a judge evaluation result. May be called any number of times.
+   * <p>
+   * No event is emitted when the result was not sampled, did not succeed, or carries no metric key
+   * or score. A {@code null} score is treated as "no score" and is distinct from {@code 0.0}.
+   *
+   * @param result the judge result; must not be {@code null}
+   */
+  void trackJudgeResult(JudgeResult result);
+
+  /**
+   * Runs the given operation, recording its duration and then its outcome and metrics.
+   * <p>
+   * The operation is timed via {@link #trackDurationOf}. If it throws, an error is recorded and the
+   * exception is rethrown. Otherwise the extractor is applied to the result; if the extractor
+   * throws, an error is recorded and the exception is rethrown. On success the extracted metrics
+   * drive {@link #trackSuccess}/{@link #trackError}, {@link #trackTokens}, and
+   * {@link #trackToolCalls}.
+   *
+   * @param metricsExtractor extracts {@link Metrics} from the operation's result
+   * @param operation the AI operation to run
+   * @param <T> the operation's result type
+   * @return the operation's result
+   * @throws Exception if the operation or the extractor throws
+   */
+  <T> T trackMetricsOf(Function<? super T, Metrics> metricsExtractor, Callable<T> operation)
+      throws Exception;
+
+  /**
+   * Returns an immutable snapshot of the metrics recorded on this tracker so far.
+   *
+   * @return the metric summary, never {@code null}
+   */
+  MetricSummary getSummary();
 }