Document LLM Observability Experiments search endpoints

examples/v2/llm-observability/AggregateLLMObsExperimentation.rb

@@ -0,0 +1,41 @@
+# Aggregate LLM Observability experimentation returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.aggregate_llm_obs_experimentation".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsExperimentationAnalyticsRequest.new({
+  data: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsDataAttributesRequest.new({
+      aggregate: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsAggregate.new({
+        compute: [
+          DatadogAPIClient::V2::LLMObsExperimentationAnalyticsCompute.new({
+            metric: "score_value",
+            name: "avg_faithfulness",
+          }),
+        ],
+        dataset_version: nil,
+        group_by: [
+          DatadogAPIClient::V2::LLMObsExperimentationAnalyticsGroupBy.new({
+            field: "span_id",
+          }),
+        ],
+        indexes: [
+          "experiment-evals",
+        ],
+        limit: 1000,
+        search: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsSearch.new({
+          query: "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012",
+        }),
+        time: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsTimeRange.new({
+          from: 1705312200000,
+          to: 1705315800000,
+        }),
+      }),
+    }),
+    type: DatadogAPIClient::V2::LLMObsExperimentationType::EXPERIMENTATION,
+  }),
+})
+p api_instance.aggregate_llm_obs_experimentation(body)

examples/v2/llm-observability/ListLLMObsExperimentEvents.rb

@@ -0,0 +1,8 @@
+# List events for an LLM Observability experiment returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.list_llm_obs_experiment_events".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+p api_instance.list_llm_obs_experiment_events("experiment_id")

examples/v2/llm-observability/SearchLLMObsExperimentation.rb

@@ -0,0 +1,35 @@
+# Search LLM Observability experimentation entities returns "Partial Content — more results are available. Use
+# `meta.after` as the next `page.cursor`." response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.search_llm_obs_experimentation".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsExperimentationSearchRequest.new({
+  data: DatadogAPIClient::V2::LLMObsExperimentationSearchDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsExperimentationSearchDataAttributesRequest.new({
+      content_preview: DatadogAPIClient::V2::LLMObsExperimentationContentPreview.new({
+        limit: 500,
+      }),
+      filter: DatadogAPIClient::V2::LLMObsExperimentationFilter.new({
+        include_deleted: false,
+        is_deleted: false,
+        query: "my experiment",
+        scope: [
+          "experiments",
+        ],
+        version: nil,
+      }),
+      include: DatadogAPIClient::V2::LLMObsExperimentationInclude.new({
+        user_data: false,
+      }),
+      page: DatadogAPIClient::V2::LLMObsExperimentationCursorPage.new({
+        limit: 100,
+      }),
+    }),
+    type: DatadogAPIClient::V2::LLMObsExperimentationType::EXPERIMENTATION,
+  }),
+})
+p api_instance.search_llm_obs_experimentation(body)

examples/v2/llm-observability/SimpleSearchLLMObsExperimentation.rb

@@ -0,0 +1,41 @@
+# Simple search experimentation entities returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.simple_search_llm_obs_experimentation".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsExperimentationSimpleSearchRequest.new({
+  data: DatadogAPIClient::V2::LLMObsExperimentationSimpleSearchDataRequest.new({
+    attributes: DatadogAPIClient::V2::LLMObsExperimentationSimpleSearchDataAttributesRequest.new({
+      content_preview: DatadogAPIClient::V2::LLMObsExperimentationContentPreview.new({
+        limit: 500,
+      }),
+      filter: DatadogAPIClient::V2::LLMObsExperimentationFilter.new({
+        include_deleted: false,
+        is_deleted: false,
+        query: "my experiment",
+        scope: [
+          "experiments",
+        ],
+        version: nil,
+      }),
+      include: DatadogAPIClient::V2::LLMObsExperimentationInclude.new({
+        user_data: false,
+      }),
+      page: DatadogAPIClient::V2::LLMObsExperimentationNumberPage.new({
+        limit: 50,
+        number: 1,
+      }),
+      sort: [
+        DatadogAPIClient::V2::LLMObsExperimentationSortField.new({
+          direction: DatadogAPIClient::V2::LLMObsExperimentationSortFieldDirection::DESC,
+          field: "created_at",
+        }),
+      ],
+    }),
+    type: DatadogAPIClient::V2::LLMObsExperimentationType::EXPERIMENTATION,
+  }),
+})
+p api_instance.simple_search_llm_obs_experimentation(body)

features/scenarios_model_mapping.rb

@@ -951,6 +951,15 @@
             "queue_id" => "String",
             "body" => "LLMObsAnnotationQueueLabelSchemaUpdateRequest",
     },
+    "v2.AggregateLLMObsExperimentation" => {
+            "body" => "LLMObsExperimentationAnalyticsRequest",
+    },
+    "v2.SearchLLMObsExperimentation" => {
+            "body" => "LLMObsExperimentationSearchRequest",
+    },
+    "v2.SimpleSearchLLMObsExperimentation" => {
+            "body" => "LLMObsExperimentationSimpleSearchRequest",
+    },
     "v2.ListLLMObsExperiments" => {
             "filter_project_id" => "String",
             "filter_dataset_id" => "String",
@@ -1030,6 +1039,11 @@
             "dataset_id" => "String",
             "body" => "LLMObsDeleteDatasetRecordsRequest",
     },
+    "v2.ListLLMObsExperimentEvents" => {
+            "experiment_id" => "String",
+            "page_limit" => "Integer",
+            "page_cursor" => "String",
+    },
     "v2.CreateDatastore" => {
             "body" => "CreateAppsDatastoreRequest",
     },

features/v2/llm_observability.feature

@@ -53,6 +53,22 @@ Feature: LLM Observability
     When the request is sent
     Then the response status is 404 Not Found
 
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Aggregate LLM Observability experimentation returns "Bad Request" response
+    Given operation "AggregateLLMObsExperimentation" enabled
+    And new "AggregateLLMObsExperimentation" request
+    And body with value {"data": {"attributes": {"aggregate": {"compute": [{"metric": "score_value", "name": "avg_faithfulness"}], "dataset_version": null, "group_by": [{"field": "span_id"}], "indexes": ["experiment-evals"], "limit": 1000, "search": {"query": "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012"}, "time": {"from": 1705312200000, "to": 1705315800000}}}, "type": "experimentation"}}
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Aggregate LLM Observability experimentation returns "OK" response
+    Given operation "AggregateLLMObsExperimentation" enabled
+    And new "AggregateLLMObsExperimentation" request
+    And body with value {"data": {"attributes": {"aggregate": {"compute": [{"metric": "score_value", "name": "avg_faithfulness"}], "dataset_version": null, "group_by": [{"field": "span_id"}], "indexes": ["experiment-evals"], "limit": 1000, "search": {"query": "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012"}, "time": {"from": 1705312200000, "to": 1705315800000}}}, "type": "experimentation"}}
+    When the request is sent
+    Then the response status is 200 OK
+
   @generated @skip @team:DataDog/ml-observability
   Scenario: Append records to an LLM Observability dataset returns "Bad Request" response
     Given operation "CreateLLMObsDatasetRecords" enabled
@@ -542,6 +558,30 @@ Feature: LLM Observability
     When the request is sent
     Then the response status is 200 OK
 
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: List events for an LLM Observability experiment returns "Bad Request" response
+    Given operation "ListLLMObsExperimentEvents" enabled
+    And new "ListLLMObsExperimentEvents" request
+    And request contains "experiment_id" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: List events for an LLM Observability experiment returns "Not Found" response
+    Given operation "ListLLMObsExperimentEvents" enabled
+    And new "ListLLMObsExperimentEvents" request
+    And request contains "experiment_id" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 404 Not Found
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: List events for an LLM Observability experiment returns "OK" response
+    Given operation "ListLLMObsExperimentEvents" enabled
+    And new "ListLLMObsExperimentEvents" request
+    And request contains "experiment_id" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 200 OK
+
   @generated @skip @team:DataDog/ml-observability
   Scenario: Push events for an LLM Observability experiment returns "Accepted" response
     Given operation "CreateLLMObsExperimentEvents" enabled
@@ -569,6 +609,46 @@ Feature: LLM Observability
     When the request is sent
     Then the response status is 404 Not Found
 
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Search LLM Observability experimentation entities returns "Bad Request" response
+    Given operation "SearchLLMObsExperimentation" enabled
+    And new "SearchLLMObsExperimentation" request
+    And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 100}}, "type": "experimentation"}}
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Search LLM Observability experimentation entities returns "OK — all results returned in a single page." response
+    Given operation "SearchLLMObsExperimentation" enabled
+    And new "SearchLLMObsExperimentation" request
+    And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 100}}, "type": "experimentation"}}
+    When the request is sent
+    Then the response status is 200 OK — all results returned in a single page.
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Search LLM Observability experimentation entities returns "Partial Content — more results are available. Use `meta.after` as the next `page.cursor`." response
+    Given operation "SearchLLMObsExperimentation" enabled
+    And new "SearchLLMObsExperimentation" request
+    And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 100}}, "type": "experimentation"}}
+    When the request is sent
+    Then the response status is 206 Partial Content — more results are available. Use `meta.after` as the next `page.cursor`.
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Simple search experimentation entities returns "Bad Request" response
+    Given operation "SimpleSearchLLMObsExperimentation" enabled
+    And new "SimpleSearchLLMObsExperimentation" request
+    And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 50, "number": 1}, "sort": [{"direction": "desc", "field": "created_at"}]}, "type": "experimentation"}}
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Simple search experimentation entities returns "OK" response
+    Given operation "SimpleSearchLLMObsExperimentation" enabled
+    And new "SimpleSearchLLMObsExperimentation" request
+    And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 50, "number": 1}, "sort": [{"direction": "desc", "field": "created_at"}]}, "type": "experimentation"}}
+    When the request is sent
+    Then the response status is 200 OK
+
   @generated @skip @team:DataDog/ml-observability
   Scenario: Update LLM Observability dataset records returns "Bad Request" response
     Given operation "UpdateLLMObsDatasetRecords" enabled

features/v2/undo.json

@@ -3485,6 +3485,24 @@
       "type": "idempotent"
     }
   },
+  "AggregateLLMObsExperimentation": {
+    "tag": "LLM Observability",
+    "undo": {
+      "type": "safe"
+    }
+  },
+  "SearchLLMObsExperimentation": {
+    "tag": "LLM Observability",
+    "undo": {
+      "type": "safe"
+    }
+  },
+  "SimpleSearchLLMObsExperimentation": {
+    "tag": "LLM Observability",
+    "undo": {
+      "type": "safe"
+    }
+  },
   "ListLLMObsExperiments": {
     "tag": "LLM Observability",
     "undo": {
@@ -3494,15 +3512,19 @@
   "CreateLLMObsExperiment": {
     "tag": "LLM Observability",
     "undo": {
-      "operationId": "TODO",
-      "parameters": [],
+      "operationId": "DeleteLLMObsExperiments",
+      "parameters": [
+        {
+          "name": "body",
+          "template": "{\"data\": {\"attributes\": {\"experiment_ids\": [\"{{ data.id }}\"]}, \"type\": \"experiments\"}}"
+        }
+      ],
       "type": "unsafe"
     }
   },
   "DeleteLLMObsExperiments": {
     "tag": "LLM Observability",
     "undo": {
-      "operationId": "TODO",
       "parameters": [],
       "type": "unsafe"
     }
@@ -3516,7 +3538,6 @@
   "CreateLLMObsExperimentEvents": {
     "tag": "LLM Observability",
     "undo": {
-      "operationId": "TODO",
       "parameters": [],
       "type": "unsafe"
     }
@@ -3614,6 +3635,12 @@
       "type": "unsafe"
     }
   },
+  "ListLLMObsExperimentEvents": {
+    "tag": "LLM Observability",
+    "undo": {
+      "type": "safe"
+    }
+  },
   "SubmitLog": {
     "tag": "Logs",
     "undo": {

lib/datadog_api_client/configuration.rb

@@ -207,6 +207,7 @@ def initialize
             "v2.list_fleet_tracers": false,
             "v2.trigger_fleet_schedule": false,
             "v2.update_fleet_schedule": false,
+            "v2.aggregate_llm_obs_experimentation": false,
             "v2.create_llm_obs_annotation_queue": false,
             "v2.create_llm_obs_annotation_queue_interactions": false,
             "v2.create_llm_obs_dataset": false,
@@ -227,8 +228,11 @@ def initialize
             "v2.list_llm_obs_annotation_queues": false,
             "v2.list_llm_obs_dataset_records": false,
             "v2.list_llm_obs_datasets": false,
+            "v2.list_llm_obs_experiment_events": false,
             "v2.list_llm_obs_experiments": false,
             "v2.list_llm_obs_projects": false,
+            "v2.search_llm_obs_experimentation": false,
+            "v2.simple_search_llm_obs_experimentation": false,
             "v2.update_llm_obs_annotation_queue": false,
             "v2.update_llm_obs_annotation_queue_label_schema": false,
             "v2.update_llm_obs_custom_eval_config": false,