Skip to content

feat: Add delete_evaluation_metric to GenAI SDK #6629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Apr 23, 2026
Merged

feat: Add delete_evaluation_metric to GenAI SDK #6629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion tests/unit/vertexai/genai/replays/test_evaluation_metric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,15 @@

from tests.unit.vertexai.genai.replays import pytest_helper
from vertexai._genai import types
from google.genai import errors
import pytest


_TEST_PROJECT = "977012026409"
_TEST_LOCATION = "us-central1"


def test_create_and_get_evaluation_metric(client):
def test_create_get_delete_evaluation_metric(client):
client._api_client._http_options.api_version = "v1beta1"
result = client.evals.create_evaluation_metric(
display_name="test_metric",
Expand All @@ -40,6 +43,12 @@ def test_create_and_get_evaluation_metric(client):
assert isinstance(metric, types.EvaluationMetric)
assert metric.display_name == "test_metric"

client.evals.delete_evaluation_metric(metric_resource_name=result)

# Verify that the metric no longer exists
with pytest.raises(errors.ClientError, match="404"):
client.evals.get_evaluation_metric(metric_resource_name=result)


def test_list_evaluation_metrics(client):
client._api_client._http_options.api_version = "v1beta1"
Expand Down
210 changes: 210 additions & 0 deletions vertexai/_genai/evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,24 @@ def _CustomCodeExecutionSpec_to_vertex(
return to_object


def _DeleteEvaluationMetricParameters_to_vertex(
from_object: Union[dict[str, Any], object],
parent_object: Optional[dict[str, Any]] = None,
) -> dict[str, Any]:
to_object: dict[str, Any] = {}
if getv(from_object, ["metric_resource_name"]) is not None:
setv(
to_object,
["_url", "evaluation_metric"],
getv(from_object, ["metric_resource_name"]),
)

if getv(from_object, ["config"]) is not None:
setv(to_object, ["config"], getv(from_object, ["config"]))

return to_object


def _EvaluateInstancesRequestParameters_to_vertex(
from_object: Union[dict[str, Any], object],
parent_object: Optional[dict[str, Any]] = None,
Expand Down Expand Up @@ -1242,6 +1260,78 @@ def _create_evaluation_set(
self._api_client._verify_response(return_value)
return return_value

def _delete_evaluation_metric(
self,
*,
metric_resource_name: str,
config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None,
) -> types.DeleteEvaluationMetricOperation:
"""
Deletes an EvaluationMetric.
"""

parameter_model = types._DeleteEvaluationMetricParameters(
metric_resource_name=metric_resource_name,
config=config,
)

request_url_dict: Optional[dict[str, str]]
if not self._api_client.vertexai:
raise ValueError(
"This method is only supported in the Gemini Enterprise Agent Platform (previously known as Vertex AI) client."
)
else:
request_dict = _DeleteEvaluationMetricParameters_to_vertex(parameter_model)
request_url_dict = request_dict.get("_url")
if request_url_dict:
path = "{evaluation_metric}".format_map(request_url_dict)
else:
path = "{evaluation_metric}"

query_params = request_dict.get("_query")
if query_params:
path = f"{path}?{urlencode(query_params)}"
# TODO: remove the hack that pops config.
request_dict.pop("config", None)

http_options: Optional[types.HttpOptions] = None
if (
parameter_model.config is not None
and parameter_model.config.http_options is not None
):
http_options = parameter_model.config.http_options

request_dict = _common.convert_to_dict(request_dict)
request_dict = _common.encode_unserializable_types(request_dict)

response = self._api_client.request("delete", path, request_dict, http_options)

response_dict = {} if not response.body else json.loads(response.body)

return_value = types.DeleteEvaluationMetricOperation._from_response(
response=response_dict,
kwargs=(
{
"config": {
"response_schema": getattr(
parameter_model.config, "response_schema", None
),
"response_json_schema": getattr(
parameter_model.config, "response_json_schema", None
),
"include_all_fields": getattr(
parameter_model.config, "include_all_fields", None
),
}
}
if getattr(parameter_model, "config", None)
else {}
),
)

self._api_client._verify_response(return_value)
return return_value

def _evaluate_instances(
self,
*,
Expand Down Expand Up @@ -2907,6 +2997,29 @@ def list_evaluation_metrics(
config=config,
)

@_common.experimental_warning(
"The Vertex SDK GenAI evals.delete_evaluation_metric method is experimental, "
"and may change in future versions."
)
def delete_evaluation_metric(
self,
*,
metric_resource_name: str,
config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None,
) -> None:
"""Deletes an EvaluationMetric.

Args:
metric_resource_name: The resource name of the EvaluationMetric to delete.
Format:
`projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric}`
config: The optional configuration for the delete operation.
"""
self._delete_evaluation_metric(
metric_resource_name=metric_resource_name,
config=config,
)


class AsyncEvals(_api_module.BaseModule):

Expand Down Expand Up @@ -3234,6 +3347,80 @@ async def _create_evaluation_set(
self._api_client._verify_response(return_value)
return return_value

async def _delete_evaluation_metric(
self,
*,
metric_resource_name: str,
config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None,
) -> types.DeleteEvaluationMetricOperation:
"""
Deletes an EvaluationMetric.
"""

parameter_model = types._DeleteEvaluationMetricParameters(
metric_resource_name=metric_resource_name,
config=config,
)

request_url_dict: Optional[dict[str, str]]
if not self._api_client.vertexai:
raise ValueError(
"This method is only supported in the Gemini Enterprise Agent Platform (previously known as Vertex AI) client."
)
else:
request_dict = _DeleteEvaluationMetricParameters_to_vertex(parameter_model)
request_url_dict = request_dict.get("_url")
if request_url_dict:
path = "{evaluation_metric}".format_map(request_url_dict)
else:
path = "{evaluation_metric}"

query_params = request_dict.get("_query")
if query_params:
path = f"{path}?{urlencode(query_params)}"
# TODO: remove the hack that pops config.
request_dict.pop("config", None)

http_options: Optional[types.HttpOptions] = None
if (
parameter_model.config is not None
and parameter_model.config.http_options is not None
):
http_options = parameter_model.config.http_options

request_dict = _common.convert_to_dict(request_dict)
request_dict = _common.encode_unserializable_types(request_dict)

response = await self._api_client.async_request(
"delete", path, request_dict, http_options
)

response_dict = {} if not response.body else json.loads(response.body)

return_value = types.DeleteEvaluationMetricOperation._from_response(
response=response_dict,
kwargs=(
{
"config": {
"response_schema": getattr(
parameter_model.config, "response_schema", None
),
"response_json_schema": getattr(
parameter_model.config, "response_json_schema", None
),
"include_all_fields": getattr(
parameter_model.config, "include_all_fields", None
),
}
}
if getattr(parameter_model, "config", None)
else {}
),
)

self._api_client._verify_response(return_value)
return return_value

async def _evaluate_instances(
self,
*,
Expand Down Expand Up @@ -4548,3 +4735,26 @@ async def list_evaluation_metrics(
return await self._list_evaluation_metrics(
config=config,
)

@_common.experimental_warning(
"The Vertex SDK GenAI evals.delete_evaluation_metric method is experimental, "
"and may change in future versions."
)
async def delete_evaluation_metric(
self,
*,
metric_resource_name: str,
config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None,
) -> None:
"""Deletes an EvaluationMetric.

Args:
metric_resource_name: The resource name of the EvaluationMetric to delete.
Format:
`projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric}`
config: The optional configuration for the delete operation.
"""
await self._delete_evaluation_metric(
metric_resource_name=metric_resource_name,
config=config,
)
14 changes: 14 additions & 0 deletions vertexai/_genai/types/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
from .common import _DeleteAgentEngineSessionRequestParameters
from .common import _DeleteAgentEngineTaskRequestParameters
from .common import _DeleteDatasetRequestParameters
from .common import _DeleteEvaluationMetricParameters
from .common import _DeleteMultimodalDatasetRequestParameters
from .common import _DeletePromptVersionRequestParameters
from .common import _EvaluateInstancesRequestParameters
Expand Down Expand Up @@ -315,6 +316,12 @@
from .common import DeleteAgentEngineTaskConfig
from .common import DeleteAgentEngineTaskConfigDict
from .common import DeleteAgentEngineTaskConfigOrDict
from .common import DeleteEvaluationMetricConfig
from .common import DeleteEvaluationMetricConfigDict
from .common import DeleteEvaluationMetricConfigOrDict
from .common import DeleteEvaluationMetricOperation
from .common import DeleteEvaluationMetricOperationDict
from .common import DeleteEvaluationMetricOperationOrDict
from .common import DeletePromptConfig
from .common import DeletePromptConfigDict
from .common import DeletePromptConfigOrDict
Expand Down Expand Up @@ -1457,6 +1464,12 @@
"EvaluationSet",
"EvaluationSetDict",
"EvaluationSetOrDict",
"DeleteEvaluationMetricConfig",
"DeleteEvaluationMetricConfigDict",
"DeleteEvaluationMetricConfigOrDict",
"DeleteEvaluationMetricOperation",
"DeleteEvaluationMetricOperationDict",
"DeleteEvaluationMetricOperationOrDict",
"BleuInstance",
"BleuInstanceDict",
"BleuInstanceOrDict",
Expand Down Expand Up @@ -2399,6 +2412,7 @@
"_CreateEvaluationMetricParameters",
"_CreateEvaluationRunParameters",
"_CreateEvaluationSetParameters",
"_DeleteEvaluationMetricParameters",
"_EvaluateInstancesRequestParameters",
"_GenerateUserScenariosParameters",
"_GenerateLossClustersParameters",
Expand Down
Loading
Loading