diff --git a/tests/unit/vertexai/genai/replays/test_evaluation_metric.py b/tests/unit/vertexai/genai/replays/test_evaluation_metric.py index 1d364a5a31..24ac8e181c 100644 --- a/tests/unit/vertexai/genai/replays/test_evaluation_metric.py +++ b/tests/unit/vertexai/genai/replays/test_evaluation_metric.py @@ -17,12 +17,15 @@ from tests.unit.vertexai.genai.replays import pytest_helper from vertexai._genai import types +from google.genai import errors +import pytest + _TEST_PROJECT = "977012026409" _TEST_LOCATION = "us-central1" -def test_create_and_get_evaluation_metric(client): +def test_create_get_delete_evaluation_metric(client): client._api_client._http_options.api_version = "v1beta1" result = client.evals.create_evaluation_metric( display_name="test_metric", @@ -40,6 +43,12 @@ def test_create_and_get_evaluation_metric(client): assert isinstance(metric, types.EvaluationMetric) assert metric.display_name == "test_metric" + client.evals.delete_evaluation_metric(metric_resource_name=result) + + # Verify that the metric no longer exists + with pytest.raises(errors.ClientError, match="404"): + client.evals.get_evaluation_metric(metric_resource_name=result) + def test_list_evaluation_metrics(client): client._api_client._http_options.api_version = "v1beta1" diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 6e0f1c7582..53833b6a67 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -194,6 +194,24 @@ def _CustomCodeExecutionSpec_to_vertex( return to_object +def _DeleteEvaluationMetricParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["metric_resource_name"]) is not None: + setv( + to_object, + ["_url", "evaluation_metric"], + getv(from_object, ["metric_resource_name"]), + ) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _EvaluateInstancesRequestParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1242,6 +1260,78 @@ def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + def _delete_evaluation_metric( + self, + *, + metric_resource_name: str, + config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, + ) -> types.DeleteEvaluationMetricOperation: + """ + Deletes an EvaluationMetric. + """ + + parameter_model = types._DeleteEvaluationMetricParameters( + metric_resource_name=metric_resource_name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in the Gemini Enterprise Agent Platform (previously known as Vertex AI) client." + ) + else: + request_dict = _DeleteEvaluationMetricParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{evaluation_metric}".format_map(request_url_dict) + else: + path = "{evaluation_metric}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("delete", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationMetricOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _evaluate_instances( self, *, @@ -2907,6 +2997,29 @@ def list_evaluation_metrics( config=config, ) + @_common.experimental_warning( + "The Vertex SDK GenAI evals.delete_evaluation_metric method is experimental, " + "and may change in future versions." + ) + def delete_evaluation_metric( + self, + *, + metric_resource_name: str, + config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, + ) -> None: + """Deletes an EvaluationMetric. + + Args: + metric_resource_name: The resource name of the EvaluationMetric to delete. + Format: + `projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric}` + config: The optional configuration for the delete operation. + """ + self._delete_evaluation_metric( + metric_resource_name=metric_resource_name, + config=config, + ) + class AsyncEvals(_api_module.BaseModule): @@ -3234,6 +3347,80 @@ async def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + async def _delete_evaluation_metric( + self, + *, + metric_resource_name: str, + config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, + ) -> types.DeleteEvaluationMetricOperation: + """ + Deletes an EvaluationMetric. + """ + + parameter_model = types._DeleteEvaluationMetricParameters( + metric_resource_name=metric_resource_name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in the Gemini Enterprise Agent Platform (previously known as Vertex AI) client." + ) + else: + request_dict = _DeleteEvaluationMetricParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{evaluation_metric}".format_map(request_url_dict) + else: + path = "{evaluation_metric}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "delete", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationMetricOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _evaluate_instances( self, *, @@ -4548,3 +4735,26 @@ async def list_evaluation_metrics( return await self._list_evaluation_metrics( config=config, ) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.delete_evaluation_metric method is experimental, " + "and may change in future versions." + ) + async def delete_evaluation_metric( + self, + *, + metric_resource_name: str, + config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, + ) -> None: + """Deletes an EvaluationMetric. + + Args: + metric_resource_name: The resource name of the EvaluationMetric to delete. + Format: + `projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric}` + config: The optional configuration for the delete operation. + """ + await self._delete_evaluation_metric( + metric_resource_name=metric_resource_name, + config=config, + ) diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index 115aa43cd5..d278f1cce3 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -48,6 +48,7 @@ from .common import _DeleteAgentEngineSessionRequestParameters from .common import _DeleteAgentEngineTaskRequestParameters from .common import _DeleteDatasetRequestParameters +from .common import _DeleteEvaluationMetricParameters from .common import _DeleteMultimodalDatasetRequestParameters from .common import _DeletePromptVersionRequestParameters from .common import _EvaluateInstancesRequestParameters @@ -315,6 +316,12 @@ from .common import DeleteAgentEngineTaskConfig from .common import DeleteAgentEngineTaskConfigDict from .common import DeleteAgentEngineTaskConfigOrDict +from .common import DeleteEvaluationMetricConfig +from .common import DeleteEvaluationMetricConfigDict +from .common import DeleteEvaluationMetricConfigOrDict +from .common import DeleteEvaluationMetricOperation +from .common import DeleteEvaluationMetricOperationDict +from .common import DeleteEvaluationMetricOperationOrDict from .common import DeletePromptConfig from .common import DeletePromptConfigDict from .common import DeletePromptConfigOrDict @@ -1457,6 +1464,12 @@ "EvaluationSet", "EvaluationSetDict", "EvaluationSetOrDict", + "DeleteEvaluationMetricConfig", + "DeleteEvaluationMetricConfigDict", + "DeleteEvaluationMetricConfigOrDict", + "DeleteEvaluationMetricOperation", + "DeleteEvaluationMetricOperationDict", + "DeleteEvaluationMetricOperationOrDict", "BleuInstance", "BleuInstanceDict", "BleuInstanceOrDict", @@ -2399,6 +2412,7 @@ "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", "_CreateEvaluationSetParameters", + "_DeleteEvaluationMetricParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", "_GenerateLossClustersParameters", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 07d82a747f..f3cae0631e 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -3541,6 +3541,92 @@ class EvaluationSetDict(TypedDict, total=False): EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict] +class DeleteEvaluationMetricConfig(_common.BaseModel): + """Config for deleting an evaluation metric.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationMetricConfigDict(TypedDict, total=False): + """Config for deleting an evaluation metric.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +DeleteEvaluationMetricConfigOrDict = Union[ + DeleteEvaluationMetricConfig, DeleteEvaluationMetricConfigDict +] + + +class _DeleteEvaluationMetricParameters(_common.BaseModel): + """Parameters for deleting an evaluation metric.""" + + metric_resource_name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationMetricConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationMetricParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation metric.""" + + metric_resource_name: Optional[str] + """""" + + config: Optional[DeleteEvaluationMetricConfigDict] + """""" + + +_DeleteEvaluationMetricParametersOrDict = Union[ + _DeleteEvaluationMetricParameters, _DeleteEvaluationMetricParametersDict +] + + +class DeleteEvaluationMetricOperation(_common.BaseModel): + """Operation for deleting an evaluation metric.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationMetricOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation metric.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationMetricOperationOrDict = Union[ + DeleteEvaluationMetricOperation, DeleteEvaluationMetricOperationDict +] + + class BleuInstance(_common.BaseModel): """Bleu instance."""