From 24dff9c26995bfd6c7a5cb66815629ba1bb830e6 Mon Sep 17 00:00:00 2001 From: Alan Date: Thu, 16 Apr 2026 14:16:01 -0500 Subject: [PATCH 1/3] fix: update litellm to >=1.83.0 to resolve security vulnerability --- setup.py | 4 ++-- tests/unit/vertexai/genai/test_evals.py | 10 ++++++---- vertexai/_genai/_evals_common.py | 9 ++++++++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 549092e8df..06d58a21d5 100644 --- a/setup.py +++ b/setup.py @@ -181,8 +181,8 @@ "jsonschema", "ruamel.yaml", "pyyaml", - "litellm>=1.75.5, <=1.82.6", - # For LiteLLM tests. Upper bound pinned: versions 1.82.7+ compromised in supply chain attack. + "litellm>=1.83.0, <2", + # For LiteLLM tests. Versions >=1.82.7,<1.83.0 compromised in supply chain attack. ] langchain_extra_require = [ diff --git a/tests/unit/vertexai/genai/test_evals.py b/tests/unit/vertexai/genai/test_evals.py index 4135c08532..06886a5729 100644 --- a/tests/unit/vertexai/genai/test_evals.py +++ b/tests/unit/vertexai/genai/test_evals.py @@ -3454,7 +3454,7 @@ def test_run_inference_with_litellm_string_prompt_format( ) as mock_litellm, mock.patch( "vertexai._genai._evals_common._call_litellm_completion" ) as mock_call_litellm_completion: - mock_litellm.utils.get_valid_models.return_value = ["gpt-4o"] + mock_litellm.get_llm_provider.return_value = ("gpt-4o", "openai", None , None) prompt_df = pd.DataFrame([{"prompt": "What is LiteLLM?"}]) expected_messages = [{"role": "user", "content": "What is LiteLLM?"}] @@ -3510,7 +3510,7 @@ def test_run_inference_with_litellm_openai_request_format( ) as mock_litellm, mock.patch( "vertexai._genai._evals_common._call_litellm_completion" ) as mock_call_litellm_completion: - mock_litellm.utils.get_valid_models.return_value = ["gpt-4o"] + mock_litellm.get_llm_provider.return_value = ("gpt-4o", "openai", None , None) prompt_df = pd.DataFrame( [ { @@ -3579,7 +3579,9 @@ def test_run_inference_with_unsupported_model_string( with mock.patch( "vertexai._genai._evals_common.litellm" ) as mock_litellm_package: - mock_litellm_package.utils.get_valid_models.return_value = [] + mock_litellm_package.get_llm_provider.side_effect = ValueError( + "unsupported model" + ) evals_module = evals.Evals(api_client_=mock_api_client_fixture) prompt_df = pd.DataFrame([{"prompt": "test"}]) @@ -3646,7 +3648,7 @@ def test_run_inference_with_litellm_parsing( # fmt: off with mock.patch("vertexai._genai._evals_common.litellm") as mock_litellm: # fmt: on - mock_litellm.utils.get_valid_models.return_value = ["gpt-4o"] + mock_litellm.get_llm_provider.return_value = ("gpt-4o", "openai", None , None) inference_result = self.client.evals.run_inference( model="gpt-4o", src=mock_df, diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index 201135b731..7e640cf2ce 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -761,7 +761,14 @@ def _is_litellm_vertex_maas_model(model: str) -> bool: def _is_litellm_model(model: str) -> bool: """Checks if the model name corresponds to a valid LiteLLM model name.""" - return model in litellm.utils.get_valid_models(model) + if litellm is None: + return False + + try: + litellm.get_llm_provider(model) + return True + except ValueError: + return False def _is_gemini_model(model: str) -> bool: From 8cda8916122eb3ea03520ce4f4784899618d584e Mon Sep 17 00:00:00 2001 From: Alan Date: Mon, 20 Apr 2026 18:26:13 -0500 Subject: [PATCH 2/3] fix linting errors --- tests/unit/vertexai/genai/test_evals.py | 18 ++++++++++++------ vertexai/_genai/_evals_common.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/unit/vertexai/genai/test_evals.py b/tests/unit/vertexai/genai/test_evals.py index a46d8c0b72..c384cfd40a 100644 --- a/tests/unit/vertexai/genai/test_evals.py +++ b/tests/unit/vertexai/genai/test_evals.py @@ -3675,12 +3675,18 @@ def test_run_inference_with_litellm_openai_request_format( mock_api_client_fixture, ): """Tests inference with LiteLLM where the row contains a chat completion request body.""" - with mock.patch( - "vertexai._genai._evals_common.litellm" - ) as mock_litellm, mock.patch( - "vertexai._genai._evals_common._call_litellm_completion" - ) as mock_call_litellm_completion: - mock_litellm.get_llm_provider.return_value = ("gpt-4o", "openai", None , None) + with ( + mock.patch("vertexai._genai._evals_common.litellm") as mock_litellm, + mock.patch( + "vertexai._genai._evals_common._call_litellm_completion" + ) as mock_call_litellm_completion, + ): + mock_litellm.get_llm_provider.return_value = ( + "gpt-4o", + "openai", + None, + None, + ) prompt_df = pd.DataFrame( [ { diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index ead09a798a..b51adc1783 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -740,7 +740,7 @@ def _is_litellm_model(model: str) -> bool: try: litellm.get_llm_provider(model) - return True + return True except ValueError: return False From fb1efd0ed79a18a382093a920b82c8f3647a24b4 Mon Sep 17 00:00:00 2001 From: Alan Date: Tue, 21 Apr 2026 10:43:00 -0500 Subject: [PATCH 3/3] Removed version pinning from PR --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 06d58a21d5..549092e8df 100644 --- a/setup.py +++ b/setup.py @@ -181,8 +181,8 @@ "jsonschema", "ruamel.yaml", "pyyaml", - "litellm>=1.83.0, <2", - # For LiteLLM tests. Versions >=1.82.7,<1.83.0 compromised in supply chain attack. + "litellm>=1.75.5, <=1.82.6", + # For LiteLLM tests. Upper bound pinned: versions 1.82.7+ compromised in supply chain attack. ] langchain_extra_require = [