From b9375f14fd197d426e5d11df55d4b3481a0a86a3 Mon Sep 17 00:00:00 2001 From: Shruti Iyer Date: Fri, 10 Jul 2026 10:44:52 -0400 Subject: [PATCH] Prototype: BYO admin-connected judge models via project Responses API shim AzureOpenAIGrader routes admin-connected judge models ('connection/deployment') through the Foundry project Responses API using a chat.completions-compatible shim (_byo_judge.py), so the platform resolves the connection. Judges cannot use chat.completions for BYO (404 DeploymentNotFound). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3 --- .../azure/ai/evaluation/_aoai/aoai_grader.py | 30 ++++ .../azure/ai/evaluation/_byo_judge.py | 157 ++++++++++++++++++ .../tests/unittests/test_byo_judge.py | 111 +++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py index 07e771303326..79f70fbbf634 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py @@ -61,6 +61,27 @@ def __init__( def _validate_model_config(self) -> None: """Validate the model configuration that this grader wrapper is using.""" + # Prototype: admin-connected (BYO) judge models are served via the project Responses + # API and require a credential + project endpoint (no api_key / azure_endpoint). + from azure.ai.evaluation._byo_judge import is_byo_model_config + + if is_byo_model_config(self._model_config): + if not self._credential: + raise EvaluationException( + message=f"{type(self).__name__}: BYO (admin-connected) judge models require a credential.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.AOAI_GRADER, + ) + if not self._model_config.get("project_endpoint"): + raise EvaluationException( + message=f"{type(self).__name__}: BYO judge model_config requires 'project_endpoint'.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.AOAI_GRADER, + ) + return + msg = None if self._is_azure_model_config(self._model_config): if not any(auth for auth in (self._model_config.get("api_key"), self._credential)): @@ -104,6 +125,15 @@ def get_client(self) -> Any: """ default_headers = {"User-Agent": UserAgentSingleton().value} model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration] = self._model_config + + # Prototype: admin-connected (BYO) judge model referenced as "connection/deployment". + # Route judge calls through the Foundry project Responses API via a + # chat.completions-compatible shim, so the platform resolves the connection. + from azure.ai.evaluation._byo_judge import is_byo_model_config, build_byo_judge_client + + if is_byo_model_config(model_config): + return build_byo_judge_client(model_config, self._credential) + api_key: Optional[str] = model_config.get("api_key") if self._is_azure_model_config(model_config): diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py new file mode 100644 index 000000000000..99e22078988f --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py @@ -0,0 +1,157 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +"""Prototype: use admin-connected (BYO) models as judge / grader models. + +Admin-connected models (Foundry "ModelGateway" / "API Management" connections, referenced +as ``"connection-name/deployment-name"``) are only invokable through the Foundry project +**Responses API** — the platform resolves the connection and handles every auth type +(API key / managed identity / OAuth), ``deploymentInPath``, api-version and custom headers. + +LLM-as-judge evaluators in this library call ``client.chat.completions.create(...)``. This +module provides a small OpenAI-compatible **shim** that routes those calls to the project +Responses API for a BYO model, so judge/grader code can use admin-connected connections +**without any change to its calling code**. +""" +import time +from typing import Any, Dict, List, Optional + +from typing_extensions import TypedDict + +from azure.core.credentials import TokenCredential + + +class BYOProjectModelConfiguration(TypedDict, total=False): + """Model configuration for an admin-connected (BYO) judge model. + + :keyword byo_model: The admin-connected model reference, ``"connection-name/deployment-name"``. + :keyword project_endpoint: The Foundry project endpoint, + e.g. ``https://.services.ai.azure.com/api/projects/``. + """ + + byo_model: str + project_endpoint: str + + +def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + """Map chat-completions messages ({role, content}) to Responses API input items.""" + items: List[Dict[str, Any]] = [] + for message in messages or []: + items.append( + { + "type": "message", + "role": message.get("role", "user"), + "content": message.get("content", ""), + } + ) + return items + + +def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Map a curated set of chat-completions sampling params to Responses API params.""" + mapped: Dict[str, Any] = {} + if "temperature" in kwargs: + mapped["temperature"] = kwargs["temperature"] + if "top_p" in kwargs: + mapped["top_p"] = kwargs["top_p"] + for key in ("max_output_tokens", "max_completion_tokens", "max_tokens"): + if key in kwargs: + mapped["max_output_tokens"] = kwargs[key] + break + return mapped + + +class _ChatMessage: + def __init__(self, content: str) -> None: + self.role = "assistant" + self.content = content + self.tool_calls = None + + +class _Choice: + def __init__(self, content: str) -> None: + self.index = 0 + self.message = _ChatMessage(content) + self.finish_reason = "stop" + + +class _ChatCompletion: + """Minimal chat.completions-shaped view over a Responses API result.""" + + def __init__(self, response: Any) -> None: + self.id = getattr(response, "id", None) + self.model = getattr(response, "model", None) + self.usage = getattr(response, "usage", None) + self.object = "chat.completion" + self.created = int(time.time()) + self.choices = [_Choice(getattr(response, "output_text", "") or "")] + + +class _ChatCompletions: + def __init__(self, owner: "ByoProjectResponsesClient") -> None: + self._owner = owner + + def create(self, *, model: Optional[str] = None, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> _ChatCompletion: + # ``model`` from the caller is ignored — the BYO model is fixed by the shim's config. + response = self._owner._responses_create(messages=messages, **kwargs) + return _ChatCompletion(response) + + +class _Chat: + def __init__(self, owner: "ByoProjectResponsesClient") -> None: + self.completions = _ChatCompletions(owner) + + +class ByoProjectResponsesClient: + """OpenAI-compatible shim that routes ``chat.completions.create()`` to the Foundry + project Responses API for an admin-connected (BYO) model. + + Judge/grader code that calls ``client.chat.completions.create(model=..., messages=...)`` + works unchanged; the request is served by the platform, which resolves the connection. + A ``responses`` passthrough is also exposed for callers that use the Responses API directly. + """ + + def __init__(self, byo_model: str, project_endpoint: str, credential: TokenCredential) -> None: + self._byo_model = byo_model + self._project_endpoint = project_endpoint + self._credential = credential + self._client: Any = None + self.chat = _Chat(self) + + def _openai(self) -> Any: + if self._client is None: + from azure.ai.projects import AIProjectClient + + self._client = AIProjectClient( + endpoint=self._project_endpoint, credential=self._credential + ).get_openai_client() + return self._client + + def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any: + return self._openai().responses.create( + model=self._byo_model, + input=_to_responses_input(messages), + **_map_params(kwargs), + ) + + @property + def responses(self) -> Any: + return self._openai().responses + + +def is_byo_model_config(model_config: Dict[str, Any]) -> bool: + """Return True if the model configuration references an admin-connected (BYO) model.""" + return bool(model_config) and bool(model_config.get("byo_model")) + + +def build_byo_judge_client(model_config: Dict[str, Any], credential: TokenCredential) -> ByoProjectResponsesClient: + """Build a chat.completions-compatible client for an admin-connected (BYO) judge model.""" + if not model_config.get("byo_model") or not model_config.get("project_endpoint"): + raise ValueError("BYOProjectModelConfiguration requires both 'byo_model' and 'project_endpoint'.") + if credential is None: + raise ValueError("A TokenCredential is required to call the project Responses API for BYO judge models.") + return ByoProjectResponsesClient( + byo_model=model_config["byo_model"], + project_endpoint=model_config["project_endpoint"], + credential=credential, + ) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py new file mode 100644 index 000000000000..f9fadd16e945 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py @@ -0,0 +1,111 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +"""Unit tests for the admin-connected (BYO) judge-model prototype.""" +from unittest.mock import MagicMock, patch + +import pytest + +from azure.ai.evaluation._byo_judge import ( + ByoProjectResponsesClient, + build_byo_judge_client, + is_byo_model_config, + _to_responses_input, + _map_params, +) + + +class TestByoJudgeHelpers: + def test_is_byo_model_config(self): + assert is_byo_model_config({"byo_model": "c/d", "project_endpoint": "https://x"}) + assert not is_byo_model_config({"azure_endpoint": "https://x"}) + assert not is_byo_model_config({}) + + def test_to_responses_input(self): + assert _to_responses_input([{"role": "user", "content": "hi"}]) == [ + {"type": "message", "role": "user", "content": "hi"} + ] + + def test_map_params_curates_and_renames(self): + assert _map_params({"temperature": 0.0, "max_tokens": 50, "top_p": 0.9, "stream": True}) == { + "temperature": 0.0, + "top_p": 0.9, + "max_output_tokens": 50, + } + + +class TestByoJudgeClient: + @patch("azure.ai.projects.AIProjectClient") + def test_chat_completions_routes_to_responses(self, mock_aipc): + resp = MagicMock() + resp.output_text = "judge says 5" + resp.usage = MagicMock() + resp.id = "resp_1" + resp.model = "my-conn/gpt-4o" + oai = MagicMock() + oai.responses.create.return_value = resp + mock_aipc.return_value.get_openai_client.return_value = oai + + client = build_byo_judge_client( + {"byo_model": "my-conn/gpt-4o", "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1"}, + credential=MagicMock(), + ) + # Judge/grader code calls chat.completions.create unchanged. + result = client.chat.completions.create( + model="ignored", + messages=[{"role": "user", "content": "score this 1-5"}], + temperature=0.0, + max_tokens=10, + ) + + # chat.completions-shaped result over the Responses output. + assert result.choices[0].message.content == "judge says 5" + assert result.choices[0].message.role == "assistant" + + # Underlying call is responses.create with the BYO model + mapped input/params. + mock_aipc.assert_called_once() + _, pkwargs = mock_aipc.call_args + assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p1" + _, rkwargs = oai.responses.create.call_args + assert rkwargs["model"] == "my-conn/gpt-4o" + assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "score this 1-5"}] + assert rkwargs["temperature"] == 0.0 + assert rkwargs["max_output_tokens"] == 10 + + def test_build_requires_project_endpoint(self): + with pytest.raises(ValueError): + build_byo_judge_client({"byo_model": "c/d"}, credential=MagicMock()) + + def test_build_requires_credential(self): + with pytest.raises(ValueError): + build_byo_judge_client({"byo_model": "c/d", "project_endpoint": "https://x"}, credential=None) + + +class TestAzureOpenAIGraderByo: + @patch("azure.ai.projects.AIProjectClient") + def test_grader_get_client_returns_shim_for_byo(self, _mock_aipc): + from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader + + grader = AzureOpenAIGrader( + model_config={ + "byo_model": "my-conn/gpt-4o", + "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1", + }, + grader_config={}, + credential=MagicMock(), + ) + assert isinstance(grader.get_client(), ByoProjectResponsesClient) + + def test_grader_byo_requires_credential(self): + from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader + from azure.ai.evaluation._exceptions import EvaluationException + + with pytest.raises(EvaluationException): + AzureOpenAIGrader( + model_config={ + "byo_model": "my-conn/gpt-4o", + "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1", + }, + grader_config={}, + credential=None, + )