Prototype: BYO admin-connected judge models via project Responses API…

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py

@@ -61,6 +61,27 @@ def __init__(
 
     def _validate_model_config(self) -> None:
         """Validate the model configuration that this grader wrapper is using."""
+        # Prototype: admin-connected (BYO) judge models are served via the project Responses
+        # API and require a credential + project endpoint (no api_key / azure_endpoint).
+        from azure.ai.evaluation._byo_judge import is_byo_model_config
+
+        if is_byo_model_config(self._model_config):
+            if not self._credential:
+                raise EvaluationException(
+                    message=f"{type(self).__name__}: BYO (admin-connected) judge models require a credential.",
+                    blame=ErrorBlame.USER_ERROR,
+                    category=ErrorCategory.INVALID_VALUE,
+                    target=ErrorTarget.AOAI_GRADER,
+                )
+            if not self._model_config.get("project_endpoint"):
+                raise EvaluationException(
+                    message=f"{type(self).__name__}: BYO judge model_config requires 'project_endpoint'.",
+                    blame=ErrorBlame.USER_ERROR,
+                    category=ErrorCategory.INVALID_VALUE,
+                    target=ErrorTarget.AOAI_GRADER,
+                )
+            return
+
         msg = None
         if self._is_azure_model_config(self._model_config):
             if not any(auth for auth in (self._model_config.get("api_key"), self._credential)):
@@ -104,6 +125,15 @@ def get_client(self) -> Any:
         """
         default_headers = {"User-Agent": UserAgentSingleton().value}
         model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration] = self._model_config
+
+        # Prototype: admin-connected (BYO) judge model referenced as "connection/deployment".
+        # Route judge calls through the Foundry project Responses API via a
+        # chat.completions-compatible shim, so the platform resolves the connection.
+        from azure.ai.evaluation._byo_judge import is_byo_model_config, build_byo_judge_client
+
+        if is_byo_model_config(model_config):
+            return build_byo_judge_client(model_config, self._credential)
+
         api_key: Optional[str] = model_config.get("api_key")
 
         if self._is_azure_model_config(model_config):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py

@@ -0,0 +1,157 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Prototype: use admin-connected (BYO) models as judge / grader models.
+
+Admin-connected models (Foundry "ModelGateway" / "API Management" connections, referenced
+as ``"connection-name/deployment-name"``) are only invokable through the Foundry project
+**Responses API** — the platform resolves the connection and handles every auth type
+(API key / managed identity / OAuth), ``deploymentInPath``, api-version and custom headers.
+
+LLM-as-judge evaluators in this library call ``client.chat.completions.create(...)``. This
+module provides a small OpenAI-compatible **shim** that routes those calls to the project
+Responses API for a BYO model, so judge/grader code can use admin-connected connections
+**without any change to its calling code**.
+"""
+import time
+from typing import Any, Dict, List, Optional
+
+from typing_extensions import TypedDict
+
+from azure.core.credentials import TokenCredential
+
+
+class BYOProjectModelConfiguration(TypedDict, total=False):
+    """Model configuration for an admin-connected (BYO) judge model.
+
+    :keyword byo_model: The admin-connected model reference, ``"connection-name/deployment-name"``.
+    :keyword project_endpoint: The Foundry project endpoint,
+        e.g. ``https://<account>.services.ai.azure.com/api/projects/<project>``.
+    """
+
+    byo_model: str
+    project_endpoint: str
+
+
+def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+    """Map chat-completions messages ({role, content}) to Responses API input items."""
+    items: List[Dict[str, Any]] = []
+    for message in messages or []:
+        items.append(
+            {
+                "type": "message",
+                "role": message.get("role", "user"),
+                "content": message.get("content", ""),
+            }
+        )
+    return items
+
+
+def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """Map a curated set of chat-completions sampling params to Responses API params."""
+    mapped: Dict[str, Any] = {}
+    if "temperature" in kwargs:
+        mapped["temperature"] = kwargs["temperature"]
+    if "top_p" in kwargs:
+        mapped["top_p"] = kwargs["top_p"]
+    for key in ("max_output_tokens", "max_completion_tokens", "max_tokens"):
+        if key in kwargs:
+            mapped["max_output_tokens"] = kwargs[key]
+            break
+    return mapped
+
+
+class _ChatMessage:
+    def __init__(self, content: str) -> None:
+        self.role = "assistant"
+        self.content = content
+        self.tool_calls = None
+
+
+class _Choice:
+    def __init__(self, content: str) -> None:
+        self.index = 0
+        self.message = _ChatMessage(content)
+        self.finish_reason = "stop"
+
+
+class _ChatCompletion:
+    """Minimal chat.completions-shaped view over a Responses API result."""
+
+    def __init__(self, response: Any) -> None:
+        self.id = getattr(response, "id", None)
+        self.model = getattr(response, "model", None)
+        self.usage = getattr(response, "usage", None)
+        self.object = "chat.completion"
+        self.created = int(time.time())
+        self.choices = [_Choice(getattr(response, "output_text", "") or "")]
+
+
+class _ChatCompletions:
+    def __init__(self, owner: "ByoProjectResponsesClient") -> None:
+        self._owner = owner
+
+    def create(self, *, model: Optional[str] = None, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> _ChatCompletion:
+        # ``model`` from the caller is ignored — the BYO model is fixed by the shim's config.
+        response = self._owner._responses_create(messages=messages, **kwargs)
+        return _ChatCompletion(response)
+
+
+class _Chat:
+    def __init__(self, owner: "ByoProjectResponsesClient") -> None:
+        self.completions = _ChatCompletions(owner)
+
+
+class ByoProjectResponsesClient:
+    """OpenAI-compatible shim that routes ``chat.completions.create()`` to the Foundry
+    project Responses API for an admin-connected (BYO) model.
+
+    Judge/grader code that calls ``client.chat.completions.create(model=..., messages=...)``
+    works unchanged; the request is served by the platform, which resolves the connection.
+    A ``responses`` passthrough is also exposed for callers that use the Responses API directly.
+    """
+
+    def __init__(self, byo_model: str, project_endpoint: str, credential: TokenCredential) -> None:
+        self._byo_model = byo_model
+        self._project_endpoint = project_endpoint
+        self._credential = credential
+        self._client: Any = None
+        self.chat = _Chat(self)
+
+    def _openai(self) -> Any:
+        if self._client is None:
+            from azure.ai.projects import AIProjectClient
+
+            self._client = AIProjectClient(
+                endpoint=self._project_endpoint, credential=self._credential
+            ).get_openai_client()
+        return self._client
+
+    def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
+        return self._openai().responses.create(
+            model=self._byo_model,
+            input=_to_responses_input(messages),
+            **_map_params(kwargs),
+        )
+
+    @property
+    def responses(self) -> Any:
+        return self._openai().responses
+
+
+def is_byo_model_config(model_config: Dict[str, Any]) -> bool:
+    """Return True if the model configuration references an admin-connected (BYO) model."""
+    return bool(model_config) and bool(model_config.get("byo_model"))
+
+
+def build_byo_judge_client(model_config: Dict[str, Any], credential: TokenCredential) -> ByoProjectResponsesClient:
+    """Build a chat.completions-compatible client for an admin-connected (BYO) judge model."""
+    if not model_config.get("byo_model") or not model_config.get("project_endpoint"):
+        raise ValueError("BYOProjectModelConfiguration requires both 'byo_model' and 'project_endpoint'.")
+    if credential is None:
+        raise ValueError("A TokenCredential is required to call the project Responses API for BYO judge models.")
+    return ByoProjectResponsesClient(
+        byo_model=model_config["byo_model"],
+        project_endpoint=model_config["project_endpoint"],
+        credential=credential,
+    )

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py

@@ -0,0 +1,111 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Unit tests for the admin-connected (BYO) judge-model prototype."""
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from azure.ai.evaluation._byo_judge import (
+    ByoProjectResponsesClient,
+    build_byo_judge_client,
+    is_byo_model_config,
+    _to_responses_input,
+    _map_params,
+)
+
+
+class TestByoJudgeHelpers:
+    def test_is_byo_model_config(self):
+        assert is_byo_model_config({"byo_model": "c/d", "project_endpoint": "https://x"})
+        assert not is_byo_model_config({"azure_endpoint": "https://x"})
+        assert not is_byo_model_config({})
+
+    def test_to_responses_input(self):
+        assert _to_responses_input([{"role": "user", "content": "hi"}]) == [
+            {"type": "message", "role": "user", "content": "hi"}
+        ]
+
+    def test_map_params_curates_and_renames(self):
+        assert _map_params({"temperature": 0.0, "max_tokens": 50, "top_p": 0.9, "stream": True}) == {
+            "temperature": 0.0,
+            "top_p": 0.9,
+            "max_output_tokens": 50,
+        }
+
+
+class TestByoJudgeClient:
+    @patch("azure.ai.projects.AIProjectClient")
+    def test_chat_completions_routes_to_responses(self, mock_aipc):
+        resp = MagicMock()
+        resp.output_text = "judge says 5"
+        resp.usage = MagicMock()
+        resp.id = "resp_1"
+        resp.model = "my-conn/gpt-4o"
+        oai = MagicMock()
+        oai.responses.create.return_value = resp
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client = build_byo_judge_client(
+            {"byo_model": "my-conn/gpt-4o", "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1"},
+            credential=MagicMock(),
+        )
+        # Judge/grader code calls chat.completions.create unchanged.
+        result = client.chat.completions.create(
+            model="ignored",
+            messages=[{"role": "user", "content": "score this 1-5"}],
+            temperature=0.0,
+            max_tokens=10,
+        )
+
+        # chat.completions-shaped result over the Responses output.
+        assert result.choices[0].message.content == "judge says 5"
+        assert result.choices[0].message.role == "assistant"
+
+        # Underlying call is responses.create with the BYO model + mapped input/params.
+        mock_aipc.assert_called_once()
+        _, pkwargs = mock_aipc.call_args
+        assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p1"
+        _, rkwargs = oai.responses.create.call_args
+        assert rkwargs["model"] == "my-conn/gpt-4o"
+        assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "score this 1-5"}]
+        assert rkwargs["temperature"] == 0.0
+        assert rkwargs["max_output_tokens"] == 10
+
+    def test_build_requires_project_endpoint(self):
+        with pytest.raises(ValueError):
+            build_byo_judge_client({"byo_model": "c/d"}, credential=MagicMock())
+
+    def test_build_requires_credential(self):
+        with pytest.raises(ValueError):
+            build_byo_judge_client({"byo_model": "c/d", "project_endpoint": "https://x"}, credential=None)
+
+
+class TestAzureOpenAIGraderByo:
+    @patch("azure.ai.projects.AIProjectClient")
+    def test_grader_get_client_returns_shim_for_byo(self, _mock_aipc):
+        from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader
+
+        grader = AzureOpenAIGrader(
+            model_config={
+                "byo_model": "my-conn/gpt-4o",
+                "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
+            },
+            grader_config={},
+            credential=MagicMock(),
+        )
+        assert isinstance(grader.get_client(), ByoProjectResponsesClient)
+
+    def test_grader_byo_requires_credential(self):
+        from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader
+        from azure.ai.evaluation._exceptions import EvaluationException
+
+        with pytest.raises(EvaluationException):
+            AzureOpenAIGrader(
+                model_config={
+                    "byo_model": "my-conn/gpt-4o",
+                    "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
+                },
+                grader_config={},
+                credential=None,
+            )