diff --git a/sdk/ai/azure-ai-projects/.env.template b/sdk/ai/azure-ai-projects/.env.template index 76c795cfaf62..c1dd0df6c39a 100644 --- a/sdk/ai/azure-ai-projects/.env.template +++ b/sdk/ai/azure-ai-projects/.env.template @@ -46,6 +46,13 @@ SHAREPOINT_USER_INPUT= FABRIC_USER_INPUT= BING_CUSTOM_USER_INPUT= A2A_USER_INPUT= +EVALUATOR_NAME= +DATASET_NAME= +DATASET_VERSION= +POLL_INTERVAL_SECONDS= +EVAL_MODEL= +OPTIMIZATION_MODEL= +JOB_ID= ####################################################################### # diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index 46e2133eea41..c09665e8b58b 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -169,6 +169,7 @@ The table below lists the operation groups supported by the client library, with | Agents (create, run, stream) | [Agents overview](https://learn.microsoft.com/azure/foundry/agents/overview) | `samples/agents/` | | Hosted agents (preview) | [Hosted agents concepts](https://learn.microsoft.com/azure/foundry/agents/concepts/hosted-agents), [Deploy your first hosted agent](https://learn.microsoft.com/azure/foundry/agents/quickstarts/quickstart-hosted-agent) | `samples/hosted_agents/` | | Agents tools | [Tool catalog](https://learn.microsoft.com/azure/foundry/agents/concepts/tool-catalog) | `samples/agents/tools/` | +| Agents optimization | | `samples/agents/optimization/` | | Connections | [Add a new connection to your project](https://learn.microsoft.com/azure/foundry/how-to/connections-add?tabs=foundry-portal)| `samples/connections/` | | Datasets | | `samples/datasets/` | | Deployments | [Deployment types](https://learn.microsoft.com/azure/foundry/foundry-models/concepts/deployment-types) | `samples/deployments/` | diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py new file mode 100644 index 000000000000..575a7ca885f3 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py @@ -0,0 +1,127 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to create an agent + optimization job, poll it to completion, and read the results. + + Agent optimization automatically improves an agent's system prompt, model + choice, or tool definitions by running candidate variants against your + training dataset and scoring them with the evaluators you specify. + +USAGE: + python sample_optimization_job_basic.py + + Before running the sample: + + pip install "azure-ai-projects>=2.3.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry portal. + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". +""" + +import os +import time + +from dotenv import load_dotenv + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + OptimizationAgentIdentifier as AgentIdentifier, + OptimizationEvaluatorRef as EvaluatorRef, + JobStatus, + OptimizationJob, + OptimizationJobInputs, + OptimizationOptions, + OptimizationReferenceDatasetInput as ReferenceDatasetInput, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ["FOUNDRY_AGENT_NAME"] +dataset_name = os.environ["DATASET_NAME"] +evaluator_name = os.environ["EVALUATOR_NAME"] +dataset_version = os.environ.get("DATASET_VERSION", "1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") +optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + + # ------------------------------------------------------------------ + # 1. Create an optimization job. + # ------------------------------------------------------------------ + print("Creating optimization job...") + job = project_client.beta.agents.create_optimization_job( + job=OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ) + ) + ) + print(f"Created job: id={job.id}, status={job.status}") + + # ------------------------------------------------------------------ + # 2. Poll until the job reaches a terminal state. + # ------------------------------------------------------------------ + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: + time.sleep(poll_interval) + job = project_client.beta.agents.get_optimization_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final status: {job.status}") + + if job.warnings: + for warning in job.warnings: + print(f"[WARNING] {warning}") + + if job.status == JobStatus.FAILED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Optimization job `{job.id}` failed: {message}") + + # ------------------------------------------------------------------ + # 3. Inspect the results. + # ------------------------------------------------------------------ + if job.status == JobStatus.SUCCEEDED and job.result: + result = job.result + print(f"\nBaseline candidate: {result.baseline}") + print(f"Best candidate: {result.best}") + print(f"Candidates ({len(result.candidates or [])}):") + for candidate in result.candidates or []: + print( + f" - {candidate.name}" + f" | avg_score={candidate.avg_score:.4f}" + f" | avg_tokens={candidate.avg_tokens:.0f}" + ) + if candidate.eval_id: + print(f" eval_id={candidate.eval_id}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py new file mode 100644 index 000000000000..6a636a03c5a0 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py @@ -0,0 +1,130 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Async version of sample_optimization_job_basic.py. Demonstrates how to + create an agent optimization job, poll it to completion, and read the + results using the async AIProjectClient. + +USAGE: + python sample_optimization_job_basic_async.py + + Before running the sample: + + pip install "azure-ai-projects>=2.3.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry portal. + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". +""" + +import asyncio +import os + +from dotenv import load_dotenv + +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import ( + OptimizationAgentIdentifier as AgentIdentifier, + OptimizationEvaluatorRef as EvaluatorRef, + JobStatus, + OptimizationJob, + OptimizationJobInputs, + OptimizationOptions, + OptimizationReferenceDatasetInput as ReferenceDatasetInput, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ["FOUNDRY_AGENT_NAME"] +dataset_name = os.environ["DATASET_NAME"] +evaluator_name = os.environ["EVALUATOR_NAME"] +dataset_version = os.environ.get("DATASET_VERSION", "1") +eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") +optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + + +async def main() -> None: + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + + # ------------------------------------------------------------------ + # 1. Create an optimization job. + # ------------------------------------------------------------------ + print("Creating optimization job...") + job = await project_client.beta.agents.create_optimization_job( + job=OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ) + ) + ) + print(f"Created job: id={job.id}, status={job.status}") + + # ------------------------------------------------------------------ + # 2. Poll until terminal state. + # ------------------------------------------------------------------ + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: + await asyncio.sleep(poll_interval) + job = await project_client.beta.agents.get_optimization_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final status: {job.status}") + + if job.warnings: + for warning in job.warnings: + print(f"[WARNING] {warning}") + + if job.status == JobStatus.FAILED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Optimization job `{job.id}` failed: {message}") + + # ------------------------------------------------------------------ + # 3. Inspect the results. + # ------------------------------------------------------------------ + if job.status == JobStatus.SUCCEEDED and job.result: + result = job.result + print(f"\nBaseline candidate: {result.baseline}") + print(f"Best candidate: {result.best}") + print(f"Candidates ({len(result.candidates or [])}):") + for candidate in result.candidates or []: + print( + f" - {candidate.name}" + f" | avg_score={candidate.avg_score:.4f}" + f" | avg_tokens={candidate.avg_tokens:.0f}" + ) + if candidate.eval_id: + print(f" eval_id={candidate.eval_id}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py new file mode 100644 index 000000000000..9dd011a14394 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py @@ -0,0 +1,90 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to create an agent + optimization job and immediately cancel it. + +USAGE: + python sample_optimization_job_cancel.py + + Before running the sample: + + pip install "azure-ai-projects>=2.3.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry portal. + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 7) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". + +""" + +import os + +from dotenv import load_dotenv + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + OptimizationAgentIdentifier as AgentIdentifier, + OptimizationEvaluatorRef as EvaluatorRef, + OptimizationJob, + OptimizationJobInputs, + OptimizationOptions, + OptimizationReferenceDatasetInput as ReferenceDatasetInput, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ["FOUNDRY_AGENT_NAME"] +dataset_name = os.environ["DATASET_NAME"] +evaluator_name = os.environ["EVALUATOR_NAME"] +dataset_version = os.environ.get("DATASET_VERSION", "1") +eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") +optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") + + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + + # ------------------------------------------------------------------ + # 1. Create a job. + # ------------------------------------------------------------------ + print("Creating optimization job...") + job = project_client.beta.agents.create_optimization_job( + job=OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ) + ) + ) + print(f"Created job: id={job.id}, status={job.status}") + + # ------------------------------------------------------------------ + # 2. Cancel it immediately. + # ------------------------------------------------------------------ + print(f"Cancelling job {job.id}...") + cancelled = project_client.beta.agents.cancel_optimization_job(job_id=job.id) + print(f"Job {cancelled.id} status: {cancelled.status}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_list_get_delete.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_list_get_delete.py new file mode 100644 index 000000000000..4ffa45625fb9 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_list_get_delete.py @@ -0,0 +1,98 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to list optimization + jobs (with optional filters), get a specific job by ID, and delete a job. + +USAGE: + python sample_optimization_job_list_get_delete.py + + Before running the sample: + + pip install "azure-ai-projects>=2.3.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry portal. + 2) FOUNDRY_AGENT_NAME - Required. Filter the list to jobs for this agent. + 3) JOB_ID - Required. If set, fetches and deletes this specific job. +""" + +import os + +from dotenv import load_dotenv + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import JobStatus + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ["FOUNDRY_AGENT_NAME"] +job_id = os.environ["JOB_ID"] + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + + # ------------------------------------------------------------------ + # 1. List the most recent 10 optimization jobs (unfiltered). + # ------------------------------------------------------------------ + print("Listing optimization jobs (limit=10):") + count = 0 + for job in project_client.beta.agents.list_optimization_jobs(limit=10): + agent_str = job.agent.agent_name if job.agent else "?" + print(f" {job.id} | status={job.status} | agent={agent_str}") + count += 1 + print(f" ({count} jobs listed)\n") + + # ------------------------------------------------------------------ + # 2. List jobs filtered by agent name (if provided). + # ------------------------------------------------------------------ + if agent_name: + print(f"Listing jobs for agent '{agent_name}' (limit=10):") + count = 0 + for job in project_client.beta.agents.list_optimization_jobs(agent_name=agent_name, limit=10): + print(f" {job.id} | status={job.status}") + count += 1 + print(f" ({count} jobs)\n") + + # ------------------------------------------------------------------ + # 3. List jobs filtered by status. + # ------------------------------------------------------------------ + print(f"Listing succeeded jobs (limit=5):") + count = 0 + for job in project_client.beta.agents.list_optimization_jobs(status=JobStatus.SUCCEEDED, limit=5): + print(f" {job.id}") + count += 1 + print(f" ({count} succeeded jobs)\n") + + # ------------------------------------------------------------------ + # 4. Get a specific job by ID (if provided). + # ------------------------------------------------------------------ + if job_id: + print(f"Getting job {job_id}...") + job = project_client.beta.agents.get_optimization_job(job_id=job_id) + print(f" status={job.status}") + if job.inputs: + print(f" agent={job.inputs.agent.agent_name if job.inputs.agent else '?'}") + if job.result: + print(f" baseline={job.result.baseline}, best={job.result.best}") + print(f" candidates: {len(job.result.candidates or [])}") + if job.warnings: + for w in job.warnings: + print(f" [WARNING] {w}") + + # ------------------------------------------------------------------ + # 5. Delete the job. + # ------------------------------------------------------------------ + print(f"\nDeleting job {job_id}...") + project_client.beta.agents.delete_optimization_job(job_id=job_id) + print(" Deleted.")