diff --git a/src/uipath/_cli/_evals/_conversational_utils.py b/src/uipath/_cli/_evals/_conversational_utils.py index cd47fe229..d4dbf0cdd 100644 --- a/src/uipath/_cli/_evals/_conversational_utils.py +++ b/src/uipath/_cli/_evals/_conversational_utils.py @@ -12,6 +12,7 @@ UiPathConversationToolCall, UiPathConversationToolCallData, UiPathConversationToolCallResult, + UiPathExternalValue, UiPathInlineValue, ) @@ -146,13 +147,21 @@ def legacy_conversational_eval_input_to_uipath_message_list( else [] ) - # TODO: Add attachments if present - # if message.attachments: - # for attachment in message.attachments: - # content_parts.append( - # UiPathConversationContentPart(...) - # ) - + if eval_message.attachments: + for attachment in eval_message.attachments: + content_parts.append( + UiPathConversationContentPart( + content_part_id=str(uuid.uuid4()), + mime_type=attachment.mime_type, + data=UiPathExternalValue( + uri=f"urn:uipath:cas:file:orchestrator:{attachment.id}" + ), + name=attachment.full_name, + citations=[], + created_at=timestamp, + updated_at=timestamp, + ) + ) messages.append( UiPathConversationMessage( message_id=str(uuid.uuid4()), @@ -228,12 +237,21 @@ def legacy_conversational_eval_input_to_uipath_message_list( else [] ) - # TODO Add attachments if present - # if eval_input.current_user_prompt.attachments: - # for attachment in eval_input.current_user_prompt.attachments: - # content_parts.append( - # UiPathConversationContentPart(...) - # ) + if eval_input.current_user_prompt.attachments: + for attachment in eval_input.current_user_prompt.attachments: + content_parts.append( + UiPathConversationContentPart( + content_part_id=str(uuid.uuid4()), + mime_type=attachment.mime_type, + data=UiPathExternalValue( + uri=f"urn:uipath:cas:file:orchestrator:{attachment.id}" + ), + name=attachment.full_name, + citations=[], + created_at=timestamp, + updated_at=timestamp, + ) + ) messages.append( UiPathConversationMessage( diff --git a/src/uipath/_cli/cli_eval.py b/src/uipath/_cli/cli_eval.py index 0c32ad6e9..4e497ffdb 100644 --- a/src/uipath/_cli/cli_eval.py +++ b/src/uipath/_cli/cli_eval.py @@ -267,7 +267,9 @@ def eval( # Load eval set to resolve the path eval_set_path = eval_set or EvalHelpers.auto_discover_eval_set() - _, resolved_eval_set_path = EvalHelpers.load_eval_set(eval_set_path, eval_ids) + _, resolved_eval_set_path = EvalHelpers.load_eval_set( + eval_set_path, eval_ids, input_overrides=input_overrides + ) eval_context.report_coverage = report_coverage eval_context.input_overrides = input_overrides @@ -338,7 +340,9 @@ async def execute_eval(): # Load eval set (path is already resolved in cli_eval.py) eval_context.evaluation_set, _ = EvalHelpers.load_eval_set( - resolved_eval_set_path, eval_ids + resolved_eval_set_path, + eval_ids, + input_overrides=input_overrides, ) # Resolve model settings override from eval set diff --git a/src/uipath/eval/helpers.py b/src/uipath/eval/helpers.py index 8134f65c5..456fd8bb9 100644 --- a/src/uipath/eval/helpers.py +++ b/src/uipath/eval/helpers.py @@ -25,6 +25,51 @@ EVAL_SETS_DIRECTORY_NAME = "evaluations/eval-sets" +def _apply_file_overrides_to_conversational_inputs( + conversational_inputs: Any, + overrides: dict[str, Any], +) -> None: + """Apply file overrides to conversational input attachments before mapper conversion. + + Extracts file objects from override values (single dict or array), matches them + to existing attachments by FullName, and replaces attachment fields in-place. + No-op if there are no file overrides or no matching attachments. + """ + if not overrides: + return + + file_overrides: list[dict[str, Any]] = [] + for value in overrides.values(): + if isinstance(value, list): + file_overrides.extend(f for f in value if isinstance(f, dict) and "ID" in f) + elif isinstance(value, dict) and "ID" in value: + file_overrides.append(value) + + if not file_overrides: + return + + override_by_name = {f["FullName"]: f for f in file_overrides if "FullName" in f} + + def _override_attachments(attachments: list[Any] | None) -> None: + if not attachments: + return + for attachment in attachments: + override = override_by_name.get(attachment.full_name) + if override: + attachment.id = override["ID"] + if "FullName" in override: + attachment.full_name = override["FullName"] + if "MimeType" in override: + attachment.mime_type = override["MimeType"] + + _override_attachments(conversational_inputs.current_user_prompt.attachments) + + for exchange in conversational_inputs.conversation_history: + for message in exchange: + if hasattr(message, "attachments"): + _override_attachments(message.attachments) + + def discriminate_eval_set(data: dict[str, Any]) -> EvaluationSet | LegacyEvaluationSet: """Discriminate and parse evaluation set based on version field. @@ -91,13 +136,19 @@ def auto_discover_eval_set() -> str: @staticmethod def load_eval_set( - eval_set_path: str, eval_ids: list[str] | None = None + eval_set_path: str, + eval_ids: list[str] | None = None, + input_overrides: dict[str, Any] | None = None, ) -> tuple[EvaluationSet, str]: """Load the evaluation set from file. Args: eval_set_path: Path to the evaluation set file eval_ids: Optional list of evaluation IDs to filter + input_overrides: Optional input field overrides per evaluation ID. + For conversational agents, file overrides are applied to attachments + before the legacy-to-messages conversion so that overridden IDs + are baked into the messages before mapping. Returns: Tuple of (EvaluationSet, resolved_path) @@ -148,6 +199,16 @@ def migrate_evaluation_item( ) if evaluation.conversational_inputs: + overrides_for_eval = ( + input_overrides.get(evaluation.id, {}) + if input_overrides + else {} + ) + _apply_file_overrides_to_conversational_inputs( + evaluation.conversational_inputs, + overrides_for_eval, + ) + conversational_messages_input = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list( evaluation.conversational_inputs ) diff --git a/tests/cli/eval/test_apply_file_overrides.py b/tests/cli/eval/test_apply_file_overrides.py new file mode 100644 index 000000000..8a5dae151 --- /dev/null +++ b/tests/cli/eval/test_apply_file_overrides.py @@ -0,0 +1,378 @@ +"""Tests for _apply_file_overrides_to_conversational_inputs in uipath.eval.helpers.""" + +from typing import Any + +from uipath._cli._evals._conversational_utils import ( + LegacyConversationalEvalInput, + LegacyConversationalEvalInputAgentMessage, + LegacyConversationalEvalJobAttachmentReference, + LegacyConversationalEvalUserMessage, +) +from uipath.eval.helpers import _apply_file_overrides_to_conversational_inputs + + +def _make_attachment( + id: str, full_name: str, mime_type: str = "application/pdf" +) -> LegacyConversationalEvalJobAttachmentReference: + return LegacyConversationalEvalJobAttachmentReference( + ID=id, FullName=full_name, MimeType=mime_type + ) + + +def _make_conversational_inputs( + current_prompt_text: str = "hello", + current_prompt_attachments: list[LegacyConversationalEvalJobAttachmentReference] + | None = None, + conversation_history: list[list[Any]] | None = None, +) -> LegacyConversationalEvalInput: + current_user_prompt = LegacyConversationalEvalUserMessage( + role="user", + text=current_prompt_text, + attachments=current_prompt_attachments, + ) + return LegacyConversationalEvalInput( + conversationHistory=conversation_history or [], + currentUserPrompt=current_user_prompt, + ) + + +class TestApplyFileOverridesNoOp: + """Tests where the function should be a no-op.""" + + def test_empty_overrides(self) -> None: + attachment = _make_attachment("old-id", "file.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + _apply_file_overrides_to_conversational_inputs(inputs, {}) + assert attachment.id == "old-id" + + def test_no_overrides(self) -> None: + attachment = _make_attachment("old-id", "file.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + _apply_file_overrides_to_conversational_inputs(inputs, {}) + assert attachment.id == "old-id" + + def test_overrides_without_id_key_are_ignored(self) -> None: + """Dicts without 'ID' key should not be treated as file overrides.""" + attachment = _make_attachment("old-id", "file.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = {"someField": {"name": "not-a-file"}} + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "old-id" + + def test_string_override_values_are_ignored(self) -> None: + """Non-dict, non-list override values should be skipped.""" + attachment = _make_attachment("old-id", "file.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = {"someField": "just a string", "anotherField": 42} + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "old-id" + + def test_no_matching_attachment_by_name(self) -> None: + """Override with a FullName that doesn't match any attachment.""" + attachment = _make_attachment("old-id", "file.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": { + "ID": "new-id", + "FullName": "other.pdf", + "MimeType": "application/pdf", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "old-id" + + def test_no_attachments_on_current_prompt(self) -> None: + """No error when current_user_prompt has no attachments.""" + inputs = _make_conversational_inputs(current_prompt_attachments=None) + overrides = { + "files": { + "ID": "new-id", + "FullName": "file.pdf", + "MimeType": "application/pdf", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + + def test_empty_attachments_list(self) -> None: + """No error when current_user_prompt has empty attachments list.""" + inputs = _make_conversational_inputs(current_prompt_attachments=[]) + overrides = { + "files": { + "ID": "new-id", + "FullName": "file.pdf", + "MimeType": "application/pdf", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + + +class TestApplyFileOverridesSingleDict: + """Tests for single-dict override values.""" + + def test_override_updates_id(self) -> None: + attachment = _make_attachment("old-id", "report.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": { + "ID": "new-id", + "FullName": "report.pdf", + "MimeType": "application/pdf", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "new-id" + + def test_override_updates_full_name(self) -> None: + attachment = _make_attachment("old-id", "report.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": { + "ID": "new-id", + "FullName": "report.pdf", + "MimeType": "text/plain", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.full_name == "report.pdf" + + def test_override_updates_mime_type(self) -> None: + attachment = _make_attachment("old-id", "report.pdf", "application/pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": { + "ID": "new-id", + "FullName": "report.pdf", + "MimeType": "text/plain", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.mime_type == "text/plain" + + def test_override_without_mime_type_preserves_original(self) -> None: + attachment = _make_attachment("old-id", "report.pdf", "application/pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = {"files": {"ID": "new-id", "FullName": "report.pdf"}} + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "new-id" + assert attachment.mime_type == "application/pdf" + + +class TestApplyFileOverridesArrayValues: + """Tests for array-based override values.""" + + def test_override_from_array_of_files(self) -> None: + attachment = _make_attachment("old-id", "doc.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": [ + { + "ID": "new-id-1", + "FullName": "doc.pdf", + "MimeType": "application/pdf", + }, + { + "ID": "new-id-2", + "FullName": "other.pdf", + "MimeType": "application/pdf", + }, + ] + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "new-id-1" + + def test_multiple_attachments_matched_from_array(self) -> None: + att1 = _make_attachment("old-1", "a.pdf") + att2 = _make_attachment("old-2", "b.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[att1, att2]) + overrides = { + "files": [ + {"ID": "new-1", "FullName": "a.pdf", "MimeType": "application/pdf"}, + {"ID": "new-2", "FullName": "b.pdf", "MimeType": "text/plain"}, + ] + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert att1.id == "new-1" + assert att2.id == "new-2" + assert att2.mime_type == "text/plain" + + def test_non_dict_items_in_array_are_skipped(self) -> None: + """Non-dict items in an override array should be safely ignored.""" + attachment = _make_attachment("old-id", "doc.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": [ + "not-a-dict", + 42, + {"ID": "new-id", "FullName": "doc.pdf", "MimeType": "application/pdf"}, + ] + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "new-id" + + def test_dicts_without_id_in_array_are_skipped(self) -> None: + """Dicts in array that lack 'ID' should not be treated as file overrides.""" + attachment = _make_attachment("old-id", "doc.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": [ + {"name": "not-a-file"}, + {"ID": "new-id", "FullName": "doc.pdf", "MimeType": "application/pdf"}, + ] + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "new-id" + + +class TestApplyFileOverridesMultipleOverrideKeys: + """Tests for overrides with multiple top-level keys.""" + + def test_file_overrides_collected_across_multiple_keys(self) -> None: + att1 = _make_attachment("old-1", "a.pdf") + att2 = _make_attachment("old-2", "b.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[att1, att2]) + overrides = { + "primaryFile": { + "ID": "new-1", + "FullName": "a.pdf", + "MimeType": "application/pdf", + }, + "secondaryFiles": [ + {"ID": "new-2", "FullName": "b.pdf", "MimeType": "text/plain"}, + ], + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert att1.id == "new-1" + assert att2.id == "new-2" + + +class TestApplyFileOverridesEdgeCases: + """Edge cases for the override logic.""" + + def test_file_override_without_full_name_is_not_indexed(self) -> None: + """File dicts with ID but no FullName cannot match any attachment.""" + attachment = _make_attachment("old-id", "file.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = {"files": {"ID": "new-id"}} + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "old-id" + + def test_duplicate_full_names_last_wins(self) -> None: + """When multiple overrides share the same FullName, the last one wins in the dict comprehension.""" + attachment = _make_attachment("old-id", "file.pdf") + inputs = _make_conversational_inputs(current_prompt_attachments=[attachment]) + overrides = { + "files": [ + { + "ID": "first-id", + "FullName": "file.pdf", + "MimeType": "application/pdf", + }, + {"ID": "second-id", "FullName": "file.pdf", "MimeType": "text/plain"}, + ] + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert attachment.id == "second-id" + assert attachment.mime_type == "text/plain" + + +class TestApplyFileOverridesConversationHistory: + """Tests for overrides applied to conversation history attachments.""" + + def test_override_applied_to_history_user_message(self) -> None: + history_attachment = _make_attachment("old-hist", "history.pdf") + history_msg = LegacyConversationalEvalUserMessage( + role="user", text="past message", attachments=[history_attachment] + ) + inputs = _make_conversational_inputs( + conversation_history=[[history_msg]], + ) + overrides = { + "files": { + "ID": "new-hist", + "FullName": "history.pdf", + "MimeType": "application/pdf", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert history_attachment.id == "new-hist" + + def test_override_applied_to_both_current_and_history(self) -> None: + current_att = _make_attachment("old-curr", "current.pdf") + history_att = _make_attachment("old-hist", "history.pdf") + history_msg = LegacyConversationalEvalUserMessage( + role="user", text="past message", attachments=[history_att] + ) + inputs = _make_conversational_inputs( + current_prompt_attachments=[current_att], + conversation_history=[[history_msg]], + ) + overrides = { + "files": [ + { + "ID": "new-curr", + "FullName": "current.pdf", + "MimeType": "application/pdf", + }, + { + "ID": "new-hist", + "FullName": "history.pdf", + "MimeType": "application/pdf", + }, + ] + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert current_att.id == "new-curr" + assert history_att.id == "new-hist" + + def test_agent_messages_without_attachments_are_skipped(self) -> None: + """Agent messages don't have attachments; they should be safely iterated.""" + current_att = _make_attachment("old-id", "file.pdf") + agent_msg = LegacyConversationalEvalInputAgentMessage( + role="agent", text="agent response" + ) + user_msg = LegacyConversationalEvalUserMessage( + role="user", text="user follow-up", attachments=None + ) + inputs = _make_conversational_inputs( + current_prompt_attachments=[current_att], + conversation_history=[[user_msg, agent_msg]], + ) + overrides = { + "files": { + "ID": "new-id", + "FullName": "file.pdf", + "MimeType": "application/pdf", + } + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert current_att.id == "new-id" + + def test_multiple_exchanges_in_history(self) -> None: + att1 = _make_attachment("old-1", "first.pdf") + att2 = _make_attachment("old-2", "second.pdf") + exchange1 = [ + LegacyConversationalEvalUserMessage( + role="user", text="msg1", attachments=[att1] + ), + ] + exchange2 = [ + LegacyConversationalEvalUserMessage( + role="user", text="msg2", attachments=[att2] + ), + ] + inputs = _make_conversational_inputs( + conversation_history=[exchange1, exchange2], + ) + overrides = { + "files": [ + {"ID": "new-1", "FullName": "first.pdf", "MimeType": "application/pdf"}, + { + "ID": "new-2", + "FullName": "second.pdf", + "MimeType": "application/pdf", + }, + ] + } + _apply_file_overrides_to_conversational_inputs(inputs, overrides) + assert att1.id == "new-1" + assert att2.id == "new-2" diff --git a/tests/cli/eval/test_conversational_utils.py b/tests/cli/eval/test_conversational_utils.py index 2030eca7f..b2dee0084 100644 --- a/tests/cli/eval/test_conversational_utils.py +++ b/tests/cli/eval/test_conversational_utils.py @@ -5,13 +5,14 @@ LegacyConversationalEvalInputAgentMessage, LegacyConversationalEvalInputToolCall, LegacyConversationalEvalInputToolCallResult, + LegacyConversationalEvalJobAttachmentReference, LegacyConversationalEvalOutput, LegacyConversationalEvalOutputAgentMessage, LegacyConversationalEvalOutputToolCall, LegacyConversationalEvalUserMessage, UiPathLegacyEvalChatMessagesMapper, ) -from uipath.core.chat import UiPathInlineValue +from uipath.core.chat import UiPathExternalValue, UiPathInlineValue class TestLegacyConversationalEvalInputToUiPathMessages: @@ -333,6 +334,220 @@ def test_blank_text_with_tool_calls_creates_empty_content_parts(self): # Tool calls should still be present assert len(agent_message.tool_calls) == 1 + def test_converts_user_message_with_single_attachment(self): + """Should convert user message with a single attachment to UiPathExternalValue content part.""" + eval_input = LegacyConversationalEvalInput( + conversationHistory=[ + [ + LegacyConversationalEvalUserMessage( + text="See attached", + attachments=[ + LegacyConversationalEvalJobAttachmentReference( + ID="abc-123", + FullName="report.pdf", + MimeType="application/pdf", + ) + ], + ), + LegacyConversationalEvalInputAgentMessage(text="Got it"), + ] + ], + currentUserPrompt=LegacyConversationalEvalUserMessage(text="Next"), + ) + + result = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list( + eval_input + ) + + user_msg = result[0] + assert len(user_msg.content_parts) == 2 + # First part is text + assert isinstance(user_msg.content_parts[0].data, UiPathInlineValue) + assert user_msg.content_parts[0].data.inline == "See attached" + # Second part is attachment + assert isinstance(user_msg.content_parts[1].data, UiPathExternalValue) + assert ( + user_msg.content_parts[1].data.uri + == "urn:uipath:cas:file:orchestrator:abc-123" + ) + assert user_msg.content_parts[1].name == "report.pdf" + assert user_msg.content_parts[1].mime_type == "application/pdf" + + def test_converts_user_message_with_multiple_attachments(self): + """Should convert user message with multiple attachments.""" + eval_input = LegacyConversationalEvalInput( + conversationHistory=[ + [ + LegacyConversationalEvalUserMessage( + text="Here are the files", + attachments=[ + LegacyConversationalEvalJobAttachmentReference( + ID="id-1", + FullName="file1.csv", + MimeType="text/csv", + ), + LegacyConversationalEvalJobAttachmentReference( + ID="id-2", + FullName="file2.xlsx", + MimeType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ), + ], + ), + LegacyConversationalEvalInputAgentMessage(text="Received"), + ] + ], + currentUserPrompt=LegacyConversationalEvalUserMessage(text="Analyze"), + ) + + result = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list( + eval_input + ) + + user_msg = result[0] + assert len(user_msg.content_parts) == 3 # 1 text + 2 attachments + assert isinstance(user_msg.content_parts[1].data, UiPathExternalValue) + assert ( + user_msg.content_parts[1].data.uri + == "urn:uipath:cas:file:orchestrator:id-1" + ) + assert user_msg.content_parts[1].name == "file1.csv" + assert isinstance(user_msg.content_parts[2].data, UiPathExternalValue) + assert ( + user_msg.content_parts[2].data.uri + == "urn:uipath:cas:file:orchestrator:id-2" + ) + assert user_msg.content_parts[2].name == "file2.xlsx" + + def test_converts_current_user_prompt_with_attachment(self): + """Should convert current user prompt attachments.""" + eval_input = LegacyConversationalEvalInput( + conversationHistory=[], + currentUserPrompt=LegacyConversationalEvalUserMessage( + text="Process this", + attachments=[ + LegacyConversationalEvalJobAttachmentReference( + ID="prompt-att-1", + FullName="data.json", + MimeType="application/json", + ) + ], + ), + ) + + result = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list( + eval_input + ) + + assert len(result) == 1 + prompt_msg = result[0] + assert len(prompt_msg.content_parts) == 2 + assert isinstance(prompt_msg.content_parts[1].data, UiPathExternalValue) + assert ( + prompt_msg.content_parts[1].data.uri + == "urn:uipath:cas:file:orchestrator:prompt-att-1" + ) + assert prompt_msg.content_parts[1].name == "data.json" + assert prompt_msg.content_parts[1].mime_type == "application/json" + + def test_user_message_with_no_attachments_unchanged(self): + """Should not add attachment content parts when attachments is None.""" + eval_input = LegacyConversationalEvalInput( + conversationHistory=[ + [ + LegacyConversationalEvalUserMessage( + text="No attachments here", attachments=None + ), + LegacyConversationalEvalInputAgentMessage(text="Ok"), + ] + ], + currentUserPrompt=LegacyConversationalEvalUserMessage(text="Done"), + ) + + result = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list( + eval_input + ) + + user_msg = result[0] + assert len(user_msg.content_parts) == 1 + assert isinstance(user_msg.content_parts[0].data, UiPathInlineValue) + + def test_user_message_with_empty_text_and_attachment(self): + """Should create only attachment content part when text is empty.""" + eval_input = LegacyConversationalEvalInput( + conversationHistory=[ + [ + LegacyConversationalEvalUserMessage( + text="", + attachments=[ + LegacyConversationalEvalJobAttachmentReference( + ID="att-only", + FullName="image.png", + MimeType="image/png", + ) + ], + ), + LegacyConversationalEvalInputAgentMessage(text="I see the image"), + ] + ], + currentUserPrompt=LegacyConversationalEvalUserMessage(text="Thanks"), + ) + + result = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list( + eval_input + ) + + user_msg = result[0] + # No text part (empty text), only attachment + assert len(user_msg.content_parts) == 1 + assert isinstance(user_msg.content_parts[0].data, UiPathExternalValue) + assert ( + user_msg.content_parts[0].data.uri + == "urn:uipath:cas:file:orchestrator:att-only" + ) + assert user_msg.content_parts[0].name == "image.png" + + def test_attachment_content_parts_have_unique_ids(self): + """Should generate unique content part IDs for attachment parts.""" + eval_input = LegacyConversationalEvalInput( + conversationHistory=[ + [ + LegacyConversationalEvalUserMessage( + text="Files", + attachments=[ + LegacyConversationalEvalJobAttachmentReference( + ID="a1", + FullName="f1.txt", + MimeType="text/plain", + ), + LegacyConversationalEvalJobAttachmentReference( + ID="a2", + FullName="f2.txt", + MimeType="text/plain", + ), + ], + ), + LegacyConversationalEvalInputAgentMessage(text="Ok"), + ] + ], + currentUserPrompt=LegacyConversationalEvalUserMessage( + text="More", + attachments=[ + LegacyConversationalEvalJobAttachmentReference( + ID="a3", + FullName="f3.txt", + MimeType="text/plain", + ) + ], + ), + ) + + result = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list( + eval_input + ) + + all_ids = [part.content_part_id for msg in result for part in msg.content_parts] + assert len(all_ids) == len(set(all_ids)) + class TestLegacyConversationalEvalOutputToUiPathMessageData: """Tests for converting legacy eval output to UiPath message data."""