From c34c41302f22182d59ad2fb560456872ca4bea50 Mon Sep 17 00:00:00 2001 From: ssjia Date: Fri, 29 May 2026 07:29:56 -0700 Subject: [PATCH 1/3] [llama4][tests][2/N] Report disabled stito runner tests as executed Pull Request resolved: https://github.com/pytorch/executorch/pull/19868 Applies the same disabled-test treatment as the prior diff in this stack to the llama4 stito runner e2e tests. Some test runners preserve stale failure state when tests report through unittest skip results, so this replaces the disabled coverage with local decorators that keep the tests discoverable, log their disabled reason, and produce an executed result. Adds two decorators: disable_test mirrors unittest.skip for the unconditionally disabled test_replay_logged_tensors and test_transcription tests, and disable_if mirrors unittest.skipIf (evaluating the condition at decoration time) for the MODEL_VARIANT-gated test_speech_features and test_multiturn_speech_features_embeddings tests. ghstack-source-id: 387421286 @exported-using-ghexport Differential Revision: [D106736355](https://our.internmc.facebook.com/intern/diff/D106736355/) From 6e3b139fa2530a6f60bfd37d7770b51f30b6ed1b Mon Sep 17 00:00:00 2001 From: ssjia Date: Fri, 29 May 2026 07:29:56 -0700 Subject: [PATCH 2/3] [qualcomm][tests][3/N] Report disabled QNN simulator tests as executed Applies the same disabled-test treatment as the prior diffs in this stack to the QNN x86 simulator delegate tests. Some test runners preserve stale failure state when tests report through unittest skip results, so this replaces the disabled coverage with a local decorator that keeps the tests discoverable, logs their disabled reason, and produces an executed result. Replaces the six unittest.skip decorators on the simulator-unsupported runtime-option tests (HTP performance, profile etdump, heap profile) across the quantized and floating-point utils simulator classes with the local disable_test decorator. The now-unused unittest import is removed. Differential Revision: [D106736356](https://our.internmc.facebook.com/intern/diff/D106736356/) From dbc2e02d8cb6ef214b42e56a87837bbd9884b2fc Mon Sep 17 00:00:00 2001 From: ssjia Date: Fri, 29 May 2026 07:30:02 -0700 Subject: [PATCH 3/3] [devtools][tests][4/N] Report disabled inspector tests as executed Applies the same disabled-test treatment as the prior diffs in this stack to the devtools inspector tests. Some test runners preserve stale failure state when tests report through unittest skip results, so this replaces the conditionally disabled coverage with a local decorator that keeps the tests discoverable, logs their disabled reason, and produces an executed result. Adds a disable_if decorator that mirrors unittest.skipIf (evaluating the condition at decoration time) and converts the three Windows-gated test cases to use it. Differential Revision: [D106736354](https://our.internmc.facebook.com/intern/diff/D106736354/) ghstack-source-id: 387629542 Pull-Request: https://github.com/pytorch/executorch/pull/19874 --- devtools/inspector/tests/inspector_test.py | 29 +++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/devtools/inspector/tests/inspector_test.py b/devtools/inspector/tests/inspector_test.py index b33c5b37164..4c59190650c 100644 --- a/devtools/inspector/tests/inspector_test.py +++ b/devtools/inspector/tests/inspector_test.py @@ -7,6 +7,7 @@ # pyre-unsafe import copy +import functools import os import random import statistics @@ -90,6 +91,28 @@ def forward(self, indices: torch.Tensor, values: torch.Tensor) -> torch.Tensor: ETRECORD_PATH = "unittest_etrecord_path" +def disable_if(condition, reason): + """Disable a test when condition is true, still reporting it as executed. + + Conditional analogue of unittest.skipIf that keeps disabled tests visible in + logs instead of producing a skipped result, which some test runners handle + inconsistently. + """ + + def decorator(fn): + if not condition: + return fn + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + print(f"DISABLED_TEST: {fn.__qualname__}: {reason}") + return None + + return wrapper + + return decorator + + # TODO: write an E2E test: create an inspector instance, mock just the file reads, and then verify the external correctness class TestInspector(unittest.TestCase): def test_perf_data(self) -> None: @@ -1504,7 +1527,7 @@ def test_calculate_numeric_gap_with_edge_dialect_exported_program_name(self): self.assertIsInstance(df, pd.DataFrame) self.assertEqual(len(df), 1) - @unittest.skipIf(sys.platform.startswith("win"), "Skipping on Windows") + @disable_if(sys.platform.startswith("win"), "Skipping on Windows") def test_transformer_block_xnnpack_numeric_gap_within_tolerance(self): """ Test that the numeric gap between AOT and runtime intermediate outputs @@ -1693,7 +1716,7 @@ def forward( f"Stack trace for {op_name} doesn't contain file info", ) - @unittest.skipIf(sys.platform.startswith("win"), "Skipping on Windows") + @disable_if(sys.platform.startswith("win"), "Skipping on Windows") def test_intermediate_tensor_comparison_with_torch_export(self): """Test intermediate tensor comparison using torch.export.export and to_edge_transform_and_lower. @@ -1840,7 +1863,7 @@ def _gen_random_runtime_output( ) -> List[Union[None, List[torch.Tensor], bool, float, int, str, torch.Tensor]]: return [torch.randn(RAW_DATA_SIZE)] - @unittest.skipIf(sys.platform.startswith("win"), "Skipping on Windows") + @disable_if(sys.platform.startswith("win"), "Skipping on Windows") def test_disable_debug_handle_validation_with_symbolic_shapes(self): """ Test that demonstrates the issue with symbolic shape related nodes losing from_node info