From 8a96d5be5d89e90e7d16e7084638681941587ea9 Mon Sep 17 00:00:00 2001 From: AdityaGupta716 Date: Sat, 28 Mar 2026 01:49:15 +0530 Subject: [PATCH] systemtests: add per-test max_time override for precice-config --- changelog-entries/402.md | 1 + tools/tests/README.md | 2 ++ tools/tests/generate_reference_results.py | 10 ++++++--- tools/tests/systemtests.py | 10 ++++++--- tools/tests/systemtests/Systemtest.py | 26 +++++++++++++++++++++++ tools/tests/systemtests/TestSuite.py | 16 +++++++++++++- 6 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 changelog-entries/402.md diff --git a/changelog-entries/402.md b/changelog-entries/402.md new file mode 100644 index 000000000..6e1cf81a8 --- /dev/null +++ b/changelog-entries/402.md @@ -0,0 +1 @@ +- Add optional `max_time` and `max_time_windows` fields to `tests.yaml` entries to override `` or `` in `precice-config.xml` per test case. diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..e0c416c89 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -118,6 +118,8 @@ In order for the systemtests to pick up the tutorial we need to define a `metada To add a testsuite just open the `tests.yaml` file and use the output of `python print_case_combinations.py` to add the right case combinations you want to test. Note that you can specify a `reference_result` which is not yet present. The `generate_reference_data.py` will pick that up and create it for you. Note that its important to carefully check the paths of the `reference_result` in order to not have typos in there. Also note that same cases in different testsuites should use the same `reference_result`. +To cap the preCICE simulation time for a specific test without editing `precice-config.xml`, add an optional `max_time` (positive float, overrides ``) or `max_time_windows` (positive integer, overrides ``) field to the tutorial entry. Applies to both test runs and reference result generation. + ### Generate reference results Since we need data to compare against, you need to run `python generate_reference_data.py`. This process might take a while. diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 055e7b31c..23f9c96d2 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -108,10 +108,14 @@ def main(): for test_suite in test_suites: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + max_times = test_suite.max_times.get(tutorial, []) + mtw_list = test_suite.max_time_windows.get(tutorial, []) + for i, (case, reference_result) in enumerate(zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): + max_time = max_times[i] if i < len(max_times) else None + max_time_windows = mtw_list[i] if i < len(mtw_list) else None systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 8a37670eb..4968c7486 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -58,10 +58,14 @@ def main(): for test_suite in test_suites_to_execute: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + max_times = test_suite.max_times.get(tutorial, []) + mtw_list = test_suite.max_time_windows.get(tutorial, []) + for i, (case, reference_result) in enumerate(zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): + max_time = max_times[i] if i < len(max_times) else None + max_time_windows = mtw_list[i] if i < len(mtw_list) else None systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6abc5a029..0db99eb28 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -134,6 +134,8 @@ class Systemtest: arguments: SystemtestArguments case_combination: CaseCombination reference_result: ReferenceResult + max_time: float | None = None + max_time_windows: int | None = None params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -513,11 +515,35 @@ def __write_logs(self, stdout_data: List[str], stderr_data: List[str]): with open(self.system_test_dir / "stderr.log", 'w') as stderr_file: stderr_file.write("\n".join(stderr_data)) + def __apply_max_time_override(self): + """Overwrite or value in precice-config.xml.""" + if self.max_time is None and self.max_time_windows is None: + return + config_path = self.system_test_dir / "precice-config.xml" + text = config_path.read_text() + new_text = text + if self.max_time is not None: + pattern = r'({self.max_time}\2', new_text) + if count == 0: + logging.warning(f"No tag found in {config_path}") + else: + logging.info(f"Overwrote to {self.max_time} in {config_path}") + if self.max_time_windows is not None: + pattern = r'({self.max_time_windows}\2', new_text) + if count == 0: + logging.warning(f"No tag found in {config_path}") + else: + logging.info(f"Overwrote to {self.max_time_windows} in {config_path}") + config_path.write_text(new_text) + def __prepare_for_run(self, run_directory: Path): """ Prepares the run_directory with folders and datastructures needed for every systemtest execution """ self.__copy_tutorial_into_directory(run_directory) + self.__apply_max_time_override() self.__copy_tools(run_directory) self.__put_gitignore(run_directory) host_uid, host_gid = self.__get_uid_gid() diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 9d8c2ac72..7bb50189f 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -10,6 +10,8 @@ class TestSuite: name: str cases_of_tutorial: Dict[Tutorial, List[CaseCombination]] reference_results: Dict[Tutorial, List[ReferenceResult]] + max_times: Dict[Tutorial, list] = field(default_factory=dict) + max_time_windows: Dict[Tutorial, list] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -48,6 +50,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): for test_suite_name in test_suites_raw: case_combinations_of_tutorial = {} reference_results_of_tutorial = {} + max_times_of_tutorial = {} + max_time_windows_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -57,6 +61,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if tutorial not in case_combinations_of_tutorial: case_combinations_of_tutorial[tutorial] = [] reference_results_of_tutorial[tutorial] = [] + max_times_of_tutorial[tutorial] = [] + max_time_windows_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -65,12 +71,20 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): case_combinations_of_tutorial[tutorial].append(case_combination_requested) reference_results_of_tutorial[tutorial].append(ReferenceResult( tutorial_case['reference_result'], case_combination_requested)) + max_time_raw = tutorial_case.get('max_time', None) + if max_time_raw is not None and (not isinstance(max_time_raw, (int, float)) or max_time_raw <= 0): + raise ValueError(f"max_time must be a positive number, got {max_time_raw!r}") + max_times_of_tutorial[tutorial].append(max_time_raw) + mtw_raw = tutorial_case.get('max_time_windows', None) + if mtw_raw is not None and (not isinstance(mtw_raw, int) or mtw_raw <= 0): + raise ValueError(f"max_time_windows must be a positive integer, got {mtw_raw!r}") + max_time_windows_of_tutorial[tutorial].append(mtw_raw) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial)) + reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial)) return cls(testsuites)