diff --git a/script/dependency-parser/main.py b/script/dependency-parser/main.py index f345362b269..4f4dbe22edc 100644 --- a/script/dependency-parser/main.py +++ b/script/dependency-parser/main.py @@ -66,6 +66,9 @@ def main(): parser_test.add_argument( "--output", help="Output JSON file", default="tests_to_run.json" ) + parser_test.add_argument( + "--fixturemap", help="Optional path to file containing the test <-> gtest fixture mapping", default="" + ) # Code auditing parser_audit = subparsers.add_parser( @@ -95,6 +98,8 @@ def main(): filter_args.append("--all") if args.output: filter_args += ["--output", args.output] + if args.fixturemap: + filter_args += ["--fixturemap", args.fixturemap] run_selective_test_filter(filter_args) elif args.command == "audit": run_selective_test_filter([args.depmap_json, "--audit"]) diff --git a/script/dependency-parser/src/all_gtest_fixtures.py b/script/dependency-parser/src/all_gtest_fixtures.py new file mode 100644 index 00000000000..3f8dccd91e9 --- /dev/null +++ b/script/dependency-parser/src/all_gtest_fixtures.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import os +import stat +import subprocess +import json +import sys +from pathlib import Path + +def is_executable(file_path: Path) -> bool: + """Check if a file is an executable (not a directory).""" + return ( + file_path.is_file() + and os.access(file_path, os.X_OK) + and not file_path.name.startswith('.') # skip hidden files + ) + +def list_gtest_fixtures(executable: Path): + """Run the executable with --gtest_list_tests and return fixture names.""" + try: + # Run the command and capture output + result = subprocess.run( + [str(executable), "--gtest_list_tests"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10 # prevent hanging + ) + if result.returncode != 0: + print(f"Warning: {executable} returned non-zero exit code. Assuming it is not a gtest.", file=sys.stderr) + return [] + + fixtures = [] + for line in result.stdout.splitlines(): + if line.strip() and not line.startswith(" ") and line.endswith("."): # non-indented = fixture name + fixtures.append(line.strip()) + return fixtures + + except subprocess.TimeoutExpired: + print(f"Error: {executable} timed out", file=sys.stderr) + return [] + except Exception as e: + print(f"Error running {executable}: {e}", file=sys.stderr) + return [] + +def main(directory: str, output_file: str): + dir_path = Path(directory) + if not dir_path.is_dir(): + print(f"Error: {directory} is not a valid directory", file=sys.stderr) + sys.exit(1) + + results = {} + fixture_count = 0 + for file in dir_path.iterdir(): + if is_executable(file): + fixtures = list_gtest_fixtures(file) + if fixtures: +# src_file = file.name[5:] + ".cpp" # For MIOpen gtests, src_file.cpp <-> test_src_file + src_file = f"bin/{file.name}" # however, we're currently using the exe's name + results[src_file] = fixtures + fixture_count += len(fixtures) + + # Write results to JSON + try: + with open(output_file, "w", encoding="utf-8") as f: + json.dump(results, f, indent=4) + print(f"List of {fixture_count} fixtures from {len(results)} files written to {output_file}") + except Exception as e: + print(f"Error writing JSON file: {e}", file=sys.stderr) + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + main(sys.argv[1], sys.argv[2]) + diff --git a/script/dependency-parser/src/enhanced_ninja_parser.py b/script/dependency-parser/src/enhanced_ninja_parser.py index ebcd8789158..5d17f88c2d0 100644 --- a/script/dependency-parser/src/enhanced_ninja_parser.py +++ b/script/dependency-parser/src/enhanced_ninja_parser.py @@ -163,11 +163,20 @@ def _build_file_to_executable_mapping(self): """Build the final mapping from files to executables.""" print("Building file-to-executable mapping...") + # For monorepo, truncate the path before and including projects/ + self.project = None + rl_regex = rf"rocm-libraries[\\/]+projects[\\/]+([^\\/]+)[\\/]+(.*)" for exe, object_files in self.executable_to_objects.items(): for obj_file in object_files: # Add all dependencies of this object file if obj_file in self.object_to_all_deps: for dep_file in self.object_to_all_deps[obj_file]: + match = re.search(rl_regex, dep_file, re.IGNORECASE) + if match: + dep_file = match.group(2) + if not self.project: + print(f"Found rocm-libraries project: '{match.group(1)}'") + self.project = match.group(1) # Filter out system files and focus on project files if self._is_project_file(dep_file): self.file_to_executables[dep_file].add(exe) @@ -244,6 +253,10 @@ def export_to_json(self, output_file): exe_to_files[exe].add(file_path) mapping_data = { + "repo": { + "type": "monorepo" if self.project else "component", + "project": self.project + }, "file_to_executables": { file_path: list(exes) for file_path, exes in self.file_to_executables.items() diff --git a/script/dependency-parser/src/selective_test_filter.py b/script/dependency-parser/src/selective_test_filter.py index 83f7f7eebee..c2767f49c1a 100644 --- a/script/dependency-parser/src/selective_test_filter.py +++ b/script/dependency-parser/src/selective_test_filter.py @@ -31,7 +31,7 @@ import os -def get_changed_files(ref1, ref2): +def get_changed_files(ref1, ref2, project: str = None): """Return a set of files changed between two git refs.""" try: result = subprocess.run( @@ -40,7 +40,21 @@ def get_changed_files(ref1, ref2): text=True, check=True, ) - files = set(line.strip() for line in result.stdout.splitlines() if line.strip()) + + raw_files = set(line.strip() for line in result.stdout.splitlines() if line.strip()) + + if project is None: + files = raw_files + print(f"Identified {len(files)} modified files") + else: + root = f"projects/{project}/" + root_len = len(root) + files = set() + for f in raw_files: + if f.startswith(root): + files.add(f[root_len:]) + print(f"Identified {len(files)} files modified in project '{project}'") + return files except subprocess.CalledProcessError as e: print(f"Error running git diff: {e}") @@ -52,8 +66,18 @@ def load_depmap(depmap_json): with open(depmap_json, "r") as f: data = json.load(f) # Support both old and new formats + json_project = None + if "repo" in data and data["repo"]["type"] == "monorepo": + json_project = data["repo"]["project"] if "file_to_executables" in data: - return data["file_to_executables"] + return data["file_to_executables"], json_project + return data, json_project + + +def load_fixturemap(fixture_file): + """Load the dependency mapping JSON.""" + with open(fixture_file, "r") as f: + data = json.load(f) return data @@ -70,6 +94,23 @@ def select_tests(file_to_executables, changed_files, filter_mode): return sorted(affected) +def get_gtest_filter(tests, fixturemap): + """Maps the set of tests to be executed to a gtest_filter""" + gtest_filter = "" + fixture_count = 0 + for t in tests: + if t in fixturemap: + for f in fixturemap[t]: + gtest_filter += f + "*:" + fixture_count += 1 + else: + print(f"Warning: Diff references test {t}. However, it is not in the fixturemap") + if gtest_filter: + gtest_filter = gtest_filter[:-1] + print(f"Added {fixture_count} fixtures to gtest_filter") + return gtest_filter + + def main(): if "--audit" in sys.argv: if len(sys.argv) < 2: @@ -118,6 +159,7 @@ def main(): ref2 = sys.argv[3] filter_mode = "all" output_json = "tests_to_run.json" + fixture_file = "" if "--test-prefix" in sys.argv: filter_mode = "test_prefix" @@ -127,23 +169,35 @@ def main(): idx = sys.argv.index("--output") if idx + 1 < len(sys.argv): output_json = sys.argv[idx + 1] - + if "--fixturemap" in sys.argv: + idx = sys.argv.index("--fixturemap") + if idx + 1 < len(sys.argv): + fixture_file = sys.argv[idx + 1] if not os.path.exists(depmap_json): print(f"Dependency map JSON not found: {depmap_json}") sys.exit(1) - changed_files = get_changed_files(ref1, ref2) + file_to_executables, json_project = load_depmap(depmap_json) + changed_files = get_changed_files(ref1, ref2, json_project) if not changed_files: print("No changed files detected.") tests = [] else: - file_to_executables = load_depmap(depmap_json) tests = select_tests(file_to_executables, changed_files, filter_mode) + gtest_filter = "" + if tests and fixture_file and os.path.exists(fixture_file): + tests_to_fixtures = load_fixturemap(fixture_file) + gtest_filter = get_gtest_filter(tests, tests_to_fixtures) with open(output_json, "w") as f: - json.dump( - {"tests_to_run": tests, "changed_files": sorted(changed_files)}, f, indent=2 - ) + if gtest_filter: + json.dump( + {"tests_to_run": tests, "gtest_filter": gtest_filter, "changed_files": sorted(changed_files)}, f, indent=2 + ) + else: + json.dump( + {"tests_to_run": tests, "changed_files": sorted(changed_files)}, f, indent=2 + ) print(f"Exported {len(tests)} tests to run to {output_json}")