diff --git a/.ci/scripts/tests/test_lint_pyproject_dependencies.py b/.ci/scripts/tests/test_lint_pyproject_dependencies.py new file mode 100644 index 00000000000..cfbecb65202 --- /dev/null +++ b/.ci/scripts/tests/test_lint_pyproject_dependencies.py @@ -0,0 +1,114 @@ +import importlib.util +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +SCRIPT_PATH = REPO_ROOT / "scripts" / "lint_pyproject_dependencies.py" +spec = importlib.util.spec_from_file_location( + "lint_pyproject_dependencies", SCRIPT_PATH +) +assert spec is not None +assert spec.loader is not None +lint_pyproject_dependencies = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = lint_pyproject_dependencies +spec.loader.exec_module(lint_pyproject_dependencies) +find_direct_reference_dependencies = ( + lint_pyproject_dependencies.find_direct_reference_dependencies +) + + +class TestLintPyprojectDependencies(unittest.TestCase): + def write_pyproject(self, content: str) -> Path: + path = Path(self.tempdir.name) / "pyproject.toml" + path.write_text(textwrap.dedent(content)) + return path + + def setUp(self) -> None: + self.tempdir = tempfile.TemporaryDirectory() + + def tearDown(self) -> None: + self.tempdir.cleanup() + + def test_allows_versioned_dependencies(self) -> None: + path = self.write_pyproject( + """ + [project] + dependencies = [ + "torch>=2.12.0a0", + ] + + [project.optional-dependencies] + openvino = [ + "openvino>=2025.1.0,<2026.0.0; platform_system == 'Linux'", + ] + """ + ) + + self.assertEqual(find_direct_reference_dependencies(path), []) + + def test_rejects_project_direct_reference_dependency(self) -> None: + path = self.write_pyproject( + """ + [project] + dependencies = [ + "example @ git+https://github.com/example/example.git@abc123", + ] + """ + ) + + violations = find_direct_reference_dependencies(path) + self.assertEqual(len(violations), 1) + self.assertEqual(violations[0].section, "project.dependencies") + self.assertEqual( + violations[0].dependency, + "example @ git+https://github.com/example/example.git@abc123", + ) + + def test_rejects_direct_reference_dependency_with_spaced_extras(self) -> None: + path = self.write_pyproject( + """ + [project] + dependencies = [ + "example [dev] @ git+https://github.com/example/example.git@abc123", + ] + """ + ) + + violations = find_direct_reference_dependencies(path) + self.assertEqual(len(violations), 1) + self.assertEqual(violations[0].section, "project.dependencies") + self.assertEqual( + violations[0].dependency, + "example [dev] @ git+https://github.com/example/example.git@abc123", + ) + + def test_rejects_optional_direct_reference_dependency(self) -> None: + path = self.write_pyproject( + """ + [project] + dependencies = [] + + [project.optional-dependencies] + cortex_m = [ + "cmsis_nn @ git+https://github.com/ARM-software/CMSIS-NN.git@abc123", + ] + """ + ) + + violations = find_direct_reference_dependencies(path) + self.assertEqual(len(violations), 1) + self.assertEqual( + violations[0].section, + "project.optional-dependencies.cortex_m", + ) + self.assertEqual( + violations[0].dependency, + "cmsis_nn @ git+https://github.com/ARM-software/CMSIS-NN.git@abc123", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b21cc527b8d..a2b2aa429b8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,6 +20,22 @@ jobs: name: Get changed files uses: ./.github/workflows/_get-changed-files.yml + pyproject-metadata: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check pyproject metadata + run: python scripts/lint_pyproject_dependencies.py pyproject.toml + lintrunner-mypy: needs: [get-changed-files] runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index bb3beda32b1..8e1b2e874bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,10 +77,6 @@ dependencies=[ ] [project.optional-dependencies] -cortex_m = [ - # Keep this in sync with AoT deps from backends/cortex_m/requirements-cortex-m.txt - "cmsis_nn @ git+https://github.com/ARM-software/CMSIS-NN.git@d933672e7ca97eec70ef43230baee7b20c2a28ae", -] vgf = [ # AoT vgf dependencies # Keep this in sync with AoT deps from backends/arm/requirements-arm-vgf.txt and diff --git a/scripts/lint_pyproject_dependencies.py b/scripts/lint_pyproject_dependencies.py new file mode 100644 index 00000000000..985d17b6eb3 --- /dev/null +++ b/scripts/lint_pyproject_dependencies.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import argparse +import re +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +_DIRECT_REFERENCE = re.compile(r"^[A-Za-z0-9_.-]+(?:\s*\[[^\]]+\])?\s*@\s*\S+") + + +@dataclass(frozen=True) +class DirectReferenceDependency: + section: str + dependency: str + line: int + + +def _line_number_for_dependency(pyproject_text: str, dependency: str) -> int: + quoted_dependency = f'"{dependency}"' + for line_number, line in enumerate(pyproject_text.splitlines(), start=1): + if quoted_dependency in line or dependency in line: + return line_number + return 1 + + +def _record_if_direct_reference( + *, + pyproject_text: str, + section: str, + dependency: Any, + violations: list[DirectReferenceDependency], +) -> None: + if not isinstance(dependency, str): + return + normalized_dependency = dependency.strip() + if _DIRECT_REFERENCE.match(normalized_dependency): + violations.append( + DirectReferenceDependency( + section=section, + dependency=dependency, + line=_line_number_for_dependency(pyproject_text, dependency), + ) + ) + + +def find_direct_reference_dependencies( + pyproject_path: Path, +) -> list[DirectReferenceDependency]: + pyproject_text = pyproject_path.read_text() + pyproject = tomllib.loads(pyproject_text) + project = pyproject.get("project", {}) + + violations: list[DirectReferenceDependency] = [] + for dependency in project.get("dependencies", []): + _record_if_direct_reference( + pyproject_text=pyproject_text, + section="project.dependencies", + dependency=dependency, + violations=violations, + ) + + optional_dependencies = project.get("optional-dependencies", {}) + for extra, dependencies in optional_dependencies.items(): + for dependency in dependencies: + _record_if_direct_reference( + pyproject_text=pyproject_text, + section=f"project.optional-dependencies.{extra}", + dependency=dependency, + violations=violations, + ) + + return violations + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Reject direct URL dependencies in project metadata. PyPI rejects " + "packages that publish dependencies like 'pkg @ git+https://...'." + ) + ) + parser.add_argument( + "pyproject", + type=Path, + nargs="?", + default=Path("pyproject.toml"), + help="Path to pyproject.toml", + ) + args = parser.parse_args() + + violations = find_direct_reference_dependencies(args.pyproject) + for violation in violations: + print( + "::error " + f"file={args.pyproject},line={violation.line}," + "title=Direct URL dependency in pyproject.toml::" + f"{violation.section} contains '{violation.dependency}'. " + "PyPI rejects direct URL dependencies in published package " + "metadata; move it to a requirements file or install script.", + file=sys.stderr, + ) + + return 1 if violations else 0 + + +if __name__ == "__main__": + sys.exit(main())