diff --git a/Makefile b/Makefile index a3d114f..866fc9e 100644 --- a/Makefile +++ b/Makefile @@ -25,10 +25,10 @@ poller-once: PYTHONPATH=src$${PYTHONPATH:+:$$PYTHONPATH} python -m kernel_ci_cloud_labs.pull_labs_poller --config $(CONFIG) --once install: build test - pip install -e . + python3.11 -m pip install -e . install-dev: - pip install -e ".[dev,test]" + python3.11 -m pip install -e ".[dev,test]" test: python -m pytest tests/ -m "not integration" diff --git a/QUICKSTART.md b/QUICKSTART.md index 87e194b..2854137 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -19,11 +19,11 @@ provision AWS resources on its own. Before continuing, make sure the pipeline can run a job end-to-end. The full walkthrough lives in [`README.md`](README.md); the minimum steps are: -1. **Install the package** in a venv — see +1. **Install the package** in a venv — Python 3.11 required, see [README → Installation](README.md#installation): ```bash - python3 -m venv .venv && source .venv/bin/activate - pip install -e . + python3.11 -m venv .venv && source .venv/bin/activate + python3.11 -m pip install -e . ``` 2. **Configure AWS credentials** — see [README 1. Configure AWS Credentials](README.md#1-configure-aws-credentials). diff --git a/README.md b/README.md index a2f65f7..26a24ff 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ This package provides the kernel-ci-cloud-runner python application as an entry ## Installation +**Python 3.11 is required.** Older interpreters (including the `python3` shipped +by Amazon Linux 2023, which is 3.9) are not supported — `pip install -e .` will +refuse to run on them. On AL2023, install `python3.11 python3.11-pip +python3.11-devel`; see [AWS_INSTALL.md](AWS_INSTALL.md) for the full host setup. + Run the package in a virtual environment. We also provide the script "tests/test-in-venv.sh" to wrap these steps in en environment. If you do not have the code already, get your copy (git URL to be defined). @@ -27,21 +32,21 @@ cd kernel-ci-cloud-labs Setup virtual environment: ```bash -# Create virtual environment -python3 -m venv .venv +# Create virtual environment (Python 3.11 required) +python3.11 -m venv .venv source .venv/bin/activate # On Windows: .venv\Scripts\activate # Install package (runtime only - boto3) -pip install -e . +python3.11 -m pip install -e . # Optional: Install with dev dependencies (pytest, black, pylint, pre-commit, pytest-cov) -pip install -e ".[dev]" +python3.11 -m pip install -e ".[dev]" # Optional: Install with analysis dependencies (pandas, matplotlib, seaborn) -pip install -e ".[analysis]" +python3.11 -m pip install -e ".[analysis]" # Recommended: Install everything (dev + analysis) -pip install -e ".[dev,analysis]" +python3.11 -m pip install -e ".[dev,analysis]" # Optional: Install pre-commit hooks (only if you plan to commit code) pre-commit install @@ -193,9 +198,9 @@ If not done already, set this up once: ```bash cd kernel-ci-cloud-labs -python3 -m venv .venv +python3.11 -m venv .venv source .venv/bin/activate -pip install -e . +python3.11 -m pip install -e . ``` Activate the virtual environment. After this, `kernel-ci-cloud-runner` is available in your shell. @@ -609,7 +614,7 @@ kernel-ci-cloud-runner aws analyze \ This downloads all `benchmark-*.csv` files from S3, combines them, compares the two kernel versions, and generates regression plots (overall, x86_64, ARM64) in `analysis/data/{run_prefix}/`. Add `--upload-analysis` to upload the results back to S3. -Requires the analysis dependencies: `pip install -e ".[analysis]"` +Requires the analysis dependencies: `python3.11 -m pip install -e ".[analysis]"` ## Automated Triggering via EventBridge diff --git a/pyproject.toml b/pyproject.toml index 84ebb8d..3562609 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 120 -target-version = ['py38', 'py39', 'py310', 'py311'] +target-version = ['py311'] include = '\.pyi?$' extend-exclude = ''' /( diff --git a/setup.py b/setup.py index 98be45f..8e39de9 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ long_description=long_description, package_dir={"": "src"}, packages=find_packages(where="src"), - python_requires=">=3.7", + python_requires=">=3.11", install_requires=[ "boto3>=1.26.0", ], diff --git a/src/kernel_ci_cloud_labs/pull_labs_poller.py b/src/kernel_ci_cloud_labs/pull_labs_poller.py index d748b5a..47366aa 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_poller.py +++ b/src/kernel_ci_cloud_labs/pull_labs_poller.py @@ -158,6 +158,46 @@ def write(self, timestamp: str) -> None: JobExecutor = Callable[[Dict[str, Any]], Tuple[List[Dict[str, Any]], Optional[str]]] +_DEFAULT_EXECUTOR_PACKAGES = ( + "kernel_ci_cloud_labs.providers", + "kernel_ci_cloud_labs.storage", + "kernel_ci_cloud_labs.auth", +) + + +def _validate_default_executor_deps() -> None: + """Eagerly import everything the default executor will need. + + Called from PullLabsPoller.__init__ when no custom job_executor is + supplied, so a missing runtime dep (boto3, an un-installed package) + surfaces at startup instead of on the first event hours later. Raises + SystemExit with a single combined message listing every problem found. + """ + from kernel_ci_cloud_labs.main import import_all_packages # noqa: PLC0415 + + problems: List[str] = [] + try: + import boto3 # noqa: F401,PLC0415 + except ImportError as e: + problems.append( + f"boto3 import failed ({e}) — run: python3.11 -m pip install -e ." + ) + + for pkg in _DEFAULT_EXECUTOR_PACKAGES: + try: + import_all_packages(pkg) + except ImportError as e: + problems.append(f"{pkg} import failed: {e}") + + if problems: + raise SystemExit( + "Default job executor dependencies are not installed:\n - " + + "\n - ".join(problems) + + "\nFix the install on this host, or pass a custom job_executor " + "to PullLabsPoller if you don't need the AWS pipeline." + ) + + def _default_job_executor(run_config: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], Optional[str]]: """Invoke the existing pipeline (provider-pluggable via registry). @@ -175,11 +215,7 @@ def _default_job_executor(run_config: Dict[str, Any]) -> Tuple[List[Dict[str, An ) from kernel_ci_cloud_labs.main import import_all_packages # noqa: PLC0415 - for pkg in [ - "kernel_ci_cloud_labs.providers", - "kernel_ci_cloud_labs.storage", - "kernel_ci_cloud_labs.auth", - ]: + for pkg in _DEFAULT_EXECUTOR_PACKAGES: import_all_packages(pkg) auth_class = AUTH_REGISTRY[run_config["auth_credentials"]["auth_provider"]] @@ -269,6 +305,9 @@ def __init__( self.job_executor: JobExecutor = job_executor or _default_job_executor self.base_config: Dict[str, Any] = config + if job_executor is None: + _validate_default_executor_deps() + # -- Credential resolution ------------------------------------------- @staticmethod diff --git a/tests/test_pull_labs_poller.py b/tests/test_pull_labs_poller.py index 2b323df..2962d31 100644 --- a/tests/test_pull_labs_poller.py +++ b/tests/test_pull_labs_poller.py @@ -12,6 +12,7 @@ import pytest +from kernel_ci_cloud_labs import pull_labs_poller as poller_mod from kernel_ci_cloud_labs.pull_labs_poller import ( DEFAULT_FROM_TIMESTAMP, FileCursorStore, @@ -20,6 +21,10 @@ _parse_kcidb_rest, ) +# Capture the real validator at import time so a specific test can restore it +# after the autouse fixture has stubbed it out. +_REAL_VALIDATE_DEFAULT_EXECUTOR_DEPS = poller_mod._validate_default_executor_deps + # --------------------------------------------------------------------------- # Helpers @@ -37,6 +42,17 @@ def _clear_kernelci_env(monkeypatch): monkeypatch.delenv(var, raising=False) +@pytest.fixture(autouse=True) +def _skip_default_executor_deps_check(monkeypatch): + """Bypass the boto3/AWS-package import check the poller runs at startup. + + The construction tests don't exercise the default executor and should not + depend on boto3 being installed in the test environment. Dedicated tests + for the validator itself temporarily restore the real function. + """ + monkeypatch.setattr(poller_mod, "_validate_default_executor_deps", lambda: None) + + def _minimal_kc(**overrides): base = { "api_base_uri": "https://api.example/latest", @@ -263,3 +279,46 @@ def test_empty_summary(self): rows, log = _extract_test_results({}) assert rows == [] assert log is None + + +# --------------------------------------------------------------------------- +# Default-executor dependency validation +# --------------------------------------------------------------------------- + + +class TestDefaultExecutorDepsValidation: + """Cover the startup check that runs when no custom job_executor is set.""" + + def test_missing_boto3_raises_systemexit(self, monkeypatch): + # Put back the real validator (autouse fixture stubbed it out). + monkeypatch.setattr( + poller_mod, + "_validate_default_executor_deps", + _REAL_VALIDATE_DEFAULT_EXECUTOR_DEPS, + ) + # Force the boto3 import inside the validator to fail. + import builtins + real_import = builtins.__import__ + + def _fail_boto3(name, *args, **kwargs): + if name == "boto3": + raise ImportError("boto3 not installed (simulated)") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _fail_boto3) + with pytest.raises(SystemExit) as ei: + PullLabsPoller(_minimal_kc()) + assert "boto3" in str(ei.value) + + def test_custom_executor_skips_validation(self, monkeypatch): + """Passing a custom executor must NOT trigger the boto3 check.""" + called = {"validator": False} + + def _fail_if_called(): + called["validator"] = True + raise SystemExit("validator should not have been called") + + monkeypatch.setattr(poller_mod, "_validate_default_executor_deps", _fail_if_called) + # Custom executor — validator must be skipped, no SystemExit. + PullLabsPoller(_minimal_kc(), job_executor=lambda cfg: ([], None)) + assert called["validator"] is False