diff --git a/doc/api/base_config.rst b/doc/api/base_config.rst new file mode 100644 index 000000000..d925014e8 --- /dev/null +++ b/doc/api/base_config.rst @@ -0,0 +1,13 @@ +.. _base_config: + +BaseConfig +========== + +.. currentmodule:: exasol.toolbox.config + +.. autoclass:: BaseConfig + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + :special-members: __init__ diff --git a/doc/api/index.rst b/doc/api/index.rst new file mode 100644 index 000000000..8ad43c961 --- /dev/null +++ b/doc/api/index.rst @@ -0,0 +1,11 @@ +.. _api: + +:octicon:`cpu` API Reference +============================= + +.. toctree:: + :maxdepth: 2 + + base_config + workflow_exceptions + workflow_patcher_config diff --git a/doc/api.rst b/doc/api/workflow_exceptions.rst similarity index 83% rename from doc/api.rst rename to doc/api/workflow_exceptions.rst index 9950235c8..de27c018a 100644 --- a/doc/api.rst +++ b/doc/api/workflow_exceptions.rst @@ -1,8 +1,3 @@ -.. _api: - -:octicon:`cpu` API Reference -============================= - .. _workflow_exceptions: Workflow Exceptions diff --git a/doc/api/workflow_patcher_config.rst b/doc/api/workflow_patcher_config.rst new file mode 100644 index 000000000..5935dd729 --- /dev/null +++ b/doc/api/workflow_patcher_config.rst @@ -0,0 +1,36 @@ +.. _workflow_patcher_config: + +WorkflowPatcherConfig +===================== + +.. currentmodule:: exasol.toolbox.util.workflows.patch_workflow + +.. autoclass:: WorkflowPatcherConfig + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + :special-members: __init__ + +.. autoclass:: Workflow + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + :special-members: __init__ + +.. autoclass:: StepCustomization + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + :special-members: __init__ + +.. autoclass:: StepContent + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + :special-members: __init__ + +.. autofunction:: validate_workflow_name diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 29462e6e1..c6b743452 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,19 +2,29 @@ ## Summary +In this major version: +* the Nox session `workflow:generate` has been added to replace the deprecated +`tbx workflow install` and `tbx workflow update`. It has the additional feature +that users may customize the PTB provided workflows with a `.workflow-patcher.yml` +file. +* the GitHub workflow templates have been modified to include step_ids and to follow +an AP-format naming convention, as such it is anticipated that updating the workflows +results in several small changes. + ## Feature * #691: Started customization of PTB workflows by defining the YML schema * #712: Added basic logging to workflow processing -* #714: Added logic to modify a workflow using the .workflow-patcher.yml -* #717: Restricted workflow names in .workflow-patcher.yml to template workflow names -* #719: Added nox session `workflow:generate` to generate/update workflows using the `.workflow-patcher.yml` (if desired) +* #714: Added logic to modify a workflow using the` .workflow-patcher.yml` +* #717: Restricted workflow names in `.workflow-patcher.yml` to template workflow names +* #719: Added Nox session `workflow:generate` to generate/update workflows using the `.workflow-patcher.yml` (if desired) * #725: Added newline after headlines for dependency changes ## Documentation * #705: Described how the versions of poetry and python are retrieved * #706: Added description how to ignore findings to the User Guide +* #721: Added documentation for Nox session `workflow:generate` ## Refactoring diff --git a/doc/index.rst b/doc/index.rst index bc00b76a2..48aa5aa29 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -52,5 +52,5 @@ Documentation of the Exasol-Toolbox developer_guide/developer_guide tools github_actions/github_actions - api + api/index changes/changelog diff --git a/doc/user_guide/configuration.rst b/doc/user_guide/configuration.rst index 0967c43f0..e7f5538ef 100644 --- a/doc/user_guide/configuration.rst +++ b/doc/user_guide/configuration.rst @@ -11,3 +11,4 @@ especially when starting a new project. features/metrics/sonar Formatting + features/github_workflows/github_project_configuration diff --git a/doc/user_guide/features/github_workflows/create_and_update.rst b/doc/user_guide/features/github_workflows/create_and_update.rst index 9aa3b7710..715796771 100644 --- a/doc/user_guide/features/github_workflows/create_and_update.rst +++ b/doc/user_guide/features/github_workflows/create_and_update.rst @@ -3,12 +3,12 @@ Creating and Updating the GitHub Workflows in Your Project ========================================================== -PTB can initially generate the GitHub workflows in your project and also +The PTB can initially generate the GitHub workflows in your project and also update existing workflows. The workflows are based on Jinja templates with variables populated by the -PTB. The PTB reads the values from various places within your project, see -:ref:`template_variables`. +PTB. The PTB reads the values from various attributes and properties of your +project's config, see :ref:`template_variables`. Please note that the PTB only updates the values in the GitHub workflows when *updating* the workflows. So, after updating the :ref:`list of Python versions @@ -36,8 +36,8 @@ Many workflows are using a Build-matrix to iterate over multiple versions of Python and/or the Exasol Docker DB. This is to make sure your code is valid, free of bugs and working correctly for each combination of these items. -PTB has a default for these versions, but you can override it in file -``noxconfig.py``, e.g. +The PTB has a default for these versions, but you can override it in the +``noxconfig.py`` file, e.g. .. code-block:: shell @@ -49,6 +49,35 @@ PTB has a default for these versions, but you can override it in file Some workflows are expected to not depend on a specific Python version and will use only the lowest Python version in the list specified above. +.. _customize_workflows: + +Customize Workflows for Your Project +------------------------------------ + +The PTB allows you to customise workflows by targeting specific jobs or steps: + +* Remove a job by its job_name. +* Replace a step (referenced by step_id) with one or more new steps. +* Insert steps after a specific step_id. + +.. note:: + + These operations do not currently cascade. For example, removing a job + without accounting for its downstream dependencies may result in errors. + You must manually adjust any subsequent steps that rely on the removed + job's or step's output. + +To utilize this feature, create a ``.workflow-patcher.yml`` file in your project's +root directory, as specified further in :ref:`workflow_patcher`. This file will be +automatically detected, validated by a pydantic model, and +used when you :ref:`install or update your workflows `. + +.. note:: + The pydantic validation checks that the yml file is in the expected format + and that the specified workflow names exist. However, when a workflow is being + generated, each removed job or modified step_id is checked to see if it exists. + If it does not exist, an exception will be raised (:ref:`workflow_exceptions`). + .. _update_workflows: Add all Workflows to Your Project @@ -56,11 +85,26 @@ Add all Workflows to Your Project .. code-block:: shell - tbx workflow install all + poetry run -- nox -s workflow:generate -- all .. warning:: - #. If you already have various workflows, you may want to run the - :code:`update` command instead of the :code:`install` command. + Some workflows depend on other workflows. Please ensure you have all + the required workflows if you do not install all of them. + +.. note:: + + The commands: + + * ``tbx workflow install all`` - used to install workflows + * ``tbx workflow update all`` - used to update workflows + + are considered historic variants of this command. + + **Deprecation Notice:** + These ``tbx`` endpoints are marked as **deprecated** and are scheduled for removal + by **April 22nd, 2026**. - #. Some workflows depend on other workflows. Please ensure you have all - the required workflows if you do not install all of them. + Please note that these legacy commands do not allow users to use their specified + ``.workflow-patcher.yml`` file to further customise or patch workflows. Users + should transition to the ``nox``-based command to leverage full customisation + features. diff --git a/doc/user_guide/features/github_workflows/github_project_configuration.rst b/doc/user_guide/features/github_workflows/github_project_configuration.rst index c962f2fb5..c3ed7eeab 100644 --- a/doc/user_guide/features/github_workflows/github_project_configuration.rst +++ b/doc/user_guide/features/github_workflows/github_project_configuration.rst @@ -7,8 +7,9 @@ Branch Protection ----------------- The best and most maintainable way to have solid branch protection -(:code:`Settings/Branches/main`) is to require the workflow :code:`CI / Allow -Merge` to pass successfully. +(:code:`Settings/Branches/main`) is to require the workflow :code:`merge-gate / Allow +Merge` to pass successfully. Additionally, if it makes sense for your project, +you can further require that ``SonarCloud Code Analysis`` passes. .. note:: Setting the required status checks to pass before merging is only possible @@ -17,7 +18,7 @@ Merge` to pass successfully. Manual Approval --------------- -If your CI workflow involves slow or expensive steps you can guard these to be +If your CI workflow involves slow or expensive steps, you can guard these to be executed only after manual approval. The CI workflow will automatically create a GitHub environment named :code:`manual-approval`. You only need to add reviewers in (:code:`Settings/Environments/manual-approval`) and move the @@ -27,11 +28,11 @@ file :code:`.github/workflows/merge-gate.yml`. Secrets ------- -For accessing specific services in the Internet, your project often needs a +For accessing specific services in the internet, your project often needs a related *token* or other credentials. These credentials can be acquired by registering on the service's website. -In many cases your company or organization may manage the credentials +In many cases, your company or organization may manage the credentials centrally and enable the use in multiple projects. The credentials can be managed as Secrets in GitHub and can be made accessible to GitHub projects and used by their workflows. diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 13a8b3d7c..6393a1b1b 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -10,6 +10,7 @@ GitHub Workflow Templates github_project_configuration create_and_update template_variables + workflow_patcher The PTB ships with configurable GitHub workflow templates covering the most common CI/CD setup variants for Python projects. The templates are defined in: @@ -20,13 +21,15 @@ workflows from the templates. .. code-block:: bash - poetry run -- tbx workflow --help + poetry run -- nox -s workflow:generate --help .. attention:: In most cases, we recommend using _all_ workflows without change to ensure consistent interdependencies. For more details, see :ref:`ci_actions`. + The deprecated alternate is to use the CLI provided by + ``poetry run -- tbx workflow --help``. This will be removed by April 22nd, 2026. Workflows --------- diff --git a/doc/user_guide/features/github_workflows/template_variables.rst b/doc/user_guide/features/github_workflows/template_variables.rst index f8355e3fa..f6ff6078b 100644 --- a/doc/user_guide/features/github_workflows/template_variables.rst +++ b/doc/user_guide/features/github_workflows/template_variables.rst @@ -6,9 +6,10 @@ Template Variables Underlying the CLI, the PTB uses Jinja to dynamically generate project-specific workflows. The rendering process is supported by the ``github_template_dict`` found in your ``noxconfig.py::PROJECT_CONFIG``. This dictionary is inherited by default from -``BaseConfig.py``, ensuring a standardized baseline that can be easily overridden, if -necessary. +:py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict`, ensuring a +standardized baseline that can be easily overridden, if necessary. .. literalinclude:: ../../../../exasol/toolbox/config.py :language: python :start-at: github_template_dict + :end-before: @computed_field diff --git a/doc/user_guide/features/github_workflows/workflow_patcher.rst b/doc/user_guide/features/github_workflows/workflow_patcher.rst new file mode 100644 index 000000000..d73adeca4 --- /dev/null +++ b/doc/user_guide/features/github_workflows/workflow_patcher.rst @@ -0,0 +1,57 @@ +.. _workflow_patcher: + +Workflow Patcher +================ + +Underlying the CLI, the PTB uses, if defined, a ``.workflow_patcher.yml`` file to +customize generated project-specific workflows. The rendering process is supported by +the ``github_workflow_patcher_yaml`` found in your ``noxconfig.py::PROJECT_CONFIG``. +This filepath is inherited by default from +:py:attr:`exasol.toolbox.config.BaseConfig.github_workflow_patcher_yaml` +ensuring a standardized baseline that can be easily overridden, if necessary. + +.. literalinclude:: ../../../../exasol/toolbox/config.py + :language: python + :start-at: github_workflow_patcher_yaml + +Model +------- + +.. code-block:: yaml + + workflows: + - name: pr-merge + remove_jobs: + - publish-docs + step_customizations: + - action: REPLACE | INSERT_AFTER + job: run-unit-tests + step_id: check-out-repository + content: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + +* ``name``: Name of the GitHub workflow to customize. The PTB supports these workflows: + `exasol/toolbox/templates/github/workflows `__. +* ``remove_jobs``: List of job names to remove from the workflow. + This is useful when a job like ``publish-docs`` is not applicable for a project. +* ``step_customizations``: List of customizations: + + * ``action``: Type of customization + + * ``REPLACE``: Replace an existing step with the new content + * ``INSERT_AFTER``: Insert the content **after** the specified step + + * ``job``: Name of the job inside the workflow that should be modified, e.g. ``run-unit-tests``. + * ``step_id``: ID of the step to replace or after which to insert the new step + * ``content``: Content of the new step. The PTB does not validate that this will work on + GitHub, but it does validate that it is valid YAML content. + +.. note:: + + For more information, see the API documentation for + :class:`exasol.toolbox.util.workflows.patch_workflow.WorkflowPatcherConfig`. diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index e0fdd3bfb..3cd02df43 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -44,7 +44,7 @@ def filter_not_specified_methods( ) -> tuple[str, ...]: """ Filter methods which were specified with a @hookimpl but where not specified - in `exasol.toolbox.nox.plugins.NoxTasks`. + in ``exasol.toolbox.nox.plugins.NoxTasks``. """ return tuple(name for name, _ in methods if name not in METHODS_SPECIFIED_FOR_HOOKS) @@ -52,7 +52,7 @@ def filter_not_specified_methods( def validate_plugin_hook(plugin_class: type[Any]): """ Validate methods in a class for at least one pluggy @hookimpl marker and verifies - that this method is also specified in `exasol.toolbox.nox.plugins.NoxTasks`. + that this method is also specified in ``exasol.toolbox.nox.plugins.NoxTasks``. """ methods_with_hook = get_methods_with_hook_implementation(plugin_class=plugin_class) @@ -154,17 +154,18 @@ class BaseConfig(BaseModel): This is used to extend the default excluded_python_paths. If a more general path that would be seen in other projects, like .venv, needs to be added into this argument, please instead modify the - `exasol.toolbox.config.DEFAULT_EXCLUDED_PATHS`. + ``exasol.toolbox.config.DEFAULT_EXCLUDED_PATHS``. """, ) plugins_for_nox_sessions: tuple[ValidPluginHook, ...] = Field( default=(), description=""" This is used to provide hooks to extend one or more of the Nox sessions provided - by the python-toolbox. As described on the plugins pages: - - https://exasol.github.io/python-toolbox/main/user_guide/customization.html#plugins - - https://exasol.github.io/python-toolbox/main/developer_guide/plugins.html, - possible plugin options are defined in `exasol.toolbox.nox.plugins.NoxTasks`. + by the python-toolbox. As described on the plugins pages:\n\n + * https://exasol.github.io/python-toolbox/main/user_guide/customization.html#plugins + * https://exasol.github.io/python-toolbox/main/developer_guide/plugins.html + + possible plugin options are defined in ``exasol.toolbox.nox.plugins.NoxTasks``. """, ) dependency_manager: DependencyManager = Field( @@ -172,8 +173,8 @@ class BaseConfig(BaseModel): description=""" This is used to define which dependency manager is used to install dependencies in the CI. For more details on which PTB version pairs with which - dependency manager, see: - https://exasol.github.io/python-toolbox/main/user_guide/dependencies.html + dependency manager, see:\n\n + * https://exasol.github.io/python-toolbox/main/user_guide/dependencies.html """, ) os_version: str = Field( @@ -199,10 +200,10 @@ def documentation_path(self) -> Path: @property def minimum_python_version(self) -> str: """ - Minimum Python version declared from the `python_versions` list + Minimum Python version declared from the ``python_versions`` list This is used in specific testing scenarios where it would be either - costly to run the tests for all `python_versions` or we need a single metric. + costly to run the tests for all ``python_versions`` or we need a single metric. """ versioned = [Version.from_string(v) for v in self.python_versions] min_version = min(versioned) @@ -219,9 +220,10 @@ def excluded_python_paths(self) -> tuple[str, ...]: - lint:code - lint:security - lint:typing + where it is desired to restrict which Python files are considered within the - PROJECT_CONFIG.source_code_path path, like excluding `dist`, `.eggs`. As such, - this property is used to exclude such undesired paths. + ``PROJECT_CONFIG.source_code_path`` path, like excluding ``dist``, ``.eggs``. + As such, this property is used to exclude such undesired paths. """ return tuple( sorted(DEFAULT_EXCLUDED_PATHS.union(set(self.add_to_excluded_python_paths))) @@ -244,7 +246,7 @@ def pyupgrade_argument(self) -> tuple[str, ...]: @property def sonar_code_path(self) -> Path: """ - Relative path needed in nox session `sonar:check` to create the coverage XML + Relative path needed in Nox session ``sonar:check`` to create the coverage XML. """ return self.source_code_path.relative_to(self.root_path) @@ -269,6 +271,9 @@ def version_filepath(self) -> Path: @computed_field # type: ignore[misc] @property def github_workflow_directory(self) -> Path: + """ + Path to the GitHub workflow directory. + """ return self.root_path / ".github" / "workflows" @computed_field # type: ignore[misc] @@ -290,12 +295,9 @@ def github_template_dict(self) -> dict[str, Any]: def github_workflow_patcher_yaml(self) -> Path | None: """ For customizing the GitHub workflow templates provided by the PTB, - a project can define a `.workflow-patcher.yml` file containing instructions to - delete or modify jobs in the PTB template. Modification includes replacing and - inserting steps. - - This feature is a work-in-progress that will be completed with: - https://github.com/exasol/python-toolbox/issues/690 + a project can define a ``.workflow-patcher.yml`` file containing + instructions to delete or modify jobs in the PTB template. + Modification includes replacing and inserting steps. """ workflow_patcher_yaml = self.root_path / ".workflow-patcher.yml" if workflow_patcher_yaml.exists(): diff --git a/exasol/toolbox/util/workflows/patch_workflow.py b/exasol/toolbox/util/workflows/patch_workflow.py index ad80f9865..322efeeec 100644 --- a/exasol/toolbox/util/workflows/patch_workflow.py +++ b/exasol/toolbox/util/workflows/patch_workflow.py @@ -32,8 +32,8 @@ class StepContent(BaseModel): The :class:`StepContent` is used to lightly validate the content which would be used to REPLACE or INSERT_AFTER the specified step in the GitHub workflow. - With the value `ConfigDict(extra="allow")`, this model allows for further fields - (e.g. `dummy`) to be specified without any validation. This design choice was + With the value ``ConfigDict(extra="allow")``, this model allows for further fields + (e.g. ``dummy``) to be specified without any validation. This design choice was intentional, as GitHub already allows additional fields and may specify more fields than what has been specified in this model. @@ -54,12 +54,13 @@ class StepContent(BaseModel): class StepCustomization(BaseModel): """ - The :class:`StepCustomization` is used to specify the desired modification: - * REPLACE - means that the contents of the specified `step_id` should be replaced - with whatever `content` is provided. - * INSERT_AFTER - means that the specified `content` should be inserted after - the specified `step_id`. - For a given step + The :class:`StepCustomization` is used to specify the desired modification. + An ``action`` of ``ActionType``: + + * ``REPLACE`` - means that the contents of the specified ``step_id`` should be + replaced with whatever ``content`` is provided. + * ``INSERT_AFTER`` - means that the specified `content` should be inserted after + the specified ``step_id``. """ action: ActionType @@ -69,6 +70,10 @@ class StepCustomization(BaseModel): def validate_workflow_name(workflow_name: str) -> str: + """ + Validates that the given ``workflow_name`` is a valid workflow name provided by + the PTB. + """ if workflow_name not in WORKFLOW_TEMPLATE_OPTIONS.keys(): raise ValueError( f"Invalid workflow: {workflow_name}. Must be one of {WORKFLOW_TEMPLATE_OPTIONS.keys()}" @@ -82,10 +87,12 @@ def validate_workflow_name(workflow_name: str) -> str: class Workflow(BaseModel): """ The :class:`Workflow` is used to specify which workflow should be modified. - This is determined by the workflow `name`. A workflow can be modified by specifying: - * `remove_jobs` - job names in this list will be removed from the workflow. - * `step_customization` - items in this list indicate which job's step - should be modified. + This is determined by the workflow ``name``. A workflow can be modified by + specifying: + + * ``remove_jobs`` - job names in this list will be removed from the workflow. + * ``step_customization`` - items in this list indicate which job's step + should be modified. """ name: WorkflowName @@ -95,9 +102,9 @@ class Workflow(BaseModel): class WorkflowPatcherConfig(BaseModel): """ - The :class:`WorkflowPatcherConfig` is used to validate the expected format for - the `.workflow-patcher.yml`, which is used to modify the workflow templates provided - by the PTB. + The :class:`WorkflowPatcherConfig` is used to validate the expected format + for the ``.workflow-patcher.yml``, which is used to modify the workflow + templates provided by the PTB. """ workflows: list[Workflow] diff --git a/test/integration/project-template/nox_test.py b/test/integration/project-template/nox_test.py index 5a0d55e66..d41d67248 100644 --- a/test/integration/project-template/nox_test.py +++ b/test/integration/project-template/nox_test.py @@ -64,3 +64,12 @@ def test_release_prepare(self, poetry_path, run_command): output = run_command(release_prepare) assert output.returncode == 0 + + def test_install_github_workflows(self, poetry_path, run_command): + install_workflows = self._command(poetry_path, task="workflow:generate", + add_ons=["all"]) + output = run_command(install_workflows) + assert output.returncode == 0 + + file_list = run_command(["ls", ".github/workflows"]).stdout.splitlines() + assert len(file_list) == 13