From ba26a9eb5ba95a5c328ec6b5d86beb1971bd99bb Mon Sep 17 00:00:00 2001 From: Cody Oss Date: Mon, 29 Jun 2026 19:39:03 +0000 Subject: [PATCH 1/2] chore: remove legacylibrarian generator This project now uses release-please for all releasing which was the last usage of the legacylibrarian system. This code may now be deleted. Internal Bug: b/524618595 --- .generator/Dockerfile | 137 ---- .generator/cli.py | 1117 -------------------------- .generator/requirements-test.in | 21 - .generator/requirements.in | 6 - .generator/test_cli.py | 1293 ------------------------------- .github/workflows/generator.yml | 35 - cloudbuild-exitgate.yaml | 25 - cloudbuild-test.yaml | 37 - 8 files changed, 2671 deletions(-) delete mode 100644 .generator/Dockerfile delete mode 100644 .generator/cli.py delete mode 100644 .generator/requirements-test.in delete mode 100644 .generator/requirements.in delete mode 100644 .generator/test_cli.py delete mode 100644 .github/workflows/generator.yml delete mode 100644 cloudbuild-exitgate.yaml delete mode 100644 cloudbuild-test.yaml diff --git a/.generator/Dockerfile b/.generator/Dockerfile deleted file mode 100644 index ec4d67165010..000000000000 --- a/.generator/Dockerfile +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# --- Builder Stage --- -# This stage installs all build dependencies and compiles all Python versions. -FROM marketplace.gcr.io/google/ubuntu2404 AS builder - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - # Essential for compiling C code - build-essential \ - # For downloading and extracting secure files - git \ - wget \ - ca-certificates \ - unzip \ - # --- Critical libraries for a complete Python build --- - libssl-dev \ - zlib1g-dev \ - libbz2-dev \ - libffi-dev \ - libsqlite3-dev \ - libreadline-dev \ - # Needed for `google-cloud-bigquery-storage` to avoid - # the error `ModuleNotFoundError: No module named '_lzma'` - # described in https://github.com/googleapis/google-cloud-python/issues/14884 - liblzma-dev \ - # ------------------------------------------------------ - && apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV PYTHON_VERSION=3.14 - -# The full Python version, including the minor version, is needed for download/install -ENV PYTHON_VERSION_WITH_MINOR=3.14.2 - -# `make altinstall` is used to prevent replacing the system's default python binary. -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION_WITH_MINOR}/Python-${PYTHON_VERSION_WITH_MINOR}.tgz && \ - tar -xvf Python-${PYTHON_VERSION_WITH_MINOR}.tgz && \ - cd Python-${PYTHON_VERSION_WITH_MINOR} && \ - ./configure --enable-optimizations --prefix=/usr/local && \ - make -j$(nproc) && \ - make altinstall && \ - cd / && \ - rm -rf Python-${PYTHON_VERSION_WITH_MINOR}* - - -RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && \ - python${PYTHON_VERSION} /tmp/get-pip.py && \ - rm /tmp/get-pip.py - -# Download/extract protoc -RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-linux-x86_64.zip -RUN unzip protoc-25.3-linux-x86_64.zip -d protoc - -# Download/extract pandoc -# Pandoc is required by gapic-generator-python for parsing documentation -# version-scanner: ignore-next-line -ENV PANDOC_VERSION=3.8.2 -RUN mkdir pandoc-binary -RUN wget https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz -RUN tar -xvf pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz -C pandoc-binary --strip-components=1 - -# Pin synthtool for a more hermetic build -# This needs to be a single command so that the git clone command is not cached -RUN git clone https://github.com/googleapis/synthtool.git synthtool && \ - cd synthtool && \ - git checkout 96f416c959fbe8048200b6c16000de32b352902e - -# --- Final Stage --- -# This stage creates the lightweight final image, copying only the -# necessary artifacts from the builder stage. -FROM marketplace.gcr.io/google/ubuntu2404 - -# Tell synthtool to pull templates from this docker image instead of from -# the live repo. -ENV SYNTHTOOL_TEMPLATES="/synthtool/synthtool/gcp/templates" - -ENV PYTHON_VERSION_DEFAULT=3.14 - -# Install only the essential runtime libraries for Python. -# These are the non "-dev" versions of the libraries used in the builder. -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - # TODO(https://github.com/googleapis/google-cloud-python/issues/14992): Remove gdb - # Once this bug is fixed. - # Temporarily add gdb to assist with remote debugging for issue 14992. - gdb \ - # This is needed to avoid the following error: - # `ImportError: libsqlite3.so.0: cannot open shared object file: No such file or directory`. - # `libsqlite3-0` is used by the `coverage` PyPI package which is used when testing libraries - libsqlite3-0 \ - && apt-get clean autoclean \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* \ - && rm -f /var/cache/apt/archives/*.deb - -COPY --from=builder protoc/bin /usr/local/bin -COPY --from=builder protoc/include /usr/local/include - -COPY --from=builder pandoc-binary/bin /usr/local/bin -COPY --from=builder synthtool /synthtool - -COPY --from=builder /usr/local/bin/python${PYTHON_VERSION_DEFAULT} /usr/local/bin/ -COPY --from=builder /usr/local/lib/python${PYTHON_VERSION_DEFAULT} /usr/local/lib/python${PYTHON_VERSION_DEFAULT} - -# Set the working directory in the container. -WORKDIR /app - -# Install dependencies of the CLI such as click. -# Install gapic-generator which is used to generate libraries. -# Install nox which is used for running client library tests. -# Install starlark-pyo3 which is used to parse BUILD.bazel files. -COPY .generator/requirements.in . -RUN python${PYTHON_VERSION_DEFAULT} -m pip install -r requirements.in -RUN python${PYTHON_VERSION_DEFAULT} -m pip install /synthtool - -# Install build which is used to get the metadata of package config files. -COPY .generator/requirements.in . -RUN python${PYTHON_VERSION_DEFAULT} -m pip install -r requirements.in - -# Copy the CLI script into the container. -COPY .generator/cli.py . -RUN chmod a+rx ./cli.py - -ENTRYPOINT ["python3.14", "./cli.py"] diff --git a/.generator/cli.py b/.generator/cli.py deleted file mode 100644 index fda6cad91d16..000000000000 --- a/.generator/cli.py +++ /dev/null @@ -1,1117 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import itertools -import json -import logging -import os -import re -import shutil -import subprocess -import sys -import yaml -from datetime import date, datetime -from functools import lru_cache -from pathlib import Path -from typing import Dict, List -import build.util - - -logger = logging.getLogger() - -BUILD_REQUEST_FILE = "build-request.json" -CONFIGURE_REQUEST_FILE = "configure-request.json" -RELEASE_STAGE_REQUEST_FILE = "release-stage-request.json" -STATE_YAML_FILE = "state.yaml" - -INPUT_DIR = "input" -LIBRARIAN_DIR = "librarian" -OUTPUT_DIR = "output" -REPO_DIR = "repo" -SOURCE_DIR = "source" -_GITHUB_BASE = "https://github.com" - -_GENERATOR_INPUT_HEADER_TEXT = ( - "# DO NOT EDIT THIS FILE OUTSIDE OF `.librarian/generator-input`\n" - "# The source of truth for this file is `.librarian/generator-input`\n" -) - - -def _read_text_file(path: str) -> str: - """Helper function that reads a text file path and returns the content. - - Args: - path(str): The file path to read. - - Returns: - str: The contents of the file. - """ - - with open(path, "r") as f: - return f.read() - - -def _write_text_file(path: str, updated_content: str): - """Helper function that writes a text file path with the given content. - - Args: - path(str): The file path to write. - updated_content(str): The contents to write to the file. - """ - - os.makedirs(Path(path).parent, exist_ok=True) - with open(path, "w") as f: - f.write(updated_content) - - -def _read_json_file(path: str) -> Dict: - """Helper function that reads a json file path and returns the loaded json content. - - Args: - path(str): The file path to read. - - Returns: - dict: The parsed JSON content. - - Raises: - FileNotFoundError: If the file is not found at the specified path. - json.JSONDecodeError: If the file does not contain valid JSON. - IOError: If there is an issue reading the file. - """ - with open(path, "r") as f: - return json.load(f) - - -def _write_json_file(path: str, updated_content: Dict): - """Helper function that writes a json file with the given dictionary. - - Args: - path(str): The file path to write. - updated_content(Dict): The dictionary to write. - """ - - with open(path, "w") as f: - json.dump(updated_content, f, indent=2) - f.write("\n") - - -def _add_new_library_source_roots(library_config: Dict, library_id: str) -> None: - """Adds the default source_roots to the library configuration if not present. - - Args: - library_config(Dict): The library configuration. - library_id(str): The id of the library. - """ - if library_config["source_roots"] is None: - library_config["source_roots"] = [f"packages/{library_id}"] - - -def _add_new_library_preserve_regex(library_config: Dict, library_id: str) -> None: - """Adds the default preserve_regex to the library configuration if not present. - - Args: - library_config(Dict): The library configuration. - library_id(str): The id of the library. - """ - if library_config["preserve_regex"] is None: - library_config["preserve_regex"] = [ - f"packages/{library_id}/CHANGELOG.md", - "docs/CHANGELOG.md", - "samples/README.txt", - "scripts/client-post-processing", - "samples/snippets/README.rst", - "tests/system", - ] - - -def _add_new_library_remove_regex(library_config: Dict, library_id: str) -> None: - """Adds the default remove_regex to the library configuration if not present. - - Args: - library_config(Dict): The library configuration. - library_id(str): The id of the library. - """ - if library_config["remove_regex"] is None: - library_config["remove_regex"] = [f"packages/{library_id}"] - - -def _add_new_library_tag_format(library_config: Dict) -> None: - """Adds the default tag_format to the library configuration if not present. - - Args: - library_config(Dict): The library configuration. - """ - if "tag_format" not in library_config: - library_config["tag_format"] = "{id}-v{version}" - - -def _get_new_library_config(request_data: Dict) -> Dict: - """Finds and returns the configuration for a new library. - - Args: - request_data(Dict): The request data from which to extract the new - library config. - - Returns: - Dict: The unmodified configuration of a new library, or an empty - dictionary if not found. - """ - for library_config in request_data.get("libraries", []): - all_apis = library_config.get("apis", []) - for api in all_apis: - if api.get("status") == "new": - return library_config - return {} - - -def _add_new_library_version(library_config: Dict) -> None: - """Adds the library version to the configuration if it's not present. - - Args: - library_config(Dict): The library configuration. - """ - if "version" not in library_config or not library_config["version"]: - library_config["version"] = "0.0.0" - - -def _prepare_new_library_config(library_config: Dict) -> Dict: - """ - Prepares the new library's configuration by removing temporary keys and - adding default values. - - Args: - library_config (Dict): The raw library configuration. - - Returns: - Dict: The prepared library configuration. - """ - # remove status key from new library config. - all_apis = library_config.get("apis", []) - for api in all_apis: - if "status" in api: - del api["status"] - - library_id = _get_library_id(library_config) - _add_new_library_source_roots(library_config, library_id) - _add_new_library_preserve_regex(library_config, library_id) - _add_new_library_remove_regex(library_config, library_id) - _add_new_library_tag_format(library_config) - _add_new_library_version(library_config) - - return library_config - - -def _create_new_changelog_for_library(library_id: str, output: str): - """Creates a new changelog for the library. - Args: - library_id(str): The id of the library. - output(str): Path to the directory in the container where code - should be generated. - """ - package_changelog_path = f"{output}/packages/{library_id}/CHANGELOG.md" - docs_changelog_path = f"{output}/packages/{library_id}/docs/CHANGELOG.md" - - changelog_content = f"# Changelog\n\n[PyPI History][1]\n\n[1]: https://pypi.org/project/{library_id}/#history\n" - - os.makedirs(os.path.dirname(package_changelog_path), exist_ok=True) - _write_text_file(package_changelog_path, changelog_content) - - os.makedirs(os.path.dirname(docs_changelog_path), exist_ok=True) - _write_text_file(docs_changelog_path, changelog_content) - - -def handle_configure( - librarian: str = LIBRARIAN_DIR, - source: str = SOURCE_DIR, - repo: str = REPO_DIR, - input: str = INPUT_DIR, - output: str = OUTPUT_DIR, -): - """Onboards a new library by completing its configuration. - - This function reads a partial library definition from `configure-request.json`, - fills in missing fields like the version, source roots, and preservation - rules, and writes the complete configuration to `configure-response.json`. - It ensures that new libraries conform to the repository's standard structure. - - See https://github.com/googleapis/librarian/blob/main/doc/container-contract.md#configure-container-command - - Args: - librarian(str): Path to the directory in the container which contains - the librarian configuration. - source(str): Path to the directory in the container which contains - API protos. - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - input(str): The path to the directory in the container - which contains additional generator input. - output(str): Path to the directory in the container where code - should be generated. - - Raises: - ValueError: If configuring a new library fails. - """ - try: - # configure-request.json contains the library definitions. - request_data = _read_json_file(f"{librarian}/{CONFIGURE_REQUEST_FILE}") - new_library_config = _get_new_library_config(request_data) - - _update_global_changelog( - f"{repo}/CHANGELOG.md", - f"{output}/CHANGELOG.md", - [new_library_config], - ) - prepared_config = _prepare_new_library_config(new_library_config) - - is_mono_repo = _is_mono_repo(input) - library_id = _get_library_id(prepared_config) - path_to_library = f"packages/{library_id}" if is_mono_repo else "." - if not Path(f"{repo}/{path_to_library}").exists(): - # Create a `CHANGELOG.md` and `docs/CHANGELOG.md` file for the new library - _create_new_changelog_for_library(library_id, output) - - # Write the new library configuration to configure-response.json. - _write_json_file(f"{librarian}/configure-response.json", prepared_config) - - except Exception as e: - raise ValueError("Configuring a new library failed.") from e - logger.info("'configure' command executed.") - - -def _get_library_id(request_data: Dict) -> str: - """Retrieve the library id from the given request dictionary - - Args: - request_data(Dict): The contents `generate-request.json`. - - Raises: - ValueError: If the key `id` does not exist in `request_data`. - - Returns: - str: The id of the library in `generate-request.json` - """ - library_id = request_data.get("id") - if not library_id: - raise ValueError("Request file is missing required 'id' field.") - return library_id - - - -def _get_repo_metadata_file_path(base: str, library_id: str, is_mono_repo: bool): - """Constructs the full path to the .repo-metadata.json file. - - Args: - base (str): The base directory where the library is located. - library_id (str): The ID of the library. - is_mono_repo (bool): True if the current repository is a mono-repo. - - Returns: - str: The absolute path to the .repo-metadata.json file. - """ - path_to_library = f"packages/{library_id}" if is_mono_repo else "." - return f"{base}/{path_to_library}/.repo-metadata.json" - - -@lru_cache(maxsize=None) -def _get_repo_name_from_repo_metadata(base: str, library_id: str, is_mono_repo: bool): - """Retrieves the repository name from the .repo-metadata.json file. - - This function is cached to avoid redundant file I/O. - - Args: - base (str): The base directory where the library is located. - library_id (str): The ID of the library. - is_mono_repo (bool): True if the current repository is a mono-repo. - - Returns: - str: The name of the repository (e.g., 'googleapis/google-cloud-python'). - - Raises: - ValueError: If the '.repo-metadata.json' file is missing the 'repo' field. - """ - if is_mono_repo: - return "googleapis/google-cloud-python" - file_path = _get_repo_metadata_file_path(base, library_id, is_mono_repo) - repo_metadata = _read_json_file(file_path) - repo_name = repo_metadata.get("repo") - if not repo_name: - raise ValueError("`.repo-metadata.json` file is missing required 'repo' field.") - return repo_name - - - -def _run_nox_sessions(library_id: str, repo: str, is_mono_repo: bool): - """Calls nox for all specified sessions. - - Args: - library_id(str): The library id under test. - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global files declared in - the config.yaml. - is_mono_repo(bool): True if the current repository is a mono-repo. - """ - session_runtime = "3.14" - # TODO(https://github.com/googleapis/google-cloud-python/issues/14992): Switch the protobuf - # implementation back to upb once we identify the root cause of the crash that occurs during testing. - # It's not trivial to debug this since it only happens in cloud build. - sessions = [ - f"unit-{session_runtime}(protobuf_implementation='python')", - ] - current_session = None - try: - for nox_session in sessions: - current_session = nox_session - _run_individual_session(nox_session, library_id, repo, is_mono_repo) - - except Exception as e: - raise ValueError(f"Failed to run the nox session: {current_session}") from e - - -def _run_individual_session( - nox_session: str, library_id: str, repo: str, is_mono_repo: bool -): - """ - Calls nox with the specified sessions. - - Args: - nox_session(str): The nox session to run. - library_id(str): The library id under test. - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - is_mono_repo(bool): True if the current repository is a mono-repo. - """ - - if is_mono_repo: - path_to_library = f"packages/{library_id}" - library_path = f"{repo}/{path_to_library}" - else: - library_path = repo - command = [ - "nox", - "-s", - nox_session, - "-f", - f"{library_path}/noxfile.py", - ] - # TODO(#14992): Revert to 600 seconds (10 minutes) after debugging is complete. - result = subprocess.run(command, text=True, check=True, timeout=1200) - logger.info(result) - - -def _determine_library_namespace( - gapic_parent_path: Path, package_root_path: Path -) -> str: - """ - Determines the namespace from the gapic file's parent path relative - to its package root. - - Args: - gapic_parent_path (Path): The absolute path to the directory containing - gapic_version.py (e.g., .../google/cloud/language). - package_root_path (Path): The absolute path to the root of the package - (e.g., .../packages/google-cloud-language). - """ - # This robustly calculates the relative path, e.g., "google/cloud/language" - relative_path = gapic_parent_path.relative_to(package_root_path) - - # relative_path.parts will be like: ('google', 'cloud', 'language') - # We want all parts *except* the last one (the service dir) to form the namespace. - namespace_parts = relative_path.parts[:-1] - - if not namespace_parts and relative_path.parts: - # This handles the edge case where the parts are just ('google',). - # This implies the namespace is just "google". - return ".".join(relative_path.parts) - - return ".".join(namespace_parts) - - -def _verify_library_namespace(library_id: str, repo: str, is_mono_repo: bool): - """ - Verifies that all found package namespaces are one of - the hardcoded `exception_namespaces` or - `valid_namespaces`. - - Args: - library_id (str): The library id under test (e.g., "google-cloud-language"). - repo (str): The path to the root of the repository. - is_mono_repo(bool): True if the current repository is a mono-repo. - """ - # TODO(https://github.com/googleapis/google-cloud-python/issues/14376): Update the list of namespaces which are exceptions. - exception_namespaces = [ - "google.area120", - "google.api", - "google.apps.script", - "google.apps.script.type", - "google.cloud.alloydb", - "google.cloud.billing", - "google.cloud.devtools", - "google.cloud.gkeconnect", - "google.cloud.gkehub_v1", - "google.cloud.orchestration.airflow", - "google.cloud.orgpolicy", - "google.cloud.security", - "google.cloud.video", - "google.cloud.workflows", - "google.iam", - "google.gapic", - "google.identity.accesscontextmanager", - "google.logging", - "google.monitoring", - "google.rpc", - ] - valid_namespaces = [ - "google", - "google.ads", - "google.ai", - "google.analytics", - "google.apps", - "google.cloud", - "google.geo", - "google.maps", - "google.pubsub", - "google.shopping", - "grafeas", - *exception_namespaces, - ] - gapic_version_file = "gapic_version.py" - proto_file = "*.proto" - - if is_mono_repo: - path_to_library = f"packages/{library_id}" - library_path = Path(f"{repo}/{path_to_library}") - else: - library_path = Path(repo) - - if not library_path.is_dir(): - raise ValueError(f"Error: Path is not a directory: {library_path}") - - # Use a set to store unique parent directories of relevant directories - relevant_dirs = set() - - # Find all parent directories for 'gapic_version.py' files - for gapic_file in library_path.rglob(gapic_version_file): - relevant_dirs.add(gapic_file.parent) - - # Find all parent directories for '*.proto' files - for proto_file in library_path.rglob(proto_file): - proto_path = str(proto_file.parent.relative_to(library_path)) - # Exclude proto paths which are not intended to be used for code generation. - # Generally any protos under the `samples` or `tests` directories or in a - # directory called `proto` are not used for code generation. - if ( - proto_path.startswith("tests") - or proto_path.startswith("samples") - or proto_path.endswith("proto") - ): - continue - relevant_dirs.add(proto_file.parent) - - if not relevant_dirs: - raise ValueError( - f"Error: namespace cannot be determined for {library_id}." - f" Library is missing a `{gapic_version_file}` or `{proto_file}` file." - ) - - for relevant_dir in relevant_dirs: - library_namespace = _determine_library_namespace(relevant_dir, library_path) - - if library_namespace not in valid_namespaces: - raise ValueError( - f"The namespace `{library_namespace}` for `{library_id}` must be one of {valid_namespaces}." - ) - - -def _get_library_dist_name(library_id: str, repo: str, is_mono_repo: bool) -> str: - """ - Gets the package name by programmatically building the metadata. - - Args: - library_id: id of the library. - repo: This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - is_mono_repo(bool): True if the current repository is a mono-repo. - Returns: - str: The library name string if found, otherwise None. - """ - if is_mono_repo: - path_to_library = f"packages/{library_id}" - library_path = Path(f"{repo}/{path_to_library}") - else: - library_path = Path(repo) - metadata = build.util.project_wheel_metadata(library_path) - return metadata.get("name") - - -def _verify_library_dist_name(library_id: str, repo: str, is_mono_repo: bool): - """Verifies the library distribution name against its config files. - - This function ensures that: - 1. At least one of `setup.py` or `pyproject.toml` exists and is valid. - 2. Any existing config file's 'name' property matches the `library_id`. - - Args: - library_id: id of the library. - repo: This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - is_mono_repo(bool): True if the current repository is a mono-repo. - - Raises: - ValueError: If a name in an existing config file does not match the `library_id`. - """ - dist_name = _get_library_dist_name(library_id, repo, is_mono_repo) - if dist_name != library_id: - raise ValueError( - f"The distribution name `{dist_name}` does not match the folder `{library_id}`." - ) - - -def handle_build(librarian: str = LIBRARIAN_DIR, repo: str = REPO_DIR): - """The main coordinator for validating client library generation.""" - try: - is_mono_repo = _is_mono_repo(repo) - request_data = _read_json_file(f"{librarian}/{BUILD_REQUEST_FILE}") - library_id = _get_library_id(request_data) - _verify_library_namespace(library_id, repo, is_mono_repo) - _verify_library_dist_name(library_id, repo, is_mono_repo) - _run_nox_sessions(library_id, repo, is_mono_repo) - except Exception as e: - raise ValueError("Build failed.") from e - - logger.info("'build' command executed.") - - -def _get_libraries_to_prepare_for_release(library_entries: Dict) -> List[dict]: - """Get libraries which should be prepared for release. Only libraries - which have the `release_triggered` field set to `True` will be returned. - - Args: - library_entries(Dict): Dictionary containing all of the libraries to - evaluate. - - Returns: - List[dict]: List of all libraries which should be prepared for release, - along with the corresponding information for the release. - """ - return [ - library - for library in library_entries["libraries"] - if library.get("release_triggered") - ] - - -def _update_global_changelog( - changelog_src: str, changelog_dest: str, all_libraries: List[dict] -): - """Updates the versions of libraries in the main CHANGELOG.md. - - Args: - changelog_src(str): Path to the changelog file to read. - changelog_dest(str): Path to the changelog file to write. - all_libraries(Dict): Dictionary containing all of the library versions to - modify. - """ - - def replace_version_in_changelog(content): - new_content = content - for library in all_libraries: - library_id = library["id"] - version = library["version"] - # Find the entry for the given library in the format`==` - # Replace the `` part of the string. - pattern = re.compile(f"(\\[{re.escape(library_id)})(==)([\\d\\.]+)(\\])") - replacement = f"\\g<1>=={version}\\g<4>" - new_content = pattern.sub(replacement, new_content) - return new_content - - updated_content = replace_version_in_changelog(_read_text_file(changelog_src)) - _write_text_file(changelog_dest, updated_content) - - -def _process_version_file(content, version, version_path) -> str: - """This function searches for a version string in the - given content, replaces the version and returns the content. - - Args: - content(str): The contents where the version string should be replaced. - version(str): The new version of the library. - version_path(str): The relative path to the version file - - Raises: ValueError if the version string could not be found in the given content - - Returns: A string with the modified content. - """ - if version_path.name.endswith("gapic_version.py") or version_path.name.endswith( - "version.py" - ): - pattern = r"(__version__\s*=\s*[\"'])([^\"']+)([\"'].*)" - else: - pattern = r"(version\s*=\s*[\"'])([^\"']+)([\"'].*)" - replacement_string = f"\\g<1>{version}\\g<3>" - new_content, num_replacements = re.subn(pattern, replacement_string, content) - if num_replacements == 0: - raise ValueError( - f"Could not find version string in {version_path}. File was not modified." - ) - - # Optionally update the `__release_date__` date string, if it exists, in the format YYYY-MM-DD - date_pattern = r"(__release_date__\s*=\s*[\"'])([^\"']+)([\"'].*)" - today_iso = date.today().isoformat() # Get today's date in YYYY-MM-DD format - date_replacement_string = f"\\g<1>{today_iso}\\g<3>" - new_content, _ = re.subn(date_pattern, date_replacement_string, new_content) - return new_content - - -def _update_version_for_library( - repo: str, output: str, path_to_library: str, version: str -): - """Updates the version string in `**/gapic_version.py`, `**/version.py`, `setup.py`, - `pyproject.toml` and `samples/**/snippet_metadata.json` for a - given library, if applicable. - - Args: - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - output(str): Path to the directory in the container where modified - code should be placed. - path_to_library(str): Relative path to the library to update - version(str): The new version of the library - - Raises: `ValueError` if a version string could not be located in `**/gapic_version.py` - or `**/version.py` within the given library. - """ - - # Find and update version.py or gapic_version.py files - search_base = Path(f"{repo}/{path_to_library}") - version_files = [] - patterns = ["**/gapic_version.py", "**/version.py"] - excluded_dirs = { - ".nox", - ".venv", - "venv", - "site-packages", - ".git", - "build", - "dist", - "__pycache__", - "tests", - } - for pattern in patterns: - version_files.extend( - [ - p - for p in search_base.rglob(pattern) - if not any(part in excluded_dirs for part in p.parts) - ] - ) - - if not version_files: - # Fallback to `pyproject.toml`` or `setup.py``. Proto-only libraries have - # version information in `setup.py` or `pyproject.toml` instead of `gapic_version.py`. - pyproject_toml = Path(f"{repo}/{path_to_library}/pyproject.toml") - setup_py = Path(f"{repo}/{path_to_library}/setup.py") - version_files = [pyproject_toml if pyproject_toml.exists() else setup_py] - - for version_file in version_files: - # Do not process version files in the types directory as some - # GAPIC libraries have `version.py` which are generated from - # `version.proto` and do not include SDK versions. - if version_file.parent.name == "types": - continue - updated_content = _process_version_file( - _read_text_file(version_file), version, version_file - ) - output_path = f"{output}/{version_file.relative_to(repo)}" - _write_text_file(output_path, updated_content) - - # Find and update snippet_metadata.json files - snippet_metadata_files = Path(f"{repo}/{path_to_library}/samples").rglob( - "**/*snippet*.json" - ) - for metadata_file in snippet_metadata_files: - output_path = f"{output}/{metadata_file.relative_to(repo)}" - os.makedirs(Path(output_path).parent, exist_ok=True) - shutil.copy(metadata_file, output_path) - - metadata_contents = _read_json_file(metadata_file) - metadata_contents["clientLibrary"]["version"] = version - _write_json_file(output_path, metadata_contents) - - -def _get_previous_version(library_id: str, librarian: str) -> str: - """Gets the previous version of the library from state.yaml. - - Args: - library_id(str): id of the library. - librarian(str): Path to the directory in the container which contains - the `state.yaml` file. - - Returns: - str: The version for a given library in state.yaml - """ - state_yaml_path = f"{librarian}/{STATE_YAML_FILE}" - - with open(state_yaml_path, "r") as state_yaml_file: - state_yaml = yaml.safe_load(state_yaml_file) - for library in state_yaml.get("libraries", []): - if library.get("id") == library_id: - return library.get("version") - - raise ValueError( - f"Could not determine previous version for {library_id} from state.yaml" - ) - - -def _create_main_version_header( - version: str, - previous_version: str, - library_id: str, - repo_name: str, - tag_format: str, -) -> str: - """This function creates a header to be used in a changelog. The header has the following format: - `## [{version}](https://github.com/googleapis/google-cloud-python/compare/{tag_format}{previous_version}...{tag_format}{version}) (YYYY-MM-DD)` - - Args: - version(str): The new version of the library. - previous_version(str): The previous version of the library. - library_id(str): The id of the library where the changelog should - be updated. - repo_name(str): The name of the repository (e.g., 'googleapis/google-cloud-python'). - tag_format(str): The format of the git tag. - - Returns: - A header to be used in the changelog. - """ - current_date = datetime.now().strftime("%Y-%m-%d") - - # We will assume that version is always at the end of the tag. - tag_format = tag_format.replace("{version}", "") - - if "{id}" in tag_format: - tag_format = tag_format.format(**{"id": library_id}) - - # Return the main version header - return ( - f"## [{version}]({_GITHUB_BASE}/{repo_name}/compare/{tag_format}{previous_version}" - f"...{tag_format}{version}) ({current_date})" - ) - - -def _process_changelog( - content: str, - library_changes: List[Dict], - version: str, - previous_version: str, - library_id: str, - repo_name: str, - tag_format: str, -): - """This function searches the given content for the anchor pattern - `[1]: https://pypi.org/project/{library_id}/#history` - and adds an entry in the following format: - - ## [{version}](https://github.com/googleapis/google-cloud-python/compare/{tag_format}{previous_version}...{tag_format}{version}) (YYYY-MM-DD) - - ### Documentation - - * Update import statement example in README ([868b006](https://github.com/googleapis/google-cloud-python/commit/868b0069baf1a4bf6705986e0b6885419b35cdcc)) - - Args: - content(str): The contents of an existing changelog. - library_changes(List[Dict]): List of dictionaries containing the changes - for a given library. - version(str): The new version of the library. - previous_version(str): The previous version of the library. - library_id(str): The id of the library where the changelog should - be updated. - repo_name(str): The name of the repository (e.g., 'googleapis/google-cloud-python'). - tag_format(str): The format of the git tag. - - Raises: ValueError if the anchor pattern string could not be found in the given content - - Returns: A string with the modified content. - """ - entry_parts = [] - entry_parts.append( - _create_main_version_header( - version=version, - previous_version=previous_version, - library_id=library_id, - repo_name=repo_name, - tag_format=tag_format, - ) - ) - - # Group changes by type (e.g., feat, fix, docs) - type_key = "type" - commit_hash_key = "commit_hash" - subject_key = "subject" - library_changes.sort(key=lambda x: x[type_key]) - grouped_changes = itertools.groupby(library_changes, key=lambda x: x[type_key]) - - change_type_map = { - "feat": "Features", - "fix": "Bug Fixes", - "docs": "Documentation", - } - for library_change_type, library_changes in grouped_changes: - # We only care about feat, fix, docs - adjusted_change_type = library_change_type.replace("!", "") - if adjusted_change_type in change_type_map: - entry_parts.append(f"\n\n### {change_type_map[adjusted_change_type]}\n") - for change in library_changes: - commit_link = f"([{change[commit_hash_key]}]({_GITHUB_BASE}/{repo_name}/commit/{change[commit_hash_key]}))" - entry_parts.append(f"* {change[subject_key]} {commit_link}") - - new_entry_text = "\n".join(entry_parts) - anchor_pattern = re.compile( - rf"(\[1\]: https://pypi\.org/project/{library_id}/#history)", - re.MULTILINE, - ) - replacement_text = f"\\g<1>\n\n{new_entry_text}" - updated_content, num_subs = anchor_pattern.subn(replacement_text, content, count=1) - if num_subs == 0: - raise ValueError("Changelog anchor '[1]: ...#history' not found.") - - return updated_content - - -def _update_changelog_for_library( - repo: str, - output: str, - library_changes: List[Dict], - version: str, - previous_version: str, - library_id: str, - is_mono_repo: bool, - tag_format: str, -): - """Prepends a new release entry with multiple, grouped changes, to a changelog. - - Args: - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - output(str): Path to the directory in the container where modified - code should be placed. - library_changes(List[Dict]): List of dictionaries containing the changes - for a given library - version(str): The desired version - previous_version(str): The version in state.yaml for a given library - library_id(str): The id of the library where the changelog should - be updated. - is_mono_repo(bool): True if the current repository is a mono-repo. - tag_format(str): The format of the git tag. - """ - if is_mono_repo: - relative_path = f"packages/{library_id}/CHANGELOG.md" - docs_relative_path = f"packages/{library_id}/docs/CHANGELOG.md" - else: - relative_path = "CHANGELOG.md" - docs_relative_path = f"docs/CHANGELOG.md" - - changelog_src = f"{repo}/{relative_path}" - changelog_dest = f"{output}/{relative_path}" - repo_name = _get_repo_name_from_repo_metadata(repo, library_id, is_mono_repo) - updated_content = _process_changelog( - _read_text_file(changelog_src), - library_changes, - version, - previous_version, - library_id, - repo_name, - tag_format, - ) - _write_text_file(changelog_dest, updated_content) - - docs_changelog_src = f"{repo}/{docs_relative_path}" - if os.path.lexists(docs_changelog_src): - docs_changelog_dst = f"{output}/{docs_relative_path}" - _write_text_file(docs_changelog_dst, updated_content) - - -def _is_mono_repo(repo: str) -> bool: - """Determines if a library is generated or handwritten. - - Args: - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - - Returns: True if the library is generated, False otherwise. - """ - return Path(f"{repo}/packages").exists() - - -def handle_release_stage( - librarian: str = LIBRARIAN_DIR, repo: str = REPO_DIR, output: str = OUTPUT_DIR -): - """The main coordinator for the release preparation process. - - This function prepares for the release of client libraries by reading a - `librarian/release-stage-request.json` file. The primary responsibility is - to update all required files with the new version and commit information - for libraries that have the `release_triggered` field set to `True`. - - See https://github.com/googleapis/librarian/blob/main/doc/container-contract.md#generate-container-command - - Args: - librarian(str): Path to the directory in the container which contains - the `release-stage-request.json` file. - repo(str): This directory will contain all directories that make up a - library, the .librarian folder, and any global file declared in - the config.yaml. - output(str): Path to the directory in the container where modified - code should be placed. - - Raises: - ValueError: if the version in `release-stage-request.json` is - the same as the version in state.yaml or if the - `release-stage-request.json` file in the given - librarian directory cannot be read. - """ - try: - is_mono_repo = _is_mono_repo(repo) - - # Read a release-stage-request.json file - request_data = _read_json_file(f"{librarian}/{RELEASE_STAGE_REQUEST_FILE}") - libraries_to_prep_for_release = _get_libraries_to_prepare_for_release( - request_data - ) - - if is_mono_repo: - # only a mono repo has a global changelog - _update_global_changelog( - f"{repo}/CHANGELOG.md", - f"{output}/CHANGELOG.md", - libraries_to_prep_for_release, - ) - - # Prepare the release for each library by updating the - # library specific version files and library specific changelog. - for library_release_data in libraries_to_prep_for_release: - version = library_release_data["version"] - library_id = library_release_data["id"] - # changes is optional - library_changes = library_release_data.get("changes") - tag_format = library_release_data["tag_format"] - - # Get previous version from state.yaml - previous_version = _get_previous_version(library_id, librarian) - if previous_version == version: - raise ValueError( - f"The version in {RELEASE_STAGE_REQUEST_FILE} is the same as the version in {STATE_YAML_FILE}\n" - f"{library_id} version: {previous_version}\n" - ) - - path_to_library = f"packages/{library_id}" if is_mono_repo else "." - - _update_version_for_library(repo, output, path_to_library, version) - if library_changes is not None: - _update_changelog_for_library( - repo, - output, - library_changes, - version, - previous_version, - library_id, - is_mono_repo, - tag_format, - ) - - except Exception as e: - raise ValueError(f"Release stage failed: {e}") from e - - logger.info("'release-stage' command executed.") - - -if __name__ == "__main__": # pragma: NO COVER - parser = argparse.ArgumentParser(description="A simple CLI tool.") - subparsers = parser.add_subparsers( - dest="command", required=True, help="Available commands" - ) - - # Define commands and their corresponding handler functions - handler_map = { - "configure": handle_configure, - "build": handle_build, - "release-stage": handle_release_stage, - } - - for command_name, help_text in [ - ("configure", "Onboard a new library or an api path to Librarian workflow."), - ("build", "Run unit tests via nox for the generated library."), - ("release-stage", "Prepare to release a given set of libraries"), - ]: - parser_cmd = subparsers.add_parser(command_name, help=help_text) - parser_cmd.set_defaults(func=handler_map[command_name]) - parser_cmd.add_argument( - "--librarian", - type=str, - help="Path to the directory in the container which contains the librarian configuration", - default=LIBRARIAN_DIR, - ) - parser_cmd.add_argument( - "--input", - type=str, - help="Path to the directory in the container which contains additional generator input", - default=INPUT_DIR, - ) - parser_cmd.add_argument( - "--output", - type=str, - help="Path to the directory in the container where code should be generated", - default=OUTPUT_DIR, - ) - parser_cmd.add_argument( - "--source", - type=str, - help="Path to the directory in the container which contains API protos", - default=SOURCE_DIR, - ) - parser_cmd.add_argument( - "--repo", - type=str, - help="Path to the directory in the container which contains google-cloud-python repository", - default=REPO_DIR, - ) - - if len(sys.argv) == 1: - parser.print_help(sys.stderr) - sys.exit(1) - - args = parser.parse_args() - - # Pass specific arguments to the handler functions for build - if args.command == "configure": - args.func( - librarian=args.librarian, - source=args.source, - repo=args.repo, - input=args.input, - output=args.output, - ) - elif args.command == "build": - args.func(librarian=args.librarian, repo=args.repo) - elif args.command == "release-stage": - args.func(librarian=args.librarian, repo=args.repo, output=args.output) - else: - args.func() diff --git a/.generator/requirements-test.in b/.generator/requirements-test.in deleted file mode 100644 index c72643d50d01..000000000000 --- a/.generator/requirements-test.in +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -pytest -pytest-cov -pytest-mock -gcp-synthtool @ git+https://github.com/googleapis/synthtool@5aa438a342707842d11fbbb302c6277fbf9e4655 -starlark-pyo3>=2025.1 -build -ruff==0.14.14 diff --git a/.generator/requirements.in b/.generator/requirements.in deleted file mode 100644 index cb9f2bad32c0..000000000000 --- a/.generator/requirements.in +++ /dev/null @@ -1,6 +0,0 @@ -click -gapic-generator==1.30.13 # https://github.com/googleapis/gapic-generator-python/releases/tag/v1.30.13 -nox -starlark-pyo3>=2025.1 -build -ruff==0.14.14 diff --git a/.generator/test_cli.py b/.generator/test_cli.py deleted file mode 100644 index 27ea62fa3fab..000000000000 --- a/.generator/test_cli.py +++ /dev/null @@ -1,1293 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging -import os -import pathlib -import re -import subprocess -import yaml -import unittest.mock -from datetime import date, datetime -from pathlib import Path -from unittest.mock import MagicMock, mock_open - -import pytest -from cli import ( - BUILD_REQUEST_FILE, - CONFIGURE_REQUEST_FILE, - RELEASE_STAGE_REQUEST_FILE, - SOURCE_DIR, - STATE_YAML_FILE, - LIBRARIAN_DIR, - REPO_DIR, - _create_main_version_header, - _determine_library_namespace, - _get_library_dist_name, - _get_library_id, - _get_libraries_to_prepare_for_release, - _get_new_library_config, - _get_previous_version, - _get_repo_name_from_repo_metadata, - _add_new_library_version, - _prepare_new_library_config, - _process_changelog, - _process_version_file, - _read_json_file, - _read_text_file, - _run_individual_session, - _run_nox_sessions, - _update_changelog_for_library, - _update_global_changelog, - _update_version_for_library, - _verify_library_dist_name, - _verify_library_namespace, - _write_json_file, - _write_text_file, - _create_new_changelog_for_library, - handle_build, - handle_configure, - handle_release_stage, -) - - -_MOCK_LIBRARY_CHANGES = [ - { - "type": "feat", - "subject": "add new UpdateRepository API", - "body": "This adds the ability to update a repository's properties.", - "piper_cl_number": "786353207", - "commit_hash": "9461532e7d19c8d71709ec3b502e5d81340fb661", - }, - { - "type": "fix", - "subject": "some fix", - "body": "some body", - "piper_cl_number": "786353208", - "commit_hash": "1231532e7d19c8d71709ec3b502e5d81340fb661", - }, - { - "type": "fix", - "subject": "another fix", - "body": "", - "piper_cl_number": "786353209", - "commit_hash": "1241532e7d19c8d71709ec3b502e5d81340fb661", - }, - { - "type": "docs", - "subject": "fix typo in BranchRule comment", - "body": "", - "piper_cl_number": "786353210", - "commit_hash": "9461532e7d19c8d71709ec3b502e5d81340fb661", - }, -] - - - -@pytest.fixture -def setup_dirs(tmp_path): - """Creates input and output directories.""" - input_dir = tmp_path / "input" - output_dir = tmp_path / "output" - input_dir.mkdir() - output_dir.mkdir() - return input_dir, output_dir - - -@pytest.fixture(autouse=True) -def _clear_lru_cache(): - """Automatically clears the cache of all LRU-cached functions after each test.""" - yield - _get_repo_name_from_repo_metadata.cache_clear() - - -@pytest.fixture -def mock_build_request_file(tmp_path, monkeypatch): - """Creates the mock request file at the correct path inside a temp dir.""" - # Create the path as expected by the script: .librarian/build-request.json - request_path = f"{LIBRARIAN_DIR}/{BUILD_REQUEST_FILE}" - request_dir = tmp_path / os.path.dirname(request_path) - request_dir.mkdir() - request_file = request_dir / os.path.basename(request_path) - - request_content = { - "id": "google-cloud-language", - "apis": [{"path": "google/cloud/language/v1"}], - } - request_file.write_text(json.dumps(request_content)) - - # Change the current working directory to the temp path for the test. - monkeypatch.chdir(tmp_path) - return request_file - - -@pytest.fixture -def mock_configure_request_data(): - """Returns mock data for configure-request.json.""" - return { - "libraries": [ - { - "id": "google-cloud-language", - "apis": [{"path": "google/cloud/language/v1", "status": "new"}], - "version": "", - } - ] - } - - -@pytest.fixture -def mock_configure_request_file(tmp_path, monkeypatch, mock_configure_request_data): - """Creates the mock request file at the correct path inside a temp dir.""" - # Create the path as expected by the script: .librarian/configure-request.json - request_path = f"{LIBRARIAN_DIR}/{CONFIGURE_REQUEST_FILE}" - request_dir = tmp_path / os.path.dirname(request_path) - request_dir.mkdir(parents=True, exist_ok=True) - request_file = request_dir / os.path.basename(request_path) - - request_file.write_text(json.dumps(mock_configure_request_data)) - - # Change the current working directory to the temp path for the test. - monkeypatch.chdir(tmp_path) - return request_file - - -@pytest.fixture -def mock_generate_request_data_for_nox(): - """Returns mock data for generate-request.json for nox tests.""" - return { - "id": "mock-library", - "apis": [ - {"path": "google/mock/v1"}, - ], - } - - -@pytest.fixture -def mock_release_stage_request_file(tmp_path, monkeypatch): - """Creates the mock request file at the correct path inside a temp dir.""" - # Create the path as expected by the script: .librarian/release-request.json - request_path = f"{LIBRARIAN_DIR}/{RELEASE_STAGE_REQUEST_FILE}" - request_dir = tmp_path / os.path.dirname(request_path) - request_dir.mkdir() - request_file = request_dir / os.path.basename(request_path) - - request_content = { - "libraries": [ - { - "id": "google-cloud-another-library", - "apis": [{"path": "google/cloud/another/library/v1"}], - "release_triggered": False, - "version": "1.2.3", - "changes": [], - "tag_format": "{id}-v{version}", - }, - { - "id": "google-cloud-language", - "apis": [{"path": "google/cloud/language/v1"}], - "release_triggered": True, - "version": "1.2.3", - "changes": [], - "tag_format": "{id}-v{version}", - }, - ] - } - request_file.write_text(json.dumps(request_content)) - - # Change the current working directory to the temp path for the test. - monkeypatch.chdir(tmp_path) - return request_file - - -@pytest.fixture -def mock_state_file(tmp_path, monkeypatch): - """Creates the state file at the correct path inside a temp dir.""" - # Create the path as expected by the script: .librarian/state.yaml - request_path = f"{LIBRARIAN_DIR}/{STATE_YAML_FILE}" - request_dir = tmp_path / os.path.dirname(request_path) - request_dir.mkdir() - request_file = request_dir / os.path.basename(request_path) - - state_yaml_contents = { - "libraries": [{"id": "google-cloud-language", "version": "1.2.3"}] - } - request_file.write_text(yaml.dump(state_yaml_contents)) - - # Change the current working directory to the temp path for the test. - monkeypatch.chdir(tmp_path) - return request_file - - -def test_handle_configure_success(mock_configure_request_file, mocker): - """Tests the successful execution path of handle_configure.""" - mocker.patch("cli._update_global_changelog", return_value=None) - mock_write_json = mocker.patch("cli._write_json_file") - mock_prepare_config = mocker.patch( - "cli._prepare_new_library_config", return_value={"id": "prepared"} - ) - mock_create_changelog = mocker.patch("cli._create_new_changelog_for_library") - - handle_configure() - - mock_prepare_config.assert_called_once() - mock_write_json.assert_called_once_with( - f"{LIBRARIAN_DIR}/configure-response.json", {"id": "prepared"} - ) - - -def test_handle_configure_no_new_library(mocker): - """Tests that handle_configure fails if no new library is found.""" - mocker.patch("cli._read_json_file", return_value={"libraries": []}) - # The call to _prepare_new_library_config with an empty dict will raise a ValueError - # because _get_library_id will fail. - with pytest.raises(ValueError, match="Configuring a new library failed."): - handle_configure() - - -def test_create_new_changelog_for_library(mocker): - """Tests that the changelog files are created correctly.""" - library_id = "google-cloud-language" - output = "output" - mock_makedirs = mocker.patch("os.makedirs") - mock_write_text_file = mocker.patch("cli._write_text_file") - - _create_new_changelog_for_library(library_id, output) - - package_changelog_path = f"{output}/packages/{library_id}/CHANGELOG.md" - docs_changelog_path = f"{output}/packages/{library_id}/docs/CHANGELOG.md" - - # Check that makedirs was called for both parent directories - mock_makedirs.assert_any_call( - os.path.dirname(package_changelog_path), exist_ok=True - ) - mock_makedirs.assert_any_call(os.path.dirname(docs_changelog_path), exist_ok=True) - assert mock_makedirs.call_count == 2 - - # Check that the files were "written" with the correct content - changelog_content = f"# Changelog\n\n[PyPI History][1]\n\n[1]: https://pypi.org/project/{library_id}/#history\n" - mock_write_text_file.assert_any_call(package_changelog_path, changelog_content) - mock_write_text_file.assert_any_call(docs_changelog_path, changelog_content) - assert mock_write_text_file.call_count == 2 - - -def test_get_new_library_config_found(mock_configure_request_data): - """Tests that the new library configuration is returned when found.""" - config = _get_new_library_config(mock_configure_request_data) - assert config["id"] == "google-cloud-language" - # Assert that the config is NOT modified - assert "status" in config["apis"][0] - - -def test_get_new_library_config_not_found(): - """Tests that an empty dictionary is returned when no new library is found.""" - request_data = { - "libraries": [ - { - "id": "existing-library", - "apis": [{"path": "path/v1", "status": "existing"}], - }, - ] - } - config = _get_new_library_config(request_data) - assert config == {} - - -def test_get_new_library_config_empty_input(): - """Tests that an empty dictionary is returned for empty input.""" - config = _get_new_library_config({}) - assert config == {} - - -def test_prepare_new_library_config(mocker): - """Tests the preparation of a new library's configuration.""" - raw_config = { - "id": "google-cloud-language", - "apis": [{"path": "google/cloud/language/v1", "status": "new"}], - "source_roots": None, - "preserve_regex": None, - "remove_regex": None, - "version": "", - } - - prepared_config = _prepare_new_library_config(raw_config) - - # Check that status is removed - assert "status" not in prepared_config["apis"][0] - # Check that defaults are added - assert prepared_config["source_roots"] == ["packages/google-cloud-language"] - assert ( - "packages/google-cloud-language/CHANGELOG.md" - in prepared_config["preserve_regex"] - ) - assert prepared_config["remove_regex"] == ["packages/google-cloud-language"] - assert prepared_config["tag_format"] == "{id}-v{version}" - assert prepared_config["version"] == "0.0.0" - - -def test_prepare_new_library_config_preserves_existing_values(mocker): - """Tests that existing values in the config are not overwritten.""" - raw_config = { - "id": "google-cloud-language", - "apis": [{"path": "google/cloud/language/v1", "status": "new"}], - "source_roots": ["packages/google-cloud-language-custom"], - "preserve_regex": ["custom/regex"], - "remove_regex": ["custom/remove"], - "tag_format": "custom-format-{{version}}", - "version": "4.5.6", - } - - prepared_config = _prepare_new_library_config(raw_config) - - # Check that status is removed - assert "status" not in prepared_config["apis"][0] - # Check that existing values are preserved - assert prepared_config["source_roots"] == ["packages/google-cloud-language-custom"] - assert prepared_config["preserve_regex"] == ["custom/regex"] - assert prepared_config["remove_regex"] == ["custom/remove"] - assert prepared_config["tag_format"] == "custom-format-{{version}}" - assert prepared_config["version"] == "4.5.6" - - -def test_add_new_library_version_populates_version(mocker): - """Tests that the version is populated if it's missing.""" - config = {"version": ""} - _add_new_library_version(config) - assert config["version"] == "0.0.0" - - -def test_add_new_library_version_preserves_version(): - """Tests that an existing version is preserved.""" - config = {"version": "4.5.6"} - _add_new_library_version(config) - assert config["version"] == "4.5.6" - - -def test_get_library_id_success(): - """Tests that _get_library_id returns the correct ID when present.""" - request_data = {"id": "test-library", "name": "Test Library"} - library_id = _get_library_id(request_data) - assert library_id == "test-library" - - -def test_get_library_id_missing_id(): - """Tests that _get_library_id raises ValueError when 'id' is missing.""" - request_data = {"name": "Test Library"} - with pytest.raises( - ValueError, match="Request file is missing required 'id' field." - ): - _get_library_id(request_data) - - -def test_get_library_id_empty_id(): - """Tests that _get_library_id raises ValueError when 'id' is an empty string.""" - request_data = {"id": "", "name": "Test Library"} - with pytest.raises( - ValueError, match="Request file is missing required 'id' field." - ): - _get_library_id(request_data) - - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_run_individual_session_success(mocker, caplog, is_mono_repo): - """Tests that _run_individual_session calls nox with correct arguments and logs success.""" - caplog.set_level(logging.INFO) - - mock_subprocess_run = mocker.patch( - "cli.subprocess.run", return_value=MagicMock(returncode=0) - ) - - test_session = "unit-3.10" - test_library_id = "test-library" - repo = "repo" - _run_individual_session(test_session, test_library_id, repo, is_mono_repo) - - expected_command = [ - "nox", - "-s", - test_session, - "-f", - ( - f"{REPO_DIR}/packages/{test_library_id}/noxfile.py" - if is_mono_repo - else f"{REPO_DIR}/noxfile.py" - ), - ] - mock_subprocess_run.assert_called_once_with( - expected_command, text=True, check=True, timeout=1200 - ) - - -def test_run_individual_session_failure(mocker): - """Tests that _run_individual_session raises CalledProcessError if nox command fails.""" - mocker.patch( - "cli.subprocess.run", - side_effect=subprocess.CalledProcessError( - 1, "nox", stderr="Nox session failed" - ), - ) - - with pytest.raises(subprocess.CalledProcessError): - _run_individual_session("lint", "another-library", "repo", True) - - -@pytest.mark.parametrize( - "is_mono_repo, nox_session_python_runtime", - [ - (False, "3.14"), - (True, "3.14"), - ], -) -def test_run_nox_sessions_success( - mocker, - mock_generate_request_data_for_nox, - is_mono_repo, - nox_session_python_runtime, -): - """Tests that _run_nox_sessions successfully runs all specified sessions.""" - mocker.patch("cli._read_json_file", return_value=mock_generate_request_data_for_nox) - mocker.patch("cli._get_library_id", return_value="mock-library") - mock_run_individual_session = mocker.patch("cli._run_individual_session") - - sessions_to_run = [ - f"unit-{nox_session_python_runtime}(protobuf_implementation='python')", - ] - _run_nox_sessions("mock-library", "repo", is_mono_repo) - - assert mock_run_individual_session.call_count == len(sessions_to_run) - mock_run_individual_session.assert_has_calls( - [ - mocker.call( - f"unit-{nox_session_python_runtime}(protobuf_implementation='python')", - "mock-library", - "repo", - is_mono_repo, - ), - ] - ) - - -def test_run_nox_sessions_read_file_failure(mocker): - """Tests that _run_nox_sessions raises ValueError if _read_json_file fails.""" - mocker.patch("cli._read_json_file", side_effect=FileNotFoundError("file not found")) - - with pytest.raises(ValueError, match="Failed to run the nox session"): - _run_nox_sessions("mock-library", "repo", True) - - -def test_run_nox_sessions_get_library_id_failure(mocker): - """Tests that _run_nox_sessions raises ValueError if _get_library_id fails.""" - mocker.patch("cli._read_json_file", return_value={"apis": []}) # Missing 'id' - mocker.patch( - "cli._get_library_id", - side_effect=ValueError("Request file is missing required 'id' field."), - ) - - with pytest.raises(ValueError, match="Failed to run the nox session"): - _run_nox_sessions("mock-library", "repo", True) - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_run_nox_sessions_individual_session_failure( - mocker, mock_generate_request_data_for_nox, is_mono_repo -): - """Tests that _run_nox_sessions raises ValueError if _run_individual_session fails.""" - mocker.patch("cli._read_json_file", return_value=mock_generate_request_data_for_nox) - mocker.patch("cli._get_library_id", return_value="mock-library") - mock_run_individual_session = mocker.patch( - "cli._run_individual_session", - side_effect=[subprocess.CalledProcessError(1, "nox", "session failed")], - ) - - with pytest.raises(ValueError, match="Failed to run the nox session"): - _run_nox_sessions("mock-library", "repo", is_mono_repo) - - # Check that _run_individual_session was called at least once - assert mock_run_individual_session.call_count > 0 - - -def test_handle_build_success(caplog, mocker, mock_build_request_file): - """ - Tests the successful execution path of handle_build. - """ - caplog.set_level(logging.INFO) - - mocker.patch("cli._run_nox_sessions") - mocker.patch("cli._verify_library_namespace") - mocker.patch("cli._verify_library_dist_name") - handle_build() - - assert "'build' command executed." in caplog.text - - -def test_handle_build_fail(caplog): - """ - Tests the failed to read `librarian/build-request.json` file in handle_generates. - """ - with pytest.raises(ValueError): - handle_build() - - -def test_read_valid_json(mocker): - """Tests reading a valid JSON file.""" - mock_content = '{"key": "value"}' - mocker.patch("builtins.open", mocker.mock_open(read_data=mock_content)) - result = _read_json_file("fake/path.json") - assert result == {"key": "value"} - - -def test_json_file_not_found(mocker): - """Tests behavior when the file does not exist.""" - mocker.patch("builtins.open", side_effect=FileNotFoundError("No such file")) - - with pytest.raises(FileNotFoundError): - _read_json_file("non/existent/path.json") - - -def test_invalid_json(mocker): - """Tests reading a file with malformed JSON.""" - mock_content = '{"key": "value",}' - mocker.patch("builtins.open", mocker.mock_open(read_data=mock_content)) - - with pytest.raises(json.JSONDecodeError): - _read_json_file("fake/path.json") - - - -def test_get_libraries_to_prepare_for_release(mock_release_stage_request_file): - """ - Tests that only libraries with the `release_triggered` field set to `True` are - returned. - """ - request_data = _read_json_file(f"{LIBRARIAN_DIR}/{RELEASE_STAGE_REQUEST_FILE}") - libraries_to_prep_for_release = _get_libraries_to_prepare_for_release(request_data) - assert len(libraries_to_prep_for_release) == 1 - assert "google-cloud-language" in libraries_to_prep_for_release[0]["id"] - assert libraries_to_prep_for_release[0]["release_triggered"] - - -def test_handle_release_stage_success(mocker, mock_release_stage_request_file): - """ - Simply tests that `handle_release_stage` runs without errors. - """ - mocker.patch("cli._update_global_changelog", return_value=None) - mocker.patch("cli._update_version_for_library", return_value=None) - mocker.patch("cli._get_previous_version", return_value=None) - mocker.patch("cli._update_changelog_for_library", return_value=None) - handle_release_stage() - - -def test_handle_release_stage_is_generated_success( - mocker, mock_release_stage_request_file -): - """ - Tests that `handle_release_stage` calls `_update_global_changelog` when the - `packages` directory exists. - """ - mocker.patch("pathlib.Path.exists", return_value=True) - mock_update_global_changelog = mocker.patch("cli._update_global_changelog") - mocker.patch("cli._update_version_for_library") - mocker.patch("cli._get_previous_version", return_value="1.2.2") - mocker.patch("cli._update_changelog_for_library") - - handle_release_stage() - - mock_update_global_changelog.assert_called_once() - - -def test_handle_release_stage_fail_value_error_file(): - """ - Tests that handle_release_stage fails to read `librarian/release-stage-request.json`. - """ - with pytest.raises(ValueError, match="No such file or directory"): - handle_release_stage() - - -def test_handle_release_stage_fail_value_error_version(mocker): - m = mock_open() - - mock_release_stage_request_content = { - "libraries": [ - { - "id": "google-cloud-language", - "apis": [{"path": "google/cloud/language/v1"}], - "release_triggered": True, - "version": "1.2.2", - "changes": [], - "tag_format": "{id}-v{version}", - }, - ] - } - with unittest.mock.patch("cli.open", m): - mocker.patch( - "cli._get_libraries_to_prepare_for_release", - return_value=mock_release_stage_request_content["libraries"], - ) - mocker.patch("cli._get_previous_version", return_value="1.2.2") - mocker.patch("cli._process_changelog", return_value=None) - mocker.patch( - "cli._read_json_file", return_value=mock_release_stage_request_content - ) - with pytest.raises( - ValueError, match="is the same as the version in state.yaml" - ): - handle_release_stage() - - -def test_read_valid_text_file(mocker): - """Tests reading a valid text file.""" - mock_content = "some text" - mocker.patch("builtins.open", mocker.mock_open(read_data=mock_content)) - result = _read_text_file("fake/path.txt") - assert result == "some text" - - -def test_text_file_not_found(mocker): - """Tests behavior when the file does not exist.""" - mocker.patch("builtins.open", side_effect=FileNotFoundError("No such file")) - - with pytest.raises(FileNotFoundError): - _read_text_file("non/existent/path.text") - - -def test_write_text_file(): - """Tests writing a text file. - See https://docs.python.org/3/library/unittest.mock.html#mock-open - """ - m = mock_open() - - with unittest.mock.patch("cli.open", m): - _write_text_file("fake_path.txt", "modified content") - - handle = m() - handle.write.assert_called_once_with("modified content") - - -def test_write_json_file(): - """Tests writing a json file. - See https://docs.python.org/3/library/unittest.mock.html#mock-open - """ - m = mock_open() - - expected_dict = {"name": "call me json"} - - with unittest.mock.patch("cli.open", m): - _write_json_file("fake_path.json", expected_dict) - - handle = m() - # Get all the arguments passed to the mock's write method - # and join them into a single string. - written_content = "".join( - [call.args[0] for call in handle.write.call_args_list] - ) - - # Create the expected output string with the correct formatting. - expected_output = json.dumps(expected_dict, indent=2) + "\n" - - # Assert that the content written to the mock file matches the expected output. - assert written_content == expected_output - - -def test_update_global_changelog(mocker, mock_release_stage_request_file): - """Tests that the global changelog is updated - with the new version for a given library. - See https://docs.python.org/3/library/unittest.mock.html#mock-open - """ - m = mock_open() - request_data = _read_json_file(f"{LIBRARIAN_DIR}/{RELEASE_STAGE_REQUEST_FILE}") - libraries = _get_libraries_to_prepare_for_release(request_data) - - with unittest.mock.patch("cli.open", m): - mocker.patch( - "cli._read_text_file", return_value="[google-cloud-language==1.2.2]" - ) - _update_global_changelog("source", "output", libraries) - - handle = m() - handle.write.assert_called_once_with("[google-cloud-language==1.2.3]") - - -def test_update_version_for_library_success_gapic(mocker): - mock_content = '__version__ = "1.2.2"' - mock_json_metadata = {"clientLibrary": {"version": "0.1.0"}} - mock_shutil_copy = mocker.patch("shutil.copy") - - m = mock_open() - - mock_rglob = mocker.patch("pathlib.Path.rglob") - mock_rglob.side_effect = [ - [ - pathlib.Path("repo/gapic_version.py"), - pathlib.Path("repo/tests/gapic_version.py"), - ], # 1st call (gapic_version.py) - [pathlib.Path("repo/types/version.py")], # 2nd call (types/version.py). - [pathlib.Path("repo/samples/snippet_metadata.json")], # 3rd call (snippets) - ] - mock_read_text_file = mocker.patch("cli._read_text_file") - mock_read_text_file.side_effect = [ - mock_content, # 1st call (gapic_version.py) - # Do not process version files in the `types` directory as some - # GAPIC libraries have `version.py` which are generated from - # `version.proto` and do not include SDK versions. - # Leave the content as empty because it doesn't contain version information - "", # 2nd call (tests/gapic_version.py) - "", # 3rd call (types/version.py) - ] - - with unittest.mock.patch("cli.open", m): - mocker.patch("cli._read_json_file", return_value=mock_json_metadata) - _update_version_for_library( - "repo", "output", "packages/google-cloud-language", "1.2.3" - ) - - handle = m() - assert handle.write.call_args_list[0].args[0] == '__version__ = "1.2.3"' - # Get all the arguments passed to the mock's write method - # and join them into a single string. - written_content = "".join( - [call.args[0] for call in handle.write.call_args_list[1:]] - ) - # Create the expected output string with the correct formatting. - assert ( - written_content - == '{\n "clientLibrary": {\n "version": "1.2.3"\n }\n}\n' - ) - - -def test_update_version_for_library_success_proto_only_setup_py(mocker): - m = mock_open() - - mock_rglob = mocker.patch("pathlib.Path.rglob") - mock_rglob.side_effect = [ - [], - [pathlib.Path("repo/setup.py")], - [pathlib.Path("repo/samples/snippet_metadata.json")], - ] - mock_shutil_copy = mocker.patch("shutil.copy") - mock_content = 'version = "1.2.2"' - mock_json_metadata = {"clientLibrary": {"version": "0.1.0"}} - - with unittest.mock.patch("cli.open", m): - mocker.patch("cli._read_text_file", return_value=mock_content) - mocker.patch("cli._read_json_file", return_value=mock_json_metadata) - _update_version_for_library( - "repo", "output", "packages/google-cloud-language", "1.2.3" - ) - - handle = m() - assert handle.write.call_args_list[0].args[0] == 'version = "1.2.3"' - # Get all the arguments passed to the mock's write method - # and join them into a single string. - written_content = "".join( - [call.args[0] for call in handle.write.call_args_list[1:]] - ) - # Create the expected output string with the correct formatting. - assert ( - written_content - == '{\n "clientLibrary": {\n "version": "1.2.3"\n }\n}\n' - ) - - -def test_update_version_for_library_success_with_date_string(mocker): - m = mock_open() - - mock_rglob = mocker.patch("pathlib.Path.rglob") - mock_rglob.side_effect = [ - [], - [pathlib.Path("repo/setup.py")], - [pathlib.Path("repo/samples/snippet_metadata.json")], - ] - mock_shutil_copy = mocker.patch("shutil.copy") - mock_content = 'version = "1.2.2"\n__release_date__ = "2025-11-03"' - mock_json_metadata = {"clientLibrary": {"version": "0.1.0"}} - today_iso = date.today().isoformat() - - with unittest.mock.patch("cli.open", m): - mocker.patch("cli._read_text_file", return_value=mock_content) - mocker.patch("cli._read_json_file", return_value=mock_json_metadata) - _update_version_for_library( - "repo", "output", "packages/google-cloud-language", "1.2.3" - ) - - handle = m() - assert ( - handle.write.call_args_list[0].args[0] - == f'version = "1.2.3"\n__release_date__ = "{today_iso}"' - ) - # Get all the arguments passed to the mock's write method - # and join them into a single string. - written_content = "".join( - [call.args[0] for call in handle.write.call_args_list[1:]] - ) - # Create the expected output string with the correct formatting. - assert ( - written_content - == '{\n "clientLibrary": {\n "version": "1.2.3"\n }\n}\n' - ) - - -def test_update_version_for_library_success_proto_only_pyproject_toml(mocker): - m = mock_open() - - mock_path_exists = mocker.patch("pathlib.Path.exists", return_value=True) - mock_rglob = mocker.patch("pathlib.Path.rglob") - mock_rglob.side_effect = [ - [], # gapic_version.py - [], # version.py - [pathlib.Path("repo/samples/snippet_metadata.json")], - ] - mock_shutil_copy = mocker.patch("shutil.copy") - mock_content = 'version = "1.2.2"' - mock_json_metadata = {"clientLibrary": {"version": "0.1.0"}} - - with unittest.mock.patch("cli.open", m): - mocker.patch("cli._read_text_file", return_value=mock_content) - mocker.patch("cli._read_json_file", return_value=mock_json_metadata) - _update_version_for_library( - "repo", "output", "packages/google-cloud-language", "1.2.3" - ) - - handle = m() - assert handle.write.call_args_list[0].args[0] == 'version = "1.2.3"' - # Get all the arguments passed to the mock's write method - # and join them into a single string. - written_content = "".join( - [call.args[0] for call in handle.write.call_args_list[1:]] - ) - # Create the expected output string with the correct formatting. - assert ( - written_content - == '{\n "clientLibrary": {\n "version": "1.2.3"\n }\n}\n' - ) - - -def test_update_version_for_library_failure(mocker): - """Tests that value error is raised if the version string cannot be found""" - m = mock_open() - - mock_rglob = mocker.patch( - "pathlib.Path.rglob", return_value=[pathlib.Path("repo/gapic_version.py")] - ) - mock_content = "not found" - with pytest.raises(ValueError): - with unittest.mock.patch("cli.open", m): - mocker.patch("cli._read_text_file", return_value=mock_content) - _update_version_for_library( - "repo", "output", "packages/google-cloud-language", "1.2.3" - ) - - -def test_get_previous_version_success(mock_state_file): - """Test that the version can be retrieved from the state.yaml for a given library""" - previous_version = _get_previous_version("google-cloud-language", LIBRARIAN_DIR) - assert previous_version == "1.2.3" - - -def test_get_previous_version_failure(mock_state_file): - """Test that ValueError is raised when a library does not exist in state.yaml""" - with pytest.raises(ValueError): - _get_previous_version("google-cloud-does-not-exist", LIBRARIAN_DIR) - - -def test_update_changelog_for_library_writes_both_changelogs(mocker): - """Tests that _update_changelog_for_library writes to both changelogs.""" - mock_content = """# Changelog - -[PyPI History][1] - -[1]: https://pypi.org/project/google-cloud-language/#history -""" - mock_read = mocker.patch("cli._read_text_file", return_value=mock_content) - mock_write = mocker.patch("cli._write_text_file") - mock_path_exists = mocker.patch("cli.os.path.lexists", return_value=True) - _update_changelog_for_library( - "repo", - "output", - _MOCK_LIBRARY_CHANGES, - "1.2.3", - "1.2.2", - "google-cloud-language", - True, - "{id}-v{version}", - ) - - assert mock_write.call_count == 2 - mock_write.assert_any_call( - "output/packages/google-cloud-language/CHANGELOG.md", mocker.ANY - ) - mock_write.assert_any_call( - "output/packages/google-cloud-language/docs/CHANGELOG.md", mocker.ANY - ) - - -def test_update_changelog_for_library_single_repo(mocker): - """Tests that _update_changelog_for_library writes to both changelogs in a single repo.""" - mock_content = """# Changelog - -[PyPI History][1] - -[1]: https://pypi.org/project/google-cloud-language/#history -""" - mock_read = mocker.patch("cli._read_text_file", return_value=mock_content) - mock_write = mocker.patch("cli._write_text_file") - mock_path_exists = mocker.patch("cli.os.path.lexists", return_value=True) - mocker.patch( - "cli._get_repo_name_from_repo_metadata", - return_value="googleapis/google-cloud-python", - ) - _update_changelog_for_library( - "repo", - "output", - _MOCK_LIBRARY_CHANGES, - "1.2.3", - "1.2.2", - "google-cloud-language", - False, - "v{version}", - ) - - assert mock_write.call_count == 2 - mock_write.assert_any_call("output/CHANGELOG.md", mocker.ANY) - mock_write.assert_any_call("output/docs/CHANGELOG.md", mocker.ANY) - - -def test_process_changelog_success(): - """Tests that value error is raised if the changelog anchor string cannot be found""" - current_date = datetime.now().strftime("%Y-%m-%d") - mock_content = """# Changelog\n[PyPI History][1]\n[1]: https://pypi.org/project/google-cloud-language/#history\n -## [1.2.2](https://github.com/googleapis/google-cloud-python/compare/google-cloud-language-v1.2.1...google-cloud-language-v1.2.2) (2025-06-11)""" - expected_result = f"""# Changelog\n[PyPI History][1]\n[1]: https://pypi.org/project/google-cloud-language/#history\n -## [1.2.3](https://github.com/googleapis/google-cloud-python/compare/google-cloud-language-v1.2.2...google-cloud-language-v1.2.3) ({current_date})\n\n -### Documentation\n -* fix typo in BranchRule comment ([9461532e7d19c8d71709ec3b502e5d81340fb661](https://github.com/googleapis/google-cloud-python/commit/9461532e7d19c8d71709ec3b502e5d81340fb661))\n\n -### Features\n -* add new UpdateRepository API ([9461532e7d19c8d71709ec3b502e5d81340fb661](https://github.com/googleapis/google-cloud-python/commit/9461532e7d19c8d71709ec3b502e5d81340fb661))\n\n -### Bug Fixes\n -* some fix ([1231532e7d19c8d71709ec3b502e5d81340fb661](https://github.com/googleapis/google-cloud-python/commit/1231532e7d19c8d71709ec3b502e5d81340fb661)) -* another fix ([1241532e7d19c8d71709ec3b502e5d81340fb661](https://github.com/googleapis/google-cloud-python/commit/1241532e7d19c8d71709ec3b502e5d81340fb661))\n -## [1.2.2](https://github.com/googleapis/google-cloud-python/compare/google-cloud-language-v1.2.1...google-cloud-language-v1.2.2) (2025-06-11)""" - version = "1.2.3" - previous_version = "1.2.2" - library_id = "google-cloud-language" - tag_format = "{id}-v{version}" - - result = _process_changelog( - mock_content, - _MOCK_LIBRARY_CHANGES, - version, - previous_version, - library_id, - "googleapis/google-cloud-python", - tag_format, - ) - assert result == expected_result - - -def test_process_changelog_failure(): - """Tests that value error is raised if the changelog anchor string cannot be found""" - with pytest.raises(ValueError): - _process_changelog("", [], "", "", "", "googleapis/google-cloud-python", "") - - -def test_update_changelog_for_library_failure(mocker): - m = mock_open() - - mock_content = """# Changelog""" - - with pytest.raises(ValueError): - with unittest.mock.patch("cli.open", m): - mocker.patch("cli._read_text_file", return_value=mock_content) - _update_changelog_for_library( - "repo", - "output", - _MOCK_LIBRARY_CHANGES, - "1.2.3", - "1.2.2", - "google-cloud-language", - True, - "{id}-v{version}", - ) - - -def test_process_version_file_success(): - version_file_contents = 'version = "1.2.2"' - new_version = "1.2.3" - modified_content = _process_version_file( - version_file_contents, new_version, Path("file.txt") - ) - assert modified_content == f'version = "{new_version}"' - - -def test_process_version_file_failure(): - """Tests that value error is raised if the version string cannot be found""" - with pytest.raises(ValueError): - _process_version_file("", "", Path("")) - - -@pytest.mark.parametrize( - "tag_format,expected_tag_result", - [(r"{id}-v{version}", "google-cloud-language-v"), (r"v{version}", "v")], -) -def test_create_main_version_header(tag_format, expected_tag_result): - current_date = datetime.now().strftime("%Y-%m-%d") - expected_header = f"## [1.2.3](https://github.com/googleapis/google-cloud-python/compare/{expected_tag_result}1.2.2...{expected_tag_result}1.2.3) ({current_date})" - previous_version = "1.2.2" - version = "1.2.3" - library_id = "google-cloud-language" - tag_format = tag_format - actual_header = _create_main_version_header( - version, - previous_version, - library_id, - "googleapis/google-cloud-python", - tag_format, - ) - assert actual_header == expected_header - - -@pytest.fixture -def mock_path_class(mocker): - """ - A mock instance is pre-configured as its return_value. - """ - mock_instance = MagicMock(spec=Path) - mock_class_patch = mocker.patch("cli.Path", return_value=mock_instance) - return mock_class_patch - - -@pytest.mark.parametrize( - "pkg_root_str, gapic_parent_str, expected_namespace", - [ - ( - "repo/packages/google-cloud-lang", - "repo/packages/google-cloud-lang/google/cloud/language", - "google.cloud", - ), - ( - "repo/packages/google-ads", - "repo/packages/google-ads/google/ads/v17", - "google.ads", - ), - ( - "repo/packages/google-auth", - "repo/packages/google-auth/google/auth", - "google", - ), - ("repo/packages/google-api", "repo/packages/google-api/google", "google"), - ], -) -def test_determine_library_namespace_success( - pkg_root_str, gapic_parent_str, expected_namespace -): - """Tests that the refactored namespace logic correctly calculates the relative namespace.""" - pkg_root_path = Path(pkg_root_str) - gapic_parent_path = Path(gapic_parent_str) - - namespace = _determine_library_namespace(gapic_parent_path, pkg_root_path) - assert namespace == expected_namespace - - -def test_determine_library_namespace_fails_not_subpath(): - """Tests that a ValueError is raised if the gapic path is not inside the package root.""" - pkg_root_path = Path("repo/packages/my-lib") - gapic_parent_path = Path("SOME/OTHER/PATH/google/cloud/api") - - with pytest.raises(ValueError): - _determine_library_namespace(gapic_parent_path, pkg_root_path) - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_get_library_dist_name_success(mocker, is_mono_repo): - mock_metadata = {"name": "my-lib", "version": "1.0.0"} - mocker.patch("build.util.project_wheel_metadata", return_value=mock_metadata) - assert _get_library_dist_name("my-lib", "repo", is_mono_repo) == "my-lib" - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_verify_library_dist_name_setup_success(mocker, is_mono_repo): - """Tests success when a library distribution name in setup.py is valid.""" - mock_setup_file = mocker.patch("cli._get_library_dist_name", return_value="my-lib") - _verify_library_dist_name("my-lib", "repo", is_mono_repo) - mock_setup_file.assert_called_once_with("my-lib", "repo", is_mono_repo) - - -def test_verify_library_dist_name_fail(mocker): - """Tests failure when a library-id does not match the libary distribution name.""" - mocker.patch("cli._get_library_dist_name", return_value="invalid-lib") - with pytest.raises(ValueError): - _verify_library_dist_name("my-lib", "repo", True) - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_verify_library_namespace_success_valid(mocker, mock_path_class, is_mono_repo): - """Tests success when a single valid namespace is found.""" - - # 1. Get the mock instance from the mock class's return_value - mock_instance = mock_path_class.return_value # This is library_path - - # 2. Configure the mock instance - mock_instance.is_dir.return_value = True - mock_files_gapic_version = MagicMock(spec=Path) - mock_gapic_parent = MagicMock(spec=Path) - mock_gapic_parent.__str__.return_value = ( - "/abs/repo/packages/my-lib/google/cloud/language" - if is_mono_repo - else "/abs/repo/google/cloud/language" - ) - mock_files_gapic_version.parent = mock_gapic_parent - mock_files_proto = MagicMock(spec=Path) - mock_proto_parent = MagicMock(spec=Path) - mock_files_proto.parent = mock_proto_parent - mock_proto_parent.relative_to.return_value = MagicMock() - mock_proto_parent.relative_to.return_value.__str__.return_value = ( - "google/cloud/language/v1" - ) - mock_proto_parent.__str__.return_value = ( - "/abs/repo/packages/my-lib/google/cloud/language/v1/proto" - if is_mono_repo - else "/abs/repo/google/cloud/language/v1/proto" - ) - mock_instance.rglob.return_value = [mock_files_gapic_version, mock_files_proto] - - mock_determine_ns = mocker.patch( - "cli._determine_library_namespace", return_value="google.cloud" - ) - - _verify_library_namespace("my-lib", "/abs/repo", is_mono_repo) - - # 3. Assert against the mock CLASS (from the fixture) - mock_path_class.assert_called_once_with( - "/abs/repo/packages/my-lib" if is_mono_repo else "/abs/repo" - ) - - # 4. Verify the helper was called with the correct instance - assert mock_determine_ns.call_count == 2 - mock_determine_ns.assert_any_call(mock_gapic_parent, mock_instance) - mock_determine_ns.assert_any_call(mock_proto_parent, mock_instance) - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_verify_library_namespace_excludes_proto_dir( - mocker, mock_path_class, is_mono_repo -): - """Tests that a proto file path ending in 'proto' is correctly excluded.""" - - mock_instance = mock_path_class.return_value # This is library_path - mock_instance.is_dir.return_value = True - - mock_exclude_file = MagicMock(spec=Path) - mock_exclude_parent = MagicMock(spec=Path) - mock_exclude_file.parent = mock_exclude_parent - - mock_relative_result = MagicMock() - mock_relative_result.__str__.return_value = "google/cloud/language/v1/proto" - mock_exclude_parent.relative_to.return_value = mock_relative_result - mock_exclude_parent.__str__.return_value = ( - "/abs/repo/packages/my-lib/google/cloud/language/v1/proto" - if is_mono_repo - else "/abs/repo/google/cloud/language/v1/proto" - ) - - mock_instance.rglob.side_effect = [[], [mock_exclude_file]] - mock_determine_ns = mocker.patch("cli._determine_library_namespace", autospec=True) - - with pytest.raises(ValueError) as excinfo: - _verify_library_namespace("my-lib", "/abs/repo", is_mono_repo) - - assert "namespace cannot be determined" in str(excinfo.value) - mock_determine_ns.assert_not_called() - mock_path_class.assert_called_once_with( - "/abs/repo/packages/my-lib" if is_mono_repo else "/abs/repo" - ) - - -def test_verify_library_namespace_failure_invalid(mocker, mock_path_class): - """Tests failure when a namespace is found that is NOT in the valid list.""" - mock_instance = mock_path_class.return_value - mock_instance.is_dir.return_value = True - - mock_file = MagicMock(spec=Path) - mock_parent = MagicMock(spec=Path) - mock_parent.__str__.return_value = "/abs/repo/packages/my-lib/google/api/core" - mock_file.parent = mock_parent - mock_relative_result = MagicMock() - mock_relative_result.__str__.return_value = ( - "google/api/core" # Does not end with 'proto' or start with 'samples' - ) - mock_parent.relative_to.return_value = mock_relative_result - mock_instance.rglob.return_value = [mock_file] - mock_determine_ns = mocker.patch( - "cli._determine_library_namespace", - return_value="google.apis", # NOT in valid_namespaces - ) - with pytest.raises(ValueError) as excinfo: - _verify_library_namespace("my-lib", "/abs/repo", True) - assert "The namespace `google.apis` for `my-lib` must be one of" in str( - excinfo.value - ) - - # Verify the class was still called correctly - mock_path_class.assert_called_once_with("/abs/repo/packages/my-lib") - mock_determine_ns.assert_called_once_with(mock_parent, mock_instance) - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_verify_library_namespace_error_no_directory( - mocker, mock_path_class, is_mono_repo -): - """Tests that the specific ValueError is raised if the path isn't a directory.""" - mock_instance = mock_path_class.return_value - mock_instance.is_dir.return_value = False # Configure the failure case - - with pytest.raises(ValueError, match="Error: Path is not a directory"): - _verify_library_namespace("my-lib", "repo", is_mono_repo) - - # Verify the function was called and triggered the check - mock_path_class.assert_called_once_with( - "repo/packages/my-lib" if is_mono_repo else "repo" - ) - - -@pytest.mark.parametrize("is_mono_repo", [False, True]) -def test_verify_library_namespace_error_no_gapic_file( - mocker, mock_path_class, is_mono_repo -): - """Tests that the specific ValueError is raised if no gapic files are found.""" - mock_instance = mock_path_class.return_value - mock_instance.is_dir.return_value = True - mock_instance.rglob.return_value = [] # rglob returns an empty list - - with pytest.raises(ValueError, match="Library is missing a `gapic_version.py`"): - _verify_library_namespace("my-lib", "repo", is_mono_repo) - - # Verify the initial path logic still ran - mock_path_class.assert_called_once_with( - "repo/packages/my-lib" if is_mono_repo else "repo" - ) - - -def test_get_repo_name_from_repo_metadata_success(mocker): - """Tests that the repo name is returned when it exists.""" - mocker.patch( - "cli._read_json_file", return_value={"repo": "googleapis/google-cloud-python"} - ) - repo_name = _get_repo_name_from_repo_metadata("base", "library_id", False) - assert repo_name == "googleapis/google-cloud-python" - - -def test_get_repo_name_from_repo_metadata_missing_repo(mocker): - """Tests that a ValueError is raised when the repo field is missing.""" - mocker.patch("cli._read_json_file", return_value={}) - with pytest.raises(ValueError): - _get_repo_name_from_repo_metadata("base", "library_id", False) diff --git a/.github/workflows/generator.yml b/.github/workflows/generator.yml deleted file mode 100644 index b7944e066e86..000000000000 --- a/.github/workflows/generator.yml +++ /dev/null @@ -1,35 +0,0 @@ -on: - pull_request: - branches: - - main -name: generator - -permissions: - contents: read - -jobs: - test_generator_cli: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - # Use a fetch-depth of 2 - # See https://github.com/googleapis/google-cloud-python/issues/12013 - # and https://github.com/actions/checkout#checkout-head. - with: - fetch-depth: 2 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.14" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r .generator/requirements-test.in - - name: Run generator_cli tests - run: | - pytest .generator - - name: Check coverage - run: | - pytest --cov=. --cov-report=term-missing --cov-fail-under=95 - working-directory: .generator diff --git a/cloudbuild-exitgate.yaml b/cloudbuild-exitgate.yaml deleted file mode 100644 index fa43568da860..000000000000 --- a/cloudbuild-exitgate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# TODO(https://github.com/googleapis/google-cloud-python/issues/14142): -# Reduce this timeout by moving the installation of Python runtimes to a separate base image -timeout: 7200s # 2 hours for the first uncached run, can be lowered later. -steps: - - name: 'gcr.io/cloud-builders/docker' - args: ['build','-f', '.generator/Dockerfile', '-t', 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/python-librarian-generator', '.'] -images: - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/python-librarian-generator' -options: - logging: CLOUD_LOGGING_ONLY - machineType: E2_HIGHCPU_32 diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml deleted file mode 100644 index 164359ebd5ed..000000000000 --- a/cloudbuild-test.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# TODO(https://github.com/googleapis/google-cloud-python/issues/14142): -# Reduce this timeout by moving the installation of Python runtimes to a separate base image -timeout: 7200s # 2 hours for the first uncached run, can be lowered later. -steps: - # Build the generator image using Kaniko and push it to the registry as a - # verification that the image builds successfully. - - name: 'gcr.io/kaniko-project/executor:latest' - id: 'build-generator' - args: - # Specifies the Dockerfile path - - '--dockerfile=.generator/Dockerfile' - # Specifies the build context directory - - '--context=.' - # The final destination for the image - - '--destination=gcr.io/$PROJECT_ID/python-librarian-generator:latest' - # Enables Kaniko's remote registry caching - - '--cache=true' - # Sets a time-to-live for cache layers - - '--cache-ttl=24h' - -options: - default_logs_bucket_behavior: REGIONAL_USER_OWNED_BUCKET - machineType: E2_HIGHCPU_32 From eee155e57979882b0952790dc5fdc27be0032ee0 Mon Sep 17 00:00:00 2001 From: Cody Oss Date: Mon, 29 Jun 2026 20:08:25 +0000 Subject: [PATCH 2/2] rm skip release --- librarian.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/librarian.yaml b/librarian.yaml index 075cf34ad674..0d0abe910e5a 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -59,7 +59,6 @@ default: libraries: - name: bigframes version: 2.44.0 - skip_release: true python: library_type: INTEGRATION - name: bigquery-magics @@ -2227,7 +2226,6 @@ libraries: default_version: v1 - name: google-crc32c version: 1.8.0 - skip_release: true python: library_type: OTHER - name: google-developer-knowledge @@ -2572,7 +2570,6 @@ libraries: default_version: apiVersion - name: pandas-gbq version: 0.35.0 - skip_release: true python: library_type: INTEGRATION - name: proto-plus @@ -2581,7 +2578,6 @@ libraries: library_type: CORE - name: sqlalchemy-bigquery version: 1.17.0 - skip_release: true python: library_type: INTEGRATION - name: sqlalchemy-spanner