diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..015cd136b0c6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git/ +.github/ +.kokoro/ +.vscode/ +bazel-*/ +googleapis/ +**/output/ +**/target/ +**/*.jar diff --git a/.gitignore b/.gitignore index 241e3aa3b52c..66a7aa817548 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ MODULE.bazel* .flattened-pom.xml +# vscode files +.vscode/settings.json + # Vim files *.swp *.swo @@ -79,4 +82,4 @@ monorepo *.tfstate.*.backup *.tfstate.lock.info -.jqwik-database \ No newline at end of file +.jqwik-database diff --git a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 3bfe19506e1d..e3731adf192b 100644 --- a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -51,7 +51,7 @@ steps: cd /workspace git clone https://github.com/googleapis/google-cloud-java cd google-cloud-java - git checkout chore/test-hermetic-build + git checkout chore/test-hermetic-build-parallel mkdir ../golden cd ../golden cp -r ../google-cloud-java/java-apigee-connect . @@ -82,13 +82,12 @@ steps: "run", "--rm", "-v", "/workspace/google-cloud-java:/workspace", - "-v", "/workspace/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config", "-v", "/workspace/googleapis:/workspace/apis", # Fix gapic-generator-java so that the generation result stays # the same. "-v", "/workspace/gapic-generator-java.jar:/home/.library_generation/gapic-generator-java.jar", "${_TEST_IMAGE}", - "--generation-config-path=/workspace/config/generation_config.yaml", + "--generation-config-path=/workspace/generation_config.yaml", "--api-definitions-path=/workspace/apis" ] env: diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py b/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py index e9d14685c911..70b9f1aef713 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py @@ -166,3 +166,30 @@ def _construct_effective_arg( arguments += ["--destination_path", temp_destination_path] return arguments + + +import sys +from io import StringIO +import traceback + + +def library_generation_worker(config, library_path, library, repo_config): + buffer = StringIO() + has_local = hasattr(sys.stdout, "local") + if has_local: + sys.stdout.local.buffer = buffer + sys.stderr.local.buffer = buffer + error_msg = None + try: + generate_composed_library(config, library_path, library, repo_config) + except Exception as e: + error_msg = f"{e}\n{traceback.format_exc()}" + finally: + logs = buffer.getvalue() + buffer.close() + if has_local: + if hasattr(sys.stdout.local, "buffer"): + del sys.stdout.local.buffer + if hasattr(sys.stderr.local, "buffer"): + del sys.stderr.local.buffer + return logs, error_msg diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_library.sh b/sdk-platform-java/hermetic_build/library_generation/generate_library.sh index e7c5896ea707..c05e717ef483 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_library.sh +++ b/sdk-platform-java/hermetic_build/library_generation/generate_library.sh @@ -115,7 +115,9 @@ if [ -z "${artifact}" ]; then artifact="" fi -temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM" +# Use mktemp to guarantee collision-free unique directories when multiple +# library generation processes run concurrently in a shared output folder +temp_destination_path=$(mktemp -d -p "${output_folder}" temp_preprocessed-XXXXXX) mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py index 634ddffd384f..1807c4f29550 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py @@ -13,14 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import os import shutil +import sys +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Optional import library_generation.utils.utilities as util from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig from common.utils.proto_path_utils import ends_with_version -from library_generation.generate_composed_library import generate_composed_library +from library_generation.generate_composed_library import ( + generate_composed_library, + library_generation_worker, +) from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing from common.model.gapic_config import GapicConfig @@ -57,14 +64,13 @@ def generate_from_yaml( ) # copy api definition to output folder. shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) - for library_path, library in repo_config.get_libraries().items(): - print(f"generating library {library.get_library_name()}") - generate_composed_library( - config=config, - library_path=library_path, - library=library, - repo_config=repo_config, - ) + sys.stdout = ThreadLocalStream(original_stdout) + sys.stderr = ThreadLocalStream(original_stderr) + try: + _generate_libraries_in_parallel(config, repo_config) + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr if not config.is_monorepo() or config.contains_common_protos(): return @@ -152,3 +158,63 @@ def _get_target_libraries_from_api_path( target_libraries.append(target_library) return target_libraries return [] + + + +class ThreadLocalStream: + """ + Thread-safe interceptor to route print() statements into thread-local buffers. + Necessary because sys.stdout is a global stream; direct threading writes interleave outputs. + """ + + def __init__(self, original_stream): + self.original_stream = original_stream + self.local = threading.local() + + @property + def buffer(self): + return getattr(self.local, "buffer", None) + + def write(self, data): + writer = self.buffer if self.buffer is not None else self.original_stream + writer.write(data) + + def flush(self): + if self.buffer is not None: + self.buffer.flush() + self.original_stream.flush() + + +original_stdout, original_stderr = sys.stdout, sys.stderr + +print_lock = threading.Lock() + + +def _print_worker_result(lib_name, logs, err): + """ + Atomically prints the buffered output of a worker thread directly to original_stdout, + preventing output interleaving in the console. + """ + with print_lock: + status = "[FAILURE]" if err else "[SUCCESS]" + original_stdout.write(f"\n{'='*40}\n{status} Logs for {lib_name}:\n{'='*40}\n") + original_stdout.write(logs) + if err: + original_stdout.write(f"\nError details:\n{err}\n") + original_stdout.flush() + + +def _generate_libraries_in_parallel(config, repo_config): + cores = os.cpu_count() or 4 + with ThreadPoolExecutor(max_workers=min(cores, 5)) as executor: + futures = { + executor.submit( + library_generation_worker, config, path, lib, repo_config + ): lib.get_library_name() + for path, lib in repo_config.get_libraries().items() + } + + for future in as_completed(futures): + lib_name = futures[future] + logs, err = future.result() + _print_worker_result(lib_name, logs, err) diff --git a/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh b/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh index 3118f32a2b09..9947a3f5c210 100755 --- a/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh +++ b/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh @@ -60,7 +60,9 @@ echo "...done" # write or restore pom.xml files echo "Generating missing pom.xml..." -python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}" +# Under parallel multi-library generation, fix_poms.py modifies the shared versions_file. +# We use flock to serialize edits safely across concurrent processes. +flock "${versions_file}" python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}" echo "...done" # write or restore clirr-ignored-differences.xml diff --git a/sdk-platform-java/tracing-sample b/sdk-platform-java/tracing-sample new file mode 160000 index 000000000000..9712ecf5c5c7 --- /dev/null +++ b/sdk-platform-java/tracing-sample @@ -0,0 +1 @@ +Subproject commit 9712ecf5c5c76d3d671c3f76db55d6eb3a72db1a