From e262563c3df0fdbae8cfaa68df13adc8e38ff0e2 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 16 Apr 2026 08:43:18 -0400 Subject: [PATCH 1/8] chore(hermetic-build): explore parallel generation --- .dockerignore | 9 ++ ...d-library-generation-integration-test.yaml | 2 +- .../generate_composed_library.py | 15 ++++ .../library_generation/generate_library.sh | 4 +- .../library_generation/generate_repo.py | 85 +++++++++++++++++-- .../owlbot/bin/entrypoint.sh | 4 +- 6 files changed, 107 insertions(+), 12 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..015cd136b0c6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git/ +.github/ +.kokoro/ +.vscode/ +bazel-*/ +googleapis/ +**/output/ +**/target/ +**/*.jar diff --git a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 3bfe19506e1d..1e6eb9e9d51d 100644 --- a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -51,7 +51,7 @@ steps: cd /workspace git clone https://github.com/googleapis/google-cloud-java cd google-cloud-java - git checkout chore/test-hermetic-build + git checkout chore/test-hermetic-build-parallel mkdir ../golden cd ../golden cp -r ../google-cloud-java/java-apigee-connect . diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py b/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py index e9d14685c911..20920124cb6a 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py @@ -166,3 +166,18 @@ def _construct_effective_arg( arguments += ["--destination_path", temp_destination_path] return arguments + + +import sys +from io import StringIO +import traceback + + +def library_generation_worker(config, library_path, library, repo_config): + error_msg = None + try: + generate_composed_library(config, library_path, library, repo_config) + except Exception as e: + error_msg = f"{e}\n{traceback.format_exc()}" + + return "", error_msg diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_library.sh b/sdk-platform-java/hermetic_build/library_generation/generate_library.sh index e7c5896ea707..c05e717ef483 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_library.sh +++ b/sdk-platform-java/hermetic_build/library_generation/generate_library.sh @@ -115,7 +115,9 @@ if [ -z "${artifact}" ]; then artifact="" fi -temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM" +# Use mktemp to guarantee collision-free unique directories when multiple +# library generation processes run concurrently in a shared output folder +temp_destination_path=$(mktemp -d -p "${output_folder}" temp_preprocessed-XXXXXX) mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py index 634ddffd384f..3486c5485aa3 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py @@ -20,7 +20,10 @@ from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig from common.utils.proto_path_utils import ends_with_version -from library_generation.generate_composed_library import generate_composed_library +from library_generation.generate_composed_library import ( + generate_composed_library, + library_generation_worker, +) from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing from common.model.gapic_config import GapicConfig @@ -57,14 +60,9 @@ def generate_from_yaml( ) # copy api definition to output folder. shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) - for library_path, library in repo_config.get_libraries().items(): - print(f"generating library {library.get_library_name()}") - generate_composed_library( - config=config, - library_path=library_path, - library=library, - repo_config=repo_config, - ) + _generate_libraries_in_parallel(config, repo_config) + sys.stdout = original_stdout + sys.stderr = original_stderr if not config.is_monorepo() or config.contains_common_protos(): return @@ -152,3 +150,72 @@ def _get_target_libraries_from_api_path( target_libraries.append(target_library) return target_libraries return [] + + +import os +import sys +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed + + +class ThreadLocalStream: + """ + Thread-safe interceptor to route print() statements into thread-local buffers. + Necessary because sys.stdout is a global stream; direct threading writes interleave outputs. + """ + + def __init__(self, original_stream): + self.original_stream = original_stream + self.local = threading.local() + + @property + def buffer(self): + return getattr(self.local, "buffer", None) + + def write(self, data): + writer = self.buffer if self.buffer is not None else self.original_stream + writer.write(data) + + def flush(self): + if self.buffer is not None: + return + self.original_stream.flush() + + +original_stdout, original_stderr = sys.stdout, sys.stderr + +print_lock = threading.Lock() + + +def _print_worker_result(lib_name, logs, err): + """ + Atomically prints the buffered output of a worker thread directly to original_stdout, + preventing output interleaving in the console. + """ + print_lock.acquire() + status = "[FAILURE]" if err else "[SUCCESS]" + original_stdout.write(f"\n{'='*40}\n{status} Logs for {lib_name}:\n{'='*40}\n") + original_stdout.write(logs) + if err: + original_stdout.write(f"\nError details:\n{err}\n") + original_stdout.flush() + print_lock.release() + + +def _generate_libraries_in_parallel(config, repo_config): + cores = os.cpu_count() or 4 + executor = ThreadPoolExecutor(max_workers=min(cores, 5)) + + futures = { + executor.submit( + library_generation_worker, config, path, lib, repo_config + ): lib.get_library_name() + for path, lib in repo_config.get_libraries().items() + } + + for future in as_completed(futures): + lib_name = futures[future] + logs, err = future.result() + _print_worker_result(lib_name, logs, err) + + executor.shutdown() diff --git a/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh b/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh index 3118f32a2b09..9947a3f5c210 100755 --- a/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh +++ b/sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh @@ -60,7 +60,9 @@ echo "...done" # write or restore pom.xml files echo "Generating missing pom.xml..." -python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}" +# Under parallel multi-library generation, fix_poms.py modifies the shared versions_file. +# We use flock to serialize edits safely across concurrent processes. +flock "${versions_file}" python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}" echo "...done" # write or restore clirr-ignored-differences.xml From 84eb79d0de9fd8fecab9bbd259265b9cdcbc183b Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 16 Apr 2026 09:25:39 -0400 Subject: [PATCH 2/8] chore: address gemini comments --- .../generate_composed_library.py | 16 +++++- .../library_generation/generate_repo.py | 51 +++++++++---------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py b/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py index 20920124cb6a..70b9f1aef713 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py @@ -174,10 +174,22 @@ def _construct_effective_arg( def library_generation_worker(config, library_path, library, repo_config): + buffer = StringIO() + has_local = hasattr(sys.stdout, "local") + if has_local: + sys.stdout.local.buffer = buffer + sys.stderr.local.buffer = buffer error_msg = None try: generate_composed_library(config, library_path, library, repo_config) except Exception as e: error_msg = f"{e}\n{traceback.format_exc()}" - - return "", error_msg + finally: + logs = buffer.getvalue() + buffer.close() + if has_local: + if hasattr(sys.stdout.local, "buffer"): + del sys.stdout.local.buffer + if hasattr(sys.stderr.local, "buffer"): + del sys.stderr.local.buffer + return logs, error_msg diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py index 3486c5485aa3..fd698a64d9b6 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py @@ -13,7 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import os import shutil +import sys +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Optional import library_generation.utils.utilities as util @@ -152,11 +156,6 @@ def _get_target_libraries_from_api_path( return [] -import os -import sys -import threading -from concurrent.futures import ThreadPoolExecutor, as_completed - class ThreadLocalStream: """ @@ -192,30 +191,26 @@ def _print_worker_result(lib_name, logs, err): Atomically prints the buffered output of a worker thread directly to original_stdout, preventing output interleaving in the console. """ - print_lock.acquire() - status = "[FAILURE]" if err else "[SUCCESS]" - original_stdout.write(f"\n{'='*40}\n{status} Logs for {lib_name}:\n{'='*40}\n") - original_stdout.write(logs) - if err: - original_stdout.write(f"\nError details:\n{err}\n") - original_stdout.flush() - print_lock.release() + with print_lock: + status = "[FAILURE]" if err else "[SUCCESS]" + original_stdout.write(f"\n{'='*40}\n{status} Logs for {lib_name}:\n{'='*40}\n") + original_stdout.write(logs) + if err: + original_stdout.write(f"\nError details:\n{err}\n") + original_stdout.flush() def _generate_libraries_in_parallel(config, repo_config): cores = os.cpu_count() or 4 - executor = ThreadPoolExecutor(max_workers=min(cores, 5)) - - futures = { - executor.submit( - library_generation_worker, config, path, lib, repo_config - ): lib.get_library_name() - for path, lib in repo_config.get_libraries().items() - } - - for future in as_completed(futures): - lib_name = futures[future] - logs, err = future.result() - _print_worker_result(lib_name, logs, err) - - executor.shutdown() + with ThreadPoolExecutor(max_workers=min(cores, 5)) as executor: + futures = { + executor.submit( + library_generation_worker, config, path, lib, repo_config + ): lib.get_library_name() + for path, lib in repo_config.get_libraries().items() + } + + for future in as_completed(futures): + lib_name = futures[future] + logs, err = future.result() + _print_worker_result(lib_name, logs, err) From e69b564e4e75c396e1fe6d81c5fd8cacb805faf6 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 16 Apr 2026 09:25:57 -0400 Subject: [PATCH 3/8] chore: triggere whole repo generation. --- .../cloudbuild-library-generation-integration-test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml index 1e6eb9e9d51d..e3731adf192b 100644 --- a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml @@ -82,13 +82,12 @@ steps: "run", "--rm", "-v", "/workspace/google-cloud-java:/workspace", - "-v", "/workspace/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config", "-v", "/workspace/googleapis:/workspace/apis", # Fix gapic-generator-java so that the generation result stays # the same. "-v", "/workspace/gapic-generator-java.jar:/home/.library_generation/gapic-generator-java.jar", "${_TEST_IMAGE}", - "--generation-config-path=/workspace/config/generation_config.yaml", + "--generation-config-path=/workspace/generation_config.yaml", "--api-definitions-path=/workspace/apis" ] env: From 05624e01b772f084de2a3f1ca1b90a65159ce3bd Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 16 Apr 2026 09:48:58 -0400 Subject: [PATCH 4/8] fix: hook to streamlined stdout --- .vscode/settings.json | 30 +++++++++++++++++++ googleapis | 1 + .../library_generation/generate_repo.py | 10 +++++-- sdk-platform-java/tracing-sample | 1 + 4 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 .vscode/settings.json create mode 160000 googleapis create mode 160000 sdk-platform-java/tracing-sample diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000000..ca8c6cb6a0a4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,30 @@ +{ + "workbench.colorCustomizations": { + "activityBar.activeBackground": "#6dc273", + "activityBar.background": "#6dc273", + "activityBar.foreground": "#15202b", + "activityBar.inactiveForeground": "#15202b99", + "activityBarBadge.background": "#6862bd", + "activityBarBadge.foreground": "#e7e7e7", + "commandCenter.border": "#15202b99", + "sash.hoverBorder": "#6dc273", + "statusBar.background": "#4ab252", + "statusBar.foreground": "#15202b", + "statusBarItem.hoverBackground": "#3b8e41", + "statusBarItem.remoteBackground": "#4ab252", + "statusBarItem.remoteForeground": "#15202b", + "titleBar.activeBackground": "#4ab252", + "titleBar.activeForeground": "#15202b", + "titleBar.inactiveBackground": "#4ab25299", + "titleBar.inactiveForeground": "#15202b99" + }, + "peacock.color": "#4ab252", + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/.DS_Store": true, + "**/Thumbs.db": true, + "java-*/**": true, + } +} \ No newline at end of file diff --git a/googleapis b/googleapis new file mode 160000 index 000000000000..13ca8dbc7975 --- /dev/null +++ b/googleapis @@ -0,0 +1 @@ +Subproject commit 13ca8dbc797515144104ee799e429f8e29b45c08 diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py index fd698a64d9b6..a9565b2a9c9f 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py @@ -64,9 +64,13 @@ def generate_from_yaml( ) # copy api definition to output folder. shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) - _generate_libraries_in_parallel(config, repo_config) - sys.stdout = original_stdout - sys.stderr = original_stderr + sys.stdout = ThreadLocalStream(original_stdout) + sys.stderr = ThreadLocalStream(original_stderr) + try: + _generate_libraries_in_parallel(config, repo_config) + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr if not config.is_monorepo() or config.contains_common_protos(): return diff --git a/sdk-platform-java/tracing-sample b/sdk-platform-java/tracing-sample new file mode 160000 index 000000000000..9712ecf5c5c7 --- /dev/null +++ b/sdk-platform-java/tracing-sample @@ -0,0 +1 @@ +Subproject commit 9712ecf5c5c76d3d671c3f76db55d6eb3a72db1a From 817d11a76a7af5687c141022c83afc4748a0841b Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 16 Apr 2026 09:57:20 -0400 Subject: [PATCH 5/8] chore: remove unwanted file --- .vscode/settings.json | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index ca8c6cb6a0a4..000000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "workbench.colorCustomizations": { - "activityBar.activeBackground": "#6dc273", - "activityBar.background": "#6dc273", - "activityBar.foreground": "#15202b", - "activityBar.inactiveForeground": "#15202b99", - "activityBarBadge.background": "#6862bd", - "activityBarBadge.foreground": "#e7e7e7", - "commandCenter.border": "#15202b99", - "sash.hoverBorder": "#6dc273", - "statusBar.background": "#4ab252", - "statusBar.foreground": "#15202b", - "statusBarItem.hoverBackground": "#3b8e41", - "statusBarItem.remoteBackground": "#4ab252", - "statusBarItem.remoteForeground": "#15202b", - "titleBar.activeBackground": "#4ab252", - "titleBar.activeForeground": "#15202b", - "titleBar.inactiveBackground": "#4ab25299", - "titleBar.inactiveForeground": "#15202b99" - }, - "peacock.color": "#4ab252", - "files.exclude": { - "**/.git": true, - "**/.svn": true, - "**/.hg": true, - "**/.DS_Store": true, - "**/Thumbs.db": true, - "java-*/**": true, - } -} \ No newline at end of file From d01a1359d8eec7b176624e33a0d1d50725564d5d Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 16 Apr 2026 09:58:13 -0400 Subject: [PATCH 6/8] chore: ignore .vscode folder --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 241e3aa3b52c..66a7aa817548 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ MODULE.bazel* .flattened-pom.xml +# vscode files +.vscode/settings.json + # Vim files *.swp *.swo @@ -79,4 +82,4 @@ monorepo *.tfstate.*.backup *.tfstate.lock.info -.jqwik-database \ No newline at end of file +.jqwik-database From 7fc7f144fa14e8b75e3a3c8e07f86074d3554d2d Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Thu, 16 Apr 2026 14:04:27 +0000 Subject: [PATCH 7/8] chore: generate libraries at Thu Apr 16 14:01:43 UTC 2026 --- googleapis | 1 - 1 file changed, 1 deletion(-) delete mode 160000 googleapis diff --git a/googleapis b/googleapis deleted file mode 160000 index 13ca8dbc7975..000000000000 --- a/googleapis +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 13ca8dbc797515144104ee799e429f8e29b45c08 From 0fa43c0af8f2db0368f255f7d61ad5130fe6a728 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 16 Apr 2026 13:32:38 -0400 Subject: [PATCH 8/8] fix: flush buffer --- .../hermetic_build/library_generation/generate_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py index a9565b2a9c9f..1807c4f29550 100755 --- a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py +++ b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py @@ -181,7 +181,7 @@ def write(self, data): def flush(self): if self.buffer is not None: - return + self.buffer.flush() self.original_stream.flush()