chore: explore parallel library generation

.dockerignore

@@ -0,0 +1,9 @@
+.git/
+.github/
+.kokoro/
+.vscode/
+bazel-*/
+googleapis/
+**/output/
+**/target/
+**/*.jar

.gitignore

@@ -22,6 +22,9 @@ MODULE.bazel*
 
 .flattened-pom.xml
 
+# vscode files
+.vscode/settings.json
+
 # Vim files
 *.swp
 *.swo
@@ -79,4 +82,4 @@ monorepo
 *.tfstate.*.backup
 *.tfstate.lock.info
 
-.jqwik-database
+.jqwik-database

sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml

@@ -51,7 +51,7 @@ steps:
     cd /workspace
     git clone https://github.com/googleapis/google-cloud-java
     cd google-cloud-java
-    git checkout chore/test-hermetic-build
+    git checkout chore/test-hermetic-build-parallel
     mkdir ../golden
     cd ../golden
     cp -r ../google-cloud-java/java-apigee-connect .
@@ -82,13 +82,12 @@ steps:
     "run",
     "--rm",
     "-v", "/workspace/google-cloud-java:/workspace",
-    "-v", "/workspace/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config",
     "-v", "/workspace/googleapis:/workspace/apis",
     # Fix gapic-generator-java so that the generation result stays
     # the same.
     "-v", "/workspace/gapic-generator-java.jar:/home/.library_generation/gapic-generator-java.jar",
     "${_TEST_IMAGE}",
-    "--generation-config-path=/workspace/config/generation_config.yaml",
+    "--generation-config-path=/workspace/generation_config.yaml",
     "--api-definitions-path=/workspace/apis"
   ]
   env:

sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py

@@ -166,3 +166,30 @@ def _construct_effective_arg(
     arguments += ["--destination_path", temp_destination_path]
 
     return arguments
+
+
+import sys
+from io import StringIO
+import traceback
+
+
+def library_generation_worker(config, library_path, library, repo_config):
+    buffer = StringIO()
+    has_local = hasattr(sys.stdout, "local")
+    if has_local:
+        sys.stdout.local.buffer = buffer
+        sys.stderr.local.buffer = buffer
+    error_msg = None
+    try:
+        generate_composed_library(config, library_path, library, repo_config)
+    except Exception as e:
+        error_msg = f"{e}\n{traceback.format_exc()}"
+    finally:
+        logs = buffer.getvalue()
+        buffer.close()
+        if has_local:
+            if hasattr(sys.stdout.local, "buffer"):
+                del sys.stdout.local.buffer
+            if hasattr(sys.stderr.local, "buffer"):
+                del sys.stderr.local.buffer
+    return logs, error_msg

sdk-platform-java/hermetic_build/library_generation/generate_library.sh

@@ -115,7 +115,9 @@ if [ -z "${artifact}" ]; then
   artifact=""
 fi
 
-temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM"
+# Use mktemp to guarantee collision-free unique directories when multiple 
+# library generation processes run concurrently in a shared output folder
+temp_destination_path=$(mktemp -d -p "${output_folder}" temp_preprocessed-XXXXXX)
 mkdir -p "${output_folder}/${destination_path}"
 if [ -d "${temp_destination_path}" ]; then
   # we don't want the preprocessed sources of a previous run

sdk-platform-java/hermetic_build/library_generation/generate_repo.py

@@ -13,14 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import copy
+import os
 import shutil
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from typing import Optional
 import library_generation.utils.utilities as util
 from common.model.generation_config import GenerationConfig
 from common.model.library_config import LibraryConfig
 from common.utils.proto_path_utils import ends_with_version
-from library_generation.generate_composed_library import generate_composed_library
+from library_generation.generate_composed_library import (
+    generate_composed_library,
+    library_generation_worker,
+)
 from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing
 
 from common.model.gapic_config import GapicConfig
@@ -57,14 +64,13 @@ def generate_from_yaml(
     )
     # copy api definition to output folder.
     shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True)
-    for library_path, library in repo_config.get_libraries().items():
-        print(f"generating library {library.get_library_name()}")
-        generate_composed_library(
-            config=config,
-            library_path=library_path,
-            library=library,
-            repo_config=repo_config,
-        )
+    sys.stdout = ThreadLocalStream(original_stdout)
+    sys.stderr = ThreadLocalStream(original_stderr)
+    try:
+        _generate_libraries_in_parallel(config, repo_config)
+    finally:
+        sys.stdout = original_stdout
+        sys.stderr = original_stderr
 
     if not config.is_monorepo() or config.contains_common_protos():
         return
@@ -152,3 +158,63 @@ def _get_target_libraries_from_api_path(
             target_libraries.append(target_library)
             return target_libraries
     return []
+
+
+
+class ThreadLocalStream:
+    """
+    Thread-safe interceptor to route print() statements into thread-local buffers.
+    Necessary because sys.stdout is a global stream; direct threading writes interleave outputs.
+    """
+
+    def __init__(self, original_stream):
+        self.original_stream = original_stream
+        self.local = threading.local()
+
+    @property
+    def buffer(self):
+        return getattr(self.local, "buffer", None)
+
+    def write(self, data):
+        writer = self.buffer if self.buffer is not None else self.original_stream
+        writer.write(data)
+
+    def flush(self):
+        if self.buffer is not None:
+            self.buffer.flush()
+        self.original_stream.flush()
+
+
+original_stdout, original_stderr = sys.stdout, sys.stderr
+
+print_lock = threading.Lock()
+
+
+def _print_worker_result(lib_name, logs, err):
+    """
+    Atomically prints the buffered output of a worker thread directly to original_stdout,
+    preventing output interleaving in the console.
+    """
+    with print_lock:
+        status = "[FAILURE]" if err else "[SUCCESS]"
+        original_stdout.write(f"\n{'='*40}\n{status} Logs for {lib_name}:\n{'='*40}\n")
+        original_stdout.write(logs)
+        if err:
+            original_stdout.write(f"\nError details:\n{err}\n")
+        original_stdout.flush()
+
+
+def _generate_libraries_in_parallel(config, repo_config):
+    cores = os.cpu_count() or 4
+    with ThreadPoolExecutor(max_workers=min(cores, 5)) as executor:
+        futures = {
+            executor.submit(
+                library_generation_worker, config, path, lib, repo_config
+            ): lib.get_library_name()
+            for path, lib in repo_config.get_libraries().items()
+        }
+
+        for future in as_completed(futures):
+            lib_name = futures[future]
+            logs, err = future.result()
+            _print_worker_result(lib_name, logs, err)

sdk-platform-java/hermetic_build/library_generation/owlbot/bin/entrypoint.sh

@@ -60,7 +60,9 @@ echo "...done"
 
 # write or restore pom.xml files
 echo "Generating missing pom.xml..."
-python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}"
+# Under parallel multi-library generation, fix_poms.py modifies the shared versions_file.
+# We use flock to serialize edits safely across concurrent processes.
+flock "${versions_file}" python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}"
 echo "...done"
 
 # write or restore clirr-ignored-differences.xml