diff --git a/.librarian/generator-input/client-post-processing/storage-integration.yaml b/.librarian/generator-input/client-post-processing/storage-integration.yaml index 42d18b232129..99bab51cd92e 100644 --- a/.librarian/generator-input/client-post-processing/storage-integration.yaml +++ b/.librarian/generator-input/client-post-processing/storage-integration.yaml @@ -43,8 +43,9 @@ replacements: "Credentials must be set via environment variable GOOGLE_APPLICATION_CREDENTIALS" ) - # Install all test dependencies - session.install("mock", "pytest", "pytest-cov", "brotli") + # Install dependencies for the unit test environment + unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES + session.install(*unit_deps_all) # Install dependencies needed for system tests session.install( @@ -59,7 +60,8 @@ replacements: prerel_deps = [ "google-api-core", - "google-auth", + # Exclude google-auth 3.0.0.dev0 which was yanked + "google-auth!=3.0.0.dev0", "google-cloud-core", "google-crc32c", "google-resumable-media", @@ -88,9 +90,7 @@ replacements: f"import {version_namespace}; print({version_namespace}.__version__)", ) # Remaining dependencies - other_deps = [ - "requests", - ] + other_deps = ["requests", "pyopenssl"] session.install(*other_deps) session.run( diff --git a/librarian.yaml b/librarian.yaml index d612822c614d..c885164b5869 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -3593,7 +3593,6 @@ libraries: apis: - path: google/storage/v2 description_override: 'is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally.' - skip_generate: true python: library_type: GAPIC_MANUAL opt_args_by_api: diff --git a/packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml index f62a6395828c..820688940074 100644 --- a/packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -8,128 +8,130 @@ substitutions: steps: - # Step 0: Generate a persistent SSH key for this build run. - # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "generate-ssh-key" - entrypoint: "bash" - args: - - "-c" - - | - mkdir -p /workspace/.ssh - # Generate the SSH key - ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb - # Save the public key content to a file for the cleanup step - cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub - waitFor: ["-"] - - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "cleanup-old-keys" - entrypoint: "bash" - args: - - "-c" - - | - #!/bin/bash - set -e - - echo "Fetching OS Login SSH keys..." - echo "Removing all keys." - echo "---------------------------------------------------------------------" - - FINGERPRINTS_TO_DELETE=$$(gcloud compute os-login ssh-keys list \ - --format="value(fingerprint)") - - echo "Keys to delete: $$FINGERPRINTS_TO_DELETE" - - if [ -z "$$FINGERPRINTS_TO_DELETE" ]; then - echo "No keys found to delete. Nothing to do." - exit 0 - fi - - while IFS= read -r FINGERPRINT; do - if [ -n "$$FINGERPRINT" ]; then - echo "Deleting key with fingerprint: $$FINGERPRINT" - gcloud compute os-login ssh-keys remove \ - --key="$$FINGERPRINT" \ - --quiet || true - fi - done <<< "$$FINGERPRINTS_TO_DELETE" - - echo "---------------------------------------------------------------------" - echo "Cleanup complete." - - # Step 1 Create a GCE VM to run the tests. - # The VM is created in the same zone as the buckets to test rapid storage features. - # It's given the 'cloud-platform' scope to allow it to access GCS and other services. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-vm" - entrypoint: "gcloud" - args: - - "compute" - - "instances" - - "create" - - "${_VM_NAME}" - - "--project=${PROJECT_ID}" - - "--zone=${_ZONE}" - - "--machine-type=e2-medium" - - "--image-family=debian-13" - - "--image-project=debian-cloud" - - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" - - "--scopes=https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/devstorage.read_write,https://www.googleapis.com/auth/cloudkms" - - "--metadata=enable-oslogin=TRUE" - waitFor: ["-"] - - # Step 2: Run the integration tests inside the newly created VM and cleanup. - # This step uses 'gcloud compute ssh' to execute a remote script. - # The VM is deleted after tests are run, regardless of success. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "run-tests-and-delete-vm" - entrypoint: "bash" - args: - - "-c" - - | - set -e - # Wait for the VM to be fully initialized and SSH to be ready. - for i in {1..10}; do - if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then - break - fi - echo "Waiting for VM to become available... (attempt $i/10)" - sleep 15 - done - # copy the script to the VM - gcloud compute scp packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine - - # Execute the script on the VM via SSH. - # Capture the exit code to ensure cleanup happens before the build fails. - set +e - gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh" - EXIT_CODE=$? - set -e - - echo "--- Deleting GCE VM ---" - gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet - - # Exit with the original exit code from the test script. - exit $$EXIT_CODE - waitFor: - - "create-vm" - - "generate-ssh-key" - - "cleanup-old-keys" - - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "cleanup-ssh-key" - entrypoint: "bash" - args: - - "-c" - - | - echo "--- Removing SSH key from OS Login profile to prevent accumulation ---" - gcloud compute os-login ssh-keys remove \ - --key-file=/workspace/gcb_ssh_key.pub || true - waitFor: - - "run-tests-and-delete-vm" - -timeout: "3600s" # 60 minutes + - name: 'bash' + args: ['echo', 'This job is currently disabled. See https://github.com/googleapis/google-cloud-python/issues/16487. Skipping tests...'] +# # Step 0: Generate a persistent SSH key for this build run. +# # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. +# - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" +# id: "generate-ssh-key" +# entrypoint: "bash" +# args: +# - "-c" +# - | +# mkdir -p /workspace/.ssh +# # Generate the SSH key +# ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb +# # Save the public key content to a file for the cleanup step +# cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub +# waitFor: ["-"] + +# - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" +# id: "cleanup-old-keys" +# entrypoint: "bash" +# args: +# - "-c" +# - | +# #!/bin/bash +# set -e + +# echo "Fetching OS Login SSH keys..." +# echo "Removing all keys." +# echo "---------------------------------------------------------------------" + +# FINGERPRINTS_TO_DELETE=$$(gcloud compute os-login ssh-keys list \ +# --format="value(fingerprint)") + +# echo "Keys to delete: $$FINGERPRINTS_TO_DELETE" + +# if [ -z "$$FINGERPRINTS_TO_DELETE" ]; then +# echo "No keys found to delete. Nothing to do." +# exit 0 +# fi + +# while IFS= read -r FINGERPRINT; do +# if [ -n "$$FINGERPRINT" ]; then +# echo "Deleting key with fingerprint: $$FINGERPRINT" +# gcloud compute os-login ssh-keys remove \ +# --key="$$FINGERPRINT" \ +# --quiet || true +# fi +# done <<< "$$FINGERPRINTS_TO_DELETE" + +# echo "---------------------------------------------------------------------" +# echo "Cleanup complete." + +# # Step 1 Create a GCE VM to run the tests. +# # The VM is created in the same zone as the buckets to test rapid storage features. +# # It's given the 'cloud-platform' scope to allow it to access GCS and other services. +# - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" +# id: "create-vm" +# entrypoint: "gcloud" +# args: +# - "compute" +# - "instances" +# - "create" +# - "${_VM_NAME}" +# - "--project=${PROJECT_ID}" +# - "--zone=${_ZONE}" +# - "--machine-type=e2-medium" +# - "--image-family=debian-13" +# - "--image-project=debian-cloud" +# - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" +# - "--scopes=https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/devstorage.read_write,https://www.googleapis.com/auth/cloudkms" +# - "--metadata=enable-oslogin=TRUE" +# waitFor: ["-"] + +# # Step 2: Run the integration tests inside the newly created VM and cleanup. +# # This step uses 'gcloud compute ssh' to execute a remote script. +# # The VM is deleted after tests are run, regardless of success. +# - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" +# id: "run-tests-and-delete-vm" +# entrypoint: "bash" +# args: +# - "-c" +# - | +# set -e +# # Wait for the VM to be fully initialized and SSH to be ready. +# for i in {1..10}; do +# if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then +# break +# fi +# echo "Waiting for VM to become available... (attempt $i/10)" +# sleep 15 +# done +# # copy the script to the VM +# gcloud compute scp packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + +# # Execute the script on the VM via SSH. +# # Capture the exit code to ensure cleanup happens before the build fails. +# set +e +# gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh" +# EXIT_CODE=$? +# set -e + +# echo "--- Deleting GCE VM ---" +# gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet + +# # Exit with the original exit code from the test script. +# exit $$EXIT_CODE +# waitFor: +# - "create-vm" +# - "generate-ssh-key" +# - "cleanup-old-keys" + +# - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" +# id: "cleanup-ssh-key" +# entrypoint: "bash" +# args: +# - "-c" +# - | +# echo "--- Removing SSH key from OS Login profile to prevent accumulation ---" +# gcloud compute os-login ssh-keys remove \ +# --key-file=/workspace/gcb_ssh_key.pub || true +# waitFor: +# - "run-tests-and-delete-vm" + +# timeout: "3600s" # 60 minutes options: logging: CLOUD_LOGGING_ONLY diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py index ab76631a94be..48f786c7654d 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py @@ -18,8 +18,7 @@ from google.api_core.bidi_async import AsyncBidiRpc from google.cloud import _storage_v2 -from google.cloud.storage import Blob -from google.cloud.storage import _grpc_conversions +from google.cloud.storage import Blob, _grpc_conversions from google.cloud.storage.asyncio import _utils from google.cloud.storage.asyncio.async_abstract_object_stream import ( _AsyncAbstractObjectStream, diff --git a/packages/google-cloud-storage/noxfile.py b/packages/google-cloud-storage/noxfile.py index 4f6a1e0402bd..25c24ccb8b6f 100644 --- a/packages/google-cloud-storage/noxfile.py +++ b/packages/google-cloud-storage/noxfile.py @@ -567,8 +567,9 @@ def prerelease_deps(session, protobuf_implementation): "Credentials must be set via environment variable GOOGLE_APPLICATION_CREDENTIALS" ) - # Install all test dependencies - session.install("mock", "pytest", "pytest-cov", "brotli") + # Install dependencies for the unit test environment + unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES + session.install(*unit_deps_all) # Install dependencies needed for system tests session.install( @@ -583,7 +584,8 @@ def prerelease_deps(session, protobuf_implementation): prerel_deps = [ "google-api-core", - "google-auth", + # Exclude google-auth 3.0.0.dev0 which was yanked + "google-auth!=3.0.0.dev0", "google-cloud-core", "google-crc32c", "google-resumable-media", @@ -612,9 +614,7 @@ def prerelease_deps(session, protobuf_implementation): f"import {version_namespace}; print({version_namespace}.__version__)", ) # Remaining dependencies - other_deps = [ - "requests", - ] + other_deps = ["requests", "pyopenssl"] session.install(*other_deps) session.run( diff --git a/packages/google-cloud-storage/tests/system/test_zonal.py b/packages/google-cloud-storage/tests/system/test_zonal.py index 30eccd4039b1..b4a94585b619 100644 --- a/packages/google-cloud-storage/tests/system/test_zonal.py +++ b/packages/google-cloud-storage/tests/system/test_zonal.py @@ -11,13 +11,12 @@ import pytest from google.api_core.exceptions import FailedPrecondition, NotFound, OutOfRange +# current library imports +from google.cloud import kms from google.cloud.storage.asyncio.async_appendable_object_writer import ( _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, ) - -# current library imports -from google.cloud import kms from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, diff --git a/packages/google-cloud-storage/tests/unit/asyncio/retry/test_reads_resumption_strategy.py b/packages/google-cloud-storage/tests/unit/asyncio/retry/test_reads_resumption_strategy.py index 54ba6bfbf9aa..dc27cb701974 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/retry/test_reads_resumption_strategy.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/retry/test_reads_resumption_strategy.py @@ -16,8 +16,8 @@ import io import unittest -from google.api_core import exceptions import google_crc32c +from google.api_core import exceptions from google.cloud import _storage_v2 as storage_v2 from google.cloud._storage_v2.types.storage import BidiReadObjectRedirectedError diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py index 87ed431ff97a..c0e62eb54242 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -19,10 +19,10 @@ import pytest from google.api_core import exceptions from google.rpc import status_pb2 -from google.cloud.storage import Blob from google.cloud._storage_v2.types import storage as storage_type from google.cloud._storage_v2.types.storage import BidiWriteObjectRedirectedError +from google.cloud.storage import Blob from google.cloud.storage.asyncio.async_appendable_object_writer import ( _DEFAULT_FLUSH_INTERVAL_BYTES, _MAX_CHUNK_SIZE_BYTES, diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index 6c90fc10f5d8..24a632b68131 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -17,9 +17,9 @@ from unittest import mock from unittest.mock import AsyncMock +import google_crc32c import pytest from google.api_core import exceptions -import google_crc32c from google.cloud import _storage_v2 from google.cloud.storage.asyncio import async_read_object_stream diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py index af003a5b5bb3..cd6bdbdb440a 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py @@ -19,8 +19,7 @@ import pytest from google.cloud import _storage_v2 -from google.cloud.storage import Blob -from google.cloud.storage import Bucket +from google.cloud.storage import Blob, Bucket from google.cloud.storage.asyncio.async_write_object_stream import ( _AsyncWriteObjectStream, )