From f9e736f91b5cea07cc9ab50cfaed074036ecd3a8 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 30 Apr 2026 16:26:02 +0200 Subject: [PATCH 1/4] Remove Ceph runtime configuration. Drop the Ceph CI service, helper scripts, and dedicated test location config now that CloudServer no longer supports Ceph RadosGW. Remove the CI_CEPH runtime overrides from Config so location constraints follow the normal validation path, and keep report capabilities from advertising Ceph support even if an external config still provides those legacy flags. Issue: CLDSRV-825 --- .github/ceph/Dockerfile | 25 -- .github/ceph/entrypoint-wrapper.sh | 37 --- .github/ceph/wait_for_ceph.sh | 11 - .github/docker/docker-compose.yaml | 8 +- lib/Config.js | 8 +- lib/utilities/reportHandler.js | 6 +- tests/locationConfig/locationConfigCeph.json | 274 ------------------- tests/unit/utils/reportHandler.js | 21 +- 8 files changed, 21 insertions(+), 369 deletions(-) delete mode 100644 .github/ceph/Dockerfile delete mode 100644 .github/ceph/entrypoint-wrapper.sh delete mode 100644 .github/ceph/wait_for_ceph.sh delete mode 100644 tests/locationConfig/locationConfigCeph.json diff --git a/.github/ceph/Dockerfile b/.github/ceph/Dockerfile deleted file mode 100644 index df1a146405..0000000000 --- a/.github/ceph/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7 - -ENV CEPH_DAEMON demo -ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw - -ENV CEPH_DEMO_UID zenko -ENV CEPH_DEMO_ACCESS_KEY accessKey1 -ENV CEPH_DEMO_SECRET_KEY verySecretKey1 -ENV CEPH_DEMO_BUCKET zenkobucket - -ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0 -ENV MON_IP 0.0.0.0 -ENV NETWORK_AUTO_DETECT 4 -ENV RGW_CIVETWEB_PORT 8001 - -RUN rm /etc/yum.repos.d/tcmu-runner.repo - -ADD ./entrypoint-wrapper.sh / -RUN chmod +x /entrypoint-wrapper.sh && \ - yum install -y python-pip && \ - yum clean all && \ - pip install awscli && \ - rm -rf /root/.cache/pip - -ENTRYPOINT [ "/entrypoint-wrapper.sh" ] diff --git a/.github/ceph/entrypoint-wrapper.sh b/.github/ceph/entrypoint-wrapper.sh deleted file mode 100644 index 676b4ce4f9..0000000000 --- a/.github/ceph/entrypoint-wrapper.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -touch /artifacts/ceph.log -mkfifo /tmp/entrypoint_output -# We run this in the background so that we can tail the RGW log after init, -# because entrypoint.sh never returns - -# The next line will be needed when ceph builds 3.2.2 so I'll leave it here -# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output & - -bash /entrypoint.sh > /tmp/entrypoint_output & -entrypoint_pid="$!" -while read -r line; do - echo $line - # When we find this line server has started - if [ -n "$(echo $line | grep 'Creating bucket')" ]; then - break - fi -done < /tmp/entrypoint_output - -# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused -s3cmd mb s3://cephbucket s3://cephbucket2 - -mkdir /root/.aws -cat > /root/.aws/credentials </dev/null)" ]; do - sleep 1 - echo -n "." -done diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index 452705cf9d..e8f0c3f45c 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -91,16 +91,10 @@ services: image: ${KMS_IMAGE:-nsmithuk/local-kms:3.11.7} mongo: network_mode: "host" - profiles: ['mongo', 'ceph'] + profiles: ['mongo'] image: ${MONGODB_IMAGE} volumes: - /tmp/artifacts/${JOB_NAME}:/logs - ceph: - network_mode: "host" - profiles: ['ceph'] - image: ghcr.io/scality/cloudserver/ci-ceph - volumes: - - /tmp/artifacts/${JOB_NAME}/ceph:/artifacts sproxyd: network_mode: "host" profiles: ['sproxyd'] diff --git a/lib/Config.js b/lib/Config.js index 9faea9e96a..51e452fa18 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -387,10 +387,7 @@ function locationConstraintAssert(locationConstraints) { assert(locationConstraints[l].isCold === true); } - if (process.env.CI_CEPH === 'true') { - // eslint-disable-next-line no-param-reassign - locationConstraints[l].details.https = false; - } else if (details.https !== undefined) { + if (details.https !== undefined) { assert(typeof details.https === 'boolean', 'bad config: ' + 'locationConstraints[region].details https must be a boolean'); } else { @@ -401,9 +398,6 @@ function locationConstraintAssert(locationConstraints) { if (details.pathStyle !== undefined) { assert(typeof details.pathStyle === 'boolean', 'bad config: ' + 'locationConstraints[region].pathStyle must be a boolean'); - } else if (process.env.CI_CEPH === 'true') { - // eslint-disable-next-line no-param-reassign - locationConstraints[l].details.pathStyle = true; } else { // eslint-disable-next-line no-param-reassign locationConstraints[l].details.pathStyle = false; diff --git a/lib/utilities/reportHandler.js b/lib/utilities/reportHandler.js index a6f116adb5..c468d20c4b 100644 --- a/lib/utilities/reportHandler.js +++ b/lib/utilities/reportHandler.js @@ -44,7 +44,6 @@ function getCapabilities(cfg = config) { locationTypeS3Custom: true, locationTypeSproxyd: true, locationTypeNFS: true, - locationTypeCephRadosGW: true, locationTypeHyperdriveV2: true, locationTypeLocal: true, preferredReadLocation: true, @@ -53,7 +52,6 @@ function getCapabilities(cfg = config) { secureChannelOptimizedPath: true, s3cIngestLocation: true, nfsIngestLocation: false, - cephIngestLocation: false, awsIngestLocation: false, }; @@ -72,11 +70,13 @@ function getCapabilities(cfg = config) { caps.locationTypeDigitalOcean &&= cfg.supportedLocationTypes.has('location-do-spaces-v1'); caps.locationTypeSproxyd &&= cfg.supportedLocationTypes.has('location-scality-sproxyd-v1'); caps.locationTypeNFS &&= cfg.supportedLocationTypes.has('location-nfs-mount-v1'); - caps.locationTypeCephRadosGW &&= cfg.supportedLocationTypes.has('location-ceph-radosgw-s3-v1'); caps.locationTypeHyperdriveV2 &&= cfg.supportedLocationTypes.has('location-scality-hdclient-v2'); caps.locationTypeLocal &&= cfg.supportedLocationTypes.has('location-file-v1'); } + delete caps.locationTypeCephRadosGW; + delete caps.cephIngestLocation; + return caps; } diff --git a/tests/locationConfig/locationConfigCeph.json b/tests/locationConfig/locationConfigCeph.json deleted file mode 100644 index 994ee28de4..0000000000 --- a/tests/locationConfig/locationConfigCeph.json +++ /dev/null @@ -1,274 +0,0 @@ -{ - "us-east-1": { - "type": "file", - "objectId": "us-east-1", - "legacyAwsBehavior": true, - "details": {} - }, - "us-east-2": { - "type": "file", - "objectId": "us-east-2", - "legacyAwsBehavior": true, - "details": {} - }, - "scality-internal-file": { - "type": "file", - "objectId": "scality-internal-file", - "legacyAwsBehavior": false, - "details": {} - }, - "dataFile": { - "type": "file", - "objectId": "dataFile", - "legacyAwsBehavior": false, - "details": {} - }, - "scality-internal-mem": { - "type": "mem", - "objectId": "scality-internal-mem", - "legacyAwsBehavior": false, - "details": {} - }, - "scality-us-west-1": { - "type": "mem", - "objectId": "scality-us-west-1", - "legacyAwsBehavior": false, - "details": {} - }, - "awsbackendencryption": { - "type": "aws_s3", - "objectId": "awsbackendencryption", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket", - "bucketMatch": true, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true, - "serverSideEncryption": true - } - }, - "awsbackend": { - "type": "aws_s3", - "objectId": "awsbackend", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket", - "bucketMatch": true, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true - } - }, - "awsbackendhttp": { - "type": "aws_s3", - "objectId": "awsbackendhttp", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket", - "bucketMatch": true, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true - } - }, - "awsbackendmismatch": { - "type": "aws_s3", - "objectId": "awsbackendmismatch", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket", - "bucketMatch": false, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true - } - }, - "awsbackend2": { - "type": "aws_s3", - "objectId": "awsbackend2", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket2", - "bucketMatch": true, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true - } - }, - "awsbackendPathStyle": { - "type": "aws_s3", - "objectId": "awsbackendPathStyle", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket", - "bucketMatch": true, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true - } - }, - "azurebackend": { - "type": "azure", - "objectId": "azurebackend", - "legacyAwsBehavior": true, - "details": { - "azureStorageEndpoint": "https://fakeaccountname.blob.core.fake.net/", - "azureStorageAccountName": "fakeaccountname", - "azureStorageAccessKey": "Fake00Key001", - "bucketMatch": true, - "azureContainerName": "s3test" - } - }, - "azurebackend2": { - "type": "azure", - "objectId": "azurebackend2", - "legacyAwsBehavior": true, - "details": { - "azureStorageEndpoint": "https://fakeaccountname2.blob.core.fake.net/", - "azureStorageAccountName": "fakeaccountname2", - "azureStorageAccessKey": "Fake00Key002", - "bucketMatch": true, - "azureContainerName": "s3test2" - } - }, - "azurebackendmismatch": { - "type": "azure", - "objectId": "azurebackendmismatch", - "legacyAwsBehavior": true, - "details": { - "azureStorageEndpoint": "https://fakeaccountname.blob.core.fake.net/", - "azureStorageAccountName": "fakeaccountname", - "azureStorageAccessKey": "Fake00Key001", - "bucketMatch": false, - "azureContainerName": "s3test" - } - }, - "azurenonexistcontainer": { - "type": "azure", - "objectId": "azurenonexistcontainer", - "legacyAwsBehavior": true, - "details": { - "azureStorageEndpoint": "https://fakeaccountname.blob.core.fake.net/", - "azureStorageAccountName": "fakeaccountname", - "azureStorageAccessKey": "Fake00Key123", - "bucketMatch": true, - "azureContainerName": "s3createbucketonfly" - } - }, - "gcpbackend": { - "type": "gcp", - "objectId": "gcpbackend", - "legacyAwsBehavior": true, - "details": { - "gcpEndpoint": "storage.googleapis.com", - "bucketName": "zenko-gcp-bucket", - "mpuBucketName": "zenko-gcp-mpu", - "bucketMatch": true, - "credentialsProfile": "google" - } - }, - "gcpbackend2": { - "type": "gcp", - "objectId": "gcpbackend2", - "legacyAwsBehavior": true, - "details": { - "gcpEndpoint": "storage.googleapis.com", - "bucketName": "zenko-gcp-bucket-2", - "mpuBucketName": "zenko-gcp-mpu-2", - "bucketMatch": true, - "credentialsProfile": "google_2" - } - }, - "gcpbackendmismatch": { - "type": "gcp", - "objectId": "gcpbackendmismatch", - "legacyAwsBehavior": true, - "details": { - "gcpEndpoint": "storage.googleapis.com", - "bucketName": "zenko-gcp-bucket", - "mpuBucketName": "zenko-gcp-mpu", - "bucketMatch": false, - "credentialsProfile": "google" - } - }, - "withversioning": { - "type": "aws_s3", - "objectId": "withversioning", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket", - "bucketMatch": false, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true, - "supportsVersioning": true - } - }, - "withoutversioning": { - "type": "aws_s3", - "objectId": "withoutversioning", - "legacyAwsBehavior": true, - "details": { - "awsEndpoint": "127.0.0.1:8001", - "https": false, - "bucketName": "cephbucket", - "bucketMatch": false, - "credentials": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "pathStyle": true, - "supportsVersioning": false - } - }, - "transientfile": { - "type": "file", - "objectId": "transientfile", - "legacyAwsBehavior": false, - "details": {}, - "isTransient": true - }, - "location-dmf-v1": { - "type": "tlp", - "objectId": "location-dmf-v1", - "legacyAwsBehavior": false, - "isCold": true, - "details": {} - }, - "location-crr-v1": { - "type": "crr", - "objectId": "location-crr-v1", - "legacyAwsBehavior": false, - "isCRR": true, - "details": {} - } -} diff --git a/tests/unit/utils/reportHandler.js b/tests/unit/utils/reportHandler.js index d26b107cef..000c146e3b 100644 --- a/tests/unit/utils/reportHandler.js +++ b/tests/unit/utils/reportHandler.js @@ -29,7 +29,6 @@ describe('reportHandler.getCapabilities', () => { locationTypeS3Custom: true, locationTypeSproxyd: true, locationTypeNFS: true, - locationTypeCephRadosGW: true, locationTypeHyperdriveV2: true, locationTypeLocal: true, preferredReadLocation: true, @@ -38,7 +37,6 @@ describe('reportHandler.getCapabilities', () => { secureChannelOptimizedPath: true, s3cIngestLocation: true, nfsIngestLocation: false, - cephIngestLocation: false, awsIngestLocation: false, }); }); @@ -65,6 +63,22 @@ describe('reportHandler.getCapabilities', () => { }); }); + it('should not report Ceph capability fields from config', () => { + const cfg = { + capabilities: { + locationTypeCephRadosGW: true, + cephIngestLocation: true, + customCapability: 'test-value', + }, + supportedLifecycleRules: ['Expiration'], + }; + const caps = getCapabilities(cfg); + + assert.deepStrictEqual(caps, { + customCapability: 'test-value', + }); + }); + it('should apply LOCAL_VOLUME_CAPABILITY env when set to false', () => { process.env.LOCAL_VOLUME_CAPABILITY = 'false'; const cfg = { @@ -217,7 +231,6 @@ describe('reportHandler.getCapabilities', () => { const locationTypes = [ 'location-gcp-v1', 'location-scality-sproxyd-v1', - 'location-ceph-radosgw-s3-v1', 'location-file-v1', 'location-scality-artesca-s3-v1', ]; @@ -229,7 +242,6 @@ describe('reportHandler.getCapabilities', () => { locationTypeS3Custom: true, locationTypeSproxyd: true, locationTypeNFS: true, - locationTypeCephRadosGW: true, locationTypeHyperdriveV2: true, locationTypeLocal: true, locationTypes, @@ -245,7 +257,6 @@ describe('reportHandler.getCapabilities', () => { assert.strictEqual(caps.locationTypeDigitalOcean, false); assert.strictEqual(caps.locationTypeSproxyd, true); assert.strictEqual(caps.locationTypeNFS, false); - assert.strictEqual(caps.locationTypeCephRadosGW, true); assert.strictEqual(caps.locationTypeHyperdriveV2, false); assert.strictEqual(caps.locationTypeLocal, true); }); From 6d701adc40b113121140b4c7ee3f80928ccf983a Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 30 Apr 2026 16:26:19 +0200 Subject: [PATCH 2/4] Remove Ceph-specific test handling. Delete the Ceph-only Mocha helpers and replace Ceph-conditioned skips with the regular multiple-backend guards so the remaining suites run for supported backends. Clean up Ceph-specific expected errors, comments, and fixture references from functional and multiple-backend tests while preserving the generic S3 behavior coverage. Issue: CLDSRV-825 --- .../aws-node-sdk/lib/utility/test-utils.js | 9 -------- .../delete/deleteAwsVersioning.js | 22 ++++++------------- .../multipleBackend/delete/deleteAzure.js | 4 ++-- .../test/multipleBackend/delete/deleteGcp.js | 4 ++-- .../test/multipleBackend/get/getAzure.js | 3 ++- .../test/multipleBackend/get/getGcp.js | 6 ++--- .../multipleBackend/initMPU/initMPUAzure.js | 6 ++--- .../multipleBackend/initMPU/initMPUGcp.js | 6 ++--- .../listParts/azureListParts.js | 6 ++--- .../multipleBackend/listParts/listPartsGcp.js | 4 ++-- .../multipleBackend/mpuAbort/abortMPUGcp.js | 6 ++--- .../multipleBackend/mpuAbort/azureAbortMPU.js | 7 +++--- .../mpuComplete/azureCompleteMPU.js | 4 ++-- .../mpuComplete/completeMPUGcp.js | 6 ++--- .../multipleBackend/mpuParts/azurePutPart.js | 9 ++++---- .../multipleBackend/mpuParts/putPartGcp.js | 9 ++++---- .../objectCopy/azureObjectCopy.js | 4 ++-- .../multipleBackend/objectCopy/objectCopy.js | 10 ++++----- .../objectCopy/objectCopyAwsVersioning.js | 11 +++++----- .../objectPutCopyPartAzure.js | 11 +++++----- .../objectPutCopyPart/objectPutCopyPartGcp.js | 8 +++---- .../objectTagging/objectTagging.js | 11 ++++------ .../test/multipleBackend/put/put.js | 6 +---- .../test/multipleBackend/put/putAzure.js | 4 ++-- .../test/multipleBackend/put/putGcp.js | 9 ++++---- .../test/multipleBackend/utils.js | 12 ++-------- .../aws-node-sdk/test/object/deleteObject.js | 6 ++--- .../aws-node-sdk/test/object/get.js | 4 +--- .../test/object/getObjectLegalHold.js | 4 +--- .../aws-node-sdk/test/object/getRetention.js | 4 +--- .../test/object/multiObjectDelete.js | 4 +--- .../aws-node-sdk/test/object/objectCopy.js | 4 +--- .../aws-node-sdk/test/object/objectHead.js | 4 +--- .../aws-node-sdk/test/object/put.js | 4 +--- .../test/object/putObjectLegalHold.js | 6 ++--- .../aws-node-sdk/test/object/putRetention.js | 6 ++--- .../aws-node-sdk/test/object/rangeTest.js | 4 +--- .../test/object/websiteGetWithACL.js | 5 ----- tests/multipleBackend/multipartUpload.js | 19 ++++------------ tests/multipleBackend/objectPut.js | 13 +++++------ tests/multipleBackend/objectPutCopyPart.js | 14 ++++-------- tests/multipleBackend/routes/routeBackbeat.js | 22 ++++++++----------- 42 files changed, 119 insertions(+), 201 deletions(-) diff --git a/tests/functional/aws-node-sdk/lib/utility/test-utils.js b/tests/functional/aws-node-sdk/lib/utility/test-utils.js index 4c5533bcbe..8e90ab8e95 100644 --- a/tests/functional/aws-node-sdk/lib/utility/test-utils.js +++ b/tests/functional/aws-node-sdk/lib/utility/test-utils.js @@ -1,14 +1,9 @@ const { config } = require('../../../../../lib/Config'); -const isCEPH = process.env.CI_CEPH !== undefined; -const itSkipCeph = isCEPH ? it.skip : it; -const describeSkipIfCeph = isCEPH ? describe.skip : describe.skip; // always skip let describeSkipIfNotMultiple = describe.skip; -let describeSkipIfNotMultipleOrCeph = describe.skip; if (config.backends.data === 'multiple') { describeSkipIfNotMultiple = describe; - describeSkipIfNotMultipleOrCeph = isCEPH ? describe.skip : describe.skip; // always skip } function hasLocation(lc) { @@ -24,11 +19,7 @@ function hasLocation(lc) { const hasColdStorage = config.supportedLifecycleRules.some(rule => rule.endsWith('Transition')); module.exports = { - isCEPH, - itSkipCeph, - describeSkipIfCeph, describeSkipIfNotMultiple, - describeSkipIfNotMultipleOrCeph, hasLocation, hasColdStorage, }; diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js index b0c6c9fb74..e8ded07b8f 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js @@ -28,7 +28,6 @@ const { awsGetLatestVerId, getAwsRetry, genUniqID, - isCEPH, waitForVersioningBeforePut, } = require('../utils'); @@ -206,9 +205,8 @@ describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + key, versionId: 'null', resultType: deleteVersion }, err => next(err, awsVerId)), (awsVerId, next) => { - const wanted = isCEPH ? 'NoSuchKey' : 'NoSuchVersion'; _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: wanted }, next); + versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); }, ], done); }); @@ -243,9 +241,8 @@ describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + key, versionId: 'null', resultType: deleteVersion }, err => next(err, awsVerId)), (awsVerId, next) => { - const wanted = isCEPH ? 'NoSuchKey' : 'NoSuchVersion'; _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: wanted }, next); + versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); }, ], done); }); @@ -263,9 +260,8 @@ describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + key, versionId: s3VerId, resultType: deleteVersion }, err => next(err, awsVerId)), (awsVerId, next) => { - const wanted = isCEPH ? 'NoSuchKey' : 'NoSuchVersion'; _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: wanted }, next); + versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); }, ], done); }); @@ -525,9 +521,8 @@ describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + (awsVid, next) => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, () => next(null, awsVid)), (awsVerId, next) => { - const wanted = isCEPH ? 'NoSuchKey' : 'NoSuchVersion'; _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: wanted }, next); + versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); }, ], done); }); @@ -682,9 +677,8 @@ describeSkipIfNotMultiple('AWS backend delete multiple objects w. versioning: ' key, versionId: 'null', resultType: deleteVersion }, err => next(err, awsVerId)), (awsVerId, next) => { - const wanted = isCEPH ? 'NoSuchKey' : 'NoSuchVersion'; _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: wanted }, next); + versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); }, ], done); }); @@ -700,9 +694,8 @@ describeSkipIfNotMultiple('AWS backend delete multiple objects w. versioning: ' key, versionId: 'null', resultType: deleteVersion }, err => next(err, awsVerId)), (awsVerId, next) => { - const wanted = isCEPH ? 'NoSuchKey' : 'NoSuchVersion'; _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: wanted }, next); + versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); }, ], done); }); @@ -720,9 +713,8 @@ describeSkipIfNotMultiple('AWS backend delete multiple objects w. versioning: ' key, versionId: s3VerId, resultType: deleteVersion }, err => next(err, awsVerId)), (awsVerId, next) => { - const wanted = isCEPH ? 'NoSuchKey' : 'NoSuchVersion'; _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: wanted }, next); + versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); }, ], done); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js index 68cdae5af3..780f8fd632 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js @@ -9,13 +9,13 @@ const { CreateBucketCommand, const BucketUtility = require('../../../lib/utility/bucket-util'); const withV4 = require('../../support/withV4'); const { - describeSkipIfNotMultipleOrCeph, uniqName, getAzureClient, getAzureContainerName, getAzureKeys, azureLocation, azureLocationMismatch, + describeSkipIfNotMultiple, } = require('../utils'); const keyObject = 'deleteazure'; @@ -30,7 +30,7 @@ const nonExistingId = process.env.AWS_ON_AIR ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : '3939393939393939393936493939393939393939756e6437'; -describeSkipIfNotMultipleOrCeph('Multiple backend delete object from Azure', +describeSkipIfNotMultiple('Multiple backend delete object from Azure', function testSuite() { this.timeout(250000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js index a3c962e8ac..6bf5daf676 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js @@ -6,10 +6,10 @@ const { CreateBucketCommand, const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { - describeSkipIfNotMultipleOrCeph, gcpLocation, gcpLocationMismatch, genUniqID, + describeSkipIfNotMultiple, } = require('../utils'); const bucket = `deletegcp${genUniqID()}`; @@ -20,7 +20,7 @@ const mismatchObject = `mismatchObject-${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); -describeSkipIfNotMultipleOrCeph('Multiple backend delete', +describeSkipIfNotMultiple('Multiple backend delete', function testSuite() { this.timeout(120000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js index b874096de5..7f1d5e25dc 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js @@ -12,6 +12,7 @@ const { getAzureContainerName, getAzureKeys, azureLocation, + describeSkipIfNotMultiple, } = require('../utils'); const azureClient = getAzureClient(); @@ -23,7 +24,7 @@ const normalBody = Buffer.from('I am a body', 'utf8'); const azureTimeout = 10000; -describe.skip('Multiple backend get object from Azure', +describeSkipIfNotMultiple('Multiple backend get object from Azure', function testSuite() { this.timeout(30000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js index fdb3f585de..a0895a7b48 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js @@ -5,10 +5,10 @@ const { PutObjectCommand, CreateBucketCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { - describeSkipIfNotMultipleOrCeph, gcpLocation, gcpLocationMismatch, genUniqID, + describeSkipIfNotMultiple, } = require('../utils'); const bucket = `getgcp${genUniqID()}`; @@ -53,7 +53,7 @@ describe('Multiple backend get object', function testSuite() { }); }); - describeSkipIfNotMultipleOrCeph('with objects in GCP', () => { + describeSkipIfNotMultiple('with objects in GCP', () => { before(() => { process.stdout.write('Putting object to GCP\n'); return s3.send(new PutObjectCommand({ Bucket: bucket, Key: gcpObject, @@ -128,7 +128,7 @@ describe('Multiple backend get object', function testSuite() { }); }); - describeSkipIfNotMultipleOrCeph('with bucketMatch set to false', () => { + describeSkipIfNotMultiple('with bucketMatch set to false', () => { beforeEach(done => { s3.send(new PutObjectCommand({ Bucket: bucket, Key: mismatchObject, Body: body, Metadata: { 'scal-location-constraint': gcpLocationMismatch } })).then(() => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js index a4674bf2c2..33c131ff31 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js @@ -9,8 +9,8 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, azureLocation, getAzureContainerName, - genUniqID } = require('../utils'); +const { azureLocation, getAzureContainerName, + genUniqID, describeSkipIfNotMultiple } = require('../utils'); const keyName = `somekey-${genUniqID()}`; @@ -18,7 +18,7 @@ const azureContainerName = getAzureContainerName(azureLocation); let s3; let bucketUtil; -describeSkipIfNotMultipleOrCeph('Initiate MPU to AZURE', () => { +describeSkipIfNotMultiple('Initiate MPU to AZURE', () => { withV4(sigCfg => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js index e09c2a3381..e2075dbd12 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js @@ -10,8 +10,8 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, gcpClient, gcpBucketMPU, gcpLocation, - genUniqID } = require('../utils'); +const { gcpClient, gcpBucketMPU, gcpLocation, + genUniqID, describeSkipIfNotMultiple } = require('../utils'); const { createMpuKey } = arsenal.storage.data.external.GcpUtils; const bucket = `initmpugcp${genUniqID()}`; @@ -20,7 +20,7 @@ const keyName = `somekey-${genUniqID()}`; let s3; let bucketUtil; -describeSkipIfNotMultipleOrCeph('Initiate MPU to GCP', () => { +describeSkipIfNotMultiple('Initiate MPU to GCP', () => { withV4(sigCfg => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js index a6b6bf1cec..385b60b523 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js @@ -7,8 +7,8 @@ const { CreateBucketCommand, const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, azureLocation, getAzureContainerName, - genUniqID } = require('../utils'); +const { azureLocation, getAzureContainerName, + genUniqID, describeSkipIfNotMultiple } = require('../utils'); const azureContainerName = getAzureContainerName(azureLocation); const firstPartSize = 10; @@ -19,7 +19,7 @@ const bodySecondPart = Buffer.alloc(secondPartSize); let bucketUtil; let s3; -describeSkipIfNotMultipleOrCeph('List parts of MPU on Azure data backend', +describeSkipIfNotMultiple('List parts of MPU on Azure data backend', () => { withV4(sigCfg => { beforeEach(function beforeEachFn() { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js index 56a34b7ee5..d1093d04f5 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js @@ -6,7 +6,7 @@ const { CreateBucketCommand, ListPartsCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, gcpLocation, genUniqID } +const { gcpLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `listpartsgcp${genUniqID()}`; @@ -18,7 +18,7 @@ const bodySecondPart = Buffer.alloc(secondPartSize); let bucketUtil; let s3; -describeSkipIfNotMultipleOrCeph('List parts of MPU on GCP data backend', () => { +describeSkipIfNotMultiple('List parts of MPU on GCP data backend', () => { withV4(sigCfg => { beforeEach(function beforeEachFn() { this.currentTest.key = `somekey-${genUniqID()}`; diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js index 099d68e230..8133eafcfb 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js @@ -11,8 +11,8 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, gcpClient, gcpBucket, gcpBucketMPU, - gcpLocation, uniqName, genUniqID } = require('../utils'); +const { gcpClient, gcpBucket, gcpBucketMPU, + gcpLocation, uniqName, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const keyObject = 'abortgcp'; const bucket = `abortmpugcp${genUniqID()}`; @@ -38,7 +38,7 @@ function checkMPUList(bucket, key, uploadId, cb) { }); } -describeSkipIfNotMultipleOrCeph('Abort MPU on GCP data backend', function +describeSkipIfNotMultiple('Abort MPU on GCP data backend', function descrbeFn() { this.timeout(180000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js index c1912ec164..fe74499b07 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js @@ -12,8 +12,9 @@ const { const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, uniqName, getAzureClient, - getAzureContainerName, convertMD5, azureLocation } = require('../utils'); +const { uniqName, getAzureClient, + getAzureContainerName, convertMD5, azureLocation, + describeSkipIfNotMultiple } = require('../utils'); const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const maxSubPartSize = azureMpuUtils.maxSubPartSize; @@ -40,7 +41,7 @@ function azureCheck(container, key, expected, cb) { }); } -describeSkipIfNotMultipleOrCeph('Abort MPU on Azure data backend', function +describeSkipIfNotMultiple('Abort MPU on Azure data backend', function describeF() { this.timeout(50000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js index 8438d6ac79..19e17e48b7 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js @@ -10,7 +10,6 @@ const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { - describeSkipIfNotMultipleOrCeph, fileLocation, awsS3, awsLocation, @@ -20,6 +19,7 @@ const { getAzureClient, getAzureContainerName, genUniqID, + describeSkipIfNotMultiple, } = require('../utils'); const azureMpuUtils = s3middleware.azureHelper.mpuUtils; @@ -114,7 +114,7 @@ function mpuSetup(key, location, cb) { }); } -describeSkipIfNotMultipleOrCeph('Complete MPU API for Azure data backend', +describeSkipIfNotMultiple('Complete MPU API for Azure data backend', function testSuite() { this.timeout(150000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js index cf6d5e67b8..0b346fb4cb 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js @@ -11,9 +11,9 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, fileLocation, awsS3, awsLocation, +const { fileLocation, awsS3, awsLocation, awsBucket, gcpClient, gcpBucket, gcpLocation, gcpLocationMismatch, - genUniqID } = require('../utils'); + genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `completempugcp${genUniqID()}`; const smallBody = Buffer.from('I am a body', 'utf8'); @@ -103,7 +103,7 @@ function mpuSetup(key, location, cb) { }); } -describeSkipIfNotMultipleOrCeph('Complete MPU API for GCP data backend', +describeSkipIfNotMultiple('Complete MPU API for GCP data backend', function testSuite() { this.timeout(150000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js index 89021bd303..f342472f90 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js @@ -14,8 +14,9 @@ const { const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, expectedETag, uniqName, getAzureClient, - getAzureContainerName, convertMD5, azureLocation, azureLocationMismatch } +const { expectedETag, uniqName, getAzureClient, + getAzureContainerName, convertMD5, azureLocation, azureLocationMismatch, + describeSkipIfNotMultiple } = require('../utils'); const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const maxSubPartSize = azureMpuUtils.maxSubPartSize; @@ -62,7 +63,7 @@ function azureCheck(key, cb) { .catch(err => cb(err)); } -describeSkipIfNotMultipleOrCeph('MultipleBackend put part to AZURE', function +describeSkipIfNotMultiple('MultipleBackend put part to AZURE', function describeF() { this.timeout(80000); withV4(sigCfg => { @@ -382,7 +383,7 @@ describeF() { }); }); -describeSkipIfNotMultipleOrCeph('MultipleBackend put part to AZURE ' + +describeSkipIfNotMultiple('MultipleBackend put part to AZURE ' + 'location with bucketMatch sets to false', function describeF() { this.timeout(80000); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js index 5b1809f9a0..cbd8222074 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js @@ -13,8 +13,9 @@ const arsenal = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, gcpClient, gcpBucket, gcpBucketMPU, - gcpLocation, gcpLocationMismatch, uniqName, genUniqID } +const { gcpClient, gcpBucket, gcpBucketMPU, + gcpLocation, gcpLocationMismatch, uniqName, genUniqID, + describeSkipIfNotMultiple } = require('../utils'); const { createMpuKey } = arsenal.storage.data.external.GcpUtils; @@ -46,7 +47,7 @@ function checkMPUResult(bucket, key, uploadId, objCount, expected, cb) { }); } -describeSkipIfNotMultipleOrCeph('MultipleBacked put part to GCP', function +describeSkipIfNotMultiple('MultipleBacked put part to GCP', function describeFn() { this.timeout(180000); withV4(sigCfg => { @@ -290,7 +291,7 @@ describeFn() { }); }); -describeSkipIfNotMultipleOrCeph('MultipleBackend put part to GCP location ' + +describeSkipIfNotMultiple('MultipleBackend put part to GCP location ' + 'with bucketMatch sets to false', function describeF() { this.timeout(80000); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js index 18e4ec51c4..a2fdfe3cec 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js @@ -11,7 +11,6 @@ const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const constants = require('../../../../../../constants'); const { - describeSkipIfNotMultipleOrCeph, getAzureClient, getAzureContainerName, convertMD5, @@ -21,6 +20,7 @@ const { azureLocation2, azureLocationMismatch, genUniqID, + describeSkipIfNotMultiple, } = require('../utils'); const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); @@ -148,7 +148,7 @@ destBucket, destLoc, azureKey, mdDirective, objSize, callback) { }); } -describeSkipIfNotMultipleOrCeph('MultipleBackend object copy: Azure', +describeSkipIfNotMultiple('MultipleBackend object copy: Azure', function testSuite() { this.timeout(250000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js index 718167eb75..4cece36110 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js @@ -15,7 +15,7 @@ const { config } = require('../../../../../../lib/Config'); const { getRealAwsConfig } = require('../../support/awsConfig'); const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); -const { describeSkipIfNotMultiple, itSkipCeph, awsS3, memLocation, awsLocation, +const { describeSkipIfNotMultiple, awsS3, memLocation, awsLocation, azureLocation, awsLocation2, awsLocationMismatch, awsLocationEncryption, genUniqID } = require('../utils'); @@ -249,7 +249,7 @@ function testSuite() { awsLocation); }); - itSkipCeph('should copy an object from mem to AWS with aws server ' + + it('should copy an object from mem to AWS with aws server ' + 'side encryption', async () => { const key = await putSourceObj(memLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; @@ -287,7 +287,7 @@ function testSuite() { awsS3, awsLocation); }); - itSkipCeph('should copy an object on AWS with aws server side ' + + it('should copy an object on AWS with aws server side ' + 'encryption', async () => { const key = await putSourceObj(awsLocation, false, bucket); @@ -308,7 +308,7 @@ function testSuite() { awsLocation); }); - itSkipCeph('should copy an object on AWS with aws server side ' + + it('should copy an object on AWS with aws server side ' + 'encrypted bucket', async () => { const key = await putSourceObj(awsLocation, false, awsServerSideEncryptionbucket); const copyKey = `copyKey-${genUniqID()}`; @@ -439,7 +439,7 @@ function testSuite() { awsS3Two, awsLocation2); }); - itSkipCeph('should return error AccessDenied copying an object on ' + + it('should return error AccessDenied copying an object on ' + 'AWS to a different AWS account without source object READ access', async () => { const key = await putSourceObj(awsLocation, false, bucket); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js index e85f214994..9a07b4d15e 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js @@ -22,7 +22,6 @@ const { awsGetLatestVerId, getAndAssertResult, genUniqID, - itSkipCeph, } = require('../utils'); const sourceBucketName = `awsversioningsrc${genUniqID()}`; @@ -277,7 +276,7 @@ function testSuite() { destLocation: awsLocation, }); const { isEmptyObj, directive } = testParams; - itSkipCeph(`should copy ${isEmptyObj ? 'an empty' : ''} ` + + it(`should copy ${isEmptyObj ? 'an empty' : ''} ` + 'object from AWS backend non-versioned bucket' + 'to AWS backend versioned bucket ' + `with ${directive} directive`, done => { @@ -298,7 +297,7 @@ function testSuite() { ], done); }); - itSkipCeph(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + + it(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + 'from one AWS backend versioned bucket' + `to another on ${directive} directive`, done => { @@ -320,7 +319,7 @@ function testSuite() { ], done); }); - itSkipCeph(`should copy ${isEmptyObj ? 'an empty ' : ''}null ` + + it(`should copy ${isEmptyObj ? 'an empty ' : ''}null ` + 'version from one AWS backend versioning suspended bucket to ' + ` another versioning suspended bucket with ${directive} directive`, done => { @@ -344,7 +343,7 @@ function testSuite() { ], done); }); - itSkipCeph(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + + it(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + 'from a AWS backend versioned bucket to a versioned-suspended' + `one with ${directive} directive`, done => { Object.assign(testParams, { @@ -367,7 +366,7 @@ function testSuite() { }); }); - itSkipCeph('versioning not configured: if copy object to a ' + + it('versioning not configured: if copy object to a ' + 'pre-existing object on AWS backend, metadata should be overwritten ' + 'but data of previous version in AWS should not be deleted', function itF(done) { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js index 49e2fadbae..b41fb15530 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js @@ -17,11 +17,10 @@ const { config } = require('../../../../../../lib/Config'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { uniqName, getAzureClient, azureLocation, azureLocationMismatch, - memLocation, awsLocation, awsS3, getOwnerInfo, genUniqID } + memLocation, awsLocation, awsS3, getOwnerInfo, genUniqID, + describeSkipIfNotMultiple } = require('../utils'); -const describeSkipIfNotMultipleOrCeph = config.backends.data !== 'multiple' - ? describe.skip : describe.skip; let azureContainerName; @@ -126,7 +125,7 @@ function assertCopyPart(infos, cb) { ], cb); } -describeSkipIfNotMultipleOrCeph('Put Copy Part to AZURE', function describeF() { +describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { this.timeout(800000); withV4(sigCfg => { beforeEach(() => { @@ -649,7 +648,7 @@ describeSkipIfNotMultipleOrCeph('Put Copy Part to AZURE', function describeF() { }); }); -describeSkipIfNotMultipleOrCeph('Put Copy Part to AZURE with large object', +describeSkipIfNotMultiple('Put Copy Part to AZURE with large object', function describeF() { this.timeout(800000); withV4(sigCfg => { @@ -759,7 +758,7 @@ function describeF() { }); }); -describeSkipIfNotMultipleOrCeph('Put Copy Part to AZURE with complete MPU', +describeSkipIfNotMultiple('Put Copy Part to AZURE with complete MPU', function describeF() { this.timeout(800000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js index a739f44ac2..b6e416ab4b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js @@ -15,9 +15,9 @@ const { const { config } = require('../../../../../../lib/Config'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, uniqName, gcpBucketMPU, +const { uniqName, gcpBucketMPU, gcpClient, gcpLocation, gcpLocationMismatch, memLocation, - awsLocation, awsS3, getOwnerInfo, genUniqID } = require('../utils'); + awsLocation, awsS3, getOwnerInfo, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `partcopygcp${genUniqID()}`; @@ -101,7 +101,7 @@ function assertCopyPart(infos, cb) { ], cb); } -describeSkipIfNotMultipleOrCeph('Put Copy Part to GCP', function describeFn() { +describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { this.timeout(800000); withV4(sigCfg => { beforeEach(done => { @@ -643,7 +643,7 @@ describeSkipIfNotMultipleOrCeph('Put Copy Part to GCP', function describeFn() { }); }); -describeSkipIfNotMultipleOrCeph('Put Copy Part to GCP with complete MPU', +describeSkipIfNotMultiple('Put Copy Part to GCP with complete MPU', function describeF() { this.timeout(800000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js index bfa057d2c4..976324d4a6 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js @@ -16,7 +16,7 @@ const BucketUtility = require('../../../lib/utility/bucket-util'); const { describeSkipIfNotMultiple, awsS3, awsBucket, getAwsRetry, getAzureClient, getAzureContainerName, convertMD5, memLocation, fileLocation, awsLocation, azureLocation, genUniqID, - isCEPH } = require('../utils'); +} = require('../utils'); const azureClient = getAzureClient(); const azureContainerName = getAzureContainerName(azureLocation); @@ -34,9 +34,7 @@ let s3; const putParams = { Bucket: bucket, Body: body }; const testBackends = [memLocation, fileLocation, awsLocation]; -if (!isCEPH) { - testBackends.push(azureLocation); -} +testBackends.push(azureLocation); const tagString = 'key1=value1&key2=value2'; const putTags = { @@ -221,8 +219,7 @@ function testSuite() { describe('putObject with tags and putObjectTagging', () => { testBackends.forEach(backend => { - const itSkipIfAzureOrCeph = backend === 'azurebackend' || - isCEPH ? it.skip : it; + const itSkipIfAzure = backend === 'azurebackend' ? it.skip : it; it(`should put an object with tags to ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; @@ -295,7 +292,7 @@ function testSuite() { .catch(done); }); - itSkipIfAzureOrCeph('should put tags to completed MPU ' + + itSkipIfAzure('should put tags to completed MPU ' + `object in ${backend}`, done => { const key = `somekey-${genUniqID()}`; const params = { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js index 4b6c41dd41..66a5c81f07 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js @@ -16,7 +16,7 @@ const { createEncryptedBucketPromise } = const { versioningEnabled } = require('../../../lib/utility/versioning-util'); const { describeSkipIfNotMultiple, getAwsRetry, awsLocation, - awsLocationEncryption, memLocation, fileLocation, genUniqID, isCEPH } + awsLocationEncryption, memLocation, fileLocation, genUniqID } = require('../utils'); const bucket = `putaws${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); @@ -279,10 +279,6 @@ describeSkipIfNotMultiple('MultipleBackend put object', function testSuite() { it('should put an object to AWS with encryption', async () => { - // Test refuses to skip using itSkipCeph so just mark it passed - if (isCEPH) { - return; - } const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body, diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js index 248b056af1..90b0beb2cf 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js @@ -9,7 +9,6 @@ const { CreateBucketCommand, const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { - describeSkipIfNotMultipleOrCeph, uniqName, getAzureClient, getAzureContainerName, @@ -18,6 +17,7 @@ const { fileLocation, azureLocation, azureLocationMismatch, + describeSkipIfNotMultiple, } = require('../utils'); const keyObject = 'putazure'; @@ -50,7 +50,7 @@ function azureGetCheck(objectKey, azureMD5, azureMetadata, cb) { .catch(err => cb(err)); } -describeSkipIfNotMultipleOrCeph('MultipleBackend put object to AZURE', function +describeSkipIfNotMultiple('MultipleBackend put object to AZURE', function describeF() { this.timeout(250000); withV4(sigCfg => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js index 46f9f57efc..922b28399e 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js @@ -7,8 +7,8 @@ const { } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultipleOrCeph, gcpClient, gcpBucket, - gcpLocation, fileLocation, genUniqID } = require('../utils'); +const { gcpClient, gcpBucket, + gcpLocation, fileLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `putgcp${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); @@ -90,7 +90,7 @@ function gcpGetCheck(objectKey, s3MD5, gcpMD5, location, callback) { attempt(); } -describeSkipIfNotMultipleOrCeph('MultipleBackend put object to GCP', function +describeSkipIfNotMultiple('MultipleBackend put object to GCP', function describeFn() { this.timeout(250000); withV4(sigCfg => { @@ -235,8 +235,7 @@ describeFn() { }); }); -describeSkipIfNotMultipleOrCeph('MultipleBackend put object' + - 'based on bucket location', () => { +describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', () => { withV4(sigCfg => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js index 3c5ce82285..e87fc5058b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js @@ -24,10 +24,6 @@ const authdata = require('../../../../../conf/authdata.json'); const { describeSkipIfNotMultiple, - describeSkipIfNotMultipleOrCeph, - describeSkipIfCeph, - isCEPH, - itSkipCeph, } = require('../../lib/utility/test-utils'); const memLocation = 'scality-internal-mem'; @@ -79,8 +75,6 @@ if (config.backends.data === 'multiple') { const utils = { describeSkipIfNotMultiple, - describeSkipIfNotMultipleOrCeph, - describeSkipIfCeph, awsS3, awsBucket, gcpClient, @@ -99,8 +93,6 @@ const utils = { gcpLocation, gcpLocation2, gcpLocationMismatch, - isCEPH, - itSkipCeph, }; utils.genUniqID = () => uuidv4().replace(/-/g, ''); @@ -324,7 +316,7 @@ utils.mapToAwsPuts = async (s3, bucket, key, dataArray, callback) => { utils.putVersionsToAws = async (s3, bucket, key, versions, callback) => { try { await utils.enableVersioning(s3, bucket); - // Wait for versioning to be enabled before PUT to ensure Ceph returns VersionId + // Wait for versioning to be enabled before PUT to ensure VersionId is available await utils.waitForVersioningBeforePut(s3, bucket); const results = await utils.mapToAwsPuts(s3, bucket, key, versions); if (callback) { @@ -345,7 +337,7 @@ utils.putNullVersionsToAws = async (s3, bucket, key, versions, callback) => { try { await utils.suspendVersioning(s3, bucket); // Note: When versioning is suspended, we don't need to wait for "Enabled" status - // The wait is only needed when enabling versioning to ensure Ceph returns VersionId + // The wait is only needed when enabling versioning const results = await utils.mapToAwsPuts(s3, bucket, key, versions); if (callback) { callback(null, results); diff --git a/tests/functional/aws-node-sdk/test/object/deleteObject.js b/tests/functional/aws-node-sdk/test/object/deleteObject.js index a111f73094..ee14d27d06 100644 --- a/tests/functional/aws-node-sdk/test/object/deleteObject.js +++ b/tests/functional/aws-node-sdk/test/object/deleteObject.js @@ -18,8 +18,6 @@ const changeObjectLock = require('../../../../utilities/objectLock-util'); const objectName = 'key'; const objectNameTwo = 'secondkey'; -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; describe('DELETE object', () => { withV4(sigCfg => { @@ -109,7 +107,7 @@ describe('DELETE object', () => { }); }); - describeSkipIfCeph('with object lock', () => { + describe('with object lock', () => { const bucketName = 'testdeleteobjectlockbucket'; let versionIdOne; let versionIdTwo; @@ -229,7 +227,7 @@ describe('DELETE object', () => { }); }); - describeSkipIfCeph('with object lock and legal hold', () => { + describe('with object lock and legal hold', () => { const bucketName = 'testdeletelocklegalholdbucket'; const objectName = 'key'; let versionId; diff --git a/tests/functional/aws-node-sdk/test/object/get.js b/tests/functional/aws-node-sdk/test/object/get.js index f8182ac36a..5b2c7102d0 100644 --- a/tests/functional/aws-node-sdk/test/object/get.js +++ b/tests/functional/aws-node-sdk/test/object/get.js @@ -1043,10 +1043,8 @@ describe('GET object', () => { }); }); -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('GET object with object lock', () => { +describe('GET object with object lock', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; diff --git a/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js b/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js index 01789b92b6..1424897ced 100644 --- a/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js +++ b/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js @@ -20,10 +20,8 @@ const unlockedBucket = 'mock-bucket-no-lock'; const key = 'mock-object-legalhold'; const keyNoHold = 'mock-object-no-legalhold'; -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('GET object legal hold', () => { +describe('GET object legal hold', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; diff --git a/tests/functional/aws-node-sdk/test/object/getRetention.js b/tests/functional/aws-node-sdk/test/object/getRetention.js index 74203ab5f9..3c97a0fbaa 100644 --- a/tests/functional/aws-node-sdk/test/object/getRetention.js +++ b/tests/functional/aws-node-sdk/test/object/getRetention.js @@ -32,10 +32,8 @@ const expectedConfig = { RetainUntilDate: new Date(retainDate), }; -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('GET object retention', () => { +describe('GET object retention', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; diff --git a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js index 2caecbaf04..7aa6d9babf 100644 --- a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js +++ b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js @@ -291,10 +291,8 @@ describe('Multi-Object Delete Access', function access() { }); }); -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('Multi-Object Delete with Object Lock', () => { +describe('Multi-Object Delete with Object Lock', () => { let bucketUtil; let s3; const versionIds = []; diff --git a/tests/functional/aws-node-sdk/test/object/objectCopy.js b/tests/functional/aws-node-sdk/test/object/objectCopy.js index 78a5f59a16..c7b6ceedcc 100644 --- a/tests/functional/aws-node-sdk/test/object/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/object/objectCopy.js @@ -1362,10 +1362,8 @@ describe('Object Copy', () => { }); }); -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('Object Copy with object lock enabled on both destination ' + +describe('Object Copy with object lock enabled on both destination ' + 'bucket and source bucket', () => { withV4(sigCfg => { let bucketUtil; diff --git a/tests/functional/aws-node-sdk/test/object/objectHead.js b/tests/functional/aws-node-sdk/test/object/objectHead.js index a9479a89c3..f376afc885 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead.js @@ -538,10 +538,8 @@ describe('HEAD object, conditions', () => { }); }); -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('HEAD object with object lock', () => { +describe('HEAD object with object lock', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; diff --git a/tests/functional/aws-node-sdk/test/object/put.js b/tests/functional/aws-node-sdk/test/object/put.js index 35fb245f69..aa1514c87d 100644 --- a/tests/functional/aws-node-sdk/test/object/put.js +++ b/tests/functional/aws-node-sdk/test/object/put.js @@ -313,10 +313,8 @@ describe('PUT object', () => { }); }); -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('PUT object with object lock', () => { +describe('PUT object with object lock', () => { const bucket = 'bucket2putstuffin4324242-lock'; withV4(sigCfg => { let bucketUtil; diff --git a/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js b/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js index af43270d83..7dc9a2c23c 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js +++ b/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js @@ -48,10 +48,8 @@ function createLegalHoldParams(bucket, key, status, versionId) { return params; } -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; -describeSkipIfCeph('PUT object legal hold', () => { +describe('PUT object legal hold', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; @@ -215,7 +213,7 @@ describeSkipIfCeph('PUT object legal hold', () => { // Use bucket policy to test iam action for putObjectLegalHold with version id // It used to need non standard s3:PutObjectVersionLegalHold action but was fixed // by ARSN-297 ARTESCA-7107 -describeSkipIfCeph('PUT object legal hold iam action and version id', () => { +describe('PUT object legal hold iam action and version id', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; diff --git a/tests/functional/aws-node-sdk/test/object/putRetention.js b/tests/functional/aws-node-sdk/test/object/putRetention.js index b73cbba1bc..a01d163724 100644 --- a/tests/functional/aws-node-sdk/test/object/putRetention.js +++ b/tests/functional/aws-node-sdk/test/object/putRetention.js @@ -24,12 +24,10 @@ const retentionConfig = { RetainUntilDate: moment().add(1, 'd').add(123, 'ms').toDate(), }; -const isCEPH = process.env.CI_CEPH !== undefined; -const describeSkipIfCeph = isCEPH ? describe.skip : describe; const changeObjectLockPromise = promisify(changeObjectLock); -describeSkipIfCeph('PUT object retention', () => { +describe('PUT object retention', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; @@ -146,7 +144,7 @@ describeSkipIfCeph('PUT object retention', () => { // Use bucket policy to test iam action for putObjectRetention with version id // It used to need non standard s3:PutObjectVersionRetention action but was fixed // by ARSN-297 ARTESCA-7107 -describeSkipIfCeph('PUT object retention iam action and version id', () => { +describe('PUT object retention iam action and version id', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; diff --git a/tests/functional/aws-node-sdk/test/object/rangeTest.js b/tests/functional/aws-node-sdk/test/object/rangeTest.js index 143ba8a1f6..686456ab19 100644 --- a/tests/functional/aws-node-sdk/test/object/rangeTest.js +++ b/tests/functional/aws-node-sdk/test/object/rangeTest.js @@ -16,8 +16,6 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { describeSkipIfCeph } = require('../../lib/utility/test-utils'); - const bucket = 'bucket-for-range-test'; const key = 'key-for-range-test'; let s3; @@ -103,7 +101,7 @@ function createHashedFile(bytes) { return execFileAsync('./getRangeExec', ['--size', bytes, name]); } -describeSkipIfCeph('aws-node-sdk range tests', () => { +describe('aws-node-sdk range tests', () => { before(() => execFileAsync('gcc', ['-o', 'getRangeExec', 'lib/utility/getRange.c'])); after(() => execAsync('rm getRangeExec')); diff --git a/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js b/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js index 6661094e2f..7bc84b7fd2 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js +++ b/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js @@ -26,14 +26,12 @@ const aclEquivalent = { }; const aclTests = [ - // CEPH: test_website_private_bucket_list_private_index_blockederrordoc { it: 'should return 403 if private bucket index and error documents', bucketACL: 'private', objects: { index: 'private', error: 'private' }, html: '403-access-denied', }, - // CEPH: test_website_public_bucket_list_private_index_blockederrordoc { it: 'should return 403 if public bucket - private index - public ' + 'error documents', @@ -85,7 +83,6 @@ const aclTests = [ html: 'error-user-404', }, - // CEPH: test_website_private_bucket_list_empty_blockederrordoc { it: 'should return 403 if private bucket - without index - ' + 'private error documents', @@ -94,7 +91,6 @@ const aclTests = [ html: '403-access-denied', }, - // CEPH: test_website_public_bucket_list_empty_blockederrordoc { it: 'should return 404 if public bucket - without index - ' + 'private error documents', @@ -103,7 +99,6 @@ const aclTests = [ html: '404-not-found', }, - // CEPH: test_website_public_bucket_list_empty_missingerrordoc { it: 'should return 404 if public bucket - without index - ' + 'without error documents', diff --git a/tests/multipleBackend/multipartUpload.js b/tests/multipleBackend/multipartUpload.js index b01c8e2861..b5d537d73d 100644 --- a/tests/multipleBackend/multipartUpload.js +++ b/tests/multipleBackend/multipartUpload.js @@ -52,11 +52,6 @@ const awsMismatchBucket = config.locationConstraints[awsLocationMismatch] const smallBody = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); const locMetaHeader = 'scal-location-constraint'; -const isCEPH = (config.locationConstraints[awsLocation] - .details.awsEndpoint !== undefined && - config.locationConstraints[awsLocation] - .details.awsEndpoint.indexOf('amazon') === -1); -const itSkipCeph = isCEPH ? it.skip : it; const bucketPutRequest = { bucketName, namespace, @@ -481,13 +476,10 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { abortMPU(uploadId, getAwsParams(objectKey), () => { const listParams = getListParams(objectKey, uploadId); listParts(authInfo, listParams, log, err => { - let wantedDesc = 'Error returned from AWS: ' + + const wantedDesc = 'Error returned from AWS: ' + 'The specified upload does not exist. The upload ID ' + 'may be invalid, or the upload may have been aborted' + ' or completed.'; - if (isCEPH) { - wantedDesc = 'Error returned from AWS: null'; - } assert.strictEqual(err.is.ServiceUnavailable, true); assert.deepStrictEqual(err.description, wantedDesc); done(); @@ -509,8 +501,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { })).then(() => { assert.fail('Expected an error listing parts of aborted MPU'); }).catch(err => { - const wantedError = isCEPH ? 'NoSuchKey' : 'NoSuchUpload'; - assert.strictEqual(err.name, wantedError); + assert.strictEqual(err.name, 'NoSuchUpload'); done(); }); }); @@ -531,8 +522,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { })).then(() => { assert.fail('Expected an error listing parts of aborted MPU'); }).catch(err => { - const wantedError = isCEPH ? 'NoSuchKey' : 'NoSuchUpload'; - assert.strictEqual(err.name, wantedError); + assert.strictEqual(err.name, 'NoSuchUpload'); done(); }); }); @@ -675,8 +665,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); }); - // Ceph doesn't care about order - itSkipCeph('should return invalidPartOrder error', done => { + it('should return invalidPartOrder error', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { const errorBody = '' + diff --git a/tests/multipleBackend/objectPut.js b/tests/multipleBackend/objectPut.js index 325662e419..e76dc24cdc 100644 --- a/tests/multipleBackend/objectPut.js +++ b/tests/multipleBackend/objectPut.js @@ -21,7 +21,6 @@ const fileLocation = 'scality-internal-file'; const memLocation = 'scality-internal-mem'; const sproxydLocation = 'scality-internal-sproxyd'; -const isCEPH = process.env.CI_CEPH !== undefined; const describeSkipIfE2E = process.env.S3_END_TO_END ? describe.skip : describe; function put(bucketLoc, objLoc, requestHost, objectName, cb, errorDescription) { @@ -124,13 +123,11 @@ describeSkipIfE2E('objectPutAPI with multiple backends', function testSuite() { }, ]; - if (!isCEPH) { - putCases.push({ - name: 'sproxyd', - bucketLoc: sproxydLocation, - objLoc: null, - }); - } + putCases.push({ + name: 'sproxyd', + bucketLoc: sproxydLocation, + objLoc: null, + }); function isDataStoredInMem(testCase) { return testCase.objLoc === memLocation diff --git a/tests/multipleBackend/objectPutCopyPart.js b/tests/multipleBackend/objectPutCopyPart.js index a0fec0c1de..c319ca9df4 100644 --- a/tests/multipleBackend/objectPutCopyPart.js +++ b/tests/multipleBackend/objectPutCopyPart.js @@ -43,12 +43,6 @@ const awsLocationMismatch = 'awsbackendmismatch'; const partETag = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; const describeSkipIfE2E = process.env.S3_END_TO_END ? describe.skip : describe; -const { config } = require('../../lib/Config'); -const isCEPH = (config.locationConstraints[awsLocation] - .details.awsEndpoint !== undefined && - config.locationConstraints[awsLocation] - .details.awsEndpoint.indexOf('amazon') === -1); -const itSkipCeph = isCEPH ? it.skip : it; function getSourceAndDestKeys() { const timestamp = Date.now(); @@ -212,7 +206,7 @@ function testSuite() { }); }); - itSkipCeph('should copy part to AWS based on mpu location', done => { + it('should copy part to AWS based on mpu location', done => { copyPutPart(memLocation, awsLocation, null, 'localhost', (keys, uploadId) => { assert.strictEqual(ds.length, 2); @@ -259,7 +253,7 @@ function testSuite() { }); }); - itSkipCeph('should copy part to AWS based on bucket location', done => { + it('should copy part to AWS based on bucket location', done => { copyPutPart(awsLocation, null, null, 'localhost', (keys, uploadId) => { assert.deepStrictEqual(ds, []); const awsReq = getAwsParams(keys.destObjName, uploadId); @@ -278,7 +272,7 @@ function testSuite() { }); }); - itSkipCeph('should copy part an object on AWS location that has ' + + it('should copy part an object on AWS location that has ' + 'bucketMatch equals false to a mpu with a different AWS location', done => { copyPutPart(null, awsLocation, awsLocationMismatch, 'localhost', (keys, uploadId) => { @@ -299,7 +293,7 @@ function testSuite() { }); }); - itSkipCeph('should copy part an object on AWS to a mpu with a different ' + + it('should copy part an object on AWS to a mpu with a different ' + 'AWS location that has bucketMatch equals false', done => { copyPutPart(null, awsLocationMismatch, awsLocation, 'localhost', (keys, uploadId) => { diff --git a/tests/multipleBackend/routes/routeBackbeat.js b/tests/multipleBackend/routes/routeBackbeat.js index d8a43b7b75..7d47404522 100644 --- a/tests/multipleBackend/routes/routeBackbeat.js +++ b/tests/multipleBackend/routes/routeBackbeat.js @@ -24,9 +24,8 @@ const { makeid } = require('../../unit/helpers'); const { makeRequest, makeBackbeatRequest } = require('../../functional/raw-node/utils/makeRequest'); const BucketUtility = require('../../functional/aws-node-sdk/lib/utility/bucket-util'); const { - describeSkipIfNotMultipleOrCeph, - itSkipCeph, - hasLocation + hasLocation, + describeSkipIfNotMultiple, } = require('../../functional/aws-node-sdk/lib/utility/test-utils'); const { awsLocation, @@ -192,14 +191,12 @@ function generateUniqueBucketName(prefix, suffix = uuidv4()) { return `${prefix}-${suffix.substring(0, 8)}`.substring(0, 63); } const describeIfLocationAws = hasLocation(awsLocation) ? describe : describe.skip; -const itIfLocationAwsSkipCeph = hasLocation(awsLocation) ? itSkipCeph : it.skip; const itIfLocationAws = hasLocation(awsLocation) ? it : it.skip; const itIfLocationAzure = hasLocation(azureLocation) ? it : it.skip; const itSkipS3C = process.env.S3_END_TO_END ? it.skip : it; -// FIXME: does not pass for Ceph, see CLDSRV-443 -describeSkipIfNotMultipleOrCeph('backbeat DELETE routes', () => { - it('abort MPU', done => { +describeSkipIfNotMultiple('backbeat DELETE routes', () => { + itIfLocationAws('abort MPU', done => { const awsKey = 'backbeat-mpu-test'; async.waterfall([ next => { @@ -2337,7 +2334,7 @@ describe('backbeat routes', () => { }); }); - itIfLocationAwsSkipCeph('should PUT tags for a non-versioned bucket (awslocation)', function test(done) { + itIfLocationAws('should PUT tags for a non-versioned bucket (awslocation)', function test(done) { this.timeout(10000); const bucket = NONVERSIONED_BUCKET; const awsKey = uuidv4(); @@ -3055,8 +3052,7 @@ describe('backbeat routes', () => { describeIfLocationAws('backbeat multipart upload operations (external location)', function test() { this.timeout(10000); - // The ceph image does not support putting tags during initiate MPU. - itSkipCeph('should put tags if the source is AWS and tags are ' + + it('should put tags if the source is AWS and tags are ' + 'provided when initiating the multipart upload', done => { const awsKey = uuidv4(); const multipleBackendPath = @@ -3277,7 +3273,7 @@ describe('backbeat routes', () => { ], done); }); - itIfLocationAwsSkipCeph('should batch delete a versioned AWS location', done => { + itIfLocationAws('should batch delete a versioned AWS location', done => { let versionId; const awsKey = `${TEST_BUCKET}/batch-delete-test-key-${makeid(8)}`; @@ -3432,7 +3428,7 @@ describe('backbeat routes', () => { ], done); }); - itIfLocationAwsSkipCeph('should not put tags if the source is not Azure and ' + + itIfLocationAws('should not put tags if the source is not Azure and ' + 'if-unmodified-since condition is not met', done => { const awsKey = uuidv4(); async.series([ @@ -3479,7 +3475,7 @@ describe('backbeat routes', () => { ], done); }); - itIfLocationAwsSkipCeph('should put tags if the source is not Azure and ' + + itIfLocationAws('should put tags if the source is not Azure and ' + 'if-unmodified-since condition is met', done => { const awsKey = uuidv4(); let lastModified; From c8925f87a2d9887e27342c83b692bf845055c00d Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 30 Apr 2026 16:26:35 +0200 Subject: [PATCH 3/4] Update range tests for SDK v3 streams. Adapt the range functional test to the SDK v3 response body API by materializing the returned byte array before writing it to disk. Return uploaded part responses so CompleteMultipartUpload receives the expected ETags, and tolerate NoSuchUpload during cleanup after the upload has already completed. Issue: CLDSRV-825 --- .../aws-node-sdk/test/object/rangeTest.js | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/rangeTest.js b/tests/functional/aws-node-sdk/test/object/rangeTest.js index 686456ab19..3e0a66b606 100644 --- a/tests/functional/aws-node-sdk/test/object/rangeTest.js +++ b/tests/functional/aws-node-sdk/test/object/rangeTest.js @@ -49,7 +49,7 @@ function checkRanges(range, bytes) { Key: key, Range: `bytes=${range}`, })) - .then(res => { + .then(async res => { const { begin, end } = getOuterRange(range, bytes); const total = (end - begin) + 1; // If the range header is '-' (i.e., it is invalid), content range @@ -59,18 +59,20 @@ function checkRanges(range, bytes) { assert.deepStrictEqual(res.ContentLength, total); assert.deepStrictEqual(res.ContentRange, contentRange); - assert.deepStrictEqual(res.ContentType, 'application/octet-stream'); + assert(res.ContentType === undefined || + res.ContentType === 'application/octet-stream'); assert.deepStrictEqual(res.Metadata, {}); - // Write a file using the buffer so getRangeExec can then check bytes. - // If the getRangeExec program fails, then the range is incorrect. - return writeFileAsync(`hashedFile.${bytes}.${range}`, res.Body) - .then(() => execFileAsync('./getRangeExec', ['--check', '--size', total, - '--offset', begin, `hashedFile.${bytes}.${range}`])); + const bodyBytes = await res.Body.transformToByteArray(); + const bodyBuffer = Buffer.from(bodyBytes); + await writeFileAsync(`hashedFile.${bytes}.${range}`, bodyBuffer); + return execFileAsync('./getRangeExec', ['--check', '--size', total, + '--offset', begin, `hashedFile.${bytes}.${range}`]); }); } -// Create 5MB parts and upload them as parts of a MPU +// Create 5MB parts and upload them as parts of a MPU. Returns array of part +// responses (with ETag) for CompleteMultipartUpload. async function uploadParts(bytes, uploadId) { const name = `hashedFile.${bytes}`; return Promise.all([1, 2].map(async part => { @@ -78,17 +80,18 @@ async function uploadParts(bytes, uploadId) { await execFileAsync('dd', [ `if=${name}`, `of=${name}.mpuPart${part}`, - 'bs=5242880', + 'bs=5242880', `skip=${part - 1}`, 'count=1', ]); - await s3.send(new UploadPartCommand({ + const res = await s3.send(new UploadPartCommand({ Bucket: bucket, Key: key, PartNumber: part, UploadId: uploadId, Body: createReadStream(`${name}.mpuPart${part}`), })); + return res; } catch (error) { throw new Error(`Error uploading part ${part}: ${error.message}`); } @@ -149,12 +152,13 @@ describe('aws-node-sdk range tests', () => { Key: key, UploadId: uploadId, }))) - .catch(err => new Promise((resolve, reject) => { - if (err.code !== 'NoSuchUpload') { - reject(err); + .catch(err => { + // Upload was already completed in beforeEach; abort is no-op + if (err.name === 'NoSuchUpload' || err.code === 'NoSuchUpload') { + return; } - resolve(); - })) + throw err; + }) .then(() => bucketUtil.deleteOne(bucket)) .then(() => execAsync(`rm hashedFile.${fileSize}*`)) ); From f64bab2a007e6d79ba49c3bf63e7d8b7d54ff2f9 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 30 Apr 2026 16:26:45 +0200 Subject: [PATCH 4/4] Bump Arsenal to 8.4.1. Update the Arsenal dependency to 8.4.1 so CloudServer consumes the backend capability changes that complete the Ceph support removal. Refresh yarn.lock for the new Arsenal revision and its hdclient dependency requirement. Issue: CLDSRV-825 --- package.json | 2 +- yarn.lock | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 47c80f083f..1b42de351c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.3.9", + "arsenal": "git+https://github.com/scality/Arsenal#8.4.1", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/yarn.lock b/yarn.lock index 52cc8efd45..e454932745 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3471,6 +3471,14 @@ httpagent "github:scality/httpagent#1.1.0" werelogs "github:scality/werelogs#8.2.2" +"@scality/hdclient@^1.3.2": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@scality/hdclient/-/hdclient-1.3.2.tgz#544d08a5b88869a9c30107c05d774e13595966fb" + integrity sha512-voy67AlH1irNmaXno0KP/KpiEBYzzdW8EoGjBXsVztLFjGe+RQnNtAyzCn05secZZy43jBYLOrZ7032gorxvrg== + dependencies: + httpagent "github:scality/httpagent#1.1.0" + werelogs "github:scality/werelogs#8.2.2" + "@senx/warp10@^2.0.3": version "2.0.3" resolved "https://registry.yarnpkg.com/@senx/warp10/-/warp10-2.0.3.tgz#dcce3890d491c6380f2967abcf126909ed208969" @@ -6202,9 +6210,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.3.9": - version "8.3.9" - resolved "git+https://github.com/scality/Arsenal#51e5b761f7f0612a722c828fa3d43b438c50ab7c" +"arsenal@git+https://github.com/scality/Arsenal#8.4.1": + version "8.4.1" + resolved "git+https://github.com/scality/Arsenal#6b3b58b152ac23d29176ab1f24f49f8eda3145b2" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" @@ -6213,7 +6221,7 @@ arraybuffer.prototype.slice@^1.0.4: "@azure/identity" "^4.13.0" "@azure/storage-blob" "^12.31.0" "@js-sdsl/ordered-set" "^4.4.2" - "@scality/hdclient" "^1.3.1" + "@scality/hdclient" "^1.3.2" "@smithy/node-http-handler" "^4.3.0" "@smithy/protocol-http" "^5.3.5" JSONStream "^1.3.5"