From c127faa92682ebb838765d77e8aa5169292f4e93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Wed, 3 Jun 2026 16:54:08 +0200 Subject: [PATCH 1/6] Reformat with prettier Issue: CLDSRV-906 --- lib/api/apiUtils/object/versioning.js | 171 +++--- lib/api/objectDeleteTagging.js | 145 ++--- lib/api/objectPutLegalHold.js | 152 ++--- lib/api/objectPutRetention.js | 183 +++--- lib/api/objectPutTagging.js | 154 +++--- lib/services.js | 615 +++++++++++---------- lib/utilities/collectResponseHeaders.js | 66 +-- tests/unit/api/apiUtils/versioning.js | 546 +++++++++--------- tests/unit/lib/services.spec.js | 90 +-- tests/unit/utils/collectResponseHeaders.js | 23 +- 10 files changed, 1089 insertions(+), 1056 deletions(-) diff --git a/lib/api/apiUtils/object/versioning.js b/lib/api/apiUtils/object/versioning.js index 418224a4a3..1ece12c652 100644 --- a/lib/api/apiUtils/object/versioning.js +++ b/lib/api/apiUtils/object/versioning.js @@ -10,8 +10,7 @@ const { scaledMsPerDay } = config.getTimeOptions(); const versionIdUtils = versioning.VersionID; // Use Arsenal function to generate a version ID used internally by metadata // for null versions that are created before bucket versioning is configured -const nonVersionedObjId = - versionIdUtils.getInfVid(config.replicationGroupId); +const nonVersionedObjId = versionIdUtils.getInfVid(config.replicationGroupId); /** decodeVID - decode the version id * @param {string} versionId - version ID @@ -101,25 +100,24 @@ function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb) nullVersionMD.originOp = 's3:StoreNullVersion'; metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => { if (err) { - log.debug('error from metadata storing null version as new version', - { error: err }); + log.debug('error from metadata storing null version as new version', { error: err }); } - + cb(err); }); } /** check existence and get location of null version data for deletion -* @param {string} bucketName - name of bucket -* @param {string} objKey - name of object key -* @param {object} options - metadata options for getting object MD -* @param {string} options.versionId - version to get from metadata -* @param {object} mst - info about the master version -* @param {string} mst.versionId - the master version's version id -* @param {RequestLogger} log - logger instanceof -* @param {function} cb - callback -* @return {undefined} - and call callback with (err, dataToDelete) -*/ + * @param {string} bucketName - name of bucket + * @param {string} objKey - name of object key + * @param {object} options - metadata options for getting object MD + * @param {string} options.versionId - version to get from metadata + * @param {object} mst - info about the master version + * @param {string} mst.versionId - the master version's version id + * @param {RequestLogger} log - logger instanceof + * @param {function} cb - callback + * @return {undefined} - and call callback with (err, dataToDelete) + */ function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) { const nullOptions = {}; if (!options.deleteData) { @@ -135,38 +133,40 @@ function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) // PUT via this option nullOptions.deleteNullKey = true; } - return metadata.getObjectMD(bucketName, objKey, options, log, - (err, versionMD) => { - if (err) { - // the null key may not exist, hence it's a normal - // situation to have a NoSuchKey error, in which case - // there is nothing to delete - if (err.is.NoSuchKey) { - log.debug('null version does not exist', { - method: '_prepareNullVersionDeletion', - }); - } else { - log.warn('could not get null version metadata', { - error: err, - method: '_prepareNullVersionDeletion', - }); - } - return cb(err); - } - if (versionMD.location) { - const dataToDelete = Array.isArray(versionMD.location) ? - versionMD.location : [versionMD.location]; - nullOptions.dataToDelete = dataToDelete; + return metadata.getObjectMD(bucketName, objKey, options, log, (err, versionMD) => { + if (err) { + // the null key may not exist, hence it's a normal + // situation to have a NoSuchKey error, in which case + // there is nothing to delete + if (err.is.NoSuchKey) { + log.debug('null version does not exist', { + method: '_prepareNullVersionDeletion', + }); + } else { + log.warn('could not get null version metadata', { + error: err, + method: '_prepareNullVersionDeletion', + }); } - return cb(null, nullOptions); - }); + return cb(err); + } + if (versionMD.location) { + const dataToDelete = Array.isArray(versionMD.location) ? versionMD.location : [versionMD.location]; + nullOptions.dataToDelete = dataToDelete; + } + return cb(null, nullOptions); + }); } function _deleteNullVersionMD(bucketName, objKey, options, log, cb) { return metadata.deleteObjectMD(bucketName, objKey, options, log, err => { if (err) { - log.warn('metadata error deleting null versioned key', - { bucketName, objKey, error: err, method: '_deleteNullVersionMD' }); + log.warn('metadata error deleting null versioned key', { + bucketName, + objKey, + error: err, + method: '_deleteNullVersionMD', + }); } return cb(err); }); @@ -193,7 +193,7 @@ function _deleteNullVersionMD(bucketName, objKey, options, log, cb) { version key, if needed */ function processVersioningState(mst, vstat, nullVersionCompatMode) { - const versioningSuspended = (vstat === 'Suspended'); + const versioningSuspended = vstat === 'Suspended'; const masterIsNull = mst.exists && (mst.isNull || !mst.versionId); if (versioningSuspended) { @@ -244,7 +244,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) { if (masterIsNull) { // if master is a null version or a non-versioned key, // copy it to a new null key - const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId; + const nullVersionId = mst.isNull && mst.versionId ? mst.versionId : nonVersionedObjId; if (nullVersionCompatMode) { options.extraMD = { nullVersionId, @@ -311,8 +311,7 @@ function getMasterState(objMD) { }; if (objMD.location) { - mst.objLocation = Array.isArray(objMD.location) ? - objMD.location : [objMD.location]; + mst.objLocation = Array.isArray(objMD.location) ? objMD.location : [objMD.location]; } return mst; @@ -332,8 +331,7 @@ function getMasterState(objMD) { * options.versioning - (true/undefined) metadata instruction to create new ver * options.isNull - (true/undefined) whether new version is null or not */ -function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, - log, callback) { +function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, log, callback) { const mst = getMasterState(objMD); const vCfg = bucketMD.getVersioningConfiguration(); @@ -342,50 +340,57 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, return process.nextTick(callback, null, options); } - const { options, nullVersionId, delOptions } = - processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode); - - return async.series([ - function storeNullVersionMD(next) { - if (!nullVersionId) { - return process.nextTick(next); - } + const { options, nullVersionId, delOptions } = processVersioningState( + mst, + vCfg.Status, + config.nullVersionCompatMode, + ); + + return async.series( + [ + function storeNullVersionMD(next) { + if (!nullVersionId) { + return process.nextTick(next); + } - options.nullVersionId = nullVersionId; - return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next); - }, - function prepareNullVersionDeletion(next) { - if (!delOptions) { - return process.nextTick(next); - } - return _prepareNullVersionDeletion( - bucketName, objectKey, delOptions, mst, log, - (err, nullOptions) => { + options.nullVersionId = nullVersionId; + return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next); + }, + function prepareNullVersionDeletion(next) { + if (!delOptions) { + return process.nextTick(next); + } + return _prepareNullVersionDeletion(bucketName, objectKey, delOptions, mst, log, (err, nullOptions) => { if (err) { return next(err); } Object.assign(options, nullOptions); return next(); }); - }, - function deleteNullVersionMD(next) { - if (delOptions && - delOptions.versionId && - delOptions.versionId !== 'null') { - // backward-compat: delete old null versioned key - return _deleteNullVersionMD( - bucketName, objectKey, { versionId: delOptions.versionId, overheadField }, log, next); + }, + function deleteNullVersionMD(next) { + if (delOptions && delOptions.versionId && delOptions.versionId !== 'null') { + // backward-compat: delete old null versioned key + return _deleteNullVersionMD( + bucketName, + objectKey, + { versionId: delOptions.versionId, overheadField }, + log, + next, + ); + } + return process.nextTick(next); + }, + ], + err => { + // it's possible there was a prior request that deleted the + // null version, so proceed with putting a new version + if (err && err.is.NoSuchKey) { + return callback(null, options); } - return process.nextTick(next); + return callback(err, options); }, - ], err => { - // it's possible there was a prior request that deleted the - // null version, so proceed with putting a new version - if (err && err.is.NoSuchKey) { - return callback(null, options); - } - return callback(err, options); - }); + ); } /** Return options to pass to Metadata layer for version-specific @@ -545,7 +550,7 @@ function overwritingVersioning(objMD, metadataStoreParams) { restoreRequestedAt: objMD.archive?.restoreRequestedAt, restoreRequestedDays: objMD.archive?.restoreRequestedDays, restoreCompletedAt: new Date(now), - restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), }; /* eslint-enable no-param-reassign */ diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js index 71115ffe5a..45d67a0f97 100644 --- a/lib/api/objectDeleteTagging.js +++ b/lib/api/objectDeleteTagging.js @@ -1,8 +1,11 @@ const async = require('async'); const { errors } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } - = require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -48,75 +51,81 @@ function objectDeleteTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectDeleteTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectDeleteTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectDeleteTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.tags = {}; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectDeleteTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectDeleteTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectDeleteTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + objectMD.tags = {}; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + // eslint-disable-next-line no-param-reassign + objectMD.originOp = 's3:ObjectTagging:Delete'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + (bucket, objectMD, next) => + // if external backends handles tagging + data.objectTagging('Delete', objectKey, bucket.getName(), objectMD, log, err => + next(err, bucket, objectMD), + ), + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectDeleteTagging' }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteObjectTagging'); + } else { + pushMetric('deleteObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectTagging:Delete'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => - next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - (bucket, objectMD, next) => - // if external backends handles tagging - data.objectTagging('Delete', objectKey, bucket.getName(), objectMD, - log, err => next(err, bucket, objectMD)), - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectDeleteTagging' }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteObjectTagging'); - } else { - pushMetric('deleteObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectDeleteTagging; diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js index c16f2c84e8..d687f77ce6 100644 --- a/lib/api/objectPutLegalHold.js +++ b/lib/api/objectPutLegalHold.js @@ -2,8 +2,11 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); @@ -47,78 +50,87 @@ function objectPutLegalHold(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutLegalHold', error: err }); + return next(err); + } + if (!objectMD) { + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutLegalHold', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutLegalHold' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectPutLegalHold' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + bucket, + ); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + log.trace('parsing legal hold'); + parseLegalHoldXml(request.post, log, (err, res) => next(err, bucket, res, objectMD)); + }, + (bucket, legalHold, objectMD, next) => { + // eslint-disable-next-line no-param-reassign + objectMD.legalHold = legalHold; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + // eslint-disable-next-line no-param-reassign + objectMD.originOp = 's3:ObjectLegalHold:Put'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - log.trace('request authorization failed', - { method: 'objectPutLegalHold', error: err }); - return next(err); - } - if (!objectMD) { - const err = versionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutLegalHold', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutLegalHold' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectPutLegalHold' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration' - ), bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - log.trace('parsing legal hold'); - parseLegalHoldXml(request.post, log, (err, res) => - next(err, bucket, res, objectMD)); - }, - (bucket, legalHold, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.legalHold = legalHold; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { - // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + log.trace('error processing request', { error: err, method: 'objectPutLegalHold' }); + } else { + pushMetric('putObjectLegalHold', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectLegalHold:Put'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'objectPutLegalHold' }); - } else { - pushMetric('putObjectLegalHold', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutLegalHold; diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js index 6a7a2c8441..b8182646e9 100644 --- a/lib/api/objectPutRetention.js +++ b/lib/api/objectPutRetention.js @@ -1,10 +1,12 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); -const { ObjectLockInfo, hasGovernanceBypassHeader } = - require('./apiUtils/object/objectLockHelpers'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); +const { ObjectLockInfo, hasGovernanceBypassHeader } = require('./apiUtils/object/objectLockHelpers'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); @@ -50,99 +52,106 @@ function objectPutRetention(authInfo, request, log, callback) { const hasGovernanceBypass = hasGovernanceBypassHeader(request.headers); - return async.waterfall([ - next => { - log.trace('parsing retention information'); - parseRetentionXml(request.post, log, - (err, retentionInfo) => { + return async.waterfall( + [ + next => { + log.trace('parsing retention information'); + parseRetentionXml(request.post, log, (err, retentionInfo) => { if (err) { - log.trace('error parsing retention information', - { error: err }); + log.trace('error parsing retention information', { error: err }); return next(err); } - const remainingDays = Math.ceil( - (new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24)); + const remainingDays = Math.ceil((new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24)); metadataValParams.request.objectLockRetentionDays = remainingDays; return next(null, retentionInfo); }); - }, - (retentionInfo, next) => standardMetadataValidateBucketAndObj(metadataValParams, - request.actionImplicitDenies, log, (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectPutRetention', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutRetention', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutRetention' }); - return next(errors.MethodNotAllowed, bucket); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectPutRetention' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration' - ), bucket); - } - return next(null, bucket, retentionInfo, objectMD); - }), - (bucket, retentionInfo, objectMD, next) => { - const objLockInfo = new ObjectLockInfo({ - mode: objectMD.retentionMode, - date: objectMD.retentionDate, - legalHold: objectMD.legalHold, - }); + }, + (retentionInfo, next) => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutRetention', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutRetention', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutRetention' }); + return next(errors.MethodNotAllowed, bucket); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectPutRetention' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + bucket, + ); + } + return next(null, bucket, retentionInfo, objectMD); + }, + ), + (bucket, retentionInfo, objectMD, next) => { + const objLockInfo = new ObjectLockInfo({ + mode: objectMD.retentionMode, + date: objectMD.retentionDate, + legalHold: objectMD.legalHold, + }); - if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) { - return next(errors.AccessDenied, bucket); - } + if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) { + return next(errors.AccessDenied, bucket); + } - return next(null, bucket, retentionInfo, objectMD); - }, - (bucket, retentionInfo, objectMD, next) => { - /* eslint-disable no-param-reassign */ - objectMD.retentionMode = retentionInfo.mode; - objectMD.retentionDate = retentionInfo.date; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + return next(null, bucket, retentionInfo, objectMD); + }, + (bucket, retentionInfo, objectMD, next) => { + /* eslint-disable no-param-reassign */ + objectMD.retentionMode = retentionInfo.mode; + objectMD.retentionDate = retentionInfo.date; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + objectMD.originOp = 's3:ObjectRetention:Put'; + /* eslint-enable no-param-reassign */ + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectPutRetention' }); + } else { + pushMetric('putObjectRetention', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - objectMD.originOp = 's3:ObjectRetention:Put'; - /* eslint-enable no-param-reassign */ - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'objectPutRetention' }); - } else { - pushMetric('putObjectRetention', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutRetention; diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js index ef23dcf64d..85bb652787 100644 --- a/lib/api/objectPutTagging.js +++ b/lib/api/objectPutTagging.js @@ -1,8 +1,11 @@ const async = require('async'); const { errors, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -47,80 +50,85 @@ function objectPutTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectPutTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - log.trace('parsing tag(s)'); - parseTagXml(request.post, log, (err, tags) => - next(err, bucket, tags, objectMD)); - }, - (bucket, tags, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.tags = tags; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + log.trace('parsing tag(s)'); + parseTagXml(request.post, log, (err, tags) => next(err, bucket, tags, objectMD)); + }, + (bucket, tags, objectMD, next) => { + // eslint-disable-next-line no-param-reassign + objectMD.tags = tags; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + objectMD.originOp = 's3:ObjectTagging:Put'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + (bucket, objectMD, next) => + // if external backend handles tagging + data.objectTagging('Put', objectKey, bucket.getName(), objectMD, log, err => + next(err, bucket, objectMD), + ), + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectPutTagging' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectTagging'); + } else { + pushMetric('putObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectTagging:Put'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => - next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - (bucket, objectMD, next) => - // if external backend handles tagging - data.objectTagging('Put', objectKey, bucket.getName(), objectMD, - log, err => next(err, bucket, objectMD)), - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectPutTagging' }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'putObjectTagging'); - } else { - pushMetric('putObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutTagging; diff --git a/lib/services.js b/lib/services.js index 97116329a4..5bc5036442 100644 --- a/lib/services.js +++ b/lib/services.js @@ -13,8 +13,7 @@ const constants = require('../constants'); const { config } = require('./Config'); const { data } = require('./data/wrapper'); const metadata = require('./metadata/wrapper'); -const { setObjectLockInformation } - = require('./api/apiUtils/object/objectLockHelpers'); +const { setObjectLockInformation } = require('./api/apiUtils/object/objectLockHelpers'); const removeAWSChunked = require('./api/apiUtils/object/removeAWSChunked'); const { parseTagFromQuery } = s3middleware.tagging; @@ -41,52 +40,53 @@ const services = { // (without special increase) // TODO: Consider implementing pagination like object listing // with respect to bucket listing so can go beyond 10000 - metadata.listObject(bucketUsers, { prefix, maxKeys: 10000 }, log, - (err, listResponse) => { - // If MD responds with NoSuchBucket, this means the - // hidden usersBucket has not yet been created for - // the domain. If this is the case, it means - // that no buckets in this domain have been created so - // it follows that this particular user has no buckets. - // So, the get service listing should not have any - // buckets to list. By returning an empty array, the - // getService API will just respond with the user info - // without listing any buckets. - if (err?.is?.NoSuchBucket) { - log.trace('no buckets found'); - // If we checked the old user bucket, that means we - // already checked the new user bucket. If neither the - // old user bucket or the new user bucket exist, no buckets - // have yet been created in the namespace so an empty - // listing should be returned - if (overrideUserbucket) { - return cb(null, [], splitter); - } - // Since there were no results from checking the - // new users bucket, we check the old users bucket - return this.getService(authInfo, request, log, - constants.oldSplitter, cb, oldUsersBucket); - } - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); + metadata.listObject(bucketUsers, { prefix, maxKeys: 10000 }, log, (err, listResponse) => { + // If MD responds with NoSuchBucket, this means the + // hidden usersBucket has not yet been created for + // the domain. If this is the case, it means + // that no buckets in this domain have been created so + // it follows that this particular user has no buckets. + // So, the get service listing should not have any + // buckets to list. By returning an empty array, the + // getService API will just respond with the user info + // without listing any buckets. + if (err?.is?.NoSuchBucket) { + log.trace('no buckets found'); + // If we checked the old user bucket, that means we + // already checked the new user bucket. If neither the + // old user bucket or the new user bucket exist, no buckets + // have yet been created in the namespace so an empty + // listing should be returned + if (overrideUserbucket) { + return cb(null, [], splitter); } - return cb(null, listResponse.Contents, splitter); - }); + // Since there were no results from checking the + // new users bucket, we check the old users bucket + return this.getService(authInfo, request, log, constants.oldSplitter, cb, oldUsersBucket); + } + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); + } + return cb(null, listResponse.Contents, splitter); + }); }, - /** - * Check that hashedStream.completedHash matches header contentMd5. - * @param {object} contentMD5 - content-md5 header - * @param {string} completedHash - hashed stream once completed - * @param {RequestLogger} log - the current request logger - * @return {boolean} - true if contentMD5 matches or is undefined, - * false otherwise - */ + /** + * Check that hashedStream.completedHash matches header contentMd5. + * @param {object} contentMD5 - content-md5 header + * @param {string} completedHash - hashed stream once completed + * @param {RequestLogger} log - the current request logger + * @return {boolean} - true if contentMD5 matches or is undefined, + * false otherwise + */ checkHashMatchMD5(contentMD5, completedHash, log) { if (contentMD5 && completedHash && contentMD5 !== completedHash) { - log.debug('contentMD5 and completedHash does not match', - { method: 'checkHashMatchMD5', completedHash, contentMD5 }); + log.debug('contentMD5 and completedHash does not match', { + method: 'checkHashMatchMD5', + completedHash, + contentMD5, + }); return false; } return true; @@ -103,15 +103,46 @@ const services = { * @return {function} executes callback with err or ETag as arguments */ metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) { - const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders, - contentType, cacheControl, contentDisposition, contentEncoding, - expires, multipart, headers, overrideMetadata, log, - lastModifiedDate, versioning, versionId, uploadId, - tagging, taggingCopy, replicationInfo, defaultRetention, - dataStoreName, creationTime, retentionMode, retentionDate, - legalHold, originOp, updateMicroVersionId, archive, oldReplayId, - deleteNullKey, amzStorageClass, overheadField, needOplogUpdate, - restoredEtag, bucketOwnerId } = params; + const { + objectKey, + authInfo, + size, + contentMD5, + checksum, + metaHeaders, + contentType, + cacheControl, + contentDisposition, + contentEncoding, + expires, + multipart, + headers, + overrideMetadata, + log, + lastModifiedDate, + versioning, + versionId, + uploadId, + tagging, + taggingCopy, + replicationInfo, + defaultRetention, + dataStoreName, + creationTime, + retentionMode, + retentionDate, + legalHold, + originOp, + updateMicroVersionId, + archive, + oldReplayId, + deleteNullKey, + amzStorageClass, + overheadField, + needOplogUpdate, + restoredEtag, + bucketOwnerId, + } = params; log.trace('storing object in metadata'); assert.strictEqual(typeof bucketName, 'string'); const md = new ObjectMD(); @@ -196,12 +227,15 @@ const services = { // update restore if (archive) { md.setAmzStorageClass(amzStorageClass); - md.setArchive(new ObjectMDArchive( - archive.archiveInfo, - archive.restoreRequestedAt, - archive.restoreRequestedDays, - archive.restoreCompletedAt, - archive.restoreWillExpireAt)); + md.setArchive( + new ObjectMDArchive( + archive.archiveInfo, + archive.restoreRequestedAt, + archive.restoreRequestedDays, + archive.restoreCompletedAt, + archive.restoreWillExpireAt, + ), + ); md.setAmzRestore({ 'ongoing-request': false, 'expiry-date': archive.restoreWillExpireAt, @@ -287,55 +321,57 @@ const services = { // If this is not the completion of a multipart upload or // the creation of a delete marker, parse the headers to // get the ACL's if any - return async.waterfall([ - callback => { - if (multipart || md.getIsDeleteMarker()) { - return callback(); + return async.waterfall( + [ + callback => { + if (multipart || md.getIsDeleteMarker()) { + return callback(); + } + const parseAclParams = { + headers, + resourceType: 'object', + acl: md.getAcl(), + log, + }; + log.trace('parsing acl from headers'); + acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => { + if (err) { + log.debug('error parsing acl', { error: err }); + return callback(err); + } + md.setAcl(parsedACL); + return callback(); + }); + return null; + }, + callback => metadata.putObjectMD(bucketName, objectKey, md, options, log, callback), + ], + (err, data) => { + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); } - const parseAclParams = { - headers, - resourceType: 'object', - acl: md.getAcl(), - log, - }; - log.trace('parsing acl from headers'); - acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => { - if (err) { - log.debug('error parsing acl', { error: err }); - return callback(err); + log.trace('object successfully stored in metadata'); + // if versioning is enabled, data will be returned from metadata + // as JSON containing a versionId which some APIs will need sent + // back to them + let versionId; + if (data) { + if (params.isNull && params.isDeleteMarker) { + versionId = 'null'; + } else if (!params.isNull) { + versionId = JSON.parse(data).versionId; } - md.setAcl(parsedACL); - return callback(); + } + return cb(err, { + lastModified: md.getLastModified(), + tags: md.getTags(), + contentMD5, + versionId, + checksum: md.getChecksum(), }); - return null; }, - callback => metadata.putObjectMD(bucketName, objectKey, md, - options, log, callback), - ], (err, data) => { - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); - } - log.trace('object successfully stored in metadata'); - // if versioning is enabled, data will be returned from metadata - // as JSON containing a versionId which some APIs will need sent - // back to them - let versionId; - if (data) { - if (params.isNull && params.isDeleteMarker) { - versionId = 'null'; - } else if (!params.isNull) { - versionId = JSON.parse(data).versionId; - } - } - return cb(err, { - lastModified: md.getLastModified(), - tags: md.getTags(), - contentMD5, - versionId, - checksum: md.getChecksum(), - }); - }); + ); }, /** @@ -358,7 +394,11 @@ const services = { assert.strictEqual(typeof objectMD, 'object'); function deleteMDandData() { - return metadata.deleteObjectMD(bucketName, objectKey, options, log, + return metadata.deleteObjectMD( + bucketName, + objectKey, + options, + log, (err, res) => { if (err) { return cb(err, res); @@ -369,8 +409,7 @@ const services = { } if (deferLocationDeletion) { - return cb(null, Array.isArray(objectMD.location) - ? objectMD.location : [objectMD.location]); + return cb(null, Array.isArray(objectMD.location) ? objectMD.location : [objectMD.location]); } if (!Array.isArray(objectMD.location)) { @@ -384,14 +423,15 @@ const services = { } return cb(null, res); }); - }, originOp); + }, + originOp, + ); } const objGetInfo = objectMD.location; // special case that prevents azure blocks from unecessary deletion // will return null if no need - return data.protectAzureBlocks(bucketName, objectKey, objGetInfo, - log, err => { + return data.protectAzureBlocks(bucketName, objectKey, objGetInfo, log, err => { if (err) { return cb(err); } @@ -411,16 +451,14 @@ const services = { */ getObjectListing(bucketName, listingParams, log, cb) { assert.strictEqual(typeof bucketName, 'string'); - log.trace('performing metadata get object listing', - { listingParams }); - metadata.listObject(bucketName, listingParams, log, - (err, listResponse) => { - if (err) { - log.debug('error from metadata', { error: err }); - return cb(err); - } - return cb(null, listResponse); - }); + log.trace('performing metadata get object listing', { listingParams }); + metadata.listObject(bucketName, listingParams, log, (err, listResponse) => { + if (err) { + log.debug('error from metadata', { error: err }); + return cb(err); + } + return cb(null, listResponse); + }); }, /** @@ -463,10 +501,8 @@ const services = { } // Check each version in current batch for matching uploadId - const matchedVersion = (listResponse.Versions || []).find(version => - version.key === objectKey && - version.value && - version.value.uploadId === uploadId + const matchedVersion = (listResponse.Versions || []).find( + version => version.key === objectKey && version.value && version.value.uploadId === uploadId, ); if (matchedVersion) { @@ -484,7 +520,7 @@ const services = { return callback(); }); }, - err => cb(err, err ? null : foundVersion) + err => cb(err, err ? null : foundVersion), ); }, @@ -500,16 +536,14 @@ const services = { */ getLifecycleListing(bucketName, listingParams, log, cb) { assert.strictEqual(typeof bucketName, 'string'); - log.trace('performing metadata get object listing for lifecycle', - { listingParams }); - metadata.listLifecycleObject(bucketName, listingParams, log, - (err, listResponse) => { - if (err) { - log.debug('error from metadata', { error: err }); - return cb(err); - } - return cb(null, listResponse); - }); + log.trace('performing metadata get object listing for lifecycle', { listingParams }); + metadata.listLifecycleObject(bucketName, listingParams, log, (err, listResponse) => { + if (err) { + log.debug('error from metadata', { error: err }); + return cb(err); + } + return cb(null, listResponse); + }); }, metadataStoreMPObject(bucketName, cipherBundle, params, log, cb) { @@ -522,9 +556,7 @@ const services = { // the splitter. // 2) UploadId's are UUID version 4 const splitter = params.splitter; - const longMPUIdentifier = - `overview${splitter}${params.objectKey}` + - `${splitter}${params.uploadId}`; + const longMPUIdentifier = `overview${splitter}${params.objectKey}` + `${splitter}${params.uploadId}`; const multipartObjectMD = {}; multipartObjectMD.id = params.uploadId; multipartObjectMD.eventualStorageBucket = params.eventualStorageBucket; @@ -546,28 +578,19 @@ const services = { multipartObjectMD.key = params.objectKey; multipartObjectMD.uploadId = params.uploadId; multipartObjectMD['cache-control'] = params.headers['cache-control']; - multipartObjectMD['content-disposition'] = - params.headers['content-disposition']; - multipartObjectMD['content-encoding'] = - removeAWSChunked(params.headers['content-encoding']); - multipartObjectMD['content-type'] = - params.headers['content-type']; - multipartObjectMD.expires = - params.headers.expires; - multipartObjectMD['x-amz-storage-class'] = params.storageClass; // TODO: removed CLDSRV-639 - multipartObjectMD['x-amz-website-redirect-location'] = - params.headers['x-amz-website-redirect-location']; + multipartObjectMD['content-disposition'] = params.headers['content-disposition']; + multipartObjectMD['content-encoding'] = removeAWSChunked(params.headers['content-encoding']); + multipartObjectMD['content-type'] = params.headers['content-type']; + multipartObjectMD.expires = params.headers.expires; + multipartObjectMD['x-amz-storage-class'] = params.storageClass; // TODO: removed CLDSRV-639 + multipartObjectMD['x-amz-website-redirect-location'] = params.headers['x-amz-website-redirect-location']; if (cipherBundle) { - multipartObjectMD['x-amz-server-side-encryption'] = - cipherBundle.algorithm; + multipartObjectMD['x-amz-server-side-encryption'] = cipherBundle.algorithm; if (cipherBundle.masterKeyId) { - multipartObjectMD[ - 'x-amz-server-side-encryption-aws-kms-key-id'] = - cipherBundle.masterKeyId; + multipartObjectMD['x-amz-server-side-encryption-aws-kms-key-id'] = cipherBundle.masterKeyId; } } - multipartObjectMD.controllingLocationConstraint = - params.controllingLocationConstraint; + multipartObjectMD.controllingLocationConstraint = params.controllingLocationConstraint; multipartObjectMD.dataStoreName = params.dataStoreName; if (params.tagging) { const validationTagRes = parseTagFromQuery(params.tagging); @@ -613,15 +636,14 @@ const services = { return cb(err); } multipartObjectMD.acl = parsedACL; - metadata.putObjectMD(bucketName, longMPUIdentifier, - multipartObjectMD, {}, log, err => { - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); - } + metadata.putObjectMD(bucketName, longMPUIdentifier, multipartObjectMD, {}, log, err => { + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); + } - return cb(null, multipartObjectMD); - }); + return cb(null, multipartObjectMD); + }); return undefined; }); }, @@ -648,12 +670,10 @@ const services = { assert.strictEqual(typeof params.splitter, 'string'); assert.strictEqual(typeof params.storedMetadata, 'object'); const splitter = params.splitter; - const longMPUIdentifier = - `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; + const longMPUIdentifier = `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; const multipartObjectMD = Object.assign({}, params.storedMetadata); multipartObjectMD.completeInProgress = true; - metadata.putObjectMD(params.bucketName, longMPUIdentifier, multipartObjectMD, - {}, log, err => { + metadata.putObjectMD(params.bucketName, longMPUIdentifier, multipartObjectMD, {}, log, err => { if (err) { log.error('error from metadata', { error: err }); return cb(err); @@ -685,30 +705,28 @@ const services = { const mpuBucketName = `${constants.mpuBucketPrefix}${params.bucketName}`; const splitter = params.splitter; - const mpuOverviewKey = - `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; - return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, - (err, res) => { - if (err) { - if (err.is && err.is.NoSuchKey) { - // The overview key no longer exists, meaning completeMultipartUpload - // already ran to completion and cleaned up the MPU bucket. - // This is a race condition: objectPutPart checked for old - // part locations after completeMultipartUpload deleted the overview. - // Returning true (complete in progress) prevents objectPutPart - // from deleting part data that may have already been committed - // as the final object. - return cb(null, true); - } - log.error('error getting the overview object from mpu bucket', { - error: err, - method: 'services.isCompleteMPUInProgress', - params, - }); - return cb(err); + const mpuOverviewKey = `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; + return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, (err, res) => { + if (err) { + if (err.is && err.is.NoSuchKey) { + // The overview key no longer exists, meaning completeMultipartUpload + // already ran to completion and cleaned up the MPU bucket. + // This is a race condition: objectPutPart checked for old + // part locations after completeMultipartUpload deleted the overview. + // Returning true (complete in progress) prevents objectPutPart + // from deleting part data that may have already been committed + // as the final object. + return cb(null, true); } - return cb(null, Boolean(res.completeInProgress)); - }); + log.error('error getting the overview object from mpu bucket', { + error: err, + method: 'services.isCompleteMPUInProgress', + params, + }); + return cb(err); + } + return cb(null, Boolean(res.completeInProgress)); + }); }, /** @@ -725,8 +743,7 @@ const services = { * - the overview key stored metadata */ metadataValidateMultipart(params, cb) { - const { bucketName, uploadId, authInfo, - objectKey, requestType, log } = params; + const { bucketName, uploadId, authInfo, objectKey, requestType, log } = params; assert.strictEqual(typeof bucketName, 'string'); // This checks whether the mpu bucket exists. @@ -734,13 +751,11 @@ const services = { const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`; metadata.getBucket(mpuBucketName, log, (err, mpuBucket) => { if (err?.is?.NoSuchBucket) { - log.debug('bucket not found in metadata', { error: err, - method: 'services.metadataValidateMultipart' }); + log.debug('bucket not found in metadata', { error: err, method: 'services.metadataValidateMultipart' }); return cb(errors.NoSuchUpload); } if (err) { - log.error('error from metadata', { error: err, - method: 'services.metadataValidateMultipart' }); + log.error('error from metadata', { error: err, method: 'services.metadataValidateMultipart' }); return cb(err); } @@ -749,87 +764,78 @@ const services = { if (mpuBucket.getMdBucketModelVersion() < 2) { splitter = constants.oldSplitter; } - const mpuOverviewKey = - `overview${splitter}${objectKey}${splitter}${uploadId}`; + const mpuOverviewKey = `overview${splitter}${objectKey}${splitter}${uploadId}`; - metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey, - {}, log, (err, storedMetadata) => { - if (err) { - if (err.is && err.is.NoSuchKey) { - return cb(errors.NoSuchUpload); - } - log.error('error from metadata', { error: err }); - return cb(err); + metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey, {}, log, (err, storedMetadata) => { + if (err) { + if (err.is && err.is.NoSuchKey) { + return cb(errors.NoSuchUpload); } + log.error('error from metadata', { error: err }); + return cb(err); + } - const initiatorID = storedMetadata.initiator.ID; - const ownerID = storedMetadata['owner-id']; - const mpuOverview = { - key: storedMetadata.key, - id: storedMetadata.id, - eventualStorageBucket: - storedMetadata.eventualStorageBucket, - initiatorID, - initiatorDisplayName: - storedMetadata.initiator.DisplayName, - ownerID, - ownerDisplayName: - storedMetadata['owner-display-name'], - storageClass: - storedMetadata['x-amz-storage-class'], - initiated: storedMetadata.initiated, - controllingLocationConstraint: - storedMetadata.controllingLocationConstraint, - checksumAlgorithm: storedMetadata.checksumAlgorithm, - checksumType: storedMetadata.checksumType, - checksumIsDefault: storedMetadata.checksumIsDefault, - }; + const initiatorID = storedMetadata.initiator.ID; + const ownerID = storedMetadata['owner-id']; + const mpuOverview = { + key: storedMetadata.key, + id: storedMetadata.id, + eventualStorageBucket: storedMetadata.eventualStorageBucket, + initiatorID, + initiatorDisplayName: storedMetadata.initiator.DisplayName, + ownerID, + ownerDisplayName: storedMetadata['owner-display-name'], + storageClass: storedMetadata['x-amz-storage-class'], + initiated: storedMetadata.initiated, + controllingLocationConstraint: storedMetadata.controllingLocationConstraint, + checksumAlgorithm: storedMetadata.checksumAlgorithm, + checksumType: storedMetadata.checksumType, + checksumIsDefault: storedMetadata.checksumIsDefault, + }; - const tagging = storedMetadata['x-amz-tagging']; - if (tagging) { - mpuOverview.tagging = tagging; - } - // If access was provided by the destination bucket's - // bucket policies, go ahead. - if (requestType === 'bucketPolicyGoAhead') { - return cb(null, mpuBucket, mpuOverview, storedMetadata); - } + const tagging = storedMetadata['x-amz-tagging']; + if (tagging) { + mpuOverview.tagging = tagging; + } + // If access was provided by the destination bucket's + // bucket policies, go ahead. + if (requestType === 'bucketPolicyGoAhead') { + return cb(null, mpuBucket, mpuOverview, storedMetadata); + } - const requesterID = authInfo.isRequesterAnIAMUser() ? - authInfo.getArn() : authInfo.getCanonicalID(); - const isRequesterInitiator = - initiatorID === requesterID; - const isRequesterParentAccountOfInitiator = - ownerID === authInfo.getCanonicalID(); - if (requestType === 'putPart or complete') { - // Only the initiator of the multipart - // upload can upload a part or complete the mpu - if (!isRequesterInitiator) { - return cb(errors.AccessDenied); - } + const requesterID = authInfo.isRequesterAnIAMUser() ? authInfo.getArn() : authInfo.getCanonicalID(); + const isRequesterInitiator = initiatorID === requesterID; + const isRequesterParentAccountOfInitiator = ownerID === authInfo.getCanonicalID(); + if (requestType === 'putPart or complete') { + // Only the initiator of the multipart + // upload can upload a part or complete the mpu + if (!isRequesterInitiator) { + return cb(errors.AccessDenied); } - if (requestType === 'deleteMPU' - || requestType === 'listParts') { - // In order for account/user to be - // authorized must either be the - // bucket owner or intitator of - // the multipart upload request - // (or parent account of initiator). - // In addition if the bucket policy - // designates someone else with - // s3:AbortMultipartUpload or - // s3:ListMultipartUploadPartsrights, - // as applicable, that account/user will have the right. - // If got to this step, it means there is - // no bucket policy on this. - if (mpuBucket.getOwner() !== authInfo.getCanonicalID() - && !isRequesterInitiator - && !isRequesterParentAccountOfInitiator) { - return cb(errors.AccessDenied); - } + } + if (requestType === 'deleteMPU' || requestType === 'listParts') { + // In order for account/user to be + // authorized must either be the + // bucket owner or intitator of + // the multipart upload request + // (or parent account of initiator). + // In addition if the bucket policy + // designates someone else with + // s3:AbortMultipartUpload or + // s3:ListMultipartUploadPartsrights, + // as applicable, that account/user will have the right. + // If got to this step, it means there is + // no bucket policy on this. + if ( + mpuBucket.getOwner() !== authInfo.getCanonicalID() && + !isRequesterInitiator && + !isRequesterParentAccountOfInitiator + ) { + return cb(errors.AccessDenied); } - return cb(null, mpuBucket, mpuOverview, storedMetadata); - }); + } + return cb(null, mpuBucket, mpuOverview, storedMetadata); + }); return undefined; }); }, @@ -851,13 +857,11 @@ const services = { * @param {function} cb - callback to send error or move to next task * @return {undefined} */ - metadataStorePart(mpuBucketName, partLocations, - metaStoreParams, log, cb) { + metadataStorePart(mpuBucketName, partLocations, metaStoreParams, log, cb) { assert.strictEqual(typeof mpuBucketName, 'string'); - const { partNumber, contentMD5, size, uploadId, lastModified, splitter, overheadField, ownerId } - = metaStoreParams; - const dateModified = typeof lastModified === 'string' ? - lastModified : new Date().toJSON(); + const { partNumber, contentMD5, size, uploadId, lastModified, splitter, overheadField, ownerId } = + metaStoreParams; + const dateModified = typeof lastModified === 'string' ? lastModified : new Date().toJSON(); assert.strictEqual(typeof splitter, 'string'); const partKey = `${uploadId}${splitter}${partNumber}`; const omVal = { @@ -865,7 +869,7 @@ const services = { // from an object to an array 'md-model-version': 3, partLocations, - 'key': partKey, + key: partKey, 'last-modified': dateModified, 'content-md5': contentMD5, 'content-length': size, @@ -887,14 +891,14 @@ const services = { }, /** - * Gets list of open multipart uploads in bucket - * @param {object} MPUbucketName - bucket in which objectMetadata is stored - * @param {object} listingParams - params object passing on - * needed items from request object - * @param {object} log - Werelogs logger - * @param {function} cb - callback to listMultipartUploads.js - * @return {undefined} - */ + * Gets list of open multipart uploads in bucket + * @param {object} MPUbucketName - bucket in which objectMetadata is stored + * @param {object} listingParams - params object passing on + * needed items from request object + * @param {object} log - Werelogs logger + * @param {function} cb - callback to listMultipartUploads.js + * @return {undefined} + */ getMultipartUploadListing(MPUbucketName, listingParams, log, cb) { assert.strictEqual(typeof MPUbucketName, 'string'); assert.strictEqual(typeof listingParams.splitter, 'string'); @@ -921,8 +925,7 @@ const services = { if (bucket.getMdBucketModelVersion() < 2) { listParams.splitter = constants.oldSplitter; } - metadata.listMultipartUploads(MPUbucketName, listParams, log, - cb); + metadata.listMultipartUploads(MPUbucketName, listParams, log, cb); return undefined; }); }, @@ -944,24 +947,26 @@ const services = { if (err?.is?.NoSuchBucket) { log.trace('no buckets found'); const creationDate = new Date().toJSON(); - const mpuBucket = new BucketInfo(MPUBucketName, + const mpuBucket = new BucketInfo( + MPUBucketName, destinationBucket.getOwner(), - destinationBucket.getOwnerDisplayName(), creationDate, - BucketInfo.currentModelVersion()); + destinationBucket.getOwnerDisplayName(), + creationDate, + BucketInfo.currentModelVersion(), + ); // Note that unlike during the creation of a normal bucket, // we do NOT add this bucket to the lists of a user's buckets. // By not adding this bucket to the lists of a user's buckets, // a getService request should not return a reference to this // bucket. This is the desired behavior since this should be // a hidden bucket. - return metadata.createBucket(MPUBucketName, mpuBucket, log, - err => { - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); - } - return cb(null, mpuBucket); - }); + return metadata.createBucket(MPUBucketName, mpuBucket, log, err => { + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); + } + return cb(null, mpuBucket); + }); } if (err) { log.error('error from metadata', { @@ -986,8 +991,7 @@ const services = { }, getSomeMPUparts(params, cb) { - const { uploadId, mpuBucketName, maxParts, partNumberMarker, log } = - params; + const { uploadId, mpuBucketName, maxParts, partNumberMarker, log } = params; assert.strictEqual(typeof mpuBucketName, 'string'); assert.strictEqual(typeof params.splitter, 'string'); const paddedPartNumber = `000000${partNumberMarker}`.substr(-5); @@ -1004,9 +1008,14 @@ const services = { // If have efficient way to batch delete metadata, should so this // all at once in production implementation assert.strictEqual(typeof mpuBucketName, 'string'); - async.eachLimit(keysToDelete, 5, (key, callback) => { - metadata.deleteObjectMD(mpuBucketName, key, { overheadField: constants.overheadField }, log, callback); - }, cb); + async.eachLimit( + keysToDelete, + 5, + (key, callback) => { + metadata.deleteObjectMD(mpuBucketName, key, { overheadField: constants.overheadField }, log, callback); + }, + cb, + ); }, }; diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js index a754300c62..a8136cfe25 100644 --- a/lib/utilities/collectResponseHeaders.js +++ b/lib/utilities/collectResponseHeaders.js @@ -1,6 +1,5 @@ const { getVersionIdResHeader } = require('../api/apiUtils/object/versioning'); -const checkUserMetadataSize - = require('../api/apiUtils/object/checkUserMetadataSize'); +const checkUserMetadataSize = require('../api/apiUtils/object/checkUserMetadataSize'); const { getAmzRestoreResHeader } = require('../api/apiUtils/object/coldStorage'); const { config } = require('../Config'); const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface'); @@ -16,38 +15,36 @@ const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface'); * @return {object} responseMetaHeaders headers with object metadata to include * in response to client */ -function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, - returnTagCount) { +function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagCount) { // Add user meta headers from objectMD let responseMetaHeaders = Object.assign({}, corsHeaders); - Object.keys(objectMD).filter(val => (val.startsWith('x-amz-meta-'))) - .forEach(id => { responseMetaHeaders[id] = objectMD[id]; }); + Object.keys(objectMD) + .filter(val => val.startsWith('x-amz-meta-')) + .forEach(id => { + responseMetaHeaders[id] = objectMD[id]; + }); // Check user metadata size responseMetaHeaders = checkUserMetadataSize(responseMetaHeaders); // TODO: When implement lifecycle, add additional response headers // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html - responseMetaHeaders['x-amz-version-id'] = - getVersionIdResHeader(versioningCfg, objectMD); + responseMetaHeaders['x-amz-version-id'] = getVersionIdResHeader(versioningCfg, objectMD); if (objectMD['x-amz-website-redirect-location']) { - responseMetaHeaders['x-amz-website-redirect-location'] = - objectMD['x-amz-website-redirect-location']; + responseMetaHeaders['x-amz-website-redirect-location'] = objectMD['x-amz-website-redirect-location']; } if (objectMD['x-amz-storage-class'] !== 'STANDARD') { - responseMetaHeaders['x-amz-storage-class'] = - objectMD['x-amz-storage-class']; + responseMetaHeaders['x-amz-storage-class'] = objectMD['x-amz-storage-class']; } if (objectMD['x-amz-server-side-encryption']) { - responseMetaHeaders['x-amz-server-side-encryption'] - = objectMD['x-amz-server-side-encryption']; + responseMetaHeaders['x-amz-server-side-encryption'] = objectMD['x-amz-server-side-encryption']; } const kmsKey = objectMD['x-amz-server-side-encryption-aws-kms-key-id']; - if (kmsKey && - objectMD['x-amz-server-side-encryption'] === 'aws:kms') { - responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id'] - = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKey) : kmsKey; + if (kmsKey && objectMD['x-amz-server-side-encryption'] === 'aws:kms') { + responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id'] = config.kmsHideScalityArn + ? getKeyIdFromArn(kmsKey) + : kmsKey; } const restoreHeader = getAmzRestoreResHeader(objectMD); @@ -65,8 +62,7 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, responseMetaHeaders['Cache-Control'] = objectMD['cache-control']; } if (objectMD['content-disposition']) { - responseMetaHeaders['Content-Disposition'] - = objectMD['content-disposition']; + responseMetaHeaders['Content-Disposition'] = objectMD['content-disposition']; } if (objectMD['content-encoding']) { responseMetaHeaders['Content-Encoding'] = objectMD['content-encoding']; @@ -78,40 +74,30 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, // Note: ETag must have a capital "E" and capital "T" for cosbench // to work. responseMetaHeaders.ETag = `"${objectMD['content-md5']}"`; - responseMetaHeaders['Last-Modified'] = - new Date(objectMD['last-modified']).toUTCString(); + responseMetaHeaders['Last-Modified'] = new Date(objectMD['last-modified']).toUTCString(); if (objectMD['content-type']) { responseMetaHeaders['Content-Type'] = objectMD['content-type']; } - if (returnTagCount && objectMD.tags && - Object.keys(objectMD.tags).length > 0) { - responseMetaHeaders['x-amz-tagging-count'] = - Object.keys(objectMD.tags).length; + if (returnTagCount && objectMD.tags && Object.keys(objectMD.tags).length > 0) { + responseMetaHeaders['x-amz-tagging-count'] = Object.keys(objectMD.tags).length; } - const hasRetentionInfo = objectMD.retentionMode - && objectMD.retentionDate; + const hasRetentionInfo = objectMD.retentionMode && objectMD.retentionDate; if (hasRetentionInfo) { - responseMetaHeaders['x-amz-object-lock-retain-until-date'] - = objectMD.retentionDate; - responseMetaHeaders['x-amz-object-lock-mode'] - = objectMD.retentionMode; + responseMetaHeaders['x-amz-object-lock-retain-until-date'] = objectMD.retentionDate; + responseMetaHeaders['x-amz-object-lock-mode'] = objectMD.retentionMode; } if (objectMD.legalHold !== undefined) { - responseMetaHeaders['x-amz-object-lock-legal-hold'] - = objectMD.legalHold ? 'ON' : 'OFF'; + responseMetaHeaders['x-amz-object-lock-legal-hold'] = objectMD.legalHold ? 'ON' : 'OFF'; } if (objectMD.replicationInfo && objectMD.replicationInfo.status) { - responseMetaHeaders['x-amz-replication-status'] = - objectMD.replicationInfo.status; + responseMetaHeaders['x-amz-replication-status'] = objectMD.replicationInfo.status; } if (Array.isArray(objectMD?.replicationInfo?.backends)) { objectMD.replicationInfo.backends.forEach(backend => { const { status, site, dataStoreVersionId } = backend; - responseMetaHeaders[`x-amz-meta-${site}-replication-status`] = - status; + responseMetaHeaders[`x-amz-meta-${site}-replication-status`] = status; if (status === 'COMPLETED' && dataStoreVersionId) { - responseMetaHeaders[`x-amz-meta-${site}-version-id`] = - dataStoreVersionId; + responseMetaHeaders[`x-amz-meta-${site}-version-id`] = dataStoreVersionId; } }); } diff --git a/tests/unit/api/apiUtils/versioning.js b/tests/unit/api/apiUtils/versioning.js index 1edea6d0b0..1bd1e876f0 100644 --- a/tests/unit/api/apiUtils/versioning.js +++ b/tests/unit/api/apiUtils/versioning.js @@ -6,11 +6,13 @@ const INF_VID = versioning.VersionID.getInfVid(config.replicationGroupId); const { scaledMsPerDay } = config.getTimeOptions(); const sinon = require('sinon'); -const { processVersioningState, getMasterState, - getVersionSpecificMetadataOptions, - preprocessingVersioningDelete, - overwritingVersioning } = - require('../../../../lib/api/apiUtils/object/versioning'); +const { + processVersioningState, + getMasterState, + getVersionSpecificMetadataOptions, + preprocessingVersioningDelete, + overwritingVersioning, +} = require('../../../../lib/api/apiUtils/object/versioning'); describe('versioning helpers', () => { describe('getMasterState+processVersioningState', () => { @@ -518,17 +520,22 @@ describe('versioning helpers', () => { }, ].forEach(testCase => [false, true].forEach(nullVersionCompatMode => - ['Enabled', 'Suspended'].forEach(versioningStatus => it( - `${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}` + - `, versioning Status=${versioningStatus}`, - () => { - const mst = getMasterState(testCase.objMD); - const res = processVersioningState(mst, versioningStatus, nullVersionCompatMode); - const resultName = `versioning${versioningStatus}` + - `${nullVersionCompatMode ? 'Compat' : ''}ExpectedRes`; - const expectedRes = testCase[resultName]; - assert.deepStrictEqual(res, expectedRes); - })))); + ['Enabled', 'Suspended'].forEach(versioningStatus => + it( + `${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}` + + `, versioning Status=${versioningStatus}`, + () => { + const mst = getMasterState(testCase.objMD); + const res = processVersioningState(mst, versioningStatus, nullVersionCompatMode); + const resultName = + `versioning${versioningStatus}` + `${nullVersionCompatMode ? 'Compat' : ''}ExpectedRes`; + const expectedRes = testCase[resultName]; + assert.deepStrictEqual(res, expectedRes); + }, + ), + ), + ), + ); }); describe('getVersionSpecificMetadataOptions', () => { @@ -583,14 +590,13 @@ describe('versioning helpers', () => { }, ].forEach(testCase => [false, true].forEach(nullVersionCompatMode => - it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, - () => { - const options = getVersionSpecificMetadataOptions( - testCase.objMD, nullVersionCompatMode); - const expectedResAttr = nullVersionCompatMode ? - 'expectedResCompat' : 'expectedRes'; + it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, () => { + const options = getVersionSpecificMetadataOptions(testCase.objMD, nullVersionCompatMode); + const expectedResAttr = nullVersionCompatMode ? 'expectedResCompat' : 'expectedRes'; assert.deepStrictEqual(options, testCase[expectedResAttr]); - }))); + }), + ), + ); }); describe('preprocessingVersioningDelete', () => { @@ -669,24 +675,28 @@ describe('versioning helpers', () => { }, ].forEach(testCase => [false, true].forEach(nullVersionCompatMode => - it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, - () => { + it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, () => { const mockBucketMD = { getVersioningConfiguration: () => ({ Status: 'Enabled' }), }; const options = preprocessingVersioningDelete( - 'foobucket', mockBucketMD, testCase.objMD, testCase.reqVersionId, - nullVersionCompatMode); - const expectedResAttr = nullVersionCompatMode ? - 'expectedResCompat' : 'expectedRes'; + 'foobucket', + mockBucketMD, + testCase.objMD, + testCase.reqVersionId, + nullVersionCompatMode, + ); + const expectedResAttr = nullVersionCompatMode ? 'expectedResCompat' : 'expectedRes'; assert.deepStrictEqual(options, testCase[expectedResAttr]); - }))); + }), + ), + ); }); describe('overwritingVersioning', () => { const days = 3; const archiveInfo = { - 'archiveID': '126783123678', + archiveID: '126783123678', }; const now = Date.now(); let clock; @@ -702,70 +712,70 @@ describe('versioning helpers', () => { [ { description: 'Should update archive with restore infos', - objMD: { - 'versionId': '2345678', + objMD: { + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep user mds and tags', hasUserMD: true, objMD: { - 'versionId': '2345678', + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-meta-test': 'test', 'x-amz-meta-test2': 'test2', - 'tags': { 'testtag': 'testtag', 'testtag2': 'testtag2' }, + tags: { testtag: 'testtag', testtag2: 'testtag2' }, 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'metaHeaders': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + metaHeaders: { 'x-amz-meta-test': 'test', 'x-amz-meta-test2': 'test2', }, - 'taggingCopy': { 'testtag': 'testtag', 'testtag2': 'testtag2' }, - 'amzStorageClass': 'cold-location', - 'archive': { + taggingCopy: { testtag: 'testtag', testtag2: 'testtag2' }, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } + restoreRequestedDays: days, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, }, }, { @@ -773,257 +783,243 @@ describe('versioning helpers', () => { objMD: { 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', - 'nullVersionId': 'vnull', - 'isNull': true, + originOp: 's3:PutObject', + nullVersionId: 'vnull', + isNull: true, 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'amzStorageClass': 'cold-location', - 'taggingCopy': undefined, - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + amzStorageClass: 'cold-location', + taggingCopy: undefined, + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should not keep x-amz-meta-scal-s3-restore-attempt user MD', hasUserMD: true, objMD: { - 'versionId': '2345678', + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-meta-test': 'test', 'x-amz-meta-scal-s3-restore-attempt': 14, 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'metaHeaders': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + metaHeaders: { 'x-amz-meta-test': 'test', }, - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep replication infos', objMD: { - 'versionId': '2345678', - 'creation-time': now, - 'last-modified': now, - 'originOp': 's3:PutObject', - 'x-amz-storage-class': 'cold-location', - 'replicationInfo': { - 'status': 'COMPLETED', - 'backends': [ - { - 'site': 'azure-blob', - 'status': 'COMPLETED', - 'dataStoreVersionId': '' - } - ], - 'content': [ - 'DATA', - 'METADATA' - ], - 'destination': 'arn:aws:s3:::replicate-cold', - 'storageClass': 'azure-blob', - 'role': 'arn:aws:iam::root:role/s3-replication-role', - 'storageType': 'azure', - 'dataStoreVersionId': '', - }, - archive: { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + versionId: '2345678', + 'creation-time': now, + 'last-modified': now, + originOp: 's3:PutObject', + 'x-amz-storage-class': 'cold-location', + replicationInfo: { + status: 'COMPLETED', + backends: [ + { + site: 'azure-blob', + status: 'COMPLETED', + dataStoreVersionId: '', + }, + ], + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::replicate-cold', + storageClass: 'azure-blob', + role: 'arn:aws:iam::root:role/s3-replication-role', + storageType: 'azure', + dataStoreVersionId: '', + }, + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'amzStorageClass': 'cold-location', - 'replicationInfo': { - 'status': 'COMPLETED', - 'backends': [ + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + amzStorageClass: 'cold-location', + replicationInfo: { + status: 'COMPLETED', + backends: [ { - 'site': 'azure-blob', - 'status': 'COMPLETED', - 'dataStoreVersionId': '' - } + site: 'azure-blob', + status: 'COMPLETED', + dataStoreVersionId: '', + }, ], - 'content': [ - 'DATA', - 'METADATA' - ], - 'destination': 'arn:aws:s3:::replicate-cold', - 'storageClass': 'azure-blob', - 'role': 'arn:aws:iam::root:role/s3-replication-role', - 'storageType': 'azure', - 'dataStoreVersionId': '', - }, - 'taggingCopy': undefined, + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::replicate-cold', + storageClass: 'azure-blob', + role: 'arn:aws:iam::root:role/s3-replication-role', + storageType: 'azure', + dataStoreVersionId: '', + }, + taggingCopy: undefined, archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep legalHold', objMD: { - 'versionId': '2345678', - 'creation-time': now, - 'last-modified': now, - 'originOp': 's3:PutObject', - 'legalHold': true, - 'x-amz-storage-class': 'cold-location', - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + versionId: '2345678', + 'creation-time': now, + 'last-modified': now, + originOp: 's3:PutObject', + legalHold: true, + 'x-amz-storage-class': 'cold-location', + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'legalHold': true, - 'amzStorageClass': 'cold-location', - 'taggingCopy': undefined, - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + legalHold: true, + amzStorageClass: 'cold-location', + taggingCopy: undefined, + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, { description: 'Should keep ACLs', objMD: { - 'versionId': '2345678', - 'creation-time': now, - 'last-modified': now, - 'originOp': 's3:PutObject', - 'x-amz-storage-class': 'cold-location', - 'acl': { - 'Canned': '', - 'FULL_CONTROL': [ - '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185' - ], - 'WRITE_ACP': [], - 'READ': [ - 'http://acs.amazonaws.com/groups/global/AllUsers' - ], - 'READ_ACP': [] - }, - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } + versionId: '2345678', + 'creation-time': now, + 'last-modified': now, + originOp: 's3:PutObject', + 'x-amz-storage-class': 'cold-location', + acl: { + Canned: '', + FULL_CONTROL: ['872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'], + WRITE_ACP: [], + READ: ['http://acs.amazonaws.com/groups/global/AllUsers'], + READ_ACP: [], + }, + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, }, expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'acl': { - 'Canned': '', - 'FULL_CONTROL': [ - '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185' - ], - 'WRITE_ACP': [], - 'READ': [ - 'http://acs.amazonaws.com/groups/global/AllUsers' - ], - 'READ_ACP': [] - }, - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + acl: { + Canned: '', + FULL_CONTROL: ['872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'], + WRITE_ACP: [], + READ: ['http://acs.amazonaws.com/groups/global/AllUsers'], + READ_ACP: [], + }, + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, }, }, - { - description: 'Should keep contentMD5 of the original object', - objMD: { - 'versionId': '2345678', + { + description: 'Should keep contentMD5 of the original object', + objMD: { + versionId: '2345678', 'creation-time': now, 'last-modified': now, - 'originOp': 's3:PutObject', + originOp: 's3:PutObject', 'x-amz-storage-class': 'cold-location', 'content-md5': '123456789-5', - 'acl': {}, - 'archive': { - 'restoreRequestedDays': days, - 'restoreRequestedAt': now, - archiveInfo - } - }, - metadataStoreParams: { - 'contentMD5': '987654321-3', - }, - expectedRes: { - 'creationTime': now, - 'lastModifiedDate': now, - 'updateMicroVersionId': true, - 'originOp': 's3:ObjectRestore:Completed', - 'contentMD5': '123456789-5', - 'restoredEtag': '987654321-3', - 'acl': {}, - 'taggingCopy': undefined, - 'amzStorageClass': 'cold-location', - 'archive': { - archiveInfo, - 'restoreRequestedDays': 3, - 'restoreRequestedAt': now, - 'restoreCompletedAt': new Date(now), - 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), - } - } + acl: {}, + archive: { + restoreRequestedDays: days, + restoreRequestedAt: now, + archiveInfo, + }, + }, + metadataStoreParams: { + contentMD5: '987654321-3', + }, + expectedRes: { + creationTime: now, + lastModifiedDate: now, + updateMicroVersionId: true, + originOp: 's3:ObjectRestore:Completed', + contentMD5: '123456789-5', + restoredEtag: '987654321-3', + acl: {}, + taggingCopy: undefined, + amzStorageClass: 'cold-location', + archive: { + archiveInfo, + restoreRequestedDays: 3, + restoreRequestedAt: now, + restoreCompletedAt: new Date(now), + restoreWillExpireAt: new Date(now + days * scaledMsPerDay), + }, + }, }, ].forEach(testCase => { it(testCase.description, () => { diff --git a/tests/unit/lib/services.spec.js b/tests/unit/lib/services.spec.js index f76ee39502..2ee61aabb6 100644 --- a/tests/unit/lib/services.spec.js +++ b/tests/unit/lib/services.spec.js @@ -94,8 +94,12 @@ describe('services', () => { IsTruncated: false, }); - services.findObjectVersionByUploadId(bucketName, objectKey, 'non-existent-upload-id', - log, (err, foundVersion) => { + services.findObjectVersionByUploadId( + bucketName, + objectKey, + 'non-existent-upload-id', + log, + (err, foundVersion) => { assert.ifError(err); sinon.assert.calledTwice(getObjectListingStub); @@ -104,7 +108,8 @@ describe('services', () => { assert.strictEqual(secondCallParams.versionIdMarker, 'version-marker'); assert.strictEqual(foundVersion, null); done(); - }); + }, + ); }); it('should find a version on the first page of many and stop listing', done => { @@ -176,10 +181,10 @@ describe('services', () => { let putObjectMDStub; beforeEach(() => { - putObjectMDStub = sinon.stub(metadata, 'putObjectMD') + putObjectMDStub = sinon + .stub(metadata, 'putObjectMD') .callsFake((bucket, key, md, opts, reqLog, cb) => cb(null)); - sinon.stub(acl, 'parseAclFromHeaders') - .callsFake((params, cb) => cb(null, { Canned: 'private' })); + sinon.stub(acl, 'parseAclFromHeaders').callsFake((params, cb) => cb(null, { Canned: 'private' })); }); it('should store checksumAlgorithm, checksumType and checksumIsDefault when provided', done => { @@ -241,8 +246,7 @@ describe('services', () => { const authInfo = makeAuthInfo('accessKey1'); const ownerID = authInfo.getCanonicalID(); const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`; - const mpuOverviewKey = `overview${constants.splitter}${objectKey}` + - `${constants.splitter}${uploadId}`; + const mpuOverviewKey = `overview${constants.splitter}${objectKey}` + `${constants.splitter}${uploadId}`; const mpuBucket = { getName: () => mpuBucketName, getMdBucketModelVersion: () => 2, @@ -268,46 +272,48 @@ describe('services', () => { assert.strictEqual(name, mpuBucketName); done(null, mpuBucket); }); - sinon.stub(metadata, 'getObjectMD') - .callsFake((bucket, key, params, reqLog, done) => { - assert.strictEqual(bucket, mpuBucketName); - assert.strictEqual(key, mpuOverviewKey); - done(null, { - ...storedMetadata, - ...storedMetadataOverride, - }); + sinon.stub(metadata, 'getObjectMD').callsFake((bucket, key, params, reqLog, done) => { + assert.strictEqual(bucket, mpuBucketName); + assert.strictEqual(key, mpuOverviewKey); + done(null, { + ...storedMetadata, + ...storedMetadataOverride, }); + }); - services.metadataValidateMultipart({ - bucketName, - objectKey, - uploadId, - authInfo, - requestType: 'listParts', - log, - }, cb); + services.metadataValidateMultipart( + { + bucketName, + objectKey, + uploadId, + authInfo, + requestType: 'listParts', + log, + }, + cb, + ); } - it('should expose checksum fields from stored MPU overview metadata', - done => { - validateMultipart({ - checksumAlgorithm: 'sha256', - checksumType: 'COMPOSITE', - checksumIsDefault: false, - }, (err, bucket, mpuOverview, returnedStoredMetadata) => { - assert.ifError(err); - assert.strictEqual(bucket, mpuBucket); - assert.strictEqual(returnedStoredMetadata.checksumAlgorithm, - 'sha256'); - assert.strictEqual(mpuOverview.checksumAlgorithm, 'sha256'); - assert.strictEqual(mpuOverview.checksumType, 'COMPOSITE'); - assert.strictEqual(mpuOverview.checksumIsDefault, false); - done(); - }); + it('should expose checksum fields from stored MPU overview metadata', done => { + validateMultipart( + { + checksumAlgorithm: 'sha256', + checksumType: 'COMPOSITE', + checksumIsDefault: false, + }, + (err, bucket, mpuOverview, returnedStoredMetadata) => { + assert.ifError(err); + assert.strictEqual(bucket, mpuBucket); + assert.strictEqual(returnedStoredMetadata.checksumAlgorithm, 'sha256'); + assert.strictEqual(mpuOverview.checksumAlgorithm, 'sha256'); + assert.strictEqual(mpuOverview.checksumType, 'COMPOSITE'); + assert.strictEqual(mpuOverview.checksumIsDefault, false); + done(); + }, + ); }); - it('should leave checksum fields undefined for legacy MPU overview metadata', - done => { + it('should leave checksum fields undefined for legacy MPU overview metadata', done => { validateMultipart({}, (err, bucket, mpuOverview) => { assert.ifError(err); assert.strictEqual(bucket, mpuBucket); diff --git a/tests/unit/utils/collectResponseHeaders.js b/tests/unit/utils/collectResponseHeaders.js index 75cd3c134f..ced3c18e48 100644 --- a/tests/unit/utils/collectResponseHeaders.js +++ b/tests/unit/utils/collectResponseHeaders.js @@ -1,6 +1,5 @@ const assert = require('assert'); -const collectResponseHeaders = - require('../../../lib/utilities/collectResponseHeaders'); +const collectResponseHeaders = require('../../../lib/utilities/collectResponseHeaders'); describe('Middleware: Collect Response Headers', () => { it('should be able to set replication status when config is set', () => { @@ -21,22 +20,19 @@ describe('Middleware: Collect Response Headers', () => { }; const headers = collectResponseHeaders(objectMD); assert.deepStrictEqual(headers['x-amz-replication-status'], 'COMPLETED'); - assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'], - 'COMPLETED'); + assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'], 'COMPLETED'); assert.deepStrictEqual(headers['x-amz-meta-us-east-1-version-id'], '123'); - assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'], - 'COMPLETED'); + assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'], 'COMPLETED'); assert.deepStrictEqual(headers['x-amz-meta-us-west-2-version-id'], undefined); }); - + [ { md: { replicationInfo: null }, test: 'when config is not set' }, { md: {}, test: 'for older objects' }, ].forEach(item => { it(`should skip replication header ${item.test}`, () => { const headers = collectResponseHeaders(item.md); - assert.deepStrictEqual(headers['x-amz-replication-status'], - undefined); + assert.deepStrictEqual(headers['x-amz-replication-status'], undefined); }); }); @@ -45,19 +41,16 @@ describe('Middleware: Collect Response Headers', () => { assert.strictEqual(headers['Accept-Ranges'], 'bytes'); }); - it('should return an undefined value when x-amz-website-redirect-location' + - ' is empty', () => { + it('should return an undefined value when x-amz-website-redirect-location' + ' is empty', () => { const objectMD = { 'x-amz-website-redirect-location': '' }; const headers = collectResponseHeaders(objectMD); - assert.strictEqual(headers['x-amz-website-redirect-location'], - undefined); + assert.strictEqual(headers['x-amz-website-redirect-location'], undefined); }); it('should return the (nonempty) value of WebsiteRedirectLocation', () => { const obj = { 'x-amz-website-redirect-location': 'google.com' }; const headers = collectResponseHeaders(obj); - assert.strictEqual(headers['x-amz-website-redirect-location'], - 'google.com'); + assert.strictEqual(headers['x-amz-website-redirect-location'], 'google.com'); }); it('should not set flag when transition not in progress', () => { From bf3b2aeab62f2ce82985049dc6c9f14eb41cefd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Thu, 28 May 2026 23:42:47 +0200 Subject: [PATCH 2/6] Bump arsenal dependency Issue: CLDSRV-906 --- package.json | 2 +- yarn.lock | 62 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 0e28ebe250..619b64c9af 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@opentelemetry/instrumentation-ioredis": "~0.64.0", "@opentelemetry/instrumentation-mongodb": "~0.69.0", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.4.7", + "arsenal": "git+https://github.com/scality/Arsenal#f6b6e2a37675e76186326e22c17b7a4cef77f5e0", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/yarn.lock b/yarn.lock index 138a4c068f..2a05031e96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3279,6 +3279,13 @@ dependencies: "@opentelemetry/semantic-conventions" "^1.29.0" +"@opentelemetry/core@2.8.0": + version "2.8.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-2.8.0.tgz#f6e86de3688bdb54a6ca8f4935363a5b588ae91c" + integrity sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww== + dependencies: + "@opentelemetry/semantic-conventions" "^1.29.0" + "@opentelemetry/exporter-logs-otlp-grpc@0.218.0": version "0.218.0" resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.218.0.tgz#7e5a7e624074d6449590c851d58efe60096314b3" @@ -3501,7 +3508,7 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/redis-common/-/redis-common-0.38.3.tgz#31a0464a48a991c29408614e3725d94db7c11aee" integrity sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw== -"@opentelemetry/resources@2.7.1", "@opentelemetry/resources@^2.7.1": +"@opentelemetry/resources@2.7.1": version "2.7.1" resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-2.7.1.tgz#3b2a9179f6119bb1f2cddefe41ba9b2855504a5d" integrity sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ== @@ -3509,6 +3516,14 @@ "@opentelemetry/core" "2.7.1" "@opentelemetry/semantic-conventions" "^1.29.0" +"@opentelemetry/resources@2.8.0", "@opentelemetry/resources@^2.7.1": + version "2.8.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-2.8.0.tgz#9bcb658ab6254f33099f4a95544b40d6f53cc946" + integrity sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg== + dependencies: + "@opentelemetry/core" "2.8.0" + "@opentelemetry/semantic-conventions" "^1.29.0" + "@opentelemetry/sdk-logs@0.218.0": version "0.218.0" resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-logs/-/sdk-logs-0.218.0.tgz#78886fe300b82802cee9963208b2af326b928af5" @@ -3558,7 +3573,7 @@ "@opentelemetry/sdk-trace-node" "2.7.1" "@opentelemetry/semantic-conventions" "^1.29.0" -"@opentelemetry/sdk-trace-base@2.7.1", "@opentelemetry/sdk-trace-base@^2.7.1": +"@opentelemetry/sdk-trace-base@2.7.1": version "2.7.1" resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.7.1.tgz#9160c3af9ef2219c26563abd136e22fb7d19b34f" integrity sha512-NAYIlsF8MPUsKqJMiDQJTMPOmlbawC1Iz/omMLygZ1C9am8fTKYjTaI+OZM+WTY3t3Glo0wnOg/6/pac6RGPPw== @@ -3567,6 +3582,15 @@ "@opentelemetry/resources" "2.7.1" "@opentelemetry/semantic-conventions" "^1.29.0" +"@opentelemetry/sdk-trace-base@^2.7.1": + version "2.8.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.8.0.tgz#ec9c1d69e2e6fba256c9df0c8e8d67d42386d52b" + integrity sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ== + dependencies: + "@opentelemetry/core" "2.8.0" + "@opentelemetry/resources" "2.8.0" + "@opentelemetry/semantic-conventions" "^1.29.0" + "@opentelemetry/sdk-trace-node@2.7.1": version "2.7.1" resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.7.1.tgz#54dedb8e77fa51a6d02fc2192097739266c82168" @@ -3618,11 +3642,6 @@ resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== -"@protobufjs/inquire@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.2.tgz#ae64fbc014ff44c8bfad03dd4c93cd2d6a4c82db" - integrity sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw== - "@protobufjs/path@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" @@ -6151,9 +6170,9 @@ undici-types "~6.20.0" "@types/node@>=13.7.0": - version "25.9.1" - resolved "https://registry.yarnpkg.com/@types/node/-/node-25.9.1.tgz#3bda556db500ae4319c08e7fc9ab94f19013ba0b" - integrity sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg== + version "25.9.3" + resolved "https://registry.yarnpkg.com/@types/node/-/node-25.9.3.tgz#11dfe7a33e68fa5c560f0aa76cc5595621ef26b9" + integrity sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg== dependencies: undici-types ">=7.24.0 <7.24.7" @@ -6298,9 +6317,9 @@ acorn@^8.14.0: integrity sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg== acorn@^8.15.0: - version "8.16.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.16.0.tgz#4ce79c89be40afe7afe8f3adb902a1f1ce9ac08a" - integrity sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw== + version "8.17.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.17.0.tgz#1785adb84faf8d8add10369b93826fc2bd08f1fe" + integrity sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg== agent-base@^4.3.0: version "4.3.0" @@ -6576,9 +6595,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.4.7": +"arsenal@git+https://github.com/scality/Arsenal#f6b6e2a37675e76186326e22c17b7a4cef77f5e0": version "8.4.7" - resolved "git+https://github.com/scality/Arsenal#096cb33f1092b35113a11521bbbb62ef615aed79" + resolved "git+https://github.com/scality/Arsenal#f6b6e2a37675e76186326e22c17b7a4cef77f5e0" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" @@ -9011,9 +9030,9 @@ import-fresh@^3.2.1: resolve-from "^4.0.0" import-in-the-middle@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-3.0.1.tgz#8a0a1230c9b865c0e12698171646ae1e3fff691d" - integrity sha512-pYkiyXVL2Mf3pozdlDGV6NAObxQx13Ae8knZk1UJRJ6uRW/ZRmTGHlQYtrsSl7ubuE5F8CD1z+s1n4RHNuTtuA== + version "3.1.0" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-3.1.0.tgz#0d0e88c93c276599bd4bf81835946d723656efab" + integrity sha512-c0AeAV8VcwZzfYE7euTZY3H+VXUPMVugiovdosq80lqEXJmOekg3zGUAYg6KImHMaMuBoTUfTv7xNpUFdy0hJA== dependencies: acorn "^8.15.0" acorn-import-attributes "^1.9.5" @@ -11230,9 +11249,9 @@ promise-retry@^2.0.1: retry "^0.12.0" protobufjs@^7.5.5: - version "7.6.2" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.6.2.tgz#2659f77bd8d54778814c274dc0df808f54c88918" - integrity sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ== + version "7.6.4" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.6.4.tgz#8bb000300026efd63eb7951d26e5dbb38f5658f2" + integrity sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" @@ -11240,7 +11259,6 @@ protobufjs@^7.5.5: "@protobufjs/eventemitter" "^1.1.1" "@protobufjs/fetch" "^1.1.1" "@protobufjs/float" "^1.0.2" - "@protobufjs/inquire" "^1.1.2" "@protobufjs/path" "^1.1.2" "@protobufjs/pool" "^1.1.0" "@protobufjs/utf8" "^1.1.1" From 16a098d89b3172ef6365c8e143edc23c52942055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 29 May 2026 01:39:46 +0200 Subject: [PATCH 3/6] Use isReplica field for x-amz-replication-status response header Prefer ReplicationInfo.isReplica over replicationInfo.status when producing x-amz-replication-status. The legacy status === 'REPLICA' branch is kept as a fallback so objects written before the feature still surface the correct value. Issue: CLDSRV-906 --- lib/utilities/collectResponseHeaders.js | 7 +++++-- tests/unit/utils/collectResponseHeaders.js | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js index a8136cfe25..8480cf8d9f 100644 --- a/lib/utilities/collectResponseHeaders.js +++ b/lib/utilities/collectResponseHeaders.js @@ -89,8 +89,11 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagC if (objectMD.legalHold !== undefined) { responseMetaHeaders['x-amz-object-lock-legal-hold'] = objectMD.legalHold ? 'ON' : 'OFF'; } - if (objectMD.replicationInfo && objectMD.replicationInfo.status) { - responseMetaHeaders['x-amz-replication-status'] = objectMD.replicationInfo.status; + if (objectMD.replicationInfo) { + const { isReplica, status } = objectMD.replicationInfo; + if (isReplica || status) { + responseMetaHeaders['x-amz-replication-status'] = isReplica ? 'REPLICA' : status; + } } if (Array.isArray(objectMD?.replicationInfo?.backends)) { objectMD.replicationInfo.backends.forEach(backend => { diff --git a/tests/unit/utils/collectResponseHeaders.js b/tests/unit/utils/collectResponseHeaders.js index ced3c18e48..de17b64321 100644 --- a/tests/unit/utils/collectResponseHeaders.js +++ b/tests/unit/utils/collectResponseHeaders.js @@ -8,6 +8,22 @@ describe('Middleware: Collect Response Headers', () => { assert.deepStrictEqual(headers['x-amz-replication-status'], 'REPLICA'); }); + it('should set REPLICA header from isReplica even when status is PENDING', () => { + const objectMD = { + replicationInfo: { status: 'PENDING', isReplica: true }, + }; + const headers = collectResponseHeaders(objectMD); + assert.deepStrictEqual(headers['x-amz-replication-status'], 'REPLICA'); + }); + + it('should use replicationInfo.status when isReplica is false', () => { + const objectMD = { + replicationInfo: { status: 'PENDING', isReplica: false }, + }; + const headers = collectResponseHeaders(objectMD); + assert.deepStrictEqual(headers['x-amz-replication-status'], 'PENDING'); + }); + it('should set the replication status of each site', () => { const objectMD = { replicationInfo: { From be378e8a04b151f256fa8232d26221d31f861ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 29 May 2026 01:39:55 +0200 Subject: [PATCH 4/6] Bump microVersionId on object metadata writes Bump microVersionId on every user write that changes object metadata other than replicationInfo, providing a unique revision identifier needed by upcoming cascaded CRR loop detection. Issue: CLDSRV-906 --- lib/api/apiUtils/object/bumpMicroVersionId.js | 27 ++++++++ lib/api/objectDeleteTagging.js | 2 + lib/api/objectPutLegalHold.js | 2 + lib/api/objectPutRetention.js | 2 + lib/api/objectPutTagging.js | 2 + lib/metadata/acl.js | 2 + lib/services.js | 3 +- .../api/apiUtils/object/bumpMicroVersionId.js | 31 +++++++++ tests/unit/api/objectReplicationMD.js | 65 +++++++++++++++++++ tests/unit/lib/services.spec.js | 35 ++++++++++ 10 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 lib/api/apiUtils/object/bumpMicroVersionId.js create mode 100644 tests/unit/api/apiUtils/object/bumpMicroVersionId.js diff --git a/lib/api/apiUtils/object/bumpMicroVersionId.js b/lib/api/apiUtils/object/bumpMicroVersionId.js new file mode 100644 index 0000000000..1a76970726 --- /dev/null +++ b/lib/api/apiUtils/object/bumpMicroVersionId.js @@ -0,0 +1,27 @@ +const { versioning } = require('arsenal'); +const { config } = require('../../../Config'); + +/** + * Bump objectMD.microVersionId. microVersionId is a generic + * metadata-revision marker, not a CRR-specific field, but cascaded CRR + * is its only consumer today - so we gate on replicationInfo to avoid + * inflating storage on objects that wouldn't use it. The gate can be + * widened later if another consumer needs it on every object. + * Pass `force = true` to bump unconditionally. + * + * @param {object} objectMD - object MD POJO or `md.getValue()` + * @param {boolean} [force] - bump even without replicationInfo + * @return {undefined} + */ +function bumpMicroVersionId(objectMD, force) { + if (!force && !objectMD?.replicationInfo) { + return; + } + + const { instanceId, replicationGroupId } = config; + + // eslint-disable-next-line no-param-reassign + objectMD.microVersionId = versioning.VersionID.generateVersionId(instanceId, replicationGroupId); +} + +module.exports = bumpMicroVersionId; diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js index 45d67a0f97..9c8420c737 100644 --- a/lib/api/objectDeleteTagging.js +++ b/lib/api/objectDeleteTagging.js @@ -13,6 +13,7 @@ const monitoring = require('../utilities/monitoringHandler'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const { data } = require('../data/wrapper'); const { config } = require('../Config'); const REPLICATION_ACTION = 'DELETE_TAGGING'; @@ -93,6 +94,7 @@ function objectDeleteTagging(authInfo, request, log, callback) { if (replicationInfo) { // eslint-disable-next-line no-param-reassign objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + bumpMicroVersionId(objectMD); } // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectTagging:Delete'; diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js index d687f77ce6..5fe844a1e8 100644 --- a/lib/api/objectPutLegalHold.js +++ b/lib/api/objectPutLegalHold.js @@ -8,6 +8,7 @@ const { getVersionSpecificMetadataOptions, } = require('./apiUtils/object/versioning'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -105,6 +106,7 @@ function objectPutLegalHold(authInfo, request, log, callback) { if (replicationInfo) { // eslint-disable-next-line no-param-reassign objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + bumpMicroVersionId(objectMD); } // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectLegalHold:Put'; diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js index b8182646e9..e281b71f91 100644 --- a/lib/api/objectPutRetention.js +++ b/lib/api/objectPutRetention.js @@ -10,6 +10,7 @@ const { ObjectLockInfo, hasGovernanceBypassHeader } = require('./apiUtils/object const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { config } = require('../Config'); @@ -126,6 +127,7 @@ function objectPutRetention(authInfo, request, log, callback) { ); if (replicationInfo) { objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + bumpMicroVersionId(objectMD); } objectMD.originOp = 's3:ObjectRetention:Put'; /* eslint-enable no-param-reassign */ diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js index 85bb652787..cd5326bb98 100644 --- a/lib/api/objectPutTagging.js +++ b/lib/api/objectPutTagging.js @@ -11,6 +11,7 @@ const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUt const { pushMetric } = require('../utapi/utilities'); const monitoring = require('../utilities/monitoringHandler'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { data } = require('../data/wrapper'); @@ -96,6 +97,7 @@ function objectPutTagging(authInfo, request, log, callback) { if (replicationInfo) { // eslint-disable-next-line no-param-reassign objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + bumpMicroVersionId(objectMD); } // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectTagging:Put'; diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index c67487897c..211bf47828 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -1,6 +1,7 @@ const { errors } = require('arsenal'); const getReplicationInfo = require('../api/apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('../api/apiUtils/object/bumpMicroVersionId'); const aclUtils = require('../utilities/aclUtils'); const constants = require('../../constants'); const metadata = require('../metadata/wrapper'); @@ -73,6 +74,7 @@ const acl = { ...replicationInfo, backends, }; + bumpMicroVersionId(objectMD); } return metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, cb); diff --git a/lib/services.js b/lib/services.js index 5bc5036442..3d515e7f8d 100644 --- a/lib/services.js +++ b/lib/services.js @@ -220,9 +220,8 @@ const services = { md.setUploadId(uploadId); options.replayId = uploadId; } - // update microVersionId when overwriting metadata. if (updateMicroVersionId) { - md.updateMicroVersionId(); + md.updateMicroVersionId(config.instanceId, config.replicationGroupId); } // update restore if (archive) { diff --git a/tests/unit/api/apiUtils/object/bumpMicroVersionId.js b/tests/unit/api/apiUtils/object/bumpMicroVersionId.js new file mode 100644 index 0000000000..2dc50c0515 --- /dev/null +++ b/tests/unit/api/apiUtils/object/bumpMicroVersionId.js @@ -0,0 +1,31 @@ +const assert = require('assert'); + +const bumpMicroVersionId = require('../../../../../lib/api/apiUtils/object/bumpMicroVersionId'); + +describe('bumpMicroVersionId', () => { + it('should set a fresh microVersionId when replicationInfo is present', () => { + const objectMD = { replicationInfo: {} }; + bumpMicroVersionId(objectMD); + assert(objectMD.microVersionId, 'expected microVersionId to be set'); + }); + + it('should produce a different value on each call', () => { + const objectMD = { replicationInfo: {} }; + bumpMicroVersionId(objectMD); + const first = objectMD.microVersionId; + bumpMicroVersionId(objectMD); + assert.notStrictEqual(objectMD.microVersionId, first); + }); + + it('should do nothing when replicationInfo is absent', () => { + const objectMD = {}; + bumpMicroVersionId(objectMD); + assert.strictEqual(objectMD.microVersionId, undefined); + }); + + it('should bump unconditionally when force is true', () => { + const objectMD = {}; + bumpMicroVersionId(objectMD, true); + assert(objectMD.microVersionId, 'expected microVersionId to be set when force=true'); + }); +}); diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index 75333ea873..c5a25acd42 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -16,6 +16,8 @@ const completeMultipartUpload = require('../../../lib/api/completeMultipartUploa const objectPutACL = require('../../../lib/api/objectPutACL'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging'); +const objectPutLegalHold = require('../../../lib/api/objectPutLegalHold'); +const objectPutRetention = require('../../../lib/api/objectPutRetention'); const { config } = require('../../../lib/Config'); const log = new DummyRequestLogger(); @@ -69,6 +71,27 @@ function getObjectPutReq(key, hasContent) { const taggingPutReq = new TaggingConfigTester().createObjectTaggingRequest('PUT', bucketName, keyA); const taggingDeleteReq = new TaggingConfigTester().createObjectTaggingRequest('DELETE', bucketName, keyA); +const legalHoldReq = { + bucketName, + objectKey: keyA, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: '' + 'ON', + actionImplicitDenies: false, +}; + +const retentionFutureDate = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString(); +const retentionReq = { + bucketName, + objectKey: keyA, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: + '' + + 'GOVERNANCE' + + `${retentionFutureDate}` + + '', + actionImplicitDenies: false, +}; + const emptyReplicationMD = { status: '', backends: [], @@ -885,3 +908,45 @@ describe('Replication object MD with CRR and cloud destinations on the same obje assert.strictEqual(cloudBackend, undefined, 'new cloud backend should not be added'); }); }); + +const metadataOnlyWrites = [ + { name: 'objectPutTagging', fn: objectPutTagging, req: taggingPutReq }, + { name: 'objectDeleteTagging', fn: objectDeleteTagging, req: taggingDeleteReq }, + { name: 'objectPutACL', fn: objectPutACL, req: objectACLReq, skipSubsequentBump: true }, + { name: 'objectPutLegalHold', fn: objectPutLegalHold, req: legalHoldReq, requiresObjectLock: true }, + { name: 'objectPutRetention', fn: objectPutRetention, req: retentionReq, requiresObjectLock: true }, +]; + +describe('microVersionId is bumped on metadata-only writes', () => { + const getMD = key => metadata.keyMaps.get(bucketName).get(key); + const objectPutAsync = promisify(objectPut); + + beforeEach(() => { + cleanup(); + createBucketWithReplication(true); + metadata.buckets.get(bucketName).setObjectLockEnabled(true); + }); + + afterEach(() => cleanup()); + + metadataOnlyWrites.forEach(({ name, fn, req, skipSubsequentBump }) => { + const asyncFn = promisify(fn); + + it(`should set microVersionId on ${name}`, async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + await asyncFn(authInfo, req, log); + assert(getMD(keyA).microVersionId, `expected microVersionId to be set after ${name}`); + }); + + if (!skipSubsequentBump) { + it(`should bump microVersionId on subsequent ${name}`, async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + await asyncFn(authInfo, req, log); + const before = getMD(keyA).microVersionId; + await asyncFn(authInfo, req, log); + const after = getMD(keyA).microVersionId; + assert(after && after !== before, `expected microVersionId to change after ${name}`); + }); + } + }); +}); diff --git a/tests/unit/lib/services.spec.js b/tests/unit/lib/services.spec.js index 2ee61aabb6..ba8b4499b2 100644 --- a/tests/unit/lib/services.spec.js +++ b/tests/unit/lib/services.spec.js @@ -324,4 +324,39 @@ describe('services', () => { }); }); }); + + describe('metadataStoreObject', () => { + const authInfo = makeAuthInfo('accessKey1'); + const params = { + objectKey, + authInfo, + size: 0, + contentMD5: 'd41d8cd98f00b204e9800998ecf8427e', + metaHeaders: {}, + headers: {}, + log, + }; + + beforeEach(() => { + sinon.stub(metadata, 'putObjectMD').yields(null); + }); + + it('should set microVersionId on the stored MD when updateMicroVersionId is true', done => { + services.metadataStoreObject(bucketName, null, null, { ...params, updateMicroVersionId: true }, err => { + assert.ifError(err); + const storedMD = metadata.putObjectMD.firstCall.args[2]; + assert(storedMD.getMicroVersionId(), 'expected microVersionId to be set on stored MD'); + done(); + }); + }); + + it('should not set microVersionId when updateMicroVersionId is not set', done => { + services.metadataStoreObject(bucketName, null, null, params, err => { + assert.ifError(err); + const storedMD = metadata.putObjectMD.firstCall.args[2]; + assert.strictEqual(storedMD.getMicroVersionId(), undefined); + done(); + }); + }); + }); }); From 4b8ecad120ea2cd66acf6c8f7e18865b85147141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 29 May 2026 01:37:03 +0200 Subject: [PATCH 5/6] Clear isReplica on direct user writes overwriting a replica When a user updates tags, ACL, retention, or legal-hold on an object that arrived via replication (isReplica=true), the resulting object is no longer a replica - clear the flag so the x-amz-replication-status response header reflects the new state. Issue: CLDSRV-906 --- lib/api/apiUtils/object/getReplicationInfo.js | 2 ++ tests/unit/api/apiUtils/getReplicationInfo.js | 6 ++++ tests/unit/api/objectReplicationMD.js | 32 +++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js index 4349483abe..2320c56e69 100644 --- a/lib/api/apiUtils/object/getReplicationInfo.js +++ b/lib/api/apiUtils/object/getReplicationInfo.js @@ -100,6 +100,8 @@ function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operation content, role: ReplicationConfiguration.resolveSourceRole(config.role), isNFS: bucketMD.isNFS(), + // `undefined` so JSON/BSON drop the field on persist (saves bytes). + isReplica: undefined, }; } diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js index 07c4d680dc..2cb4f8ae67 100644 --- a/tests/unit/api/apiUtils/getReplicationInfo.js +++ b/tests/unit/api/apiUtils/getReplicationInfo.js @@ -95,6 +95,7 @@ describe('getReplicationInfo helper', () => { content: ['METADATA'], role: 'arn:aws:iam::root:role/src-role', isNFS: undefined, + isReplica: undefined, }); }); @@ -166,6 +167,7 @@ describe('getReplicationInfo helper', () => { content: ['METADATA'], role: 'arn:aws:iam::root:role/src-role', isNFS: undefined, + isReplica: undefined, }); }); @@ -200,6 +202,7 @@ describe('getReplicationInfo helper', () => { content: ['METADATA'], role: 'arn:aws:iam::root:role/src-role', isNFS: undefined, + isReplica: undefined, }); }); @@ -302,6 +305,7 @@ describe('getReplicationInfo helper', () => { content: ['METADATA'], role: 'arn:aws:iam::root:role/src-role', isNFS: undefined, + isReplica: undefined, }); }); @@ -331,6 +335,7 @@ describe('getReplicationInfo helper', () => { content: ['METADATA'], role: 'arn:aws:iam::root:role/src-role', isNFS: undefined, + isReplica: undefined, }); }); @@ -363,6 +368,7 @@ describe('getReplicationInfo helper', () => { content: ['METADATA'], role: 'arn:aws:iam::root:role/src-role', isNFS: undefined, + isReplica: undefined, }); }); diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index c5a25acd42..8b8e809c9e 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -604,6 +604,8 @@ describe('Replication object MD without bucket replication config', () => { site, status: 'PENDING', dataStoreVersionId: '', + destination: 'arn:aws:s3:::destination-bucket', + role: 'arn:aws:iam::account-id:role/resource', })); describe('Object metadata replicationInfo for cloud backends', () => { const expectedReplicationInfo = { @@ -950,3 +952,33 @@ describe('microVersionId is bumped on metadata-only writes', () => { } }); }); + +describe('isReplica is cleared on direct user writes overwriting a replica', () => { + const getMD = key => metadata.keyMaps.get(bucketName).get(key); + const objectPutAsync = promisify(objectPut); + + beforeEach(() => { + cleanup(); + createBucketWithReplication(true); + metadata.buckets.get(bucketName).setObjectLockEnabled(true); + }); + + afterEach(() => cleanup()); + + metadataOnlyWrites.forEach(({ name, fn, req }) => { + const asyncFn = promisify(fn); + + it(`should clear isReplica on ${name} when prior MD has it true`, async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + getMD(keyA).replicationInfo.isReplica = true; + await asyncFn(authInfo, req, log); + assert.strictEqual(getMD(keyA).replicationInfo.isReplica, undefined); + }); + + it(`should leave isReplica undefined on ${name} when prior MD does not have it`, async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + await asyncFn(authInfo, req, log); + assert.strictEqual(getMD(keyA).replicationInfo.isReplica, undefined); + }); + }); +}); From 5579a2006bf8b5ad94d4846231f3aa3869df602c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 19 Jun 2026 16:33:01 +0200 Subject: [PATCH 6/6] Use isReplica field to detect destination-side writes Update the replication metric guard and the NFS replica check in the backbeat route to read replicationInfo.isReplica directly instead of inferring it from replicationInfo.status === 'REPLICA', aligning with the new authoritative marker introduced in this branch. Issue: CLDSRV-906 --- lib/routes/routeBackbeat.js | 2 +- lib/routes/utilities/pushReplicationMetric.js | 2 +- tests/unit/utils/pushReplicationMetric.js | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index e462b28ad0..cde9c63a50 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -672,7 +672,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // then we want to create a version for the replica object even though // none was provided in the object metadata value. if (omVal.replicationInfo.isNFS) { - const isReplica = omVal.replicationInfo.status === 'REPLICA'; + const { isReplica } = omVal.replicationInfo; versioning = isReplica; omVal.replicationInfo.isNFS = !isReplica; } diff --git a/lib/routes/utilities/pushReplicationMetric.js b/lib/routes/utilities/pushReplicationMetric.js index c81027a494..8bec307855 100644 --- a/lib/routes/utilities/pushReplicationMetric.js +++ b/lib/routes/utilities/pushReplicationMetric.js @@ -4,7 +4,7 @@ const { pushMetric } = require('../../utapi/utilities'); function getMetricToPush(prevObjectMD, newObjectMD) { // We only want to update metrics for a destination bucket. - if (newObjectMD.getReplicationStatus() !== 'REPLICA') { + if (!newObjectMD.getReplicationIsReplica()) { return null; } diff --git a/tests/unit/utils/pushReplicationMetric.js b/tests/unit/utils/pushReplicationMetric.js index 02b9c944e6..c944dce504 100644 --- a/tests/unit/utils/pushReplicationMetric.js +++ b/tests/unit/utils/pushReplicationMetric.js @@ -10,7 +10,7 @@ describe('getMetricToPush', () => { .setVersionId('1'); const objectMD = new ObjectMD() .setVersionId('2') - .setReplicationStatus('REPLICA'); + .setReplicationIsReplica(true); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, 'replicateObject'); }); @@ -28,7 +28,7 @@ describe('getMetricToPush', () => { .setVersionId('1'); const objectMD = new ObjectMD() .setVersionId('1') - .setReplicationStatus('REPLICA') + .setReplicationIsReplica(true) .setTags({ 'object-tag-key': 'object-tag-value' }); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, 'replicateTags'); @@ -38,7 +38,7 @@ describe('getMetricToPush', () => { () => { const prevObjectMD = new ObjectMD() .setTags({ 'object-tag-key': 'object-tag-value' }); - const objectMD = new ObjectMD().setReplicationStatus('REPLICA'); + const objectMD = new ObjectMD().setReplicationIsReplica(true); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, 'replicateTags'); }); @@ -51,7 +51,7 @@ describe('getMetricToPush', () => { .setTags({ 'object-tag-key': 'object-tag-value' }); const objectMD = new ObjectMD() .setVersionId('1') - .setReplicationStatus('REPLICA') + .setReplicationIsReplica(true) .setTags({ 'object-tag-key': 'object-tag-value' }); const result = getMetricToPush(prevObjectMD, objectMD); assert.strictEqual(result, null); @@ -65,7 +65,7 @@ describe('getMetricToPush', () => { const publicACL = objectMD.getAcl(); publicACL.Canned = 'public-read'; objectMD - .setReplicationStatus('REPLICA') + .setReplicationIsReplica(true) .setAcl(publicACL) .setVersionId('1'); const result = getMetricToPush(prevObjectMD, objectMD); @@ -79,14 +79,14 @@ describe('getMetricToPush', () => { const publicACL = prevObjectMD.getAcl(); publicACL.Canned = 'public-read'; prevObjectMD - .setReplicationStatus('REPLICA') + .setReplicationIsReplica(true) .setAcl(publicACL) .setVersionId('1'); const objectMD = new ObjectMD(); const privateACL = objectMD.getAcl(); privateACL.Canned = 'private'; objectMD - .setReplicationStatus('REPLICA') + .setReplicationIsReplica(true) .setAcl(privateACL) .setVersionId('1'); const result = getMetricToPush(prevObjectMD, objectMD); @@ -100,7 +100,7 @@ describe('getMetricToPush', () => { const publicACL = objectMD.getAcl(); publicACL.Canned = 'public-read'; objectMD - .setReplicationStatus('REPLICA') + .setReplicationIsReplica(true) .setAcl(publicACL) .setVersionId('1'); const prevObjectMD = new ObjectMD()