CLDSRV-895: ListParts return part checksum

lib/api/listParts.js

@@ -14,6 +14,7 @@ const { pushMetric } = require('../utapi/utilities');
 const monitoring = require('../utilities/monitoringHandler');
 const { data } = require('../data/wrapper');
 const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
+const { algorithms } = require('./apiUtils/integrity/validateChecksums');
 
 /*
   Format of xml response:
@@ -64,6 +65,40 @@ function buildXML(xmlParams, xml, encodingFn) {
     });
 }
 
+function getPartNumber(item, splitter, splitterLen) {
+    // Metadata listings identify parts by key; AWS external listings are
+    // already normalized by Arsenal with a partNumber field.
+    if (item.key) {
+        // key form: {uploadId}{splitter}{partNumber}
+        const key = item.key;
+        const index = key.lastIndexOf(splitter);
+        if (index !== -1) {
+            return parseInt(key.substring(index + splitterLen), 10);
+        }
+    }
+    return item.partNumber;
+}
+
+function getPartChecksum(item) {
+    return {
+        checksumAlgorithm: item.value.ChecksumAlgorithm,
+        checksumValue: item.value.ChecksumValue,
+    };
+}
+
+function getPartChecksumXML(checksumAlgorithm, checksumValue) {
+    if (!checksumAlgorithm || !checksumValue) {
+        return undefined;
+    }
+    const algorithm = checksumAlgorithm.toLowerCase();
+    const xmlTag = algorithms[algorithm] &&
+        algorithms[algorithm].getObjectAttributesXMLTag;
+    if (!xmlTag) {
+        return undefined;
+    }
+    return { tag: xmlTag, value: checksumValue };
+}
+
 /**
  * listParts - List parts of an open multipart upload
  * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
@@ -203,25 +238,15 @@ function listParts(authInfo, request, log, callback) {
             const isTruncated = storedParts.IsTruncated;
             const splitterLen = splitter.length;
             const partListing = storedParts.Contents.map(item => {
-                // key form:
-                // - {uploadId}
-                // - {splitter}
-                // - {partNumber}
-                let partNumber;
-                if (item.key) {
-                    const index = item.key.lastIndexOf(splitter);
-                    partNumber =
-                        parseInt(item.key.substring(index + splitterLen), 10);
-                } else {
-                    // if partListing came from real AWS backend,
-                    // item.partNumber is present instead of item.key
-                    partNumber = item.partNumber;
-                }
+                const value = item.value;
+                const partChecksum = getPartChecksum(item);
                 return {
-                    partNumber,
-                    lastModified: item.value.LastModified,
-                    ETag: item.value.ETag,
-                    size: item.value.Size,
+                    partNumber: getPartNumber(item, splitter, splitterLen),
+                    lastModified: value.LastModified,
+                    ETag: value.ETag,
+                    size: value.Size,
+                    checksumAlgorithm: partChecksum.checksumAlgorithm,
+                    checksumValue: partChecksum.checksumValue,
                 };
             });
             const lastPartShown = partListing.length > 0 ?
@@ -246,6 +271,16 @@ function listParts(authInfo, request, log, callback) {
                 { tag: 'Key', value: objectKey },
                 { tag: 'UploadId', value: uploadId },
             ], xml, encodingFn);
+            const showChecksum = !mpuOverviewObj.checksumIsDefault &&
+                mpuOverviewObj.checksumAlgorithm &&
+                mpuOverviewObj.checksumType;
+            if (showChecksum) {
+                buildXML([
+                    { tag: 'ChecksumAlgorithm',
+                        value: mpuOverviewObj.checksumAlgorithm.toUpperCase() },
+                    { tag: 'ChecksumType', value: mpuOverviewObj.checksumType },
+                ], xml, encodingFn);
+            }
             xml.push('<Initiator>');
             buildXML([
                 { tag: 'ID', value: mpuOverviewObj.initiatorID },
@@ -271,13 +306,20 @@ function listParts(authInfo, request, log, callback) {
             ], xml, encodingFn);
 
             partListing.forEach(part => {
+                const partChecksumXML = showChecksum ?
+                    getPartChecksumXML(
+                        part.checksumAlgorithm, part.checksumValue) :
+                    undefined;
                 xml.push('<Part>');
                 buildXML([
                     { tag: 'PartNumber', value: part.partNumber },
                     { tag: 'LastModified', value: part.lastModified },
                     { tag: 'ETag', value: `"${part.ETag}"` },
                     { tag: 'Size', value: part.size },
                 ], xml, encodingFn);
+                if (partChecksumXML) {
+                    buildXML([partChecksumXML], xml, encodingFn);
+                }
                 xml.push('</Part>');
             });
             xml.push('</ListPartsResult>');

lib/services.js

@@ -780,6 +780,9 @@ const services = {
                         initiated: storedMetadata.initiated,
                         controllingLocationConstraint:
                             storedMetadata.controllingLocationConstraint,
+                        checksumAlgorithm: storedMetadata.checksumAlgorithm,
+                        checksumType: storedMetadata.checksumType,
+                        checksumIsDefault: storedMetadata.checksumIsDefault,
                     };
 
                     const tagging = storedMetadata['x-amz-tagging'];

package.json

@@ -33,7 +33,7 @@
     "@azure/storage-blob": "^12.28.0",
     "@hapi/joi": "^17.1.1",
     "@smithy/node-http-handler": "^3.0.0",
-    "arsenal": "git+https://github.com/scality/Arsenal#8.3.10",
+    "arsenal": "git+https://github.com/scality/Arsenal#8.3.11",
     "async": "2.6.4",
     "bucketclient": "scality/bucketclient#8.2.7",
     "bufferutil": "^4.0.8",

tests/functional/aws-node-sdk/test/object/listPartsChecksum.js

@@ -0,0 +1,174 @@
+const assert = require('assert');
+const {
+    AbortMultipartUploadCommand,
+    CreateBucketCommand,
+    CreateMultipartUploadCommand,
+    DeleteBucketCommand,
+    ListPartsCommand,
+    UploadPartCommand,
+} = require('@aws-sdk/client-s3');
+
+const withV4 = require('../support/withV4');
+const BucketUtility = require('../../lib/utility/bucket-util');
+const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
+
+const bucket = `list-parts-checksum-test-${Date.now()}`;
+const defaultKey = 'default-no-checksum';
+const checksumBodies = [
+    Buffer.from('first checksummed part', 'utf8'),
+    Buffer.from('second checksummed part', 'utf8'),
+];
+const defaultBodies = [
+    Buffer.from('first default checksum part', 'utf8'),
+    Buffer.from('second default checksum part', 'utf8'),
+];
+const checksumFieldByAlgorithm = {
+    CRC32: 'ChecksumCRC32',
+    CRC32C: 'ChecksumCRC32C',
+    CRC64NVME: 'ChecksumCRC64NVME',
+    SHA1: 'ChecksumSHA1',
+    SHA256: 'ChecksumSHA256',
+};
+const checksumTypesByAlgorithm = {
+    CRC32: ['COMPOSITE', 'FULL_OBJECT'],
+    CRC32C: ['COMPOSITE', 'FULL_OBJECT'],
+    CRC64NVME: ['FULL_OBJECT'],
+    SHA1: ['COMPOSITE'],
+    SHA256: ['COMPOSITE'],
+};
+
+function assertNoListPartsChecksum(partList) {
+    assert.strictEqual(partList.ChecksumAlgorithm, undefined);
+    assert.strictEqual(partList.ChecksumType, undefined);
+    partList.Parts.forEach(part => {
+        assert.strictEqual(part.ChecksumCRC32, undefined);
+        assert.strictEqual(part.ChecksumCRC32C, undefined);
+        assert.strictEqual(part.ChecksumCRC64NVME, undefined);
+        assert.strictEqual(part.ChecksumSHA1, undefined);
+        assert.strictEqual(part.ChecksumSHA256, undefined);
+    });
+}
+
+describe('ListParts checksum fields', () =>
+    withV4(sigCfg => {
+        let bucketUtil;
+        let s3;
+        const openMPUs = [];
+
+        async function abortUpload(key, uploadId) {
+            await s3.send(new AbortMultipartUploadCommand({
+                Bucket: bucket,
+                Key: key,
+                UploadId: uploadId,
+            }));
+            const index = openMPUs.findIndex(upload =>
+                upload.key === key && upload.uploadId === uploadId);
+            if (index !== -1) {
+                openMPUs.splice(index, 1);
+            }
+        }
+
+        before(async () => {
+            bucketUtil = new BucketUtility('default', {
+                ...sigCfg,
+                requestChecksumCalculation: 'WHEN_REQUIRED',
+            });
+            s3 = bucketUtil.s3;
+            await s3.send(new CreateBucketCommand({ Bucket: bucket }));
+        });
+
+        after(async () => {
+            await Promise.all(openMPUs.map(upload =>
+                s3.send(new AbortMultipartUploadCommand({
+                    Bucket: bucket,
+                    Key: upload.key,
+                    UploadId: upload.uploadId,
+                })).catch(() => undefined)));
+            await bucketUtil.empty(bucket);
+            await s3.send(new DeleteBucketCommand({ Bucket: bucket }));
+        });
+
+        for (const [checksumAlgorithm, checksumTypes] of
+            Object.entries(checksumTypesByAlgorithm)) {
+            for (const checksumType of checksumTypes) {
+                it(`should include ${checksumAlgorithm}/${checksumType} root ` +
+                'and part checksum fields', async () => {
+                    const key = `explicit-${checksumAlgorithm}-${checksumType}`;
+                    const checksumField =
+                        checksumFieldByAlgorithm[checksumAlgorithm];
+                    const internalAlgorithm = checksumAlgorithm.toLowerCase();
+                    const { UploadId } =
+                        await s3.send(new CreateMultipartUploadCommand({
+                            Bucket: bucket,
+                            Key: key,
+                            ChecksumAlgorithm: checksumAlgorithm,
+                            ChecksumType: checksumType,
+                        }));
+                    openMPUs.push({ key, uploadId: UploadId });
+
+                    const partChecksums = await Promise.all(
+                        checksumBodies.map(body =>
+                            algorithms[internalAlgorithm].digest(body)));
+
+                    await Promise.all(checksumBodies.map((body, index) =>
+                        s3.send(new UploadPartCommand({
+                            Bucket: bucket,
+                            Key: key,
+                            UploadId,
+                            PartNumber: index + 1,
+                            Body: body,
+                            [checksumField]: partChecksums[index],
+                        }))));
+
+                    const partList = await s3.send(new ListPartsCommand({
+                        Bucket: bucket,
+                        Key: key,
+                        UploadId,
+                    }));
+
+                    assert.strictEqual(partList.ChecksumAlgorithm,
+                        checksumAlgorithm);
+                    assert.strictEqual(partList.ChecksumType, checksumType);
+                    assert.strictEqual(partList.Parts.length,
+                        checksumBodies.length);
+                    partList.Parts.forEach((part, index) => {
+                        assert.strictEqual(part.PartNumber, index + 1);
+                        assert.strictEqual(part[checksumField],
+                            partChecksums[index]);
+                    });
+
+                    await abortUpload(key, UploadId);
+                });
+            }
+        }
+
+        it('should omit default checksum fields when no checksum headers are sent',
+        async () => {
+            const { UploadId } = await s3.send(new CreateMultipartUploadCommand({
+                Bucket: bucket,
+                Key: defaultKey,
+            }));
+            openMPUs.push({ key: defaultKey, uploadId: UploadId });
+
+            await Promise.all(defaultBodies.map((body, index) =>
+                s3.send(new UploadPartCommand({
+                    Bucket: bucket,
+                    Key: defaultKey,
+                    UploadId,
+                    PartNumber: index + 1,
+                    Body: body,
+                }))));
+
+            const partList = await s3.send(new ListPartsCommand({
+                Bucket: bucket,
+                Key: defaultKey,
+                UploadId,
+            }));
+
+            assert.strictEqual(partList.Parts.length, defaultBodies.length);
+            assertNoListPartsChecksum(partList);
+
+            await abortUpload(defaultKey, UploadId);
+        });
+    })
+);

tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js

@@ -5,6 +5,7 @@ const {
     AbortMultipartUploadCommand,
     UploadPartCommand,
     DeleteBucketCommand,
+    ListPartsCommand,
 } = require('@aws-sdk/client-s3');
 
 const withV4 = require('../support/withV4');
@@ -42,6 +43,18 @@ before(async () => {
     }
 });
 
+async function assertPartChecksumStored(s3, uploadId, partNumber,
+    checksumHeader, expectedChecksum) {
+    const listRes = await s3.send(new ListPartsCommand({
+        Bucket: bucket,
+        Key: key,
+        UploadId: uploadId,
+    }));
+    const found = listRes.Parts.find(part => part.PartNumber === partNumber);
+    assert(found, `Expected part ${partNumber} in ListParts response`);
+    assert.strictEqual(found[checksumHeader], expectedChecksum);
+}
+
 describe('UploadPart checksum validation', () =>
     withV4(sigCfg => {
         let bucketUtil;
@@ -82,12 +95,15 @@ describe('UploadPart checksum validation', () =>
                 });
 
                 it(`should accept ${mpuAlgo} with correct digest`, async () => {
+                    const partNumber = 1;
                     const res = await s3.send(new UploadPartCommand({
                         Bucket: bucket, Key: key, UploadId: uploadId,
-                        PartNumber: 1, Body: partBody,
+                        PartNumber: partNumber, Body: partBody,
                         [checksumField[mpuAlgo]]: correctDigest[mpuAlgo],
                     }));
                     assert.strictEqual(res[checksumField[mpuAlgo]], correctDigest[mpuAlgo]);
+                    await assertPartChecksumStored(s3, uploadId, partNumber,
+                        checksumField[mpuAlgo], correctDigest[mpuAlgo]);
                 });
 
                 it(`should reject ${mpuAlgo} with wrong digest (BadDigest)`, async () => {