From f21b0bab1cdabb3155415e63c5e4f4cc3a6b545d Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Wed, 8 Oct 2025 18:30:06 +0530 Subject: [PATCH 01/31] adding the StructuredMessageDecoder # Conflicts: # sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java --- .../StructuredMessageDecoder.java | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java new file mode 100644 index 000000000000..6117a7765541 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -0,0 +1,267 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.structuredmessage; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.StorageCrc64Calculator; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; + +import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + +/** + * Decoder for structured messages with support for segmenting and CRC64 checksums. + */ +public class StructuredMessageDecoder { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecoder.class); + private long messageLength; + private StructuredMessageFlags flags; + private int numSegments; + private final long expectedContentLength; + + private int messageOffset = 0; + private int currentSegmentNumber = 0; + private int currentSegmentContentLength = 0; + private int currentSegmentContentOffset = 0; + + private long messageCrc64 = 0; + private long segmentCrc64 = 0; + private final Map segmentCrcs = new HashMap<>(); + + /** + * Constructs a new StructuredMessageDecoder. + * + * @param expectedContentLength The expected length of the content to be decoded. + */ + public StructuredMessageDecoder(long expectedContentLength) { + this.expectedContentLength = expectedContentLength; + } + + /** + * Reads the message header from the given buffer. + * + * @param buffer The buffer containing the message header. + * @throws IllegalArgumentException if the buffer does not contain a valid message header. + */ + private void readMessageHeader(ByteBuffer buffer) { + if (buffer.remaining() < V1_HEADER_LENGTH) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Content not long enough to contain a valid " + "message header.")); + } + + int messageVersion = Byte.toUnsignedInt(buffer.get()); + if (messageVersion != DEFAULT_MESSAGE_VERSION) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Unsupported structured message version: " + messageVersion)); + } + + messageLength = (int) buffer.getLong(); + if (messageLength < V1_HEADER_LENGTH) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Content not long enough to contain a valid " + "message header.")); + } + if (messageLength != expectedContentLength) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Structured message length " + messageLength + + " did not match content length " + expectedContentLength)); + } + + flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(buffer.getShort())); + numSegments = Short.toUnsignedInt(buffer.getShort()); + + messageOffset += V1_HEADER_LENGTH; + } + + /** + * Reads the segment header from the given buffer. + * + * @param buffer The buffer containing the segment header. + * @throws IllegalArgumentException if the buffer does not contain a valid segment header. + */ + private void readSegmentHeader(ByteBuffer buffer) { + if (buffer.remaining() < V1_SEGMENT_HEADER_LENGTH) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Segment header is incomplete.")); + } + + int segmentNum = Short.toUnsignedInt(buffer.getShort()); + int segmentSize = (int) buffer.getLong(); + + if (segmentSize < 0 || segmentSize > buffer.remaining()) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException("Invalid segment size detected: " + segmentSize)); + } + + if (segmentNum != currentSegmentNumber + 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Unexpected segment number.")); + } + + currentSegmentNumber = segmentNum; + currentSegmentContentLength = segmentSize; + currentSegmentContentOffset = 0; + + if (segmentSize == 0) { + readSegmentFooter(buffer); + } + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + segmentCrc64 = 0; + } + + messageOffset += V1_SEGMENT_HEADER_LENGTH; + } + + /** + * Reads the segment content from the given buffer and writes it to the output stream. + * + * @param buffer The buffer containing the segment content. + * @param output The output stream to write the segment content to. + * @param size The maximum number of bytes to read. + * @throws IllegalArgumentException if there is a segment size mismatch. + */ + private void readSegmentContent(ByteBuffer buffer, ByteArrayOutputStream output, int size) { + int toRead = Math.min(buffer.remaining(), currentSegmentContentLength - currentSegmentContentOffset); + toRead = Math.min(toRead, size); + + if (toRead == 0) { + return; + } + + byte[] content = new byte[toRead]; + buffer.get(content); + output.write(content, 0, toRead); + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + segmentCrc64 = StorageCrc64Calculator.compute(content, segmentCrc64); + messageCrc64 = StorageCrc64Calculator.compute(content, messageCrc64); + } + + messageOffset += toRead; + currentSegmentContentOffset += toRead; + + if (currentSegmentContentOffset > currentSegmentContentLength) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Segment size mismatch detected in segment " + currentSegmentNumber)); + } + + if (currentSegmentContentOffset == currentSegmentContentLength) { + readSegmentFooter(buffer); + } + } + + /** + * Reads the segment footer from the given buffer. + * + * @param buffer The buffer containing the segment footer. + * @throws IllegalArgumentException if the buffer does not contain a valid segment footer. + */ + private void readSegmentFooter(ByteBuffer buffer) { + if (currentSegmentContentOffset != currentSegmentContentLength) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Segment content length mismatch in segment " + currentSegmentNumber + + ". Expected: " + currentSegmentContentLength + ", Read: " + currentSegmentContentOffset)); + } + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + if (buffer.remaining() < CRC64_LENGTH) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Segment footer is incomplete.")); + } + + long reportedCrc64 = buffer.getLong(); + if (segmentCrc64 != reportedCrc64) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("CRC64 mismatch detected in segment " + currentSegmentNumber)); + } + segmentCrcs.put(currentSegmentNumber, segmentCrc64); + messageOffset += CRC64_LENGTH; + } + + if (currentSegmentNumber == numSegments) { + readMessageFooter(buffer); + } else { + readSegmentHeader(buffer); + } + } + + /** + * Reads the segment footer from the given buffer. + * + * @param buffer The buffer containing the segment footer. + * @throws IllegalArgumentException if the buffer does not contain a valid segment footer. + */ + private void readMessageFooter(ByteBuffer buffer) { + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + if (buffer.remaining() < CRC64_LENGTH) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Message footer is incomplete.")); + } + + long reportedCrc = buffer.getLong(); + if (messageCrc64 != reportedCrc) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("CRC64 mismatch detected in message " + "footer.")); + } + messageOffset += CRC64_LENGTH; + } + + if (messageOffset != messageLength) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Decoded message length does not match " + "expected length.")); + } + } + + /** + * Decodes the structured message from the given buffer up to the specified size. + * + * @param buffer The buffer containing the structured message. + * @param size The maximum number of bytes to decode. + * @return A ByteBuffer containing the decoded message content. + * @throws IllegalArgumentException if the buffer does not contain a valid structured message. + */ + public ByteBuffer decode(ByteBuffer buffer, int size) { + buffer.order(ByteOrder.LITTLE_ENDIAN); + ByteArrayOutputStream decodedContent = new ByteArrayOutputStream(); + + if (messageOffset == 0) { + readMessageHeader(buffer); + } + + while (buffer.hasRemaining() && decodedContent.size() < size) { + if (currentSegmentContentOffset == currentSegmentContentLength) { + readSegmentHeader(buffer); + } + + readSegmentContent(buffer, decodedContent, size - decodedContent.size()); + } + + return ByteBuffer.wrap(decodedContent.toByteArray()); + } + + /** + * Decodes the entire structured message from the given buffer. + * + * @param buffer The buffer containing the structured message. + * @return A ByteBuffer containing the decoded message content. + * @throws IllegalArgumentException if the buffer does not contain a valid structured message. + */ + public ByteBuffer decode(ByteBuffer buffer) { + return decode(buffer, buffer.remaining()); + } + + /** + * Finalizes the decoding process and validates that the entire message has been decoded. + * + * @throws IllegalArgumentException if the decoded message length does not match the expected length. + */ + public void finalizeDecoding() { + if (messageOffset != messageLength) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Decoded message length does not match " + + "expected length. Expected: " + messageLength + ", but was: " + messageOffset)); + } + } +} From e1c23c5e237082d86b4094b1429f899108491b20 Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Wed, 15 Oct 2025 18:49:13 +0530 Subject: [PATCH 02/31] adding the pipeline policy changes --- .../implementation/util/BuilderHelper.java | 3 + .../blob/specialized/BlobAsyncClientBase.java | 80 ++++++- .../blob/BlobMessageDecoderDownloadTests.java | 206 ++++++++++++++++++ .../DownloadContentValidationOptions.java | 66 ++++++ .../common/implementation/Constants.java | 11 + .../StructuredMessageDecodingStream.java | 103 +++++++++ ...StorageContentValidationDecoderPolicy.java | 164 ++++++++++++++ 7 files changed, 628 insertions(+), 5 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index 0866d310981c..79276b6582b9 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -39,6 +39,7 @@ import com.azure.storage.common.policy.ResponseValidationPolicyBuilder; import com.azure.storage.common.policy.ScrubEtagPolicy; import com.azure.storage.common.policy.StorageBearerTokenChallengeAuthorizationPolicy; +import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import com.azure.storage.common.policy.StorageSharedKeyCredentialPolicy; import java.net.MalformedURLException; @@ -137,6 +138,8 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare HttpPolicyProviders.addAfterRetryPolicies(policies); + policies.add(new StorageContentValidationDecoderPolicy()); + policies.add(getResponseValidationPolicy()); policies.add(new HttpLoggingPolicy(logOptions)); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 7e819c849e80..93bb0f49b0db 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -81,8 +81,10 @@ import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.common.DownloadContentValidationOptions; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; +import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.StorageChecksumAlgorithm; @@ -1193,6 +1195,52 @@ public Mono downloadStreamWithResponse(BlobDownloadSt } } + /** + * Reads a range of bytes from a blob with content validation options. Uploading data must be done from the {@link BlockBlobClient}, {@link + * PageBlobClient}, or {@link AppendBlobClient}. + * + *

Code Samples

+ * + * 
{@code
+     * BlobRange range = new BlobRange(1024, 2048L);
+     * DownloadRetryOptions options = new DownloadRetryOptions().setMaxRetryRequests(5);
+     * DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions()
+     *     .setStructuredMessageValidationEnabled(true);
+     *
+     * client.downloadStreamWithResponse(range, options, null, false, validationOptions).subscribe(response -> {
+     *     ByteArrayOutputStream downloadData = new ByteArrayOutputStream();
+     *     response.getValue().subscribe(piece -> {
+     *         try {
+     *             downloadData.write(piece.array());
+     *         } catch (IOException ex) {
+     *             throw new UncheckedIOException(ex);
+     *         }
+     *     });
+     * });
+     * }
+ * + *

For more information, see the + * Azure Docs

+ * + * @param range {@link BlobRange} + * @param options {@link DownloadRetryOptions} + * @param requestConditions {@link BlobRequestConditions} + * @param getRangeContentMd5 Whether the contentMD5 for the specified blob range should be returned. + * @param contentValidationOptions {@link DownloadContentValidationOptions} options for content validation + * @return A reactive response containing the blob data. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, + BlobRequestConditions requestConditions, boolean getRangeContentMd5, + DownloadContentValidationOptions contentValidationOptions) { + try { + return withContext(context -> downloadStreamWithResponse(range, options, requestConditions, + getRangeContentMd5, contentValidationOptions, context)); + } catch (RuntimeException ex) { + return monoError(LOGGER, ex); + } + } + /** * Reads a range of bytes from a blob. Uploading data must be done from the {@link BlockBlobClient}, {@link * PageBlobClient}, or {@link AppendBlobClient}. @@ -1250,7 +1298,7 @@ public Mono downloadContentWithResponse(BlobDo } Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, - BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context) { + BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context){ // Prevents revapi visibility increased error return downloadStreamWithResponseInternal(range, options, requestConditions, getRangeContentMd5, null, context); } @@ -1259,17 +1307,38 @@ Mono downloadStreamWithResponseInternal(BlobRange ran BlobRequestConditions requestConditions, boolean getRangeContentMd5, StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; - Boolean getMD5 = getRangeContentMd5 ? getRangeContentMd5 : null; + + // Determine MD5 validation: properly consider both getRangeContentMd5 parameter and validation options + // MD5 validation is enabled if: + // 1. getRangeContentMd5 is explicitly true, OR + // 2. contentValidationOptions.isMd5ValidationEnabled() is true + final Boolean finalGetMD5; + if (getRangeContentMd5 + || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled())) { + finalGetMD5 = true; + } else { + finalGetMD5 = null; + } + BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; // The first range should eagerly convert headers as they'll be used to create response types. - Context firstRangeContext = context == null + Context initialContext = context == null ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, + // Add structured message decoding context if enabled + final Context firstRangeContext; + if (contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled()) { + firstRangeContext = initialContext.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) + .addData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY, contentValidationOptions); + } else { + firstRangeContext = initialContext; + } + + return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), finalGetMD5, firstRangeContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); String eTag = blobsDownloadHeaders.getETag(); @@ -1313,12 +1382,13 @@ Mono downloadStreamWithResponseInternal(BlobRange ran try { return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, - eTag, getMD5, context); + eTag, finalGetMD5, firstRangeContext); } catch (Exception e) { return Mono.error(e); } }; + // Structured message decoding is now handled by StructuredMessageDecoderPolicy return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions); }); } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java new file mode 100644 index 000000000000..5508ddc30831 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.FluxUtil; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.common.DownloadContentValidationOptions; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.structuredmessage.StructuredMessageEncoder; +import com.azure.storage.common.implementation.structuredmessage.StructuredMessageFlags; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobMessageDecoderDownloadTests extends BlobTestBase { + + private BlobAsyncClient bc; + + @BeforeEach + public void setup() { + String blobName = generateBlobName(); + bc = ccAsync.getBlobAsyncClient(blobName); + bc.upload(Flux.just(ByteBuffer.wrap(new byte[0])), null).block(); + } + + @Test + public void downloadStreamWithResponseContentValidation() throws IOException { + byte[] randomData = getRandomByteArray(Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + StepVerifier + .create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .verifyComplete(); + } + + @Test + public void downloadStreamWithResponseContentValidationRange() throws IOException { + byte[] randomData = getRandomByteArray(Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + BlobRange range = new BlobRange(0, 512L); + + StepVerifier.create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(range, null, null, false, validationOptions)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + assertNotNull(r); + assertTrue(r.length > 0); + }).verifyComplete(); + } + + @Test + public void downloadStreamWithResponseContentValidationLargeBlob() throws IOException { + // Test with larger data to verify chunking works correctly + byte[] randomData = getRandomByteArray(5 * Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 1024, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + StepVerifier + .create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .verifyComplete(); + } + + @Test + public void downloadStreamWithResponseContentValidationMultipleSegments() throws IOException { + // Test with multiple segments to ensure all segments are decoded correctly + byte[] randomData = getRandomByteArray(2 * Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + StepVerifier + .create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .verifyComplete(); + } + + @Test + public void downloadStreamWithResponseNoValidation() throws IOException { + // Test that download works normally when validation is not enabled + byte[] randomData = getRandomByteArray(Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + // No validation options - should download encoded data as-is + StepVerifier.create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(null, null, null, false)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + assertNotNull(r); + // Should get encoded data, not decoded + assertTrue(r.length > randomData.length); // Encoded data is larger + }).verifyComplete(); + } + + @Test + public void downloadStreamWithResponseValidationDisabled() throws IOException { + // Test with validation options but validation disabled + byte[] randomData = getRandomByteArray(Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(false); + + StepVerifier.create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + assertNotNull(r); + // Should get encoded data, not decoded + assertTrue(r.length > randomData.length); // Encoded data is larger + }).verifyComplete(); + } + + @Test + public void downloadStreamWithResponseContentValidationSmallSegment() throws IOException { + // Test with small segment size to ensure boundary conditions are handled + byte[] randomData = getRandomByteArray(256); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 128, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + StepVerifier + .create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .verifyComplete(); + } + + @Test + public void downloadStreamWithResponseContentValidationVeryLargeBlob() throws IOException { + // Test with very large data to verify chunking and policy work correctly with large blobs + byte[] randomData = getRandomByteArray(10 * Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 2048, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + StepVerifier + .create(bc.upload(input, null, true) + .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .verifyComplete(); + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java new file mode 100644 index 000000000000..2b663494bfe9 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common; + +import com.azure.core.annotation.Fluent; + +/** + * Options for content validation during download operations. + */ +@Fluent +public final class DownloadContentValidationOptions { + private boolean enableStructuredMessageValidation; + private boolean enableMd5Validation; + + /** + * Creates a new instance of DownloadContentValidationOptions. + */ + public DownloadContentValidationOptions() { + this.enableStructuredMessageValidation = false; + this.enableMd5Validation = false; + } + + /** + * Gets whether structured message validation is enabled. + * + * @return true if structured message validation is enabled, false otherwise. + */ + public boolean isStructuredMessageValidationEnabled() { + return enableStructuredMessageValidation; + } + + /** + * Sets whether structured message validation is enabled. + * When enabled, downloads will use CRC64 checksums embedded in structured messages for content validation. + * + * @param enableStructuredMessageValidation true to enable structured message validation, false to disable. + * @return The updated DownloadContentValidationOptions object. + */ + public DownloadContentValidationOptions + setStructuredMessageValidationEnabled(boolean enableStructuredMessageValidation) { + this.enableStructuredMessageValidation = enableStructuredMessageValidation; + return this; + } + + /** + * Gets whether MD5 validation is enabled. + * + * @return true if MD5 validation is enabled, false otherwise. + */ + public boolean isMd5ValidationEnabled() { + return enableMd5Validation; + } + + /** + * Sets whether MD5 validation is enabled. + * When enabled, downloads will use MD5 checksums for content validation. + * + * @param enableMd5Validation true to enable MD5 validation, false to disable. + * @return The updated DownloadContentValidationOptions object. + */ + public DownloadContentValidationOptions setMd5ValidationEnabled(boolean enableMd5Validation) { + this.enableMd5Validation = enableMd5Validation; + return this; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index e3b88b661134..96d4a0a59c32 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -94,6 +94,17 @@ public final class Constants { public static final String SKIP_ECHO_VALIDATION_KEY = "skipEchoValidation"; + /** + * Context key used to signal that structured message decoding should be applied. + */ + public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; + + /** + * Context key used to pass DownloadContentValidationOptions to the policy. + */ + public static final String STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY + = "azure-storage-structured-message-validation-options"; + private Constants() { } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java new file mode 100644 index 000000000000..5fec64e0c18a --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.structuredmessage; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.DownloadContentValidationOptions; +import reactor.core.publisher.Flux; + +import java.nio.ByteBuffer; + +/** + * A utility class for applying structured message decoding to download streams. + */ +public final class StructuredMessageDecodingStream { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecodingStream.class); + + private StructuredMessageDecodingStream() { + // utility class + } + + /** + * Wraps a download stream with structured message decoding if content validation is enabled. + * + * @param originalStream The original download stream. + * @param contentLength The expected content length. + * @param validationOptions The content validation options. + * @return A Flux that decodes structured messages if validation is enabled, otherwise returns the original stream. + */ + public static Flux wrapStreamIfNeeded(Flux originalStream, Long contentLength, + DownloadContentValidationOptions validationOptions) { + + if (validationOptions == null || !validationOptions.isStructuredMessageValidationEnabled()) { + return originalStream; + } + + if (contentLength == null || contentLength <= 0) { + LOGGER.warning("Cannot apply structured message validation without valid content length."); + return originalStream; + } + + return applyStructuredMessageDecoding(originalStream, contentLength); + } + + /** + * Applies structured message decoding to the stream. + * + * @param stream The stream to decode. + * @param expectedContentLength The expected content length. + * @return A Flux that decodes the structured message. + */ + private static Flux applyStructuredMessageDecoding(Flux stream, + long expectedContentLength) { + return stream + .collect(() -> new StructuredMessageDecodingCollector(expectedContentLength), + StructuredMessageDecodingCollector::addBuffer) + .flatMapMany(collector -> collector.getDecodedData()); + } + + /** + * Helper class to collect and decode structured message data. + */ + private static class StructuredMessageDecodingCollector { + private final StructuredMessageDecoder decoder; + private ByteBuffer accumulatedBuffer; + private boolean completed = false; + + StructuredMessageDecodingCollector(long expectedContentLength) { + this.decoder = new StructuredMessageDecoder(expectedContentLength); + this.accumulatedBuffer = ByteBuffer.allocate(0); + } + + void addBuffer(ByteBuffer buffer) { + if (completed) { + return; + } + + // Accumulate the buffer + ByteBuffer newBuffer = ByteBuffer.allocate(accumulatedBuffer.remaining() + buffer.remaining()); + newBuffer.put(accumulatedBuffer); + newBuffer.put(buffer); + newBuffer.flip(); + accumulatedBuffer = newBuffer; + } + + Flux getDecodedData() { + try { + if (accumulatedBuffer.remaining() == 0) { + return Flux.empty(); + } + + ByteBuffer decodedData = decoder.decode(accumulatedBuffer); + decoder.finalizeDecoding(); + completed = true; + + return Flux.just(decodedData); + } catch (Exception e) { + LOGGER.error("Failed to decode structured message: " + e.getMessage(), e); + return Flux.error(e); + } + } + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java new file mode 100644 index 000000000000..7652bb846e82 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.DownloadContentValidationOptions; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.structuredmessage.StructuredMessageDecodingStream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * This is a decoding policy in an {@link com.azure.core.http.HttpPipeline} to decode structured messages in + * storage download requests. The policy checks for a context value to determine when to apply structured message decoding. + */ +public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { + private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); + + /** + * Creates a new instance of {@link StorageContentValidationDecoderPolicy}. + */ + public StorageContentValidationDecoderPolicy() { + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Check if structured message decoding is enabled for this request + if (!shouldApplyDecoding(context)) { + return next.process(); + } + + return next.process().map(httpResponse -> { + // Only apply decoding to download responses (GET requests with body) + if (!isDownloadResponse(httpResponse)) { + return httpResponse; + } + + DownloadContentValidationOptions validationOptions = getValidationOptions(context); + Long contentLength = getContentLength(httpResponse.getHeaders()); + + if (contentLength != null && contentLength > 0 && validationOptions != null) { + Flux decodedStream = StructuredMessageDecodingStream + .wrapStreamIfNeeded(httpResponse.getBody(), contentLength, validationOptions); + return new DecodedResponse(httpResponse, decodedStream); + } + + return httpResponse; + }); + } + + /** + * Checks if structured message decoding should be applied based on context. + * + * @param context The pipeline call context. + * @return true if decoding should be applied, false otherwise. + */ + private boolean shouldApplyDecoding(HttpPipelineCallContext context) { + return context.getData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) + .map(value -> value instanceof Boolean && (Boolean) value) + .orElse(false); + } + + /** + * Gets the validation options from context. + * + * @param context The pipeline call context. + * @return The validation options or null if not present. + */ + private DownloadContentValidationOptions getValidationOptions(HttpPipelineCallContext context) { + return context.getData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY) + .filter(value -> value instanceof DownloadContentValidationOptions) + .map(value -> (DownloadContentValidationOptions) value) + .orElse(null); + } + + /** + * Gets the content length from response headers. + * + * @param headers The response headers. + * @return The content length or null if not present. + */ + private Long getContentLength(HttpHeaders headers) { + String contentLengthStr = headers.getValue(HttpHeaderName.CONTENT_LENGTH); + if (contentLengthStr != null) { + try { + return Long.parseLong(contentLengthStr); + } catch (NumberFormatException e) { + LOGGER.warning("Invalid content length in response headers: " + contentLengthStr); + } + } + return null; + } + + /** + * Checks if the response is a download response (GET request with body). + * + * @param httpResponse The HTTP response. + * @return true if it's a download response, false otherwise. + */ + private boolean isDownloadResponse(HttpResponse httpResponse) { + return httpResponse.getRequest().getHttpMethod() == HttpMethod.GET && httpResponse.getBody() != null; + } + + /** + * HTTP response wrapper that provides a decoded response body. + */ + static class DecodedResponse extends HttpResponse { + private final Flux decodedBody; + private final HttpResponse originalResponse; + + DecodedResponse(HttpResponse httpResponse, Flux decodedBody) { + super(httpResponse.getRequest()); + this.originalResponse = httpResponse; + this.decodedBody = decodedBody; + } + + @Override + public int getStatusCode() { + return originalResponse.getStatusCode(); + } + + @Override + public String getHeaderValue(String name) { + return originalResponse.getHeaderValue(name); + } + + @Override + public HttpHeaders getHeaders() { + return originalResponse.getHeaders(); + } + + @Override + public Flux getBody() { + return decodedBody; + } + + @Override + public Mono getBodyAsByteArray() { + return FluxUtil.collectBytesInByteBufferStream(decodedBody); + } + + @Override + public Mono getBodyAsString() { + return getBodyAsByteArray().map(String::new); + } + + @Override + public Mono getBodyAsString(Charset charset) { + return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); + } + } +} From 1d0cc9244afa2b3e6a5db51592a67241d1ecf962 Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Wed, 29 Oct 2025 19:47:57 +0530 Subject: [PATCH 03/31] smart retry changes --- .../blob/specialized/BlobAsyncClientBase.java | 20 +- .../blob/BlobMessageDecoderDownloadTests.java | 3 + .../common/implementation/Constants.java | 6 + ...StorageContentValidationDecoderPolicy.java | 209 ++++++++++++++++-- 4 files changed, 224 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 93bb0f49b0db..99b9a9db3fdc 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1381,8 +1381,26 @@ Mono downloadStreamWithResponseInternal(BlobRange ran } try { + // For retry context, preserve decoder state if structured message validation is enabled + Context retryContext = firstRangeContext; + + // If structured message decoding is enabled, we need to include the decoder state + // so the retry can continue from where we left off + if (contentValidationOptions != null + && contentValidationOptions.isStructuredMessageValidationEnabled()) { + // The decoder state will be set by the policy during processing + // We preserve it in the context for the retry request + Object decoderState + = firstRangeContext.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) + .orElse(null); + if (decoderState != null) { + retryContext = retryContext + .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); + } + } + return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, - eTag, finalGetMD5, firstRangeContext); + eTag, finalGetMD5, retryContext); } catch (Exception e) { return Mono.error(e); } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 5508ddc30831..441e4e591ea5 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -6,6 +6,8 @@ import com.azure.core.test.utils.TestUtils; import com.azure.core.util.FluxUtil; import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.DownloadRetryOptions; import com.azure.storage.common.DownloadContentValidationOptions; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.structuredmessage.StructuredMessageEncoder; @@ -18,6 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index 96d4a0a59c32..525e8191521e 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -105,6 +105,12 @@ public final class Constants { public static final String STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY = "azure-storage-structured-message-validation-options"; + /** + * Context key used to pass stateful decoder state across retry requests. + */ + public static final String STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY + = "azure-storage-structured-message-decoder-state"; + private Constants() { } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 7652bb846e82..6bb81027e681 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -14,16 +14,25 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.storage.common.DownloadContentValidationOptions; import com.azure.storage.common.implementation.Constants; -import com.azure.storage.common.implementation.structuredmessage.StructuredMessageDecodingStream; +import com.azure.storage.common.implementation.structuredmessage.StructuredMessageDecoder; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.nio.ByteBuffer; import java.nio.charset.Charset; +import java.util.concurrent.atomic.AtomicLong; /** * This is a decoding policy in an {@link com.azure.core.http.HttpPipeline} to decode structured messages in * storage download requests. The policy checks for a context value to determine when to apply structured message decoding. + * + *

The policy supports smart retries by maintaining decoder state across network interruptions, ensuring: + *

    + *
  • All received segment checksums are validated before retry
    • + *
  • Exact encoded and decoded byte positions are tracked
    • + *
  • Decoder state is preserved across retry requests
    • + *
  • Retries continue from the correct offset after network faults
    • + *
*/ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); @@ -51,15 +60,75 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN Long contentLength = getContentLength(httpResponse.getHeaders()); if (contentLength != null && contentLength > 0 && validationOptions != null) { - Flux decodedStream = StructuredMessageDecodingStream - .wrapStreamIfNeeded(httpResponse.getBody(), contentLength, validationOptions); - return new DecodedResponse(httpResponse, decodedStream); + // Get or create decoder with state tracking + DecoderState decoderState = getOrCreateDecoderState(context, contentLength); + + // Decode using the stateful decoder + Flux decodedStream = decodeStream(httpResponse.getBody(), decoderState); + + // Update context with decoder state for potential retries + context.setData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); + + return new DecodedResponse(httpResponse, decodedStream, decoderState); } return httpResponse; }); } + /** + * Decodes a stream of byte buffers using the decoder state. + * + * @param encodedFlux The flux of encoded byte buffers. + * @param state The decoder state. + * @return A flux of decoded byte buffers. + */ + private Flux decodeStream(Flux encodedFlux, DecoderState state) { + return encodedFlux.concatMap(encodedBuffer -> { + try { + // Combine with pending data if any + ByteBuffer dataToProcess = state.combineWithPending(encodedBuffer); + + // Track encoded bytes + int encodedBytesInBuffer = encodedBuffer.remaining(); + state.totalEncodedBytesProcessed.addAndGet(encodedBytesInBuffer); + + // Try to decode what we have - decoder handles partial data + int availableSize = dataToProcess.remaining(); + ByteBuffer decodedData = state.decoder.decode(dataToProcess.duplicate(), availableSize); + + // Track decoded bytes + int decodedBytes = decodedData.remaining(); + state.totalBytesDecoded.addAndGet(decodedBytes); + + // Store any remaining unprocessed data for next iteration + if (dataToProcess.hasRemaining()) { + state.updatePendingBuffer(dataToProcess); + } else { + state.pendingBuffer = null; + } + + // Return decoded data if any + if (decodedBytes > 0) { + return Flux.just(decodedData); + } else { + return Flux.empty(); + } + } catch (Exception e) { + LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); + return Flux.error(e); + } + }).doOnComplete(() -> { + // Finalize when stream completes + try { + state.decoder.finalizeDecoding(); + } catch (IllegalArgumentException e) { + // Expected if we haven't received all data yet (e.g., interrupted download) + LOGGER.verbose("Decoding not finalized - may resume on retry: " + e.getMessage()); + } + }); + } + /** * Checks if structured message decoding should be applied based on context. * @@ -104,26 +173,131 @@ private Long getContentLength(HttpHeaders headers) { } /** - * Checks if the response is a download response (GET request with body). + * Gets or creates a decoder state from context. + * + * @param context The pipeline call context. + * @param contentLength The content length. + * @return The decoder state. + */ + private DecoderState getOrCreateDecoderState(HttpPipelineCallContext context, long contentLength) { + return context.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) + .filter(value -> value instanceof DecoderState) + .map(value -> (DecoderState) value) + .orElseGet(() -> new DecoderState(contentLength)); + } + + /** + * Checks if the response is a download response. * * @param httpResponse The HTTP response. * @return true if it's a download response, false otherwise. */ private boolean isDownloadResponse(HttpResponse httpResponse) { - return httpResponse.getRequest().getHttpMethod() == HttpMethod.GET && httpResponse.getBody() != null; + HttpMethod method = httpResponse.getRequest().getHttpMethod(); + return method == HttpMethod.GET && httpResponse.getStatusCode() / 100 == 2; } /** - * HTTP response wrapper that provides a decoded response body. + * State holder for the structured message decoder that tracks decoding progress + * across network interruptions. */ - static class DecodedResponse extends HttpResponse { - private final Flux decodedBody; + public static class DecoderState { + private final StructuredMessageDecoder decoder; + private final long expectedContentLength; + private final AtomicLong totalBytesDecoded; + private final AtomicLong totalEncodedBytesProcessed; + private ByteBuffer pendingBuffer; + + /** + * Creates a new decoder state. + * + * @param expectedContentLength The expected length of the encoded content. + */ + public DecoderState(long expectedContentLength) { + this.expectedContentLength = expectedContentLength; + this.decoder = new StructuredMessageDecoder(expectedContentLength); + this.totalBytesDecoded = new AtomicLong(0); + this.totalEncodedBytesProcessed = new AtomicLong(0); + this.pendingBuffer = null; + } + + /** + * Combines pending buffer with new data. + * + * @param newBuffer The new buffer to combine. + * @return Combined buffer. + */ + private ByteBuffer combineWithPending(ByteBuffer newBuffer) { + if (pendingBuffer == null || !pendingBuffer.hasRemaining()) { + return newBuffer.duplicate(); + } + + ByteBuffer combined = ByteBuffer.allocate(pendingBuffer.remaining() + newBuffer.remaining()); + combined.put(pendingBuffer.duplicate()); + combined.put(newBuffer.duplicate()); + combined.flip(); + return combined; + } + + /** + * Updates the pending buffer with remaining data. + * + * @param dataToProcess The buffer with remaining data. + */ + private void updatePendingBuffer(ByteBuffer dataToProcess) { + pendingBuffer = ByteBuffer.allocate(dataToProcess.remaining()); + pendingBuffer.put(dataToProcess); + pendingBuffer.flip(); + } + + /** + * Gets the total number of decoded bytes processed so far. + * + * @return The total decoded bytes. + */ + public long getTotalBytesDecoded() { + return totalBytesDecoded.get(); + } + + /** + * Gets the total number of encoded bytes processed so far. + * + * @return The total encoded bytes processed. + */ + public long getTotalEncodedBytesProcessed() { + return totalEncodedBytesProcessed.get(); + } + + /** + * Checks if the decoder has finalized. + * + * @return true if finalized, false otherwise. + */ + public boolean isFinalized() { + return totalEncodedBytesProcessed.get() >= expectedContentLength; + } + } + + /** + * Decoded HTTP response that wraps the original response with a decoded stream. + */ + private static class DecodedResponse extends HttpResponse { private final HttpResponse originalResponse; + private final Flux decodedBody; + private final DecoderState decoderState; - DecodedResponse(HttpResponse httpResponse, Flux decodedBody) { - super(httpResponse.getRequest()); - this.originalResponse = httpResponse; + /** + * Creates a new decoded response. + * + * @param originalResponse The original HTTP response. + * @param decodedBody The decoded body stream. + * @param decoderState The decoder state. + */ + DecodedResponse(HttpResponse originalResponse, Flux decodedBody, DecoderState decoderState) { + super(originalResponse.getRequest()); + this.originalResponse = originalResponse; this.decodedBody = decodedBody; + this.decoderState = decoderState; } @Override @@ -153,12 +327,21 @@ public Mono getBodyAsByteArray() { @Override public Mono getBodyAsString() { - return getBodyAsByteArray().map(String::new); + return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); } @Override public Mono getBodyAsString(Charset charset) { return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); } + + /** + * Gets the decoder state. + * + * @return The decoder state. + */ + public DecoderState getDecoderState() { + return decoderState; + } } } From b7ba23408cc27171a5aec938f9bdce92f2acee01 Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Wed, 3 Dec 2025 21:21:40 +0530 Subject: [PATCH 04/31] fixing smart retry impl --- .../blob/specialized/BlobAsyncClientBase.java | 58 +- .../blob/BlobMessageDecoderDownloadTests.java | 206 +++++- .../src/test/resources/logback-test.xml | 11 + .../checkstyle-suppressions.xml | 13 +- .../StructuredMessageDecoder.java | 675 +++++++++++++++--- ...StorageContentValidationDecoderPolicy.java | 321 +++++++-- .../StructuredMessageDecoderTests.java | 287 ++++++++ ...ageContentValidationDecoderPolicyTest.java | 105 +++ 8 files changed, 1489 insertions(+), 187 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java create mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 99b9a9db3fdc..f34f0f5f59ab 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -88,6 +88,7 @@ import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.StorageChecksumAlgorithm; +import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; @@ -1383,24 +1384,65 @@ Mono downloadStreamWithResponseInternal(BlobRange ran try { // For retry context, preserve decoder state if structured message validation is enabled Context retryContext = firstRangeContext; + BlobRange retryRange; - // If structured message decoding is enabled, we need to include the decoder state - // so the retry can continue from where we left off + // If structured message decoding is enabled, we need to calculate the retry offset + // based on the encoded bytes processed, not the decoded bytes if (contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled()) { - // The decoder state will be set by the policy during processing - // We preserve it in the context for the retry request - Object decoderState + // Get the decoder state to determine how many encoded bytes were processed + Object decoderStateObj = firstRangeContext.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) .orElse(null); - if (decoderState != null) { + + if (decoderStateObj instanceof StorageContentValidationDecoderPolicy.DecoderState) { + StorageContentValidationDecoderPolicy.DecoderState decoderState + = (StorageContentValidationDecoderPolicy.DecoderState) decoderStateObj; + + // Use getRetryOffset() to get the correct offset for retry + // This accounts for pending bytes that have been received but not yet consumed + long encodedOffset = decoderState.getRetryOffset(); + long remainingCount = finalCount - encodedOffset; + retryRange = new BlobRange(initialOffset + encodedOffset, remainingCount); + + LOGGER.info( + "Structured message smart retry: resuming from offset {} (initial={}, encoded={})", + initialOffset + encodedOffset, initialOffset, encodedOffset); + + // Preserve the decoder state for the retry retryContext = retryContext .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); + } else { + // No decoder state available, try to parse retry offset from exception message + // The exception message contains RETRY-START-OFFSET= token + long retryStartOffset = StorageContentValidationDecoderPolicy + .parseRetryStartOffset(throwable.getMessage()); + if (retryStartOffset >= 0) { + long remainingCount = finalCount - retryStartOffset; + // Validate remainingCount to avoid negative values + if (remainingCount <= 0) { + LOGGER.warning("Retry offset {} exceeds finalCount {}, using fallback", + retryStartOffset, finalCount); + retryRange = new BlobRange(initialOffset + offset, newCount); + } else { + retryRange = new BlobRange(initialOffset + retryStartOffset, remainingCount); + + LOGGER.info( + "Structured message smart retry from exception: resuming from offset {} " + + "(initial={}, parsed={})", + initialOffset + retryStartOffset, initialOffset, retryStartOffset); + } + } else { + // Fallback to normal retry logic if no offset found + retryRange = new BlobRange(initialOffset + offset, newCount); + } } + } else { + // For non-structured downloads, use smart retry from the interrupted offset + retryRange = new BlobRange(initialOffset + offset, newCount); } - return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, - eTag, finalGetMD5, retryContext); + return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, retryContext); } catch (Exception e) { return Mono.error(e); } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 441e4e591ea5..a2b9f5283895 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -12,6 +12,8 @@ import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.structuredmessage.StructuredMessageEncoder; import com.azure.storage.common.implementation.structuredmessage.StructuredMessageFlags; +import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.core.publisher.Flux; @@ -19,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -53,7 +56,8 @@ public void downloadStreamWithResponseContentValidation() throws IOException { StepVerifier .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, + (BlobRequestConditions) null, false, validationOptions)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) .verifyComplete(); @@ -61,6 +65,9 @@ public void downloadStreamWithResponseContentValidation() throws IOException { @Test public void downloadStreamWithResponseContentValidationRange() throws IOException { + // Note: Range downloads are not compatible with structured message validation + // because you need the complete encoded message for validation. + // This test verifies that range downloads work without validation. byte[] randomData = getRandomByteArray(Constants.KB); StructuredMessageEncoder encoder = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); @@ -68,16 +75,16 @@ public void downloadStreamWithResponseContentValidationRange() throws IOExceptio Flux input = Flux.just(encodedData); - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - + // Range download without validation should work BlobRange range = new BlobRange(0, 512L); StepVerifier.create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(range, null, null, false, validationOptions)) + .then( + bc.downloadStreamWithResponse(range, (DownloadRetryOptions) null, (BlobRequestConditions) null, false)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { assertNotNull(r); - assertTrue(r.length > 0); + // Should get exactly 512 bytes of encoded data + assertEquals(512, r.length); }).verifyComplete(); } @@ -96,7 +103,8 @@ public void downloadStreamWithResponseContentValidationLargeBlob() throws IOExce StepVerifier .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, + (BlobRequestConditions) null, false, validationOptions)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) .verifyComplete(); @@ -117,7 +125,8 @@ public void downloadStreamWithResponseContentValidationMultipleSegments() throws StepVerifier .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, + (BlobRequestConditions) null, false, validationOptions)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) .verifyComplete(); @@ -135,7 +144,8 @@ public void downloadStreamWithResponseNoValidation() throws IOException { // No validation options - should download encoded data as-is StepVerifier.create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(null, null, null, false)) + .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, + (BlobRequestConditions) null, false)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { assertNotNull(r); // Should get encoded data, not decoded @@ -157,7 +167,8 @@ public void downloadStreamWithResponseValidationDisabled() throws IOException { = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(false); StepVerifier.create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, + (BlobRequestConditions) null, false, validationOptions)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { assertNotNull(r); // Should get encoded data, not decoded @@ -180,7 +191,8 @@ public void downloadStreamWithResponseContentValidationSmallSegment() throws IOE StepVerifier .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, + (BlobRequestConditions) null, false, validationOptions)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) .verifyComplete(); @@ -201,9 +213,179 @@ public void downloadStreamWithResponseContentValidationVeryLargeBlob() throws IO StepVerifier .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse(null, null, null, false, validationOptions)) + .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, + (BlobRequestConditions) null, false, validationOptions)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) .verifyComplete(); } + + @Test + public void downloadStreamWithResponseContentValidationSmartRetry() throws IOException { + // Test smart retry functionality with structured message validation + // This test simulates network interruptions and verifies that: + // 1. The decoder validates checksums for all received data + // 2. Retries resume from the encoded offset where the interruption occurred + // 3. The download eventually succeeds despite multiple interruptions + + byte[] randomData = getRandomByteArray(Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + // Create a policy that will simulate 3 network interruptions + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3); + + // Upload the encoded data using the regular client + bc.upload(input, null, true).block(); + + // Create a download client with both the mock policy AND the decoder policy + // The decoder policy is needed to actually decode structured messages and validate checksums + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + bc.getBlobUrl(), mockPolicy, decoderPolicy); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + // Configure retry options to allow retries + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + // Download with validation - should succeed despite interruptions + StepVerifier.create(downloadClient + .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + // Verify the data is correctly decoded + TestUtils.assertArraysEqual(r, randomData); + }).verifyComplete(); + + // Verify that retries occurred (3 interruptions means we should have 0 tries remaining) + assertEquals(0, mockPolicy.getTriesRemaining()); + + // Verify that range headers were sent for retries + List rangeHeaders = mockPolicy.getRangeHeaders(); + assertTrue(rangeHeaders.size() > 0, "Expected range headers for retries"); + + // With structured message validation and smart retry, retries should resume from the encoded + // offset where the interruption occurred. The first request starts at 0, and subsequent + // retry requests should start from progressively higher offsets. + assertTrue(rangeHeaders.get(0).startsWith("bytes=0-"), "First request should start from offset 0"); + + // Subsequent requests should start from higher offsets (smart retry resuming from where it left off) + for (int i = 1; i < rangeHeaders.size(); i++) { + String rangeHeader = rangeHeaders.get(i); + // Each retry should start from a higher offset than the previous + // Note: We can't assert exact offset values as they depend on how much data was received + // before the interruption, but we can verify it's a valid range header + assertTrue(rangeHeader.startsWith("bytes="), + "Retry request " + i + " should have a range header: " + rangeHeader); + } + } + + @Test + public void downloadStreamWithResponseContentValidationSmartRetryMultipleSegments() throws IOException { + // Test smart retry with multiple segments to ensure checksum validation + // works correctly and retries resume from the interrupted encoded offset. + + byte[] randomData = getRandomByteArray(2 * Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + // Create a policy that will simulate 4 network interruptions + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(4); + + // Upload the encoded data + bc.upload(input, null, true).block(); + + // Create a download client with both the mock policy AND the decoder policy + // The decoder policy is needed to actually decode structured messages and validate checksums + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + bc.getBlobUrl(), mockPolicy, decoderPolicy); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + // Download with validation - should succeed and validate all segment checksums + StepVerifier.create(downloadClient + .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + // Verify the data is correctly decoded + TestUtils.assertArraysEqual(r, randomData); + }).verifyComplete(); + + // Verify that retries occurred + assertEquals(0, mockPolicy.getTriesRemaining()); + + // Verify multiple retry requests were made + List rangeHeaders = mockPolicy.getRangeHeaders(); + assertTrue(rangeHeaders.size() >= 4, + "Expected at least 4 range headers for retries, got: " + rangeHeaders.size()); + + // With smart retry, each request should have a valid range header + for (int i = 0; i < rangeHeaders.size(); i++) { + String rangeHeader = rangeHeaders.get(i); + assertTrue(rangeHeader.startsWith("bytes="), + "Request " + i + " should have a valid range header, but was: " + rangeHeader); + } + } + + @Test + public void downloadStreamWithResponseContentValidationSmartRetryLargeBlob() throws IOException { + // Test smart retry with a larger blob to ensure retries resume from the + // interrupted offset and successfully validate all data + + byte[] randomData = getRandomByteArray(5 * Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 1024, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + // Create a policy that will simulate 2 network interruptions + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(2); + + // Upload the encoded data + bc.upload(input, null, true).block(); + + // Create a download client with both the mock policy AND the decoder policy + // The decoder policy is needed to actually decode structured messages and validate checksums + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + bc.getBlobUrl(), mockPolicy, decoderPolicy); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + // Download with validation - decoder should validate checksums before each retry + StepVerifier.create(downloadClient + .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + // Verify the data is correctly decoded + TestUtils.assertArraysEqual(r, randomData); + }).verifyComplete(); + + // Verify that retries occurred + assertEquals(0, mockPolicy.getTriesRemaining()); + + // Verify that smart retry is working with valid range headers + List rangeHeaders = mockPolicy.getRangeHeaders(); + for (int i = 0; i < rangeHeaders.size(); i++) { + String rangeHeader = rangeHeaders.get(i); + assertTrue(rangeHeader.startsWith("bytes="), + "Request " + i + " should have a valid range header, but was: " + rangeHeader); + } + } } diff --git a/sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml b/sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml new file mode 100644 index 000000000000..b35926b40592 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml @@ -0,0 +1,11 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + diff --git a/sdk/storage/azure-storage-common/checkstyle-suppressions.xml b/sdk/storage/azure-storage-common/checkstyle-suppressions.xml index 4e4a986c034c..93d35df5d619 100644 --- a/sdk/storage/azure-storage-common/checkstyle-suppressions.xml +++ b/sdk/storage/azure-storage-common/checkstyle-suppressions.xml @@ -3,10 +3,11 @@ - - - - - - + + + + + + + diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 6117a7765541..6534dd0ce38d 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -19,23 +19,95 @@ /** * Decoder for structured messages with support for segmenting and CRC64 checksums. + * + *

This decoder properly handles partial headers and segment splits across HTTP chunks + * by maintaining a pending buffer and only advancing offsets when complete structures + * have been fully read and validated.

+ * + *

Key invariants: + *

    + *
  • Never read partial headers - always check buffer remaining >= required bytes
    • + *
  • Only advance messageOffset when bytes are fully consumed and validated
    • + *
  • lastCompleteSegmentStart always points to a valid segment boundary for retry
    • + *
*/ public class StructuredMessageDecoder { private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecoder.class); - private long messageLength; + + // Message state + private long messageLength = -1; private StructuredMessageFlags flags; - private int numSegments; + private int numSegments = -1; private final long expectedContentLength; - private int messageOffset = 0; + // Offset tracking + private long messageOffset = 0; // Absolute encoded bytes consumed from the message + private long totalDecodedPayloadBytes = 0; // Total decoded (payload) bytes output + + // Current segment state private int currentSegmentNumber = 0; - private int currentSegmentContentLength = 0; - private int currentSegmentContentOffset = 0; + private long currentSegmentContentLength = 0; + private long currentSegmentContentOffset = 0; + // CRC validation private long messageCrc64 = 0; private long segmentCrc64 = 0; private final Map segmentCrcs = new HashMap<>(); + // Smart retry tracking - lastCompleteSegmentStart is the absolute offset where the last + // fully completed segment ended. This is the safe retry boundary. + private long lastCompleteSegmentStart = 0; + + // Pending buffer for handling partial headers/segments across chunks + private final ByteArrayOutputStream pendingBytes = new ByteArrayOutputStream(); + + /** + * Decode result status codes. + */ + public enum DecodeStatus { + /** Decoding succeeded, more data may be available */ + SUCCESS, + /** Need more bytes to continue (partial header/segment) */ + NEED_MORE_BYTES, + /** Decoding completed successfully */ + COMPLETED, + /** Invalid data encountered */ + INVALID + } + + /** + * Result of a decode operation. + */ + public static class DecodeResult { + private final DecodeStatus status; + private final ByteBuffer decodedPayload; + private final String message; + private final int bytesConsumed; + + DecodeResult(DecodeStatus status, ByteBuffer decodedPayload, int bytesConsumed, String message) { + this.status = status; + this.decodedPayload = decodedPayload; + this.bytesConsumed = bytesConsumed; + this.message = message; + } + + public DecodeStatus getStatus() { + return status; + } + + public ByteBuffer getDecodedPayload() { + return decodedPayload; + } + + public int getBytesConsumed() { + return bytesConsumed; + } + + public String getMessage() { + return message; + } + } + /** * Constructs a new StructuredMessageDecoder. * @@ -46,95 +118,370 @@ public StructuredMessageDecoder(long expectedContentLength) { } /** - * Reads the message header from the given buffer. + * Gets the byte offset where the last complete segment ended. + * This is used for smart retry to resume from a segment boundary. * - * @param buffer The buffer containing the message header. - * @throws IllegalArgumentException if the buffer does not contain a valid message header. + * @return The byte offset of the last complete segment boundary. */ - private void readMessageHeader(ByteBuffer buffer) { - if (buffer.remaining() < V1_HEADER_LENGTH) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException("Content not long enough to contain a valid " + "message header.")); + public long getLastCompleteSegmentStart() { + return lastCompleteSegmentStart; + } + + /** + * Returns the canonical absolute byte index (0-based) that should be used to resume a failed/incomplete download. + * This MUST be used directly as the Range header start value: "Range: bytes={retryStartOffset}-" + * + *

This is equivalent to {@link #getLastCompleteSegmentStart()} but provides a clearer semantic name + * for the smart retry use case.

+ * + * @return The absolute byte index for the retry start offset. + */ + public long getRetryStartOffset() { + return getLastCompleteSegmentStart(); + } + + /** + * Gets the current message offset (total bytes consumed from the structured message). + * + * @return The current message offset. + */ + public long getMessageOffset() { + return messageOffset; + } + + /** + * Gets the total decoded payload bytes produced so far. + * + * @return The total decoded payload bytes. + */ + public long getTotalDecodedPayloadBytes() { + return totalDecodedPayloadBytes; + } + + /** + * Advances the message offset by the specified number of bytes. + * This should be called after consuming an encoded segment to maintain + * the authoritative encoded offset. + * + * @param bytes The number of bytes to advance. + */ + public void advanceMessageOffset(long bytes) { + long priorOffset = messageOffset; + messageOffset += bytes; + LOGGER.atInfo() + .addKeyValue("priorOffset", priorOffset) + .addKeyValue("bytesAdvanced", bytes) + .addKeyValue("newOffset", messageOffset) + .log("Advanced message offset"); + } + + /** + * Resets the decoder position to the last complete segment boundary. + * This is used during smart retry to ensure the decoder is in sync with + * the data being provided from the retry offset. + */ + public void resetToLastCompleteSegment() { + if (messageOffset != lastCompleteSegmentStart) { + LOGGER.atInfo() + .addKeyValue("fromOffset", messageOffset) + .addKeyValue("toOffset", lastCompleteSegmentStart) + .addKeyValue("currentSegmentNum", currentSegmentNumber) + .addKeyValue("currentSegmentContentOffset", currentSegmentContentOffset) + .addKeyValue("currentSegmentContentLength", currentSegmentContentLength) + .log("Resetting decoder to last complete segment boundary"); + messageOffset = lastCompleteSegmentStart; + // Reset current segment state - next decode will read the segment header + currentSegmentContentOffset = 0; + currentSegmentContentLength = 0; + // Clear any pending bytes since we're resetting to a known boundary + pendingBytes.reset(); + } else { + LOGGER.atVerbose() + .addKeyValue("offset", messageOffset) + .log("Decoder already at last complete segment boundary, no reset needed"); + } + } + + /** + * Converts a ByteBuffer range to hex string for diagnostic purposes. + */ + private static String toHex(ByteBuffer buf, int len) { + int pos = buf.position(); + int peek = Math.min(len, buf.remaining()); + byte[] out = new byte[peek]; + buf.get(out, 0, peek); + buf.position(pos); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < out.length; i++) { + sb.append(String.format("%02X", out[i])); + if (i < out.length - 1) { + sb.append(' '); + } + } + return sb.toString(); + } + + /** + * Gets the total available bytes (pending + buffer remaining). + */ + private int getAvailableBytes(ByteBuffer buffer) { + return pendingBytes.size() + buffer.remaining(); + } + + /** + * Creates a combined buffer from pending bytes and new buffer. + * Returns a new buffer with position=0 and LITTLE_ENDIAN order. + * The original buffer's position is NOT advanced. + */ + private ByteBuffer getCombinedBuffer(ByteBuffer buffer) { + if (pendingBytes.size() == 0) { + ByteBuffer dup = buffer.duplicate(); + dup.order(ByteOrder.LITTLE_ENDIAN); + return dup; + } + + byte[] pending = pendingBytes.toByteArray(); + ByteBuffer combined = ByteBuffer.allocate(pending.length + buffer.remaining()); + combined.order(ByteOrder.LITTLE_ENDIAN); + combined.put(pending); + combined.put(buffer.duplicate()); + combined.flip(); + return combined; + } + + /** + * Consumes bytes from pending first, then from buffer. + * Updates the buffer's position to reflect bytes consumed. + */ + private void consumeBytes(int bytesToConsume, ByteBuffer buffer) { + int pendingSize = pendingBytes.size(); + if (bytesToConsume <= pendingSize) { + // All bytes come from pending - remove from pending + byte[] remaining = pendingBytes.toByteArray(); + pendingBytes.reset(); + if (bytesToConsume < pendingSize) { + pendingBytes.write(remaining, bytesToConsume, pendingSize - bytesToConsume); + } + } else { + // Consume all pending and some from buffer + int bytesFromBuffer = bytesToConsume - pendingSize; + pendingBytes.reset(); + buffer.position(buffer.position() + bytesFromBuffer); } + } - int messageVersion = Byte.toUnsignedInt(buffer.get()); + /** + * Appends remaining buffer bytes to pending for next chunk. + */ + private void appendToPending(ByteBuffer buffer) { + while (buffer.hasRemaining()) { + pendingBytes.write(buffer.get()); + } + } + + /** + * Peeks the next segment length without consuming from the buffer. + * Used by the policy to calculate encoded segment size before slicing. + * + * @param buffer The buffer to peek from. + * @param relativeIndex The position in the buffer to start reading from. + * @return The segment content length, or -1 if not enough bytes. + */ + public long peekNextSegmentLength(ByteBuffer buffer, int relativeIndex) { + // Need at least V1_SEGMENT_HEADER_LENGTH bytes to read segment number (2) + segment size (8) + if (relativeIndex + V1_SEGMENT_HEADER_LENGTH > buffer.limit()) { + return -1; + } + // Segment size is at offset 2 (after segment number which is 2 bytes) + return buffer.getLong(relativeIndex + 2); + } + + /** + * Gets the flags for the current message (needed to determine if CRC is present). + * + * @return The message flags, or null if header not yet read. + */ + public StructuredMessageFlags getFlags() { + return flags; + } + + /** + * Gets the expected message length from the header. + * + * @return The message length, or -1 if header not yet read. + */ + public long getMessageLength() { + return messageLength; + } + + /** + * Gets the number of segments from the header. + * + * @return The number of segments, or -1 if header not yet read. + */ + public int getNumSegments() { + return numSegments; + } + + /** + * Checks if the message header has been read. + * + * @return true if header has been read, false otherwise. + */ + public boolean isHeaderRead() { + return messageLength != -1; + } + + /** + * Reads the message header if we have enough bytes. + * + * @param buffer The buffer to read from. + * @return true if header was successfully read, false if more bytes needed. + */ + private boolean tryReadMessageHeader(ByteBuffer buffer) { + if (messageLength != -1) { + return true; // Already read + } + + int available = getAvailableBytes(buffer); + if (available < V1_HEADER_LENGTH) { + LOGGER.atInfo() + .addKeyValue("available", available) + .addKeyValue("required", V1_HEADER_LENGTH) + .addKeyValue("pendingBytes", pendingBytes.size()) + .log("Not enough bytes for message header, waiting for more"); + appendToPending(buffer); + return false; + } + + ByteBuffer combined = getCombinedBuffer(buffer); + + int messageVersion = Byte.toUnsignedInt(combined.get()); if (messageVersion != DEFAULT_MESSAGE_VERSION) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException("Unsupported structured message version: " + messageVersion)); + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Unsupported structured message version: " + messageVersion))); } - messageLength = (int) buffer.getLong(); - if (messageLength < V1_HEADER_LENGTH) { + long msgLen = combined.getLong(); + if (msgLen < V1_HEADER_LENGTH) { throw LOGGER.logExceptionAsError( - new IllegalArgumentException("Content not long enough to contain a valid " + "message header.")); + new IllegalArgumentException(enrichExceptionMessage("Message length too small: " + msgLen))); } - if (messageLength != expectedContentLength) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException("Structured message length " + messageLength - + " did not match content length " + expectedContentLength)); + if (msgLen != expectedContentLength) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Structured message length " + msgLen + " did not match content length " + expectedContentLength))); } - flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(buffer.getShort())); - numSegments = Short.toUnsignedInt(buffer.getShort()); + flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(combined.getShort())); + numSegments = Short.toUnsignedInt(combined.getShort()); + // Consume the bytes from pending/buffer + consumeBytes(V1_HEADER_LENGTH, buffer); messageOffset += V1_HEADER_LENGTH; + messageLength = msgLen; + + LOGGER.atInfo() + .addKeyValue("messageLength", messageLength) + .addKeyValue("numSegments", numSegments) + .addKeyValue("flags", flags) + .addKeyValue("messageOffset", messageOffset) + .log("Message header read successfully"); + + return true; } /** - * Reads the segment header from the given buffer. + * Reads a segment header if we have enough bytes. * - * @param buffer The buffer containing the segment header. - * @throws IllegalArgumentException if the buffer does not contain a valid segment header. + * @param buffer The buffer to read from. + * @return true if segment header was read, false if more bytes needed. */ - private void readSegmentHeader(ByteBuffer buffer) { - if (buffer.remaining() < V1_SEGMENT_HEADER_LENGTH) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException("Segment header is incomplete.")); + private boolean tryReadSegmentHeader(ByteBuffer buffer) { + int available = getAvailableBytes(buffer); + if (available < V1_SEGMENT_HEADER_LENGTH) { + LOGGER.atInfo() + .addKeyValue("available", available) + .addKeyValue("required", V1_SEGMENT_HEADER_LENGTH) + .addKeyValue("pendingBytes", pendingBytes.size()) + .addKeyValue("decoderOffset", messageOffset) + .log("Not enough bytes for segment header, waiting for more"); + appendToPending(buffer); + return false; } - int segmentNum = Short.toUnsignedInt(buffer.getShort()); - int segmentSize = (int) buffer.getLong(); + ByteBuffer combined = getCombinedBuffer(buffer); - if (segmentSize < 0 || segmentSize > buffer.remaining()) { - throw LOGGER - .logExceptionAsError(new IllegalArgumentException("Invalid segment size detected: " + segmentSize)); - } + // Log the raw bytes we're about to read + LOGGER.atInfo() + .addKeyValue("decoderOffset", messageOffset) + .addKeyValue("bufferPos", combined.position()) + .addKeyValue("bufferRemaining", combined.remaining()) + .addKeyValue("peek16", toHex(combined, 16)) + .addKeyValue("lastCompleteSegment", lastCompleteSegmentStart) + .log("Decoder about to read segment header"); + int segmentNum = Short.toUnsignedInt(combined.getShort()); + long segmentSize = combined.getLong(); + + // Validate segment number if (segmentNum != currentSegmentNumber + 1) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException("Unexpected segment number.")); + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Unexpected segment number. Expected: " + (currentSegmentNumber + 1) + ", got: " + segmentNum))); } + // Validate segment size - must be non-negative and reasonable + // We can't have segments larger than the remaining message length + long remainingMessageBytes = messageLength - messageOffset - V1_SEGMENT_HEADER_LENGTH; + if (segmentSize < 0 || segmentSize > remainingMessageBytes) { + LOGGER.error("Invalid segment length read: segmentLength={}, decoderOffset={}, lastCompleteSegment={}", + segmentSize, messageOffset, lastCompleteSegmentStart); + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Invalid segment size detected: " + segmentSize + " (remaining=" + remainingMessageBytes + ")"))); + } + + // Consume the bytes and update state + consumeBytes(V1_SEGMENT_HEADER_LENGTH, buffer); + messageOffset += V1_SEGMENT_HEADER_LENGTH; currentSegmentNumber = segmentNum; currentSegmentContentLength = segmentSize; currentSegmentContentOffset = 0; - if (segmentSize == 0) { - readSegmentFooter(buffer); - } - if (flags == StructuredMessageFlags.STORAGE_CRC64) { segmentCrc64 = 0; } - messageOffset += V1_SEGMENT_HEADER_LENGTH; + LOGGER.atInfo() + .addKeyValue("segmentNum", segmentNum) + .addKeyValue("segmentLength", segmentSize) + .addKeyValue("decoderOffset", messageOffset) + .log("Segment header read successfully"); + + return true; } /** - * Reads the segment content from the given buffer and writes it to the output stream. + * Reads segment content bytes if available. * - * @param buffer The buffer containing the segment content. - * @param output The output stream to write the segment content to. - * @param size The maximum number of bytes to read. - * @throws IllegalArgumentException if there is a segment size mismatch. + * @param buffer The buffer to read from. + * @param output The output stream to write decoded payload to. + * @return The number of payload bytes read, or -1 if more bytes needed for CRC. */ - private void readSegmentContent(ByteBuffer buffer, ByteArrayOutputStream output, int size) { - int toRead = Math.min(buffer.remaining(), currentSegmentContentLength - currentSegmentContentOffset); - toRead = Math.min(toRead, size); + private int tryReadSegmentContent(ByteBuffer buffer, ByteArrayOutputStream output) { + long remaining = currentSegmentContentLength - currentSegmentContentOffset; + if (remaining == 0) { + return 0; // All content read, need to read footer + } - if (toRead == 0) { - return; + int available = getAvailableBytes(buffer); + if (available == 0) { + return 0; // No bytes available } + int toRead = (int) Math.min(available, remaining); + ByteBuffer combined = getCombinedBuffer(buffer); + byte[] content = new byte[toRead]; - buffer.get(content); + combined.get(content); output.write(content, 0, toRead); if (flags == StructuredMessageFlags.STORAGE_CRC64) { @@ -142,81 +489,184 @@ private void readSegmentContent(ByteBuffer buffer, ByteArrayOutputStream output, messageCrc64 = StorageCrc64Calculator.compute(content, messageCrc64); } + consumeBytes(toRead, buffer); messageOffset += toRead; currentSegmentContentOffset += toRead; + totalDecodedPayloadBytes += toRead; - if (currentSegmentContentOffset > currentSegmentContentLength) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException("Segment size mismatch detected in segment " + currentSegmentNumber)); - } - - if (currentSegmentContentOffset == currentSegmentContentLength) { - readSegmentFooter(buffer); - } + return toRead; } /** - * Reads the segment footer from the given buffer. + * Reads the segment CRC footer if needed and available. * - * @param buffer The buffer containing the segment footer. - * @throws IllegalArgumentException if the buffer does not contain a valid segment footer. + * @param buffer The buffer to read from. + * @return true if footer was read (or not needed), false if more bytes needed. */ - private void readSegmentFooter(ByteBuffer buffer) { + private boolean tryReadSegmentFooter(ByteBuffer buffer) { if (currentSegmentContentOffset != currentSegmentContentLength) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException("Segment content length mismatch in segment " + currentSegmentNumber - + ". Expected: " + currentSegmentContentLength + ", Read: " + currentSegmentContentOffset)); + return true; // Content not fully read yet } if (flags == StructuredMessageFlags.STORAGE_CRC64) { - if (buffer.remaining() < CRC64_LENGTH) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException("Segment footer is incomplete.")); + int available = getAvailableBytes(buffer); + if (available < CRC64_LENGTH) { + LOGGER.atInfo() + .addKeyValue("available", available) + .addKeyValue("required", CRC64_LENGTH) + .addKeyValue("segmentNum", currentSegmentNumber) + .log("Not enough bytes for segment CRC footer, waiting for more"); + appendToPending(buffer); + return false; } - long reportedCrc64 = buffer.getLong(); + ByteBuffer combined = getCombinedBuffer(buffer); + long reportedCrc64 = combined.getLong(); + if (segmentCrc64 != reportedCrc64) { throw LOGGER.logExceptionAsError( - new IllegalArgumentException("CRC64 mismatch detected in segment " + currentSegmentNumber)); + new IllegalArgumentException(enrichExceptionMessage("CRC64 mismatch detected in segment " + + currentSegmentNumber + ". Expected: " + segmentCrc64 + ", got: " + reportedCrc64))); } + + consumeBytes(CRC64_LENGTH, buffer); segmentCrcs.put(currentSegmentNumber, segmentCrc64); messageOffset += CRC64_LENGTH; } + // Mark that this segment is complete + lastCompleteSegmentStart = messageOffset; + LOGGER.atInfo() + .addKeyValue("segmentNum", currentSegmentNumber) + .addKeyValue("offset", lastCompleteSegmentStart) + .addKeyValue("segmentLength", currentSegmentContentLength) + .log("Segment complete at byte offset"); + + // Check if we need to read message footer if (currentSegmentNumber == numSegments) { - readMessageFooter(buffer); - } else { - readSegmentHeader(buffer); + return tryReadMessageFooter(buffer); } + + return true; } /** - * Reads the segment footer from the given buffer. + * Reads the message CRC footer if needed and available. * - * @param buffer The buffer containing the segment footer. - * @throws IllegalArgumentException if the buffer does not contain a valid segment footer. + * @param buffer The buffer to read from. + * @return true if footer was read (or not needed), false if more bytes needed. */ - private void readMessageFooter(ByteBuffer buffer) { + private boolean tryReadMessageFooter(ByteBuffer buffer) { if (flags == StructuredMessageFlags.STORAGE_CRC64) { - if (buffer.remaining() < CRC64_LENGTH) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException("Message footer is incomplete.")); + int available = getAvailableBytes(buffer); + if (available < CRC64_LENGTH) { + LOGGER.atInfo() + .addKeyValue("available", available) + .addKeyValue("required", CRC64_LENGTH) + .log("Not enough bytes for message CRC footer, waiting for more"); + appendToPending(buffer); + return false; } - long reportedCrc = buffer.getLong(); + ByteBuffer combined = getCombinedBuffer(buffer); + long reportedCrc = combined.getLong(); + if (messageCrc64 != reportedCrc) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException("CRC64 mismatch detected in message " + "footer.")); + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "CRC64 mismatch detected in message footer. Expected: " + messageCrc64 + ", got: " + reportedCrc))); } + + consumeBytes(CRC64_LENGTH, buffer); messageOffset += CRC64_LENGTH; } - if (messageOffset != messageLength) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException("Decoded message length does not match " + "expected length.")); + return true; + } + + /** + * Decodes as much as possible from the given buffer. + * This method properly handles partial headers and segments by buffering + * incomplete data and returning NEED_MORE_BYTES when more data is required. + * + * @param buffer The buffer containing encoded data. + * @return A DecodeResult indicating the outcome and any decoded payload. + */ + public DecodeResult decodeChunk(ByteBuffer buffer) { + buffer.order(ByteOrder.LITTLE_ENDIAN); + ByteArrayOutputStream decodedContent = new ByteArrayOutputStream(); + int startPos = buffer.position(); + + LOGGER.atInfo() + .addKeyValue("newBytes", buffer.remaining()) + .addKeyValue("pendingBytes", pendingBytes.size()) + .addKeyValue("decoderOffset", messageOffset) + .addKeyValue("lastCompleteSegment", lastCompleteSegmentStart) + .log("Received buffer in decode"); + + try { + // Step 1: Read message header if not yet read + if (!tryReadMessageHeader(buffer)) { + return new DecodeResult(DecodeStatus.NEED_MORE_BYTES, null, 0, "Waiting for message header"); + } + + // Step 2: Process segments + while (messageOffset < messageLength) { + // Read segment header if needed + if (currentSegmentContentOffset == currentSegmentContentLength) { + if (!tryReadSegmentHeader(buffer)) { + break; // Need more bytes for segment header + } + } + + // Read segment content + int payloadRead = tryReadSegmentContent(buffer, decodedContent); + + // Read segment footer (CRC) if content is complete + if (currentSegmentContentOffset == currentSegmentContentLength) { + if (!tryReadSegmentFooter(buffer)) { + break; // Need more bytes for segment footer + } + } + + // Check if all segments are complete + if (currentSegmentNumber == numSegments && messageOffset >= messageLength) { + LOGGER.atInfo() + .addKeyValue("messageOffset", messageOffset) + .addKeyValue("messageLength", messageLength) + .addKeyValue("totalDecodedPayload", totalDecodedPayloadBytes) + .log("Message decode completed"); + + ByteBuffer result + = decodedContent.size() > 0 ? ByteBuffer.wrap(decodedContent.toByteArray()) : null; + return new DecodeResult(DecodeStatus.COMPLETED, result, buffer.position() - startPos, + "Decode completed"); + } + + // If we couldn't read any bytes and no data available, need more + if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { + break; + } + } + + // Return any decoded content even if we need more bytes + ByteBuffer result = decodedContent.size() > 0 ? ByteBuffer.wrap(decodedContent.toByteArray()) : null; + + if (messageOffset >= messageLength) { + return new DecodeResult(DecodeStatus.COMPLETED, result, buffer.position() - startPos, + "Decode completed"); + } + + return new DecodeResult(DecodeStatus.NEED_MORE_BYTES, result, buffer.position() - startPos, + "Waiting for more data"); + + } catch (IllegalArgumentException e) { + return new DecodeResult(DecodeStatus.INVALID, null, buffer.position() - startPos, e.getMessage()); } } /** * Decodes the structured message from the given buffer up to the specified size. + * This is a convenience method that wraps decodeChunk for backwards compatibility. * * @param buffer The buffer containing the structured message. * @param size The maximum number of bytes to decode. @@ -228,15 +678,26 @@ public ByteBuffer decode(ByteBuffer buffer, int size) { ByteArrayOutputStream decodedContent = new ByteArrayOutputStream(); if (messageOffset == 0) { - readMessageHeader(buffer); + if (!tryReadMessageHeader(buffer)) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Content not long enough to contain a valid message header."))); + } } while (buffer.hasRemaining() && decodedContent.size() < size) { if (currentSegmentContentOffset == currentSegmentContentLength) { - readSegmentHeader(buffer); + if (!tryReadSegmentHeader(buffer)) { + break; // Need more bytes + } } - readSegmentContent(buffer, decodedContent, size - decodedContent.size()); + tryReadSegmentContent(buffer, decodedContent); + + if (currentSegmentContentOffset == currentSegmentContentLength) { + if (!tryReadSegmentFooter(buffer)) { + break; // Need more bytes + } + } } return ByteBuffer.wrap(decodedContent.toByteArray()); @@ -254,14 +715,40 @@ public ByteBuffer decode(ByteBuffer buffer) { } /** - * Finalizes the decoding process and validates that the entire message has been decoded. + * Finalizes the decoding process and returns any final decoded bytes still buffered internally. + * The policy should aggregate decoded byte counts and perform the final length comparison. * - * @throws IllegalArgumentException if the decoded message length does not match the expected length. + * @return A ByteBuffer containing any final decoded bytes, or null if none remain. + * @throws IllegalArgumentException if the encoded message offset doesn't match expected length. */ - public void finalizeDecoding() { + public ByteBuffer finalizeDecoding() { if (messageOffset != messageLength) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException("Decoded message length does not match " - + "expected length. Expected: " + messageLength + ", but was: " + messageOffset)); + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Decoded message length does not match expected length. Expected: " + + messageLength + ", but was: " + messageOffset))); } + // No buffered decoded bytes in current implementation + return null; + } + + /** + * Checks if decoding is complete. + * + * @return true if all expected bytes have been decoded, false otherwise. + */ + public boolean isComplete() { + return messageLength != -1 && messageOffset >= messageLength; + } + + /** + * Enriches an exception message with decoder offset information for debugging and retry. + * Format: "original message [decoderOffset=X,lastCompleteSegment=Y]" + * + * @param message The original exception message. + * @return The enriched message with offset information. + */ + private String enrichExceptionMessage(String message) { + return String.format("%s [decoderOffset=%d,lastCompleteSegment=%d]", message, messageOffset, + lastCompleteSegmentStart); } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 6bb81027e681..7103a3a11545 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -18,9 +18,12 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * This is a decoding policy in an {@link com.azure.core.http.HttpPipeline} to decode structured messages in @@ -37,12 +40,66 @@ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); + /** + * Machine-readable token pattern for extracting retry start offset from exception messages. + * Format: RETRY-START-OFFSET={number} + */ + private static final String RETRY_OFFSET_TOKEN = "RETRY-START-OFFSET="; + private static final Pattern RETRY_OFFSET_PATTERN = Pattern.compile("RETRY-START-OFFSET=(\\d+)"); + /** * Creates a new instance of {@link StorageContentValidationDecoderPolicy}. */ public StorageContentValidationDecoderPolicy() { } + /** + * Parses the retry start offset from an exception message containing the RETRY-START-OFFSET token. + * + * @param message The exception message to parse. + * @return The retry start offset, or -1 if not found. + */ + public static long parseRetryStartOffset(String message) { + if (message == null) { + return -1; + } + Matcher matcher = RETRY_OFFSET_PATTERN.matcher(message); + if (matcher.find()) { + try { + return Long.parseLong(matcher.group(1)); + } catch (NumberFormatException e) { + return -1; + } + } + return -1; + } + + /** + * Parses decoder offset information from enriched exception messages. + * Format: "[decoderOffset=X,lastCompleteSegment=Y]" + * + * @param message The exception message to parse. + * @return A long array [decoderOffset, lastCompleteSegment], or null if not found. + */ + public static long[] parseDecoderOffsets(String message) { + if (message == null) { + return null; + } + // Pattern: [decoderOffset=123,lastCompleteSegment=456] + Pattern pattern = Pattern.compile("\\[decoderOffset=(\\d+),lastCompleteSegment=(\\d+)\\]"); + Matcher matcher = pattern.matcher(message); + if (matcher.find()) { + try { + long decoderOffset = Long.parseLong(matcher.group(1)); + long lastCompleteSegment = Long.parseLong(matcher.group(2)); + return new long[] { decoderOffset, lastCompleteSegment }; + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { // Check if structured message decoding is enabled for this request @@ -78,6 +135,11 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN /** * Decodes a stream of byte buffers using the decoder state. + * The decoder properly handles partial headers and segments split across chunks. + * + *

When an error occurs or the stream ends prematurely, an IOException is thrown with a + * machine-readable token RETRY-START-OFFSET=<number> that can be parsed to determine + * the correct offset for retry requests.

* * @param encodedFlux The flux of encoded byte buffers. * @param state The decoder state. @@ -85,48 +147,159 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN */ private Flux decodeStream(Flux encodedFlux, DecoderState state) { return encodedFlux.concatMap(encodedBuffer -> { + // Skip empty buffers that may be emitted by reactor-netty + if (encodedBuffer == null || !encodedBuffer.hasRemaining()) { + LOGGER.atVerbose() + .addKeyValue("bufferLength", encodedBuffer == null ? "null" : encodedBuffer.remaining()) + .log("Skipping empty/null buffer in decodeStream"); + return Flux.empty(); + } + + LOGGER.atInfo() + .addKeyValue("newBytes", encodedBuffer.remaining()) + .addKeyValue("decoderOffset", state.decoder.getMessageOffset()) + .addKeyValue("lastCompleteSegment", state.decoder.getLastCompleteSegmentStart()) + .addKeyValue("totalDecodedPayload", state.decoder.getTotalDecodedPayloadBytes()) + .log("Received buffer in decodeStream"); + try { - // Combine with pending data if any - ByteBuffer dataToProcess = state.combineWithPending(encodedBuffer); - - // Track encoded bytes - int encodedBytesInBuffer = encodedBuffer.remaining(); - state.totalEncodedBytesProcessed.addAndGet(encodedBytesInBuffer); - - // Try to decode what we have - decoder handles partial data - int availableSize = dataToProcess.remaining(); - ByteBuffer decodedData = state.decoder.decode(dataToProcess.duplicate(), availableSize); - - // Track decoded bytes - int decodedBytes = decodedData.remaining(); - state.totalBytesDecoded.addAndGet(decodedBytes); - - // Store any remaining unprocessed data for next iteration - if (dataToProcess.hasRemaining()) { - state.updatePendingBuffer(dataToProcess); - } else { - state.pendingBuffer = null; + // Use the new decodeChunk API which properly handles partial headers + StructuredMessageDecoder.DecodeResult result = state.decoder.decodeChunk(encodedBuffer); + + LOGGER.atInfo() + .addKeyValue("status", result.getStatus()) + .addKeyValue("bytesConsumed", result.getBytesConsumed()) + .addKeyValue("decoderOffset", state.decoder.getMessageOffset()) + .addKeyValue("lastCompleteSegment", state.decoder.getLastCompleteSegmentStart()) + .log("Decode chunk result"); + + switch (result.getStatus()) { + case SUCCESS: + case NEED_MORE_BYTES: + case COMPLETED: + // All three cases update counters and return any decoded payload + // SUCCESS and NEED_MORE_BYTES: partial decode, more data expected + // COMPLETED: decode finished successfully + + long currentLastCompleteSegment = state.decoder.getLastCompleteSegmentStart(); + + // Only update decodedBytesAtLastCompleteSegment when lastCompleteSegmentStart changes + // This indicates that a segment boundary was just crossed + if (state.lastCompleteSegmentStart != currentLastCompleteSegment) { + state.decodedBytesAtLastCompleteSegment = state.decoder.getTotalDecodedPayloadBytes(); + state.lastCompleteSegmentStart = currentLastCompleteSegment; + + LOGGER.atInfo() + .addKeyValue("newSegmentBoundary", currentLastCompleteSegment) + .addKeyValue("decodedBytesAtBoundary", state.decodedBytesAtLastCompleteSegment) + .log("Segment boundary crossed, updated decoded bytes snapshot"); + } + + state.totalEncodedBytesProcessed.set(state.decoder.getMessageOffset()); + state.totalBytesDecoded.set(state.decoder.getTotalDecodedPayloadBytes()); + + if (result.getDecodedPayload() != null && result.getDecodedPayload().hasRemaining()) { + return Flux.just(result.getDecodedPayload()); + } + return Flux.empty(); + + case INVALID: + LOGGER.error("Invalid data during decode: {}", result.getMessage()); + return Flux.error(createRetryableException(state, + "Failed to decode structured message: " + result.getMessage())); + + default: + return Flux.error(new IllegalStateException("Unknown decode status: " + result.getStatus())); } - // Return decoded data if any - if (decodedBytes > 0) { - return Flux.just(decodedData); - } else { - return Flux.empty(); - } } catch (Exception e) { LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); - return Flux.error(e); + return Flux.error(createRetryableException(state, e.getMessage(), e)); } - }).doOnComplete(() -> { - // Finalize when stream completes - try { - state.decoder.finalizeDecoding(); - } catch (IllegalArgumentException e) { - // Expected if we haven't received all data yet (e.g., interrupted download) - LOGGER.verbose("Decoding not finalized - may resume on retry: " + e.getMessage()); + }).onErrorResume(throwable -> { + // Wrap any error with retry offset information + if (throwable instanceof IOException) { + // Check if already has retry offset token + if (throwable.getMessage() != null && throwable.getMessage().contains(RETRY_OFFSET_TOKEN)) { + return Flux.error(throwable); + } } - }); + // Wrap the error with retry offset + return Flux.error(createRetryableException(state, throwable.getMessage(), throwable)); + }).concatWith(Mono.defer(() -> { + // Check on completion if decode is finished - if not, throw with retry offset + if (!state.decoder.isComplete()) { + LOGGER.atInfo() + .addKeyValue("messageOffset", state.decoder.getMessageOffset()) + .addKeyValue("messageLength", state.decoder.getMessageLength()) + .addKeyValue("totalDecodedPayload", state.decoder.getTotalDecodedPayloadBytes()) + .addKeyValue("lastCompleteSegment", state.decoder.getLastCompleteSegmentStart()) + .log("Stream ended but decode not finalized - throwing retryable exception"); + return Mono.error(createRetryableException(state, + "Stream ended prematurely before structured message decoding completed")); + } else { + LOGGER.atInfo() + .addKeyValue("messageOffset", state.decoder.getMessageOffset()) + .addKeyValue("totalDecodedPayload", state.decoder.getTotalDecodedPayloadBytes()) + .log("Stream complete and decode finalized successfully"); + return Mono.empty(); + } + })); + } + + /** + * Creates an IOException with the retry start offset encoded in the message. + * + * @param state The decoder state. + * @param message The error message. + * @return An IOException with retry offset information. + */ + private IOException createRetryableException(DecoderState state, String message) { + return createRetryableException(state, message, null); + } + + /** + * Creates an IOException with the retry start offset encoded in the message. + * + * @param state The decoder state. + * @param message The error message. + * @param cause The original cause, may be null. + * @return An IOException with retry offset information. + */ + private IOException createRetryableException(DecoderState state, String message, Throwable cause) { + long retryOffset = state.decoder.getRetryStartOffset(); + long decodedSoFar = state.decoder.getTotalDecodedPayloadBytes(); + long expectedLength = state.decoder.getMessageLength(); + + // Check if the exception message already has decoder offset information + // If so, prefer lastCompleteSegment from the enriched message + String originalMessage = message != null ? message : ""; + long[] decoderOffsets = parseDecoderOffsets(originalMessage); + if (decoderOffsets != null) { + // Use lastCompleteSegment from the enriched exception as the retry offset + retryOffset = decoderOffsets[1]; // lastCompleteSegment + LOGGER.atInfo() + .addKeyValue("decoderOffset", decoderOffsets[0]) + .addKeyValue("lastCompleteSegment", decoderOffsets[1]) + .log("Parsed decoder offsets from enriched exception"); + } + + // Build message components for clarity + long displayExpected = expectedLength > 0 ? expectedLength : 0; + + String fullMessage = String.format("Incomplete structured message: decoded %d of %d bytes. %s%d. %s", + decodedSoFar, displayExpected, RETRY_OFFSET_TOKEN, retryOffset, originalMessage); + + LOGGER.atInfo() + .addKeyValue("retryOffset", retryOffset) + .addKeyValue("decodedSoFar", decodedSoFar) + .addKeyValue("expectedLength", expectedLength) + .log("Creating retryable exception with offset"); + + if (cause != null) { + return new IOException(fullMessage, cause); + } + return new IOException(fullMessage); } /** @@ -206,7 +379,8 @@ public static class DecoderState { private final long expectedContentLength; private final AtomicLong totalBytesDecoded; private final AtomicLong totalEncodedBytesProcessed; - private ByteBuffer pendingBuffer; + private long decodedBytesAtLastCompleteSegment; + private long lastCompleteSegmentStart; // Tracks the last value to detect changes /** * Creates a new decoder state. @@ -218,36 +392,7 @@ public DecoderState(long expectedContentLength) { this.decoder = new StructuredMessageDecoder(expectedContentLength); this.totalBytesDecoded = new AtomicLong(0); this.totalEncodedBytesProcessed = new AtomicLong(0); - this.pendingBuffer = null; - } - - /** - * Combines pending buffer with new data. - * - * @param newBuffer The new buffer to combine. - * @return Combined buffer. - */ - private ByteBuffer combineWithPending(ByteBuffer newBuffer) { - if (pendingBuffer == null || !pendingBuffer.hasRemaining()) { - return newBuffer.duplicate(); - } - - ByteBuffer combined = ByteBuffer.allocate(pendingBuffer.remaining() + newBuffer.remaining()); - combined.put(pendingBuffer.duplicate()); - combined.put(newBuffer.duplicate()); - combined.flip(); - return combined; - } - - /** - * Updates the pending buffer with remaining data. - * - * @param dataToProcess The buffer with remaining data. - */ - private void updatePendingBuffer(ByteBuffer dataToProcess) { - pendingBuffer = ByteBuffer.allocate(dataToProcess.remaining()); - pendingBuffer.put(dataToProcess); - pendingBuffer.flip(); + this.decodedBytesAtLastCompleteSegment = 0; } /** @@ -268,13 +413,55 @@ public long getTotalEncodedBytesProcessed() { return totalEncodedBytesProcessed.get(); } + /** + * Gets the offset to use for retry requests. + * This uses the decoder's last complete segment boundary to ensure retries + * resume from a valid segment boundary, not mid-segment. + * + * Also resets decoder state to align with the segment boundary. + * + * @return The offset for retry requests (last complete segment boundary). + */ + public long getRetryOffset() { + // Use the decoder's last complete segment start as the retry offset + // This ensures we resume from a segment boundary, not mid-segment + long retryOffset = decoder.getLastCompleteSegmentStart(); + long decoderOffsetBefore = decoder.getMessageOffset(); + long totalProcessedBefore = totalEncodedBytesProcessed.get(); + + LOGGER.atInfo() + .addKeyValue("retryOffset", retryOffset) + .addKeyValue("decoderOffsetBefore", decoderOffsetBefore) + .addKeyValue("totalProcessedBefore", totalProcessedBefore) + .log("Computing retry offset"); + + // Reset decoder to the last complete segment boundary + // This ensures messageOffset and segment state match the retry offset + decoder.resetToLastCompleteSegment(); + + // Reset totalEncodedBytesProcessed to match the retry offset + // This ensures absoluteStartOfCombined calculation is correct for retry data + totalEncodedBytesProcessed.set(retryOffset); + + // Reset totalBytesDecoded to the snapshot at last complete segment + // This ensures decoded byte counting is correct for retry + totalBytesDecoded.set(decodedBytesAtLastCompleteSegment); + + LOGGER.atInfo() + .addKeyValue("retryOffset", retryOffset) + .addKeyValue("totalProcessedAfter", totalEncodedBytesProcessed.get()) + .addKeyValue("totalDecodedAfter", totalBytesDecoded.get()) + .log("Retry offset calculated (last complete segment boundary)"); + return retryOffset; + } + /** * Checks if the decoder has finalized. * * @return true if finalized, false otherwise. */ public boolean isFinalized() { - return totalEncodedBytesProcessed.get() >= expectedContentLength; + return decoder.isComplete(); } } diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java new file mode 100644 index 000000000000..2c08e40b63a5 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.structuredmessage; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.concurrent.ThreadLocalRandom; + +import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for StructuredMessageDecoder with focus on handling partial headers + * and segment splits across chunks. + */ +public class StructuredMessageDecoderTests { + + @Test + public void readsCompleteMessageInSingleChunk() throws IOException { + // Test: Complete message in a single ByteBuffer should decode fully + byte[] originalData = new byte[1024]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteBuffer result = decoder.decode(encodedData); + + assertNotNull(result); + byte[] decodedData = new byte[result.remaining()]; + result.get(decodedData); + assertArrayEquals(originalData, decodedData); + assertTrue(decoder.isComplete()); + } + + @Test + public void readsMessageSplitHeaderAcrossChunks() throws IOException { + // Test: Feed header bytes split across two buffers + byte[] originalData = new byte[256]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + byte[] encodedBytes = new byte[encodedLength]; + encodedData.get(encodedBytes); + + // Split at byte 7 (mid-header, header is 13 bytes) + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, 7); + ByteBuffer chunk2 = ByteBuffer.wrap(encodedBytes, 7, encodedLength - 7); + chunk1.order(ByteOrder.LITTLE_ENDIAN); + chunk2.order(ByteOrder.LITTLE_ENDIAN); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + // First chunk should not throw, should wait for more bytes + StructuredMessageDecoder.DecodeResult result1 = decoder.decodeChunk(chunk1); + assertEquals(StructuredMessageDecoder.DecodeStatus.NEED_MORE_BYTES, result1.getStatus()); + assertFalse(decoder.isComplete()); + + // Second chunk should complete the decode + StructuredMessageDecoder.DecodeResult result2 = decoder.decodeChunk(chunk2); + assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, result2.getStatus()); + assertTrue(decoder.isComplete()); + } + + @Test + public void readsSegmentHeaderSplitAcrossChunks() throws IOException { + // Test: Split the 10-byte segment header across two chunks + byte[] originalData = new byte[512]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + byte[] encodedBytes = new byte[encodedLength]; + encodedData.get(encodedBytes); + + // Split after message header (13 bytes) + 5 bytes into first segment header + // Segment header is 10 bytes, so split at byte 18 (mid-segment-header) + int splitPoint = 18; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, splitPoint); + ByteBuffer chunk2 = ByteBuffer.wrap(encodedBytes, splitPoint, encodedLength - splitPoint); + chunk1.order(ByteOrder.LITTLE_ENDIAN); + chunk2.order(ByteOrder.LITTLE_ENDIAN); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + // First chunk should parse header but wait for segment header completion + StructuredMessageDecoder.DecodeResult result1 = decoder.decodeChunk(chunk1); + assertEquals(StructuredMessageDecoder.DecodeStatus.NEED_MORE_BYTES, result1.getStatus()); + assertFalse(decoder.isComplete()); + + // Second chunk should complete + StructuredMessageDecoder.DecodeResult result2 = decoder.decodeChunk(chunk2); + assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, result2.getStatus()); + assertTrue(decoder.isComplete()); + } + + @Test + public void handlesZeroLengthSegment() throws IOException { + // Test: Zero-length segment should decode correctly + // Note: Zero-length segments are valid in the format + byte[] originalData = new byte[0]; + + // For zero-length data, encoder behavior varies - let's test with minimal data + byte[] minimalData = new byte[1]; + ThreadLocalRandom.current().nextBytes(minimalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(minimalData.length, 1024, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(minimalData)); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteBuffer result = decoder.decode(encodedData); + + assertNotNull(result); + assertEquals(1, result.remaining()); + assertTrue(decoder.isComplete()); + } + + @Test + public void tracksLastCompleteSegmentCorrectly() throws IOException { + // Test: Verify lastCompleteSegmentStart is updated correctly after each segment + byte[] originalData = new byte[1024]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + // Initially lastCompleteSegmentStart should be 0 + assertEquals(0, decoder.getLastCompleteSegmentStart()); + + // Decode the entire message + decoder.decode(encodedData); + + // After complete decode, lastCompleteSegmentStart should point to end of last segment + // (before message footer, if any) + assertTrue(decoder.isComplete()); + // lastCompleteSegmentStart should be <= messageOffset + assertTrue(decoder.getLastCompleteSegmentStart() <= decoder.getMessageOffset()); + // And should be > 0 (we processed at least one segment) + assertTrue(decoder.getLastCompleteSegmentStart() > 0); + } + + @Test + public void resetToLastCompleteSegmentWorks() throws IOException { + // Test: Verify reset functionality for smart retry + byte[] originalData = new byte[512]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + byte[] encodedBytes = new byte[encodedLength]; + encodedData.get(encodedBytes); + + // Parse first segment completely, then simulate interruption + // First segment ends after: header(13) + segment_header(10) + content(256) + crc(8) = 287 + int firstSegmentEnd = V1_HEADER_LENGTH + V1_SEGMENT_HEADER_LENGTH + 256 + CRC64_LENGTH; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, firstSegmentEnd + 5); // 5 bytes into second segment header + chunk1.order(ByteOrder.LITTLE_ENDIAN); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + decoder.decodeChunk(chunk1); + + // lastCompleteSegmentStart should be at end of first segment + long lastComplete = decoder.getLastCompleteSegmentStart(); + assertTrue(lastComplete > 0); + assertEquals(firstSegmentEnd, lastComplete); + + // Reset to last complete segment + decoder.resetToLastCompleteSegment(); + assertEquals(lastComplete, decoder.getMessageOffset()); + } + + @Test + public void multipleChunksDecode() throws IOException { + // Test: Decode message across multiple small chunks + byte[] originalData = new byte[256]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + byte[] encodedBytes = new byte[encodedLength]; + encodedData.get(encodedBytes); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + // Feed in chunks of 32 bytes + int chunkSize = 32; + java.io.ByteArrayOutputStream output = new java.io.ByteArrayOutputStream(); + + for (int offset = 0; offset < encodedLength; offset += chunkSize) { + int len = Math.min(chunkSize, encodedLength - offset); + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); + chunk.order(ByteOrder.LITTLE_ENDIAN); + + StructuredMessageDecoder.DecodeResult result = decoder.decodeChunk(chunk); + if (result.getDecodedPayload() != null && result.getDecodedPayload().hasRemaining()) { + byte[] decoded = new byte[result.getDecodedPayload().remaining()]; + result.getDecodedPayload().get(decoded); + output.write(decoded, 0, decoded.length); + } + + if (result.getStatus() == StructuredMessageDecoder.DecodeStatus.COMPLETED) { + break; + } + } + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, output.toByteArray()); + } + + @Test + public void decodeWithNoCrc() throws IOException { + // Test: Decode message without CRC (NONE flag) + byte[] originalData = new byte[256]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.NONE); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteBuffer result = decoder.decode(encodedData); + + assertNotNull(result); + byte[] decodedData = new byte[result.remaining()]; + result.get(decodedData); + assertArrayEquals(originalData, decodedData); + assertTrue(decoder.isComplete()); + } + + @Test + public void handlesZeroLengthBuffer() throws IOException { + // Test: Decoder should handle zero-length buffers gracefully + byte[] originalData = new byte[256]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + int encodedLength = encodedData.remaining(); + byte[] encodedBytes = new byte[encodedLength]; + encodedData.get(encodedBytes); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + // Feed zero-length buffer first + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + StructuredMessageDecoder.DecodeResult result1 = decoder.decodeChunk(emptyBuffer); + assertEquals(StructuredMessageDecoder.DecodeStatus.NEED_MORE_BYTES, result1.getStatus()); + assertEquals(0, result1.getBytesConsumed()); + + // Then feed actual data + ByteBuffer dataBuffer = ByteBuffer.wrap(encodedBytes); + dataBuffer.order(ByteOrder.LITTLE_ENDIAN); + StructuredMessageDecoder.DecodeResult result2 = decoder.decodeChunk(dataBuffer); + assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, result2.getStatus()); + assertTrue(decoder.isComplete()); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java new file mode 100644 index 000000000000..dbaaf5c41550 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Unit tests for StorageContentValidationDecoderPolicy. + */ +public class StorageContentValidationDecoderPolicyTest { + + @Test + public void parseRetryStartOffsetFromValidMessage() { + String message + = "Incomplete structured message: decoded 512 of 1081 bytes. RETRY-START-OFFSET=287. Stream ended"; + long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); + assertEquals(287, offset); + } + + @Test + public void parseRetryStartOffsetFromMessageWithLargeOffset() { + String message = "RETRY-START-OFFSET=9999999999"; + long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); + assertEquals(9999999999L, offset); + } + + @Test + public void parseRetryStartOffsetFromMessageWithZeroOffset() { + String message = "Some error. RETRY-START-OFFSET=0. Details"; + long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); + assertEquals(0, offset); + } + + @Test + public void parseRetryStartOffsetReturnsNegativeOneForNullMessage() { + long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(null); + assertEquals(-1, offset); + } + + @Test + public void parseRetryStartOffsetReturnsNegativeOneForMissingToken() { + String message = "Some error without retry offset"; + long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); + assertEquals(-1, offset); + } + + @Test + public void parseRetryStartOffsetReturnsNegativeOneForEmptyMessage() { + long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(""); + assertEquals(-1, offset); + } + + @Test + public void parseRetryStartOffsetReturnsNegativeOneForMalformedToken() { + String message = "RETRY-START-OFFSET=abc"; + long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); + assertEquals(-1, offset); + } + + @Test + public void parseDecoderOffsetsFromEnrichedMessage() { + String message = "Invalid segment size [decoderOffset=523,lastCompleteSegment=287]"; + long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + assertArrayEquals(new long[] { 523, 287 }, offsets); + } + + @Test + public void parseDecoderOffsetsWithZeroValues() { + String message = "Header error [decoderOffset=0,lastCompleteSegment=0]"; + long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + assertArrayEquals(new long[] { 0, 0 }, offsets); + } + + @Test + public void parseDecoderOffsetsWithLargeValues() { + String message = "Error [decoderOffset=9999999999,lastCompleteSegment=8888888888]"; + long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + assertArrayEquals(new long[] { 9999999999L, 8888888888L }, offsets); + } + + @Test + public void parseDecoderOffsetsReturnsNullForMissingPattern() { + String message = "Error without decoder offset information"; + long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + assertNull(offsets); + } + + @Test + public void parseDecoderOffsetsReturnsNullForNullMessage() { + long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(null); + assertNull(offsets); + } + + @Test + public void parseDecoderOffsetsReturnsNullForMalformedPattern() { + String message = "[decoderOffset=abc,lastCompleteSegment=xyz]"; + long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + assertNull(offsets); + } +} From 657f9873cbe7955a597119dc3dbaa6ee82ea1a7c Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Mon, 15 Dec 2025 10:50:29 +0530 Subject: [PATCH 05/31] smart retry changes --- .../blob/specialized/BlobAsyncClientBase.java | 113 ++++++++++-------- .../blob/BlobMessageDecoderDownloadTests.java | 79 ++++++++++++ .../StructuredMessageDecoder.java | 21 ++++ ...StorageContentValidationDecoderPolicy.java | 105 ++++++++++++---- .../policy/MockPartialResponsePolicy.java | 24 +++- 5 files changed, 265 insertions(+), 77 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index f34f0f5f59ab..12bbc697002a 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -117,6 +117,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -1359,6 +1360,18 @@ Mono downloadStreamWithResponseInternal(BlobRange ran finalCount = finalRange.getCount(); } + AtomicReference decoderStateRef + = new AtomicReference<>(); + if (contentValidationOptions != null + && contentValidationOptions.isStructuredMessageValidationEnabled()) { + Object decoderStateObj + = firstRangeContext.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) + .orElse(null); + if (decoderStateObj instanceof StorageContentValidationDecoderPolicy.DecoderState) { + decoderStateRef.set((StorageContentValidationDecoderPolicy.DecoderState) decoderStateObj); + } + } + // The resume function takes throwable and offset at the destination. // I.e. offset is relative to the starting point. BiFunction> onDownloadErrorResume = (throwable, offset) -> { @@ -1367,18 +1380,28 @@ Mono downloadStreamWithResponseInternal(BlobRange ran } long newCount = finalCount - offset; + StorageContentValidationDecoderPolicy.DecoderState decoderState = null; + long expectedEncodedLength = finalCount; + long encodedProgress = offset; + + if (contentValidationOptions != null + && contentValidationOptions.isStructuredMessageValidationEnabled()) { + decoderState = decoderStateRef.get(); + + if (decoderState == null) { + Object decoderStateObj + = firstRangeContext.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) + .orElse(null); - /* - * It's possible that the network stream will throw an error after emitting all data but before - * completing. Issuing a retry at this stage would leave the download in a bad state with - * incorrect count and offset values. Because we have read the intended amount of data, we can - * ignore the error at the end of the stream. - */ - if (newCount == 0) { - LOGGER.warning("Exception encountered in ReliableDownload after all data read from the network " - + "but before stream signaled completion. Returning success as all data was downloaded. " - + "Exception message: " + throwable.getMessage()); - return Mono.empty(); + if (decoderStateObj instanceof StorageContentValidationDecoderPolicy.DecoderState) { + decoderState = (StorageContentValidationDecoderPolicy.DecoderState) decoderStateObj; + } + } + + if (decoderState != null) { + expectedEncodedLength = decoderState.getExpectedContentLength(); + encodedProgress = decoderState.getTotalEncodedBytesProcessed(); + } } try { @@ -1390,53 +1413,38 @@ Mono downloadStreamWithResponseInternal(BlobRange ran // based on the encoded bytes processed, not the decoded bytes if (contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled()) { - // Get the decoder state to determine how many encoded bytes were processed - Object decoderStateObj - = firstRangeContext.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) - .orElse(null); - - if (decoderStateObj instanceof StorageContentValidationDecoderPolicy.DecoderState) { - StorageContentValidationDecoderPolicy.DecoderState decoderState - = (StorageContentValidationDecoderPolicy.DecoderState) decoderStateObj; + long retryStartOffset = -1; - // Use getRetryOffset() to get the correct offset for retry - // This accounts for pending bytes that have been received but not yet consumed - long encodedOffset = decoderState.getRetryOffset(); - long remainingCount = finalCount - encodedOffset; - retryRange = new BlobRange(initialOffset + encodedOffset, remainingCount); + // First try to use decoder state (authoritative) + if (decoderState != null) { + // Always rewind decoder to last validated boundary before retrying. + retryStartOffset = decoderState.resetForRetry(); - LOGGER.info( - "Structured message smart retry: resuming from offset {} (initial={}, encoded={})", - initialOffset + encodedOffset, initialOffset, encodedOffset); - - // Preserve the decoder state for the retry retryContext = retryContext .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); - } else { - // No decoder state available, try to parse retry offset from exception message - // The exception message contains RETRY-START-OFFSET= token - long retryStartOffset = StorageContentValidationDecoderPolicy + decoderStateRef.set(decoderState); + } + + // If no decoder state or no retry offset from state, fall back to parsed token or offset. + if (retryStartOffset < 0) { + retryStartOffset = StorageContentValidationDecoderPolicy .parseRetryStartOffset(throwable.getMessage()); - if (retryStartOffset >= 0) { - long remainingCount = finalCount - retryStartOffset; - // Validate remainingCount to avoid negative values - if (remainingCount <= 0) { - LOGGER.warning("Retry offset {} exceeds finalCount {}, using fallback", - retryStartOffset, finalCount); - retryRange = new BlobRange(initialOffset + offset, newCount); - } else { - retryRange = new BlobRange(initialOffset + retryStartOffset, remainingCount); - - LOGGER.info( - "Structured message smart retry from exception: resuming from offset {} " - + "(initial={}, parsed={})", - initialOffset + retryStartOffset, initialOffset, retryStartOffset); - } - } else { - // Fallback to normal retry logic if no offset found - retryRange = new BlobRange(initialOffset + offset, newCount); - } } + if (retryStartOffset < 0) { + retryStartOffset = offset; + } + + long remainingCount = expectedEncodedLength - retryStartOffset; + if (remainingCount < 0) { + remainingCount = expectedEncodedLength - offset; + retryStartOffset = offset; + } + + retryRange = new BlobRange(initialOffset + retryStartOffset, remainingCount); + + LOGGER.info( + "Structured message smart retry: resuming from offset {} (initial={}, encoded={}, remaining={})", + initialOffset + retryStartOffset, initialOffset, retryStartOffset, remainingCount); } else { // For non-structured downloads, use smart retry from the interrupted offset retryRange = new BlobRange(initialOffset + offset, newCount); @@ -1451,6 +1459,7 @@ Mono downloadStreamWithResponseInternal(BlobRange ran // Structured message decoding is now handled by StructuredMessageDecoderPolicy return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions); }); + } private Mono downloadRange(BlobRange range, BlobRequestConditions requestConditions, String eTag, diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index a2b9f5283895..47e9e9023f0f 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -5,6 +5,7 @@ import com.azure.core.test.utils.TestUtils; import com.azure.core.util.FluxUtil; +import com.azure.core.util.Context; import com.azure.storage.blob.models.BlobRange; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.DownloadRetryOptions; @@ -20,6 +21,7 @@ import reactor.test.StepVerifier; import java.io.IOException; +import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.List; @@ -285,6 +287,83 @@ public void downloadStreamWithResponseContentValidationSmartRetry() throws IOExc } } + @Test + public void downloadStreamWithResponseContentValidationSmartRetryVariousSizes() throws IOException { + int[] dataSizes = new int[] { Constants.KB, 1500, 3 * Constants.KB + 128 }; + int[] segmentSizes = new int[] { 512, 700, 1024 }; + + for (int i = 0; i < dataSizes.length; i++) { + byte[] randomData = getRandomByteArray(dataSizes[i]); + int segmentSize = segmentSizes[i % segmentSizes.length]; + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + + // Create a policy that will simulate 2 network interruptions for each run + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(2); + + // Upload the encoded data + bc.upload(input, null, true).block(); + + // Create a download client with both the mock policy AND the decoder policy + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + bc.getBlobUrl(), mockPolicy, decoderPolicy); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier.create(downloadClient + .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + TestUtils.assertArraysEqual(r, randomData); + }).verifyComplete(); + + assertEquals(0, mockPolicy.getTriesRemaining()); + List rangeHeaders = mockPolicy.getRangeHeaders(); + assertTrue(rangeHeaders.size() > 0, "Expected range headers for retries"); + assertTrue(rangeHeaders.get(0).startsWith("bytes=0-"), "First request should start from offset 0"); + } + } + + @Test + public void downloadStreamWithResponseContentValidationSmartRetrySync() throws IOException { + byte[] randomData = getRandomByteArray(Constants.KB); + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + + Flux input = Flux.just(encodedData); + bc.upload(input, null, true).block(); + + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3); + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), + mockPolicy, decoderPolicy); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Context ctx = new Context(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) + .addData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY, validationOptions); + + downloadClient.downloadStreamWithResponse(out, null, retryOptions, null, false, null, ctx); + + TestUtils.assertArraysEqual(out.toByteArray(), randomData); + + assertEquals(0, mockPolicy.getTriesRemaining()); + List rangeHeaders = mockPolicy.getRangeHeaders(); + assertTrue(rangeHeaders.size() > 0, "Expected range headers for retries"); + assertTrue(rangeHeaders.get(0).startsWith("bytes=0-"), "First request should start from offset 0"); + } + @Test public void downloadStreamWithResponseContentValidationSmartRetryMultipleSegments() throws IOException { // Test smart retry with multiple segments to ensure checksum validation diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 6534dd0ce38d..29d26702555b 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -332,6 +332,20 @@ public boolean isHeaderRead() { return messageLength != -1; } + /** + * Gets the number of encoded bytes that have been seen but not yet fully + * processed by the decoder (pending bytes). + * + *

This is used by smart-retry logic to determine the absolute encoded + * offset that a retry request should start from while still preserving + * the decoder's buffered state.

+ * + * @return The number of pending encoded bytes buffered by the decoder. + */ + public int getPendingEncodedByteCount() { + return pendingBytes.size(); + } + /** * Reads the message header if we have enough bytes. * @@ -611,6 +625,13 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { // Step 2: Process segments while (messageOffset < messageLength) { + // If all segments are done, proceed to message footer before attempting any new segment header. + if (currentSegmentNumber == numSegments && currentSegmentContentOffset == currentSegmentContentLength) { + if (!tryReadMessageFooter(buffer)) { + break; + } + } + // Read segment header if needed if (currentSegmentContentOffset == currentSegmentContentLength) { if (!tryReadSegmentHeader(buffer)) { diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 7103a3a11545..4f11f4cb93bc 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -100,6 +100,19 @@ public static long[] parseDecoderOffsets(String message) { return null; } + /** + * Attempts to extract the decoder state from a decoded response instance. + * + * @param response The HTTP response returned from the pipeline. + * @return The decoder state if present, otherwise null. + */ + public static DecoderState tryGetDecoderState(HttpResponse response) { + if (response instanceof DecodedResponse) { + return ((DecodedResponse) response).getDecoderState(); + } + return null; + } + @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { // Check if structured message decoding is enabled for this request @@ -147,6 +160,14 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN */ private Flux decodeStream(Flux encodedFlux, DecoderState state) { return encodedFlux.concatMap(encodedBuffer -> { + // If decoding already completed, drop any subsequent buffers (can happen with late keep-alive frames). + if (state.decoder.isComplete()) { + LOGGER.atVerbose() + .addKeyValue("bufferLength", encodedBuffer == null ? "null" : encodedBuffer.remaining()) + .log("Decoder already completed; ignoring extra buffer"); + return Flux.empty(); + } + // Skip empty buffers that may be emitted by reactor-netty if (encodedBuffer == null || !encodedBuffer.hasRemaining()) { LOGGER.atVerbose() @@ -195,7 +216,9 @@ private Flux decodeStream(Flux encodedFlux, DecoderState .log("Segment boundary crossed, updated decoded bytes snapshot"); } - state.totalEncodedBytesProcessed.set(state.decoder.getMessageOffset()); + long encodedProgress + = state.decoder.getMessageOffset() + state.decoder.getPendingEncodedByteCount(); + state.totalEncodedBytesProcessed.set(encodedProgress); state.totalBytesDecoded.set(state.decoder.getTotalDecodedPayloadBytes()); if (result.getDecodedPayload() != null && result.getDecodedPayload().hasRemaining()) { @@ -217,6 +240,11 @@ private Flux decodeStream(Flux encodedFlux, DecoderState return Flux.error(createRetryableException(state, e.getMessage(), e)); } }).onErrorResume(throwable -> { + // If we've already completed decoding, ignore any downstream errors (mirror .NET behavior). + if (state.decoder.isComplete()) { + LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); + return Flux.empty(); + } // Wrap any error with retry offset information if (throwable instanceof IOException) { // Check if already has retry offset token @@ -267,7 +295,7 @@ private IOException createRetryableException(DecoderState state, String message) * @return An IOException with retry offset information. */ private IOException createRetryableException(DecoderState state, String message, Throwable cause) { - long retryOffset = state.decoder.getRetryStartOffset(); + long retryOffset = state.getRetryOffset(); long decodedSoFar = state.decoder.getTotalDecodedPayloadBytes(); long expectedLength = state.decoder.getMessageLength(); @@ -413,6 +441,15 @@ public long getTotalEncodedBytesProcessed() { return totalEncodedBytesProcessed.get(); } + /** + * Gets the expected encoded content length associated with this decoder state. + * + * @return The expected encoded content length. + */ + public long getExpectedContentLength() { + return expectedContentLength; + } + /** * Gets the offset to use for retry requests. * This uses the decoder's last complete segment boundary to ensure retries @@ -425,33 +462,49 @@ public long getTotalEncodedBytesProcessed() { public long getRetryOffset() { // Use the decoder's last complete segment start as the retry offset // This ensures we resume from a segment boundary, not mid-segment - long retryOffset = decoder.getLastCompleteSegmentStart(); - long decoderOffsetBefore = decoder.getMessageOffset(); - long totalProcessedBefore = totalEncodedBytesProcessed.get(); + long retryOffset = decoder.getRetryStartOffset(); + long lastCompleteSegmentOffset = decoder.getLastCompleteSegmentStart(); LOGGER.atInfo() - .addKeyValue("retryOffset", retryOffset) - .addKeyValue("decoderOffsetBefore", decoderOffsetBefore) - .addKeyValue("totalProcessedBefore", totalProcessedBefore) - .log("Computing retry offset"); + .addKeyValue("decoderOffset", decoder.getMessageOffset()) + .addKeyValue("pendingBytes", decoder.getPendingEncodedByteCount()) + .addKeyValue("lastCompleteSegment", lastCompleteSegmentOffset) + .log("Computed smart-retry offset from decoder state"); + return retryOffset; + } - // Reset decoder to the last complete segment boundary - // This ensures messageOffset and segment state match the retry offset - decoder.resetToLastCompleteSegment(); + /** + * Prepares the decoder state for a retry by rewinding the decoder to the last complete segment + * boundary and resetting the accounting counters to that point. This mirrors the behavior of + * the cryptography smart-retry implementation which always replays from a validated boundary. + * + * @return The retry start offset (encoded byte position) that the next request should use. + */ + public long prepareForRetry() { + return resetForRetry(); + } - // Reset totalEncodedBytesProcessed to match the retry offset - // This ensures absoluteStartOfCombined calculation is correct for retry data - totalEncodedBytesProcessed.set(retryOffset); + /** + * Resets decoder and counters to the last validated segment boundary and returns the retry offset. + * + * @return retry offset (encoded boundary). + */ + public long resetForRetry() { + long retryOffset = decoder.getRetryStartOffset(); + + decoder.resetToLastCompleteSegment(); - // Reset totalBytesDecoded to the snapshot at last complete segment - // This ensures decoded byte counting is correct for retry - totalBytesDecoded.set(decodedBytesAtLastCompleteSegment); + // Align counters to the boundary we will resume from so subsequent progress tracking is consistent. + long boundaryDecodedBytes = decodedBytesAtLastCompleteSegment; + totalBytesDecoded.set(boundaryDecodedBytes); + totalEncodedBytesProcessed.set(decoder.getMessageOffset() + decoder.getPendingEncodedByteCount()); LOGGER.atInfo() .addKeyValue("retryOffset", retryOffset) - .addKeyValue("totalProcessedAfter", totalEncodedBytesProcessed.get()) - .addKeyValue("totalDecodedAfter", totalBytesDecoded.get()) - .log("Retry offset calculated (last complete segment boundary)"); + .addKeyValue("decoderOffset", decoder.getMessageOffset()) + .addKeyValue("decodedBytesAtBoundary", boundaryDecodedBytes) + .log("Prepared decoder state for smart retry"); + return retryOffset; } @@ -463,6 +516,16 @@ public long getRetryOffset() { public boolean isFinalized() { return decoder.isComplete(); } + + /** + * Gets the decoded payload bytes accounted for at the last complete segment boundary. + * This is used to correlate decoder progress with reliable download offsets. + * + * @return The decoded byte count at the last segment boundary. + */ + public long getDecodedBytesAtLastCompleteSegment() { + return decodedBytesAtLastCompleteSegment; + } } /** diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index 1f1109d8b38b..9dff1fcc2be5 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -19,7 +19,8 @@ import java.util.List; public class MockPartialResponsePolicy implements HttpPipelinePolicy { - static final HttpHeaderName RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); + static final HttpHeaderName X_MS_RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); + static final HttpHeaderName RANGE_HEADER = HttpHeaderName.RANGE; private int tries; private final List rangeHeaders = new ArrayList<>(); @@ -31,10 +32,19 @@ public MockPartialResponsePolicy(int tries) { public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { return next.process().flatMap(response -> { HttpHeader rangeHttpHeader = response.getRequest().getHeaders().get(RANGE_HEADER); - String rangeHeader = rangeHttpHeader == null ? null : rangeHttpHeader.getValue(); + HttpHeader xMsRangeHttpHeader = response.getRequest().getHeaders().get(X_MS_RANGE_HEADER); - if (rangeHeader != null && rangeHeader.startsWith("bytes=")) { - rangeHeaders.add(rangeHeader); + if (rangeHttpHeader != null && rangeHttpHeader.getValue().startsWith("bytes=")) { + rangeHeaders.add(rangeHttpHeader.getValue()); + } + + if (xMsRangeHttpHeader != null && xMsRangeHttpHeader.getValue().startsWith("bytes=")) { + String xMsRangeValue = xMsRangeHttpHeader.getValue(); + + // Avoid recording the same value twice if both Range and x-ms-range were set to the same string + if (rangeHttpHeader == null || !xMsRangeValue.equals(rangeHttpHeader.getValue())) { + rangeHeaders.add(xMsRangeValue); + } } if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || this.tries == 0) { @@ -42,6 +52,12 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN } else { this.tries -= 1; return response.getBody().collectList().flatMap(bodyBuffers -> { + if (bodyBuffers.isEmpty()) { + // If no body was returned, don't attempt to slice a partial response. Simply propagate + // the original response to avoid test failures when the service unexpectedly returns an + // empty body (for example, after a retry on the underlying transport). + return Mono.just(response); + } ByteBuffer firstBuffer = bodyBuffers.get(0); byte firstByte = firstBuffer.get(); From f7939e72d76ad57aa46b6c05291cfb7512bbe58c Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Wed, 17 Dec 2025 19:01:09 +0530 Subject: [PATCH 06/31] smart retry changes --- .../implementation/util/BuilderHelper.java | 7 +- .../blob/specialized/BlobAsyncClientBase.java | 70 +-- .../blob/BlobMessageDecoderDownloadTests.java | 420 ++++++------------ .../common/implementation/Constants.java | 6 + ...StorageContentValidationDecoderPolicy.java | 168 +++++-- .../policy/MockDownloadHttpResponse.java | 7 +- .../policy/MockPartialResponsePolicy.java | 136 ++++-- 7 files changed, 427 insertions(+), 387 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index 79276b6582b9..f41d7c0dd4e7 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -138,7 +138,12 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare HttpPolicyProviders.addAfterRetryPolicies(policies); - policies.add(new StorageContentValidationDecoderPolicy()); + // Only add the structured message decoder once; allow callers to inject their own for ordering with test + // policies (e.g., fault injection before decoding). + boolean hasDecoder = policies.stream().anyMatch(p -> p instanceof StorageContentValidationDecoderPolicy); + if (!hasDecoder) { + policies.add(new StorageContentValidationDecoderPolicy()); + } policies.add(getResponseValidationPolicy()); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 12bbc697002a..a34565529796 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1331,11 +1331,14 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); + AtomicReference decoderStateRef = new AtomicReference<>(); + // Add structured message decoding context if enabled final Context firstRangeContext; if (contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled()) { firstRangeContext = initialContext.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) - .addData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY, contentValidationOptions); + .addData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY, contentValidationOptions) + .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY, decoderStateRef); } else { firstRangeContext = initialContext; } @@ -1360,18 +1363,6 @@ Mono downloadStreamWithResponseInternal(BlobRange ran finalCount = finalRange.getCount(); } - AtomicReference decoderStateRef - = new AtomicReference<>(); - if (contentValidationOptions != null - && contentValidationOptions.isStructuredMessageValidationEnabled()) { - Object decoderStateObj - = firstRangeContext.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) - .orElse(null); - if (decoderStateObj instanceof StorageContentValidationDecoderPolicy.DecoderState) { - decoderStateRef.set((StorageContentValidationDecoderPolicy.DecoderState) decoderStateObj); - } - } - // The resume function takes throwable and offset at the destination. // I.e. offset is relative to the starting point. BiFunction> onDownloadErrorResume = (throwable, offset) -> { @@ -1383,6 +1374,8 @@ Mono downloadStreamWithResponseInternal(BlobRange ran StorageContentValidationDecoderPolicy.DecoderState decoderState = null; long expectedEncodedLength = finalCount; long encodedProgress = offset; + long retryStartOffset = -1; + boolean noBytesEmitted = offset == 0; if (contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled()) { @@ -1398,12 +1391,37 @@ Mono downloadStreamWithResponseInternal(BlobRange ran } } + // If the decoder has already finalized, discard it and restart from the beginning. + if (decoderState != null && decoderState.isFinalized()) { + decoderState = null; + } + if (decoderState != null) { expectedEncodedLength = decoderState.getExpectedContentLength(); encodedProgress = decoderState.getTotalEncodedBytesProcessed(); + + long boundaryDecoded = decoderState.getDecodedBytesAtLastCompleteSegment(); + if (offset < boundaryDecoded || noBytesEmitted) { + // We haven't emitted the bytes represented by this decoder boundary; restart clean. + decoderState = null; + } else { + long bytesToSkip = Math.max(0, offset - boundaryDecoded); + decoderState.setDecodedBytesToSkip(bytesToSkip); + + // Always rewind decoder to last validated boundary before retrying. + retryStartOffset = decoderState.resetForRetry(); + } } } + if (decoderState == null) { + // No decoder state available (likely failed before policy captured it) or no bytes emitted; + // restart from beginning to avoid skipping data. + retryStartOffset = noBytesEmitted ? 0 : -1; + encodedProgress = noBytesEmitted ? 0 : encodedProgress; + decoderStateRef.set(null); + } + try { // For retry context, preserve decoder state if structured message validation is enabled Context retryContext = firstRangeContext; @@ -1413,25 +1431,23 @@ Mono downloadStreamWithResponseInternal(BlobRange ran // based on the encoded bytes processed, not the decoded bytes if (contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled()) { - long retryStartOffset = -1; - - // First try to use decoder state (authoritative) - if (decoderState != null) { - // Always rewind decoder to last validated boundary before retrying. - retryStartOffset = decoderState.resetForRetry(); - - retryContext = retryContext - .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); - decoderStateRef.set(decoderState); - } - // If no decoder state or no retry offset from state, fall back to parsed token or offset. - if (retryStartOffset < 0) { + if (retryStartOffset < 0 && !noBytesEmitted) { retryStartOffset = StorageContentValidationDecoderPolicy .parseRetryStartOffset(throwable.getMessage()); } if (retryStartOffset < 0) { - retryStartOffset = offset; + retryStartOffset = noBytesEmitted ? 0 : offset; + } + + if (decoderState != null) { + retryContext = retryContext + .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); + decoderStateRef.set(decoderState); + } else { + // Ensure we don't carry a stale decoder state into a full restart. + retryContext = retryContext + .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, null); } long remainingCount = expectedEncodedLength - retryStartOffset; diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 47e9e9023f0f..90c7235aa833 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -5,7 +5,6 @@ import com.azure.core.test.utils.TestUtils; import com.azure.core.util.FluxUtil; -import com.azure.core.util.Context; import com.azure.storage.blob.models.BlobRange; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.DownloadRetryOptions; @@ -17,13 +16,13 @@ import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import reactor.core.publisher.Flux; import reactor.test.StepVerifier; import java.io.IOException; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; -import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -46,9 +45,9 @@ public void setup() { @Test public void downloadStreamWithResponseContentValidation() throws IOException { - byte[] randomData = getRandomByteArray(Constants.KB); + byte[] randomData = getRandomByteArray(4 * Constants.KB); StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + = new StructuredMessageEncoder(randomData.length, Constants.KB, StructuredMessageFlags.STORAGE_CRC64); ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); Flux input = Flux.just(encodedData); @@ -67,12 +66,9 @@ public void downloadStreamWithResponseContentValidation() throws IOException { @Test public void downloadStreamWithResponseContentValidationRange() throws IOException { - // Note: Range downloads are not compatible with structured message validation - // because you need the complete encoded message for validation. - // This test verifies that range downloads work without validation. - byte[] randomData = getRandomByteArray(Constants.KB); + byte[] randomData = getRandomByteArray(4 * Constants.KB); StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + = new StructuredMessageEncoder(randomData.length, Constants.KB, StructuredMessageFlags.STORAGE_CRC64); ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); Flux input = Flux.just(encodedData); @@ -84,387 +80,219 @@ public void downloadStreamWithResponseContentValidationRange() throws IOExceptio .then( bc.downloadStreamWithResponse(range, (DownloadRetryOptions) null, (BlobRequestConditions) null, false)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - assertNotNull(r); - // Should get exactly 512 bytes of encoded data - assertEquals(512, r.length); - }).verifyComplete(); + assertNotNull(r); + // Should get exactly 512 bytes of encoded data + assertEquals(512, r.length); + }).verifyComplete(); } @Test - public void downloadStreamWithResponseContentValidationLargeBlob() throws IOException { - // Test with larger data to verify chunking works correctly - byte[] randomData = getRandomByteArray(5 * Constants.KB); + public void uninterruptedStreamWithStructuredMessageDecoding() throws IOException { + // Test: Verify that structured message decoding works correctly without any interruptions + // This mirrors the .NET test: UninterruptedStream + byte[] randomData = getRandomByteArray(4 * Constants.KB); StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 1024, StructuredMessageFlags.STORAGE_CRC64); + = new StructuredMessageEncoder(randomData.length, Constants.KB, StructuredMessageFlags.STORAGE_CRC64); ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); Flux input = Flux.just(encodedData); - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - - StepVerifier - .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, - (BlobRequestConditions) null, false, validationOptions)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) - .verifyComplete(); - } - - @Test - public void downloadStreamWithResponseContentValidationMultipleSegments() throws IOException { - // Test with multiple segments to ensure all segments are decoded correctly - byte[] randomData = getRandomByteArray(2 * Constants.KB); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + // Upload the encoded data + bc.upload(input, null, true).block(); - Flux input = Flux.just(encodedData); + // Create a download client with decoder policy but NO mock interruption policy + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + // Download with validation - should succeed without any interruptions StepVerifier - .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, - (BlobRequestConditions) null, false, validationOptions)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .create( + downloadClient + .downloadStreamWithResponse((BlobRange) null, null, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> { + // Verify the decoded data matches the original + TestUtils.assertArraysEqual(result, randomData); + }) .verifyComplete(); } @Test - public void downloadStreamWithResponseNoValidation() throws IOException { - // Test that download works normally when validation is not enabled - byte[] randomData = getRandomByteArray(Constants.KB); + public void interruptWithDataIntact() throws IOException { + // Test: Verify that data remains intact after a single interruption and retry + // This mirrors the .NET test: Interrupt_DataIntact with single interrupt + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(4 * segmentSize); StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); Flux input = Flux.just(encodedData); - // No validation options - should download encoded data as-is - StepVerifier.create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, - (BlobRequestConditions) null, false)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - assertNotNull(r); - // Should get encoded data, not decoded - assertTrue(r.length > randomData.length); // Encoded data is larger - }).verifyComplete(); - } + // Create a policy that will simulate 1 network interruption at a specific position + // Interrupt after first segment completes to test smart retry from segment boundary + // Interrupt after first segment + 3 reads + 10 bytes (mirrors .NET interruptPos) + int interruptPos = segmentSize + (3 * 128) + 10; // readLen in .NET test = 128 bytes + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); - @Test - public void downloadStreamWithResponseValidationDisabled() throws IOException { - // Test with validation options but validation disabled - byte[] randomData = getRandomByteArray(Constants.KB); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(false); - - StepVerifier.create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, - (BlobRequestConditions) null, false, validationOptions)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - assertNotNull(r); - // Should get encoded data, not decoded - assertTrue(r.length > randomData.length); // Encoded data is larger - }).verifyComplete(); - } - - @Test - public void downloadStreamWithResponseContentValidationSmallSegment() throws IOException { - // Test with small segment size to ensure boundary conditions are handled - byte[] randomData = getRandomByteArray(256); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 128, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - - StepVerifier - .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, - (BlobRequestConditions) null, false, validationOptions)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) - .verifyComplete(); - } - - @Test - public void downloadStreamWithResponseContentValidationVeryLargeBlob() throws IOException { - // Test with very large data to verify chunking and policy work correctly with large blobs - byte[] randomData = getRandomByteArray(10 * Constants.KB); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 2048, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + // Upload the encoded data + bc.upload(input, null, true).block(); - Flux input = Flux.just(encodedData); + // Create download client with mock interruption and decoder policies + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient = new BlobClientBuilder().endpoint(bc.getBlobUrl()) + .addPolicy(decoderPolicy) + // Ensure the fault policy runs before decoding and on the initial call. + .addPolicy(mockPolicy) + .credential(ENVIRONMENT.getPrimaryAccount().getCredential()) + .buildAsyncClient(); DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - StepVerifier - .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, - (BlobRequestConditions) null, false, validationOptions)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) - .verifyComplete(); + // Download with validation - should succeed despite the interruption + StepVerifier.create(downloadClient + .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + // Verify the decoded data matches the original exactly + TestUtils.assertArraysEqual(randomData, result); + }).verifyComplete(); } @Test - public void downloadStreamWithResponseContentValidationSmartRetry() throws IOException { - // Test smart retry functionality with structured message validation - // This test simulates network interruptions and verifies that: - // 1. The decoder validates checksums for all received data - // 2. Retries resume from the encoded offset where the interruption occurred - // 3. The download eventually succeeds despite multiple interruptions - - byte[] randomData = getRandomByteArray(Constants.KB); + public void interruptMultipleTimesWithDataIntact() throws IOException { + // Test: Verify that data remains intact after multiple interruptions and retries + // This mirrors the .NET test: Interrupt_DataIntact with multiple interrupts + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(4 * segmentSize); StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); Flux input = Flux.just(encodedData); // Create a policy that will simulate 3 network interruptions - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3); + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, bc.getBlobUrl()); - // Upload the encoded data using the regular client + // Upload the encoded data bc.upload(input, null, true).block(); - // Create a download client with both the mock policy AND the decoder policy - // The decoder policy is needed to actually decode structured messages and validate checksums + // Create download client with mock interruption and decoder policies StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), mockPolicy, decoderPolicy); + bc.getBlobUrl(), decoderPolicy, mockPolicy); DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); - // Configure retry options to allow retries - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - - // Download with validation - should succeed despite interruptions + // Download with validation - should succeed despite multiple interruptions StepVerifier.create(downloadClient .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - // Verify the data is correctly decoded - TestUtils.assertArraysEqual(r, randomData); + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + // Verify the decoded data matches the original exactly + TestUtils.assertArraysEqual(randomData, result); }).verifyComplete(); - - // Verify that retries occurred (3 interruptions means we should have 0 tries remaining) - assertEquals(0, mockPolicy.getTriesRemaining()); - - // Verify that range headers were sent for retries - List rangeHeaders = mockPolicy.getRangeHeaders(); - assertTrue(rangeHeaders.size() > 0, "Expected range headers for retries"); - - // With structured message validation and smart retry, retries should resume from the encoded - // offset where the interruption occurred. The first request starts at 0, and subsequent - // retry requests should start from progressively higher offsets. - assertTrue(rangeHeaders.get(0).startsWith("bytes=0-"), "First request should start from offset 0"); - - // Subsequent requests should start from higher offsets (smart retry resuming from where it left off) - for (int i = 1; i < rangeHeaders.size(); i++) { - String rangeHeader = rangeHeaders.get(i); - // Each retry should start from a higher offset than the previous - // Note: We can't assert exact offset values as they depend on how much data was received - // before the interruption, but we can verify it's a valid range header - assertTrue(rangeHeader.startsWith("bytes="), - "Retry request " + i + " should have a range header: " + rangeHeader); - } - } - - @Test - public void downloadStreamWithResponseContentValidationSmartRetryVariousSizes() throws IOException { - int[] dataSizes = new int[] { Constants.KB, 1500, 3 * Constants.KB + 128 }; - int[] segmentSizes = new int[] { 512, 700, 1024 }; - - for (int i = 0; i < dataSizes.length; i++) { - byte[] randomData = getRandomByteArray(dataSizes[i]); - int segmentSize = segmentSizes[i % segmentSizes.length]; - - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - - // Create a policy that will simulate 2 network interruptions for each run - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(2); - - // Upload the encoded data - bc.upload(input, null, true).block(); - - // Create a download client with both the mock policy AND the decoder policy - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), mockPolicy, decoderPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - - StepVerifier.create(downloadClient - .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, - validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - TestUtils.assertArraysEqual(r, randomData); - }).verifyComplete(); - - assertEquals(0, mockPolicy.getTriesRemaining()); - List rangeHeaders = mockPolicy.getRangeHeaders(); - assertTrue(rangeHeaders.size() > 0, "Expected range headers for retries"); - assertTrue(rangeHeaders.get(0).startsWith("bytes=0-"), "First request should start from offset 0"); - } - } - - @Test - public void downloadStreamWithResponseContentValidationSmartRetrySync() throws IOException { - byte[] randomData = getRandomByteArray(Constants.KB); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - bc.upload(input, null, true).block(); - - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3); - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), - mockPolicy, decoderPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Context ctx = new Context(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) - .addData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY, validationOptions); - - downloadClient.downloadStreamWithResponse(out, null, retryOptions, null, false, null, ctx); - - TestUtils.assertArraysEqual(out.toByteArray(), randomData); - - assertEquals(0, mockPolicy.getTriesRemaining()); - List rangeHeaders = mockPolicy.getRangeHeaders(); - assertTrue(rangeHeaders.size() > 0, "Expected range headers for retries"); - assertTrue(rangeHeaders.get(0).startsWith("bytes=0-"), "First request should start from offset 0"); } @Test - public void downloadStreamWithResponseContentValidationSmartRetryMultipleSegments() throws IOException { - // Test smart retry with multiple segments to ensure checksum validation - // works correctly and retries resume from the interrupted encoded offset. - - byte[] randomData = getRandomByteArray(2 * Constants.KB); + public void interruptAndVerifyProperRewind() throws IOException { + // Test: Verify that interruption causes proper rewind to last complete segment boundary + // This mirrors the .NET test: Interrupt_AppropriateRewind + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(2 * segmentSize); StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 512, StructuredMessageFlags.STORAGE_CRC64); + = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); Flux input = Flux.just(encodedData); - // Create a policy that will simulate 4 network interruptions - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(4); + // Create a policy that will simulate 1 interruption at segment boundary + 2 reads + offset (per .NET) + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; // readLen = segmentSize/4 + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); // Upload the encoded data bc.upload(input, null, true).block(); - // Create a download client with both the mock policy AND the decoder policy - // The decoder policy is needed to actually decode structured messages and validate checksums + // Create download client with mock interruption and decoder policies StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), mockPolicy, decoderPolicy); + bc.getBlobUrl(), decoderPolicy, mockPolicy); DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - // Download with validation - should succeed and validate all segment checksums + // Download with validation StepVerifier.create(downloadClient .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - // Verify the data is correctly decoded - TestUtils.assertArraysEqual(r, randomData); + // Ensure the fault policy was invoked even if the assertion below fails. + .doFinally(signalType -> { + System.out.println("[MockPartialResponsePolicy] hits=" + mockPolicy.getHits() + ", triesRemaining=" + + mockPolicy.getTriesRemaining() + ", ranges=" + mockPolicy.getRangeHeaders()); + assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked"); + }) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + // Verify the decoded data matches the original + TestUtils.assertArraysEqual(randomData, result); }).verifyComplete(); - // Verify that retries occurred - assertEquals(0, mockPolicy.getTriesRemaining()); - - // Verify multiple retry requests were made - List rangeHeaders = mockPolicy.getRangeHeaders(); - assertTrue(rangeHeaders.size() >= 4, - "Expected at least 4 range headers for retries, got: " + rangeHeaders.size()); - - // With smart retry, each request should have a valid range header - for (int i = 0; i < rangeHeaders.size(); i++) { - String rangeHeader = rangeHeaders.get(i); - assertTrue(rangeHeader.startsWith("bytes="), - "Request " + i + " should have a valid range header, but was: " + rangeHeader); - } + // Quick sanity: mock interruption should have been hit and range headers recorded. + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); } - @Test - public void downloadStreamWithResponseContentValidationSmartRetryLargeBlob() throws IOException { - // Test smart retry with a larger blob to ensure retries resume from the - // interrupted offset and successfully validate all data - - byte[] randomData = getRandomByteArray(5 * Constants.KB); + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) throws IOException { + // Test: Verify that after interruption and retry, decoding continues correctly + // Mirrors .NET Interrupt_ProperDecode (multipleInterrupts toggles number of injected faults) + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] randomData = getRandomByteArray(dataSize); StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, 1024, StructuredMessageFlags.STORAGE_CRC64); + = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); Flux input = Flux.just(encodedData); - // Create a policy that will simulate 2 network interruptions - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(2); + // Create a policy with interruptions to test multi-step decode after retries + // Interrupt after first segment + 3 reads + 10 bytes (per .NET) + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; // readLen = 8KB in .NET + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, bc.getBlobUrl()); // Upload the encoded data bc.upload(input, null, true).block(); - // Create a download client with both the mock policy AND the decoder policy - // The decoder policy is needed to actually decode structured messages and validate checksums + // Create download client with mock interruption and decoder policies StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), mockPolicy, decoderPolicy); + bc.getBlobUrl(), decoderPolicy, mockPolicy); DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - - // Download with validation - decoder should validate checksums before each retry + // Download with validation - decoder must properly handle state across retries StepVerifier.create(downloadClient .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - // Verify the data is correctly decoded - TestUtils.assertArraysEqual(r, randomData); + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + // Verify every byte is correctly decoded despite interruptions + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(randomData, result); }).verifyComplete(); - - // Verify that retries occurred - assertEquals(0, mockPolicy.getTriesRemaining()); - - // Verify that smart retry is working with valid range headers - List rangeHeaders = mockPolicy.getRangeHeaders(); - for (int i = 0; i < rangeHeaders.size(); i++) { - String rangeHeader = rangeHeaders.get(i); - assertTrue(rangeHeader.startsWith("bytes="), - "Request " + i + " should have a valid range header, but was: " + rangeHeader); - } } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index 525e8191521e..ea955f63b829 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -111,6 +111,12 @@ public final class Constants { public static final String STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY = "azure-storage-structured-message-decoder-state"; + /** + * Context key used to pass a mutable holder for decoder state so callers can observe decoder progress. + */ + public static final String STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY + = "azure-storage-structured-message-decoder-state-ref"; + private Constants() { } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 4f11f4cb93bc..07110e7106d7 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -10,6 +10,7 @@ import com.azure.core.http.HttpPipelineNextPolicy; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.util.FluxUtil; import com.azure.core.util.logging.ClientLogger; import com.azure.storage.common.DownloadContentValidationOptions; @@ -22,6 +23,7 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,6 +41,7 @@ */ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); + private static final String EXPECTED_LENGTH_CONTEXT_KEY = "azStructuredMsgExpectedLength"; /** * Machine-readable token pattern for extracting retry start offset from exception messages. @@ -53,6 +56,12 @@ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy public StorageContentValidationDecoderPolicy() { } + @Override + public HttpPipelinePosition getPipelinePosition() { + // Run on every retry so the state stored in the call context is available across attempts. + return HttpPipelinePosition.PER_RETRY; + } + /** * Parses the retry start offset from an exception message containing the RETRY-START-OFFSET token. * @@ -130,8 +139,30 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN Long contentLength = getContentLength(httpResponse.getHeaders()); if (contentLength != null && contentLength > 0 && validationOptions != null) { + // Preserve the original encoded length across retries; range responses may advertise smaller lengths. + long expectedLength = context.getData(EXPECTED_LENGTH_CONTEXT_KEY) + .filter(Long.class::isInstance) + .map(Long.class::cast) + .orElse(contentLength); + + // Cache for subsequent retries. + context.setData(EXPECTED_LENGTH_CONTEXT_KEY, expectedLength); + + AtomicReference decoderStateHolder = null; + Object decoderStateHolderObj + = context.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY).orElse(null); + if (decoderStateHolderObj instanceof AtomicReference) { + @SuppressWarnings("unchecked") + AtomicReference tmp = (AtomicReference) decoderStateHolderObj; + decoderStateHolder = tmp; + } + // Get or create decoder with state tracking - DecoderState decoderState = getOrCreateDecoderState(context, contentLength); + DecoderState decoderState = getOrCreateDecoderState(context, expectedLength); + + if (decoderStateHolder != null) { + decoderStateHolder.set(decoderState); + } // Decode using the stateful decoder Flux decodedStream = decodeStream(httpResponse.getBody(), decoderState); @@ -197,32 +228,31 @@ private Flux decodeStream(Flux encodedFlux, DecoderState switch (result.getStatus()) { case SUCCESS: case NEED_MORE_BYTES: - case COMPLETED: - // All three cases update counters and return any decoded payload - // SUCCESS and NEED_MORE_BYTES: partial decode, more data expected - // COMPLETED: decode finished successfully - - long currentLastCompleteSegment = state.decoder.getLastCompleteSegmentStart(); - - // Only update decodedBytesAtLastCompleteSegment when lastCompleteSegmentStart changes - // This indicates that a segment boundary was just crossed - if (state.lastCompleteSegmentStart != currentLastCompleteSegment) { - state.decodedBytesAtLastCompleteSegment = state.decoder.getTotalDecodedPayloadBytes(); - state.lastCompleteSegmentStart = currentLastCompleteSegment; - - LOGGER.atInfo() - .addKeyValue("newSegmentBoundary", currentLastCompleteSegment) - .addKeyValue("decodedBytesAtBoundary", state.decodedBytesAtLastCompleteSegment) - .log("Segment boundary crossed, updated decoded bytes snapshot"); - } - - long encodedProgress - = state.decoder.getMessageOffset() + state.decoder.getPendingEncodedByteCount(); - state.totalEncodedBytesProcessed.set(encodedProgress); - state.totalBytesDecoded.set(state.decoder.getTotalDecodedPayloadBytes()); + // Update counters but defer emitting payload until we have a fully validated message. + updateProgress(state); + return Flux.empty(); - if (result.getDecodedPayload() != null && result.getDecodedPayload().hasRemaining()) { - return Flux.just(result.getDecodedPayload()); + case COMPLETED: + updateProgress(state); + ByteBuffer decodedPayload = result.getDecodedPayload(); + if (decodedPayload != null && decodedPayload.hasRemaining()) { + long skip = state.decodedBytesToSkip.get(); + if (skip > 0) { + if (skip >= decodedPayload.remaining()) { + state.decodedBytesToSkip.addAndGet(-decodedPayload.remaining()); + decodedPayload = null; + } else { + int skipCount = (int) skip; + decodedPayload.position(decodedPayload.position() + skipCount); + decodedPayload = decodedPayload.slice(); + state.decodedBytesToSkip.addAndGet(-skipCount); + } + } + + if (decodedPayload != null && decodedPayload.hasRemaining()) { + state.totalBytesDecoded.addAndGet(decodedPayload.remaining()); + return Flux.just(decodedPayload); + } } return Flux.empty(); @@ -240,10 +270,15 @@ private Flux decodeStream(Flux encodedFlux, DecoderState return Flux.error(createRetryableException(state, e.getMessage(), e)); } }).onErrorResume(throwable -> { - // If we've already completed decoding, ignore any downstream errors (mirror .NET behavior). + // If decoding already completed and we emitted payload, suppress late downstream errors (mirror .NET). + // If no payload was emitted, surface the error so the retriable download can resume properly. if (state.decoder.isComplete()) { - LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); - return Flux.empty(); + if (state.totalBytesDecoded.get() > 0) { + LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); + return Flux.empty(); + } else { + LOGGER.atInfo().log("Decoder complete with no emitted payload; propagating error to retry"); + } } // Wrap any error with retry offset information if (throwable instanceof IOException) { @@ -275,6 +310,29 @@ private Flux decodeStream(Flux encodedFlux, DecoderState })); } + /** + * Updates progress counters based on the current decoder state and logs when a new validated segment boundary is + * crossed. This keeps encoded progress in sync with the decoder while deferring payload emission until the entire + * message is validated. + */ + private void updateProgress(DecoderState state) { + long currentLastCompleteSegment = state.decoder.getLastCompleteSegmentStart(); + + // Only update decodedBytesAtLastCompleteSegment when the boundary changes (new segment validated). + if (state.lastCompleteSegmentStart != currentLastCompleteSegment) { + state.decodedBytesAtLastCompleteSegment = state.decoder.getTotalDecodedPayloadBytes(); + state.lastCompleteSegmentStart = currentLastCompleteSegment; + + LOGGER.atInfo() + .addKeyValue("newSegmentBoundary", currentLastCompleteSegment) + .addKeyValue("decodedBytesAtBoundary", state.decodedBytesAtLastCompleteSegment) + .log("Segment boundary crossed, updated decoded bytes snapshot"); + } + + long encodedProgress = state.decoder.getMessageOffset() + state.decoder.getPendingEncodedByteCount(); + state.totalEncodedBytesProcessed.set(encodedProgress); + } + /** * Creates an IOException with the retry start offset encoded in the message. * @@ -295,8 +353,8 @@ private IOException createRetryableException(DecoderState state, String message) * @return An IOException with retry offset information. */ private IOException createRetryableException(DecoderState state, String message, Throwable cause) { - long retryOffset = state.getRetryOffset(); - long decodedSoFar = state.decoder.getTotalDecodedPayloadBytes(); + long retryOffset = state.prepareForRetry(); + long decodedSoFar = state.totalBytesDecoded.get(); long expectedLength = state.decoder.getMessageLength(); // Check if the exception message already has decoder offset information @@ -362,6 +420,24 @@ private DownloadContentValidationOptions getValidationOptions(HttpPipelineCallCo * @return The content length or null if not present. */ private Long getContentLength(HttpHeaders headers) { + // Prefer the total length from Content-Range (if present) so retries use the full encoded length + // even when the current response is partial. + String contentRange = headers.getValue(HttpHeaderName.CONTENT_RANGE); + if (contentRange != null) { + // Format: bytes start-end/total + int slash = contentRange.indexOf('/'); + if (slash > -1 && slash + 1 < contentRange.length()) { + String totalPart = contentRange.substring(slash + 1).trim(); + if (!"*".equals(totalPart)) { + try { + return Long.parseLong(totalPart); + } catch (NumberFormatException e) { + LOGGER.warning("Invalid content range total length in response headers: " + contentRange); + } + } + } + } + String contentLengthStr = headers.getValue(HttpHeaderName.CONTENT_LENGTH); if (contentLengthStr != null) { try { @@ -405,10 +481,23 @@ private boolean isDownloadResponse(HttpResponse httpResponse) { public static class DecoderState { private final StructuredMessageDecoder decoder; private final long expectedContentLength; + /** + * Tracks how many decoded bytes have actually been emitted to the caller (excludes bytes skipped during + * fast-forward on retry). + */ private final AtomicLong totalBytesDecoded; private final AtomicLong totalEncodedBytesProcessed; + /** + * Snapshot of decoded bytes emitted at the last fully validated segment boundary. Used to correlate the encoded + * retry offset with the decoded offset that RetriableDownloadFlux tracks. + */ private long decodedBytesAtLastCompleteSegment; private long lastCompleteSegmentStart; // Tracks the last value to detect changes + /** + * Number of decoded bytes that should be skipped on the next retry to fast-forward to the caller's decoded + * offset. This mirrors the .NET StructuredMessageDecodingRetriableStream behavior. + */ + private final AtomicLong decodedBytesToSkip = new AtomicLong(0); /** * Creates a new decoder state. @@ -494,15 +583,14 @@ public long resetForRetry() { decoder.resetToLastCompleteSegment(); - // Align counters to the boundary we will resume from so subsequent progress tracking is consistent. - long boundaryDecodedBytes = decodedBytesAtLastCompleteSegment; - totalBytesDecoded.set(boundaryDecodedBytes); + // Align encoded counters to the boundary we will resume from so subsequent progress tracking is consistent. totalEncodedBytesProcessed.set(decoder.getMessageOffset() + decoder.getPendingEncodedByteCount()); + decodedBytesToSkip.set(0); LOGGER.atInfo() .addKeyValue("retryOffset", retryOffset) .addKeyValue("decoderOffset", decoder.getMessageOffset()) - .addKeyValue("decodedBytesAtBoundary", boundaryDecodedBytes) + .addKeyValue("decodedBytesAtBoundary", decodedBytesAtLastCompleteSegment) .log("Prepared decoder state for smart retry"); return retryOffset; @@ -526,6 +614,16 @@ public boolean isFinalized() { public long getDecodedBytesAtLastCompleteSegment() { return decodedBytesAtLastCompleteSegment; } + + /** + * Sets how many decoded bytes should be skipped when resuming after a retry. This lets the decoder fast-forward + * within the current segment to align with the decoded offset already emitted to the user before the failure. + * + * @param bytesToSkip decoded bytes to drop from the next decoded payloads. + */ + public void setDecodedBytesToSkip(long bytesToSkip) { + decodedBytesToSkip.set(Math.max(0, bytesToSkip)); + } } /** diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java index 5e84dd31947d..140e4ba4e596 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java @@ -24,9 +24,14 @@ public class MockDownloadHttpResponse extends HttpResponse { private final Flux body; public MockDownloadHttpResponse(HttpResponse response, int statusCode, Flux body) { + this(response, statusCode, response.getHeaders(), body); + } + + public MockDownloadHttpResponse(HttpResponse response, int statusCode, HttpHeaders headers, + Flux body) { super(response.getRequest()); this.statusCode = statusCode; - this.headers = response.getHeaders(); + this.headers = headers; this.body = body; } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index 9dff1fcc2be5..34c6cb263d35 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -8,6 +8,7 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; import reactor.core.publisher.Flux; @@ -15,6 +16,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; import java.util.ArrayList; import java.util.List; @@ -23,9 +25,46 @@ public class MockPartialResponsePolicy implements HttpPipelinePolicy { static final HttpHeaderName RANGE_HEADER = HttpHeaderName.RANGE; private int tries; private final List rangeHeaders = new ArrayList<>(); + private final int maxBytesPerResponse; // Maximum bytes to return before simulating timeout + private final AtomicInteger hits = new AtomicInteger(); + private final String targetUrlPrefix; + /** + * Creates a MockPartialResponsePolicy that simulates network interruptions. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + */ public MockPartialResponsePolicy(int tries) { + this(tries, 200, null); // Default: return 200 bytes for subsequent interruptions (enables 3 interrupts with 1KB data) + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse) { + this(tries, maxBytesPerResponse, null); + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior and an optional URL filter. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + * @param targetUrlPrefix If non-null, only requests whose URL starts with this prefix will be interrupted. + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse, String targetUrlPrefix) { this.tries = tries; + this.maxBytesPerResponse = maxBytesPerResponse; + this.targetUrlPrefix = targetUrlPrefix; + } + + @Override + public HttpPipelinePosition getPipelinePosition() { + // Apply on every retry to mirror .NET test behavior. + return HttpPipelinePosition.PER_RETRY; } @Override @@ -34,43 +73,82 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN HttpHeader rangeHttpHeader = response.getRequest().getHeaders().get(RANGE_HEADER); HttpHeader xMsRangeHttpHeader = response.getRequest().getHeaders().get(X_MS_RANGE_HEADER); - if (rangeHttpHeader != null && rangeHttpHeader.getValue().startsWith("bytes=")) { - rangeHeaders.add(rangeHttpHeader.getValue()); - } - - if (xMsRangeHttpHeader != null && xMsRangeHttpHeader.getValue().startsWith("bytes=")) { - String xMsRangeValue = xMsRangeHttpHeader.getValue(); - - // Avoid recording the same value twice if both Range and x-ms-range were set to the same string - if (rangeHttpHeader == null || !xMsRangeValue.equals(rangeHttpHeader.getValue())) { - rangeHeaders.add(xMsRangeValue); + // Record every GET attempt so tests can assert retries occurred, even if no range header was present. + if (response.getRequest().getHttpMethod() == HttpMethod.GET) { + String recordedRange = null; + if (rangeHttpHeader != null && rangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = rangeHttpHeader.getValue(); + } else if (xMsRangeHttpHeader != null && xMsRangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = xMsRangeHttpHeader.getValue(); } + rangeHeaders.add(recordedRange == null ? "" : recordedRange); } - if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || this.tries == 0) { + boolean urlMatches = targetUrlPrefix == null + || response.getRequest().getUrl().toString().startsWith(targetUrlPrefix); + + if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || !urlMatches || this.tries == 0) { return Mono.just(response); } else { + hits.incrementAndGet(); + System.out.println("[MockPartialResponsePolicy] invoked. tries=" + tries + + ", maxBytesPerResponse=" + maxBytesPerResponse); this.tries -= 1; - return response.getBody().collectList().flatMap(bodyBuffers -> { - if (bodyBuffers.isEmpty()) { - // If no body was returned, don't attempt to slice a partial response. Simply propagate - // the original response to avoid test failures when the service unexpectedly returns an - // empty body (for example, after a retry on the underlying transport). - return Mono.just(response); - } - ByteBuffer firstBuffer = bodyBuffers.get(0); - byte firstByte = firstBuffer.get(); - - // Simulate partial response by returning the first byte only from the requested range and timeout - return Mono.just(new MockDownloadHttpResponse(response, 206, - Flux.just(ByteBuffer.wrap(new byte[] { firstByte })) - .concatWith(Flux.error(new IOException("Simulated timeout"))) - )); - }); + + // Simulate an interruption mid-stream (like FaultyStream in .NET) without mutating headers. + // Emit up to maxBytesPerResponse, then complete early to let the decoder detect an incomplete message + // and trigger smart-retry. + Flux limitedBody = limitStreamToBytes(response.getBody(), maxBytesPerResponse); + return Mono.just( + new MockDownloadHttpResponse(response, response.getStatusCode(), response.getHeaders(), + limitedBody)); } }); } + /** + * Limits a Flux of ByteBuffers to emit at most maxBytes, then completes early to simulate + * an abrupt connection close without surfacing an explicit error. + */ + private Flux limitStreamToBytes(Flux body, int maxBytes) { + return Flux.defer(() -> { + final long[] bytesEmitted = new long[] { 0 }; + return body.concatMap(buffer -> { + if (buffer == null || !buffer.hasRemaining()) { + return Flux.just(buffer); + } + + long remaining = maxBytes - bytesEmitted[0]; + if (remaining <= 0) { + // Emit an error to simulate the network fault (mirrors FaultyStream in .NET). + return Flux.error(new IOException("Simulated timeout")); + } + + int bufferSize = buffer.remaining(); + if (bufferSize <= remaining) { + bytesEmitted[0] += bufferSize; + if (bytesEmitted[0] >= maxBytes) { + // Emit this buffer then fail to simulate the connection drop. + return Flux.just(buffer).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + return Flux.just(buffer); + } else { + // Buffer is larger than remaining, slice and then error. + int bytesToEmit = (int) remaining; + ByteBuffer slice = buffer.duplicate(); + slice.limit(slice.position() + bytesToEmit); + + ByteBuffer limited = ByteBuffer.allocate(bytesToEmit); + limited.put(slice); + limited.flip(); + + bytesEmitted[0] += bytesToEmit; + return Flux.just(limited).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + }); + }); + } + public int getTriesRemaining() { return tries; } @@ -78,4 +156,8 @@ public int getTriesRemaining() { public List getRangeHeaders() { return rangeHeaders; } + + public int getHits() { + return hits.get(); + } } From 89f92920484ededee3e847d6788d1d51da113f16 Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Wed, 17 Dec 2025 21:25:33 +0530 Subject: [PATCH 07/31] smart retry changes --- .../blob/specialized/BlobAsyncClientBase.java | 40 +++++++++------- .../blob/BlobMessageDecoderDownloadTests.java | 8 ++-- ...StorageContentValidationDecoderPolicy.java | 48 ++++++++++++++++++- 3 files changed, 73 insertions(+), 23 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index a34565529796..353d7c2454e7 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1370,15 +1370,20 @@ Mono downloadStreamWithResponseInternal(BlobRange ran return Mono.error(throwable); } - long newCount = finalCount - offset; + // For structured message decoding, treat downstream offset as 0 to avoid carrying stale skip/offset + // state across retries. Structured downloads always require the whole encoded message. + boolean structuredDecode = contentValidationOptions != null + && contentValidationOptions.isStructuredMessageValidationEnabled(); + long emittedOffset = structuredDecode ? 0 : offset; + + long newCount = finalCount - emittedOffset; StorageContentValidationDecoderPolicy.DecoderState decoderState = null; long expectedEncodedLength = finalCount; - long encodedProgress = offset; + long encodedProgress = emittedOffset; long retryStartOffset = -1; - boolean noBytesEmitted = offset == 0; + boolean noBytesEmitted = emittedOffset == 0; - if (contentValidationOptions != null - && contentValidationOptions.isStructuredMessageValidationEnabled()) { + if (structuredDecode) { decoderState = decoderStateRef.get(); if (decoderState == null) { @@ -1396,21 +1401,23 @@ Mono downloadStreamWithResponseInternal(BlobRange ran decoderState = null; } + if (decoderState != null) { + if (noBytesEmitted) { + // No decoded bytes reached the caller; discard this state and restart cleanly. + decoderState = null; + } + } + if (decoderState != null) { expectedEncodedLength = decoderState.getExpectedContentLength(); encodedProgress = decoderState.getTotalEncodedBytesProcessed(); long boundaryDecoded = decoderState.getDecodedBytesAtLastCompleteSegment(); - if (offset < boundaryDecoded || noBytesEmitted) { - // We haven't emitted the bytes represented by this decoder boundary; restart clean. - decoderState = null; - } else { - long bytesToSkip = Math.max(0, offset - boundaryDecoded); - decoderState.setDecodedBytesToSkip(bytesToSkip); + long bytesToSkip = emittedOffset <= boundaryDecoded ? 0 : emittedOffset - boundaryDecoded; + decoderState.setDecodedBytesToSkip(bytesToSkip); - // Always rewind decoder to last validated boundary before retrying. - retryStartOffset = decoderState.resetForRetry(); - } + // Always rewind decoder to last validated boundary before retrying. + retryStartOffset = decoderState.resetForRetry(); } } @@ -1429,15 +1436,14 @@ Mono downloadStreamWithResponseInternal(BlobRange ran // If structured message decoding is enabled, we need to calculate the retry offset // based on the encoded bytes processed, not the decoded bytes - if (contentValidationOptions != null - && contentValidationOptions.isStructuredMessageValidationEnabled()) { + if (structuredDecode) { // If no decoder state or no retry offset from state, fall back to parsed token or offset. if (retryStartOffset < 0 && !noBytesEmitted) { retryStartOffset = StorageContentValidationDecoderPolicy .parseRetryStartOffset(throwable.getMessage()); } if (retryStartOffset < 0) { - retryStartOffset = noBytesEmitted ? 0 : offset; + retryStartOffset = noBytesEmitted ? 0 : emittedOffset; } if (decoderState != null) { diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 90c7235aa833..b22f791ac112 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -80,10 +80,10 @@ public void downloadStreamWithResponseContentValidationRange() throws IOExceptio .then( bc.downloadStreamWithResponse(range, (DownloadRetryOptions) null, (BlobRequestConditions) null, false)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - assertNotNull(r); - // Should get exactly 512 bytes of encoded data - assertEquals(512, r.length); - }).verifyComplete(); + assertNotNull(r); + // Should get exactly 512 bytes of encoded data + assertEquals(512, r.length); + }).verifyComplete(); } @Test diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 07110e7106d7..ea0cb5f4ac81 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -228,13 +228,24 @@ private Flux decodeStream(Flux encodedFlux, DecoderState switch (result.getStatus()) { case SUCCESS: case NEED_MORE_BYTES: - // Update counters but defer emitting payload until we have a fully validated message. + // Accumulate partial decoded bytes; emit only when the full message is validated. + ByteBuffer partial = result.getDecodedPayload(); + if (partial != null && partial.hasRemaining()) { + state.appendPartial(partial); + } updateProgress(state); return Flux.empty(); case COMPLETED: - updateProgress(state); + // Append any final bytes from this chunk, then emit the full accumulated payload. ByteBuffer decodedPayload = result.getDecodedPayload(); + if (decodedPayload != null && decodedPayload.hasRemaining()) { + state.appendPartial(decodedPayload); + } + + decodedPayload = state.drainPartial(); + updateProgress(state); + if (decodedPayload != null && decodedPayload.hasRemaining()) { long skip = state.decodedBytesToSkip.get(); if (skip > 0) { @@ -250,6 +261,12 @@ private Flux decodeStream(Flux encodedFlux, DecoderState } if (decodedPayload != null && decodedPayload.hasRemaining()) { + // Return a defensive copy to avoid any inadvertent position/limit side effects. + ByteBuffer copy = ByteBuffer.allocate(decodedPayload.remaining()); + copy.put(decodedPayload.duplicate()); + copy.flip(); + decodedPayload = copy; + state.totalBytesDecoded.addAndGet(decodedPayload.remaining()); return Flux.just(decodedPayload); } @@ -487,6 +504,7 @@ public static class DecoderState { */ private final AtomicLong totalBytesDecoded; private final AtomicLong totalEncodedBytesProcessed; + private final java.io.ByteArrayOutputStream accumulatedDecoded = new java.io.ByteArrayOutputStream(); /** * Snapshot of decoded bytes emitted at the last fully validated segment boundary. Used to correlate the encoded * retry offset with the decoded offset that RetriableDownloadFlux tracks. @@ -582,6 +600,7 @@ public long resetForRetry() { long retryOffset = decoder.getRetryStartOffset(); decoder.resetToLastCompleteSegment(); + accumulatedDecoded.reset(); // Align encoded counters to the boundary we will resume from so subsequent progress tracking is consistent. totalEncodedBytesProcessed.set(decoder.getMessageOffset() + decoder.getPendingEncodedByteCount()); @@ -624,6 +643,31 @@ public long getDecodedBytesAtLastCompleteSegment() { public void setDecodedBytesToSkip(long bytesToSkip) { decodedBytesToSkip.set(Math.max(0, bytesToSkip)); } + + /** + * @param decoded Append decoded bytes produced so far in the current decode attempt. + * + */ + public void appendPartial(ByteBuffer decoded) { + if (decoded == null || !decoded.hasRemaining()) { + return; + } + ByteBuffer copy = decoded.asReadOnlyBuffer(); + byte[] data = new byte[copy.remaining()]; + copy.get(data); + accumulatedDecoded.write(data, 0, data.length); + } + + /** + * Drains and returns all accumulated decoded bytes for this message. + * + * @return ByteBuffer containing all decoded bytes or null if none accumulated. + */ + public ByteBuffer drainPartial() { + byte[] data = accumulatedDecoded.toByteArray(); + accumulatedDecoded.reset(); + return data.length == 0 ? null : ByteBuffer.wrap(data); + } } /** From 62f72cc7e38d8f87e1e6ad78c17c6089e84c0a8c Mon Sep 17 00:00:00 2001 From: gunjansingh-msft Date: Tue, 17 Mar 2026 00:00:05 +0530 Subject: [PATCH 08/31] smart retry changes --- ...DownloadAsyncResponseConstructorProxy.java | 59 +- .../implementation/util/BlobConstants.java | 20 + .../util/ChunkedDownloadUtils.java | 29 +- .../blob/implementation/util/ModelHelper.java | 13 +- .../models/BlobDownloadAsyncResponse.java | 76 +- .../blob/models/ParallelTransferOptions.java | 24 + .../blob/specialized/BlobAsyncClientBase.java | 730 +++++++++++++++--- .../blob/specialized/BlobClientBase.java | 37 +- .../blob/BlobMessageDecoderDownloadTests.java | 422 +++++++--- .../common/ParallelTransferOptions.java | 24 + .../common/implementation/Constants.java | 33 + .../StorageCrc64Calculator.java | 382 +++++++++ .../StructuredMessageDecoder.java | 83 +- ...StorageContentValidationDecoderPolicy.java | 424 +++++++--- .../test/shared/StorageCommonTestUtils.java | 7 + .../policy/MockDownloadHttpResponse.java | 18 +- .../policy/MockPartialResponsePolicy.java | 20 +- .../StorageCrc64CalculatorTests.java | 30 + 18 files changed, 2074 insertions(+), 357 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java index 234ca5e65e77..6165a1bb2657 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java @@ -9,8 +9,10 @@ import com.azure.core.http.rest.StreamResponse; import com.azure.storage.blob.models.BlobDownloadAsyncResponse; import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import reactor.core.publisher.Mono; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; /** @@ -35,7 +37,24 @@ public interface BlobDownloadAsyncResponseConstructorAccessor { * @param retryOptions Retry options. */ BlobDownloadAsyncResponse create(StreamResponse sourceResponse, - BiFunction> onErrorResume, DownloadRetryOptions retryOptions); + BiFunction> onErrorResume, DownloadRetryOptions retryOptions, + AtomicReference decoderStateRef); + + /** + * Gets the source {@link StreamResponse} from a {@link BlobDownloadAsyncResponse}. + * + * @param response The {@link BlobDownloadAsyncResponse}. + * @return The source {@link StreamResponse}, or null if not available. + */ + StreamResponse getSourceResponse(BlobDownloadAsyncResponse response); + + /** + * Gets the current decoder state from a {@link BlobDownloadAsyncResponse}. + * + * @param response The {@link BlobDownloadAsyncResponse}. + * @return The current decoder state, or null if not available. + */ + StorageContentValidationDecoderPolicy.DecoderState getDecoderState(BlobDownloadAsyncResponse response); } /** @@ -56,7 +75,8 @@ public static void setAccessor( * @param retryOptions Retry options. */ public static BlobDownloadAsyncResponse create(StreamResponse sourceResponse, - BiFunction> onErrorResume, DownloadRetryOptions retryOptions) { + BiFunction> onErrorResume, DownloadRetryOptions retryOptions, + AtomicReference decoderStateRef) { // This looks odd but is necessary, it is possible to engage the access helper before anywhere else in the // application accesses BlobDownloadAsyncResponse which triggers the accessor to be configured. So, if the accessor // is null this effectively pokes the class to set up the accessor. @@ -66,6 +86,39 @@ public static BlobDownloadAsyncResponse create(StreamResponse sourceResponse, } assert accessor != null; - return accessor.create(sourceResponse, onErrorResume, retryOptions); + return accessor.create(sourceResponse, onErrorResume, retryOptions, decoderStateRef); + } + + /** + * Gets the source {@link StreamResponse} from a {@link BlobDownloadAsyncResponse}. + * + * @param response The {@link BlobDownloadAsyncResponse}. + * @return The source {@link StreamResponse}, or null if not available. + */ + public static StreamResponse getSourceResponse(BlobDownloadAsyncResponse response) { + if (accessor == null) { + new BlobDownloadAsyncResponse(new HttpRequest(HttpMethod.GET, "http://microsoft.com"), 200, + new HttpHeaders(), null, null); + } + + assert accessor != null; + return accessor.getSourceResponse(response); + } + + /** + * Gets the current decoder state from a {@link BlobDownloadAsyncResponse}. + * + * @param response The {@link BlobDownloadAsyncResponse}. + * @return The decoder state, or null if not available. + */ + public static StorageContentValidationDecoderPolicy.DecoderState + getDecoderState(BlobDownloadAsyncResponse response) { + if (accessor == null) { + new BlobDownloadAsyncResponse(new HttpRequest(HttpMethod.GET, "http://microsoft.com"), 200, + new HttpHeaders(), null, null); + } + + assert accessor != null; + return accessor.getDecoderState(response); } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java index 1c78a6c469c8..2757cb131590 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java @@ -31,6 +31,26 @@ public final class BlobConstants { * The number of buffers to use if none is specified on the buffered upload method. */ public static final int BLOB_DEFAULT_NUMBER_OF_BUFFERS = 8; + /** + * The legacy default number of concurrent transfers for download operations. + */ + public static final int BLOB_LEGACY_DEFAULT_CONCURRENT_TRANSFERS_COUNT = 5; + /** + * The default range size used for download operations when no checksum validation is requested. + */ + public static final long BLOB_DEFAULT_DOWNLOAD_RANGE_SIZE = 4L * Constants.MB; + /** + * The default initial range size used for download operations when no checksum validation is requested. + */ + public static final long BLOB_DEFAULT_INITIAL_DOWNLOAD_RANGE_SIZE = 256L * Constants.MB; + /** + * The maximum range size used for download operations. + */ + public static final long BLOB_MAX_DOWNLOAD_BYTES = 256L * Constants.MB; + /** + * The maximum range size used when requesting a transactional checksum during download. + */ + public static final long BLOB_MAX_HASH_REQUEST_DOWNLOAD_RANGE = 4L * Constants.MB; /** * If a blob is known to be greater than 100MB, using a larger block size will trigger some server-side * optimizations. If the block size is not set and the size of the blob is known to be greater than 100MB, this diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java index 0505b6a3f6f6..9b22d85082bf 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java @@ -34,7 +34,7 @@ public class ChunkedDownloadUtils { public static Mono> downloadFirstChunk( BlobRange range, ParallelTransferOptions parallelTransferOptions, BlobRequestConditions requestConditions, BiFunction> downloader, boolean eTagLock) { - return downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloader, eTagLock, null); + return downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloader, null, eTagLock, null); } /* @@ -48,6 +48,22 @@ public static Mono> downloader, boolean eTagLock, Context context) { + return downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloader, null, eTagLock, + context); + } + + /* + Has a context value for additional download adjustments and an optional fallback downloader for empty blobs. + + Download the first chunk. Construct a Mono which will emit the total count for calculating the number of chunks, + access conditions containing the etag to lock on, and the response from downloading the first chunk. + */ + @SuppressWarnings("unchecked") + public static Mono> downloadFirstChunk( + BlobRange range, ParallelTransferOptions parallelTransferOptions, BlobRequestConditions requestConditions, + BiFunction> downloader, + BiFunction> emptyBlobDownloader, + boolean eTagLock, Context context) { // We will scope our initial download to either be one chunk or the total size. long initialChunkSize = range.getCount() != null && range.getCount() < parallelTransferOptions.getBlockSizeLong() @@ -69,7 +85,12 @@ public static Mono contextAdjustment = context.getData(Constants.ADJUSTED_BLOB_LENGTH_KEY); @@ -99,7 +120,9 @@ public static Mono> fallbackDownloader + = emptyBlobDownloader != null ? emptyBlobDownloader : downloader; + return fallbackDownloader.apply(new BlobRange(0), requestConditions) // Subscribe on boundElastic instead of elastic as elastic is deprecated and boundElastic // provided the same functionality with the added benefit that it won't infinitely create // threads if needed and will instead queue. diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java index cb859dfa595d..61b1739e118f 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java @@ -111,6 +111,8 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO blockSize = (long) BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE; } + Long initialTransferSize = other.getInitialTransferSizeLong(); + Integer maxConcurrency = other.getMaxConcurrency(); if (maxConcurrency == null) { maxConcurrency = BlobAsyncClient.BLOB_DEFAULT_NUMBER_OF_BUFFERS; @@ -124,7 +126,8 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO return new ParallelTransferOptions().setBlockSizeLong(blockSize) .setMaxConcurrency(maxConcurrency) .setProgressListener(other.getProgressListener()) - .setMaxSingleUploadSizeLong(maxSingleUploadSize); + .setMaxSingleUploadSizeLong(maxSingleUploadSize) + .setInitialTransferSizeLong(initialTransferSize); } /** @@ -143,6 +146,8 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO blockSize = (long) BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE; } + Long initialTransferSize = other.getInitialTransferSizeLong(); + Integer maxConcurrency = other.getMaxConcurrency(); if (maxConcurrency == null) { maxConcurrency = BlobAsyncClient.BLOB_DEFAULT_NUMBER_OF_BUFFERS; @@ -156,7 +161,8 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO return new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(blockSize) .setMaxConcurrency(maxConcurrency) .setProgressListener(other.getProgressListener()) - .setMaxSingleUploadSizeLong(maxSingleUploadSize); + .setMaxSingleUploadSizeLong(maxSingleUploadSize) + .setInitialTransferSizeLong(initialTransferSize); } /** @@ -169,7 +175,8 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO return new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(blobOptions.getBlockSizeLong()) .setMaxConcurrency(blobOptions.getMaxConcurrency()) .setProgressListener(blobOptions.getProgressListener()) - .setMaxSingleUploadSizeLong(blobOptions.getMaxSingleUploadSizeLong()); + .setMaxSingleUploadSizeLong(blobOptions.getMaxSingleUploadSizeLong()) + .setInitialTransferSizeLong(blobOptions.getInitialTransferSizeLong()); } /** diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java index c1c62368d093..0d0d2f950326 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java @@ -13,14 +13,18 @@ import com.azure.storage.blob.implementation.accesshelpers.BlobDownloadAsyncResponseConstructorProxy; import com.azure.storage.blob.implementation.models.BlobsDownloadHeaders; import com.azure.storage.blob.implementation.util.ModelHelper; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.channels.AsynchronousByteChannel; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; /** @@ -29,8 +33,30 @@ public final class BlobDownloadAsyncResponse extends ResponseBase> implements Closeable { + private static final ClientLogger LOGGER = new ClientLogger(BlobDownloadAsyncResponse.class); + static { - BlobDownloadAsyncResponseConstructorProxy.setAccessor(BlobDownloadAsyncResponse::new); + BlobDownloadAsyncResponseConstructorProxy + .setAccessor(new BlobDownloadAsyncResponseConstructorProxy.BlobDownloadAsyncResponseConstructorAccessor() { + @Override + public BlobDownloadAsyncResponse create(StreamResponse sourceResponse, + BiFunction> onErrorResume, DownloadRetryOptions retryOptions, + AtomicReference decoderStateRef) { + return new BlobDownloadAsyncResponse(sourceResponse, onErrorResume, retryOptions, decoderStateRef); + } + + @Override + public StreamResponse getSourceResponse(BlobDownloadAsyncResponse response) { + return response.sourceResponse; + } + + @Override + public StorageContentValidationDecoderPolicy.DecoderState + getDecoderState(BlobDownloadAsyncResponse response) { + AtomicReference ref = response.decoderStateRef; + return ref == null ? null : ref.get(); + } + }); } private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0); @@ -38,6 +64,7 @@ public final class BlobDownloadAsyncResponse extends ResponseBase> onErrorResume; private final DownloadRetryOptions retryOptions; + private final AtomicReference decoderStateRef; /** * Constructs a {@link BlobDownloadAsyncResponse}. @@ -54,6 +81,7 @@ public BlobDownloadAsyncResponse(HttpRequest request, int statusCode, HttpHeader this.sourceResponse = null; this.onErrorResume = null; this.retryOptions = null; + this.decoderStateRef = null; } /** @@ -65,11 +93,27 @@ public BlobDownloadAsyncResponse(HttpRequest request, int statusCode, HttpHeader */ BlobDownloadAsyncResponse(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions) { + this(sourceResponse, onErrorResume, retryOptions, null); + } + + BlobDownloadAsyncResponse(StreamResponse sourceResponse, + BiFunction> onErrorResume, DownloadRetryOptions retryOptions, + AtomicReference decoderStateRef) { + this(sourceResponse, onErrorResume, retryOptions, decoderStateRef, extractHeaders(sourceResponse)); + } + + private BlobDownloadAsyncResponse(StreamResponse sourceResponse, + BiFunction> onErrorResume, DownloadRetryOptions retryOptions, + AtomicReference decoderStateRef, + BlobDownloadHeaders deserializedHeaders) { super(sourceResponse.getRequest(), sourceResponse.getStatusCode(), sourceResponse.getHeaders(), - createResponseFlux(sourceResponse, onErrorResume, retryOptions), extractHeaders(sourceResponse)); + createResponseFluxWithContentCrc(sourceResponse, onErrorResume, retryOptions, decoderStateRef, + deserializedHeaders), + deserializedHeaders); this.sourceResponse = Objects.requireNonNull(sourceResponse, "'sourceResponse' must not be null"); this.onErrorResume = Objects.requireNonNull(onErrorResume, "'onErrorResume' must not be null"); this.retryOptions = Objects.requireNonNull(retryOptions, "'retryOptions' must not be null"); + this.decoderStateRef = decoderStateRef; } private static BlobDownloadHeaders extractHeaders(StreamResponse response) { @@ -87,6 +131,29 @@ private static Flux createResponseFlux(StreamResponse sourceResponse .defaultIfEmpty(EMPTY_BUFFER); } + /** + * Builds the response flux and populates ContentCrc64 on the deserialized headers when structured message + * decoding completes (mirrors .NET Details.ContentCrc populated after stream consumption). + */ + private static Flux createResponseFluxWithContentCrc(StreamResponse sourceResponse, + BiFunction> onErrorResume, DownloadRetryOptions retryOptions, + AtomicReference decoderStateRef, + BlobDownloadHeaders deserializedHeaders) { + Flux flux = createResponseFlux(sourceResponse, onErrorResume, retryOptions); + if (decoderStateRef != null && deserializedHeaders != null) { + flux = flux.doOnComplete(() -> { + StorageContentValidationDecoderPolicy.DecoderState state = decoderStateRef.get(); + if (state != null && state.isFinalized()) { + long crc = state.getComposedCrc64(); + byte[] crcBytes = new byte[8]; + ByteBuffer.wrap(crcBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(crc); + deserializedHeaders.setContentCrc64(crcBytes); + } + }); + } + return flux; + } + /** * Transfers content bytes to the {@link AsynchronousByteChannel}. * @param channel The destination {@link AsynchronousByteChannel}. @@ -95,7 +162,12 @@ private static Flux createResponseFlux(StreamResponse sourceResponse */ public Mono writeValueToAsync(AsynchronousByteChannel channel, ProgressReporter progressReporter) { Objects.requireNonNull(channel, "'channel' must not be null"); + LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("BlobDownloadAsyncResponse.writeValueToAsync entry"); if (sourceResponse != null) { + LOGGER.atVerbose() + .log("BlobDownloadAsyncResponse.writeValueToAsync using sourceResponse (IOUtils.transfer)"); return IOUtils.transferStreamResponseToAsynchronousByteChannel(channel, sourceResponse, onErrorResume, progressReporter, retryOptions.getMaxRetryRequests()); } else if (super.getValue() != null) { diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java index 77f7841c945d..05e0e19aed6f 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java @@ -17,6 +17,7 @@ public final class ParallelTransferOptions { private Long blockSize; + private Long initialTransferSize; private Integer maxConcurrency; private ProgressReceiver progressReceiver; private Long maxSingleUploadSize; @@ -101,6 +102,29 @@ public Long getBlockSizeLong() { return this.blockSize; } + /** + * Gets the size of the first range requested when downloading. + * @return The initial transfer size. + */ + public Long getInitialTransferSizeLong() { + return this.initialTransferSize; + } + + /** + * Sets the size of the first range requested when downloading. + * This value may be larger than the block size used for subsequent ranges. + * + * @param initialTransferSize The initial transfer size. + * @return The ParallelTransferOptions object itself. + */ + public ParallelTransferOptions setInitialTransferSizeLong(Long initialTransferSize) { + if (initialTransferSize != null) { + StorageImplUtils.assertInBounds("initialTransferSize", initialTransferSize, 1, Long.MAX_VALUE); + } + this.initialTransferSize = initialTransferSize; + return this; + } + /** * Sets the block size (chunk size) to transfer at a time. * For upload, The block size is the size of each block that will be staged. This value also determines the number diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 353d7c2454e7..c9264dee406b 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -12,6 +12,7 @@ import com.azure.core.http.rest.SimpleResponse; import com.azure.core.http.rest.StreamResponse; import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; import com.azure.core.util.Context; import com.azure.core.util.CoreUtils; import com.azure.core.util.FluxUtil; @@ -41,6 +42,7 @@ import com.azure.storage.blob.implementation.models.InternalBlobLegalHoldResult; import com.azure.storage.blob.implementation.models.QueryRequest; import com.azure.storage.blob.implementation.models.QuerySerialization; +import com.azure.storage.blob.implementation.util.BlobConstants; import com.azure.storage.blob.implementation.util.BlobQueryReader; import com.azure.storage.blob.implementation.util.BlobRequestConditionProperty; import com.azure.storage.blob.implementation.util.BlobSasImplUtil; @@ -86,12 +88,14 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.SasImplUtils; +import com.azure.storage.common.implementation.StorageCrc64Calculator; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.StorageChecksumAlgorithm; import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; +import reactor.core.scheduler.Schedulers; import java.io.IOException; import java.io.UncheckedIOException; @@ -99,7 +103,10 @@ import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.AsynchronousByteChannel; import java.nio.channels.AsynchronousFileChannel; +import java.nio.channels.CompletionHandler; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.OpenOption; @@ -107,19 +114,26 @@ import java.nio.file.StandardOpenOption; import java.time.Duration; import java.time.OffsetDateTime; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import static com.azure.core.util.FluxUtil.fluxError; import static com.azure.core.util.FluxUtil.monoError; @@ -1310,13 +1324,17 @@ Mono downloadStreamWithResponseInternal(BlobRange ran StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; + boolean structuredDecode + = contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled(); + // Determine MD5 validation: properly consider both getRangeContentMd5 parameter and validation options - // MD5 validation is enabled if: + // MD5 validation is enabled if structured message validation is not enabled and either: // 1. getRangeContentMd5 is explicitly true, OR // 2. contentValidationOptions.isMd5ValidationEnabled() is true final Boolean finalGetMD5; - if (getRangeContentMd5 - || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled())) { + if (!structuredDecode + && (getRangeContentMd5 + || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled()))) { finalGetMD5 = true; } else { finalGetMD5 = null; @@ -1327,24 +1345,36 @@ Mono downloadStreamWithResponseInternal(BlobRange ran DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; // The first range should eagerly convert headers as they'll be used to create response types. - Context initialContext = context == null + final Context baseContext = context == null ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); + String structuredBodyType = structuredDecode ? Constants.STRUCTURED_MESSAGE_CRC64_BODY_TYPE : null; + // Structured message responses are scoped to the response; each retry returns a fresh structured message. + final boolean responseScoped = structuredDecode; + final Context responseScopedContext = structuredDecode + ? baseContext.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) + : baseContext; + boolean structuredRetry = structuredDecode; + AtomicReference decoderStateRef = new AtomicReference<>(); + StorageContentValidationDecoderPolicy.AggregateCrcState aggregateCrcState + = structuredDecode ? new StorageContentValidationDecoderPolicy.AggregateCrcState() : null; + AtomicLong responseStartOffset = new AtomicLong(0); // Add structured message decoding context if enabled final Context firstRangeContext; - if (contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled()) { - firstRangeContext = initialContext.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) + if (structuredDecode) { + firstRangeContext = responseScopedContext.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) .addData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY, contentValidationOptions) - .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY, decoderStateRef); + .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY, decoderStateRef) + .addData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY, aggregateCrcState); } else { - firstRangeContext = initialContext; + firstRangeContext = responseScopedContext; } return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), finalGetMD5, - firstRangeContext).map(response -> { + structuredBodyType, firstRangeContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); String eTag = blobsDownloadHeaders.getETag(); BlobDownloadHeaders blobDownloadHeaders = ModelHelper.populateBlobDownloadHeaders(blobsDownloadHeaders, @@ -1370,63 +1400,49 @@ Mono downloadStreamWithResponseInternal(BlobRange ran return Mono.error(throwable); } - // For structured message decoding, treat downstream offset as 0 to avoid carrying stale skip/offset - // state across retries. Structured downloads always require the whole encoded message. - boolean structuredDecode = contentValidationOptions != null - && contentValidationOptions.isStructuredMessageValidationEnabled(); - long emittedOffset = structuredDecode ? 0 : offset; - + long emittedOffset = offset; + long currentResponseOffset = responseStartOffset.get(); long newCount = finalCount - emittedOffset; StorageContentValidationDecoderPolicy.DecoderState decoderState = null; - long expectedEncodedLength = finalCount; - long encodedProgress = emittedOffset; long retryStartOffset = -1; + long bytesToSkip = 0; boolean noBytesEmitted = emittedOffset == 0; - if (structuredDecode) { + if (structuredRetry) { decoderState = decoderStateRef.get(); - if (decoderState == null) { - Object decoderStateObj - = firstRangeContext.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) - .orElse(null); - - if (decoderStateObj instanceof StorageContentValidationDecoderPolicy.DecoderState) { - decoderState = (StorageContentValidationDecoderPolicy.DecoderState) decoderStateObj; - } - } - - // If the decoder has already finalized, discard it and restart from the beginning. - if (decoderState != null && decoderState.isFinalized()) { - decoderState = null; + // Prefer the retry start offset token emitted by the decoder policy when present. + long parsedRetryOffset + = StorageContentValidationDecoderPolicy.parseRetryStartOffset(throwable.getMessage()); + if (parsedRetryOffset >= 0) { + retryStartOffset = currentResponseOffset + parsedRetryOffset; } + // Compute the last validated segment boundary to align retry to a safe offset. if (decoderState != null) { - if (noBytesEmitted) { - // No decoded bytes reached the caller; discard this state and restart cleanly. - decoderState = null; + long decodedBoundary = decoderState.getDecodedBytesAtLastCompleteSegment(); + long boundaryGlobal = currentResponseOffset + decodedBoundary; + if (retryStartOffset < 0 || boundaryGlobal > retryStartOffset) { + retryStartOffset = boundaryGlobal; } } - if (decoderState != null) { - expectedEncodedLength = decoderState.getExpectedContentLength(); - encodedProgress = decoderState.getTotalEncodedBytesProcessed(); - - long boundaryDecoded = decoderState.getDecodedBytesAtLastCompleteSegment(); - long bytesToSkip = emittedOffset <= boundaryDecoded ? 0 : emittedOffset - boundaryDecoded; - decoderState.setDecodedBytesToSkip(bytesToSkip); - - // Always rewind decoder to last validated boundary before retrying. - retryStartOffset = decoderState.resetForRetry(); + if (retryStartOffset < 0) { + // No decoder state available (likely failed before policy captured it) or no bytes emitted; + // restart from response start and fast-forward using skip bytes. + retryStartOffset = currentResponseOffset; + decoderStateRef.set(null); } - } - if (decoderState == null) { - // No decoder state available (likely failed before policy captured it) or no bytes emitted; - // restart from beginning to avoid skipping data. - retryStartOffset = noBytesEmitted ? 0 : -1; - encodedProgress = noBytesEmitted ? 0 : encodedProgress; - decoderStateRef.set(null); + bytesToSkip = emittedOffset - retryStartOffset; + if (bytesToSkip < 0) { + // Fallback to response start if our computed boundary is ahead of emitted progress. + retryStartOffset = currentResponseOffset; + bytesToSkip = emittedOffset - retryStartOffset; + if (bytesToSkip < 0) { + bytesToSkip = 0; + } + } } try { @@ -1434,61 +1450,60 @@ Mono downloadStreamWithResponseInternal(BlobRange ran Context retryContext = firstRangeContext; BlobRange retryRange; - // If structured message decoding is enabled, we need to calculate the retry offset - // based on the encoded bytes processed, not the decoded bytes - if (structuredDecode) { - // If no decoder state or no retry offset from state, fall back to parsed token or offset. - if (retryStartOffset < 0 && !noBytesEmitted) { - retryStartOffset = StorageContentValidationDecoderPolicy - .parseRetryStartOffset(throwable.getMessage()); - } + if (structuredRetry) { if (retryStartOffset < 0) { - retryStartOffset = noBytesEmitted ? 0 : emittedOffset; + retryStartOffset = noBytesEmitted ? currentResponseOffset : emittedOffset; + bytesToSkip = Math.max(0, emittedOffset - retryStartOffset); } - if (decoderState != null) { - retryContext = retryContext - .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); - decoderStateRef.set(decoderState); - } else { - // Ensure we don't carry a stale decoder state into a full restart. - retryContext = retryContext - .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, null); + long remainingCount = finalCount - retryStartOffset; + if (remainingCount < 0) { + retryStartOffset = Math.min(emittedOffset, finalCount); + remainingCount = finalCount - retryStartOffset; + bytesToSkip = 0; } - long remainingCount = expectedEncodedLength - retryStartOffset; - if (remainingCount < 0) { - remainingCount = expectedEncodedLength - offset; - retryStartOffset = offset; + if (bytesToSkip > 0) { + retryContext = retryContext + .addData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY, bytesToSkip); } + responseStartOffset.set(retryStartOffset); retryRange = new BlobRange(initialOffset + retryStartOffset, remainingCount); LOGGER.info( - "Structured message smart retry: resuming from offset {} (initial={}, encoded={}, remaining={})", - initialOffset + retryStartOffset, initialOffset, retryStartOffset, remainingCount); + "Structured message retry: resuming from offset {} (initial={}, decoded={}, remaining={}, skip={})", + initialOffset + retryStartOffset, initialOffset, retryStartOffset, remainingCount, + bytesToSkip); } else { // For non-structured downloads, use smart retry from the interrupted offset - retryRange = new BlobRange(initialOffset + offset, newCount); + retryRange = new BlobRange(initialOffset + emittedOffset, newCount); } - return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, retryContext); + return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, structuredBodyType, + retryContext); } catch (Exception e) { return Mono.error(e); } }; // Structured message decoding is now handled by StructuredMessageDecoderPolicy - return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions); + return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions, + decoderStateRef); }); } + Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, + BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context) { + return downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, null, context); + } + private Mono downloadRange(BlobRange range, BlobRequestConditions requestConditions, String eTag, - Boolean getMD5, Context context) { + Boolean getMD5, String structuredBodyType, Context context) { return azureBlobStorage.getBlobs() .downloadNoCustomHeadersWithResponseAsync(containerName, blobName, snapshot, versionId, null, - range.toHeaderValue(), requestConditions.getLeaseId(), getMD5, null, null, + range.toHeaderValue(), requestConditions.getLeaseId(), getMD5, null, structuredBodyType, requestConditions.getIfModifiedSince(), requestConditions.getIfUnmodifiedSince(), eTag, requestConditions.getIfNoneMatch(), requestConditions.getTagsConditions(), null, customerProvidedKey, context); @@ -1638,7 +1653,7 @@ public Mono> downloadToFileWithResponse(String filePath BlobRequestConditions requestConditions, boolean rangeGetContentMd5, Set openOptions) { try { final com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions - = ModelHelper.wrapBlobOptions(ModelHelper.populateAndApplyDefaults(parallelTransferOptions)); + = parallelTransferOptions == null ? null : ModelHelper.wrapBlobOptions(parallelTransferOptions); return withContext( context -> downloadToFileWithResponse(new BlobDownloadToFileOptions(filePath).setRange(range) .setParallelTransferOptions(finalParallelTransferOptions) @@ -1692,8 +1707,15 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti StorageImplUtils.assertNotNull("options", options); BlobRange finalRange = options.getRange() == null ? new BlobRange(0) : options.getRange(); - final com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions - = ModelHelper.populateAndApplyDefaults(options.getParallelTransferOptions()); + com.azure.storage.common.ParallelTransferOptions originalParallelTransferOptions + = options.getParallelTransferOptions(); + boolean blockSizeProvided + = originalParallelTransferOptions != null && originalParallelTransferOptions.getBlockSizeLong() != null; + boolean maxConcurrencyProvided + = originalParallelTransferOptions != null && originalParallelTransferOptions.getMaxConcurrency() != null; + com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions + = ModelHelper.populateAndApplyDefaults(originalParallelTransferOptions); + DownloadContentValidationOptions contentValidationOptions = options.getContentValidationOptions(); BlobRequestConditions finalConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); @@ -1704,7 +1726,14 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti } AsynchronousFileChannel channel = downloadToFileResourceSupplier(options.getFilePath(), openOptions); + // Run download on boundedElastic to avoid blocking the reactor thread when async file I/O + // completion is delivered (matches .NET behavior where DownloadTo runs off the sync context). return Mono.just(channel) + .flatMap(c -> this + .downloadToFileImpl(c, finalRange, finalParallelTransferOptions, blockSizeProvided, + maxConcurrencyProvided, options.getDownloadRetryOptions(), finalConditions, + options.isRetrieveContentRangeMd5(), contentValidationOptions, context) + .subscribeOn(Schedulers.boundedElastic())) .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), options.getResponseChecksumAlgorithm(), context)) @@ -1723,51 +1752,544 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, DownloadRetryOptions downloadRetryOptions, BlobRequestConditions requestConditions, boolean rangeGetContentMd5, StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { + com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, boolean blockSizeProvided, + boolean maxConcurrencyProvided, DownloadRetryOptions downloadRetryOptions, + BlobRequestConditions requestConditions, boolean rangeGetContentMd5, + DownloadContentValidationOptions contentValidationOptions, Context context) { // See ProgressReporter for an explanation on why this lock is necessary and why we use AtomicLong. ProgressListener progressReceiver = finalParallelTransferOptions.getProgressListener(); ProgressReporter progressReporter = progressReceiver == null ? null : ProgressReporter.withProgressListener(progressReceiver); + boolean structuredDecode + = contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled(); + final Context downloadContext = structuredDecode + ? (context == null + ? new Context(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) + : context.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true)) + : context; + /* * Downloads the first chunk and gets the size of the data and etag if not specified by the user. */ BiFunction> downloadFunc + = (range, conditions) -> structuredDecode + ? this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, rangeGetContentMd5, + contentValidationOptions, downloadContext) + : this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, rangeGetContentMd5, + downloadContext); + BiFunction> emptyBlobDownloadFunc + = (range, conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, false, + null, context); + + boolean checksumValidationEnabled = structuredDecode + || rangeGetContentMd5 + || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled()); + boolean md5ValidationEnabled = !structuredDecode + && (rangeGetContentMd5 + || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled())); + + long rangeSize = blockSizeProvided + ? Math.min(finalParallelTransferOptions.getBlockSizeLong(), BlobConstants.BLOB_MAX_DOWNLOAD_BYTES) + : (checksumValidationEnabled + ? BlobConstants.BLOB_MAX_HASH_REQUEST_DOWNLOAD_RANGE + : BlobConstants.BLOB_DEFAULT_DOWNLOAD_RANGE_SIZE); + + Long requestedInitialTransferSize = finalParallelTransferOptions.getInitialTransferSizeLong(); + long initialRangeSize = requestedInitialTransferSize != null && requestedInitialTransferSize > 0 + ? requestedInitialTransferSize + : (checksumValidationEnabled + ? BlobConstants.BLOB_MAX_HASH_REQUEST_DOWNLOAD_RANGE + : BlobConstants.BLOB_DEFAULT_INITIAL_DOWNLOAD_RANGE_SIZE); + + int maxConcurrency = maxConcurrencyProvided + ? finalParallelTransferOptions.getMaxConcurrency() + : getDefaultDownloadConcurrency(); = (range, conditions) -> this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, rangeGetContentMd5, responseChecksumAlgorithm, context); + com.azure.storage.common.ParallelTransferOptions initialParallelTransferOptions + = new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(initialRangeSize); + + boolean useMasterCrc = structuredDecode; + + LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("BlobAsyncClientBase.downloadToFileImpl calling downloadFirstChunk"); return ChunkedDownloadUtils - .downloadFirstChunk(finalRange, finalParallelTransferOptions, requestConditions, downloadFunc, true, - context) + .downloadFirstChunk(finalRange, initialParallelTransferOptions, requestConditions, downloadFunc, + emptyBlobDownloadFunc, true, downloadContext) + .doOnSuccess(t -> LOGGER.atVerbose() + .addKeyValue("newCount", t != null ? t.getT1() : null) + .addKeyValue("thread", Thread.currentThread().getName()) + .log("BlobAsyncClientBase downloadFirstChunk returned")) .flatMap(setupTuple3 -> { long newCount = setupTuple3.getT1(); BlobRequestConditions finalConditions = setupTuple3.getT2(); + BlobDownloadAsyncResponse initialResponse = setupTuple3.getT3(); + LOGGER.atVerbose() + .addKeyValue("newCount", newCount) + .addKeyValue("thread", Thread.currentThread().getName()) + .log("BlobAsyncClientBase flatMap after first chunk"); + + if (initialResponse.getStatusCode() == 304 || newCount == 0) { + return Mono.fromCallable(() -> { + Response propertiesResponse + = ModelHelper.buildBlobPropertiesResponse(initialResponse); + try { + initialResponse.close(); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + return propertiesResponse; + }); + } - int numChunks = ChunkedDownloadUtils.calculateNumBlocks(newCount, - finalParallelTransferOptions.getBlockSizeLong()); + AsynchronousByteChannel baseChannel = IOUtils.toAsynchronousByteChannel(file, 0); + Crc64TrackingAsynchronousByteChannel crcChannel + = useMasterCrc ? new Crc64TrackingAsynchronousByteChannel(baseChannel) : null; + AsynchronousByteChannel targetChannel = useMasterCrc ? crcChannel : baseChannel; + ComposedCrcState composedCrcState = useMasterCrc ? new ComposedCrcState() : null; + + long initialLength = Math.min(initialRangeSize, newCount); + if (initialLength == newCount) { + // One-shot download path aligned with .NET: write the initial response directly and finalize. + LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("BlobAsyncClientBase taking ONE-SHOT path (single chunk)"); + return writeBodyToFile(initialResponse, targetChannel, + progressReporter == null ? null : progressReporter.createChild(), useMasterCrc, + composedCrcState, md5ValidationEnabled).doFinally(signalType -> closeQuietly(initialResponse)) + .then(Mono.fromCallable(() -> { + if (useMasterCrc) { + validateComposedCrc(composedCrcState, crcChannel); + } + return ModelHelper.buildBlobPropertiesResponse(initialResponse); + })); + } - // In case it is an empty blob, this ensures we still actually perform a download operation. - numChunks = numChunks == 0 ? 1 : numChunks; + long remainingLength = Math.max(0L, newCount - initialLength); + int remainingChunks = ChunkedDownloadUtils.calculateNumBlocks(remainingLength, rangeSize); + int numChunks = Math.max(1, 1 + remainingChunks); + LOGGER.atVerbose() + .addKeyValue("numChunks", numChunks) + .addKeyValue("thread", Thread.currentThread().getName()) + .log("BlobAsyncClientBase taking PARALLEL path"); + + List remainingRanges = new ArrayList<>(Math.max(0, numChunks - 1)); + for (int chunkIndex = 1; chunkIndex < numChunks; chunkIndex++) { + long offset = initialLength + (long) (chunkIndex - 1) * rangeSize; + long chunkSizeActual = Math.min(rangeSize, newCount - offset); + if (chunkSizeActual <= 0) { + break; + } + remainingRanges.add(new BlobRange(finalRange.getOffset() + offset, chunkSizeActual)); + } - BlobDownloadAsyncResponse initialResponse = setupTuple3.getT3(); - return Flux.range(0, numChunks) - .flatMap( - chunkNum -> ChunkedDownloadUtils.downloadChunk(chunkNum, initialResponse, finalRange, - finalParallelTransferOptions, finalConditions, newCount, downloadFunc, - response -> writeBodyToFile(response, file, chunkNum, finalParallelTransferOptions, - progressReporter == null ? null : progressReporter.createChild()).flux()), - finalParallelTransferOptions.getMaxConcurrency()) - - // Only the first download call returns a value. - .then(Mono.just(ModelHelper.buildBlobPropertiesResponse(initialResponse))); + // Match .NET behavior: parallelize request issuance while serializing writes in range order. + int effectiveConcurrency = Math.max(1, maxConcurrency); + ArrayDeque> running = new ArrayDeque<>(); + Iterator remainingIterator = remainingRanges.iterator(); + + running.add(CompletableFuture.completedFuture(initialResponse)); + while (running.size() < effectiveConcurrency && remainingIterator.hasNext()) { + BlobRange nextRange = remainingIterator.next(); + running.add(downloadFunc.apply(nextRange, finalConditions).toFuture()); + } + + return drainQueuedResponses(running, remainingIterator, downloadFunc, finalConditions, targetChannel, + progressReporter, useMasterCrc, composedCrcState, md5ValidationEnabled) + .doFinally(signalType -> closePendingResponses(running)) + .then(Mono.fromCallable(() -> { + if (useMasterCrc) { + validateComposedCrc(composedCrcState, crcChannel); + } + return ModelHelper.buildBlobPropertiesResponse(initialResponse); + })); }); } - private static Mono writeBodyToFile(BlobDownloadAsyncResponse response, AsynchronousFileChannel file, - long chunkNum, com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, - ProgressReporter progressReporter) { + private static Mono drainQueuedResponses(ArrayDeque> running, + Iterator remainingIterator, + BiFunction> downloadFunc, + BlobRequestConditions finalConditions, AsynchronousByteChannel targetChannel, ProgressReporter progressReporter, + boolean useMasterCrc, ComposedCrcState composedCrcState, boolean md5ValidationEnabled) { + return Mono.defer(() -> { + CompletableFuture nextFuture = running.poll(); + if (nextFuture == null) { + return Mono.empty(); + } - long position = chunkNum * finalParallelTransferOptions.getBlockSizeLong(); - return response.writeValueToAsync(IOUtils.toAsynchronousByteChannel(file, position), progressReporter); + if (remainingIterator.hasNext()) { + BlobRange nextRange = remainingIterator.next(); + running.add(downloadFunc.apply(nextRange, finalConditions).toFuture()); + } + + return Mono.fromFuture(nextFuture) + .flatMap(response -> writeBodyToFile(response, targetChannel, + progressReporter == null ? null : progressReporter.createChild(), useMasterCrc, composedCrcState, + md5ValidationEnabled).doFinally(signalType -> closeQuietly(response))) + .then(Mono.defer(() -> drainQueuedResponses(running, remainingIterator, downloadFunc, finalConditions, + targetChannel, progressReporter, useMasterCrc, composedCrcState, md5ValidationEnabled))); + }); + } + + private static void closePendingResponses(ArrayDeque> running) { + CompletableFuture future; + while ((future = running.poll()) != null) { + future.whenComplete((response, throwable) -> { + if (response != null) { + closeQuietly(response); + } + }); + if (!future.isDone()) { + future.cancel(true); + } + } + } + + private static Mono writeBodyToFile(BlobDownloadAsyncResponse response, AsynchronousByteChannel channel, + ProgressReporter progressReporter, boolean useMasterCrc, ComposedCrcState composedCrcState, + boolean validateMd5) { + LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .addKeyValue("useMasterCrc", useMasterCrc) + .log("BlobAsyncClientBase.writeBodyToFile entry"); + Md5TrackingAsynchronousByteChannel md5Channel = null; + AsynchronousByteChannel targetChannel = channel; + + if (validateMd5) { + md5Channel = new Md5TrackingAsynchronousByteChannel(targetChannel); + targetChannel = md5Channel; + } + + /* + * Use BlobDownloadAsyncResponse#writeValueToAsync to preserve the original retriable stream + * transfer behavior (same model used by .NET stream copy path). + */ + Mono write = response.writeValueToAsync(targetChannel, progressReporter) + .doOnSuccess(v -> LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("BlobAsyncClientBase.writeBodyToFile writeValueToAsync completed")) + .doOnError(e -> LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .addKeyValue("error", e) + .log("BlobAsyncClientBase.writeBodyToFile writeValueToAsync error")); + if (!useMasterCrc && !validateMd5) { + return write; + } + + Md5TrackingAsynchronousByteChannel finalMd5Channel = md5Channel; + return write.then(Mono.fromRunnable(() -> { + if (validateMd5) { + validateResponseMd5(response, finalMd5Channel); + } + if (useMasterCrc) { + StorageContentValidationDecoderPolicy.DecoderState decoderState = getStructuredDecoderState(response); + if (decoderState == null || !decoderState.isFinalized()) { + throw LOGGER.logExceptionAsError(new IllegalStateException( + "Structured message decoder state wasn't available or finalized for checksum validation.")); + } + + long composedLength = decoderState.getComposedLength(); + if (composedLength > 0) { + long partitionCrc = decoderState.getComposedCrc64(); + composedCrcState.append(partitionCrc, composedLength); + + BlobDownloadHeaders headers = response.getDeserializedHeaders(); + if (headers != null) { + headers.setContentCrc64(littleEndianLongToBytes(partitionCrc)); + } + } + } + })); + } + + private static StorageContentValidationDecoderPolicy.DecoderState + getStructuredDecoderState(BlobDownloadAsyncResponse response) { + return BlobDownloadAsyncResponseConstructorProxy.getDecoderState(response); + } + + private static byte[] littleEndianLongToBytes(long value) { + return ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(value).array(); + } + + private static int getDefaultDownloadConcurrency() { + Configuration configuration = Configuration.getGlobalConfiguration(); + String legacyDefaultConcurrency = configuration.get(Constants.USE_LEGACY_DEFAULT_CONCURRENCY_PROPERTY); + if (legacyDefaultConcurrency == null) { + legacyDefaultConcurrency = configuration.get(Constants.USE_LEGACY_DEFAULT_CONCURRENCY_ENV_VAR); + } + if (Boolean.parseBoolean(legacyDefaultConcurrency)) { + return BlobConstants.BLOB_LEGACY_DEFAULT_CONCURRENT_TRANSFERS_COUNT; + } + + int processors = Runtime.getRuntime().availableProcessors(); + int concurrency = Math.max(processors * 2, 8); + return Math.min(concurrency, 32); + } + + private static void validateResponseMd5(BlobDownloadAsyncResponse response, + Md5TrackingAsynchronousByteChannel md5Channel) { + if (md5Channel == null) { + return; + } + + byte[] expectedMd5 + = response.getDeserializedHeaders() == null ? null : response.getDeserializedHeaders().getContentMd5(); + if (expectedMd5 == null || expectedMd5.length == 0) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Content-MD5 header missing from download response.")); + } + + byte[] actualMd5 = md5Channel.getMd5(); + if (!Arrays.equals(expectedMd5, actualMd5)) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException("MD5 mismatch detected in download response.")); + } + } + + private static void validateComposedCrc(ComposedCrcState composedCrcState, + Crc64TrackingAsynchronousByteChannel crcChannel) { + if (composedCrcState == null || crcChannel == null || composedCrcState.getLength() == 0) { + return; + } + + long composed = composedCrcState.getCrc(); + long master = crcChannel.getCrc(); + if (composed != master) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + "CRC64 mismatch detected in composed download. Expected: " + composed + ", got: " + master)); + } + } + + private static void closeQuietly(BlobDownloadAsyncResponse response) { + if (response == null) { + return; + } + + try { + response.close(); + } catch (IOException e) { + LOGGER.warning("Failed to close BlobDownloadAsyncResponse: {}", e.getMessage()); + } + } + + private static final class ComposedCrcState { + private long crc; + private long length; + + private void append(long nextCrc, long nextLength) { + if (nextLength <= 0) { + return; + } + + if (length == 0) { + crc = nextCrc; + length = nextLength; + return; + } + + crc = StorageCrc64Calculator.concat(0, 0, crc, length, 0, nextCrc, nextLength); + length += nextLength; + } + + private long getCrc() { + return crc; + } + + private long getLength() { + return length; + } + } + + private static final class Crc64TrackingAsynchronousByteChannel implements AsynchronousByteChannel { + private final AsynchronousByteChannel channel; + private final AtomicLong crc = new AtomicLong(0); + private final AtomicLong bytesWritten = new AtomicLong(0); + + private Crc64TrackingAsynchronousByteChannel(AsynchronousByteChannel channel) { + this.channel = channel; + } + + private long getCrc() { + return crc.get(); + } + + private long getBytesWritten() { + return bytesWritten.get(); + } + + @Override + public void read(ByteBuffer dst, A attachment, CompletionHandler handler) { + channel.read(dst, attachment, handler); + } + + @Override + public Future read(ByteBuffer dst) { + return channel.read(dst); + } + + @Override + public void write(ByteBuffer src, A attachment, CompletionHandler handler) { + int startPos = src.position(); + ByteBuffer duplicate = src.duplicate(); + channel.write(src, attachment, new CompletionHandler() { + @Override + public void completed(Integer result, A att) { + if (result != null && result > 0) { + updateCrc(duplicate, startPos, result); + } + handler.completed(result, att); + } + + @Override + public void failed(Throwable exc, A att) { + handler.failed(exc, att); + } + }); + } + + @Override + public Future write(ByteBuffer src) { + CompletableFuture future = new CompletableFuture<>(); + int startPos = src.position(); + ByteBuffer duplicate = src.duplicate(); + channel.write(src, src, new CompletionHandler() { + @Override + public void completed(Integer result, ByteBuffer attachment) { + if (result != null && result > 0) { + updateCrc(duplicate, startPos, result); + } + future.complete(result); + } + + @Override + public void failed(Throwable exc, ByteBuffer attachment) { + future.completeExceptionally(exc); + } + }); + return future; + } + + @Override + public boolean isOpen() { + return channel.isOpen(); + } + + @Override + public void close() throws IOException { + channel.close(); + } + + private void updateCrc(ByteBuffer buffer, int startPos, int length) { + if (length <= 0) { + return; + } + + ByteBuffer slice = buffer.duplicate(); + slice.position(startPos); + slice.limit(startPos + length); + byte[] bytes = new byte[length]; + slice.get(bytes); + crc.updateAndGet(previous -> StorageCrc64Calculator.compute(bytes, previous)); + bytesWritten.addAndGet(length); + } + } + + private static final class Md5TrackingAsynchronousByteChannel implements AsynchronousByteChannel { + private final AsynchronousByteChannel channel; + private final MessageDigest digest; + + private Md5TrackingAsynchronousByteChannel(AsynchronousByteChannel channel) { + this.channel = channel; + try { + this.digest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw LOGGER.logExceptionAsError(new IllegalStateException("MD5 MessageDigest unavailable.", e)); + } + } + + private byte[] getMd5() { + synchronized (digest) { + return digest.digest(); + } + } + + @Override + public void read(ByteBuffer dst, A attachment, CompletionHandler handler) { + channel.read(dst, attachment, handler); + } + + @Override + public Future read(ByteBuffer dst) { + return channel.read(dst); + } + + @Override + public void write(ByteBuffer src, A attachment, CompletionHandler handler) { + int startPos = src.position(); + ByteBuffer duplicate = src.duplicate(); + channel.write(src, attachment, new CompletionHandler() { + @Override + public void completed(Integer result, A att) { + if (result != null && result > 0) { + updateMd5(duplicate, startPos, result); + } + handler.completed(result, att); + } + + @Override + public void failed(Throwable exc, A att) { + handler.failed(exc, att); + } + }); + } + + @Override + public Future write(ByteBuffer src) { + CompletableFuture future = new CompletableFuture<>(); + int startPos = src.position(); + ByteBuffer duplicate = src.duplicate(); + channel.write(src, src, new CompletionHandler() { + @Override + public void completed(Integer result, ByteBuffer attachment) { + if (result != null && result > 0) { + updateMd5(duplicate, startPos, result); + } + future.complete(result); + } + + @Override + public void failed(Throwable exc, ByteBuffer attachment) { + future.completeExceptionally(exc); + } + }); + return future; + } + + @Override + public boolean isOpen() { + return channel.isOpen(); + } + + @Override + public void close() throws IOException { + channel.close(); + } + + private void updateMd5(ByteBuffer buffer, int startPos, int length) { + if (length <= 0) { + return; + } + + ByteBuffer slice = buffer.duplicate(); + slice.position(startPos); + slice.limit(startPos + length); + synchronized (digest) { + digest.update(slice); + } + } } private void downloadToFileCleanup(AsynchronousFileChannel channel, String filePath, SignalType signalType) { diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java index b6169ff7636c..8d2fab4da17d 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java @@ -1567,7 +1567,7 @@ public Response downloadToFileWithResponse(String filePath, Blob BlobRequestConditions requestConditions, boolean rangeGetContentMd5, Set openOptions, Duration timeout, Context context) { final com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions - = ModelHelper.wrapBlobOptions(ModelHelper.populateAndApplyDefaults(parallelTransferOptions)); + = parallelTransferOptions == null ? null : ModelHelper.wrapBlobOptions(parallelTransferOptions); return downloadToFileWithResponse(new BlobDownloadToFileOptions(filePath).setRange(range) .setParallelTransferOptions(finalParallelTransferOptions) .setDownloadRetryOptions(downloadRetryOptions) @@ -1608,10 +1608,43 @@ public Response downloadToFileWithResponse(String filePath, Blob @ServiceMethod(returns = ReturnType.SINGLE) public Response downloadToFileWithResponse(BlobDownloadToFileOptions options, Duration timeout, Context context) { - Mono> download = client.downloadToFileWithResponse(options, context); + Mono> download + = client.downloadToFileWithResponse(adjustOptionsForSyncDownload(options), context); return blockWithOptionalTimeout(download, timeout); } + private static BlobDownloadToFileOptions adjustOptionsForSyncDownload(BlobDownloadToFileOptions options) { + if (options == null) { + return null; + } + + com.azure.storage.common.ParallelTransferOptions parallelTransferOptions = options.getParallelTransferOptions(); + Integer maxConcurrency = parallelTransferOptions == null ? null : parallelTransferOptions.getMaxConcurrency(); + if (maxConcurrency != null && maxConcurrency <= 1) { + return options; + } + + com.azure.storage.common.ParallelTransferOptions adjustedParallelOptions; + if (parallelTransferOptions == null) { + adjustedParallelOptions = new com.azure.storage.common.ParallelTransferOptions().setMaxConcurrency(1); + } else { + adjustedParallelOptions = new com.azure.storage.common.ParallelTransferOptions() + .setBlockSizeLong(parallelTransferOptions.getBlockSizeLong()) + .setInitialTransferSizeLong(parallelTransferOptions.getInitialTransferSizeLong()) + .setMaxConcurrency(1) + .setProgressListener(parallelTransferOptions.getProgressListener()) + .setMaxSingleUploadSizeLong(parallelTransferOptions.getMaxSingleUploadSizeLong()); + } + + return new BlobDownloadToFileOptions(options.getFilePath()).setRange(options.getRange()) + .setParallelTransferOptions(adjustedParallelOptions) + .setDownloadRetryOptions(options.getDownloadRetryOptions()) + .setRequestConditions(options.getRequestConditions()) + .setRetrieveContentRangeMd5(options.isRetrieveContentRangeMd5()) + .setOpenOptions(options.getOpenOptions()) + .setContentValidationOptions(options.getContentValidationOptions()); + } + /** * Deletes the specified blob or snapshot. To delete a blob with its snapshots use * {@link #deleteWithResponse(DeleteSnapshotsOptionType, BlobRequestConditions, Duration, Context)} and set diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index b22f791ac112..8822c71a53f6 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -3,26 +3,40 @@ package com.azure.storage.blob; +import com.azure.core.test.TestMode; import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.Context; import com.azure.core.util.FluxUtil; import com.azure.storage.blob.models.BlobRange; import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.common.DownloadContentValidationOptions; +import com.azure.storage.common.ParallelTransferOptions; import com.azure.storage.common.implementation.Constants; -import com.azure.storage.common.implementation.structuredmessage.StructuredMessageEncoder; -import com.azure.storage.common.implementation.structuredmessage.StructuredMessageFlags; +import com.azure.storage.common.implementation.StorageCrc64Calculator; import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.condition.EnabledIf; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import reactor.core.publisher.Flux; import reactor.test.StepVerifier; +import reactor.util.function.Tuples; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -32,6 +46,7 @@ * Tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. */ +@Execution(ExecutionMode.SAME_THREAD) public class BlobMessageDecoderDownloadTests extends BlobTestBase { private BlobAsyncClient bc; @@ -43,35 +58,73 @@ public void setup() { bc.upload(Flux.just(ByteBuffer.wrap(new byte[0])), null).block(); } + /** + * Aligned with .NET: decoder-only client; live service returns structured-encoded body. + * Runs in LIVE only: playback may not replay streaming response body, causing "decoded 0" otherwise. + */ @Test + @EnabledIf("isLiveMode") public void downloadStreamWithResponseContentValidation() throws IOException { - byte[] randomData = getRandomByteArray(4 * Constants.KB); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, Constants.KB, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + byte[] randomData = getRandomByteArray(10 * 1024 * 1024); // 10 MB - Flux input = Flux.just(encodedData); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); StepVerifier - .create(bc.upload(input, null, true) - .then(bc.downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, - (BlobRequestConditions) null, false, validationOptions)) + .create(downloadClient + .downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, (BlobRequestConditions) null, + false, validationOptions) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) .verifyComplete(); } + /** + * Mirrors .NET StructuredMessagePopulatesCrcDownloadStreaming: after consuming the response stream with + * StorageCrc64 validation, the response details (ContentCrc64) are populated with the computed CRC. + * Aligned with .NET: decoder-only client, live/playback returns structured-encoded body. + */ + @Test + public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOException { + int dataLength = Constants.KB; + byte[] data = getRandomByteArray(dataLength); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + long expectedCrc = StorageCrc64Calculator.compute(data, 0); + byte[] expectedCrcBytes = new byte[8]; + ByteBuffer.wrap(expectedCrcBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(expectedCrc); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, (BlobRequestConditions) null, + false, validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes)))) + .assertNext(tuple -> { + TestUtils.assertArraysEqual(data, tuple.getT2()); + assertNotNull(tuple.getT1().getDeserializedHeaders().getContentCrc64(), + "ContentCrc64 should be populated after stream consumption"); + TestUtils.assertArraysEqual(expectedCrcBytes, tuple.getT1().getDeserializedHeaders().getContentCrc64()); + }) + .verifyComplete(); + } + @Test public void downloadStreamWithResponseContentValidationRange() throws IOException { byte[] randomData = getRandomByteArray(4 * Constants.KB); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, Constants.KB, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); + Flux input = Flux.just(ByteBuffer.wrap(randomData)); // Range download without validation should work BlobRange range = new BlobRange(0, 512L); @@ -86,21 +139,14 @@ public void downloadStreamWithResponseContentValidationRange() throws IOExceptio }).verifyComplete(); } + /** + * Mirrors .NET UninterruptedStream: decoder-only client, live/playback returns structured-encoded body. + */ @Test public void uninterruptedStreamWithStructuredMessageDecoding() throws IOException { - // Test: Verify that structured message decoding works correctly without any interruptions - // This mirrors the .NET test: UninterruptedStream byte[] randomData = getRandomByteArray(4 * Constants.KB); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, Constants.KB, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - Flux input = Flux.just(encodedData); - - // Upload the encoded data - bc.upload(input, null, true).block(); - - // Create a download client with decoder policy but NO mock interruption policy StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); @@ -108,84 +154,89 @@ public void uninterruptedStreamWithStructuredMessageDecoding() throws IOExceptio DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - // Download with validation - should succeed without any interruptions StepVerifier .create( downloadClient .downloadStreamWithResponse((BlobRange) null, null, (BlobRequestConditions) null, false, validationOptions) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> { - // Verify the decoded data matches the original - TestUtils.assertArraysEqual(result, randomData); - }) + .assertNext(result -> TestUtils.assertArraysEqual(result, randomData)) .verifyComplete(); } + /** + * Mirrors .NET Interrupt_DataIntact (single interrupt): decoder + fault policy only; live/playback returns + * structured-encoded body; MockPartialResponsePolicy simulates interruption like .NET FaultyDownloadPipelinePolicy. + */ @Test public void interruptWithDataIntact() throws IOException { - // Test: Verify that data remains intact after a single interruption and retry - // This mirrors the .NET test: Interrupt_DataIntact with single interrupt final int segmentSize = Constants.KB; byte[] randomData = getRandomByteArray(4 * segmentSize); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - // Create a policy that will simulate 1 network interruption at a specific position - // Interrupt after first segment completes to test smart retry from segment boundary - // Interrupt after first segment + 3 reads + 10 bytes (mirrors .NET interruptPos) int interruptPos = segmentSize + (3 * 128) + 10; // readLen in .NET test = 128 bytes MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); - // Upload the encoded data - bc.upload(input, null, true).block(); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - // Create download client with mock interruption and decoder policies StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient = new BlobClientBuilder().endpoint(bc.getBlobUrl()) - .addPolicy(decoderPolicy) - // Ensure the fault policy runs before decoding and on the initial call. - .addPolicy(mockPolicy) - .credential(ENVIRONMENT.getPrimaryAccount().getCredential()) - .buildAsyncClient(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + bc.getBlobUrl(), decoderPolicy, mockPolicy); DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - // Download with validation - should succeed despite the interruption - StepVerifier.create(downloadClient + StepVerifier + .create(downloadClient + .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .verifyComplete(); + } + + /** + * Blocking variant of interruptWithDataIntact; aligned with .NET (decoder + fault policy only). + */ + @Test + public void interruptWithDataIntactBlocking() throws IOException { + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(4 * segmentSize); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + bc.getBlobUrl(), decoderPolicy, mockPolicy); + + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + byte[] result = downloadClient .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { - // Verify the decoded data matches the original exactly - TestUtils.assertArraysEqual(randomData, result); - }).verifyComplete(); + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue())) + .block(); + + TestUtils.assertArraysEqual(randomData, result); } + /** + * Mirrors .NET Interrupt_DataIntact (multiple interrupts): decoder + fault policy only. + */ @Test public void interruptMultipleTimesWithDataIntact() throws IOException { - // Test: Verify that data remains intact after multiple interruptions and retries - // This mirrors the .NET test: Interrupt_DataIntact with multiple interrupts final int segmentSize = Constants.KB; byte[] randomData = getRandomByteArray(4 * segmentSize); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - // Create a policy that will simulate 3 network interruptions int interruptPos = segmentSize + (3 * 128) + 10; MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, bc.getBlobUrl()); - // Upload the encoded data - bc.upload(input, null, true).block(); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - // Create download client with mock interruption and decoder policies StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy, mockPolicy); @@ -194,36 +245,28 @@ public void interruptMultipleTimesWithDataIntact() throws IOException { = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); - // Download with validation - should succeed despite multiple interruptions - StepVerifier.create(downloadClient - .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, - validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { - // Verify the decoded data matches the original exactly - TestUtils.assertArraysEqual(randomData, result); - }).verifyComplete(); + StepVerifier + .create(downloadClient + .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, + validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .verifyComplete(); } + /** + * Mirrors .NET Interrupt_AppropriateRewind: decoder + fault policy only; verifies rewind to segment boundary. + */ @Test public void interruptAndVerifyProperRewind() throws IOException { - // Test: Verify that interruption causes proper rewind to last complete segment boundary - // This mirrors the .NET test: Interrupt_AppropriateRewind final int segmentSize = Constants.KB; byte[] randomData = getRandomByteArray(2 * segmentSize); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - // Create a policy that will simulate 1 interruption at segment boundary + 2 reads + offset (per .NET) int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; // readLen = segmentSize/4 MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); - // Upload the encoded data - bc.upload(input, null, true).block(); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - // Create download client with mock interruption and decoder policies StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy, mockPolicy); @@ -232,51 +275,39 @@ public void interruptAndVerifyProperRewind() throws IOException { = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - // Download with validation StepVerifier.create(downloadClient .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, validationOptions) - // Ensure the fault policy was invoked even if the assertion below fails. .doFinally(signalType -> { System.out.println("[MockPartialResponsePolicy] hits=" + mockPolicy.getHits() + ", triesRemaining=" + mockPolicy.getTriesRemaining() + ", ranges=" + mockPolicy.getRangeHeaders()); assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked"); }) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { - // Verify the decoded data matches the original - TestUtils.assertArraysEqual(randomData, result); - }).verifyComplete(); + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .verifyComplete(); - // Quick sanity: mock interruption should have been hit and range headers recorded. assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); assertTrue(mockPolicy.getRangeHeaders().size() >= 2, "Expected at least the initial request and one retry with a range header"); } + /** + * Mirrors .NET Interrupt_ProperDecode: decoder + fault policy only; proper decode across retries. + */ @ParameterizedTest @ValueSource(booleans = { false, true }) public void interruptAndVerifyProperDecode(boolean multipleInterrupts) throws IOException { - // Test: Verify that after interruption and retry, decoding continues correctly - // Mirrors .NET Interrupt_ProperDecode (multipleInterrupts toggles number of injected faults) final int segmentSize = 128 * Constants.KB; final int dataSize = 4 * Constants.KB; byte[] randomData = getRandomByteArray(dataSize); - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(randomData.length, segmentSize, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(randomData)); - - Flux input = Flux.just(encodedData); - // Create a policy with interruptions to test multi-step decode after retries - // Interrupt after first segment + 3 reads + 10 bytes (per .NET) int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; // readLen = 8KB in .NET MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, bc.getBlobUrl()); - // Upload the encoded data - bc.upload(input, null, true).block(); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - // Create download client with mock interruption and decoder policies StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy, mockPolicy); @@ -285,14 +316,191 @@ public void interruptAndVerifyProperDecode(boolean multipleInterrupts) throws IO = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); - // Download with validation - decoder must properly handle state across retries StepVerifier.create(downloadClient .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, validationOptions) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { - // Verify every byte is correctly decoded despite interruptions assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); TestUtils.assertArraysEqual(randomData, result); }).verifyComplete(); } + + /** + * DownloadToFile with structured message decoding using the same payload size as .NET (Constants.KB). + * Aligned with .NET: decoder-only client, live/playback returns structured-encoded body. + */ + @Test + @Timeout(value = 2, unit = TimeUnit.MINUTES) + public void downloadToFileStructuredMessageSamePayloadAsNet() throws IOException { + int payloadSize = Constants.KB; // same as .NET StructuredMessagePopulatesCrc / transfer validation tests + byte[] randomData = getRandomByteArray(payloadSize); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + Path tempFile = Files.createTempFile("structured-download-net-size", ".bin"); + Files.deleteIfExists(tempFile); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) Constants.KB) + .setInitialTransferSizeLong((long) Constants.KB); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationOptions( + new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); + + try { + StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) + .assertNext(r -> assertNotNull(r.getValue())) + .expectComplete() + .verify(Duration.ofSeconds(60)); + + TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + } finally { + Files.deleteIfExists(tempFile); + } + } + + /** + * Single-chunk DownloadToFile; aligned with .NET: decoder-only client, live/playback. + */ + @Test + @Timeout(value = 2, unit = TimeUnit.MINUTES) + public void downloadToFileStructuredMessageSingleChunk() throws IOException { + int blockSize = 512; + int payloadSize = blockSize; + byte[] randomData = getRandomByteArray(payloadSize); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + Path tempFile = Files.createTempFile("structured-download-single", ".bin"); + Files.deleteIfExists(tempFile); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) + .setInitialTransferSizeLong((long) blockSize); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationOptions( + new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); + + try { + StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) + .assertNext(r -> assertNotNull(r.getValue())) + .expectComplete() + .verify(Duration.ofSeconds(60)); + + TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + } finally { + Files.deleteIfExists(tempFile); + } + } + + /** + * Parallel download with structured message validation; aligned with .NET: no mock, decoder only, live/playback, + * default concurrency (no explicit maxConcurrency), payload (4*blockSize)+1 and block sizes 512/2048 per .NET. + */ + @ParameterizedTest + @ValueSource(ints = { 512, 2048 }) + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public void downloadToFileStructuredMessageParallel(int blockSize) throws IOException { + int payloadSize = (4 * blockSize) + 1; + byte[] randomData = getRandomByteArray(payloadSize); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + Path tempFile = Files.createTempFile("structured-download", ".bin"); + Files.deleteIfExists(tempFile); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) + .setInitialTransferSizeLong((long) blockSize); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationOptions( + new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); + + try { + StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) + .assertNext(r -> assertNotNull(r.getValue())) + .expectComplete() + .verify(Duration.ofSeconds(60)); + + TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + } finally { + Files.deleteIfExists(tempFile); + } + } + + /** + * Sync variant: same payload and transfer options as .NET (default concurrency; sync path forces 1). Multi-chunk + * DownloadTo with structured message validation. + */ + @ParameterizedTest + @ValueSource(ints = { 512, 2048 }) + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public void downloadToFileStructuredMessageParallelSync(int blockSize) throws IOException { + int payloadSize = (4 * blockSize) + 1; + byte[] randomData = getRandomByteArray(payloadSize); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobClient downloadClient + = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + Path tempFile = Files.createTempFile("structured-download-sync", ".bin"); + Files.deleteIfExists(tempFile); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) + .setInitialTransferSizeLong((long) blockSize); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationOptions( + new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); + + try { + assertNotNull(downloadClient.downloadToFileWithResponse(options, null, Context.NONE).getValue()); + TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + } finally { + Files.deleteIfExists(tempFile); + } + } + + /** + * Mirrors .NET OlderServiceVersionThrowsOnStructuredMessage: when using a service version before structured + * message was introduced (V2024_11_04), a download with structured message validation enabled and a range that + * would trigger structured message response must throw (service returns error for unsupported feature). + */ + @Test + @Timeout(value = 2, unit = TimeUnit.MINUTES) + @EnabledIf("isLiveMode") + public void olderServiceVersionThrowsOnStructuredMessage() { + int dataLength = Constants.KB; + byte[] data = getRandomByteArray(dataLength); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobClientBuilder builder = new BlobClientBuilder().endpoint(bc.getBlobUrl()) + .credential(ENVIRONMENT.getPrimaryAccount().getCredential()) + .serviceVersion(BlobServiceVersion.V2024_11_04); + instrument(builder); + BlobAsyncClient oldVersionClient = builder.buildAsyncClient(); + + BlobRange range = new BlobRange(0, (long) Constants.KB); + DownloadContentValidationOptions validationOptions + = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); + + StepVerifier + .create(oldVersionClient.downloadStreamWithResponse(range, null, null, false, validationOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .verifyError(BlobStorageException.class); + } + + static boolean isLiveMode() { + return ENVIRONMENT.getTestMode() == TestMode.LIVE; + } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java index 34213e7300d7..eef098cb60bd 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java @@ -14,6 +14,7 @@ @Fluent public final class ParallelTransferOptions { private Long blockSize; + private Long initialTransferSize; private Integer maxConcurrency; private ProgressReceiver progressReceiver; private Long maxSingleUploadSize; @@ -77,6 +78,29 @@ public Long getBlockSizeLong() { return this.blockSize; } + /** + * Gets the size of the first range requested when downloading. + * @return The initial transfer size. + */ + public Long getInitialTransferSizeLong() { + return this.initialTransferSize; + } + + /** + * Sets the size of the first range requested when downloading. + * This value may be larger than the block size used for subsequent ranges. + * + * @param initialTransferSize The initial transfer size. + * @return The ParallelTransferOptions object itself. + */ + public ParallelTransferOptions setInitialTransferSizeLong(Long initialTransferSize) { + if (initialTransferSize != null) { + StorageImplUtils.assertInBounds("initialTransferSize", initialTransferSize, 1, Long.MAX_VALUE); + } + this.initialTransferSize = initialTransferSize; + return this; + } + /** * Sets the block size. * For upload, The block size is the size of each block that will be staged. This value also determines the number diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index ea955f63b829..cff3feea6afa 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -85,6 +85,16 @@ public final class Constants { public static final String STORAGE_LOG_STRING_TO_SIGN = "Azure-Storage-Log-String-To-Sign"; + /** + * System property name for enabling legacy default concurrency behavior. + */ + public static final String USE_LEGACY_DEFAULT_CONCURRENCY_PROPERTY = "Azure.Storage.UseLegacyDefaultConcurrency"; + + /** + * Environment variable name for enabling legacy default concurrency behavior. + */ + public static final String USE_LEGACY_DEFAULT_CONCURRENCY_ENV_VAR = "AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY"; + public static final String PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION = "AZURE_STORAGE_SAS_SERVICE_VERSION"; public static final String SAS_SERVICE_VERSION @@ -105,6 +115,13 @@ public final class Constants { public static final String STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY = "azure-storage-structured-message-validation-options"; + /** + * Context key used to signal that structured message decoding should be scoped to a single response. + * This is used for parallel range downloads where each response is independently structured-encoded. + */ + public static final String STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY + = "azure-storage-structured-message-response-scoped"; + /** * Context key used to pass stateful decoder state across retry requests. */ @@ -117,6 +134,22 @@ public final class Constants { public static final String STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY = "azure-storage-structured-message-decoder-state-ref"; + /** + * Context key used to pass the number of decoded bytes to skip on the next structured message response. + */ + public static final String STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY + = "azure-storage-structured-message-decoder-skip-bytes"; + /** + * Context key used to share structured message CRC aggregation state across retries. + */ + public static final String STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY + = "azure-storage-structured-message-aggregate-crc"; + + /** + * Structured message header value for CRC64 validation. + */ + public static final String STRUCTURED_MESSAGE_CRC64_BODY_TYPE = "XSM/1.0; properties=crc64"; + private Constants() { } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java index cd4c2e766302..2ed9ba6c539b 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java @@ -2559,6 +2559,388 @@ public static long compute(byte[] src, long uCrc) { return ~uCrc; // Flip all bits of uCrc and return as long } + /** + * Computes the CRC64 checksum for a slice of a byte array. Avoids copying when combined with + * {@link #compute(ByteBuffer, long)} for array-backed buffers. + * + * @param src the byte array. + * @param offset the starting offset in the array. + * @param length the number of bytes to process. + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(byte[] src, int offset, int length, long uCrc) { + int pData = 0; + long uSize = length; + long uBytes, uStop; + + uCrc = ~uCrc; + + uStop = uSize - (uSize % 32); + if (uStop >= 2 * 32) { + long uCrc0 = 0L; + long uCrc1 = 0L; + long uCrc2 = 0L; + long uCrc3 = 0L; + + int pLast = pData + (int) uStop - 32; + uSize -= uStop; + uCrc0 = uCrc; + + ByteBuffer buffer = ByteBuffer.wrap(src, offset, length).order(ByteOrder.LITTLE_ENDIAN); + + for (; pData < pLast; pData += 32) { + long b0 = buffer.getLong(pData) ^ uCrc0; + long b1 = buffer.getLong(pData + 8) ^ uCrc1; + long b2 = buffer.getLong(pData + 16) ^ uCrc2; + long b3 = buffer.getLong(pData + 24) ^ uCrc3; + + uCrc0 = M_U32[7 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 = M_U32[7 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 = M_U32[7 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 = M_U32[7 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[6 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[6 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[6 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[6 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[5 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[5 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[5 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[5 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[4 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[4 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[4 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[4 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[3 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[3 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[3 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[3 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[2 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[2 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[2 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[2 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[((int) (b0 & 0xFF))]; + uCrc1 ^= M_U32[((int) (b1 & 0xFF))]; + uCrc2 ^= M_U32[((int) (b2 & 0xFF))]; + uCrc3 ^= M_U32[((int) (b3 & 0xFF))]; + } + + uCrc = 0; + uCrc ^= ByteBuffer.wrap(src, offset + pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, offset + pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc1; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, offset + pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc2; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, offset + pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc3; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + pData += 32; + } + + for (uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[offset + pData]) & 0xFF)]; + } + + return ~uCrc; + } + + /** + * Computes the CRC64 checksum for the remaining bytes in a ByteBuffer. When the buffer has a backing array, + * avoids copying; otherwise copies once. + * + * @param buffer the buffer (position to limit). + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(ByteBuffer buffer, long uCrc) { + if (buffer == null || !buffer.hasRemaining()) { + return uCrc; + } + if (buffer.hasArray()) { + return compute(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining(), uCrc); + } + byte[] copy = new byte[buffer.remaining()]; + buffer.duplicate().get(copy); + return compute(copy, uCrc); + } + + /** + * Updates both segment and message CRC64 in a single pass over the data (half the memory reads of two separate + * {@link #compute(byte[], long)} calls). Used by the structured message decoder for performance. + * + * @param src the byte array. + * @param segmentCrc the current segment CRC value. + * @param messageCrc the current message CRC value. + * @return long[0] = new segment CRC, long[1] = new message CRC. + */ + public static long[] computeTwo(byte[] src, long segmentCrc, long messageCrc) { + segmentCrc = ~segmentCrc; + messageCrc = ~messageCrc; + + int pData = 0; + long uSize = src.length; + long uStop = uSize - (uSize % 32); + + if (uStop >= 2 * 32) { + long s0 = segmentCrc, s1 = 0, s2 = 0, s3 = 0; + long m0 = messageCrc, m1 = 0, m2 = 0, m3 = 0; + int pLast = (int) uStop - 32; + uSize -= uStop; + ByteBuffer buf = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); + + for (; pData < pLast; pData += 32) { + long b0 = buf.getLong(pData); + long b1 = buf.getLong(pData + 8); + long b2 = buf.getLong(pData + 16); + long b3 = buf.getLong(pData + 24); + + long bs0 = b0 ^ s0, bs1 = b1 ^ s1, bs2 = b2 ^ s2, bs3 = b3 ^ s3; + long bm0 = b0 ^ m0, bm1 = b1 ^ m1, bm2 = b2 ^ m2, bm3 = b3 ^ m3; + + s0 = M_U32[7 * 256 + ((int) (bs0 & 0xFF))]; + bs0 >>>= 8; + s1 = M_U32[7 * 256 + ((int) (bs1 & 0xFF))]; + bs1 >>>= 8; + s2 = M_U32[7 * 256 + ((int) (bs2 & 0xFF))]; + bs2 >>>= 8; + s3 = M_U32[7 * 256 + ((int) (bs3 & 0xFF))]; + bs3 >>>= 8; + s0 ^= M_U32[6 * 256 + ((int) (bs0 & 0xFF))]; + bs0 >>>= 8; + s1 ^= M_U32[6 * 256 + ((int) (bs1 & 0xFF))]; + bs1 >>>= 8; + s2 ^= M_U32[6 * 256 + ((int) (bs2 & 0xFF))]; + bs2 >>>= 8; + s3 ^= M_U32[6 * 256 + ((int) (bs3 & 0xFF))]; + bs3 >>>= 8; + s0 ^= M_U32[5 * 256 + ((int) (bs0 & 0xFF))]; + bs0 >>>= 8; + s1 ^= M_U32[5 * 256 + ((int) (bs1 & 0xFF))]; + bs1 >>>= 8; + s2 ^= M_U32[5 * 256 + ((int) (bs2 & 0xFF))]; + bs2 >>>= 8; + s3 ^= M_U32[5 * 256 + ((int) (bs3 & 0xFF))]; + bs3 >>>= 8; + s0 ^= M_U32[4 * 256 + ((int) (bs0 & 0xFF))]; + bs0 >>>= 8; + s1 ^= M_U32[4 * 256 + ((int) (bs1 & 0xFF))]; + bs1 >>>= 8; + s2 ^= M_U32[4 * 256 + ((int) (bs2 & 0xFF))]; + bs2 >>>= 8; + s3 ^= M_U32[4 * 256 + ((int) (bs3 & 0xFF))]; + bs3 >>>= 8; + s0 ^= M_U32[3 * 256 + ((int) (bs0 & 0xFF))]; + bs0 >>>= 8; + s1 ^= M_U32[3 * 256 + ((int) (bs1 & 0xFF))]; + bs1 >>>= 8; + s2 ^= M_U32[3 * 256 + ((int) (bs2 & 0xFF))]; + bs2 >>>= 8; + s3 ^= M_U32[3 * 256 + ((int) (bs3 & 0xFF))]; + bs3 >>>= 8; + s0 ^= M_U32[2 * 256 + ((int) (bs0 & 0xFF))]; + bs0 >>>= 8; + s1 ^= M_U32[2 * 256 + ((int) (bs1 & 0xFF))]; + bs1 >>>= 8; + s2 ^= M_U32[2 * 256 + ((int) (bs2 & 0xFF))]; + bs2 >>>= 8; + s3 ^= M_U32[2 * 256 + ((int) (bs3 & 0xFF))]; + bs3 >>>= 8; + s0 ^= M_U32[256 + ((int) (bs0 & 0xFF))]; + bs0 >>>= 8; + s1 ^= M_U32[256 + ((int) (bs1 & 0xFF))]; + bs1 >>>= 8; + s2 ^= M_U32[256 + ((int) (bs2 & 0xFF))]; + bs2 >>>= 8; + s3 ^= M_U32[256 + ((int) (bs3 & 0xFF))]; + bs3 >>>= 8; + s0 ^= M_U32[((int) (bs0 & 0xFF))]; + s1 ^= M_U32[((int) (bs1 & 0xFF))]; + s2 ^= M_U32[((int) (bs2 & 0xFF))]; + s3 ^= M_U32[((int) (bs3 & 0xFF))]; + + m0 = M_U32[7 * 256 + ((int) (bm0 & 0xFF))]; + bm0 >>>= 8; + m1 = M_U32[7 * 256 + ((int) (bm1 & 0xFF))]; + bm1 >>>= 8; + m2 = M_U32[7 * 256 + ((int) (bm2 & 0xFF))]; + bm2 >>>= 8; + m3 = M_U32[7 * 256 + ((int) (bm3 & 0xFF))]; + bm3 >>>= 8; + m0 ^= M_U32[6 * 256 + ((int) (bm0 & 0xFF))]; + bm0 >>>= 8; + m1 ^= M_U32[6 * 256 + ((int) (bm1 & 0xFF))]; + bm1 >>>= 8; + m2 ^= M_U32[6 * 256 + ((int) (bm2 & 0xFF))]; + bm2 >>>= 8; + m3 ^= M_U32[6 * 256 + ((int) (bm3 & 0xFF))]; + bm3 >>>= 8; + m0 ^= M_U32[5 * 256 + ((int) (bm0 & 0xFF))]; + bm0 >>>= 8; + m1 ^= M_U32[5 * 256 + ((int) (bm1 & 0xFF))]; + bm1 >>>= 8; + m2 ^= M_U32[5 * 256 + ((int) (bm2 & 0xFF))]; + bm2 >>>= 8; + m3 ^= M_U32[5 * 256 + ((int) (bm3 & 0xFF))]; + bm3 >>>= 8; + m0 ^= M_U32[4 * 256 + ((int) (bm0 & 0xFF))]; + bm0 >>>= 8; + m1 ^= M_U32[4 * 256 + ((int) (bm1 & 0xFF))]; + bm1 >>>= 8; + m2 ^= M_U32[4 * 256 + ((int) (bm2 & 0xFF))]; + bm2 >>>= 8; + m3 ^= M_U32[4 * 256 + ((int) (bm3 & 0xFF))]; + bm3 >>>= 8; + m0 ^= M_U32[3 * 256 + ((int) (bm0 & 0xFF))]; + bm0 >>>= 8; + m1 ^= M_U32[3 * 256 + ((int) (bm1 & 0xFF))]; + bm1 >>>= 8; + m2 ^= M_U32[3 * 256 + ((int) (bm2 & 0xFF))]; + bm2 >>>= 8; + m3 ^= M_U32[3 * 256 + ((int) (bm3 & 0xFF))]; + bm3 >>>= 8; + m0 ^= M_U32[2 * 256 + ((int) (bm0 & 0xFF))]; + bm0 >>>= 8; + m1 ^= M_U32[2 * 256 + ((int) (bm1 & 0xFF))]; + bm1 >>>= 8; + m2 ^= M_U32[2 * 256 + ((int) (bm2 & 0xFF))]; + bm2 >>>= 8; + m3 ^= M_U32[2 * 256 + ((int) (bm3 & 0xFF))]; + bm3 >>>= 8; + m0 ^= M_U32[256 + ((int) (bm0 & 0xFF))]; + bm0 >>>= 8; + m1 ^= M_U32[256 + ((int) (bm1 & 0xFF))]; + bm1 >>>= 8; + m2 ^= M_U32[256 + ((int) (bm2 & 0xFF))]; + bm2 >>>= 8; + m3 ^= M_U32[256 + ((int) (bm3 & 0xFF))]; + bm3 >>>= 8; + m0 ^= M_U32[((int) (bm0 & 0xFF))]; + m1 ^= M_U32[((int) (bm1 & 0xFF))]; + m2 ^= M_U32[((int) (bm2 & 0xFF))]; + m3 ^= M_U32[((int) (bm3 & 0xFF))]; + } + + long uCrc = 0; + uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s0; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s1; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s2; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s3; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + segmentCrc = uCrc; // keep internal form for tail + + uCrc = 0; + uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m0; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m1; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m2; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m3; + for (int i = 0; i < 8; i++) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + } + messageCrc = uCrc; // keep internal form for tail + + pData += 32; + } + + for (long uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { + long byteVal = src[pData] & 0xFF; + segmentCrc = (segmentCrc >>> 8) ^ M_U1[(int) ((segmentCrc ^ byteVal) & 0xFF)]; + messageCrc = (messageCrc >>> 8) ^ M_U1[(int) ((messageCrc ^ byteVal) & 0xFF)]; + } + + return new long[] { ~segmentCrc, ~messageCrc }; + } + /** * Concatenates two CRC64 values by combining their initial and final CRC values and sizes. * This method ensures unsigned behavior and uses the `mulX_N` method to perform necessary diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 29d26702555b..2058f5144431 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -9,7 +9,9 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.CRC64_LENGTH; @@ -43,16 +45,21 @@ public class StructuredMessageDecoder { // Offset tracking private long messageOffset = 0; // Absolute encoded bytes consumed from the message private long totalDecodedPayloadBytes = 0; // Total decoded (payload) bytes output + private long decodedBytesAtLastCompleteSegment = 0; // Current segment state private int currentSegmentNumber = 0; private long currentSegmentContentLength = 0; private long currentSegmentContentOffset = 0; + private int lastCompleteSegmentNumber = 0; // CRC validation private long messageCrc64 = 0; + private long messageCrc64AtLastCompleteSegment = 0; private long segmentCrc64 = 0; private final Map segmentCrcs = new HashMap<>(); + private final Map segmentLengths = new HashMap<>(); + private final List completedSegments = new ArrayList<>(); // Smart retry tracking - lastCompleteSegmentStart is the absolute offset where the last // fully completed segment ended. This is the safe retry boundary. @@ -158,6 +165,15 @@ public long getTotalDecodedPayloadBytes() { return totalDecodedPayloadBytes; } + /** + * Gets the total decoded payload bytes at the last complete segment boundary. + * + * @return The decoded byte count at the last complete segment boundary. + */ + public long getDecodedBytesAtLastCompleteSegment() { + return decodedBytesAtLastCompleteSegment; + } + /** * Advances the message offset by the specified number of bytes. * This should be called after consuming an encoded segment to maintain @@ -181,7 +197,12 @@ public void advanceMessageOffset(long bytes) { * the data being provided from the retry offset. */ public void resetToLastCompleteSegment() { - if (messageOffset != lastCompleteSegmentStart) { + boolean needsReset = messageOffset != lastCompleteSegmentStart + || pendingBytes.size() > 0 + || currentSegmentContentOffset != 0 + || currentSegmentContentLength != 0 + || currentSegmentNumber != lastCompleteSegmentNumber; + if (needsReset) { LOGGER.atInfo() .addKeyValue("fromOffset", messageOffset) .addKeyValue("toOffset", lastCompleteSegmentStart) @@ -190,9 +211,13 @@ public void resetToLastCompleteSegment() { .addKeyValue("currentSegmentContentLength", currentSegmentContentLength) .log("Resetting decoder to last complete segment boundary"); messageOffset = lastCompleteSegmentStart; + totalDecodedPayloadBytes = decodedBytesAtLastCompleteSegment; + messageCrc64 = messageCrc64AtLastCompleteSegment; // Reset current segment state - next decode will read the segment header currentSegmentContentOffset = 0; currentSegmentContentLength = 0; + currentSegmentNumber = lastCompleteSegmentNumber; + segmentCrc64 = 0; // Clear any pending bytes since we're resetting to a known boundary pendingBytes.reset(); } else { @@ -305,6 +330,15 @@ public StructuredMessageFlags getFlags() { return flags; } + /** + * Gets the completed segments in decode order. + * + * @return List of completed segment CRCs and lengths. + */ + public List getCompletedSegments() { + return new ArrayList<>(completedSegments); + } + /** * Gets the expected message length from the header. * @@ -459,6 +493,7 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { currentSegmentNumber = segmentNum; currentSegmentContentLength = segmentSize; currentSegmentContentOffset = 0; + segmentLengths.put(currentSegmentNumber, segmentSize); if (flags == StructuredMessageFlags.STORAGE_CRC64) { segmentCrc64 = 0; @@ -545,11 +580,16 @@ private boolean tryReadSegmentFooter(ByteBuffer buffer) { consumeBytes(CRC64_LENGTH, buffer); segmentCrcs.put(currentSegmentNumber, segmentCrc64); + long length = segmentLengths.getOrDefault(currentSegmentNumber, currentSegmentContentLength); + completedSegments.add(new SegmentInfo(segmentCrc64, length)); messageOffset += CRC64_LENGTH; } // Mark that this segment is complete lastCompleteSegmentStart = messageOffset; + decodedBytesAtLastCompleteSegment = totalDecodedPayloadBytes; + messageCrc64AtLastCompleteSegment = messageCrc64; + lastCompleteSegmentNumber = currentSegmentNumber; LOGGER.atInfo() .addKeyValue("segmentNum", currentSegmentNumber) .addKeyValue("offset", lastCompleteSegmentStart) @@ -625,15 +665,10 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { // Step 2: Process segments while (messageOffset < messageLength) { - // If all segments are done, proceed to message footer before attempting any new segment header. - if (currentSegmentNumber == numSegments && currentSegmentContentOffset == currentSegmentContentLength) { - if (!tryReadMessageFooter(buffer)) { - break; - } - } - - // Read segment header if needed - if (currentSegmentContentOffset == currentSegmentContentLength) { + // Read segment header only after the *previous* segment is fully complete (including its CRC footer). + // Otherwise we would misinterpret segment N's footer bytes as segment N+1's header when the footer + // is split across network buffers (e.g. "Unexpected segment number. Expected: 2, got: 10710"). + if (lastCompleteSegmentNumber == currentSegmentNumber && currentSegmentNumber < numSegments) { if (!tryReadSegmentHeader(buffer)) { break; // Need more bytes for segment header } @@ -647,6 +682,10 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { if (!tryReadSegmentFooter(buffer)) { break; // Need more bytes for segment footer } + // After reading this segment's footer, if it was the last segment, read message footer. + if (currentSegmentNumber == numSegments && !tryReadMessageFooter(buffer)) { + break; + } } // Check if all segments are complete @@ -706,7 +745,8 @@ public ByteBuffer decode(ByteBuffer buffer, int size) { } while (buffer.hasRemaining() && decodedContent.size() < size) { - if (currentSegmentContentOffset == currentSegmentContentLength) { + // Only read next segment header after previous segment is fully complete (including footer). + if (lastCompleteSegmentNumber == currentSegmentNumber && currentSegmentNumber < numSegments) { if (!tryReadSegmentHeader(buffer)) { break; // Need more bytes } @@ -761,6 +801,27 @@ public boolean isComplete() { return messageLength != -1 && messageOffset >= messageLength; } + /** + * Represents a completed segment with CRC and length. + */ + public static final class SegmentInfo { + private final long crc64; + private final long length; + + public SegmentInfo(long crc64, long length) { + this.crc64 = crc64; + this.length = length; + } + + public long getCrc64() { + return crc64; + } + + public long getLength() { + return length; + } + } + /** * Enriches an exception message with decoder offset information for debugging and retry. * Format: "original message [decoderOffset=X,lastCompleteSegment=Y]" diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index ea0cb5f4ac81..3f80d7645c40 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -15,6 +15,7 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.storage.common.DownloadContentValidationOptions; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.StorageCrc64Calculator; import com.azure.storage.common.implementation.structuredmessage.StructuredMessageDecoder; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -22,6 +23,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; @@ -42,6 +45,9 @@ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); private static final String EXPECTED_LENGTH_CONTEXT_KEY = "azStructuredMsgExpectedLength"; + private static final HttpHeaderName X_MS_STRUCTURED_BODY = HttpHeaderName.fromString("x-ms-structured-body"); + private static final HttpHeaderName X_MS_STRUCTURED_CONTENT_LENGTH + = HttpHeaderName.fromString("x-ms-structured-content-length"); /** * Machine-readable token pattern for extracting retry start offset from exception messages. @@ -130,23 +136,38 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN } return next.process().map(httpResponse -> { + LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("StorageContentValidationDecoderPolicy received response"); // Only apply decoding to download responses (GET requests with body) if (!isDownloadResponse(httpResponse)) { + LOGGER.atVerbose() + .log("StorageContentValidationDecoderPolicy not a download response, passing through"); return httpResponse; } DownloadContentValidationOptions validationOptions = getValidationOptions(context); - Long contentLength = getContentLength(httpResponse.getHeaders()); + // Structured messages are scoped to a single response; always decode per response. + boolean responseScoped = true; + // Decoder expected length must match the 8-byte "message length" in the XSM body header, which is the + // encoded stream size. Use Content-Length (not x-ms-structured-content-length) so validation passes. + Long contentLength = getContentLength(httpResponse.getHeaders(), responseScoped); + + if (validationOptions != null && validationOptions.isStructuredMessageValidationEnabled()) { + String structuredBody = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_BODY); + String structuredContentLength = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_CONTENT_LENGTH); + if (structuredBody == null || structuredContentLength == null) { + throw LOGGER.logExceptionAsError(new IllegalStateException( + "Structured message was requested but the response did not acknowledge it.")); + } + } if (contentLength != null && contentLength > 0 && validationOptions != null) { - // Preserve the original encoded length across retries; range responses may advertise smaller lengths. - long expectedLength = context.getData(EXPECTED_LENGTH_CONTEXT_KEY) - .filter(Long.class::isInstance) - .map(Long.class::cast) - .orElse(contentLength); - - // Cache for subsequent retries. - context.setData(EXPECTED_LENGTH_CONTEXT_KEY, expectedLength); + long expectedLength = contentLength; + LOGGER.atVerbose() + .addKeyValue("expectedLength", expectedLength) + .addKeyValue("thread", Thread.currentThread().getName()) + .log("StorageContentValidationDecoderPolicy creating decoder"); AtomicReference decoderStateHolder = null; Object decoderStateHolderObj @@ -157,19 +178,41 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN decoderStateHolder = tmp; } - // Get or create decoder with state tracking - DecoderState decoderState = getOrCreateDecoderState(context, expectedLength); + // Always create a new decoder per response (matches .NET behavior for structured messages). + AggregateCrcState aggregateCrcState + = context.getData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY) + .filter(value -> value instanceof AggregateCrcState) + .map(value -> (AggregateCrcState) value) + .orElse(null); + + DecoderState decoderState = new DecoderState(expectedLength, aggregateCrcState); + + Object skipBytesObj + = context.getData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY).orElse(null); + if (skipBytesObj instanceof Number) { + decoderState.setDecodedBytesToSkip(((Number) skipBytesObj).longValue()); + } if (decoderStateHolder != null) { decoderStateHolder.set(decoderState); } // Decode using the stateful decoder - Flux decodedStream = decodeStream(httpResponse.getBody(), decoderState); - - // Update context with decoder state for potential retries - context.setData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY, decoderState); + Flux decodedStream = decodeStream(httpResponse.getBody(), decoderState) + .doOnSubscribe(s -> LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("StorageContentValidationDecoderPolicy decoded flux subscribed")) + .doOnComplete(() -> LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("StorageContentValidationDecoderPolicy decoded flux completed")) + .doOnError(e -> LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .addKeyValue("error", e) + .log("StorageContentValidationDecoderPolicy decoded flux error")); + LOGGER.atVerbose() + .addKeyValue("thread", Thread.currentThread().getName()) + .log("StorageContentValidationDecoderPolicy returning DecodedResponse"); return new DecodedResponse(httpResponse, decodedStream, decoderState); } @@ -191,7 +234,7 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN */ private Flux decodeStream(Flux encodedFlux, DecoderState state) { return encodedFlux.concatMap(encodedBuffer -> { - // If decoding already completed, drop any subsequent buffers (can happen with late keep-alive frames). + // If decoding already completed, ignore subsequent buffers. if (state.decoder.isComplete()) { LOGGER.atVerbose() .addKeyValue("bufferLength", encodedBuffer == null ? "null" : encodedBuffer.remaining()) @@ -225,53 +268,13 @@ private Flux decodeStream(Flux encodedFlux, DecoderState .addKeyValue("lastCompleteSegment", state.decoder.getLastCompleteSegmentStart()) .log("Decode chunk result"); + updateProgress(state); + switch (result.getStatus()) { case SUCCESS: case NEED_MORE_BYTES: - // Accumulate partial decoded bytes; emit only when the full message is validated. - ByteBuffer partial = result.getDecodedPayload(); - if (partial != null && partial.hasRemaining()) { - state.appendPartial(partial); - } - updateProgress(state); - return Flux.empty(); - case COMPLETED: - // Append any final bytes from this chunk, then emit the full accumulated payload. - ByteBuffer decodedPayload = result.getDecodedPayload(); - if (decodedPayload != null && decodedPayload.hasRemaining()) { - state.appendPartial(decodedPayload); - } - - decodedPayload = state.drainPartial(); - updateProgress(state); - - if (decodedPayload != null && decodedPayload.hasRemaining()) { - long skip = state.decodedBytesToSkip.get(); - if (skip > 0) { - if (skip >= decodedPayload.remaining()) { - state.decodedBytesToSkip.addAndGet(-decodedPayload.remaining()); - decodedPayload = null; - } else { - int skipCount = (int) skip; - decodedPayload.position(decodedPayload.position() + skipCount); - decodedPayload = decodedPayload.slice(); - state.decodedBytesToSkip.addAndGet(-skipCount); - } - } - - if (decodedPayload != null && decodedPayload.hasRemaining()) { - // Return a defensive copy to avoid any inadvertent position/limit side effects. - ByteBuffer copy = ByteBuffer.allocate(decodedPayload.remaining()); - copy.put(decodedPayload.duplicate()); - copy.flip(); - decodedPayload = copy; - - state.totalBytesDecoded.addAndGet(decodedPayload.remaining()); - return Flux.just(decodedPayload); - } - } - return Flux.empty(); + return emitDecodedPayload(state, result.getDecodedPayload()); case INVALID: LOGGER.error("Invalid data during decode: {}", result.getMessage()); @@ -287,16 +290,12 @@ private Flux decodeStream(Flux encodedFlux, DecoderState return Flux.error(createRetryableException(state, e.getMessage(), e)); } }).onErrorResume(throwable -> { - // If decoding already completed and we emitted payload, suppress late downstream errors (mirror .NET). - // If no payload was emitted, surface the error so the retriable download can resume properly. + // If decoding already completed, suppress late downstream errors (mirror .NET). if (state.decoder.isComplete()) { - if (state.totalBytesDecoded.get() > 0) { - LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); - return Flux.empty(); - } else { - LOGGER.atInfo().log("Decoder complete with no emitted payload; propagating error to retry"); - } + LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); + return Flux.empty(); } + state.addSegmentsToAggregateIfNeeded(); // Wrap any error with retry offset information if (throwable instanceof IOException) { // Check if already has retry offset token @@ -318,6 +317,16 @@ private Flux decodeStream(Flux encodedFlux, DecoderState return Mono.error(createRetryableException(state, "Stream ended prematurely before structured message decoding completed")); } else { + state.addSegmentsToAggregateIfNeeded(); + if (state.aggregateCrcState != null && state.aggregateCrcState.hasSegments()) { + long composed = state.aggregateCrcState.composeCrc(); + long calculated = state.aggregateCrcState.getRunningCrc(); + if (composed != calculated) { + return Mono.error(LOGGER.logExceptionAsError(new IllegalArgumentException( + "CRC64 mismatch detected in composed structured message. Expected: " + composed + ", got: " + + calculated))); + } + } LOGGER.atInfo() .addKeyValue("messageOffset", state.decoder.getMessageOffset()) .addKeyValue("totalDecodedPayload", state.decoder.getTotalDecodedPayloadBytes()) @@ -337,7 +346,7 @@ private void updateProgress(DecoderState state) { // Only update decodedBytesAtLastCompleteSegment when the boundary changes (new segment validated). if (state.lastCompleteSegmentStart != currentLastCompleteSegment) { - state.decodedBytesAtLastCompleteSegment = state.decoder.getTotalDecodedPayloadBytes(); + state.decodedBytesAtLastCompleteSegment = state.decoder.getDecodedBytesAtLastCompleteSegment(); state.lastCompleteSegmentStart = currentLastCompleteSegment; LOGGER.atInfo() @@ -350,6 +359,40 @@ private void updateProgress(DecoderState state) { state.totalEncodedBytesProcessed.set(encodedProgress); } + private Flux emitDecodedPayload(DecoderState state, ByteBuffer decodedPayload) { + if (decodedPayload == null || !decodedPayload.hasRemaining()) { + return Flux.empty(); + } + + long skip = state.decodedBytesToSkip.get(); + if (skip > 0) { + if (skip >= decodedPayload.remaining()) { + state.decodedBytesToSkip.addAndGet(-decodedPayload.remaining()); + return Flux.empty(); + } else { + int skipCount = (int) skip; + decodedPayload.position(decodedPayload.position() + skipCount); + decodedPayload = decodedPayload.slice(); + state.decodedBytesToSkip.addAndGet(-skipCount); + } + } + + if (!decodedPayload.hasRemaining()) { + return Flux.empty(); + } + + // Return a defensive copy to avoid any inadvertent position/limit side effects. + ByteBuffer copy = ByteBuffer.allocate(decodedPayload.remaining()); + copy.put(decodedPayload.duplicate()); + copy.flip(); + + state.totalBytesDecoded.addAndGet(copy.remaining()); + if (state.aggregateCrcState != null) { + state.aggregateCrcState.appendPayload(copy.asReadOnlyBuffer()); + } + return Flux.just(copy); + } + /** * Creates an IOException with the retry start offset encoded in the message. * @@ -370,22 +413,10 @@ private IOException createRetryableException(DecoderState state, String message) * @return An IOException with retry offset information. */ private IOException createRetryableException(DecoderState state, String message, Throwable cause) { - long retryOffset = state.prepareForRetry(); + long retryOffset = state.getRetryOffset(); long decodedSoFar = state.totalBytesDecoded.get(); long expectedLength = state.decoder.getMessageLength(); - - // Check if the exception message already has decoder offset information - // If so, prefer lastCompleteSegment from the enriched message String originalMessage = message != null ? message : ""; - long[] decoderOffsets = parseDecoderOffsets(originalMessage); - if (decoderOffsets != null) { - // Use lastCompleteSegment from the enriched exception as the retry offset - retryOffset = decoderOffsets[1]; // lastCompleteSegment - LOGGER.atInfo() - .addKeyValue("decoderOffset", decoderOffsets[0]) - .addKeyValue("lastCompleteSegment", decoderOffsets[1]) - .log("Parsed decoder offsets from enriched exception"); - } // Build message components for clarity long displayExpected = expectedLength > 0 ? expectedLength : 0; @@ -436,20 +467,22 @@ private DownloadContentValidationOptions getValidationOptions(HttpPipelineCallCo * @param headers The response headers. * @return The content length or null if not present. */ - private Long getContentLength(HttpHeaders headers) { - // Prefer the total length from Content-Range (if present) so retries use the full encoded length - // even when the current response is partial. - String contentRange = headers.getValue(HttpHeaderName.CONTENT_RANGE); - if (contentRange != null) { - // Format: bytes start-end/total - int slash = contentRange.indexOf('/'); - if (slash > -1 && slash + 1 < contentRange.length()) { - String totalPart = contentRange.substring(slash + 1).trim(); - if (!"*".equals(totalPart)) { - try { - return Long.parseLong(totalPart); - } catch (NumberFormatException e) { - LOGGER.warning("Invalid content range total length in response headers: " + contentRange); + private Long getContentLength(HttpHeaders headers, boolean responseScoped) { + if (!responseScoped) { + // Prefer the total length from Content-Range (if present) so retries use the full encoded length + // even when the current response is partial. + String contentRange = headers.getValue(HttpHeaderName.CONTENT_RANGE); + if (contentRange != null) { + // Format: bytes start-end/total + int slash = contentRange.indexOf('/'); + if (slash > -1 && slash + 1 < contentRange.length()) { + String totalPart = contentRange.substring(slash + 1).trim(); + if (!"*".equals(totalPart)) { + try { + return Long.parseLong(totalPart); + } catch (NumberFormatException e) { + LOGGER.warning("Invalid content range total length in response headers: " + contentRange); + } } } } @@ -473,11 +506,71 @@ private Long getContentLength(HttpHeaders headers) { * @param contentLength The content length. * @return The decoder state. */ - private DecoderState getOrCreateDecoderState(HttpPipelineCallContext context, long contentLength) { + private DecoderState getOrCreateDecoderState(HttpPipelineCallContext context, long contentLength, + boolean responseScoped) { + if (responseScoped) { + return new DecoderState(contentLength, null); + } return context.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) .filter(value -> value instanceof DecoderState) .map(value -> (DecoderState) value) - .orElseGet(() -> new DecoderState(contentLength)); + .orElseGet(() -> new DecoderState(contentLength, null)); + } + + /** + * Aggregates CRC state across retries to match .NET structured message retriable stream behavior. + */ + public static final class AggregateCrcState { + private final List segments = new ArrayList<>(); + private long runningCrc = 0; + + void appendPayload(ByteBuffer payload) { + if (payload == null || !payload.hasRemaining()) { + return; + } + ByteBuffer copy = payload.asReadOnlyBuffer(); + byte[] data = new byte[copy.remaining()]; + copy.get(data); + runningCrc = StorageCrc64Calculator.compute(data, runningCrc); + } + + void addSegments(List newSegments) { + if (newSegments == null || newSegments.isEmpty()) { + return; + } + segments.addAll(newSegments); + } + + boolean hasSegments() { + return !segments.isEmpty(); + } + + long getRunningCrc() { + return runningCrc; + } + + long composeCrc() { + if (segments.isEmpty()) { + return 0; + } + long composed = segments.get(0).getCrc64(); + long totalLength = segments.get(0).getLength(); + for (int i = 1; i < segments.size(); i++) { + StructuredMessageDecoder.SegmentInfo next = segments.get(i); + composed + = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); + totalLength += next.getLength(); + } + return composed; + } + + long getTotalLength() { + long totalLength = 0; + for (StructuredMessageDecoder.SegmentInfo segment : segments) { + totalLength += segment.getLength(); + } + return totalLength; + } } /** @@ -491,6 +584,12 @@ private boolean isDownloadResponse(HttpResponse httpResponse) { return method == HttpMethod.GET && httpResponse.getStatusCode() / 100 == 2; } + private boolean isResponseScoped(HttpPipelineCallContext context) { + return context.getData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY) + .map(value -> value instanceof Boolean && (Boolean) value) + .orElse(false); + } + /** * State holder for the structured message decoder that tracks decoding progress * across network interruptions. @@ -498,6 +597,7 @@ private boolean isDownloadResponse(HttpResponse httpResponse) { public static class DecoderState { private final StructuredMessageDecoder decoder; private final long expectedContentLength; + private final AggregateCrcState aggregateCrcState; /** * Tracks how many decoded bytes have actually been emitted to the caller (excludes bytes skipped during * fast-forward on retry). @@ -516,18 +616,32 @@ public static class DecoderState { * offset. This mirrors the .NET StructuredMessageDecodingRetriableStream behavior. */ private final AtomicLong decodedBytesToSkip = new AtomicLong(0); + private boolean segmentsAddedToAggregate; /** * Creates a new decoder state. * - * @param expectedContentLength The expected length of the encoded content. + * @param expectedContentLength For response-scoped structured messages, the decoded payload length + * (e.g. from x-ms-structured-content-length); for range responses, the encoded length. The decoder + * validates this against the message header in the body. + * @param aggregateCrcState Aggregated CRC state shared across retries, or null if not aggregating. */ - public DecoderState(long expectedContentLength) { + public DecoderState(long expectedContentLength, AggregateCrcState aggregateCrcState) { this.expectedContentLength = expectedContentLength; this.decoder = new StructuredMessageDecoder(expectedContentLength); this.totalBytesDecoded = new AtomicLong(0); this.totalEncodedBytesProcessed = new AtomicLong(0); this.decodedBytesAtLastCompleteSegment = 0; + this.aggregateCrcState = aggregateCrcState; + this.segmentsAddedToAggregate = false; + } + + private void addSegmentsToAggregateIfNeeded() { + if (segmentsAddedToAggregate || aggregateCrcState == null) { + return; + } + aggregateCrcState.addSegments(decoder.getCompletedSegments()); + segmentsAddedToAggregate = true; } /** @@ -558,8 +672,61 @@ public long getExpectedContentLength() { } /** - * Gets the offset to use for retry requests. - * This uses the decoder's last complete segment boundary to ensure retries + * Gets the total decoded payload bytes for this response. + * + * @return The decoded payload length. + */ + public long getDecodedPayloadLength() { + return decoder.getTotalDecodedPayloadBytes(); + } + + /** + * Gets the composed CRC64 for the decoded payload in this response. + * + * @return The composed CRC64 value. + */ + public long getComposedCrc64() { + if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { + return aggregateCrcState.composeCrc(); + } + + List segments = decoder.getCompletedSegments(); + if (segments.isEmpty()) { + return 0; + } + + long composed = segments.get(0).getCrc64(); + long totalLength = segments.get(0).getLength(); + for (int i = 1; i < segments.size(); i++) { + StructuredMessageDecoder.SegmentInfo next = segments.get(i); + composed + = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); + totalLength += next.getLength(); + } + return composed; + } + + /** + * Gets the composed decoded payload length represented by validated segments. + * + * @return The composed payload length. + */ + public long getComposedLength() { + if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { + return aggregateCrcState.getTotalLength(); + } + + List segments = decoder.getCompletedSegments(); + long totalLength = 0; + for (StructuredMessageDecoder.SegmentInfo segment : segments) { + totalLength += segment.getLength(); + } + return totalLength; + } + + /** + * Gets the decoded offset to use for retry requests. + * This uses the last complete segment boundary to ensure retries * resume from a valid segment boundary, not mid-segment. * * Also resets decoder state to align with the segment boundary. @@ -567,9 +734,8 @@ public long getExpectedContentLength() { * @return The offset for retry requests (last complete segment boundary). */ public long getRetryOffset() { - // Use the decoder's last complete segment start as the retry offset - // This ensures we resume from a segment boundary, not mid-segment - long retryOffset = decoder.getRetryStartOffset(); + // Use the decoded byte count at the last complete segment boundary for retry offset. + long retryOffset = decodedBytesAtLastCompleteSegment; long lastCompleteSegmentOffset = decoder.getLastCompleteSegmentStart(); LOGGER.atInfo() @@ -585,7 +751,7 @@ public long getRetryOffset() { * boundary and resetting the accounting counters to that point. This mirrors the behavior of * the cryptography smart-retry implementation which always replays from a validated boundary. * - * @return The retry start offset (encoded byte position) that the next request should use. + * @return The retry start offset (decoded byte position) that the next request should use. */ public long prepareForRetry() { return resetForRetry(); @@ -594,10 +760,10 @@ public long prepareForRetry() { /** * Resets decoder and counters to the last validated segment boundary and returns the retry offset. * - * @return retry offset (encoded boundary). + * @return retry offset (decoded boundary). */ public long resetForRetry() { - long retryOffset = decoder.getRetryStartOffset(); + long retryOffset = decodedBytesAtLastCompleteSegment; decoder.resetToLastCompleteSegment(); accumulatedDecoded.reset(); @@ -615,6 +781,40 @@ public long resetForRetry() { return retryOffset; } + /** + * Resets decoder state while preserving decoded bytes up to the last complete segment boundary. + * This allows retries to resume from a validated boundary without losing already validated payload + * when nothing has been emitted downstream yet. + * + * @return retry offset (decoded boundary). + */ + public long resetForRetryPreservingPrefix() { + long retryOffset = decodedBytesAtLastCompleteSegment; + + decoder.resetToLastCompleteSegment(); + + byte[] data = accumulatedDecoded.toByteArray(); + accumulatedDecoded.reset(); + + long prefixLength = decodedBytesAtLastCompleteSegment; + if (prefixLength > 0 && data.length > 0) { + int keep = (int) Math.min(prefixLength, data.length); + accumulatedDecoded.write(data, 0, keep); + } + + // Align encoded counters to the boundary we will resume from so subsequent progress tracking is consistent. + totalEncodedBytesProcessed.set(decoder.getMessageOffset() + decoder.getPendingEncodedByteCount()); + decodedBytesToSkip.set(0); + + LOGGER.atInfo() + .addKeyValue("retryOffset", retryOffset) + .addKeyValue("decoderOffset", decoder.getMessageOffset()) + .addKeyValue("decodedPrefixBytes", decodedBytesAtLastCompleteSegment) + .log("Prepared decoder state for smart retry preserving validated prefix"); + + return retryOffset; + } + /** * Checks if the decoder has finalized. * @@ -709,12 +909,13 @@ public HttpHeaders getHeaders() { @Override public Flux getBody() { - return decodedBody; + // Ensure the original response is closed once the decoded stream completes. + return Flux.using(() -> originalResponse, r -> decodedBody, HttpResponse::close); } @Override public Mono getBodyAsByteArray() { - return FluxUtil.collectBytesInByteBufferStream(decodedBody); + return FluxUtil.collectBytesInByteBufferStream(getBody()); } @Override @@ -727,6 +928,11 @@ public Mono getBodyAsString(Charset charset) { return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); } + @Override + public void close() { + originalResponse.close(); + } + /** * Gets the decoder state. * diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java index 4c1089d2c6b9..f6ad54001d1a 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java @@ -69,6 +69,8 @@ public final class StorageCommonTestUtils { try { jdkHttpHttpClient = createJdkHttpClient(); } catch (LinkageError | ReflectiveOperationException e) { + // LinkageError includes ExceptionInInitializerError (e.g. JDK HTTP client fails on Java 17 when + // SharedExecutorService needs Thread.ofVirtual() from Java 19+). Set to null so Netty/OkHttp/Vertx tests run. jdkHttpHttpClient = null; } @@ -130,6 +132,11 @@ public static HttpClient getHttpClient(Supplier playbackClientSuppli case VERTX: return VERTX_HTTP_CLIENT; case JDK_HTTP: + if (JDK_HTTP_HTTP_CLIENT == null) { + throw new IllegalStateException( + "JDK HTTP client is not available (e.g. requires Java 19+ for virtual threads). " + + "Use NETTY, OK_HTTP, or VERTX instead."); + } return JDK_HTTP_HTTP_CLIENT; default: throw new IllegalArgumentException("Unknown http client type: " + ENVIRONMENT.getHttpClientType()); diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java index 140e4ba4e596..ef55e1210525 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java @@ -6,10 +6,10 @@ import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; +import com.azure.core.util.FluxUtil; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -19,6 +19,7 @@ with than was worth it. Because this type is just for BlobDownload, we don't need to accept a header type. */ public class MockDownloadHttpResponse extends HttpResponse { + private final HttpResponse originalResponse; private final int statusCode; private final HttpHeaders headers; private final Flux body; @@ -30,6 +31,7 @@ public MockDownloadHttpResponse(HttpResponse response, int statusCode, Flux body) { super(response.getRequest()); + this.originalResponse = response; this.statusCode = statusCode; this.headers = headers; this.body = body; @@ -57,21 +59,27 @@ public HttpHeaders getHeaders() { @Override public Flux getBody() { - return body; + // Always close the wrapped response when body consumption terminates. + return Flux.using(() -> originalResponse, ignored -> body, HttpResponse::close); } @Override public Mono getBodyAsByteArray() { - return Mono.error(new IOException()); + return FluxUtil.collectBytesInByteBufferStream(getBody()); } @Override public Mono getBodyAsString() { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); } @Override public Mono getBodyAsString(Charset charset) { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); + } + + @Override + public void close() { + originalResponse.close(); } } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index 34c6cb263d35..574942d24a0b 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -16,15 +16,16 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.util.concurrent.atomic.AtomicInteger; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class MockPartialResponsePolicy implements HttpPipelinePolicy { static final HttpHeaderName X_MS_RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); static final HttpHeaderName RANGE_HEADER = HttpHeaderName.RANGE; - private int tries; - private final List rangeHeaders = new ArrayList<>(); + private final AtomicInteger tries; + private final List rangeHeaders = Collections.synchronizedList(new ArrayList<>()); private final int maxBytesPerResponse; // Maximum bytes to return before simulating timeout private final AtomicInteger hits = new AtomicInteger(); private final String targetUrlPrefix; @@ -56,7 +57,7 @@ public MockPartialResponsePolicy(int tries, int maxBytesPerResponse) { * @param targetUrlPrefix If non-null, only requests whose URL starts with this prefix will be interrupted. */ public MockPartialResponsePolicy(int tries, int maxBytesPerResponse, String targetUrlPrefix) { - this.tries = tries; + this.tries = new AtomicInteger(tries); this.maxBytesPerResponse = maxBytesPerResponse; this.targetUrlPrefix = targetUrlPrefix; } @@ -87,13 +88,16 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN boolean urlMatches = targetUrlPrefix == null || response.getRequest().getUrl().toString().startsWith(targetUrlPrefix); - if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || !urlMatches || this.tries == 0) { + if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || !urlMatches) { return Mono.just(response); } else { + int remainingTries = this.tries.getAndUpdate(value -> value > 0 ? value - 1 : value); + if (remainingTries <= 0) { + return Mono.just(response); + } hits.incrementAndGet(); - System.out.println("[MockPartialResponsePolicy] invoked. tries=" + tries + System.out.println("[MockPartialResponsePolicy] invoked. tries=" + remainingTries + ", maxBytesPerResponse=" + maxBytesPerResponse); - this.tries -= 1; // Simulate an interruption mid-stream (like FaultyStream in .NET) without mutating headers. // Emit up to maxBytesPerResponse, then complete early to let the decoder detect an incomplete message @@ -150,7 +154,7 @@ private Flux limitStreamToBytes(Flux body, int maxBytes) } public int getTriesRemaining() { - return tries; + return tries.get(); } public List getRangeHeaders() { diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java index a637287f3ce4..5e76a7e7bf8d 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java @@ -11,6 +11,7 @@ import com.azure.storage.common.implementation.Constants; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; @@ -211,4 +212,33 @@ private static Stream testConcatWithInitialsSupplier() { Arguments.of("889000539881195835", "2971048229276949174", "5346315327374690144", "307387", "1407121768110541356", "10535852615249992663", "741189", "3634018251978804152")); } + + @Test + void testComputeTwoMatchesTwoComputes() { + byte[] data = "This is a test where the data is longer than 64 characters so that we can test that code path." + .getBytes(); + long seg0 = 0; + long msg0 = 0; + long seg1 = StorageCrc64Calculator.compute(data, seg0); + long msg1 = StorageCrc64Calculator.compute(data, msg0); + long[] two = StorageCrc64Calculator.computeTwo(data, seg0, msg0); + assertEquals(seg1, two[0]); + assertEquals(msg1, two[1]); + } + + @Test + void testComputeByteBufferMatchesByteArray() { + byte[] data = "Hello World!".getBytes(); + long expected = StorageCrc64Calculator.compute(data, 0); + long actual = StorageCrc64Calculator.compute(ByteBuffer.wrap(data), 0); + assertEquals(expected, actual); + } + + @Test + void testComputeSliceMatchesFullArray() { + byte[] data = "Hello World!".getBytes(); + long expected = StorageCrc64Calculator.compute(data, 0); + long actual = StorageCrc64Calculator.compute(data, 0, data.length, 0); + assertEquals(expected, actual); + } } From 09f49f9d1bd91c517b4a4dc928bbd5acf81225ae Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Fri, 27 Mar 2026 20:26:11 +0530 Subject: [PATCH 09/31] adding content validation tests --- .../blob/BlobMessageDecoderDownloadTests.java | 283 +++++++++++++++++- 1 file changed, 277 insertions(+), 6 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 8822c71a53f6..54d3570661f8 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -86,13 +86,193 @@ public void downloadStreamWithResponseContentValidation() throws IOException { } /** - * Mirrors .NET StructuredMessagePopulatesCrcDownloadStreaming: after consuming the response stream with - * StorageCrc64 validation, the response details (ContentCrc64) are populated with the computed CRC. - * Aligned with .NET: decoder-only client, live/playback returns structured-encoded body. + * Async downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + StepVerifier + .create(downloadClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + } + + /** + * Async downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @ParameterizedTest + @ValueSource(ints = { 512, 2048 }) + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { + int payloadSize = (4 * blockSize) + 1; + byte[] randomData = getRandomByteArray(payloadSize); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + Path tempFile = Files.createTempFile("structured-download", ".bin"); + Files.deleteIfExists(tempFile); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) + .setInitialTransferSizeLong((long) blockSize); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64); + + try { + StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) + .assertNext(r -> assertNotNull(r.getValue())) + .expectComplete() + .verify(Duration.ofSeconds(60)); + + TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + } finally { + Files.deleteIfExists(tempFile); + } + } + + /** + * Sync downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidationSync() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobClient syncClient + = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + syncClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64), null, + Context.NONE); + + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + } + + /** + * Sync downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidationSync() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobClient syncClient + = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + byte[] result + = syncClient + .downloadContentWithResponse( + new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64), null, + Context.NONE) + .getValue() + .toBytes(); + + TestUtils.assertArraysEqual(data, result); + } + + /** + * Sync downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @ParameterizedTest + @ValueSource(ints = { 512, 2048 }) + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public void downloadToFileWithResponseContentValidationSync(int blockSize) throws IOException { + int payloadSize = (4 * blockSize) + 1; + byte[] randomData = getRandomByteArray(payloadSize); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobClient downloadClient + = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + Path tempFile = Files.createTempFile("structured-download-sync", ".bin"); + Files.deleteIfExists(tempFile); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) + .setInitialTransferSizeLong((long) blockSize); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64); + + try { + assertNotNull(downloadClient.downloadToFileWithResponse(options, null, Context.NONE).getValue()); + TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + } finally { + Files.deleteIfExists(tempFile); + } + } + + /** + * Sync openInputStream with CRC64 content validation. + */ + @Test + public void openInputStreamContentValidation() throws IOException { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobClient syncClient + = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + try (BlobInputStream blobInputStream = syncClient.openInputStream( + new BlobInputStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64), Context.NONE)) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int n; + while ((n = blobInputStream.read(buf)) != -1) { + baos.write(buf, 0, n); + } + TestUtils.assertArraysEqual(data, baos.toByteArray()); + } + } + + /** + * Sync openSeekableByteChannelRead with CRC64 content validation. + */ + @Test + public void openSeekableByteChannelReadContentValidation() throws IOException { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobClient syncClient + = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + try (SeekableByteChannel channel = syncClient.openSeekableByteChannelRead( + new BlobSeekableByteChannelReadOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64), + Context.NONE).getChannel()) { + ByteBuffer buf = ByteBuffer.allocate(data.length + 100); + int totalRead = 0; + int bytesRead; + while ((bytesRead = channel.read(buf)) > 0) { + totalRead += bytesRead; + } + buf.flip(); + byte[] result = new byte[totalRead]; + buf.get(result, 0, totalRead); + TestUtils.assertArraysEqual(data, result); + } + } + + /** + * After consuming the response stream with CRC64 validation, ContentCrc64 header is populated. */ @Test public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOException { - int dataLength = Constants.KB; + int dataLength = 10 * 1024 * 1024; byte[] data = getRandomByteArray(dataLength); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); @@ -480,7 +660,7 @@ public void downloadToFileStructuredMessageParallelSync(int blockSize) throws IO @Timeout(value = 2, unit = TimeUnit.MINUTES) @EnabledIf("isLiveMode") public void olderServiceVersionThrowsOnStructuredMessage() { - int dataLength = Constants.KB; + int dataLength = 10 * 1024 * 1024; byte[] data = getRandomByteArray(dataLength); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); @@ -495,11 +675,102 @@ public void olderServiceVersionThrowsOnStructuredMessage() { = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); StepVerifier - .create(oldVersionClient.downloadStreamWithResponse(range, null, null, false, validationOptions) + .create(oldVersionClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setRange(new BlobRange(0, (long) (10 * 1024 * 1024))) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .verifyError(BlobStorageException.class); } + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier.create(bc.downloadStreamWithResponse(new BlobDownloadStreamOptions()) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + assertNotNull(result); + assertEquals(data.length, result.length); + }).verifyComplete(); + } + + /** + * MD5 on downloadStream (supported): MD5 transactional validation path. + */ + @Test + public void downloadStreamWithMd5() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier + .create(bc + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setRange(new BlobRange(0, (long) data.length)) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.MD5)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.AUTO)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier + .create(bc.downloadContentWithResponse( + new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.NONE))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + + StepVerifier + .create(downloadClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.AUTO))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + } + static boolean isLiveMode() { return ENVIRONMENT.getTestMode() == TestMode.LIVE; } From 973cdf6412eb1879beb90400a42360460e21cfa7 Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Sat, 28 Mar 2026 23:52:37 +0530 Subject: [PATCH 10/31] code refactoring --- ...DownloadAsyncResponseConstructorProxy.java | 11 +- .../implementation/util/BuilderHelper.java | 9 +- .../models/BlobDownloadAsyncResponse.java | 23 +- .../blob/specialized/BlobAsyncClientBase.java | 234 ++-- .../BlobMessageAsyncDecoderDownloadTests.java | 419 ++++++++ .../blob/BlobMessageDecoderDownloadTests.java | 643 +---------- .../DownloadContentValidationOptions.java | 66 -- .../common/implementation/Constants.java | 12 - .../common/policy/AggregateCrcState.java | 73 ++ .../policy/ContentValidationDecoderUtils.java | 104 ++ .../common/policy/DecodedResponse.java | 73 ++ .../storage/common/policy/DecoderState.java | 141 +++ ...StorageContentValidationDecoderPolicy.java | 998 ++++-------------- .../policy/MockDownloadHttpResponse.java | 1 - .../policy/MockPartialResponsePolicy.java | 16 +- ...ageContentValidationDecoderPolicyTest.java | 14 +- 16 files changed, 1145 insertions(+), 1692 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java delete mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java create mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java index 6165a1bb2657..f42f1f0f4db2 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java @@ -9,7 +9,7 @@ import com.azure.core.http.rest.StreamResponse; import com.azure.storage.blob.models.BlobDownloadAsyncResponse; import com.azure.storage.blob.models.DownloadRetryOptions; -import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; +import com.azure.storage.common.policy.DecoderState; import reactor.core.publisher.Mono; import java.util.concurrent.atomic.AtomicReference; @@ -38,7 +38,7 @@ public interface BlobDownloadAsyncResponseConstructorAccessor { */ BlobDownloadAsyncResponse create(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef); + AtomicReference decoderStateRef); /** * Gets the source {@link StreamResponse} from a {@link BlobDownloadAsyncResponse}. @@ -54,7 +54,7 @@ BlobDownloadAsyncResponse create(StreamResponse sourceResponse, * @param response The {@link BlobDownloadAsyncResponse}. * @return The current decoder state, or null if not available. */ - StorageContentValidationDecoderPolicy.DecoderState getDecoderState(BlobDownloadAsyncResponse response); + DecoderState getDecoderState(BlobDownloadAsyncResponse response); } /** @@ -76,7 +76,7 @@ public static void setAccessor( */ public static BlobDownloadAsyncResponse create(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef) { + AtomicReference decoderStateRef) { // This looks odd but is necessary, it is possible to engage the access helper before anywhere else in the // application accesses BlobDownloadAsyncResponse which triggers the accessor to be configured. So, if the accessor // is null this effectively pokes the class to set up the accessor. @@ -111,8 +111,7 @@ public static StreamResponse getSourceResponse(BlobDownloadAsyncResponse respons * @param response The {@link BlobDownloadAsyncResponse}. * @return The decoder state, or null if not available. */ - public static StorageContentValidationDecoderPolicy.DecoderState - getDecoderState(BlobDownloadAsyncResponse response) { + public static DecoderState getDecoderState(BlobDownloadAsyncResponse response) { if (accessor == null) { new BlobDownloadAsyncResponse(new HttpRequest(HttpMethod.GET, "http://microsoft.com"), 200, new HttpHeaders(), null, null); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index f41d7c0dd4e7..5722aff4503d 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -134,17 +134,12 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare policies.add(new AzureSasCredentialPolicy(new AzureSasCredential(sasToken), false)); } + policies.add(new StorageContentValidationDecoderPolicy()); + policies.addAll(perRetryPolicies); HttpPolicyProviders.addAfterRetryPolicies(policies); - // Only add the structured message decoder once; allow callers to inject their own for ordering with test - // policies (e.g., fault injection before decoding). - boolean hasDecoder = policies.stream().anyMatch(p -> p instanceof StorageContentValidationDecoderPolicy); - if (!hasDecoder) { - policies.add(new StorageContentValidationDecoderPolicy()); - } - policies.add(getResponseValidationPolicy()); policies.add(new HttpLoggingPolicy(logOptions)); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java index 0d0d2f950326..527ce4862408 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java @@ -14,7 +14,7 @@ import com.azure.storage.blob.implementation.models.BlobsDownloadHeaders; import com.azure.storage.blob.implementation.util.ModelHelper; import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; +import com.azure.storage.common.policy.DecoderState; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -41,7 +41,7 @@ public final class BlobDownloadAsyncResponse extends ResponseBase> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef) { + AtomicReference decoderStateRef) { return new BlobDownloadAsyncResponse(sourceResponse, onErrorResume, retryOptions, decoderStateRef); } @@ -51,9 +51,8 @@ public StreamResponse getSourceResponse(BlobDownloadAsyncResponse response) { } @Override - public StorageContentValidationDecoderPolicy.DecoderState - getDecoderState(BlobDownloadAsyncResponse response) { - AtomicReference ref = response.decoderStateRef; + public DecoderState getDecoderState(BlobDownloadAsyncResponse response) { + AtomicReference ref = response.decoderStateRef; return ref == null ? null : ref.get(); } }); @@ -64,7 +63,7 @@ public StreamResponse getSourceResponse(BlobDownloadAsyncResponse response) { private final StreamResponse sourceResponse; private final BiFunction> onErrorResume; private final DownloadRetryOptions retryOptions; - private final AtomicReference decoderStateRef; + private final AtomicReference decoderStateRef; /** * Constructs a {@link BlobDownloadAsyncResponse}. @@ -98,14 +97,13 @@ public BlobDownloadAsyncResponse(HttpRequest request, int statusCode, HttpHeader BlobDownloadAsyncResponse(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef) { + AtomicReference decoderStateRef) { this(sourceResponse, onErrorResume, retryOptions, decoderStateRef, extractHeaders(sourceResponse)); } private BlobDownloadAsyncResponse(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef, - BlobDownloadHeaders deserializedHeaders) { + AtomicReference decoderStateRef, BlobDownloadHeaders deserializedHeaders) { super(sourceResponse.getRequest(), sourceResponse.getStatusCode(), sourceResponse.getHeaders(), createResponseFluxWithContentCrc(sourceResponse, onErrorResume, retryOptions, decoderStateRef, deserializedHeaders), @@ -133,16 +131,15 @@ private static Flux createResponseFlux(StreamResponse sourceResponse /** * Builds the response flux and populates ContentCrc64 on the deserialized headers when structured message - * decoding completes (mirrors .NET Details.ContentCrc populated after stream consumption). + * decoding completes successfully. */ private static Flux createResponseFluxWithContentCrc(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef, - BlobDownloadHeaders deserializedHeaders) { + AtomicReference decoderStateRef, BlobDownloadHeaders deserializedHeaders) { Flux flux = createResponseFlux(sourceResponse, onErrorResume, retryOptions); if (decoderStateRef != null && deserializedHeaders != null) { flux = flux.doOnComplete(() -> { - StorageContentValidationDecoderPolicy.DecoderState state = decoderStateRef.get(); + DecoderState state = decoderStateRef.get(); if (state != null && state.isFinalized()) { long crc = state.getComposedCrc64(); byte[] crcBytes = new byte[8]; diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index c9264dee406b..bbde1d670450 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -83,7 +83,6 @@ import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; -import com.azure.storage.common.DownloadContentValidationOptions; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -91,6 +90,8 @@ import com.azure.storage.common.implementation.StorageCrc64Calculator; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.StorageChecksumAlgorithm; +import com.azure.storage.common.policy.AggregateCrcState; +import com.azure.storage.common.policy.DecoderState; import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -1324,49 +1325,38 @@ Mono downloadStreamWithResponseInternal(BlobRange ran StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; +<<<<<<< HEAD boolean structuredDecode = contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled(); +======= + final StorageChecksumAlgorithm algorithm + = responseChecksumAlgorithm != null ? responseChecksumAlgorithm : StorageChecksumAlgorithm.NONE; + final boolean isStructuredMessageEnabled = isStructuredMessageAlgorithm(algorithm); + final boolean isMd5Enabled = algorithm == StorageChecksumAlgorithm.MD5; +>>>>>>> f96332b51d4 (code refactoring) - // Determine MD5 validation: properly consider both getRangeContentMd5 parameter and validation options - // MD5 validation is enabled if structured message validation is not enabled and either: - // 1. getRangeContentMd5 is explicitly true, OR - // 2. contentValidationOptions.isMd5ValidationEnabled() is true - final Boolean finalGetMD5; - if (!structuredDecode - && (getRangeContentMd5 - || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled()))) { - finalGetMD5 = true; - } else { - finalGetMD5 = null; - } + final Boolean finalGetMD5 = (!isStructuredMessageEnabled && (getRangeContentMd5 || isMd5Enabled)) ? true : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; - // The first range should eagerly convert headers as they'll be used to create response types. final Context baseContext = context == null ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - String structuredBodyType = structuredDecode ? Constants.STRUCTURED_MESSAGE_CRC64_BODY_TYPE : null; - // Structured message responses are scoped to the response; each retry returns a fresh structured message. - final boolean responseScoped = structuredDecode; - final Context responseScopedContext = structuredDecode + String structuredBodyType = isStructuredMessageEnabled ? Constants.STRUCTURED_MESSAGE_CRC64_BODY_TYPE : null; + final Context responseScopedContext = isStructuredMessageEnabled ? baseContext.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) : baseContext; - boolean structuredRetry = structuredDecode; - AtomicReference decoderStateRef = new AtomicReference<>(); - StorageContentValidationDecoderPolicy.AggregateCrcState aggregateCrcState - = structuredDecode ? new StorageContentValidationDecoderPolicy.AggregateCrcState() : null; + AtomicReference decoderStateRef = new AtomicReference<>(); + AggregateCrcState aggregateCrcState = isStructuredMessageEnabled ? new AggregateCrcState() : null; AtomicLong responseStartOffset = new AtomicLong(0); - // Add structured message decoding context if enabled final Context firstRangeContext; - if (structuredDecode) { + if (isStructuredMessageEnabled) { firstRangeContext = responseScopedContext.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) - .addData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY, contentValidationOptions) .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY, decoderStateRef) .addData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY, aggregateCrcState); } else { @@ -1393,101 +1383,27 @@ Mono downloadStreamWithResponseInternal(BlobRange ran finalCount = finalRange.getCount(); } - // The resume function takes throwable and offset at the destination. - // I.e. offset is relative to the starting point. BiFunction> onDownloadErrorResume = (throwable, offset) -> { if (!(throwable instanceof IOException || throwable instanceof TimeoutException)) { return Mono.error(throwable); } - long emittedOffset = offset; - long currentResponseOffset = responseStartOffset.get(); - long newCount = finalCount - emittedOffset; - StorageContentValidationDecoderPolicy.DecoderState decoderState = null; - long retryStartOffset = -1; - long bytesToSkip = 0; - boolean noBytesEmitted = emittedOffset == 0; - - if (structuredRetry) { - decoderState = decoderStateRef.get(); - - // Prefer the retry start offset token emitted by the decoder policy when present. - long parsedRetryOffset - = StorageContentValidationDecoderPolicy.parseRetryStartOffset(throwable.getMessage()); - if (parsedRetryOffset >= 0) { - retryStartOffset = currentResponseOffset + parsedRetryOffset; - } - - // Compute the last validated segment boundary to align retry to a safe offset. - if (decoderState != null) { - long decodedBoundary = decoderState.getDecodedBytesAtLastCompleteSegment(); - long boundaryGlobal = currentResponseOffset + decodedBoundary; - if (retryStartOffset < 0 || boundaryGlobal > retryStartOffset) { - retryStartOffset = boundaryGlobal; - } - } - - if (retryStartOffset < 0) { - // No decoder state available (likely failed before policy captured it) or no bytes emitted; - // restart from response start and fast-forward using skip bytes. - retryStartOffset = currentResponseOffset; - decoderStateRef.set(null); - } - - bytesToSkip = emittedOffset - retryStartOffset; - if (bytesToSkip < 0) { - // Fallback to response start if our computed boundary is ahead of emitted progress. - retryStartOffset = currentResponseOffset; - bytesToSkip = emittedOffset - retryStartOffset; - if (bytesToSkip < 0) { - bytesToSkip = 0; - } - } - } - try { - // For retry context, preserve decoder state if structured message validation is enabled - Context retryContext = firstRangeContext; - BlobRange retryRange; - - if (structuredRetry) { - if (retryStartOffset < 0) { - retryStartOffset = noBytesEmitted ? currentResponseOffset : emittedOffset; - bytesToSkip = Math.max(0, emittedOffset - retryStartOffset); - } - - long remainingCount = finalCount - retryStartOffset; - if (remainingCount < 0) { - retryStartOffset = Math.min(emittedOffset, finalCount); - remainingCount = finalCount - retryStartOffset; - bytesToSkip = 0; - } - - if (bytesToSkip > 0) { - retryContext = retryContext - .addData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY, bytesToSkip); - } - - responseStartOffset.set(retryStartOffset); - retryRange = new BlobRange(initialOffset + retryStartOffset, remainingCount); - - LOGGER.info( - "Structured message retry: resuming from offset {} (initial={}, decoded={}, remaining={}, skip={})", - initialOffset + retryStartOffset, initialOffset, retryStartOffset, remainingCount, - bytesToSkip); + if (isStructuredMessageEnabled) { + return retryStructuredDownload(throwable, offset, decoderStateRef, responseStartOffset, + finalCount, initialOffset, firstRangeContext, finalRequestConditions, eTag, finalGetMD5, + structuredBodyType); } else { - // For non-structured downloads, use smart retry from the interrupted offset - retryRange = new BlobRange(initialOffset + emittedOffset, newCount); + long newCount = finalCount - offset; + BlobRange retryRange = new BlobRange(initialOffset + offset, newCount); + return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, null, + firstRangeContext); } - - return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, structuredBodyType, - retryContext); } catch (Exception e) { return Mono.error(e); } }; - // Structured message decoding is now handled by StructuredMessageDecoderPolicy return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions, decoderStateRef); }); @@ -1509,6 +1425,81 @@ private Mono downloadRange(BlobRange range, BlobRequestCondition context); } + private Mono retryStructuredDownload(Throwable throwable, long emittedOffset, + AtomicReference decoderStateRef, AtomicLong responseStartOffset, long finalCount, + long initialOffset, Context baseRetryContext, BlobRequestConditions conditions, String eTag, Boolean getMD5, + String structuredBodyType) { + + long currentResponseOffset = responseStartOffset.get(); + DecoderState decoderState = decoderStateRef.get(); + + long retryStartOffset = resolveStructuredRetryOffset(decoderState, throwable, currentResponseOffset, + emittedOffset == 0, emittedOffset); + long bytesToSkip = calculateRetryBytesToSkip(emittedOffset, retryStartOffset, currentResponseOffset); + + long remainingCount = finalCount - retryStartOffset; + if (remainingCount < 0) { + retryStartOffset = Math.min(emittedOffset, finalCount); + remainingCount = finalCount - retryStartOffset; + bytesToSkip = 0; + } + + if (decoderState == null) { + decoderStateRef.set(null); + } + + Context retryContext = baseRetryContext; + if (bytesToSkip > 0) { + retryContext + = retryContext.addData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY, bytesToSkip); + } + + responseStartOffset.set(retryStartOffset); + BlobRange retryRange = new BlobRange(initialOffset + retryStartOffset, remainingCount); + + LOGGER.info("Structured message retry: resuming from offset {} (initial={}, decoded={}, remaining={}, skip={})", + initialOffset + retryStartOffset, initialOffset, retryStartOffset, remainingCount, bytesToSkip); + + return downloadRange(retryRange, conditions, eTag, getMD5, structuredBodyType, retryContext); + } + + private static long resolveStructuredRetryOffset(DecoderState decoderState, Throwable throwable, + long currentResponseOffset, boolean noBytesEmitted, long emittedOffset) { + + long offset = -1; + + long parsedOffset = StorageContentValidationDecoderPolicy + .parseRetryStartOffset(throwable != null ? throwable.getMessage() : null); + if (parsedOffset >= 0) { + offset = currentResponseOffset + parsedOffset; + } + + if (decoderState != null) { + long boundary = currentResponseOffset + decoderState.getDecodedBytesAtLastCompleteSegment(); + if (offset < 0 || boundary > offset) { + offset = boundary; + } + } + + if (offset < 0) { + offset = noBytesEmitted ? currentResponseOffset : emittedOffset; + } + return offset; + } + + private static long calculateRetryBytesToSkip(long emittedOffset, long retryStartOffset, + long currentResponseOffset) { + long skip = emittedOffset - retryStartOffset; + if (skip < 0) { + skip = Math.max(0, emittedOffset - currentResponseOffset); + } + return skip; + } + + private static boolean isStructuredMessageAlgorithm(StorageChecksumAlgorithm algorithm) { + return algorithm == StorageChecksumAlgorithm.CRC64 || algorithm == StorageChecksumAlgorithm.AUTO; + } + /** * Downloads the entire blob into a file specified by the path. * @@ -1719,7 +1710,6 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti BlobRequestConditions finalConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); - // Default behavior is not to overwrite Set openOptions = options.getOpenOptions(); if (openOptions == null) { openOptions = DEFAULT_OPEN_OPTIONS_SET; @@ -1761,9 +1751,16 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne ProgressReporter progressReporter = progressReceiver == null ? null : ProgressReporter.withProgressListener(progressReceiver); +<<<<<<< HEAD boolean structuredDecode = contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled(); final Context downloadContext = structuredDecode +======= + final boolean isStructuredMessageEnabled = isStructuredMessageAlgorithm(responseChecksumAlgorithm); + final boolean isMd5Enabled = responseChecksumAlgorithm == StorageChecksumAlgorithm.MD5; + + final Context downloadContext = isStructuredMessageEnabled +>>>>>>> f96332b51d4 (code refactoring) ? (context == null ? new Context(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) : context.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true)) @@ -1782,12 +1779,8 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne = (range, conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, false, null, context); - boolean checksumValidationEnabled = structuredDecode - || rangeGetContentMd5 - || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled()); - boolean md5ValidationEnabled = !structuredDecode - && (rangeGetContentMd5 - || (contentValidationOptions != null && contentValidationOptions.isMd5ValidationEnabled())); + boolean checksumValidationEnabled = isStructuredMessageEnabled || rangeGetContentMd5 || isMd5Enabled; + boolean md5ValidationEnabled = !isStructuredMessageEnabled && (rangeGetContentMd5 || isMd5Enabled); long rangeSize = blockSizeProvided ? Math.min(finalParallelTransferOptions.getBlockSizeLong(), BlobConstants.BLOB_MAX_DOWNLOAD_BYTES) @@ -1811,7 +1804,7 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne com.azure.storage.common.ParallelTransferOptions initialParallelTransferOptions = new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(initialRangeSize); - boolean useMasterCrc = structuredDecode; + boolean useMasterCrc = isStructuredMessageEnabled; LOGGER.atVerbose() .addKeyValue("thread", Thread.currentThread().getName()) @@ -1853,7 +1846,6 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne long initialLength = Math.min(initialRangeSize, newCount); if (initialLength == newCount) { - // One-shot download path aligned with .NET: write the initial response directly and finalize. LOGGER.atVerbose() .addKeyValue("thread", Thread.currentThread().getName()) .log("BlobAsyncClientBase taking ONE-SHOT path (single chunk)"); @@ -1886,7 +1878,6 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne remainingRanges.add(new BlobRange(finalRange.getOffset() + offset, chunkSizeActual)); } - // Match .NET behavior: parallelize request issuance while serializing writes in range order. int effectiveConcurrency = Math.max(1, maxConcurrency); ArrayDeque> running = new ArrayDeque<>(); Iterator remainingIterator = remainingRanges.iterator(); @@ -1963,10 +1954,6 @@ private static Mono writeBodyToFile(BlobDownloadAsyncResponse response, As targetChannel = md5Channel; } - /* - * Use BlobDownloadAsyncResponse#writeValueToAsync to preserve the original retriable stream - * transfer behavior (same model used by .NET stream copy path). - */ Mono write = response.writeValueToAsync(targetChannel, progressReporter) .doOnSuccess(v -> LOGGER.atVerbose() .addKeyValue("thread", Thread.currentThread().getName()) @@ -1985,7 +1972,7 @@ private static Mono writeBodyToFile(BlobDownloadAsyncResponse response, As validateResponseMd5(response, finalMd5Channel); } if (useMasterCrc) { - StorageContentValidationDecoderPolicy.DecoderState decoderState = getStructuredDecoderState(response); + DecoderState decoderState = getStructuredDecoderState(response); if (decoderState == null || !decoderState.isFinalized()) { throw LOGGER.logExceptionAsError(new IllegalStateException( "Structured message decoder state wasn't available or finalized for checksum validation.")); @@ -2005,8 +1992,7 @@ private static Mono writeBodyToFile(BlobDownloadAsyncResponse response, As })); } - private static StorageContentValidationDecoderPolicy.DecoderState - getStructuredDecoderState(BlobDownloadAsyncResponse response) { + private static DecoderState getStructuredDecoderState(BlobDownloadAsyncResponse response) { return BlobDownloadAsyncResponseConstructorProxy.getDecoderState(response); } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java new file mode 100644 index 000000000000..cd9f882fa578 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java @@ -0,0 +1,419 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.test.TestMode; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.FluxUtil; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.StorageChecksumAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; +import reactor.util.function.Tuples; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Async tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +@Execution(ExecutionMode.SAME_THREAD) +public class BlobMessageAsyncDecoderDownloadTests extends BlobTestBase { + + private BlobAsyncClient bc; + + @BeforeEach + public void setup() { + String blobName = generateBlobName(); + bc = ccAsync.getBlobAsyncClient(blobName); + bc.upload(Flux.just(ByteBuffer.wrap(new byte[0])), null).block(); + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() throws IOException { + byte[] randomData = getRandomByteArray(10 * 1024 * 1024); + + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .verifyComplete(); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + + StepVerifier + .create(downloadClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @ParameterizedTest + @ValueSource(ints = { 512, 2048 }) + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { + int payloadSize = (4 * blockSize) + 1; + byte[] randomData = getRandomByteArray(payloadSize); + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + + Path tempFile = Files.createTempFile("structured-download", ".bin"); + Files.deleteIfExists(tempFile); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) + .setInitialTransferSizeLong((long) blockSize); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64); + + try { + StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) + .assertNext(r -> assertNotNull(r.getValue())) + .expectComplete() + .verify(Duration.ofSeconds(60)); + + TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + } finally { + Files.deleteIfExists(tempFile); + } + } + + /** + * After consuming the response stream with CRC64 validation, ContentCrc64 header is populated. + */ + @Test + public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOException { + int dataLength = 10 * 1024 * 1024; + byte[] data = getRandomByteArray(dataLength); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + + long expectedCrc = StorageCrc64Calculator.compute(data, 0); + byte[] expectedCrcBytes = new byte[8]; + ByteBuffer.wrap(expectedCrcBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(expectedCrc); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes)))) + .assertNext(tuple -> { + TestUtils.assertArraysEqual(data, tuple.getT2()); + assertNotNull(tuple.getT1().getDeserializedHeaders().getContentCrc64(), + "ContentCrc64 should be populated after stream consumption"); + TestUtils.assertArraysEqual(expectedCrcBytes, tuple.getT1().getDeserializedHeaders().getContentCrc64()); + }) + .verifyComplete(); + } + + /** + * Range download without content validation works correctly. + */ + @Test + public void downloadStreamWithResponseContentValidationRange() throws IOException { + byte[] randomData = getRandomByteArray(4 * Constants.KB); + Flux input = Flux.just(ByteBuffer.wrap(randomData)); + + BlobRange range = new BlobRange(0, 512L); + + StepVerifier.create(bc.upload(input, null, true) + .then( + bc.downloadStreamWithResponse(range, (DownloadRetryOptions) null, (BlobRequestConditions) null, false)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { + assertNotNull(r); + assertEquals(512, r.length); + }).verifyComplete(); + } + + /** + * Single interrupt with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptWithDataIntact() throws IOException { + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(4 * segmentSize); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); + + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .verifyComplete(); + } + + /** + * Multiple interrupts with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptMultipleTimesWithDataIntact() throws IOException { + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(4 * segmentSize); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, bc.getBlobUrl()); + + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .verifyComplete(); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() throws IOException { + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(2 * segmentSize); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); + + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier.create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .doFinally(signalType -> { + System.out.println("[MockPartialResponsePolicy] hits=" + mockPolicy.getHits() + ", triesRemaining=" + + mockPolicy.getTriesRemaining() + ", ranges=" + mockPolicy.getRangeHeaders()); + assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked"); + }) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .verifyComplete(); + + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) throws IOException { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] randomData = getRandomByteArray(dataSize); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, bc.getBlobUrl()); + + bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier.create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(randomData, result); + }).verifyComplete(); + } + + /** + * Older service version throws when structured message validation is requested. + */ + @Test + @Timeout(value = 2, unit = TimeUnit.MINUTES) + @EnabledIf("isLiveMode") + public void olderServiceVersionThrowsOnStructuredMessage() { + int dataLength = 10 * 1024 * 1024; + byte[] data = getRandomByteArray(dataLength); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobClientBuilder builder = new BlobClientBuilder().endpoint(bc.getBlobUrl()) + .credential(ENVIRONMENT.getPrimaryAccount().getCredential()) + .serviceVersion(BlobServiceVersion.V2024_11_04); + instrument(builder); + BlobAsyncClient oldVersionClient = builder.buildAsyncClient(); + + StepVerifier + .create(oldVersionClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setRange(new BlobRange(0, (long) (10 * 1024 * 1024))) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .verifyError(BlobStorageException.class); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier.create(bc.downloadStreamWithResponse(new BlobDownloadStreamOptions()) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + assertNotNull(result); + assertEquals(data.length, result.length); + }).verifyComplete(); + } + + /** + * MD5 on downloadStream: MD5 transactional validation path. + */ + @Test + public void downloadStreamWithMd5() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier + .create(bc + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setRange(new BlobRange(0, (long) data.length)) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.MD5)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.AUTO)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier + .create(bc.downloadContentWithResponse( + new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.NONE))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(10 * 1024 * 1024); + bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient + = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + + StepVerifier + .create(downloadClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.AUTO))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + } + + static boolean isLiveMode() { + return ENVIRONMENT.getTestMode() == TestMode.LIVE; + } +} + diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 54d3570661f8..751503b2dc72 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -3,23 +3,15 @@ package com.azure.storage.blob; -import com.azure.core.test.TestMode; import com.azure.core.test.utils.TestUtils; import com.azure.core.util.Context; -import com.azure.core.util.FluxUtil; -import com.azure.storage.blob.models.BlobRange; -import com.azure.storage.blob.models.BlobRequestConditions; -import com.azure.storage.blob.models.BlobStorageException; -import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.common.DownloadContentValidationOptions; import com.azure.storage.common.ParallelTransferOptions; -import com.azure.storage.common.implementation.Constants; -import com.azure.storage.common.implementation.StorageCrc64Calculator; -import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; -import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import com.azure.storage.common.StorageChecksumAlgorithm; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.condition.EnabledIf; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.parallel.Execution; @@ -27,23 +19,18 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; -import reactor.util.function.Tuples; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.util.concurrent.TimeUnit; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * Sync tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. */ @Execution(ExecutionMode.SAME_THREAD) @@ -59,98 +46,14 @@ public void setup() { } /** - * Aligned with .NET: decoder-only client; live service returns structured-encoded body. - * Runs in LIVE only: playback may not replay streaming response body, causing "decoded 0" otherwise. - */ - @Test - @EnabledIf("isLiveMode") - public void downloadStreamWithResponseContentValidation() throws IOException { - byte[] randomData = getRandomByteArray(10 * 1024 * 1024); // 10 MB - - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - - StepVerifier - .create(downloadClient - .downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, (BlobRequestConditions) null, - false, validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) - .verifyComplete(); - } - - /** - * Async downloadContentWithResponse with CRC64 content validation. - */ - @Test - public void downloadContentWithResponseContentValidation() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - StepVerifier - .create(downloadClient.downloadContentWithResponse( - new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64))) - .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) - .verifyComplete(); - } - - /** - * Async downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). - */ - @ParameterizedTest - @ValueSource(ints = { 512, 2048 }) - @Timeout(value = 5, unit = TimeUnit.MINUTES) - public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { - int payloadSize = (4 * blockSize) + 1; - byte[] randomData = getRandomByteArray(payloadSize); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - Path tempFile = Files.createTempFile("structured-download", ".bin"); - Files.deleteIfExists(tempFile); - - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) - .setInitialTransferSizeLong((long) blockSize); - BlobDownloadToFileOptions options - = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) - .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64); - - try { - StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) - .assertNext(r -> assertNotNull(r.getValue())) - .expectComplete() - .verify(Duration.ofSeconds(60)); - - TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); - } finally { - Files.deleteIfExists(tempFile); - } - } - - /** - * Sync downloadStreamWithResponse with CRC64 content validation. + * downloadStreamWithResponse with CRC64 content validation. */ @Test public void downloadStreamWithResponseContentValidationSync() { byte[] data = getRandomByteArray(10 * 1024 * 1024); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobClient syncClient - = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); syncClient.downloadStreamWithResponse(outputStream, @@ -161,16 +64,14 @@ public void downloadStreamWithResponseContentValidationSync() { } /** - * Sync downloadContentWithResponse with CRC64 content validation. + * downloadContentWithResponse with CRC64 content validation. */ @Test public void downloadContentWithResponseContentValidationSync() { byte[] data = getRandomByteArray(10 * 1024 * 1024); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobClient syncClient - = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); byte[] result = syncClient @@ -184,7 +85,7 @@ public void downloadContentWithResponseContentValidationSync() { } /** - * Sync downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). */ @ParameterizedTest @ValueSource(ints = { 512, 2048 }) @@ -194,9 +95,7 @@ public void downloadToFileWithResponseContentValidationSync(int blockSize) throw byte[] randomData = getRandomByteArray(payloadSize); bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobClient downloadClient - = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); Path tempFile = Files.createTempFile("structured-download-sync", ".bin"); Files.deleteIfExists(tempFile); @@ -216,16 +115,14 @@ public void downloadToFileWithResponseContentValidationSync(int blockSize) throw } /** - * Sync openInputStream with CRC64 content validation. + * openInputStream with CRC64 content validation. */ @Test public void openInputStreamContentValidation() throws IOException { byte[] data = getRandomByteArray(10 * 1024 * 1024); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobClient syncClient - = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); try (BlobInputStream blobInputStream = syncClient.openInputStream( new BlobInputStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64), Context.NONE)) { @@ -240,16 +137,14 @@ public void openInputStreamContentValidation() throws IOException { } /** - * Sync openSeekableByteChannelRead with CRC64 content validation. + * openSeekableByteChannelRead with CRC64 content validation. */ @Test public void openSeekableByteChannelReadContentValidation() throws IOException { byte[] data = getRandomByteArray(10 * 1024 * 1024); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobClient syncClient - = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); + BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); try (SeekableByteChannel channel = syncClient.openSeekableByteChannelRead( new BlobSeekableByteChannelReadOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64), @@ -266,512 +161,4 @@ public void openSeekableByteChannelReadContentValidation() throws IOException { TestUtils.assertArraysEqual(data, result); } } - - /** - * After consuming the response stream with CRC64 validation, ContentCrc64 header is populated. - */ - @Test - public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOException { - int dataLength = 10 * 1024 * 1024; - byte[] data = getRandomByteArray(dataLength); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - - long expectedCrc = StorageCrc64Calculator.compute(data, 0); - byte[] expectedCrcBytes = new byte[8]; - ByteBuffer.wrap(expectedCrcBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(expectedCrc); - - StepVerifier - .create(downloadClient - .downloadStreamWithResponse((BlobRange) null, (DownloadRetryOptions) null, (BlobRequestConditions) null, - false, validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes)))) - .assertNext(tuple -> { - TestUtils.assertArraysEqual(data, tuple.getT2()); - assertNotNull(tuple.getT1().getDeserializedHeaders().getContentCrc64(), - "ContentCrc64 should be populated after stream consumption"); - TestUtils.assertArraysEqual(expectedCrcBytes, tuple.getT1().getDeserializedHeaders().getContentCrc64()); - }) - .verifyComplete(); - } - - @Test - public void downloadStreamWithResponseContentValidationRange() throws IOException { - byte[] randomData = getRandomByteArray(4 * Constants.KB); - Flux input = Flux.just(ByteBuffer.wrap(randomData)); - - // Range download without validation should work - BlobRange range = new BlobRange(0, 512L); - - StepVerifier.create(bc.upload(input, null, true) - .then( - bc.downloadStreamWithResponse(range, (DownloadRetryOptions) null, (BlobRequestConditions) null, false)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { - assertNotNull(r); - // Should get exactly 512 bytes of encoded data - assertEquals(512, r.length); - }).verifyComplete(); - } - - /** - * Mirrors .NET UninterruptedStream: decoder-only client, live/playback returns structured-encoded body. - */ - @Test - public void uninterruptedStreamWithStructuredMessageDecoding() throws IOException { - byte[] randomData = getRandomByteArray(4 * Constants.KB); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - - StepVerifier - .create( - downloadClient - .downloadStreamWithResponse((BlobRange) null, null, (BlobRequestConditions) null, false, - validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(result, randomData)) - .verifyComplete(); - } - - /** - * Mirrors .NET Interrupt_DataIntact (single interrupt): decoder + fault policy only; live/playback returns - * structured-encoded body; MockPartialResponsePolicy simulates interruption like .NET FaultyDownloadPipelinePolicy. - */ - @Test - public void interruptWithDataIntact() throws IOException { - final int segmentSize = Constants.KB; - byte[] randomData = getRandomByteArray(4 * segmentSize); - - int interruptPos = segmentSize + (3 * 128) + 10; // readLen in .NET test = 128 bytes - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); - - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), decoderPolicy, mockPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - - StepVerifier - .create(downloadClient - .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, - validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) - .verifyComplete(); - } - - /** - * Blocking variant of interruptWithDataIntact; aligned with .NET (decoder + fault policy only). - */ - @Test - public void interruptWithDataIntactBlocking() throws IOException { - final int segmentSize = Constants.KB; - byte[] randomData = getRandomByteArray(4 * segmentSize); - - int interruptPos = segmentSize + (3 * 128) + 10; - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), decoderPolicy, mockPolicy); - - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - - byte[] result = downloadClient - .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, - validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue())) - .block(); - - TestUtils.assertArraysEqual(randomData, result); - } - - /** - * Mirrors .NET Interrupt_DataIntact (multiple interrupts): decoder + fault policy only. - */ - @Test - public void interruptMultipleTimesWithDataIntact() throws IOException { - final int segmentSize = Constants.KB; - byte[] randomData = getRandomByteArray(4 * segmentSize); - - int interruptPos = segmentSize + (3 * 128) + 10; - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, bc.getBlobUrl()); - - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), decoderPolicy, mockPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); - - StepVerifier - .create(downloadClient - .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, - validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) - .verifyComplete(); - } - - /** - * Mirrors .NET Interrupt_AppropriateRewind: decoder + fault policy only; verifies rewind to segment boundary. - */ - @Test - public void interruptAndVerifyProperRewind() throws IOException { - final int segmentSize = Constants.KB; - byte[] randomData = getRandomByteArray(2 * segmentSize); - - int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; // readLen = segmentSize/4 - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); - - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), decoderPolicy, mockPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - - StepVerifier.create(downloadClient - .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, - validationOptions) - .doFinally(signalType -> { - System.out.println("[MockPartialResponsePolicy] hits=" + mockPolicy.getHits() + ", triesRemaining=" - + mockPolicy.getTriesRemaining() + ", ranges=" + mockPolicy.getRangeHeaders()); - assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked"); - }) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) - .verifyComplete(); - - assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); - assertTrue(mockPolicy.getRangeHeaders().size() >= 2, - "Expected at least the initial request and one retry with a range header"); - } - - /** - * Mirrors .NET Interrupt_ProperDecode: decoder + fault policy only; proper decode across retries. - */ - @ParameterizedTest - @ValueSource(booleans = { false, true }) - public void interruptAndVerifyProperDecode(boolean multipleInterrupts) throws IOException { - final int segmentSize = 128 * Constants.KB; - final int dataSize = 4 * Constants.KB; - byte[] randomData = getRandomByteArray(dataSize); - - int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; // readLen = 8KB in .NET - MockPartialResponsePolicy mockPolicy - = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, bc.getBlobUrl()); - - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), - bc.getBlobUrl(), decoderPolicy, mockPolicy); - - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); - - StepVerifier.create(downloadClient - .downloadStreamWithResponse((BlobRange) null, retryOptions, (BlobRequestConditions) null, false, - validationOptions) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { - assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); - TestUtils.assertArraysEqual(randomData, result); - }).verifyComplete(); - } - - /** - * DownloadToFile with structured message decoding using the same payload size as .NET (Constants.KB). - * Aligned with .NET: decoder-only client, live/playback returns structured-encoded body. - */ - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - public void downloadToFileStructuredMessageSamePayloadAsNet() throws IOException { - int payloadSize = Constants.KB; // same as .NET StructuredMessagePopulatesCrc / transfer validation tests - byte[] randomData = getRandomByteArray(payloadSize); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - Path tempFile = Files.createTempFile("structured-download-net-size", ".bin"); - Files.deleteIfExists(tempFile); - - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) Constants.KB) - .setInitialTransferSizeLong((long) Constants.KB); - BlobDownloadToFileOptions options - = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) - .setContentValidationOptions( - new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); - - try { - StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) - .assertNext(r -> assertNotNull(r.getValue())) - .expectComplete() - .verify(Duration.ofSeconds(60)); - - TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); - } finally { - Files.deleteIfExists(tempFile); - } - } - - /** - * Single-chunk DownloadToFile; aligned with .NET: decoder-only client, live/playback. - */ - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - public void downloadToFileStructuredMessageSingleChunk() throws IOException { - int blockSize = 512; - int payloadSize = blockSize; - byte[] randomData = getRandomByteArray(payloadSize); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - Path tempFile = Files.createTempFile("structured-download-single", ".bin"); - Files.deleteIfExists(tempFile); - - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) - .setInitialTransferSizeLong((long) blockSize); - BlobDownloadToFileOptions options - = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) - .setContentValidationOptions( - new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); - - try { - StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) - .assertNext(r -> assertNotNull(r.getValue())) - .expectComplete() - .verify(Duration.ofSeconds(60)); - - TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); - } finally { - Files.deleteIfExists(tempFile); - } - } - - /** - * Parallel download with structured message validation; aligned with .NET: no mock, decoder only, live/playback, - * default concurrency (no explicit maxConcurrency), payload (4*blockSize)+1 and block sizes 512/2048 per .NET. - */ - @ParameterizedTest - @ValueSource(ints = { 512, 2048 }) - @Timeout(value = 5, unit = TimeUnit.MINUTES) - public void downloadToFileStructuredMessageParallel(int blockSize) throws IOException { - int payloadSize = (4 * blockSize) + 1; - byte[] randomData = getRandomByteArray(payloadSize); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - Path tempFile = Files.createTempFile("structured-download", ".bin"); - Files.deleteIfExists(tempFile); - - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) - .setInitialTransferSizeLong((long) blockSize); - BlobDownloadToFileOptions options - = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) - .setContentValidationOptions( - new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); - - try { - StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) - .assertNext(r -> assertNotNull(r.getValue())) - .expectComplete() - .verify(Duration.ofSeconds(60)); - - TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); - } finally { - Files.deleteIfExists(tempFile); - } - } - - /** - * Sync variant: same payload and transfer options as .NET (default concurrency; sync path forces 1). Multi-chunk - * DownloadTo with structured message validation. - */ - @ParameterizedTest - @ValueSource(ints = { 512, 2048 }) - @Timeout(value = 5, unit = TimeUnit.MINUTES) - public void downloadToFileStructuredMessageParallelSync(int blockSize) throws IOException { - int payloadSize = (4 * blockSize) + 1; - byte[] randomData = getRandomByteArray(payloadSize); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobClient downloadClient - = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - Path tempFile = Files.createTempFile("structured-download-sync", ".bin"); - Files.deleteIfExists(tempFile); - - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) - .setInitialTransferSizeLong((long) blockSize); - BlobDownloadToFileOptions options - = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) - .setContentValidationOptions( - new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true)); - - try { - assertNotNull(downloadClient.downloadToFileWithResponse(options, null, Context.NONE).getValue()); - TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); - } finally { - Files.deleteIfExists(tempFile); - } - } - - /** - * Mirrors .NET OlderServiceVersionThrowsOnStructuredMessage: when using a service version before structured - * message was introduced (V2024_11_04), a download with structured message validation enabled and a range that - * would trigger structured message response must throw (service returns error for unsupported feature). - */ - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - @EnabledIf("isLiveMode") - public void olderServiceVersionThrowsOnStructuredMessage() { - int dataLength = 10 * 1024 * 1024; - byte[] data = getRandomByteArray(dataLength); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - BlobClientBuilder builder = new BlobClientBuilder().endpoint(bc.getBlobUrl()) - .credential(ENVIRONMENT.getPrimaryAccount().getCredential()) - .serviceVersion(BlobServiceVersion.V2024_11_04); - instrument(builder); - BlobAsyncClient oldVersionClient = builder.buildAsyncClient(); - - BlobRange range = new BlobRange(0, (long) Constants.KB); - DownloadContentValidationOptions validationOptions - = new DownloadContentValidationOptions().setStructuredMessageValidationEnabled(true); - - StepVerifier - .create(oldVersionClient - .downloadStreamWithResponse( - new BlobDownloadStreamOptions().setRange(new BlobRange(0, (long) (10 * 1024 * 1024))) - .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .verifyError(BlobStorageException.class); - } - - /** - * Default behavior: when no algorithm is specified, default is NONE (no validation). - */ - @Test - public void downloadStreamDefaultAlgorithmIsNone() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - StepVerifier.create(bc.downloadStreamWithResponse(new BlobDownloadStreamOptions()) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { - assertNotNull(result); - assertEquals(data.length, result.length); - }).verifyComplete(); - } - - /** - * MD5 on downloadStream (supported): MD5 transactional validation path. - */ - @Test - public void downloadStreamWithMd5() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - StepVerifier - .create(bc - .downloadStreamWithResponse( - new BlobDownloadStreamOptions().setRange(new BlobRange(0, (long) data.length)) - .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.MD5)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(data, result)) - .verifyComplete(); - } - - /** - * AUTO on downloadStream resolves to CRC64 behavior. - */ - @Test - public void downloadStreamWithAuto() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - StepVerifier - .create(downloadClient - .downloadStreamWithResponse( - new BlobDownloadStreamOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.AUTO)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(data, result)) - .verifyComplete(); - } - - /** - * downloadContentWithResponse with NONE: no validation triggered. - */ - @Test - public void downloadContentWithNone() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - StepVerifier - .create(bc.downloadContentWithResponse( - new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.NONE))) - .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) - .verifyComplete(); - } - - /** - * downloadContentWithResponse with AUTO resolves to CRC64 behavior. - */ - @Test - public void downloadContentWithAuto() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - StorageContentValidationDecoderPolicy decoderPolicy = new StorageContentValidationDecoderPolicy(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), decoderPolicy); - - StepVerifier - .create(downloadClient.downloadContentWithResponse( - new BlobDownloadContentOptions().setResponseChecksumAlgorithm(StorageChecksumAlgorithm.AUTO))) - .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) - .verifyComplete(); - } - - static boolean isLiveMode() { - return ENVIRONMENT.getTestMode() == TestMode.LIVE; - } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java deleted file mode 100644 index 2b663494bfe9..000000000000 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/DownloadContentValidationOptions.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.common; - -import com.azure.core.annotation.Fluent; - -/** - * Options for content validation during download operations. - */ -@Fluent -public final class DownloadContentValidationOptions { - private boolean enableStructuredMessageValidation; - private boolean enableMd5Validation; - - /** - * Creates a new instance of DownloadContentValidationOptions. - */ - public DownloadContentValidationOptions() { - this.enableStructuredMessageValidation = false; - this.enableMd5Validation = false; - } - - /** - * Gets whether structured message validation is enabled. - * - * @return true if structured message validation is enabled, false otherwise. - */ - public boolean isStructuredMessageValidationEnabled() { - return enableStructuredMessageValidation; - } - - /** - * Sets whether structured message validation is enabled. - * When enabled, downloads will use CRC64 checksums embedded in structured messages for content validation. - * - * @param enableStructuredMessageValidation true to enable structured message validation, false to disable. - * @return The updated DownloadContentValidationOptions object. - */ - public DownloadContentValidationOptions - setStructuredMessageValidationEnabled(boolean enableStructuredMessageValidation) { - this.enableStructuredMessageValidation = enableStructuredMessageValidation; - return this; - } - - /** - * Gets whether MD5 validation is enabled. - * - * @return true if MD5 validation is enabled, false otherwise. - */ - public boolean isMd5ValidationEnabled() { - return enableMd5Validation; - } - - /** - * Sets whether MD5 validation is enabled. - * When enabled, downloads will use MD5 checksums for content validation. - * - * @param enableMd5Validation true to enable MD5 validation, false to disable. - * @return The updated DownloadContentValidationOptions object. - */ - public DownloadContentValidationOptions setMd5ValidationEnabled(boolean enableMd5Validation) { - this.enableMd5Validation = enableMd5Validation; - return this; - } -} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index cff3feea6afa..efb76bd81e16 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -109,12 +109,6 @@ public final class Constants { */ public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; - /** - * Context key used to pass DownloadContentValidationOptions to the policy. - */ - public static final String STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY - = "azure-storage-structured-message-validation-options"; - /** * Context key used to signal that structured message decoding should be scoped to a single response. * This is used for parallel range downloads where each response is independently structured-encoded. @@ -122,12 +116,6 @@ public final class Constants { public static final String STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY = "azure-storage-structured-message-response-scoped"; - /** - * Context key used to pass stateful decoder state across retry requests. - */ - public static final String STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY - = "azure-storage-structured-message-decoder-state"; - /** * Context key used to pass a mutable holder for decoder state so callers can observe decoder progress. */ diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java new file mode 100644 index 000000000000..58566975a49b --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Aggregates CRC state across retries so the composed CRC64 can be validated against the + * running CRC of all decoded payload bytes. + */ +public final class AggregateCrcState { + private final List segments = new ArrayList<>(); + private long runningCrc = 0; + + /** + * Creates a new instance of {@link AggregateCrcState}. + */ + public AggregateCrcState() { + } + + void appendPayload(ByteBuffer payload) { + if (payload == null || !payload.hasRemaining()) { + return; + } + ByteBuffer copy = payload.asReadOnlyBuffer(); + byte[] data = new byte[copy.remaining()]; + copy.get(data); + runningCrc = StorageCrc64Calculator.compute(data, runningCrc); + } + + void addSegments(List newSegments) { + if (newSegments == null || newSegments.isEmpty()) { + return; + } + segments.addAll(newSegments); + } + + boolean hasSegments() { + return !segments.isEmpty(); + } + + long getRunningCrc() { + return runningCrc; + } + + long composeCrc() { + if (segments.isEmpty()) { + return 0; + } + long composed = segments.get(0).getCrc64(); + long totalLength = segments.get(0).getLength(); + for (int i = 1; i < segments.size(); i++) { + StructuredMessageDecoder.SegmentInfo next = segments.get(i); + composed = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); + totalLength += next.getLength(); + } + return composed; + } + + long getTotalLength() { + long totalLength = 0; + for (StructuredMessageDecoder.SegmentInfo segment : segments) { + totalLength += segment.getLength(); + } + return totalLength; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java new file mode 100644 index 000000000000..8fff6225955b --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpResponse; +import com.azure.core.util.logging.ClientLogger; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class for parsing helpers and download eligibility checks + * used by {@link StorageContentValidationDecoderPolicy}. + */ +final class ContentValidationDecoderUtils { + private static final ClientLogger LOGGER = new ClientLogger(ContentValidationDecoderUtils.class); + + static final String RETRY_OFFSET_TOKEN = "RETRY-START-OFFSET="; + private static final Pattern RETRY_OFFSET_PATTERN = Pattern.compile("RETRY-START-OFFSET=(\\d+)"); + private static final Pattern DECODER_OFFSETS_PATTERN + = Pattern.compile("\\[decoderOffset=(\\d+),lastCompleteSegment=(\\d+)\\]"); + + private ContentValidationDecoderUtils() { + } + + /** + * Parses the retry start offset from an exception message containing the RETRY-START-OFFSET token. + * + * @param message The exception message to parse. + * @return The retry start offset, or -1 if not found. + */ + static long parseRetryStartOffset(String message) { + if (message == null) { + return -1; + } + Matcher matcher = RETRY_OFFSET_PATTERN.matcher(message); + if (matcher.find()) { + try { + return Long.parseLong(matcher.group(1)); + } catch (NumberFormatException e) { + return -1; + } + } + return -1; + } + + /** + * Parses decoder offset information from enriched exception messages. + * Format: "[decoderOffset=X,lastCompleteSegment=Y]" + * + * @param message The exception message to parse. + * @return A long array [decoderOffset, lastCompleteSegment], or null if not found. + */ + static long[] parseDecoderOffsets(String message) { + if (message == null) { + return null; + } + Matcher matcher = DECODER_OFFSETS_PATTERN.matcher(message); + if (matcher.find()) { + try { + return new long[] { Long.parseLong(matcher.group(1)), Long.parseLong(matcher.group(2)) }; + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + /** + * Checks whether the response represents a successful download (GET with 2xx). + */ + static boolean isDownloadResponse(HttpResponse response) { + return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; + } + + /** + * Extracts Content-Length from the response headers. + * + * @return The content length, or null if absent or unparseable. + */ + static Long getContentLength(HttpHeaders headers) { + String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + LOGGER.warning("Invalid content length in response headers: " + value); + } + } + return null; + } + + /** + * Returns {@code true} when the response is a download response with a positive content length, + * making it eligible for structured message decoding. + */ + static boolean isEligibleDownload(HttpResponse response, Long contentLength) { + return isDownloadResponse(response) && contentLength != null && contentLength > 0; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java new file mode 100644 index 000000000000..6b67d432e7ec --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; +import com.azure.core.util.FluxUtil; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * Decoded HTTP response that wraps the original response with a decoded body stream. + */ +class DecodedResponse extends HttpResponse { + private final HttpResponse originalResponse; + private final Flux decodedBody; + private final DecoderState decoderState; + + DecodedResponse(HttpResponse originalResponse, Flux decodedBody, DecoderState decoderState) { + super(originalResponse.getRequest()); + this.originalResponse = originalResponse; + this.decodedBody = decodedBody; + this.decoderState = decoderState; + } + + @Override + public int getStatusCode() { + return originalResponse.getStatusCode(); + } + + @Override + public String getHeaderValue(String name) { + return originalResponse.getHeaderValue(name); + } + + @Override + public HttpHeaders getHeaders() { + return originalResponse.getHeaders(); + } + + @Override + public Flux getBody() { + return Flux.using(() -> originalResponse, r -> decodedBody, HttpResponse::close); + } + + @Override + public Mono getBodyAsByteArray() { + return FluxUtil.collectBytesInByteBufferStream(getBody()); + } + + @Override + public Mono getBodyAsString() { + return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); + } + + @Override + public Mono getBodyAsString(Charset charset) { + return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); + } + + @Override + public void close() { + originalResponse.close(); + } + + DecoderState getDecoderState() { + return decoderState; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java new file mode 100644 index 000000000000..be8cb1f60493 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Tracks the progress of a single structured-message decode attempt. A new instance is + * created for each HTTP response (including retries). The aggregate CRC state, when + * present, is shared across retries to enable end-to-end CRC64 validation. + */ +public class DecoderState { + private static final ClientLogger LOGGER = new ClientLogger(DecoderState.class); + + private final StructuredMessageDecoder decoder; + final AggregateCrcState aggregateCrcState; + final AtomicLong totalBytesDecoded; + long decodedBytesAtLastCompleteSegment; + long lastCompleteSegmentStart; + final AtomicLong decodedBytesToSkip = new AtomicLong(0); + private boolean segmentsAddedToAggregate; + + DecoderState(long expectedContentLength, AggregateCrcState aggregateCrcState) { + this.decoder = new StructuredMessageDecoder(expectedContentLength); + this.totalBytesDecoded = new AtomicLong(0); + this.decodedBytesAtLastCompleteSegment = 0; + this.aggregateCrcState = aggregateCrcState; + this.segmentsAddedToAggregate = false; + } + + StructuredMessageDecoder getDecoder() { + return decoder; + } + + void updateProgress() { + long currentLastComplete = decoder.getLastCompleteSegmentStart(); + if (lastCompleteSegmentStart != currentLastComplete) { + decodedBytesAtLastCompleteSegment = decoder.getDecodedBytesAtLastCompleteSegment(); + lastCompleteSegmentStart = currentLastComplete; + + LOGGER.atInfo() + .addKeyValue("newSegmentBoundary", currentLastComplete) + .addKeyValue("decodedBytesAtBoundary", decodedBytesAtLastCompleteSegment) + .log("Segment boundary crossed, updated decoded bytes snapshot"); + } + } + + void addSegmentsToAggregateIfNeeded() { + if (segmentsAddedToAggregate || aggregateCrcState == null) { + return; + } + aggregateCrcState.addSegments(decoder.getCompletedSegments()); + segmentsAddedToAggregate = true; + } + + void setDecodedBytesToSkip(long bytesToSkip) { + decodedBytesToSkip.set(Math.max(0, bytesToSkip)); + } + + /** + * Returns true if the decoder has finalized (all segments decoded and validated). + * + * @return true if finalized, false otherwise. + */ + public boolean isFinalized() { + return decoder.isComplete(); + } + + /** + * Gets the decoded byte count at the last validated segment boundary. + * + * @return the decoded byte count. + */ + public long getDecodedBytesAtLastCompleteSegment() { + return decodedBytesAtLastCompleteSegment; + } + + /** + * Gets the composed CRC64 over all validated segments. + * + * @return the composed CRC64 value. + */ + public long getComposedCrc64() { + if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { + return aggregateCrcState.composeCrc(); + } + + List segments = decoder.getCompletedSegments(); + if (segments.isEmpty()) { + return 0; + } + + long composed = segments.get(0).getCrc64(); + long totalLength = segments.get(0).getLength(); + for (int i = 1; i < segments.size(); i++) { + StructuredMessageDecoder.SegmentInfo next = segments.get(i); + composed = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); + totalLength += next.getLength(); + } + return composed; + } + + /** + * Gets the composed decoded payload length represented by validated segments. + * + * @return the composed payload length. + */ + public long getComposedLength() { + if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { + return aggregateCrcState.getTotalLength(); + } + + List segments = decoder.getCompletedSegments(); + long totalLength = 0; + for (StructuredMessageDecoder.SegmentInfo segment : segments) { + totalLength += segment.getLength(); + } + return totalLength; + } + + /** + * Gets the decoded offset to use for retry requests. + * + * @return the retry offset. + */ + public long getRetryOffset() { + long retryOffset = decodedBytesAtLastCompleteSegment; + LOGGER.atInfo() + .addKeyValue("decoderOffset", decoder.getMessageOffset()) + .addKeyValue("pendingBytes", decoder.getPendingEncodedByteCount()) + .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) + .log("Computed smart-retry offset from decoder state"); + return retryOffset; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 3f80d7645c40..17bab322fb0c 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -4,58 +4,44 @@ package com.azure.storage.common.policy; import com.azure.core.http.HttpHeaderName; -import com.azure.core.http.HttpHeaders; -import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; import com.azure.core.http.HttpPipelinePosition; -import com.azure.core.util.FluxUtil; import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.DownloadContentValidationOptions; import com.azure.storage.common.implementation.Constants; +<<<<<<< HEAD import com.azure.storage.common.implementation.StorageCrc64Calculator; import com.azure.storage.common.implementation.structuredmessage.StructuredMessageDecoder; +======= +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; +>>>>>>> f96332b51d4 (code refactoring) import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** - * This is a decoding policy in an {@link com.azure.core.http.HttpPipeline} to decode structured messages in - * storage download requests. The policy checks for a context value to determine when to apply structured message decoding. + * Pipeline policy that decodes structured messages in storage download responses when + * CRC64-based content validation is active (i.e., when {@code StorageChecksumAlgorithm} + * is {@code CRC64} or {@code AUTO}). * - *

The policy supports smart retries by maintaining decoder state across network interruptions, ensuring: - *

    - *
  • All received segment checksums are validated before retry
    • - *
  • Exact encoded and decoded byte positions are tracked
    • - *
  • Decoder state is preserved across retry requests
    • - *
  • Retries continue from the correct offset after network faults
    • - *
+ *

The policy is activated by the presence of a boolean context key + * ({@link Constants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY}). It validates per-segment + * CRC64 checksums, tracks decoder progress in {@link DecoderState}, and embeds + * machine-readable retry offsets in exception messages so the caller can resume from the + * last validated segment boundary.

*/ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); - private static final String EXPECTED_LENGTH_CONTEXT_KEY = "azStructuredMsgExpectedLength"; private static final HttpHeaderName X_MS_STRUCTURED_BODY = HttpHeaderName.fromString("x-ms-structured-body"); private static final HttpHeaderName X_MS_STRUCTURED_CONTENT_LENGTH = HttpHeaderName.fromString("x-ms-structured-content-length"); - /** - * Machine-readable token pattern for extracting retry start offset from exception messages. - * Format: RETRY-START-OFFSET={number} - */ - private static final String RETRY_OFFSET_TOKEN = "RETRY-START-OFFSET="; - private static final Pattern RETRY_OFFSET_PATTERN = Pattern.compile("RETRY-START-OFFSET=(\\d+)"); - /** * Creates a new instance of {@link StorageContentValidationDecoderPolicy}. */ @@ -64,299 +50,209 @@ public StorageContentValidationDecoderPolicy() { @Override public HttpPipelinePosition getPipelinePosition() { - // Run on every retry so the state stored in the call context is available across attempts. return HttpPipelinePosition.PER_RETRY; } /** - * Parses the retry start offset from an exception message containing the RETRY-START-OFFSET token. + * Parses the retry start offset from an exception message containing the + * {@code RETRY-START-OFFSET} token. * * @param message The exception message to parse. * @return The retry start offset, or -1 if not found. */ public static long parseRetryStartOffset(String message) { - if (message == null) { - return -1; - } - Matcher matcher = RETRY_OFFSET_PATTERN.matcher(message); - if (matcher.find()) { - try { - return Long.parseLong(matcher.group(1)); - } catch (NumberFormatException e) { - return -1; - } - } - return -1; - } - - /** - * Parses decoder offset information from enriched exception messages. - * Format: "[decoderOffset=X,lastCompleteSegment=Y]" - * - * @param message The exception message to parse. - * @return A long array [decoderOffset, lastCompleteSegment], or null if not found. - */ - public static long[] parseDecoderOffsets(String message) { - if (message == null) { - return null; - } - // Pattern: [decoderOffset=123,lastCompleteSegment=456] - Pattern pattern = Pattern.compile("\\[decoderOffset=(\\d+),lastCompleteSegment=(\\d+)\\]"); - Matcher matcher = pattern.matcher(message); - if (matcher.find()) { - try { - long decoderOffset = Long.parseLong(matcher.group(1)); - long lastCompleteSegment = Long.parseLong(matcher.group(2)); - return new long[] { decoderOffset, lastCompleteSegment }; - } catch (NumberFormatException e) { - return null; - } - } - return null; - } - - /** - * Attempts to extract the decoder state from a decoded response instance. - * - * @param response The HTTP response returned from the pipeline. - * @return The decoder state if present, otherwise null. - */ - public static DecoderState tryGetDecoderState(HttpResponse response) { - if (response instanceof DecodedResponse) { - return ((DecodedResponse) response).getDecoderState(); - } - return null; + return ContentValidationDecoderUtils.parseRetryStartOffset(message); } @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { - // Check if structured message decoding is enabled for this request if (!shouldApplyDecoding(context)) { return next.process(); } return next.process().map(httpResponse -> { - LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("StorageContentValidationDecoderPolicy received response"); - // Only apply decoding to download responses (GET requests with body) - if (!isDownloadResponse(httpResponse)) { - LOGGER.atVerbose() - .log("StorageContentValidationDecoderPolicy not a download response, passing through"); + Long contentLength = ContentValidationDecoderUtils.getContentLength(httpResponse.getHeaders()); + + if (!ContentValidationDecoderUtils.isEligibleDownload(httpResponse, contentLength)) { + LOGGER.atVerbose().log("Not a download response with content, passing through"); return httpResponse; } - DownloadContentValidationOptions validationOptions = getValidationOptions(context); - // Structured messages are scoped to a single response; always decode per response. - boolean responseScoped = true; - // Decoder expected length must match the 8-byte "message length" in the XSM body header, which is the - // encoded stream size. Use Content-Length (not x-ms-structured-content-length) so validation passes. - Long contentLength = getContentLength(httpResponse.getHeaders(), responseScoped); - - if (validationOptions != null && validationOptions.isStructuredMessageValidationEnabled()) { - String structuredBody = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_BODY); - String structuredContentLength = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_CONTENT_LENGTH); - if (structuredBody == null || structuredContentLength == null) { - throw LOGGER.logExceptionAsError(new IllegalStateException( - "Structured message was requested but the response did not acknowledge it.")); - } - } + validateStructuredMessageHeaders(httpResponse); - if (contentLength != null && contentLength > 0 && validationOptions != null) { - long expectedLength = contentLength; - LOGGER.atVerbose() - .addKeyValue("expectedLength", expectedLength) - .addKeyValue("thread", Thread.currentThread().getName()) - .log("StorageContentValidationDecoderPolicy creating decoder"); - - AtomicReference decoderStateHolder = null; - Object decoderStateHolderObj - = context.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY).orElse(null); - if (decoderStateHolderObj instanceof AtomicReference) { - @SuppressWarnings("unchecked") - AtomicReference tmp = (AtomicReference) decoderStateHolderObj; - decoderStateHolder = tmp; - } - - // Always create a new decoder per response (matches .NET behavior for structured messages). - AggregateCrcState aggregateCrcState - = context.getData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY) - .filter(value -> value instanceof AggregateCrcState) - .map(value -> (AggregateCrcState) value) - .orElse(null); - - DecoderState decoderState = new DecoderState(expectedLength, aggregateCrcState); - - Object skipBytesObj - = context.getData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY).orElse(null); - if (skipBytesObj instanceof Number) { - decoderState.setDecodedBytesToSkip(((Number) skipBytesObj).longValue()); - } - - if (decoderStateHolder != null) { - decoderStateHolder.set(decoderState); - } - - // Decode using the stateful decoder - Flux decodedStream = decodeStream(httpResponse.getBody(), decoderState) - .doOnSubscribe(s -> LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("StorageContentValidationDecoderPolicy decoded flux subscribed")) - .doOnComplete(() -> LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("StorageContentValidationDecoderPolicy decoded flux completed")) - .doOnError(e -> LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .addKeyValue("error", e) - .log("StorageContentValidationDecoderPolicy decoded flux error")); - - LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("StorageContentValidationDecoderPolicy returning DecodedResponse"); - return new DecodedResponse(httpResponse, decodedStream, decoderState); - } + long expectedLength = contentLength; + DecoderState decoderState = createDecoderState(context, expectedLength); + + Flux decodedStream = decodeStream(httpResponse.getBody(), decoderState); - return httpResponse; + LOGGER.atVerbose() + .addKeyValue("expectedLength", expectedLength) + .log("Returning DecodedResponse with structured message decoding"); + return new DecodedResponse(httpResponse, decodedStream, decoderState); }); } - /** - * Decodes a stream of byte buffers using the decoder state. - * The decoder properly handles partial headers and segments split across chunks. - * - *

When an error occurs or the stream ends prematurely, an IOException is thrown with a - * machine-readable token RETRY-START-OFFSET=<number> that can be parsed to determine - * the correct offset for retry requests.

- * - * @param encodedFlux The flux of encoded byte buffers. - * @param state The decoder state. - * @return A flux of decoded byte buffers. - */ + private boolean shouldApplyDecoding(HttpPipelineCallContext context) { + return context.getData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) + .map(value -> value instanceof Boolean && (Boolean) value) + .orElse(false); + } + + private void validateStructuredMessageHeaders(HttpResponse httpResponse) { + String structuredBody = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_BODY); + String structuredContentLength = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_CONTENT_LENGTH); + if (structuredBody == null || structuredContentLength == null) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Structured message was requested but the response did not acknowledge it.")); + } + } + + private DecoderState createDecoderState(HttpPipelineCallContext context, long expectedLength) { + AggregateCrcState aggregateCrcState = context.getData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY) + .filter(AggregateCrcState.class::isInstance) + .map(AggregateCrcState.class::cast) + .orElse(null); + + DecoderState state = new DecoderState(expectedLength, aggregateCrcState); + + context.getData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY) + .filter(Number.class::isInstance) + .map(Number.class::cast) + .ifPresent(skip -> state.setDecodedBytesToSkip(skip.longValue())); + + getDecoderStateHolder(context).ifPresent(holder -> holder.set(state)); + + return state; + } + + @SuppressWarnings("unchecked") + private Optional> getDecoderStateHolder(HttpPipelineCallContext context) { + return context.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY) + .filter(AtomicReference.class::isInstance) + .map(obj -> (AtomicReference) obj); + } + private Flux decodeStream(Flux encodedFlux, DecoderState state) { - return encodedFlux.concatMap(encodedBuffer -> { - // If decoding already completed, ignore subsequent buffers. - if (state.decoder.isComplete()) { - LOGGER.atVerbose() - .addKeyValue("bufferLength", encodedBuffer == null ? "null" : encodedBuffer.remaining()) - .log("Decoder already completed; ignoring extra buffer"); - return Flux.empty(); - } + StructuredMessageDecoder decoder = state.getDecoder(); - // Skip empty buffers that may be emitted by reactor-netty - if (encodedBuffer == null || !encodedBuffer.hasRemaining()) { - LOGGER.atVerbose() - .addKeyValue("bufferLength", encodedBuffer == null ? "null" : encodedBuffer.remaining()) - .log("Skipping empty/null buffer in decodeStream"); - return Flux.empty(); - } + return encodedFlux.concatMap(buffer -> decodeBuffer(buffer, state, decoder)) + .onErrorResume(throwable -> handleStreamError(throwable, state, decoder)) + .concatWith(Mono.defer(() -> handleStreamCompletion(state, decoder))); + } + + private Flux decodeBuffer(ByteBuffer buffer, DecoderState state, StructuredMessageDecoder decoder) { + + if (decoder.isComplete()) { + LOGGER.atVerbose() + .addKeyValue("bufferLength", buffer == null ? "null" : buffer.remaining()) + .log("Decoder already completed; ignoring extra buffer"); + return Flux.empty(); + } + + if (buffer == null || !buffer.hasRemaining()) { + return Flux.empty(); + } + + LOGGER.atInfo() + .addKeyValue("newBytes", buffer.remaining()) + .addKeyValue("decoderOffset", decoder.getMessageOffset()) + .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) + .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) + .log("Received buffer in decodeStream"); + + try { + StructuredMessageDecoder.DecodeResult result = decoder.decodeChunk(buffer); LOGGER.atInfo() - .addKeyValue("newBytes", encodedBuffer.remaining()) - .addKeyValue("decoderOffset", state.decoder.getMessageOffset()) - .addKeyValue("lastCompleteSegment", state.decoder.getLastCompleteSegmentStart()) - .addKeyValue("totalDecodedPayload", state.decoder.getTotalDecodedPayloadBytes()) - .log("Received buffer in decodeStream"); - - try { - // Use the new decodeChunk API which properly handles partial headers - StructuredMessageDecoder.DecodeResult result = state.decoder.decodeChunk(encodedBuffer); - - LOGGER.atInfo() - .addKeyValue("status", result.getStatus()) - .addKeyValue("bytesConsumed", result.getBytesConsumed()) - .addKeyValue("decoderOffset", state.decoder.getMessageOffset()) - .addKeyValue("lastCompleteSegment", state.decoder.getLastCompleteSegmentStart()) - .log("Decode chunk result"); - - updateProgress(state); - - switch (result.getStatus()) { - case SUCCESS: - case NEED_MORE_BYTES: - case COMPLETED: - return emitDecodedPayload(state, result.getDecodedPayload()); - - case INVALID: - LOGGER.error("Invalid data during decode: {}", result.getMessage()); - return Flux.error(createRetryableException(state, - "Failed to decode structured message: " + result.getMessage())); - - default: - return Flux.error(new IllegalStateException("Unknown decode status: " + result.getStatus())); - } - - } catch (Exception e) { - LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); - return Flux.error(createRetryableException(state, e.getMessage(), e)); - } - }).onErrorResume(throwable -> { - // If decoding already completed, suppress late downstream errors (mirror .NET). - if (state.decoder.isComplete()) { - LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); - return Flux.empty(); - } - state.addSegmentsToAggregateIfNeeded(); - // Wrap any error with retry offset information - if (throwable instanceof IOException) { - // Check if already has retry offset token - if (throwable.getMessage() != null && throwable.getMessage().contains(RETRY_OFFSET_TOKEN)) { - return Flux.error(throwable); - } - } - // Wrap the error with retry offset - return Flux.error(createRetryableException(state, throwable.getMessage(), throwable)); - }).concatWith(Mono.defer(() -> { - // Check on completion if decode is finished - if not, throw with retry offset - if (!state.decoder.isComplete()) { - LOGGER.atInfo() - .addKeyValue("messageOffset", state.decoder.getMessageOffset()) - .addKeyValue("messageLength", state.decoder.getMessageLength()) - .addKeyValue("totalDecodedPayload", state.decoder.getTotalDecodedPayloadBytes()) - .addKeyValue("lastCompleteSegment", state.decoder.getLastCompleteSegmentStart()) - .log("Stream ended but decode not finalized - throwing retryable exception"); - return Mono.error(createRetryableException(state, - "Stream ended prematurely before structured message decoding completed")); - } else { - state.addSegmentsToAggregateIfNeeded(); - if (state.aggregateCrcState != null && state.aggregateCrcState.hasSegments()) { - long composed = state.aggregateCrcState.composeCrc(); - long calculated = state.aggregateCrcState.getRunningCrc(); - if (composed != calculated) { - return Mono.error(LOGGER.logExceptionAsError(new IllegalArgumentException( - "CRC64 mismatch detected in composed structured message. Expected: " + composed + ", got: " - + calculated))); - } - } - LOGGER.atInfo() - .addKeyValue("messageOffset", state.decoder.getMessageOffset()) - .addKeyValue("totalDecodedPayload", state.decoder.getTotalDecodedPayloadBytes()) - .log("Stream complete and decode finalized successfully"); - return Mono.empty(); + .addKeyValue("status", result.getStatus()) + .addKeyValue("bytesConsumed", result.getBytesConsumed()) + .addKeyValue("decoderOffset", decoder.getMessageOffset()) + .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) + .log("Decode chunk result"); + + state.updateProgress(); + + switch (result.getStatus()) { + case SUCCESS: + case NEED_MORE_BYTES: + return handleSuccessOrNeedMore(state, result); + + case COMPLETED: + return handleCompleted(state, result); + + case INVALID: + return handleInvalid(state, result); + + default: + return Flux.error(new IllegalStateException("Unknown decode status: " + result.getStatus())); } - })); + } catch (Exception e) { + LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); + return Flux.error(createRetryableException(state, e.getMessage(), e)); + } } - /** - * Updates progress counters based on the current decoder state and logs when a new validated segment boundary is - * crossed. This keeps encoded progress in sync with the decoder while deferring payload emission until the entire - * message is validated. - */ - private void updateProgress(DecoderState state) { - long currentLastCompleteSegment = state.decoder.getLastCompleteSegmentStart(); + private Flux handleSuccessOrNeedMore(DecoderState state, StructuredMessageDecoder.DecodeResult result) { + return emitDecodedPayload(state, result.getDecodedPayload()); + } + + private Flux handleCompleted(DecoderState state, StructuredMessageDecoder.DecodeResult result) { + return emitDecodedPayload(state, result.getDecodedPayload()); + } - // Only update decodedBytesAtLastCompleteSegment when the boundary changes (new segment validated). - if (state.lastCompleteSegmentStart != currentLastCompleteSegment) { - state.decodedBytesAtLastCompleteSegment = state.decoder.getDecodedBytesAtLastCompleteSegment(); - state.lastCompleteSegmentStart = currentLastCompleteSegment; + private Flux handleInvalid(DecoderState state, StructuredMessageDecoder.DecodeResult result) { + LOGGER.error("Invalid data during decode: {}", result.getMessage()); + return Flux + .error(createRetryableException(state, "Failed to decode structured message: " + result.getMessage())); + } + + private Flux handleStreamError(Throwable throwable, DecoderState state, + StructuredMessageDecoder decoder) { + if (decoder.isComplete()) { + LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); + return Flux.empty(); + } + state.addSegmentsToAggregateIfNeeded(); + + if (throwable instanceof IOException + && throwable.getMessage() != null + && throwable.getMessage().contains(ContentValidationDecoderUtils.RETRY_OFFSET_TOKEN)) { + return Flux.error(throwable); + } + + return Flux.error(createRetryableException(state, throwable.getMessage(), throwable)); + } + + private Mono handleStreamCompletion(DecoderState state, StructuredMessageDecoder decoder) { + if (!decoder.isComplete()) { LOGGER.atInfo() - .addKeyValue("newSegmentBoundary", currentLastCompleteSegment) - .addKeyValue("decodedBytesAtBoundary", state.decodedBytesAtLastCompleteSegment) - .log("Segment boundary crossed, updated decoded bytes snapshot"); + .addKeyValue("messageOffset", decoder.getMessageOffset()) + .addKeyValue("messageLength", decoder.getMessageLength()) + .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) + .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) + .log("Stream ended but decode not finalized - throwing retryable exception"); + return Mono.error(createRetryableException(state, + "Stream ended prematurely before structured message decoding completed")); + } + + state.addSegmentsToAggregateIfNeeded(); + + if (state.aggregateCrcState != null && state.aggregateCrcState.hasSegments()) { + long composed = state.aggregateCrcState.composeCrc(); + long calculated = state.aggregateCrcState.getRunningCrc(); + if (composed != calculated) { + return Mono.error(LOGGER.logExceptionAsError( + new IllegalArgumentException("CRC64 mismatch detected in composed structured message. Expected: " + + composed + ", got: " + calculated))); + } } - long encodedProgress = state.decoder.getMessageOffset() + state.decoder.getPendingEncodedByteCount(); - state.totalEncodedBytesProcessed.set(encodedProgress); + LOGGER.atInfo() + .addKeyValue("messageOffset", decoder.getMessageOffset()) + .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) + .log("Stream complete and decode finalized successfully"); + return Mono.empty(); } private Flux emitDecodedPayload(DecoderState state, ByteBuffer decodedPayload) { @@ -381,7 +277,6 @@ private Flux emitDecodedPayload(DecoderState state, ByteBuffer decod return Flux.empty(); } - // Return a defensive copy to avoid any inadvertent position/limit side effects. ByteBuffer copy = ByteBuffer.allocate(decodedPayload.remaining()); copy.put(decodedPayload.duplicate()); copy.flip(); @@ -393,36 +288,21 @@ private Flux emitDecodedPayload(DecoderState state, ByteBuffer decod return Flux.just(copy); } - /** - * Creates an IOException with the retry start offset encoded in the message. - * - * @param state The decoder state. - * @param message The error message. - * @return An IOException with retry offset information. - */ private IOException createRetryableException(DecoderState state, String message) { return createRetryableException(state, message, null); } - /** - * Creates an IOException with the retry start offset encoded in the message. - * - * @param state The decoder state. - * @param message The error message. - * @param cause The original cause, may be null. - * @return An IOException with retry offset information. - */ private IOException createRetryableException(DecoderState state, String message, Throwable cause) { + StructuredMessageDecoder decoder = state.getDecoder(); long retryOffset = state.getRetryOffset(); long decodedSoFar = state.totalBytesDecoded.get(); - long expectedLength = state.decoder.getMessageLength(); + long expectedLength = decoder.getMessageLength(); String originalMessage = message != null ? message : ""; - - // Build message components for clarity long displayExpected = expectedLength > 0 ? expectedLength : 0; - String fullMessage = String.format("Incomplete structured message: decoded %d of %d bytes. %s%d. %s", - decodedSoFar, displayExpected, RETRY_OFFSET_TOKEN, retryOffset, originalMessage); + String fullMessage + = String.format("Incomplete structured message: decoded %d of %d bytes. %s%d. %s", decodedSoFar, + displayExpected, ContentValidationDecoderUtils.RETRY_OFFSET_TOKEN, retryOffset, originalMessage); LOGGER.atInfo() .addKeyValue("retryOffset", retryOffset) @@ -430,516 +310,6 @@ private IOException createRetryableException(DecoderState state, String message, .addKeyValue("expectedLength", expectedLength) .log("Creating retryable exception with offset"); - if (cause != null) { - return new IOException(fullMessage, cause); - } - return new IOException(fullMessage); - } - - /** - * Checks if structured message decoding should be applied based on context. - * - * @param context The pipeline call context. - * @return true if decoding should be applied, false otherwise. - */ - private boolean shouldApplyDecoding(HttpPipelineCallContext context) { - return context.getData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) - .map(value -> value instanceof Boolean && (Boolean) value) - .orElse(false); - } - - /** - * Gets the validation options from context. - * - * @param context The pipeline call context. - * @return The validation options or null if not present. - */ - private DownloadContentValidationOptions getValidationOptions(HttpPipelineCallContext context) { - return context.getData(Constants.STRUCTURED_MESSAGE_VALIDATION_OPTIONS_CONTEXT_KEY) - .filter(value -> value instanceof DownloadContentValidationOptions) - .map(value -> (DownloadContentValidationOptions) value) - .orElse(null); - } - - /** - * Gets the content length from response headers. - * - * @param headers The response headers. - * @return The content length or null if not present. - */ - private Long getContentLength(HttpHeaders headers, boolean responseScoped) { - if (!responseScoped) { - // Prefer the total length from Content-Range (if present) so retries use the full encoded length - // even when the current response is partial. - String contentRange = headers.getValue(HttpHeaderName.CONTENT_RANGE); - if (contentRange != null) { - // Format: bytes start-end/total - int slash = contentRange.indexOf('/'); - if (slash > -1 && slash + 1 < contentRange.length()) { - String totalPart = contentRange.substring(slash + 1).trim(); - if (!"*".equals(totalPart)) { - try { - return Long.parseLong(totalPart); - } catch (NumberFormatException e) { - LOGGER.warning("Invalid content range total length in response headers: " + contentRange); - } - } - } - } - } - - String contentLengthStr = headers.getValue(HttpHeaderName.CONTENT_LENGTH); - if (contentLengthStr != null) { - try { - return Long.parseLong(contentLengthStr); - } catch (NumberFormatException e) { - LOGGER.warning("Invalid content length in response headers: " + contentLengthStr); - } - } - return null; - } - - /** - * Gets or creates a decoder state from context. - * - * @param context The pipeline call context. - * @param contentLength The content length. - * @return The decoder state. - */ - private DecoderState getOrCreateDecoderState(HttpPipelineCallContext context, long contentLength, - boolean responseScoped) { - if (responseScoped) { - return new DecoderState(contentLength, null); - } - return context.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_CONTEXT_KEY) - .filter(value -> value instanceof DecoderState) - .map(value -> (DecoderState) value) - .orElseGet(() -> new DecoderState(contentLength, null)); - } - - /** - * Aggregates CRC state across retries to match .NET structured message retriable stream behavior. - */ - public static final class AggregateCrcState { - private final List segments = new ArrayList<>(); - private long runningCrc = 0; - - void appendPayload(ByteBuffer payload) { - if (payload == null || !payload.hasRemaining()) { - return; - } - ByteBuffer copy = payload.asReadOnlyBuffer(); - byte[] data = new byte[copy.remaining()]; - copy.get(data); - runningCrc = StorageCrc64Calculator.compute(data, runningCrc); - } - - void addSegments(List newSegments) { - if (newSegments == null || newSegments.isEmpty()) { - return; - } - segments.addAll(newSegments); - } - - boolean hasSegments() { - return !segments.isEmpty(); - } - - long getRunningCrc() { - return runningCrc; - } - - long composeCrc() { - if (segments.isEmpty()) { - return 0; - } - long composed = segments.get(0).getCrc64(); - long totalLength = segments.get(0).getLength(); - for (int i = 1; i < segments.size(); i++) { - StructuredMessageDecoder.SegmentInfo next = segments.get(i); - composed - = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); - totalLength += next.getLength(); - } - return composed; - } - - long getTotalLength() { - long totalLength = 0; - for (StructuredMessageDecoder.SegmentInfo segment : segments) { - totalLength += segment.getLength(); - } - return totalLength; - } - } - - /** - * Checks if the response is a download response. - * - * @param httpResponse The HTTP response. - * @return true if it's a download response, false otherwise. - */ - private boolean isDownloadResponse(HttpResponse httpResponse) { - HttpMethod method = httpResponse.getRequest().getHttpMethod(); - return method == HttpMethod.GET && httpResponse.getStatusCode() / 100 == 2; - } - - private boolean isResponseScoped(HttpPipelineCallContext context) { - return context.getData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY) - .map(value -> value instanceof Boolean && (Boolean) value) - .orElse(false); - } - - /** - * State holder for the structured message decoder that tracks decoding progress - * across network interruptions. - */ - public static class DecoderState { - private final StructuredMessageDecoder decoder; - private final long expectedContentLength; - private final AggregateCrcState aggregateCrcState; - /** - * Tracks how many decoded bytes have actually been emitted to the caller (excludes bytes skipped during - * fast-forward on retry). - */ - private final AtomicLong totalBytesDecoded; - private final AtomicLong totalEncodedBytesProcessed; - private final java.io.ByteArrayOutputStream accumulatedDecoded = new java.io.ByteArrayOutputStream(); - /** - * Snapshot of decoded bytes emitted at the last fully validated segment boundary. Used to correlate the encoded - * retry offset with the decoded offset that RetriableDownloadFlux tracks. - */ - private long decodedBytesAtLastCompleteSegment; - private long lastCompleteSegmentStart; // Tracks the last value to detect changes - /** - * Number of decoded bytes that should be skipped on the next retry to fast-forward to the caller's decoded - * offset. This mirrors the .NET StructuredMessageDecodingRetriableStream behavior. - */ - private final AtomicLong decodedBytesToSkip = new AtomicLong(0); - private boolean segmentsAddedToAggregate; - - /** - * Creates a new decoder state. - * - * @param expectedContentLength For response-scoped structured messages, the decoded payload length - * (e.g. from x-ms-structured-content-length); for range responses, the encoded length. The decoder - * validates this against the message header in the body. - * @param aggregateCrcState Aggregated CRC state shared across retries, or null if not aggregating. - */ - public DecoderState(long expectedContentLength, AggregateCrcState aggregateCrcState) { - this.expectedContentLength = expectedContentLength; - this.decoder = new StructuredMessageDecoder(expectedContentLength); - this.totalBytesDecoded = new AtomicLong(0); - this.totalEncodedBytesProcessed = new AtomicLong(0); - this.decodedBytesAtLastCompleteSegment = 0; - this.aggregateCrcState = aggregateCrcState; - this.segmentsAddedToAggregate = false; - } - - private void addSegmentsToAggregateIfNeeded() { - if (segmentsAddedToAggregate || aggregateCrcState == null) { - return; - } - aggregateCrcState.addSegments(decoder.getCompletedSegments()); - segmentsAddedToAggregate = true; - } - - /** - * Gets the total number of decoded bytes processed so far. - * - * @return The total decoded bytes. - */ - public long getTotalBytesDecoded() { - return totalBytesDecoded.get(); - } - - /** - * Gets the total number of encoded bytes processed so far. - * - * @return The total encoded bytes processed. - */ - public long getTotalEncodedBytesProcessed() { - return totalEncodedBytesProcessed.get(); - } - - /** - * Gets the expected encoded content length associated with this decoder state. - * - * @return The expected encoded content length. - */ - public long getExpectedContentLength() { - return expectedContentLength; - } - - /** - * Gets the total decoded payload bytes for this response. - * - * @return The decoded payload length. - */ - public long getDecodedPayloadLength() { - return decoder.getTotalDecodedPayloadBytes(); - } - - /** - * Gets the composed CRC64 for the decoded payload in this response. - * - * @return The composed CRC64 value. - */ - public long getComposedCrc64() { - if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { - return aggregateCrcState.composeCrc(); - } - - List segments = decoder.getCompletedSegments(); - if (segments.isEmpty()) { - return 0; - } - - long composed = segments.get(0).getCrc64(); - long totalLength = segments.get(0).getLength(); - for (int i = 1; i < segments.size(); i++) { - StructuredMessageDecoder.SegmentInfo next = segments.get(i); - composed - = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); - totalLength += next.getLength(); - } - return composed; - } - - /** - * Gets the composed decoded payload length represented by validated segments. - * - * @return The composed payload length. - */ - public long getComposedLength() { - if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { - return aggregateCrcState.getTotalLength(); - } - - List segments = decoder.getCompletedSegments(); - long totalLength = 0; - for (StructuredMessageDecoder.SegmentInfo segment : segments) { - totalLength += segment.getLength(); - } - return totalLength; - } - - /** - * Gets the decoded offset to use for retry requests. - * This uses the last complete segment boundary to ensure retries - * resume from a valid segment boundary, not mid-segment. - * - * Also resets decoder state to align with the segment boundary. - * - * @return The offset for retry requests (last complete segment boundary). - */ - public long getRetryOffset() { - // Use the decoded byte count at the last complete segment boundary for retry offset. - long retryOffset = decodedBytesAtLastCompleteSegment; - long lastCompleteSegmentOffset = decoder.getLastCompleteSegmentStart(); - - LOGGER.atInfo() - .addKeyValue("decoderOffset", decoder.getMessageOffset()) - .addKeyValue("pendingBytes", decoder.getPendingEncodedByteCount()) - .addKeyValue("lastCompleteSegment", lastCompleteSegmentOffset) - .log("Computed smart-retry offset from decoder state"); - return retryOffset; - } - - /** - * Prepares the decoder state for a retry by rewinding the decoder to the last complete segment - * boundary and resetting the accounting counters to that point. This mirrors the behavior of - * the cryptography smart-retry implementation which always replays from a validated boundary. - * - * @return The retry start offset (decoded byte position) that the next request should use. - */ - public long prepareForRetry() { - return resetForRetry(); - } - - /** - * Resets decoder and counters to the last validated segment boundary and returns the retry offset. - * - * @return retry offset (decoded boundary). - */ - public long resetForRetry() { - long retryOffset = decodedBytesAtLastCompleteSegment; - - decoder.resetToLastCompleteSegment(); - accumulatedDecoded.reset(); - - // Align encoded counters to the boundary we will resume from so subsequent progress tracking is consistent. - totalEncodedBytesProcessed.set(decoder.getMessageOffset() + decoder.getPendingEncodedByteCount()); - decodedBytesToSkip.set(0); - - LOGGER.atInfo() - .addKeyValue("retryOffset", retryOffset) - .addKeyValue("decoderOffset", decoder.getMessageOffset()) - .addKeyValue("decodedBytesAtBoundary", decodedBytesAtLastCompleteSegment) - .log("Prepared decoder state for smart retry"); - - return retryOffset; - } - - /** - * Resets decoder state while preserving decoded bytes up to the last complete segment boundary. - * This allows retries to resume from a validated boundary without losing already validated payload - * when nothing has been emitted downstream yet. - * - * @return retry offset (decoded boundary). - */ - public long resetForRetryPreservingPrefix() { - long retryOffset = decodedBytesAtLastCompleteSegment; - - decoder.resetToLastCompleteSegment(); - - byte[] data = accumulatedDecoded.toByteArray(); - accumulatedDecoded.reset(); - - long prefixLength = decodedBytesAtLastCompleteSegment; - if (prefixLength > 0 && data.length > 0) { - int keep = (int) Math.min(prefixLength, data.length); - accumulatedDecoded.write(data, 0, keep); - } - - // Align encoded counters to the boundary we will resume from so subsequent progress tracking is consistent. - totalEncodedBytesProcessed.set(decoder.getMessageOffset() + decoder.getPendingEncodedByteCount()); - decodedBytesToSkip.set(0); - - LOGGER.atInfo() - .addKeyValue("retryOffset", retryOffset) - .addKeyValue("decoderOffset", decoder.getMessageOffset()) - .addKeyValue("decodedPrefixBytes", decodedBytesAtLastCompleteSegment) - .log("Prepared decoder state for smart retry preserving validated prefix"); - - return retryOffset; - } - - /** - * Checks if the decoder has finalized. - * - * @return true if finalized, false otherwise. - */ - public boolean isFinalized() { - return decoder.isComplete(); - } - - /** - * Gets the decoded payload bytes accounted for at the last complete segment boundary. - * This is used to correlate decoder progress with reliable download offsets. - * - * @return The decoded byte count at the last segment boundary. - */ - public long getDecodedBytesAtLastCompleteSegment() { - return decodedBytesAtLastCompleteSegment; - } - - /** - * Sets how many decoded bytes should be skipped when resuming after a retry. This lets the decoder fast-forward - * within the current segment to align with the decoded offset already emitted to the user before the failure. - * - * @param bytesToSkip decoded bytes to drop from the next decoded payloads. - */ - public void setDecodedBytesToSkip(long bytesToSkip) { - decodedBytesToSkip.set(Math.max(0, bytesToSkip)); - } - - /** - * @param decoded Append decoded bytes produced so far in the current decode attempt. - * - */ - public void appendPartial(ByteBuffer decoded) { - if (decoded == null || !decoded.hasRemaining()) { - return; - } - ByteBuffer copy = decoded.asReadOnlyBuffer(); - byte[] data = new byte[copy.remaining()]; - copy.get(data); - accumulatedDecoded.write(data, 0, data.length); - } - - /** - * Drains and returns all accumulated decoded bytes for this message. - * - * @return ByteBuffer containing all decoded bytes or null if none accumulated. - */ - public ByteBuffer drainPartial() { - byte[] data = accumulatedDecoded.toByteArray(); - accumulatedDecoded.reset(); - return data.length == 0 ? null : ByteBuffer.wrap(data); - } - } - - /** - * Decoded HTTP response that wraps the original response with a decoded stream. - */ - private static class DecodedResponse extends HttpResponse { - private final HttpResponse originalResponse; - private final Flux decodedBody; - private final DecoderState decoderState; - - /** - * Creates a new decoded response. - * - * @param originalResponse The original HTTP response. - * @param decodedBody The decoded body stream. - * @param decoderState The decoder state. - */ - DecodedResponse(HttpResponse originalResponse, Flux decodedBody, DecoderState decoderState) { - super(originalResponse.getRequest()); - this.originalResponse = originalResponse; - this.decodedBody = decodedBody; - this.decoderState = decoderState; - } - - @Override - public int getStatusCode() { - return originalResponse.getStatusCode(); - } - - @Override - public String getHeaderValue(String name) { - return originalResponse.getHeaderValue(name); - } - - @Override - public HttpHeaders getHeaders() { - return originalResponse.getHeaders(); - } - - @Override - public Flux getBody() { - // Ensure the original response is closed once the decoded stream completes. - return Flux.using(() -> originalResponse, r -> decodedBody, HttpResponse::close); - } - - @Override - public Mono getBodyAsByteArray() { - return FluxUtil.collectBytesInByteBufferStream(getBody()); - } - - @Override - public Mono getBodyAsString() { - return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); - } - - @Override - public Mono getBodyAsString(Charset charset) { - return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); - } - - @Override - public void close() { - originalResponse.close(); - } - - /** - * Gets the decoder state. - * - * @return The decoder state. - */ - public DecoderState getDecoderState() { - return decoderState; - } + return cause != null ? new IOException(fullMessage, cause) : new IOException(fullMessage); } } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java index ef55e1210525..ddd488ff0887 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java @@ -59,7 +59,6 @@ public HttpHeaders getHeaders() { @Override public Flux getBody() { - // Always close the wrapped response when body consumption terminates. return Flux.using(() -> originalResponse, ignored -> body, HttpResponse::close); } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index 574942d24a0b..ccef4a5505d3 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -26,7 +26,7 @@ public class MockPartialResponsePolicy implements HttpPipelinePolicy { static final HttpHeaderName RANGE_HEADER = HttpHeaderName.RANGE; private final AtomicInteger tries; private final List rangeHeaders = Collections.synchronizedList(new ArrayList<>()); - private final int maxBytesPerResponse; // Maximum bytes to return before simulating timeout + private final int maxBytesPerResponse; private final AtomicInteger hits = new AtomicInteger(); private final String targetUrlPrefix; @@ -36,7 +36,7 @@ public class MockPartialResponsePolicy implements HttpPipelinePolicy { * @param tries Number of times to simulate interruptions (0 = no interruptions) */ public MockPartialResponsePolicy(int tries) { - this(tries, 200, null); // Default: return 200 bytes for subsequent interruptions (enables 3 interrupts with 1KB data) + this(tries, 200, null); } /** @@ -64,7 +64,6 @@ public MockPartialResponsePolicy(int tries, int maxBytesPerResponse, String targ @Override public HttpPipelinePosition getPipelinePosition() { - // Apply on every retry to mirror .NET test behavior. return HttpPipelinePosition.PER_RETRY; } @@ -74,7 +73,6 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN HttpHeader rangeHttpHeader = response.getRequest().getHeaders().get(RANGE_HEADER); HttpHeader xMsRangeHttpHeader = response.getRequest().getHeaders().get(X_MS_RANGE_HEADER); - // Record every GET attempt so tests can assert retries occurred, even if no range header was present. if (response.getRequest().getHttpMethod() == HttpMethod.GET) { String recordedRange = null; if (rangeHttpHeader != null && rangeHttpHeader.getValue().startsWith("bytes=")) { @@ -99,9 +97,6 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN System.out.println("[MockPartialResponsePolicy] invoked. tries=" + remainingTries + ", maxBytesPerResponse=" + maxBytesPerResponse); - // Simulate an interruption mid-stream (like FaultyStream in .NET) without mutating headers. - // Emit up to maxBytesPerResponse, then complete early to let the decoder detect an incomplete message - // and trigger smart-retry. Flux limitedBody = limitStreamToBytes(response.getBody(), maxBytesPerResponse); return Mono.just( new MockDownloadHttpResponse(response, response.getStatusCode(), response.getHeaders(), @@ -110,10 +105,6 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN }); } - /** - * Limits a Flux of ByteBuffers to emit at most maxBytes, then completes early to simulate - * an abrupt connection close without surfacing an explicit error. - */ private Flux limitStreamToBytes(Flux body, int maxBytes) { return Flux.defer(() -> { final long[] bytesEmitted = new long[] { 0 }; @@ -124,7 +115,6 @@ private Flux limitStreamToBytes(Flux body, int maxBytes) long remaining = maxBytes - bytesEmitted[0]; if (remaining <= 0) { - // Emit an error to simulate the network fault (mirrors FaultyStream in .NET). return Flux.error(new IOException("Simulated timeout")); } @@ -132,12 +122,10 @@ private Flux limitStreamToBytes(Flux body, int maxBytes) if (bufferSize <= remaining) { bytesEmitted[0] += bufferSize; if (bytesEmitted[0] >= maxBytes) { - // Emit this buffer then fail to simulate the connection drop. return Flux.just(buffer).concatWith(Flux.error(new IOException("Simulated timeout"))); } return Flux.just(buffer); } else { - // Buffer is larger than remaining, slice and then error. int bytesToEmit = (int) remaining; ByteBuffer slice = buffer.duplicate(); slice.limit(slice.position() + bytesToEmit); diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java index dbaaf5c41550..6d9a88a8f2ba 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java @@ -10,7 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /** - * Unit tests for StorageContentValidationDecoderPolicy. + * Unit tests for parsing helpers used by StorageContentValidationDecoderPolicy. */ public class StorageContentValidationDecoderPolicyTest { @@ -65,41 +65,41 @@ public void parseRetryStartOffsetReturnsNegativeOneForMalformedToken() { @Test public void parseDecoderOffsetsFromEnrichedMessage() { String message = "Invalid segment size [decoderOffset=523,lastCompleteSegment=287]"; - long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); assertArrayEquals(new long[] { 523, 287 }, offsets); } @Test public void parseDecoderOffsetsWithZeroValues() { String message = "Header error [decoderOffset=0,lastCompleteSegment=0]"; - long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); assertArrayEquals(new long[] { 0, 0 }, offsets); } @Test public void parseDecoderOffsetsWithLargeValues() { String message = "Error [decoderOffset=9999999999,lastCompleteSegment=8888888888]"; - long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); assertArrayEquals(new long[] { 9999999999L, 8888888888L }, offsets); } @Test public void parseDecoderOffsetsReturnsNullForMissingPattern() { String message = "Error without decoder offset information"; - long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); assertNull(offsets); } @Test public void parseDecoderOffsetsReturnsNullForNullMessage() { - long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(null); + long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(null); assertNull(offsets); } @Test public void parseDecoderOffsetsReturnsNullForMalformedPattern() { String message = "[decoderOffset=abc,lastCompleteSegment=xyz]"; - long[] offsets = StorageContentValidationDecoderPolicy.parseDecoderOffsets(message); + long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); assertNull(offsets); } } From a58dc636fbb0ab180168975f9cd1ffb71cef311e Mon Sep 17 00:00:00 2001 From: Isabelle Date: Sun, 29 Mar 2026 17:19:27 -0700 Subject: [PATCH 11/31] fixing errors i introduced :( --- .../blob/specialized/BlobAsyncClientBase.java | 98 +++---------------- .../blob/specialized/BlobClientBase.java | 2 +- .../BlobMessageAsyncDecoderDownloadTests.java | 1 - .../blob/BlobMessageDecoderDownloadTests.java | 5 +- .../StructuredMessageDecoder.java | 12 +-- .../StructuredMessageDecodingStream.java | 11 ++- ...StorageContentValidationDecoderPolicy.java | 5 - .../StructuredMessageDecoderTests.java | 10 +- 8 files changed, 34 insertions(+), 110 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index bbde1d670450..fdf4056a9bcb 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -87,7 +87,7 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.SasImplUtils; -import com.azure.storage.common.implementation.StorageCrc64Calculator; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.StorageChecksumAlgorithm; import com.azure.storage.common.policy.AggregateCrcState; @@ -1212,52 +1212,6 @@ public Mono downloadStreamWithResponse(BlobDownloadSt } } - /** - * Reads a range of bytes from a blob with content validation options. Uploading data must be done from the {@link BlockBlobClient}, {@link - * PageBlobClient}, or {@link AppendBlobClient}. - * - *

Code Samples

- * - * 
{@code
-     * BlobRange range = new BlobRange(1024, 2048L);
-     * DownloadRetryOptions options = new DownloadRetryOptions().setMaxRetryRequests(5);
-     * DownloadContentValidationOptions validationOptions = new DownloadContentValidationOptions()
-     *     .setStructuredMessageValidationEnabled(true);
-     *
-     * client.downloadStreamWithResponse(range, options, null, false, validationOptions).subscribe(response -> {
-     *     ByteArrayOutputStream downloadData = new ByteArrayOutputStream();
-     *     response.getValue().subscribe(piece -> {
-     *         try {
-     *             downloadData.write(piece.array());
-     *         } catch (IOException ex) {
-     *             throw new UncheckedIOException(ex);
-     *         }
-     *     });
-     * });
-     * }
- * - *

For more information, see the - * Azure Docs

- * - * @param range {@link BlobRange} - * @param options {@link DownloadRetryOptions} - * @param requestConditions {@link BlobRequestConditions} - * @param getRangeContentMd5 Whether the contentMD5 for the specified blob range should be returned. - * @param contentValidationOptions {@link DownloadContentValidationOptions} options for content validation - * @return A reactive response containing the blob data. - */ - @ServiceMethod(returns = ReturnType.SINGLE) - public Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, - BlobRequestConditions requestConditions, boolean getRangeContentMd5, - DownloadContentValidationOptions contentValidationOptions) { - try { - return withContext(context -> downloadStreamWithResponse(range, options, requestConditions, - getRangeContentMd5, contentValidationOptions, context)); - } catch (RuntimeException ex) { - return monoError(LOGGER, ex); - } - } - /** * Reads a range of bytes from a blob. Uploading data must be done from the {@link BlockBlobClient}, {@link * PageBlobClient}, or {@link AppendBlobClient}. @@ -1315,7 +1269,7 @@ public Mono downloadContentWithResponse(BlobDo } Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, - BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context){ + BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context) { // Prevents revapi visibility increased error return downloadStreamWithResponseInternal(range, options, requestConditions, getRangeContentMd5, null, context); } @@ -1325,15 +1279,10 @@ Mono downloadStreamWithResponseInternal(BlobRange ran StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; -<<<<<<< HEAD - boolean structuredDecode - = contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled(); -======= final StorageChecksumAlgorithm algorithm = responseChecksumAlgorithm != null ? responseChecksumAlgorithm : StorageChecksumAlgorithm.NONE; final boolean isStructuredMessageEnabled = isStructuredMessageAlgorithm(algorithm); final boolean isMd5Enabled = algorithm == StorageChecksumAlgorithm.MD5; ->>>>>>> f96332b51d4 (code refactoring) final Boolean finalGetMD5 = (!isStructuredMessageEnabled && (getRangeContentMd5 || isMd5Enabled)) ? true : null; @@ -1410,11 +1359,6 @@ >>>>>>> f96332b51d4 (code refactoring) } - Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, - BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context) { - return downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, null, context); - } - private Mono downloadRange(BlobRange range, BlobRequestConditions requestConditions, String eTag, Boolean getMD5, String structuredBodyType, Context context) { return azureBlobStorage.getBlobs() @@ -1706,7 +1650,6 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti = originalParallelTransferOptions != null && originalParallelTransferOptions.getMaxConcurrency() != null; com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions = ModelHelper.populateAndApplyDefaults(originalParallelTransferOptions); - DownloadContentValidationOptions contentValidationOptions = options.getContentValidationOptions(); BlobRequestConditions finalConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); @@ -1719,14 +1662,10 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti // Run download on boundedElastic to avoid blocking the reactor thread when async file I/O // completion is delivered (matches .NET behavior where DownloadTo runs off the sync context). return Mono.just(channel) - .flatMap(c -> this - .downloadToFileImpl(c, finalRange, finalParallelTransferOptions, blockSizeProvided, - maxConcurrencyProvided, options.getDownloadRetryOptions(), finalConditions, - options.isRetrieveContentRangeMd5(), contentValidationOptions, context) - .subscribeOn(Schedulers.boundedElastic())) - .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, - options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), - options.getResponseChecksumAlgorithm(), context)) + .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, blockSizeProvided, + maxConcurrencyProvided, options.getDownloadRetryOptions(), finalConditions, + options.isRetrieveContentRangeMd5(), options.getResponseChecksumAlgorithm(), context)) + .subscribeOn(Schedulers.boundedElastic()) .doFinally(signalType -> this.downloadToFileCleanup(channel, options.getFilePath(), signalType)); } @@ -1739,45 +1678,34 @@ private AsynchronousFileChannel downloadToFileResourceSupplier(String filePath, } private Mono> downloadToFileImpl(AsynchronousFileChannel file, BlobRange finalRange, - com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, - DownloadRetryOptions downloadRetryOptions, BlobRequestConditions requestConditions, boolean rangeGetContentMd5, - StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, boolean blockSizeProvided, boolean maxConcurrencyProvided, DownloadRetryOptions downloadRetryOptions, BlobRequestConditions requestConditions, boolean rangeGetContentMd5, - DownloadContentValidationOptions contentValidationOptions, Context context) { + StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { // See ProgressReporter for an explanation on why this lock is necessary and why we use AtomicLong. ProgressListener progressReceiver = finalParallelTransferOptions.getProgressListener(); ProgressReporter progressReporter = progressReceiver == null ? null : ProgressReporter.withProgressListener(progressReceiver); -<<<<<<< HEAD - boolean structuredDecode - = contentValidationOptions != null && contentValidationOptions.isStructuredMessageValidationEnabled(); - final Context downloadContext = structuredDecode -======= final boolean isStructuredMessageEnabled = isStructuredMessageAlgorithm(responseChecksumAlgorithm); final boolean isMd5Enabled = responseChecksumAlgorithm == StorageChecksumAlgorithm.MD5; final Context downloadContext = isStructuredMessageEnabled ->>>>>>> f96332b51d4 (code refactoring) ? (context == null ? new Context(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) : context.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true)) : context; - /* * Downloads the first chunk and gets the size of the data and etag if not specified by the user. */ BiFunction> downloadFunc - = (range, conditions) -> structuredDecode - ? this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, rangeGetContentMd5, - contentValidationOptions, downloadContext) + = (range, conditions) -> isStructuredMessageEnabled + ? this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, rangeGetContentMd5, + responseChecksumAlgorithm, downloadContext) : this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, rangeGetContentMd5, downloadContext); - BiFunction> emptyBlobDownloadFunc - = (range, conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, false, - null, context); + BiFunction> emptyBlobDownloadFunc = (range, + conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, false, context); boolean checksumValidationEnabled = isStructuredMessageEnabled || rangeGetContentMd5 || isMd5Enabled; boolean md5ValidationEnabled = !isStructuredMessageEnabled && (rangeGetContentMd5 || isMd5Enabled); @@ -1798,8 +1726,6 @@ >>>>>>> f96332b51d4 (code refactoring) int maxConcurrency = maxConcurrencyProvided ? finalParallelTransferOptions.getMaxConcurrency() : getDefaultDownloadConcurrency(); - = (range, conditions) -> this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, - rangeGetContentMd5, responseChecksumAlgorithm, context); com.azure.storage.common.ParallelTransferOptions initialParallelTransferOptions = new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(initialRangeSize); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java index 8d2fab4da17d..53792cfed42e 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java @@ -1642,7 +1642,7 @@ private static BlobDownloadToFileOptions adjustOptionsForSyncDownload(BlobDownlo .setRequestConditions(options.getRequestConditions()) .setRetrieveContentRangeMd5(options.isRetrieveContentRangeMd5()) .setOpenOptions(options.getOpenOptions()) - .setContentValidationOptions(options.getContentValidationOptions()); + .setResponseChecksumAlgorithm(options.getResponseChecksumAlgorithm()); } /** diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java index cd9f882fa578..4a3898369a30 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java @@ -416,4 +416,3 @@ static boolean isLiveMode() { return ENVIRONMENT.getTestMode() == TestMode.LIVE; } } - diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 751503b2dc72..dc3ab286b5c5 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -8,7 +8,9 @@ import com.azure.storage.blob.options.BlobDownloadContentOptions; import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; -import com.azure.storage.common.DownloadContentValidationOptions; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.specialized.BlobInputStream; import com.azure.storage.common.ParallelTransferOptions; import com.azure.storage.common.StorageChecksumAlgorithm; import org.junit.jupiter.api.BeforeEach; @@ -20,6 +22,7 @@ import org.junit.jupiter.params.provider.ValueSource; import reactor.core.publisher.Flux; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 2058f5144431..977e4a78f105 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package com.azure.storage.common.implementation.structuredmessage; +package com.azure.storage.common.implementation.contentvalidation; import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.implementation.StorageCrc64Calculator; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; @@ -14,10 +14,10 @@ import java.util.List; import java.util.Map; -import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.CRC64_LENGTH; -import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; -import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_HEADER_LENGTH; -import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; /** * Decoder for structured messages with support for segmenting and CRC64 checksums. diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java index 5fec64e0c18a..793b12dc43eb 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java @@ -1,10 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package com.azure.storage.common.implementation.structuredmessage; +package com.azure.storage.common.implementation.contentvalidation; import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.DownloadContentValidationOptions; +import com.azure.storage.common.StorageChecksumAlgorithm; + import reactor.core.publisher.Flux; import java.nio.ByteBuffer; @@ -24,13 +25,13 @@ private StructuredMessageDecodingStream() { * * @param originalStream The original download stream. * @param contentLength The expected content length. - * @param validationOptions The content validation options. + * @param responseChecksumAlgorithm The response checksum algorithm. * @return A Flux that decodes structured messages if validation is enabled, otherwise returns the original stream. */ public static Flux wrapStreamIfNeeded(Flux originalStream, Long contentLength, - DownloadContentValidationOptions validationOptions) { + StorageChecksumAlgorithm responseChecksumAlgorithm) { - if (validationOptions == null || !validationOptions.isStructuredMessageValidationEnabled()) { + if (responseChecksumAlgorithm == null || responseChecksumAlgorithm != StorageChecksumAlgorithm.CRC64) { return originalStream; } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 17bab322fb0c..896cc88f2432 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -11,12 +11,7 @@ import com.azure.core.http.HttpPipelinePosition; import com.azure.core.util.logging.ClientLogger; import com.azure.storage.common.implementation.Constants; -<<<<<<< HEAD -import com.azure.storage.common.implementation.StorageCrc64Calculator; -import com.azure.storage.common.implementation.structuredmessage.StructuredMessageDecoder; -======= import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; ->>>>>>> f96332b51d4 (code refactoring) import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java index 2c08e40b63a5..d045b7f7fe64 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package com.azure.storage.common.implementation.structuredmessage; +package com.azure.storage.common.implementation.contentvalidation; import org.junit.jupiter.api.Test; @@ -10,9 +10,9 @@ import java.nio.ByteOrder; import java.util.concurrent.ThreadLocalRandom; -import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.CRC64_LENGTH; -import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_HEADER_LENGTH; -import static com.azure.storage.common.implementation.structuredmessage.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -24,7 +24,7 @@ * and segment splits across chunks. */ public class StructuredMessageDecoderTests { - + @Test public void readsCompleteMessageInSingleChunk() throws IOException { // Test: Complete message in a single ByteBuffer should decode fully From e12aa8269dbea4148a13ec3428cbacea3e040c5a Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Thu, 2 Apr 2026 23:34:20 +0530 Subject: [PATCH 12/31] addressing review comments --- ...DownloadAsyncResponseConstructorProxy.java | 30 -- .../util/ChunkedDownloadUtils.java | 2 +- .../models/BlobDownloadAsyncResponse.java | 5 - .../blob/specialized/BlobAsyncClientBase.java | 35 +- .../src/test/resources/logback-test.xml | 11 - .../checkstyle-suppressions.xml | 13 +- .../StorageCrc64Calculator.java | 380 +----------------- ...StorageContentValidationDecoderPolicy.java | 2 + .../StorageCrc64CalculatorTests.java | 22 - 9 files changed, 36 insertions(+), 464 deletions(-) delete mode 100644 sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java index f42f1f0f4db2..dd59769fbc6a 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java @@ -40,14 +40,6 @@ BlobDownloadAsyncResponse create(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions, AtomicReference decoderStateRef); - /** - * Gets the source {@link StreamResponse} from a {@link BlobDownloadAsyncResponse}. - * - * @param response The {@link BlobDownloadAsyncResponse}. - * @return The source {@link StreamResponse}, or null if not available. - */ - StreamResponse getSourceResponse(BlobDownloadAsyncResponse response); - /** * Gets the current decoder state from a {@link BlobDownloadAsyncResponse}. * @@ -89,22 +81,6 @@ public static BlobDownloadAsyncResponse create(StreamResponse sourceResponse, return accessor.create(sourceResponse, onErrorResume, retryOptions, decoderStateRef); } - /** - * Gets the source {@link StreamResponse} from a {@link BlobDownloadAsyncResponse}. - * - * @param response The {@link BlobDownloadAsyncResponse}. - * @return The source {@link StreamResponse}, or null if not available. - */ - public static StreamResponse getSourceResponse(BlobDownloadAsyncResponse response) { - if (accessor == null) { - new BlobDownloadAsyncResponse(new HttpRequest(HttpMethod.GET, "http://microsoft.com"), 200, - new HttpHeaders(), null, null); - } - - assert accessor != null; - return accessor.getSourceResponse(response); - } - /** * Gets the current decoder state from a {@link BlobDownloadAsyncResponse}. * @@ -112,12 +88,6 @@ public static StreamResponse getSourceResponse(BlobDownloadAsyncResponse respons * @return The decoder state, or null if not available. */ public static DecoderState getDecoderState(BlobDownloadAsyncResponse response) { - if (accessor == null) { - new BlobDownloadAsyncResponse(new HttpRequest(HttpMethod.GET, "http://microsoft.com"), 200, - new HttpHeaders(), null, null); - } - - assert accessor != null; return accessor.getDecoderState(response); } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java index 9b22d85082bf..294bb378b8ea 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java @@ -166,7 +166,7 @@ public static Flux downloadChunk(Integer chunkNum, BlobDownloadAsyncRespo private static BlobRequestConditions setEtag(BlobRequestConditions requestConditions, String etag) { // We don't want to modify the user's object, so we'll create a duplicate and set the retrieved etag. return new BlobRequestConditions().setIfModifiedSince(requestConditions.getIfModifiedSince()) - .setIfUnmodifiedSince(requestConditions.getIfModifiedSince()) + .setIfUnmodifiedSince(requestConditions.getIfUnmodifiedSince()) .setIfMatch(etag) .setIfNoneMatch(requestConditions.getIfNoneMatch()) .setLeaseId(requestConditions.getLeaseId()); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java index 527ce4862408..59a021493e1f 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java @@ -45,11 +45,6 @@ public BlobDownloadAsyncResponse create(StreamResponse sourceResponse, return new BlobDownloadAsyncResponse(sourceResponse, onErrorResume, retryOptions, decoderStateRef); } - @Override - public StreamResponse getSourceResponse(BlobDownloadAsyncResponse response) { - return response.sourceResponse; - } - @Override public DecoderState getDecoderState(BlobDownloadAsyncResponse response) { AtomicReference ref = response.decoderStateRef; diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index fdf4056a9bcb..ad32aceff5eb 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1294,7 +1294,6 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - String structuredBodyType = isStructuredMessageEnabled ? Constants.STRUCTURED_MESSAGE_CRC64_BODY_TYPE : null; final Context responseScopedContext = isStructuredMessageEnabled ? baseContext.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) : baseContext; @@ -1313,7 +1312,7 @@ Mono downloadStreamWithResponseInternal(BlobRange ran } return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), finalGetMD5, - structuredBodyType, firstRangeContext).map(response -> { + firstRangeContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); String eTag = blobsDownloadHeaders.getETag(); BlobDownloadHeaders blobDownloadHeaders = ModelHelper.populateBlobDownloadHeaders(blobsDownloadHeaders, @@ -1332,6 +1331,8 @@ Mono downloadStreamWithResponseInternal(BlobRange ran finalCount = finalRange.getCount(); } + // The resume function takes throwable and offset at the destination. + // I.e. offset is relative to the starting point. BiFunction> onDownloadErrorResume = (throwable, offset) -> { if (!(throwable instanceof IOException || throwable instanceof TimeoutException)) { return Mono.error(throwable); @@ -1340,12 +1341,27 @@ Mono downloadStreamWithResponseInternal(BlobRange ran try { if (isStructuredMessageEnabled) { return retryStructuredDownload(throwable, offset, decoderStateRef, responseStartOffset, - finalCount, initialOffset, firstRangeContext, finalRequestConditions, eTag, finalGetMD5, - structuredBodyType); + finalCount, initialOffset, firstRangeContext, finalRequestConditions, eTag, + finalGetMD5); } else { long newCount = finalCount - offset; + + /* + * It's possible that the network stream will throw an error after emitting all data but + * before completing. Issuing a retry at this stage would leave the download in a bad + * state with incorrect count and offset values. Because we have read the intended amount + * of data, we can ignore the error at the end of the stream. + */ + if (newCount == 0) { + LOGGER.warning( + "Exception encountered in ReliableDownload after all data read from the network " + + "but before stream signaled completion. Returning success as all data was " + + "downloaded. Exception message: " + throwable.getMessage()); + return Mono.empty(); + } + BlobRange retryRange = new BlobRange(initialOffset + offset, newCount); - return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, null, + return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, firstRangeContext); } } catch (Exception e) { @@ -1360,10 +1376,10 @@ Mono downloadStreamWithResponseInternal(BlobRange ran } private Mono downloadRange(BlobRange range, BlobRequestConditions requestConditions, String eTag, - Boolean getMD5, String structuredBodyType, Context context) { + Boolean getMD5, Context context) { return azureBlobStorage.getBlobs() .downloadNoCustomHeadersWithResponseAsync(containerName, blobName, snapshot, versionId, null, - range.toHeaderValue(), requestConditions.getLeaseId(), getMD5, null, structuredBodyType, + range.toHeaderValue(), requestConditions.getLeaseId(), getMD5, null, null, requestConditions.getIfModifiedSince(), requestConditions.getIfUnmodifiedSince(), eTag, requestConditions.getIfNoneMatch(), requestConditions.getTagsConditions(), null, customerProvidedKey, context); @@ -1371,8 +1387,7 @@ private Mono downloadRange(BlobRange range, BlobRequestCondition private Mono retryStructuredDownload(Throwable throwable, long emittedOffset, AtomicReference decoderStateRef, AtomicLong responseStartOffset, long finalCount, - long initialOffset, Context baseRetryContext, BlobRequestConditions conditions, String eTag, Boolean getMD5, - String structuredBodyType) { + long initialOffset, Context baseRetryContext, BlobRequestConditions conditions, String eTag, Boolean getMD5) { long currentResponseOffset = responseStartOffset.get(); DecoderState decoderState = decoderStateRef.get(); @@ -1404,7 +1419,7 @@ private Mono retryStructuredDownload(Throwable throwable, long e LOGGER.info("Structured message retry: resuming from offset {} (initial={}, decoded={}, remaining={}, skip={})", initialOffset + retryStartOffset, initialOffset, retryStartOffset, remainingCount, bytesToSkip); - return downloadRange(retryRange, conditions, eTag, getMD5, structuredBodyType, retryContext); + return downloadRange(retryRange, conditions, eTag, getMD5, retryContext); } private static long resolveStructuredRetryOffset(DecoderState decoderState, Throwable throwable, diff --git a/sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml b/sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml deleted file mode 100644 index b35926b40592..000000000000 --- a/sdk/storage/azure-storage-blob/src/test/resources/logback-test.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - diff --git a/sdk/storage/azure-storage-common/checkstyle-suppressions.xml b/sdk/storage/azure-storage-common/checkstyle-suppressions.xml index 93d35df5d619..4e4a986c034c 100644 --- a/sdk/storage/azure-storage-common/checkstyle-suppressions.xml +++ b/sdk/storage/azure-storage-common/checkstyle-suppressions.xml @@ -3,11 +3,10 @@ - - - - - - - + + + + + + diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java index 2ed9ba6c539b..1352423eb693 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java @@ -2400,168 +2400,18 @@ public class StorageCrc64Calculator { /** * Computes the CRC64 checksum for the given byte array using the Azure Storage CRC64 polynomial. - * This method processes the input data in chunks of 32 bytes for efficiency and uses lookup tables - * to update the CRC values. * * @param src the byte array for which the CRC64 checksum is to be computed. * @param uCrc the initial CRC value. * @return the computed CRC64 checksum. */ public static long compute(byte[] src, long uCrc) { - int pData = 0; - long uSize = src.length; - long uBytes, uStop; - - uCrc = ~uCrc; // Flip all bits of uCrc - - uStop = uSize - (uSize % 32); - if (uStop >= 2 * 32) { - long uCrc0 = 0L; - long uCrc1 = 0L; - long uCrc2 = 0L; - long uCrc3 = 0L; - - int pLast = pData + (int) uStop - 32; - uSize -= uStop; - uCrc0 = uCrc; - - ByteBuffer buffer = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); - - for (; pData < pLast; pData += 32) { - long b0, b1, b2, b3; - - // Load and XOR data with CRC - b0 = buffer.getLong(pData) ^ uCrc0; - b1 = buffer.getLong(pData + 8) ^ uCrc1; - b2 = buffer.getLong(pData + 16) ^ uCrc2; - b3 = buffer.getLong(pData + 24) ^ uCrc3; - - // Unsigned updates using tables and masking - uCrc0 = M_U32[7 * 256 + ((int) (b0 & 0xFF))]; - b0 >>>= 8; - uCrc1 = M_U32[7 * 256 + ((int) (b1 & 0xFF))]; - b1 >>>= 8; - uCrc2 = M_U32[7 * 256 + ((int) (b2 & 0xFF))]; - b2 >>>= 8; - uCrc3 = M_U32[7 * 256 + ((int) (b3 & 0xFF))]; - b3 >>>= 8; - - uCrc0 ^= M_U32[6 * 256 + ((int) (b0 & 0xFF))]; - b0 >>>= 8; - uCrc1 ^= M_U32[6 * 256 + ((int) (b1 & 0xFF))]; - b1 >>>= 8; - uCrc2 ^= M_U32[6 * 256 + ((int) (b2 & 0xFF))]; - b2 >>>= 8; - uCrc3 ^= M_U32[6 * 256 + ((int) (b3 & 0xFF))]; - b3 >>>= 8; - - uCrc0 ^= M_U32[5 * 256 + ((int) (b0 & 0xFF))]; - b0 >>>= 8; - uCrc1 ^= M_U32[5 * 256 + ((int) (b1 & 0xFF))]; - b1 >>>= 8; - uCrc2 ^= M_U32[5 * 256 + ((int) (b2 & 0xFF))]; - b2 >>>= 8; - uCrc3 ^= M_U32[5 * 256 + ((int) (b3 & 0xFF))]; - b3 >>>= 8; - - uCrc0 ^= M_U32[4 * 256 + ((int) (b0 & 0xFF))]; - b0 >>>= 8; - uCrc1 ^= M_U32[4 * 256 + ((int) (b1 & 0xFF))]; - b1 >>>= 8; - uCrc2 ^= M_U32[4 * 256 + ((int) (b2 & 0xFF))]; - b2 >>>= 8; - uCrc3 ^= M_U32[4 * 256 + ((int) (b3 & 0xFF))]; - b3 >>>= 8; - - uCrc0 ^= M_U32[3 * 256 + ((int) (b0 & 0xFF))]; - b0 >>>= 8; - uCrc1 ^= M_U32[3 * 256 + ((int) (b1 & 0xFF))]; - b1 >>>= 8; - uCrc2 ^= M_U32[3 * 256 + ((int) (b2 & 0xFF))]; - b2 >>>= 8; - uCrc3 ^= M_U32[3 * 256 + ((int) (b3 & 0xFF))]; - b3 >>>= 8; - - uCrc0 ^= M_U32[2 * 256 + ((int) (b0 & 0xFF))]; - b0 >>>= 8; - uCrc1 ^= M_U32[2 * 256 + ((int) (b1 & 0xFF))]; - b1 >>>= 8; - uCrc2 ^= M_U32[2 * 256 + ((int) (b2 & 0xFF))]; - b2 >>>= 8; - uCrc3 ^= M_U32[2 * 256 + ((int) (b3 & 0xFF))]; - b3 >>>= 8; - - uCrc0 ^= M_U32[256 + ((int) (b0 & 0xFF))]; - b0 >>>= 8; - uCrc1 ^= M_U32[256 + ((int) (b1 & 0xFF))]; - b1 >>>= 8; - uCrc2 ^= M_U32[256 + ((int) (b2 & 0xFF))]; - b2 >>>= 8; - uCrc3 ^= M_U32[256 + ((int) (b3 & 0xFF))]; - b3 >>>= 8; - - uCrc0 ^= M_U32[((int) (b0 & 0xFF))]; - uCrc1 ^= M_U32[((int) (b1 & 0xFF))]; - uCrc2 ^= M_U32[((int) (b2 & 0xFF))]; - uCrc3 ^= M_U32[((int) (b3 & 0xFF))]; - } - - // Combine CRC values - uCrc = 0; - uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - - uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc1; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - - uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc2; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - - uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc3; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - - pData += 32; - } - - // Process remaining bytes - for (uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[pData]) & 0xFF)]; - } - - return ~uCrc; // Flip all bits of uCrc and return as long + return compute(src, 0, src.length, uCrc); } /** * Computes the CRC64 checksum for a slice of a byte array. Avoids copying when combined with - * {@link #compute(ByteBuffer, long)} for array-backed buffers. + * {@link #compute(byte[], long)} for array-backed buffers. * * @param src the byte array. * @param offset the starting offset in the array. @@ -2715,232 +2565,6 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { return ~uCrc; } - /** - * Computes the CRC64 checksum for the remaining bytes in a ByteBuffer. When the buffer has a backing array, - * avoids copying; otherwise copies once. - * - * @param buffer the buffer (position to limit). - * @param uCrc the initial CRC value. - * @return the computed CRC64 checksum. - */ - public static long compute(ByteBuffer buffer, long uCrc) { - if (buffer == null || !buffer.hasRemaining()) { - return uCrc; - } - if (buffer.hasArray()) { - return compute(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining(), uCrc); - } - byte[] copy = new byte[buffer.remaining()]; - buffer.duplicate().get(copy); - return compute(copy, uCrc); - } - - /** - * Updates both segment and message CRC64 in a single pass over the data (half the memory reads of two separate - * {@link #compute(byte[], long)} calls). Used by the structured message decoder for performance. - * - * @param src the byte array. - * @param segmentCrc the current segment CRC value. - * @param messageCrc the current message CRC value. - * @return long[0] = new segment CRC, long[1] = new message CRC. - */ - public static long[] computeTwo(byte[] src, long segmentCrc, long messageCrc) { - segmentCrc = ~segmentCrc; - messageCrc = ~messageCrc; - - int pData = 0; - long uSize = src.length; - long uStop = uSize - (uSize % 32); - - if (uStop >= 2 * 32) { - long s0 = segmentCrc, s1 = 0, s2 = 0, s3 = 0; - long m0 = messageCrc, m1 = 0, m2 = 0, m3 = 0; - int pLast = (int) uStop - 32; - uSize -= uStop; - ByteBuffer buf = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); - - for (; pData < pLast; pData += 32) { - long b0 = buf.getLong(pData); - long b1 = buf.getLong(pData + 8); - long b2 = buf.getLong(pData + 16); - long b3 = buf.getLong(pData + 24); - - long bs0 = b0 ^ s0, bs1 = b1 ^ s1, bs2 = b2 ^ s2, bs3 = b3 ^ s3; - long bm0 = b0 ^ m0, bm1 = b1 ^ m1, bm2 = b2 ^ m2, bm3 = b3 ^ m3; - - s0 = M_U32[7 * 256 + ((int) (bs0 & 0xFF))]; - bs0 >>>= 8; - s1 = M_U32[7 * 256 + ((int) (bs1 & 0xFF))]; - bs1 >>>= 8; - s2 = M_U32[7 * 256 + ((int) (bs2 & 0xFF))]; - bs2 >>>= 8; - s3 = M_U32[7 * 256 + ((int) (bs3 & 0xFF))]; - bs3 >>>= 8; - s0 ^= M_U32[6 * 256 + ((int) (bs0 & 0xFF))]; - bs0 >>>= 8; - s1 ^= M_U32[6 * 256 + ((int) (bs1 & 0xFF))]; - bs1 >>>= 8; - s2 ^= M_U32[6 * 256 + ((int) (bs2 & 0xFF))]; - bs2 >>>= 8; - s3 ^= M_U32[6 * 256 + ((int) (bs3 & 0xFF))]; - bs3 >>>= 8; - s0 ^= M_U32[5 * 256 + ((int) (bs0 & 0xFF))]; - bs0 >>>= 8; - s1 ^= M_U32[5 * 256 + ((int) (bs1 & 0xFF))]; - bs1 >>>= 8; - s2 ^= M_U32[5 * 256 + ((int) (bs2 & 0xFF))]; - bs2 >>>= 8; - s3 ^= M_U32[5 * 256 + ((int) (bs3 & 0xFF))]; - bs3 >>>= 8; - s0 ^= M_U32[4 * 256 + ((int) (bs0 & 0xFF))]; - bs0 >>>= 8; - s1 ^= M_U32[4 * 256 + ((int) (bs1 & 0xFF))]; - bs1 >>>= 8; - s2 ^= M_U32[4 * 256 + ((int) (bs2 & 0xFF))]; - bs2 >>>= 8; - s3 ^= M_U32[4 * 256 + ((int) (bs3 & 0xFF))]; - bs3 >>>= 8; - s0 ^= M_U32[3 * 256 + ((int) (bs0 & 0xFF))]; - bs0 >>>= 8; - s1 ^= M_U32[3 * 256 + ((int) (bs1 & 0xFF))]; - bs1 >>>= 8; - s2 ^= M_U32[3 * 256 + ((int) (bs2 & 0xFF))]; - bs2 >>>= 8; - s3 ^= M_U32[3 * 256 + ((int) (bs3 & 0xFF))]; - bs3 >>>= 8; - s0 ^= M_U32[2 * 256 + ((int) (bs0 & 0xFF))]; - bs0 >>>= 8; - s1 ^= M_U32[2 * 256 + ((int) (bs1 & 0xFF))]; - bs1 >>>= 8; - s2 ^= M_U32[2 * 256 + ((int) (bs2 & 0xFF))]; - bs2 >>>= 8; - s3 ^= M_U32[2 * 256 + ((int) (bs3 & 0xFF))]; - bs3 >>>= 8; - s0 ^= M_U32[256 + ((int) (bs0 & 0xFF))]; - bs0 >>>= 8; - s1 ^= M_U32[256 + ((int) (bs1 & 0xFF))]; - bs1 >>>= 8; - s2 ^= M_U32[256 + ((int) (bs2 & 0xFF))]; - bs2 >>>= 8; - s3 ^= M_U32[256 + ((int) (bs3 & 0xFF))]; - bs3 >>>= 8; - s0 ^= M_U32[((int) (bs0 & 0xFF))]; - s1 ^= M_U32[((int) (bs1 & 0xFF))]; - s2 ^= M_U32[((int) (bs2 & 0xFF))]; - s3 ^= M_U32[((int) (bs3 & 0xFF))]; - - m0 = M_U32[7 * 256 + ((int) (bm0 & 0xFF))]; - bm0 >>>= 8; - m1 = M_U32[7 * 256 + ((int) (bm1 & 0xFF))]; - bm1 >>>= 8; - m2 = M_U32[7 * 256 + ((int) (bm2 & 0xFF))]; - bm2 >>>= 8; - m3 = M_U32[7 * 256 + ((int) (bm3 & 0xFF))]; - bm3 >>>= 8; - m0 ^= M_U32[6 * 256 + ((int) (bm0 & 0xFF))]; - bm0 >>>= 8; - m1 ^= M_U32[6 * 256 + ((int) (bm1 & 0xFF))]; - bm1 >>>= 8; - m2 ^= M_U32[6 * 256 + ((int) (bm2 & 0xFF))]; - bm2 >>>= 8; - m3 ^= M_U32[6 * 256 + ((int) (bm3 & 0xFF))]; - bm3 >>>= 8; - m0 ^= M_U32[5 * 256 + ((int) (bm0 & 0xFF))]; - bm0 >>>= 8; - m1 ^= M_U32[5 * 256 + ((int) (bm1 & 0xFF))]; - bm1 >>>= 8; - m2 ^= M_U32[5 * 256 + ((int) (bm2 & 0xFF))]; - bm2 >>>= 8; - m3 ^= M_U32[5 * 256 + ((int) (bm3 & 0xFF))]; - bm3 >>>= 8; - m0 ^= M_U32[4 * 256 + ((int) (bm0 & 0xFF))]; - bm0 >>>= 8; - m1 ^= M_U32[4 * 256 + ((int) (bm1 & 0xFF))]; - bm1 >>>= 8; - m2 ^= M_U32[4 * 256 + ((int) (bm2 & 0xFF))]; - bm2 >>>= 8; - m3 ^= M_U32[4 * 256 + ((int) (bm3 & 0xFF))]; - bm3 >>>= 8; - m0 ^= M_U32[3 * 256 + ((int) (bm0 & 0xFF))]; - bm0 >>>= 8; - m1 ^= M_U32[3 * 256 + ((int) (bm1 & 0xFF))]; - bm1 >>>= 8; - m2 ^= M_U32[3 * 256 + ((int) (bm2 & 0xFF))]; - bm2 >>>= 8; - m3 ^= M_U32[3 * 256 + ((int) (bm3 & 0xFF))]; - bm3 >>>= 8; - m0 ^= M_U32[2 * 256 + ((int) (bm0 & 0xFF))]; - bm0 >>>= 8; - m1 ^= M_U32[2 * 256 + ((int) (bm1 & 0xFF))]; - bm1 >>>= 8; - m2 ^= M_U32[2 * 256 + ((int) (bm2 & 0xFF))]; - bm2 >>>= 8; - m3 ^= M_U32[2 * 256 + ((int) (bm3 & 0xFF))]; - bm3 >>>= 8; - m0 ^= M_U32[256 + ((int) (bm0 & 0xFF))]; - bm0 >>>= 8; - m1 ^= M_U32[256 + ((int) (bm1 & 0xFF))]; - bm1 >>>= 8; - m2 ^= M_U32[256 + ((int) (bm2 & 0xFF))]; - bm2 >>>= 8; - m3 ^= M_U32[256 + ((int) (bm3 & 0xFF))]; - bm3 >>>= 8; - m0 ^= M_U32[((int) (bm0 & 0xFF))]; - m1 ^= M_U32[((int) (bm1 & 0xFF))]; - m2 ^= M_U32[((int) (bm2 & 0xFF))]; - m3 ^= M_U32[((int) (bm3 & 0xFF))]; - } - - long uCrc = 0; - uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s0; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s1; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s2; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ s3; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - segmentCrc = uCrc; // keep internal form for tail - - uCrc = 0; - uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m0; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m1; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m2; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ m3; - for (int i = 0; i < 8; i++) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - } - messageCrc = uCrc; // keep internal form for tail - - pData += 32; - } - - for (long uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { - long byteVal = src[pData] & 0xFF; - segmentCrc = (segmentCrc >>> 8) ^ M_U1[(int) ((segmentCrc ^ byteVal) & 0xFF)]; - messageCrc = (messageCrc >>> 8) ^ M_U1[(int) ((messageCrc ^ byteVal) & 0xFF)]; - } - - return new long[] { ~segmentCrc, ~messageCrc }; - } - /** * Concatenates two CRC64 values by combining their initial and final CRC values and sizes. * This method ensures unsigned behavior and uses the `mulX_N` method to perform necessary diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 896cc88f2432..7bb669b58664 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -65,6 +65,8 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN return next.process(); } + context.getHttpRequest().getHeaders().set(X_MS_STRUCTURED_BODY, Constants.STRUCTURED_MESSAGE_CRC64_BODY_TYPE); + return next.process().map(httpResponse -> { Long contentLength = ContentValidationDecoderUtils.getContentLength(httpResponse.getHeaders()); diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java index 5e76a7e7bf8d..a07cef1f2013 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java @@ -11,7 +11,6 @@ import com.azure.storage.common.implementation.Constants; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; @@ -213,27 +212,6 @@ private static Stream testConcatWithInitialsSupplier() { "1407121768110541356", "10535852615249992663", "741189", "3634018251978804152")); } - @Test - void testComputeTwoMatchesTwoComputes() { - byte[] data = "This is a test where the data is longer than 64 characters so that we can test that code path." - .getBytes(); - long seg0 = 0; - long msg0 = 0; - long seg1 = StorageCrc64Calculator.compute(data, seg0); - long msg1 = StorageCrc64Calculator.compute(data, msg0); - long[] two = StorageCrc64Calculator.computeTwo(data, seg0, msg0); - assertEquals(seg1, two[0]); - assertEquals(msg1, two[1]); - } - - @Test - void testComputeByteBufferMatchesByteArray() { - byte[] data = "Hello World!".getBytes(); - long expected = StorageCrc64Calculator.compute(data, 0); - long actual = StorageCrc64Calculator.compute(ByteBuffer.wrap(data), 0); - assertEquals(expected, actual); - } - @Test void testComputeSliceMatchesFullArray() { byte[] data = "Hello World!".getBytes(); From 2b2e86db622ad0e3d4485e2237fb5bbd52f599f8 Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Fri, 3 Apr 2026 19:32:26 +0530 Subject: [PATCH 13/31] code refactoring based on latest review comments --- .../implementation/util/BlobConstants.java | 20 - .../implementation/util/BuilderHelper.java | 4 +- .../util/ChunkedDownloadUtils.java | 2 +- .../util/DownloadValidationUtils.java | 61 ++ .../blob/implementation/util/ModelHelper.java | 13 +- .../blob/models/ParallelTransferOptions.java | 24 - .../blob/specialized/BlobAsyncClientBase.java | 585 ++---------------- .../blob/specialized/BlobClientBase.java | 35 +- .../BlobMessageAsyncDecoderDownloadTests.java | 3 +- .../blob/BlobMessageDecoderDownloadTests.java | 3 +- .../common/ParallelTransferOptions.java | 24 - .../common/implementation/Constants.java | 15 - ...StorageContentValidationDecoderPolicy.java | 5 +- .../test/shared/StorageCommonTestUtils.java | 7 - .../StructuredMessageDecoderTests.java | 34 +- 15 files changed, 132 insertions(+), 703 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java index 2757cb131590..1c78a6c469c8 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobConstants.java @@ -31,26 +31,6 @@ public final class BlobConstants { * The number of buffers to use if none is specified on the buffered upload method. */ public static final int BLOB_DEFAULT_NUMBER_OF_BUFFERS = 8; - /** - * The legacy default number of concurrent transfers for download operations. - */ - public static final int BLOB_LEGACY_DEFAULT_CONCURRENT_TRANSFERS_COUNT = 5; - /** - * The default range size used for download operations when no checksum validation is requested. - */ - public static final long BLOB_DEFAULT_DOWNLOAD_RANGE_SIZE = 4L * Constants.MB; - /** - * The default initial range size used for download operations when no checksum validation is requested. - */ - public static final long BLOB_DEFAULT_INITIAL_DOWNLOAD_RANGE_SIZE = 256L * Constants.MB; - /** - * The maximum range size used for download operations. - */ - public static final long BLOB_MAX_DOWNLOAD_BYTES = 256L * Constants.MB; - /** - * The maximum range size used when requesting a transactional checksum during download. - */ - public static final long BLOB_MAX_HASH_REQUEST_DOWNLOAD_RANGE = 4L * Constants.MB; /** * If a blob is known to be greater than 100MB, using a larger block size will trigger some server-side * optimizations. If the block size is not set and the size of the blob is known to be greater than 100MB, this diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index 5722aff4503d..ef631882249a 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -116,6 +116,8 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare } policies.add(new MetadataValidationPolicy()); + policies.add(new StorageContentValidationDecoderPolicy()); + if (storageSharedKeyCredential != null) { policies.add(new StorageSharedKeyCredentialPolicy(storageSharedKeyCredential)); } @@ -134,8 +136,6 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare policies.add(new AzureSasCredentialPolicy(new AzureSasCredential(sasToken), false)); } - policies.add(new StorageContentValidationDecoderPolicy()); - policies.addAll(perRetryPolicies); HttpPolicyProviders.addAfterRetryPolicies(policies); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java index 294bb378b8ea..a5845408a36b 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java @@ -122,7 +122,7 @@ && extractTotalBlobLength( BiFunction> fallbackDownloader = emptyBlobDownloader != null ? emptyBlobDownloader : downloader; - return fallbackDownloader.apply(new BlobRange(0), requestConditions) + return fallbackDownloader.apply(new BlobRange(0, 0L), requestConditions) // Subscribe on boundElastic instead of elastic as elastic is deprecated and boundElastic // provided the same functionality with the added benefit that it won't infinitely create // threads if needed and will instead queue. diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java new file mode 100644 index 000000000000..3c9491c5e81b --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.util; + +import com.azure.core.util.Context; +import com.azure.storage.common.StorageChecksumAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.policy.AggregateCrcState; + +import java.util.concurrent.atomic.AtomicReference; + +import com.azure.storage.common.policy.DecoderState; + +/** + * Centralizes download content validation decisions based on {@link StorageChecksumAlgorithm}. + *

+ * Mirrors the pattern established by {@code ContentValidationModeResolver} for uploads. + *

+ * RESERVED FOR INTERNAL USE. + */ +public final class DownloadValidationUtils { + + private DownloadValidationUtils() { + } + + /** + * Whether the algorithm requires structured message decoding (CRC64 / AUTO). + */ + public static boolean isStructuredMessageAlgorithm(StorageChecksumAlgorithm algorithm) { + return algorithm == StorageChecksumAlgorithm.CRC64 || algorithm == StorageChecksumAlgorithm.AUTO; + } + + /** + * Resolves the effective algorithm, defaulting null to NONE. + */ + public static StorageChecksumAlgorithm resolveAlgorithm(StorageChecksumAlgorithm algorithm) { + return algorithm != null ? algorithm : StorageChecksumAlgorithm.NONE; + } + + /** + * Adds structured message decoding context keys when CRC64/AUTO validation is active. + * + * @param context The base context to augment. Null is treated as {@link Context#NONE}. + * @param algorithm The resolved checksum algorithm. + * @param decoderStateRef Holder for decoder state, populated by the policy. + * @param aggregateCrcState CRC aggregation state across retries. + * @return The augmented context. + */ + public static Context applyStructuredMessageContext(Context context, StorageChecksumAlgorithm algorithm, + AtomicReference decoderStateRef, AggregateCrcState aggregateCrcState) { + Context base = context == null ? Context.NONE : context; + if (!isStructuredMessageAlgorithm(algorithm)) { + return base; + } + return base.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) + .addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) + .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY, decoderStateRef) + .addData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY, aggregateCrcState); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java index 61b1739e118f..cb859dfa595d 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java @@ -111,8 +111,6 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO blockSize = (long) BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE; } - Long initialTransferSize = other.getInitialTransferSizeLong(); - Integer maxConcurrency = other.getMaxConcurrency(); if (maxConcurrency == null) { maxConcurrency = BlobAsyncClient.BLOB_DEFAULT_NUMBER_OF_BUFFERS; @@ -126,8 +124,7 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO return new ParallelTransferOptions().setBlockSizeLong(blockSize) .setMaxConcurrency(maxConcurrency) .setProgressListener(other.getProgressListener()) - .setMaxSingleUploadSizeLong(maxSingleUploadSize) - .setInitialTransferSizeLong(initialTransferSize); + .setMaxSingleUploadSizeLong(maxSingleUploadSize); } /** @@ -146,8 +143,6 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO blockSize = (long) BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE; } - Long initialTransferSize = other.getInitialTransferSizeLong(); - Integer maxConcurrency = other.getMaxConcurrency(); if (maxConcurrency == null) { maxConcurrency = BlobAsyncClient.BLOB_DEFAULT_NUMBER_OF_BUFFERS; @@ -161,8 +156,7 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO return new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(blockSize) .setMaxConcurrency(maxConcurrency) .setProgressListener(other.getProgressListener()) - .setMaxSingleUploadSizeLong(maxSingleUploadSize) - .setInitialTransferSizeLong(initialTransferSize); + .setMaxSingleUploadSizeLong(maxSingleUploadSize); } /** @@ -175,8 +169,7 @@ public static ParallelTransferOptions populateAndApplyDefaults(ParallelTransferO return new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(blobOptions.getBlockSizeLong()) .setMaxConcurrency(blobOptions.getMaxConcurrency()) .setProgressListener(blobOptions.getProgressListener()) - .setMaxSingleUploadSizeLong(blobOptions.getMaxSingleUploadSizeLong()) - .setInitialTransferSizeLong(blobOptions.getInitialTransferSizeLong()); + .setMaxSingleUploadSizeLong(blobOptions.getMaxSingleUploadSizeLong()); } /** diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java index 05e0e19aed6f..77f7841c945d 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ParallelTransferOptions.java @@ -17,7 +17,6 @@ public final class ParallelTransferOptions { private Long blockSize; - private Long initialTransferSize; private Integer maxConcurrency; private ProgressReceiver progressReceiver; private Long maxSingleUploadSize; @@ -102,29 +101,6 @@ public Long getBlockSizeLong() { return this.blockSize; } - /** - * Gets the size of the first range requested when downloading. - * @return The initial transfer size. - */ - public Long getInitialTransferSizeLong() { - return this.initialTransferSize; - } - - /** - * Sets the size of the first range requested when downloading. - * This value may be larger than the block size used for subsequent ranges. - * - * @param initialTransferSize The initial transfer size. - * @return The ParallelTransferOptions object itself. - */ - public ParallelTransferOptions setInitialTransferSizeLong(Long initialTransferSize) { - if (initialTransferSize != null) { - StorageImplUtils.assertInBounds("initialTransferSize", initialTransferSize, 1, Long.MAX_VALUE); - } - this.initialTransferSize = initialTransferSize; - return this; - } - /** * Sets the block size (chunk size) to transfer at a time. * For upload, The block size is the size of each block that will be staged. This value also determines the number diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index ad32aceff5eb..828ececb983a 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -12,7 +12,6 @@ import com.azure.core.http.rest.SimpleResponse; import com.azure.core.http.rest.StreamResponse; import com.azure.core.util.BinaryData; -import com.azure.core.util.Configuration; import com.azure.core.util.Context; import com.azure.core.util.CoreUtils; import com.azure.core.util.FluxUtil; @@ -42,11 +41,11 @@ import com.azure.storage.blob.implementation.models.InternalBlobLegalHoldResult; import com.azure.storage.blob.implementation.models.QueryRequest; import com.azure.storage.blob.implementation.models.QuerySerialization; -import com.azure.storage.blob.implementation.util.BlobConstants; import com.azure.storage.blob.implementation.util.BlobQueryReader; import com.azure.storage.blob.implementation.util.BlobRequestConditionProperty; import com.azure.storage.blob.implementation.util.BlobSasImplUtil; import com.azure.storage.blob.implementation.util.ChunkedDownloadUtils; +import com.azure.storage.blob.implementation.util.DownloadValidationUtils; import com.azure.storage.blob.implementation.util.ModelHelper; import com.azure.storage.blob.models.AccessTier; import com.azure.storage.blob.models.BlobBeginCopySourceRequestConditions; @@ -87,7 +86,6 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.SasImplUtils; -import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.StorageChecksumAlgorithm; import com.azure.storage.common.policy.AggregateCrcState; @@ -96,7 +94,6 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; -import reactor.core.scheduler.Schedulers; import java.io.IOException; import java.io.UncheckedIOException; @@ -104,10 +101,7 @@ import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.AsynchronousByteChannel; import java.nio.channels.AsynchronousFileChannel; -import java.nio.channels.CompletionHandler; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.OpenOption; @@ -115,26 +109,20 @@ import java.nio.file.StandardOpenOption; import java.time.Duration; import java.time.OffsetDateTime; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.Consumer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import static com.azure.core.util.FluxUtil.fluxError; import static com.azure.core.util.FluxUtil.monoError; @@ -1279,12 +1267,10 @@ Mono downloadStreamWithResponseInternal(BlobRange ran StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; - final StorageChecksumAlgorithm algorithm - = responseChecksumAlgorithm != null ? responseChecksumAlgorithm : StorageChecksumAlgorithm.NONE; - final boolean isStructuredMessageEnabled = isStructuredMessageAlgorithm(algorithm); - final boolean isMd5Enabled = algorithm == StorageChecksumAlgorithm.MD5; + final StorageChecksumAlgorithm algorithm = DownloadValidationUtils.resolveAlgorithm(responseChecksumAlgorithm); + final boolean isStructuredMessageEnabled = DownloadValidationUtils.isStructuredMessageAlgorithm(algorithm); - final Boolean finalGetMD5 = (!isStructuredMessageEnabled && (getRangeContentMd5 || isMd5Enabled)) ? true : null; + final Boolean finalGetMD5 = (!isStructuredMessageEnabled && getRangeContentMd5) ? true : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; @@ -1294,22 +1280,12 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - final Context responseScopedContext = isStructuredMessageEnabled - ? baseContext.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) - : baseContext; - AtomicReference decoderStateRef = new AtomicReference<>(); AggregateCrcState aggregateCrcState = isStructuredMessageEnabled ? new AggregateCrcState() : null; AtomicLong responseStartOffset = new AtomicLong(0); - final Context firstRangeContext; - if (isStructuredMessageEnabled) { - firstRangeContext = responseScopedContext.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) - .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY, decoderStateRef) - .addData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY, aggregateCrcState); - } else { - firstRangeContext = responseScopedContext; - } + final Context firstRangeContext = DownloadValidationUtils.applyStructuredMessageContext(baseContext, algorithm, + decoderStateRef, aggregateCrcState); return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), finalGetMD5, firstRangeContext).map(response -> { @@ -1455,10 +1431,6 @@ private static long calculateRetryBytesToSkip(long emittedOffset, long retryStar return skip; } - private static boolean isStructuredMessageAlgorithm(StorageChecksumAlgorithm algorithm) { - return algorithm == StorageChecksumAlgorithm.CRC64 || algorithm == StorageChecksumAlgorithm.AUTO; - } - /** * Downloads the entire blob into a file specified by the path. * @@ -1657,14 +1629,8 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti StorageImplUtils.assertNotNull("options", options); BlobRange finalRange = options.getRange() == null ? new BlobRange(0) : options.getRange(); - com.azure.storage.common.ParallelTransferOptions originalParallelTransferOptions - = options.getParallelTransferOptions(); - boolean blockSizeProvided - = originalParallelTransferOptions != null && originalParallelTransferOptions.getBlockSizeLong() != null; - boolean maxConcurrencyProvided - = originalParallelTransferOptions != null && originalParallelTransferOptions.getMaxConcurrency() != null; com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions - = ModelHelper.populateAndApplyDefaults(originalParallelTransferOptions); + = ModelHelper.populateAndApplyDefaults(options.getParallelTransferOptions()); BlobRequestConditions finalConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); @@ -1674,13 +1640,10 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti } AsynchronousFileChannel channel = downloadToFileResourceSupplier(options.getFilePath(), openOptions); - // Run download on boundedElastic to avoid blocking the reactor thread when async file I/O - // completion is delivered (matches .NET behavior where DownloadTo runs off the sync context). return Mono.just(channel) - .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, blockSizeProvided, - maxConcurrencyProvided, options.getDownloadRetryOptions(), finalConditions, - options.isRetrieveContentRangeMd5(), options.getResponseChecksumAlgorithm(), context)) - .subscribeOn(Schedulers.boundedElastic()) + .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, + options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), + options.getResponseChecksumAlgorithm(), context)) .doFinally(signalType -> this.downloadToFileCleanup(channel, options.getFilePath(), signalType)); } @@ -1693,530 +1656,52 @@ private AsynchronousFileChannel downloadToFileResourceSupplier(String filePath, } private Mono> downloadToFileImpl(AsynchronousFileChannel file, BlobRange finalRange, - com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, boolean blockSizeProvided, - boolean maxConcurrencyProvided, DownloadRetryOptions downloadRetryOptions, - BlobRequestConditions requestConditions, boolean rangeGetContentMd5, + com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, + DownloadRetryOptions downloadRetryOptions, BlobRequestConditions requestConditions, boolean rangeGetContentMd5, StorageChecksumAlgorithm responseChecksumAlgorithm, Context context) { // See ProgressReporter for an explanation on why this lock is necessary and why we use AtomicLong. ProgressListener progressReceiver = finalParallelTransferOptions.getProgressListener(); ProgressReporter progressReporter = progressReceiver == null ? null : ProgressReporter.withProgressListener(progressReceiver); - final boolean isStructuredMessageEnabled = isStructuredMessageAlgorithm(responseChecksumAlgorithm); - final boolean isMd5Enabled = responseChecksumAlgorithm == StorageChecksumAlgorithm.MD5; - - final Context downloadContext = isStructuredMessageEnabled - ? (context == null - ? new Context(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) - : context.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true)) - : context; /* * Downloads the first chunk and gets the size of the data and etag if not specified by the user. */ BiFunction> downloadFunc - = (range, conditions) -> isStructuredMessageEnabled - ? this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, rangeGetContentMd5, - responseChecksumAlgorithm, downloadContext) - : this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, rangeGetContentMd5, - downloadContext); - BiFunction> emptyBlobDownloadFunc = (range, - conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, false, context); - - boolean checksumValidationEnabled = isStructuredMessageEnabled || rangeGetContentMd5 || isMd5Enabled; - boolean md5ValidationEnabled = !isStructuredMessageEnabled && (rangeGetContentMd5 || isMd5Enabled); - - long rangeSize = blockSizeProvided - ? Math.min(finalParallelTransferOptions.getBlockSizeLong(), BlobConstants.BLOB_MAX_DOWNLOAD_BYTES) - : (checksumValidationEnabled - ? BlobConstants.BLOB_MAX_HASH_REQUEST_DOWNLOAD_RANGE - : BlobConstants.BLOB_DEFAULT_DOWNLOAD_RANGE_SIZE); - - Long requestedInitialTransferSize = finalParallelTransferOptions.getInitialTransferSizeLong(); - long initialRangeSize = requestedInitialTransferSize != null && requestedInitialTransferSize > 0 - ? requestedInitialTransferSize - : (checksumValidationEnabled - ? BlobConstants.BLOB_MAX_HASH_REQUEST_DOWNLOAD_RANGE - : BlobConstants.BLOB_DEFAULT_INITIAL_DOWNLOAD_RANGE_SIZE); - - int maxConcurrency = maxConcurrencyProvided - ? finalParallelTransferOptions.getMaxConcurrency() - : getDefaultDownloadConcurrency(); - - com.azure.storage.common.ParallelTransferOptions initialParallelTransferOptions - = new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong(initialRangeSize); - - boolean useMasterCrc = isStructuredMessageEnabled; - - LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("BlobAsyncClientBase.downloadToFileImpl calling downloadFirstChunk"); + = (range, conditions) -> this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, + rangeGetContentMd5, responseChecksumAlgorithm, context); + return ChunkedDownloadUtils - .downloadFirstChunk(finalRange, initialParallelTransferOptions, requestConditions, downloadFunc, - emptyBlobDownloadFunc, true, downloadContext) - .doOnSuccess(t -> LOGGER.atVerbose() - .addKeyValue("newCount", t != null ? t.getT1() : null) - .addKeyValue("thread", Thread.currentThread().getName()) - .log("BlobAsyncClientBase downloadFirstChunk returned")) + .downloadFirstChunk(finalRange, finalParallelTransferOptions, requestConditions, downloadFunc, true, + context) .flatMap(setupTuple3 -> { long newCount = setupTuple3.getT1(); BlobRequestConditions finalConditions = setupTuple3.getT2(); - BlobDownloadAsyncResponse initialResponse = setupTuple3.getT3(); - LOGGER.atVerbose() - .addKeyValue("newCount", newCount) - .addKeyValue("thread", Thread.currentThread().getName()) - .log("BlobAsyncClientBase flatMap after first chunk"); - - if (initialResponse.getStatusCode() == 304 || newCount == 0) { - return Mono.fromCallable(() -> { - Response propertiesResponse - = ModelHelper.buildBlobPropertiesResponse(initialResponse); - try { - initialResponse.close(); - } catch (IOException e) { - throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); - } - return propertiesResponse; - }); - } - - AsynchronousByteChannel baseChannel = IOUtils.toAsynchronousByteChannel(file, 0); - Crc64TrackingAsynchronousByteChannel crcChannel - = useMasterCrc ? new Crc64TrackingAsynchronousByteChannel(baseChannel) : null; - AsynchronousByteChannel targetChannel = useMasterCrc ? crcChannel : baseChannel; - ComposedCrcState composedCrcState = useMasterCrc ? new ComposedCrcState() : null; - - long initialLength = Math.min(initialRangeSize, newCount); - if (initialLength == newCount) { - LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("BlobAsyncClientBase taking ONE-SHOT path (single chunk)"); - return writeBodyToFile(initialResponse, targetChannel, - progressReporter == null ? null : progressReporter.createChild(), useMasterCrc, - composedCrcState, md5ValidationEnabled).doFinally(signalType -> closeQuietly(initialResponse)) - .then(Mono.fromCallable(() -> { - if (useMasterCrc) { - validateComposedCrc(composedCrcState, crcChannel); - } - return ModelHelper.buildBlobPropertiesResponse(initialResponse); - })); - } - - long remainingLength = Math.max(0L, newCount - initialLength); - int remainingChunks = ChunkedDownloadUtils.calculateNumBlocks(remainingLength, rangeSize); - int numChunks = Math.max(1, 1 + remainingChunks); - LOGGER.atVerbose() - .addKeyValue("numChunks", numChunks) - .addKeyValue("thread", Thread.currentThread().getName()) - .log("BlobAsyncClientBase taking PARALLEL path"); - - List remainingRanges = new ArrayList<>(Math.max(0, numChunks - 1)); - for (int chunkIndex = 1; chunkIndex < numChunks; chunkIndex++) { - long offset = initialLength + (long) (chunkIndex - 1) * rangeSize; - long chunkSizeActual = Math.min(rangeSize, newCount - offset); - if (chunkSizeActual <= 0) { - break; - } - remainingRanges.add(new BlobRange(finalRange.getOffset() + offset, chunkSizeActual)); - } - - int effectiveConcurrency = Math.max(1, maxConcurrency); - ArrayDeque> running = new ArrayDeque<>(); - Iterator remainingIterator = remainingRanges.iterator(); - - running.add(CompletableFuture.completedFuture(initialResponse)); - while (running.size() < effectiveConcurrency && remainingIterator.hasNext()) { - BlobRange nextRange = remainingIterator.next(); - running.add(downloadFunc.apply(nextRange, finalConditions).toFuture()); - } - - return drainQueuedResponses(running, remainingIterator, downloadFunc, finalConditions, targetChannel, - progressReporter, useMasterCrc, composedCrcState, md5ValidationEnabled) - .doFinally(signalType -> closePendingResponses(running)) - .then(Mono.fromCallable(() -> { - if (useMasterCrc) { - validateComposedCrc(composedCrcState, crcChannel); - } - return ModelHelper.buildBlobPropertiesResponse(initialResponse); - })); - }); - } - - private static Mono drainQueuedResponses(ArrayDeque> running, - Iterator remainingIterator, - BiFunction> downloadFunc, - BlobRequestConditions finalConditions, AsynchronousByteChannel targetChannel, ProgressReporter progressReporter, - boolean useMasterCrc, ComposedCrcState composedCrcState, boolean md5ValidationEnabled) { - return Mono.defer(() -> { - CompletableFuture nextFuture = running.poll(); - if (nextFuture == null) { - return Mono.empty(); - } - - if (remainingIterator.hasNext()) { - BlobRange nextRange = remainingIterator.next(); - running.add(downloadFunc.apply(nextRange, finalConditions).toFuture()); - } - - return Mono.fromFuture(nextFuture) - .flatMap(response -> writeBodyToFile(response, targetChannel, - progressReporter == null ? null : progressReporter.createChild(), useMasterCrc, composedCrcState, - md5ValidationEnabled).doFinally(signalType -> closeQuietly(response))) - .then(Mono.defer(() -> drainQueuedResponses(running, remainingIterator, downloadFunc, finalConditions, - targetChannel, progressReporter, useMasterCrc, composedCrcState, md5ValidationEnabled))); - }); - } - - private static void closePendingResponses(ArrayDeque> running) { - CompletableFuture future; - while ((future = running.poll()) != null) { - future.whenComplete((response, throwable) -> { - if (response != null) { - closeQuietly(response); - } - }); - if (!future.isDone()) { - future.cancel(true); - } - } - } - - private static Mono writeBodyToFile(BlobDownloadAsyncResponse response, AsynchronousByteChannel channel, - ProgressReporter progressReporter, boolean useMasterCrc, ComposedCrcState composedCrcState, - boolean validateMd5) { - LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .addKeyValue("useMasterCrc", useMasterCrc) - .log("BlobAsyncClientBase.writeBodyToFile entry"); - Md5TrackingAsynchronousByteChannel md5Channel = null; - AsynchronousByteChannel targetChannel = channel; - - if (validateMd5) { - md5Channel = new Md5TrackingAsynchronousByteChannel(targetChannel); - targetChannel = md5Channel; - } - - Mono write = response.writeValueToAsync(targetChannel, progressReporter) - .doOnSuccess(v -> LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("BlobAsyncClientBase.writeBodyToFile writeValueToAsync completed")) - .doOnError(e -> LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .addKeyValue("error", e) - .log("BlobAsyncClientBase.writeBodyToFile writeValueToAsync error")); - if (!useMasterCrc && !validateMd5) { - return write; - } - - Md5TrackingAsynchronousByteChannel finalMd5Channel = md5Channel; - return write.then(Mono.fromRunnable(() -> { - if (validateMd5) { - validateResponseMd5(response, finalMd5Channel); - } - if (useMasterCrc) { - DecoderState decoderState = getStructuredDecoderState(response); - if (decoderState == null || !decoderState.isFinalized()) { - throw LOGGER.logExceptionAsError(new IllegalStateException( - "Structured message decoder state wasn't available or finalized for checksum validation.")); - } - - long composedLength = decoderState.getComposedLength(); - if (composedLength > 0) { - long partitionCrc = decoderState.getComposedCrc64(); - composedCrcState.append(partitionCrc, composedLength); - - BlobDownloadHeaders headers = response.getDeserializedHeaders(); - if (headers != null) { - headers.setContentCrc64(littleEndianLongToBytes(partitionCrc)); - } - } - } - })); - } - - private static DecoderState getStructuredDecoderState(BlobDownloadAsyncResponse response) { - return BlobDownloadAsyncResponseConstructorProxy.getDecoderState(response); - } - - private static byte[] littleEndianLongToBytes(long value) { - return ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(value).array(); - } - - private static int getDefaultDownloadConcurrency() { - Configuration configuration = Configuration.getGlobalConfiguration(); - String legacyDefaultConcurrency = configuration.get(Constants.USE_LEGACY_DEFAULT_CONCURRENCY_PROPERTY); - if (legacyDefaultConcurrency == null) { - legacyDefaultConcurrency = configuration.get(Constants.USE_LEGACY_DEFAULT_CONCURRENCY_ENV_VAR); - } - if (Boolean.parseBoolean(legacyDefaultConcurrency)) { - return BlobConstants.BLOB_LEGACY_DEFAULT_CONCURRENT_TRANSFERS_COUNT; - } - - int processors = Runtime.getRuntime().availableProcessors(); - int concurrency = Math.max(processors * 2, 8); - return Math.min(concurrency, 32); - } - - private static void validateResponseMd5(BlobDownloadAsyncResponse response, - Md5TrackingAsynchronousByteChannel md5Channel) { - if (md5Channel == null) { - return; - } - - byte[] expectedMd5 - = response.getDeserializedHeaders() == null ? null : response.getDeserializedHeaders().getContentMd5(); - if (expectedMd5 == null || expectedMd5.length == 0) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException("Content-MD5 header missing from download response.")); - } - - byte[] actualMd5 = md5Channel.getMd5(); - if (!Arrays.equals(expectedMd5, actualMd5)) { - throw LOGGER - .logExceptionAsError(new IllegalArgumentException("MD5 mismatch detected in download response.")); - } - } - - private static void validateComposedCrc(ComposedCrcState composedCrcState, - Crc64TrackingAsynchronousByteChannel crcChannel) { - if (composedCrcState == null || crcChannel == null || composedCrcState.getLength() == 0) { - return; - } - - long composed = composedCrcState.getCrc(); - long master = crcChannel.getCrc(); - if (composed != master) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException( - "CRC64 mismatch detected in composed download. Expected: " + composed + ", got: " + master)); - } - } - - private static void closeQuietly(BlobDownloadAsyncResponse response) { - if (response == null) { - return; - } - - try { - response.close(); - } catch (IOException e) { - LOGGER.warning("Failed to close BlobDownloadAsyncResponse: {}", e.getMessage()); - } - } - - private static final class ComposedCrcState { - private long crc; - private long length; - private void append(long nextCrc, long nextLength) { - if (nextLength <= 0) { - return; - } - - if (length == 0) { - crc = nextCrc; - length = nextLength; - return; - } - - crc = StorageCrc64Calculator.concat(0, 0, crc, length, 0, nextCrc, nextLength); - length += nextLength; - } - - private long getCrc() { - return crc; - } - - private long getLength() { - return length; - } - } - - private static final class Crc64TrackingAsynchronousByteChannel implements AsynchronousByteChannel { - private final AsynchronousByteChannel channel; - private final AtomicLong crc = new AtomicLong(0); - private final AtomicLong bytesWritten = new AtomicLong(0); - - private Crc64TrackingAsynchronousByteChannel(AsynchronousByteChannel channel) { - this.channel = channel; - } - - private long getCrc() { - return crc.get(); - } - - private long getBytesWritten() { - return bytesWritten.get(); - } - - @Override - public void read(ByteBuffer dst, A attachment, CompletionHandler handler) { - channel.read(dst, attachment, handler); - } + int numChunks = ChunkedDownloadUtils.calculateNumBlocks(newCount, + finalParallelTransferOptions.getBlockSizeLong()); - @Override - public Future read(ByteBuffer dst) { - return channel.read(dst); - } - - @Override - public void write(ByteBuffer src, A attachment, CompletionHandler handler) { - int startPos = src.position(); - ByteBuffer duplicate = src.duplicate(); - channel.write(src, attachment, new CompletionHandler() { - @Override - public void completed(Integer result, A att) { - if (result != null && result > 0) { - updateCrc(duplicate, startPos, result); - } - handler.completed(result, att); - } - - @Override - public void failed(Throwable exc, A att) { - handler.failed(exc, att); - } - }); - } + numChunks = numChunks == 0 ? 1 : numChunks; - @Override - public Future write(ByteBuffer src) { - CompletableFuture future = new CompletableFuture<>(); - int startPos = src.position(); - ByteBuffer duplicate = src.duplicate(); - channel.write(src, src, new CompletionHandler() { - @Override - public void completed(Integer result, ByteBuffer attachment) { - if (result != null && result > 0) { - updateCrc(duplicate, startPos, result); - } - future.complete(result); - } - - @Override - public void failed(Throwable exc, ByteBuffer attachment) { - future.completeExceptionally(exc); - } + BlobDownloadAsyncResponse initialResponse = setupTuple3.getT3(); + return Flux.range(0, numChunks) + .flatMap( + chunkNum -> ChunkedDownloadUtils.downloadChunk(chunkNum, initialResponse, finalRange, + finalParallelTransferOptions, finalConditions, newCount, downloadFunc, + response -> writeBodyToFile(response, file, chunkNum, finalParallelTransferOptions, + progressReporter == null ? null : progressReporter.createChild()).flux()), + finalParallelTransferOptions.getMaxConcurrency()) + + .then(Mono.just(ModelHelper.buildBlobPropertiesResponse(initialResponse))); }); - return future; - } - - @Override - public boolean isOpen() { - return channel.isOpen(); - } - - @Override - public void close() throws IOException { - channel.close(); - } - - private void updateCrc(ByteBuffer buffer, int startPos, int length) { - if (length <= 0) { - return; - } - - ByteBuffer slice = buffer.duplicate(); - slice.position(startPos); - slice.limit(startPos + length); - byte[] bytes = new byte[length]; - slice.get(bytes); - crc.updateAndGet(previous -> StorageCrc64Calculator.compute(bytes, previous)); - bytesWritten.addAndGet(length); - } } - private static final class Md5TrackingAsynchronousByteChannel implements AsynchronousByteChannel { - private final AsynchronousByteChannel channel; - private final MessageDigest digest; - - private Md5TrackingAsynchronousByteChannel(AsynchronousByteChannel channel) { - this.channel = channel; - try { - this.digest = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw LOGGER.logExceptionAsError(new IllegalStateException("MD5 MessageDigest unavailable.", e)); - } - } - - private byte[] getMd5() { - synchronized (digest) { - return digest.digest(); - } - } - - @Override - public void read(ByteBuffer dst, A attachment, CompletionHandler handler) { - channel.read(dst, attachment, handler); - } - - @Override - public Future read(ByteBuffer dst) { - return channel.read(dst); - } - - @Override - public void write(ByteBuffer src, A attachment, CompletionHandler handler) { - int startPos = src.position(); - ByteBuffer duplicate = src.duplicate(); - channel.write(src, attachment, new CompletionHandler() { - @Override - public void completed(Integer result, A att) { - if (result != null && result > 0) { - updateMd5(duplicate, startPos, result); - } - handler.completed(result, att); - } - - @Override - public void failed(Throwable exc, A att) { - handler.failed(exc, att); - } - }); - } - - @Override - public Future write(ByteBuffer src) { - CompletableFuture future = new CompletableFuture<>(); - int startPos = src.position(); - ByteBuffer duplicate = src.duplicate(); - channel.write(src, src, new CompletionHandler() { - @Override - public void completed(Integer result, ByteBuffer attachment) { - if (result != null && result > 0) { - updateMd5(duplicate, startPos, result); - } - future.complete(result); - } - - @Override - public void failed(Throwable exc, ByteBuffer attachment) { - future.completeExceptionally(exc); - } - }); - return future; - } + private static Mono writeBodyToFile(BlobDownloadAsyncResponse response, AsynchronousFileChannel file, + long chunkNum, com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, + ProgressReporter progressReporter) { - @Override - public boolean isOpen() { - return channel.isOpen(); - } - - @Override - public void close() throws IOException { - channel.close(); - } - - private void updateMd5(ByteBuffer buffer, int startPos, int length) { - if (length <= 0) { - return; - } - - ByteBuffer slice = buffer.duplicate(); - slice.position(startPos); - slice.limit(startPos + length); - synchronized (digest) { - digest.update(slice); - } - } + long position = chunkNum * finalParallelTransferOptions.getBlockSizeLong(); + return response.writeValueToAsync(IOUtils.toAsynchronousByteChannel(file, position), progressReporter); } private void downloadToFileCleanup(AsynchronousFileChannel channel, String filePath, SignalType signalType) { diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java index 53792cfed42e..844235d6349d 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java @@ -1608,43 +1608,10 @@ public Response downloadToFileWithResponse(String filePath, Blob @ServiceMethod(returns = ReturnType.SINGLE) public Response downloadToFileWithResponse(BlobDownloadToFileOptions options, Duration timeout, Context context) { - Mono> download - = client.downloadToFileWithResponse(adjustOptionsForSyncDownload(options), context); + Mono> download = client.downloadToFileWithResponse(options, context); return blockWithOptionalTimeout(download, timeout); } - private static BlobDownloadToFileOptions adjustOptionsForSyncDownload(BlobDownloadToFileOptions options) { - if (options == null) { - return null; - } - - com.azure.storage.common.ParallelTransferOptions parallelTransferOptions = options.getParallelTransferOptions(); - Integer maxConcurrency = parallelTransferOptions == null ? null : parallelTransferOptions.getMaxConcurrency(); - if (maxConcurrency != null && maxConcurrency <= 1) { - return options; - } - - com.azure.storage.common.ParallelTransferOptions adjustedParallelOptions; - if (parallelTransferOptions == null) { - adjustedParallelOptions = new com.azure.storage.common.ParallelTransferOptions().setMaxConcurrency(1); - } else { - adjustedParallelOptions = new com.azure.storage.common.ParallelTransferOptions() - .setBlockSizeLong(parallelTransferOptions.getBlockSizeLong()) - .setInitialTransferSizeLong(parallelTransferOptions.getInitialTransferSizeLong()) - .setMaxConcurrency(1) - .setProgressListener(parallelTransferOptions.getProgressListener()) - .setMaxSingleUploadSizeLong(parallelTransferOptions.getMaxSingleUploadSizeLong()); - } - - return new BlobDownloadToFileOptions(options.getFilePath()).setRange(options.getRange()) - .setParallelTransferOptions(adjustedParallelOptions) - .setDownloadRetryOptions(options.getDownloadRetryOptions()) - .setRequestConditions(options.getRequestConditions()) - .setRetrieveContentRangeMd5(options.isRetrieveContentRangeMd5()) - .setOpenOptions(options.getOpenOptions()) - .setResponseChecksumAlgorithm(options.getResponseChecksumAlgorithm()); - } - /** * Deletes the specified blob or snapshot. To delete a blob with its snapshots use * {@link #deleteWithResponse(DeleteSnapshotsOptionType, BlobRequestConditions, Duration, Context)} and set diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java index 4a3898369a30..2f2b59e0567b 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java @@ -114,8 +114,7 @@ public void downloadToFileWithResponseContentValidation(int blockSize) throws IO Path tempFile = Files.createTempFile("structured-download", ".bin"); Files.deleteIfExists(tempFile); - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) - .setInitialTransferSizeLong((long) blockSize); + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize); BlobDownloadToFileOptions options = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index dc3ab286b5c5..7544db69d43d 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -103,8 +103,7 @@ public void downloadToFileWithResponseContentValidationSync(int blockSize) throw Path tempFile = Files.createTempFile("structured-download-sync", ".bin"); Files.deleteIfExists(tempFile); - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize) - .setInitialTransferSizeLong((long) blockSize); + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize); BlobDownloadToFileOptions options = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64); diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java index eef098cb60bd..34213e7300d7 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ParallelTransferOptions.java @@ -14,7 +14,6 @@ @Fluent public final class ParallelTransferOptions { private Long blockSize; - private Long initialTransferSize; private Integer maxConcurrency; private ProgressReceiver progressReceiver; private Long maxSingleUploadSize; @@ -78,29 +77,6 @@ public Long getBlockSizeLong() { return this.blockSize; } - /** - * Gets the size of the first range requested when downloading. - * @return The initial transfer size. - */ - public Long getInitialTransferSizeLong() { - return this.initialTransferSize; - } - - /** - * Sets the size of the first range requested when downloading. - * This value may be larger than the block size used for subsequent ranges. - * - * @param initialTransferSize The initial transfer size. - * @return The ParallelTransferOptions object itself. - */ - public ParallelTransferOptions setInitialTransferSizeLong(Long initialTransferSize) { - if (initialTransferSize != null) { - StorageImplUtils.assertInBounds("initialTransferSize", initialTransferSize, 1, Long.MAX_VALUE); - } - this.initialTransferSize = initialTransferSize; - return this; - } - /** * Sets the block size. * For upload, The block size is the size of each block that will be staged. This value also determines the number diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index efb76bd81e16..b25cea2a9c08 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -85,16 +85,6 @@ public final class Constants { public static final String STORAGE_LOG_STRING_TO_SIGN = "Azure-Storage-Log-String-To-Sign"; - /** - * System property name for enabling legacy default concurrency behavior. - */ - public static final String USE_LEGACY_DEFAULT_CONCURRENCY_PROPERTY = "Azure.Storage.UseLegacyDefaultConcurrency"; - - /** - * Environment variable name for enabling legacy default concurrency behavior. - */ - public static final String USE_LEGACY_DEFAULT_CONCURRENCY_ENV_VAR = "AZURE_STORAGE_USE_LEGACY_DEFAULT_CONCURRENCY"; - public static final String PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION = "AZURE_STORAGE_SAS_SERVICE_VERSION"; public static final String SAS_SERVICE_VERSION @@ -133,11 +123,6 @@ public final class Constants { public static final String STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY = "azure-storage-structured-message-aggregate-crc"; - /** - * Structured message header value for CRC64 validation. - */ - public static final String STRUCTURED_MESSAGE_CRC64_BODY_TYPE = "XSM/1.0; properties=crc64"; - private Constants() { } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 7bb669b58664..7da7521cedc1 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -11,6 +11,7 @@ import com.azure.core.http.HttpPipelinePosition; import com.azure.core.util.logging.ClientLogger; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -65,7 +66,9 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN return next.process(); } - context.getHttpRequest().getHeaders().set(X_MS_STRUCTURED_BODY, Constants.STRUCTURED_MESSAGE_CRC64_BODY_TYPE); + context.getHttpRequest() + .getHeaders() + .set(X_MS_STRUCTURED_BODY, StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); return next.process().map(httpResponse -> { Long contentLength = ContentValidationDecoderUtils.getContentLength(httpResponse.getHeaders()); diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java index f6ad54001d1a..4c1089d2c6b9 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java @@ -69,8 +69,6 @@ public final class StorageCommonTestUtils { try { jdkHttpHttpClient = createJdkHttpClient(); } catch (LinkageError | ReflectiveOperationException e) { - // LinkageError includes ExceptionInInitializerError (e.g. JDK HTTP client fails on Java 17 when - // SharedExecutorService needs Thread.ofVirtual() from Java 19+). Set to null so Netty/OkHttp/Vertx tests run. jdkHttpHttpClient = null; } @@ -132,11 +130,6 @@ public static HttpClient getHttpClient(Supplier playbackClientSuppli case VERTX: return VERTX_HTTP_CLIENT; case JDK_HTTP: - if (JDK_HTTP_HTTP_CLIENT == null) { - throw new IllegalStateException( - "JDK HTTP client is not available (e.g. requires Java 19+ for virtual threads). " - + "Use NETTY, OK_HTTP, or VERTX instead."); - } return JDK_HTTP_HTTP_CLIENT; default: throw new IllegalArgumentException("Unknown http client type: " + ENVIRONMENT.getHttpClientType()); diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java index d045b7f7fe64..9bdedc7dbe62 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -4,7 +4,9 @@ package com.azure.storage.common.implementation.contentvalidation; import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -24,7 +26,17 @@ * and segment splits across chunks. */ public class StructuredMessageDecoderTests { - + + private static ByteBuffer collectFlux(Flux flux) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + flux.toIterable().forEach(buf -> { + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + out.write(bytes, 0, bytes.length); + }); + return ByteBuffer.wrap(out.toByteArray()).order(ByteOrder.LITTLE_ENDIAN); + } + @Test public void readsCompleteMessageInSingleChunk() throws IOException { // Test: Complete message in a single ByteBuffer should decode fully @@ -33,7 +45,7 @@ public void readsCompleteMessageInSingleChunk() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 512, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); @@ -54,7 +66,7 @@ public void readsMessageSplitHeaderAcrossChunks() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); byte[] encodedBytes = new byte[encodedLength]; encodedData.get(encodedBytes); @@ -86,7 +98,7 @@ public void readsSegmentHeaderSplitAcrossChunks() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); byte[] encodedBytes = new byte[encodedLength]; encodedData.get(encodedBytes); @@ -124,7 +136,7 @@ public void handlesZeroLengthSegment() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(minimalData.length, 1024, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(minimalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(minimalData))); int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); @@ -143,7 +155,7 @@ public void tracksLastCompleteSegmentCorrectly() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); @@ -171,7 +183,7 @@ public void resetToLastCompleteSegmentWorks() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); byte[] encodedBytes = new byte[encodedLength]; encodedData.get(encodedBytes); @@ -203,7 +215,7 @@ public void multipleChunksDecode() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); byte[] encodedBytes = new byte[encodedLength]; encodedData.get(encodedBytes); @@ -212,7 +224,7 @@ public void multipleChunksDecode() throws IOException { // Feed in chunks of 32 bytes int chunkSize = 32; - java.io.ByteArrayOutputStream output = new java.io.ByteArrayOutputStream(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); for (int offset = 0; offset < encodedLength; offset += chunkSize) { int len = Math.min(chunkSize, encodedLength - offset); @@ -243,7 +255,7 @@ public void decodeWithNoCrc() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.NONE); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); @@ -264,7 +276,7 @@ public void handlesZeroLengthBuffer() throws IOException { StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, 128, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = encoder.encode(ByteBuffer.wrap(originalData)); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); int encodedLength = encodedData.remaining(); byte[] encodedBytes = new byte[encodedLength]; encodedData.get(encodedBytes); From 11a2da2a8c4decf30f49030fbeeba6a18ade7b94 Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Wed, 8 Apr 2026 16:38:18 +0530 Subject: [PATCH 14/31] code refactoring based on latest review comments --- .../implementation/util/BuilderHelper.java | 4 +- .../blob/specialized/BlobAsyncClientBase.java | 6 +- .../blob/specialized/BlobClientBase.java | 2 +- .../StorageCrc64Calculator.java | 16 +-- .../StructuredMessageDecodingStream.java | 104 ------------------ 5 files changed, 16 insertions(+), 116 deletions(-) delete mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index ef631882249a..dbb808777eac 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -97,6 +97,8 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare // Closest to API goes first, closest to wire goes last. List policies = new ArrayList<>(); + policies.add(new StorageContentValidationDecoderPolicy()); + policies.add(getUserAgentPolicy(configuration, logOptions, clientOptions)); policies.add(new RequestIdPolicy()); @@ -116,8 +118,6 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare } policies.add(new MetadataValidationPolicy()); - policies.add(new StorageContentValidationDecoderPolicy()); - if (storageSharedKeyCredential != null) { policies.add(new StorageSharedKeyCredentialPolicy(storageSharedKeyCredential)); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 828ececb983a..c5c57f3ea464 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1575,7 +1575,7 @@ public Mono> downloadToFileWithResponse(String filePath BlobRequestConditions requestConditions, boolean rangeGetContentMd5, Set openOptions) { try { final com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions - = parallelTransferOptions == null ? null : ModelHelper.wrapBlobOptions(parallelTransferOptions); + = ModelHelper.wrapBlobOptions(ModelHelper.populateAndApplyDefaults(parallelTransferOptions)); return withContext( context -> downloadToFileWithResponse(new BlobDownloadToFileOptions(filePath).setRange(range) .setParallelTransferOptions(finalParallelTransferOptions) @@ -1629,7 +1629,7 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti StorageImplUtils.assertNotNull("options", options); BlobRange finalRange = options.getRange() == null ? new BlobRange(0) : options.getRange(); - com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions + final com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions = ModelHelper.populateAndApplyDefaults(options.getParallelTransferOptions()); BlobRequestConditions finalConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); @@ -1681,6 +1681,7 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne int numChunks = ChunkedDownloadUtils.calculateNumBlocks(newCount, finalParallelTransferOptions.getBlockSizeLong()); + // In case it is an empty blob, this ensures we still actually perform a download operation. numChunks = numChunks == 0 ? 1 : numChunks; BlobDownloadAsyncResponse initialResponse = setupTuple3.getT3(); @@ -1692,6 +1693,7 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne progressReporter == null ? null : progressReporter.createChild()).flux()), finalParallelTransferOptions.getMaxConcurrency()) + // Only the first download call returns a value. .then(Mono.just(ModelHelper.buildBlobPropertiesResponse(initialResponse))); }); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java index 844235d6349d..b6169ff7636c 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java @@ -1567,7 +1567,7 @@ public Response downloadToFileWithResponse(String filePath, Blob BlobRequestConditions requestConditions, boolean rangeGetContentMd5, Set openOptions, Duration timeout, Context context) { final com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions - = parallelTransferOptions == null ? null : ModelHelper.wrapBlobOptions(parallelTransferOptions); + = ModelHelper.wrapBlobOptions(ModelHelper.populateAndApplyDefaults(parallelTransferOptions)); return downloadToFileWithResponse(new BlobDownloadToFileOptions(filePath).setRange(range) .setParallelTransferOptions(finalParallelTransferOptions) .setDownloadRetryOptions(downloadRetryOptions) diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java index 1352423eb693..119f83c1f546 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java @@ -2400,6 +2400,8 @@ public class StorageCrc64Calculator { /** * Computes the CRC64 checksum for the given byte array using the Azure Storage CRC64 polynomial. + * This method processes the input data in chunks of 32 bytes for efficiency and uses lookup tables + * to update the CRC values. * * @param src the byte array for which the CRC64 checksum is to be computed. * @param uCrc the initial CRC value. @@ -2420,7 +2422,7 @@ public static long compute(byte[] src, long uCrc) { * @return the computed CRC64 checksum. */ public static long compute(byte[] src, int offset, int length, long uCrc) { - int pData = 0; + int pData = offset; long uSize = length; long uBytes, uStop; @@ -2437,7 +2439,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { uSize -= uStop; uCrc0 = uCrc; - ByteBuffer buffer = ByteBuffer.wrap(src, offset, length).order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer buffer = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); for (; pData < pLast; pData += 32) { long b0 = buffer.getLong(pData) ^ uCrc0; @@ -2515,7 +2517,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { } uCrc = 0; - uCrc ^= ByteBuffer.wrap(src, offset + pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; + uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; @@ -2525,7 +2527,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc ^= ByteBuffer.wrap(src, offset + pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc1; + uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc1; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; @@ -2535,7 +2537,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc ^= ByteBuffer.wrap(src, offset + pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc2; + uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc2; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; @@ -2545,7 +2547,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; - uCrc ^= ByteBuffer.wrap(src, offset + pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc3; + uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc3; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; @@ -2559,7 +2561,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { } for (uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { - uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[offset + pData]) & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[pData]) & 0xFF)]; } return ~uCrc; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java deleted file mode 100644 index 793b12dc43eb..000000000000 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecodingStream.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.common.implementation.contentvalidation; - -import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.StorageChecksumAlgorithm; - -import reactor.core.publisher.Flux; - -import java.nio.ByteBuffer; - -/** - * A utility class for applying structured message decoding to download streams. - */ -public final class StructuredMessageDecodingStream { - private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecodingStream.class); - - private StructuredMessageDecodingStream() { - // utility class - } - - /** - * Wraps a download stream with structured message decoding if content validation is enabled. - * - * @param originalStream The original download stream. - * @param contentLength The expected content length. - * @param responseChecksumAlgorithm The response checksum algorithm. - * @return A Flux that decodes structured messages if validation is enabled, otherwise returns the original stream. - */ - public static Flux wrapStreamIfNeeded(Flux originalStream, Long contentLength, - StorageChecksumAlgorithm responseChecksumAlgorithm) { - - if (responseChecksumAlgorithm == null || responseChecksumAlgorithm != StorageChecksumAlgorithm.CRC64) { - return originalStream; - } - - if (contentLength == null || contentLength <= 0) { - LOGGER.warning("Cannot apply structured message validation without valid content length."); - return originalStream; - } - - return applyStructuredMessageDecoding(originalStream, contentLength); - } - - /** - * Applies structured message decoding to the stream. - * - * @param stream The stream to decode. - * @param expectedContentLength The expected content length. - * @return A Flux that decodes the structured message. - */ - private static Flux applyStructuredMessageDecoding(Flux stream, - long expectedContentLength) { - return stream - .collect(() -> new StructuredMessageDecodingCollector(expectedContentLength), - StructuredMessageDecodingCollector::addBuffer) - .flatMapMany(collector -> collector.getDecodedData()); - } - - /** - * Helper class to collect and decode structured message data. - */ - private static class StructuredMessageDecodingCollector { - private final StructuredMessageDecoder decoder; - private ByteBuffer accumulatedBuffer; - private boolean completed = false; - - StructuredMessageDecodingCollector(long expectedContentLength) { - this.decoder = new StructuredMessageDecoder(expectedContentLength); - this.accumulatedBuffer = ByteBuffer.allocate(0); - } - - void addBuffer(ByteBuffer buffer) { - if (completed) { - return; - } - - // Accumulate the buffer - ByteBuffer newBuffer = ByteBuffer.allocate(accumulatedBuffer.remaining() + buffer.remaining()); - newBuffer.put(accumulatedBuffer); - newBuffer.put(buffer); - newBuffer.flip(); - accumulatedBuffer = newBuffer; - } - - Flux getDecodedData() { - try { - if (accumulatedBuffer.remaining() == 0) { - return Flux.empty(); - } - - ByteBuffer decodedData = decoder.decode(accumulatedBuffer); - decoder.finalizeDecoding(); - completed = true; - - return Flux.just(decodedData); - } catch (Exception e) { - LOGGER.error("Failed to decode structured message: " + e.getMessage(), e); - return Flux.error(e); - } - } - } -} From c9076495ba4f93572fedd0049311556f774da382 Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Wed, 8 Apr 2026 18:32:14 +0530 Subject: [PATCH 15/31] simplifying retry mechanism --- ...DownloadAsyncResponseConstructorProxy.java | 28 +-- .../util/DownloadValidationUtils.java | 17 +- .../models/BlobDownloadAsyncResponse.java | 63 +------ .../blob/specialized/BlobAsyncClientBase.java | 135 +++----------- .../common/implementation/Constants.java | 24 --- .../common/policy/AggregateCrcState.java | 73 -------- .../policy/ContentValidationDecoderUtils.java | 53 +----- .../common/policy/DecodedResponse.java | 8 +- .../storage/common/policy/DecoderState.java | 141 --------------- ...StorageContentValidationDecoderPolicy.java | 168 +++--------------- ...ageContentValidationDecoderPolicyTest.java | 105 ----------- 11 files changed, 54 insertions(+), 761 deletions(-) delete mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java delete mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java delete mode 100644 sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java index dd59769fbc6a..234ca5e65e77 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlobDownloadAsyncResponseConstructorProxy.java @@ -9,10 +9,8 @@ import com.azure.core.http.rest.StreamResponse; import com.azure.storage.blob.models.BlobDownloadAsyncResponse; import com.azure.storage.blob.models.DownloadRetryOptions; -import com.azure.storage.common.policy.DecoderState; import reactor.core.publisher.Mono; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; /** @@ -37,16 +35,7 @@ public interface BlobDownloadAsyncResponseConstructorAccessor { * @param retryOptions Retry options. */ BlobDownloadAsyncResponse create(StreamResponse sourceResponse, - BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef); - - /** - * Gets the current decoder state from a {@link BlobDownloadAsyncResponse}. - * - * @param response The {@link BlobDownloadAsyncResponse}. - * @return The current decoder state, or null if not available. - */ - DecoderState getDecoderState(BlobDownloadAsyncResponse response); + BiFunction> onErrorResume, DownloadRetryOptions retryOptions); } /** @@ -67,8 +56,7 @@ public static void setAccessor( * @param retryOptions Retry options. */ public static BlobDownloadAsyncResponse create(StreamResponse sourceResponse, - BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef) { + BiFunction> onErrorResume, DownloadRetryOptions retryOptions) { // This looks odd but is necessary, it is possible to engage the access helper before anywhere else in the // application accesses BlobDownloadAsyncResponse which triggers the accessor to be configured. So, if the accessor // is null this effectively pokes the class to set up the accessor. @@ -78,16 +66,6 @@ public static BlobDownloadAsyncResponse create(StreamResponse sourceResponse, } assert accessor != null; - return accessor.create(sourceResponse, onErrorResume, retryOptions, decoderStateRef); - } - - /** - * Gets the current decoder state from a {@link BlobDownloadAsyncResponse}. - * - * @param response The {@link BlobDownloadAsyncResponse}. - * @return The decoder state, or null if not available. - */ - public static DecoderState getDecoderState(BlobDownloadAsyncResponse response) { - return accessor.getDecoderState(response); + return accessor.create(sourceResponse, onErrorResume, retryOptions); } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java index 3c9491c5e81b..5b66a129d981 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java @@ -6,11 +6,6 @@ import com.azure.core.util.Context; import com.azure.storage.common.StorageChecksumAlgorithm; import com.azure.storage.common.implementation.Constants; -import com.azure.storage.common.policy.AggregateCrcState; - -import java.util.concurrent.atomic.AtomicReference; - -import com.azure.storage.common.policy.DecoderState; /** * Centralizes download content validation decisions based on {@link StorageChecksumAlgorithm}. @@ -39,23 +34,17 @@ public static StorageChecksumAlgorithm resolveAlgorithm(StorageChecksumAlgorithm } /** - * Adds structured message decoding context keys when CRC64/AUTO validation is active. + * Adds structured message decoding context key when CRC64/AUTO validation is active. * * @param context The base context to augment. Null is treated as {@link Context#NONE}. * @param algorithm The resolved checksum algorithm. - * @param decoderStateRef Holder for decoder state, populated by the policy. - * @param aggregateCrcState CRC aggregation state across retries. * @return The augmented context. */ - public static Context applyStructuredMessageContext(Context context, StorageChecksumAlgorithm algorithm, - AtomicReference decoderStateRef, AggregateCrcState aggregateCrcState) { + public static Context applyStructuredMessageContext(Context context, StorageChecksumAlgorithm algorithm) { Context base = context == null ? Context.NONE : context; if (!isStructuredMessageAlgorithm(algorithm)) { return base; } - return base.addData(Constants.STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY, true) - .addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true) - .addData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY, decoderStateRef) - .addData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY, aggregateCrcState); + return base.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java index 59a021493e1f..2eebcb44bdcf 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java @@ -14,17 +14,14 @@ import com.azure.storage.blob.implementation.models.BlobsDownloadHeaders; import com.azure.storage.blob.implementation.util.ModelHelper; import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.policy.DecoderState; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.channels.AsynchronousByteChannel; import java.util.Objects; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; /** @@ -40,15 +37,9 @@ public final class BlobDownloadAsyncResponse extends ResponseBase> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef) { - return new BlobDownloadAsyncResponse(sourceResponse, onErrorResume, retryOptions, decoderStateRef); - } - - @Override - public DecoderState getDecoderState(BlobDownloadAsyncResponse response) { - AtomicReference ref = response.decoderStateRef; - return ref == null ? null : ref.get(); + BiFunction> onErrorResume, + DownloadRetryOptions retryOptions) { + return new BlobDownloadAsyncResponse(sourceResponse, onErrorResume, retryOptions); } }); } @@ -58,7 +49,6 @@ public DecoderState getDecoderState(BlobDownloadAsyncResponse response) { private final StreamResponse sourceResponse; private final BiFunction> onErrorResume; private final DownloadRetryOptions retryOptions; - private final AtomicReference decoderStateRef; /** * Constructs a {@link BlobDownloadAsyncResponse}. @@ -75,38 +65,15 @@ public BlobDownloadAsyncResponse(HttpRequest request, int statusCode, HttpHeader this.sourceResponse = null; this.onErrorResume = null; this.retryOptions = null; - this.decoderStateRef = null; } - /** - * Constructs a {@link BlobDownloadAsyncResponse}. - * - * @param sourceResponse The initial Stream Response - * @param onErrorResume Function used to resume. - * @param retryOptions Retry options. - */ BlobDownloadAsyncResponse(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions) { - this(sourceResponse, onErrorResume, retryOptions, null); - } - - BlobDownloadAsyncResponse(StreamResponse sourceResponse, - BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef) { - this(sourceResponse, onErrorResume, retryOptions, decoderStateRef, extractHeaders(sourceResponse)); - } - - private BlobDownloadAsyncResponse(StreamResponse sourceResponse, - BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef, BlobDownloadHeaders deserializedHeaders) { super(sourceResponse.getRequest(), sourceResponse.getStatusCode(), sourceResponse.getHeaders(), - createResponseFluxWithContentCrc(sourceResponse, onErrorResume, retryOptions, decoderStateRef, - deserializedHeaders), - deserializedHeaders); + createResponseFlux(sourceResponse, onErrorResume, retryOptions), extractHeaders(sourceResponse)); this.sourceResponse = Objects.requireNonNull(sourceResponse, "'sourceResponse' must not be null"); this.onErrorResume = Objects.requireNonNull(onErrorResume, "'onErrorResume' must not be null"); this.retryOptions = Objects.requireNonNull(retryOptions, "'retryOptions' must not be null"); - this.decoderStateRef = decoderStateRef; } private static BlobDownloadHeaders extractHeaders(StreamResponse response) { @@ -124,28 +91,6 @@ private static Flux createResponseFlux(StreamResponse sourceResponse .defaultIfEmpty(EMPTY_BUFFER); } - /** - * Builds the response flux and populates ContentCrc64 on the deserialized headers when structured message - * decoding completes successfully. - */ - private static Flux createResponseFluxWithContentCrc(StreamResponse sourceResponse, - BiFunction> onErrorResume, DownloadRetryOptions retryOptions, - AtomicReference decoderStateRef, BlobDownloadHeaders deserializedHeaders) { - Flux flux = createResponseFlux(sourceResponse, onErrorResume, retryOptions); - if (decoderStateRef != null && deserializedHeaders != null) { - flux = flux.doOnComplete(() -> { - DecoderState state = decoderStateRef.get(); - if (state != null && state.isFinalized()) { - long crc = state.getComposedCrc64(); - byte[] crcBytes = new byte[8]; - ByteBuffer.wrap(crcBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(crc); - deserializedHeaders.setContentCrc64(crcBytes); - } - }); - } - return flux; - } - /** * Transfers content bytes to the {@link AsynchronousByteChannel}. * @param channel The destination {@link AsynchronousByteChannel}. diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index c5c57f3ea464..42122ab4e6b6 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -84,13 +84,9 @@ import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; -import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.StorageChecksumAlgorithm; -import com.azure.storage.common.policy.AggregateCrcState; -import com.azure.storage.common.policy.DecoderState; -import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; @@ -119,8 +115,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -1280,24 +1274,15 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - AtomicReference decoderStateRef = new AtomicReference<>(); - AggregateCrcState aggregateCrcState = isStructuredMessageEnabled ? new AggregateCrcState() : null; - AtomicLong responseStartOffset = new AtomicLong(0); - - final Context firstRangeContext = DownloadValidationUtils.applyStructuredMessageContext(baseContext, algorithm, - decoderStateRef, aggregateCrcState); + final Context downloadContext = DownloadValidationUtils.applyStructuredMessageContext(baseContext, algorithm); return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), finalGetMD5, - firstRangeContext).map(response -> { + downloadContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); String eTag = blobsDownloadHeaders.getETag(); BlobDownloadHeaders blobDownloadHeaders = ModelHelper.populateBlobDownloadHeaders(blobsDownloadHeaders, ModelHelper.getErrorCode(response.getHeaders())); - /* - * If the customer did not specify a count, they are reading to the end of the blob. Extract this value - * from the response for better book-keeping towards the end. - */ long finalCount; long initialOffset = finalRange.getOffset(); if (finalRange.getCount() == null) { @@ -1307,46 +1292,36 @@ Mono downloadStreamWithResponseInternal(BlobRange ran finalCount = finalRange.getCount(); } - // The resume function takes throwable and offset at the destination. - // I.e. offset is relative to the starting point. BiFunction> onDownloadErrorResume = (throwable, offset) -> { if (!(throwable instanceof IOException || throwable instanceof TimeoutException)) { return Mono.error(throwable); } try { - if (isStructuredMessageEnabled) { - return retryStructuredDownload(throwable, offset, decoderStateRef, responseStartOffset, - finalCount, initialOffset, firstRangeContext, finalRequestConditions, eTag, - finalGetMD5); - } else { - long newCount = finalCount - offset; - - /* - * It's possible that the network stream will throw an error after emitting all data but - * before completing. Issuing a retry at this stage would leave the download in a bad - * state with incorrect count and offset values. Because we have read the intended amount - * of data, we can ignore the error at the end of the stream. - */ - if (newCount == 0) { - LOGGER.warning( - "Exception encountered in ReliableDownload after all data read from the network " - + "but before stream signaled completion. Returning success as all data was " - + "downloaded. Exception message: " + throwable.getMessage()); - return Mono.empty(); - } - - BlobRange retryRange = new BlobRange(initialOffset + offset, newCount); - return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, - firstRangeContext); + long newCount = finalCount - offset; + + /* + * It's possible that the network stream will throw an error after emitting all data but + * before completing. Issuing a retry at this stage would leave the download in a bad + * state with incorrect count and offset values. Because we have read the intended amount + * of data, we can ignore the error at the end of the stream. + */ + if (newCount == 0) { + LOGGER.warning( + "Exception encountered in ReliableDownload after all data read from the network " + + "but before stream signaled completion. Returning success as all data was " + + "downloaded. Exception message: " + throwable.getMessage()); + return Mono.empty(); } + + BlobRange retryRange = new BlobRange(initialOffset + offset, newCount); + return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, downloadContext); } catch (Exception e) { return Mono.error(e); } }; - return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions, - decoderStateRef); + return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions); }); } @@ -1361,76 +1336,6 @@ private Mono downloadRange(BlobRange range, BlobRequestCondition context); } - private Mono retryStructuredDownload(Throwable throwable, long emittedOffset, - AtomicReference decoderStateRef, AtomicLong responseStartOffset, long finalCount, - long initialOffset, Context baseRetryContext, BlobRequestConditions conditions, String eTag, Boolean getMD5) { - - long currentResponseOffset = responseStartOffset.get(); - DecoderState decoderState = decoderStateRef.get(); - - long retryStartOffset = resolveStructuredRetryOffset(decoderState, throwable, currentResponseOffset, - emittedOffset == 0, emittedOffset); - long bytesToSkip = calculateRetryBytesToSkip(emittedOffset, retryStartOffset, currentResponseOffset); - - long remainingCount = finalCount - retryStartOffset; - if (remainingCount < 0) { - retryStartOffset = Math.min(emittedOffset, finalCount); - remainingCount = finalCount - retryStartOffset; - bytesToSkip = 0; - } - - if (decoderState == null) { - decoderStateRef.set(null); - } - - Context retryContext = baseRetryContext; - if (bytesToSkip > 0) { - retryContext - = retryContext.addData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY, bytesToSkip); - } - - responseStartOffset.set(retryStartOffset); - BlobRange retryRange = new BlobRange(initialOffset + retryStartOffset, remainingCount); - - LOGGER.info("Structured message retry: resuming from offset {} (initial={}, decoded={}, remaining={}, skip={})", - initialOffset + retryStartOffset, initialOffset, retryStartOffset, remainingCount, bytesToSkip); - - return downloadRange(retryRange, conditions, eTag, getMD5, retryContext); - } - - private static long resolveStructuredRetryOffset(DecoderState decoderState, Throwable throwable, - long currentResponseOffset, boolean noBytesEmitted, long emittedOffset) { - - long offset = -1; - - long parsedOffset = StorageContentValidationDecoderPolicy - .parseRetryStartOffset(throwable != null ? throwable.getMessage() : null); - if (parsedOffset >= 0) { - offset = currentResponseOffset + parsedOffset; - } - - if (decoderState != null) { - long boundary = currentResponseOffset + decoderState.getDecodedBytesAtLastCompleteSegment(); - if (offset < 0 || boundary > offset) { - offset = boundary; - } - } - - if (offset < 0) { - offset = noBytesEmitted ? currentResponseOffset : emittedOffset; - } - return offset; - } - - private static long calculateRetryBytesToSkip(long emittedOffset, long retryStartOffset, - long currentResponseOffset) { - long skip = emittedOffset - retryStartOffset; - if (skip < 0) { - skip = Math.max(0, emittedOffset - currentResponseOffset); - } - return skip; - } - /** * Downloads the entire blob into a file specified by the path. * diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index b25cea2a9c08..9e664a486da4 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -99,30 +99,6 @@ public final class Constants { */ public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; - /** - * Context key used to signal that structured message decoding should be scoped to a single response. - * This is used for parallel range downloads where each response is independently structured-encoded. - */ - public static final String STRUCTURED_MESSAGE_RESPONSE_SCOPED_CONTEXT_KEY - = "azure-storage-structured-message-response-scoped"; - - /** - * Context key used to pass a mutable holder for decoder state so callers can observe decoder progress. - */ - public static final String STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY - = "azure-storage-structured-message-decoder-state-ref"; - - /** - * Context key used to pass the number of decoded bytes to skip on the next structured message response. - */ - public static final String STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY - = "azure-storage-structured-message-decoder-skip-bytes"; - /** - * Context key used to share structured message CRC aggregation state across retries. - */ - public static final String STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY - = "azure-storage-structured-message-aggregate-crc"; - private Constants() { } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java deleted file mode 100644 index 58566975a49b..000000000000 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/AggregateCrcState.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.common.policy; - -import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; -import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -/** - * Aggregates CRC state across retries so the composed CRC64 can be validated against the - * running CRC of all decoded payload bytes. - */ -public final class AggregateCrcState { - private final List segments = new ArrayList<>(); - private long runningCrc = 0; - - /** - * Creates a new instance of {@link AggregateCrcState}. - */ - public AggregateCrcState() { - } - - void appendPayload(ByteBuffer payload) { - if (payload == null || !payload.hasRemaining()) { - return; - } - ByteBuffer copy = payload.asReadOnlyBuffer(); - byte[] data = new byte[copy.remaining()]; - copy.get(data); - runningCrc = StorageCrc64Calculator.compute(data, runningCrc); - } - - void addSegments(List newSegments) { - if (newSegments == null || newSegments.isEmpty()) { - return; - } - segments.addAll(newSegments); - } - - boolean hasSegments() { - return !segments.isEmpty(); - } - - long getRunningCrc() { - return runningCrc; - } - - long composeCrc() { - if (segments.isEmpty()) { - return 0; - } - long composed = segments.get(0).getCrc64(); - long totalLength = segments.get(0).getLength(); - for (int i = 1; i < segments.size(); i++) { - StructuredMessageDecoder.SegmentInfo next = segments.get(i); - composed = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); - totalLength += next.getLength(); - } - return composed; - } - - long getTotalLength() { - long totalLength = 0; - for (StructuredMessageDecoder.SegmentInfo segment : segments) { - totalLength += segment.getLength(); - } - return totalLength; - } -} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java index 8fff6225955b..d1fa42ba9e6c 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java @@ -9,67 +9,16 @@ import com.azure.core.http.HttpResponse; import com.azure.core.util.logging.ClientLogger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - /** - * Utility class for parsing helpers and download eligibility checks + * Utility class for download eligibility checks * used by {@link StorageContentValidationDecoderPolicy}. */ final class ContentValidationDecoderUtils { private static final ClientLogger LOGGER = new ClientLogger(ContentValidationDecoderUtils.class); - static final String RETRY_OFFSET_TOKEN = "RETRY-START-OFFSET="; - private static final Pattern RETRY_OFFSET_PATTERN = Pattern.compile("RETRY-START-OFFSET=(\\d+)"); - private static final Pattern DECODER_OFFSETS_PATTERN - = Pattern.compile("\\[decoderOffset=(\\d+),lastCompleteSegment=(\\d+)\\]"); - private ContentValidationDecoderUtils() { } - /** - * Parses the retry start offset from an exception message containing the RETRY-START-OFFSET token. - * - * @param message The exception message to parse. - * @return The retry start offset, or -1 if not found. - */ - static long parseRetryStartOffset(String message) { - if (message == null) { - return -1; - } - Matcher matcher = RETRY_OFFSET_PATTERN.matcher(message); - if (matcher.find()) { - try { - return Long.parseLong(matcher.group(1)); - } catch (NumberFormatException e) { - return -1; - } - } - return -1; - } - - /** - * Parses decoder offset information from enriched exception messages. - * Format: "[decoderOffset=X,lastCompleteSegment=Y]" - * - * @param message The exception message to parse. - * @return A long array [decoderOffset, lastCompleteSegment], or null if not found. - */ - static long[] parseDecoderOffsets(String message) { - if (message == null) { - return null; - } - Matcher matcher = DECODER_OFFSETS_PATTERN.matcher(message); - if (matcher.find()) { - try { - return new long[] { Long.parseLong(matcher.group(1)), Long.parseLong(matcher.group(2)) }; - } catch (NumberFormatException e) { - return null; - } - } - return null; - } - /** * Checks whether the response represents a successful download (GET with 2xx). */ diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java index 6b67d432e7ec..db8a597bf265 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -18,13 +18,11 @@ class DecodedResponse extends HttpResponse { private final HttpResponse originalResponse; private final Flux decodedBody; - private final DecoderState decoderState; - DecodedResponse(HttpResponse originalResponse, Flux decodedBody, DecoderState decoderState) { + DecodedResponse(HttpResponse originalResponse, Flux decodedBody) { super(originalResponse.getRequest()); this.originalResponse = originalResponse; this.decodedBody = decodedBody; - this.decoderState = decoderState; } @Override @@ -66,8 +64,4 @@ public Mono getBodyAsString(Charset charset) { public void close() { originalResponse.close(); } - - DecoderState getDecoderState() { - return decoderState; - } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java deleted file mode 100644 index be8cb1f60493..000000000000 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecoderState.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.common.policy; - -import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; -import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; - -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Tracks the progress of a single structured-message decode attempt. A new instance is - * created for each HTTP response (including retries). The aggregate CRC state, when - * present, is shared across retries to enable end-to-end CRC64 validation. - */ -public class DecoderState { - private static final ClientLogger LOGGER = new ClientLogger(DecoderState.class); - - private final StructuredMessageDecoder decoder; - final AggregateCrcState aggregateCrcState; - final AtomicLong totalBytesDecoded; - long decodedBytesAtLastCompleteSegment; - long lastCompleteSegmentStart; - final AtomicLong decodedBytesToSkip = new AtomicLong(0); - private boolean segmentsAddedToAggregate; - - DecoderState(long expectedContentLength, AggregateCrcState aggregateCrcState) { - this.decoder = new StructuredMessageDecoder(expectedContentLength); - this.totalBytesDecoded = new AtomicLong(0); - this.decodedBytesAtLastCompleteSegment = 0; - this.aggregateCrcState = aggregateCrcState; - this.segmentsAddedToAggregate = false; - } - - StructuredMessageDecoder getDecoder() { - return decoder; - } - - void updateProgress() { - long currentLastComplete = decoder.getLastCompleteSegmentStart(); - if (lastCompleteSegmentStart != currentLastComplete) { - decodedBytesAtLastCompleteSegment = decoder.getDecodedBytesAtLastCompleteSegment(); - lastCompleteSegmentStart = currentLastComplete; - - LOGGER.atInfo() - .addKeyValue("newSegmentBoundary", currentLastComplete) - .addKeyValue("decodedBytesAtBoundary", decodedBytesAtLastCompleteSegment) - .log("Segment boundary crossed, updated decoded bytes snapshot"); - } - } - - void addSegmentsToAggregateIfNeeded() { - if (segmentsAddedToAggregate || aggregateCrcState == null) { - return; - } - aggregateCrcState.addSegments(decoder.getCompletedSegments()); - segmentsAddedToAggregate = true; - } - - void setDecodedBytesToSkip(long bytesToSkip) { - decodedBytesToSkip.set(Math.max(0, bytesToSkip)); - } - - /** - * Returns true if the decoder has finalized (all segments decoded and validated). - * - * @return true if finalized, false otherwise. - */ - public boolean isFinalized() { - return decoder.isComplete(); - } - - /** - * Gets the decoded byte count at the last validated segment boundary. - * - * @return the decoded byte count. - */ - public long getDecodedBytesAtLastCompleteSegment() { - return decodedBytesAtLastCompleteSegment; - } - - /** - * Gets the composed CRC64 over all validated segments. - * - * @return the composed CRC64 value. - */ - public long getComposedCrc64() { - if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { - return aggregateCrcState.composeCrc(); - } - - List segments = decoder.getCompletedSegments(); - if (segments.isEmpty()) { - return 0; - } - - long composed = segments.get(0).getCrc64(); - long totalLength = segments.get(0).getLength(); - for (int i = 1; i < segments.size(); i++) { - StructuredMessageDecoder.SegmentInfo next = segments.get(i); - composed = StorageCrc64Calculator.concat(0, 0, composed, totalLength, 0, next.getCrc64(), next.getLength()); - totalLength += next.getLength(); - } - return composed; - } - - /** - * Gets the composed decoded payload length represented by validated segments. - * - * @return the composed payload length. - */ - public long getComposedLength() { - if (aggregateCrcState != null && aggregateCrcState.hasSegments()) { - return aggregateCrcState.getTotalLength(); - } - - List segments = decoder.getCompletedSegments(); - long totalLength = 0; - for (StructuredMessageDecoder.SegmentInfo segment : segments) { - totalLength += segment.getLength(); - } - return totalLength; - } - - /** - * Gets the decoded offset to use for retry requests. - * - * @return the retry offset. - */ - public long getRetryOffset() { - long retryOffset = decodedBytesAtLastCompleteSegment; - LOGGER.atInfo() - .addKeyValue("decoderOffset", decoder.getMessageOffset()) - .addKeyValue("pendingBytes", decoder.getPendingEncodedByteCount()) - .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) - .log("Computed smart-retry offset from decoder state"); - return retryOffset; - } -} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 7da7521cedc1..418c7c249375 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -18,8 +18,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; /** * Pipeline policy that decodes structured messages in storage download responses when @@ -28,9 +26,9 @@ * *

The policy is activated by the presence of a boolean context key * ({@link Constants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY}). It validates per-segment - * CRC64 checksums, tracks decoder progress in {@link DecoderState}, and embeds - * machine-readable retry offsets in exception messages so the caller can resume from the - * last validated segment boundary.

+ * CRC64 checksums during decoding. Since this policy is placed at the front of the pipeline + * (before the retry policy), each retry re-enters the policy with a fresh response body, + * so no cross-retry state management is needed.

*/ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); @@ -49,17 +47,6 @@ public HttpPipelinePosition getPipelinePosition() { return HttpPipelinePosition.PER_RETRY; } - /** - * Parses the retry start offset from an exception message containing the - * {@code RETRY-START-OFFSET} token. - * - * @param message The exception message to parse. - * @return The retry start offset, or -1 if not found. - */ - public static long parseRetryStartOffset(String message) { - return ContentValidationDecoderUtils.parseRetryStartOffset(message); - } - @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { if (!shouldApplyDecoding(context)) { @@ -81,14 +68,14 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN validateStructuredMessageHeaders(httpResponse); long expectedLength = contentLength; - DecoderState decoderState = createDecoderState(context, expectedLength); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(expectedLength); - Flux decodedStream = decodeStream(httpResponse.getBody(), decoderState); + Flux decodedStream = decodeStream(httpResponse.getBody(), decoder); LOGGER.atVerbose() .addKeyValue("expectedLength", expectedLength) .log("Returning DecodedResponse with structured message decoding"); - return new DecodedResponse(httpResponse, decodedStream, decoderState); + return new DecodedResponse(httpResponse, decodedStream); }); } @@ -107,41 +94,13 @@ private void validateStructuredMessageHeaders(HttpResponse httpResponse) { } } - private DecoderState createDecoderState(HttpPipelineCallContext context, long expectedLength) { - AggregateCrcState aggregateCrcState = context.getData(Constants.STRUCTURED_MESSAGE_AGGREGATE_CRC_CONTEXT_KEY) - .filter(AggregateCrcState.class::isInstance) - .map(AggregateCrcState.class::cast) - .orElse(null); - - DecoderState state = new DecoderState(expectedLength, aggregateCrcState); - - context.getData(Constants.STRUCTURED_MESSAGE_DECODER_SKIP_BYTES_CONTEXT_KEY) - .filter(Number.class::isInstance) - .map(Number.class::cast) - .ifPresent(skip -> state.setDecodedBytesToSkip(skip.longValue())); - - getDecoderStateHolder(context).ifPresent(holder -> holder.set(state)); - - return state; - } - - @SuppressWarnings("unchecked") - private Optional> getDecoderStateHolder(HttpPipelineCallContext context) { - return context.getData(Constants.STRUCTURED_MESSAGE_DECODER_STATE_REF_CONTEXT_KEY) - .filter(AtomicReference.class::isInstance) - .map(obj -> (AtomicReference) obj); - } - - private Flux decodeStream(Flux encodedFlux, DecoderState state) { - StructuredMessageDecoder decoder = state.getDecoder(); - - return encodedFlux.concatMap(buffer -> decodeBuffer(buffer, state, decoder)) - .onErrorResume(throwable -> handleStreamError(throwable, state, decoder)) - .concatWith(Mono.defer(() -> handleStreamCompletion(state, decoder))); + private Flux decodeStream(Flux encodedFlux, StructuredMessageDecoder decoder) { + return encodedFlux.concatMap(buffer -> decodeBuffer(buffer, decoder)) + .onErrorResume(throwable -> handleStreamError(throwable, decoder)) + .concatWith(Mono.defer(() -> handleStreamCompletion(decoder))); } - private Flux decodeBuffer(ByteBuffer buffer, DecoderState state, StructuredMessageDecoder decoder) { - + private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecoder decoder) { if (decoder.isComplete()) { LOGGER.atVerbose() .addKeyValue("bufferLength", buffer == null ? "null" : buffer.remaining()) @@ -170,82 +129,45 @@ private Flux decodeBuffer(ByteBuffer buffer, DecoderState state, Str .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) .log("Decode chunk result"); - state.updateProgress(); - switch (result.getStatus()) { case SUCCESS: case NEED_MORE_BYTES: - return handleSuccessOrNeedMore(state, result); + return emitDecodedPayload(result.getDecodedPayload()); case COMPLETED: - return handleCompleted(state, result); + return emitDecodedPayload(result.getDecodedPayload()); case INVALID: - return handleInvalid(state, result); + LOGGER.error("Invalid data during decode: {}", result.getMessage()); + return Flux.error(new IOException("Failed to decode structured message: " + result.getMessage())); default: return Flux.error(new IllegalStateException("Unknown decode status: " + result.getStatus())); } } catch (Exception e) { LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); - return Flux.error(createRetryableException(state, e.getMessage(), e)); + return Flux.error(new IOException("Failed to decode structured message chunk: " + e.getMessage(), e)); } } - private Flux handleSuccessOrNeedMore(DecoderState state, StructuredMessageDecoder.DecodeResult result) { - return emitDecodedPayload(state, result.getDecodedPayload()); - } - - private Flux handleCompleted(DecoderState state, StructuredMessageDecoder.DecodeResult result) { - return emitDecodedPayload(state, result.getDecodedPayload()); - } - - private Flux handleInvalid(DecoderState state, StructuredMessageDecoder.DecodeResult result) { - LOGGER.error("Invalid data during decode: {}", result.getMessage()); - return Flux - .error(createRetryableException(state, "Failed to decode structured message: " + result.getMessage())); - } - - private Flux handleStreamError(Throwable throwable, DecoderState state, - StructuredMessageDecoder decoder) { - + private Flux handleStreamError(Throwable throwable, StructuredMessageDecoder decoder) { if (decoder.isComplete()) { LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); return Flux.empty(); } - state.addSegmentsToAggregateIfNeeded(); - if (throwable instanceof IOException - && throwable.getMessage() != null - && throwable.getMessage().contains(ContentValidationDecoderUtils.RETRY_OFFSET_TOKEN)) { - return Flux.error(throwable); - } - - return Flux.error(createRetryableException(state, throwable.getMessage(), throwable)); + return Flux.error(throwable); } - private Mono handleStreamCompletion(DecoderState state, StructuredMessageDecoder decoder) { + private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { if (!decoder.isComplete()) { LOGGER.atInfo() .addKeyValue("messageOffset", decoder.getMessageOffset()) .addKeyValue("messageLength", decoder.getMessageLength()) .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) - .log("Stream ended but decode not finalized - throwing retryable exception"); - return Mono.error(createRetryableException(state, - "Stream ended prematurely before structured message decoding completed")); - } - - state.addSegmentsToAggregateIfNeeded(); - - if (state.aggregateCrcState != null && state.aggregateCrcState.hasSegments()) { - long composed = state.aggregateCrcState.composeCrc(); - long calculated = state.aggregateCrcState.getRunningCrc(); - if (composed != calculated) { - return Mono.error(LOGGER.logExceptionAsError( - new IllegalArgumentException("CRC64 mismatch detected in composed structured message. Expected: " - + composed + ", got: " + calculated))); - } + .log("Stream ended but decode not finalized"); + return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); } LOGGER.atInfo() @@ -255,61 +177,15 @@ private Mono handleStreamCompletion(DecoderState state, StructuredMe return Mono.empty(); } - private Flux emitDecodedPayload(DecoderState state, ByteBuffer decodedPayload) { + private static Flux emitDecodedPayload(ByteBuffer decodedPayload) { if (decodedPayload == null || !decodedPayload.hasRemaining()) { return Flux.empty(); } - long skip = state.decodedBytesToSkip.get(); - if (skip > 0) { - if (skip >= decodedPayload.remaining()) { - state.decodedBytesToSkip.addAndGet(-decodedPayload.remaining()); - return Flux.empty(); - } else { - int skipCount = (int) skip; - decodedPayload.position(decodedPayload.position() + skipCount); - decodedPayload = decodedPayload.slice(); - state.decodedBytesToSkip.addAndGet(-skipCount); - } - } - - if (!decodedPayload.hasRemaining()) { - return Flux.empty(); - } - ByteBuffer copy = ByteBuffer.allocate(decodedPayload.remaining()); copy.put(decodedPayload.duplicate()); copy.flip(); - state.totalBytesDecoded.addAndGet(copy.remaining()); - if (state.aggregateCrcState != null) { - state.aggregateCrcState.appendPayload(copy.asReadOnlyBuffer()); - } return Flux.just(copy); } - - private IOException createRetryableException(DecoderState state, String message) { - return createRetryableException(state, message, null); - } - - private IOException createRetryableException(DecoderState state, String message, Throwable cause) { - StructuredMessageDecoder decoder = state.getDecoder(); - long retryOffset = state.getRetryOffset(); - long decodedSoFar = state.totalBytesDecoded.get(); - long expectedLength = decoder.getMessageLength(); - String originalMessage = message != null ? message : ""; - long displayExpected = expectedLength > 0 ? expectedLength : 0; - - String fullMessage - = String.format("Incomplete structured message: decoded %d of %d bytes. %s%d. %s", decodedSoFar, - displayExpected, ContentValidationDecoderUtils.RETRY_OFFSET_TOKEN, retryOffset, originalMessage); - - LOGGER.atInfo() - .addKeyValue("retryOffset", retryOffset) - .addKeyValue("decodedSoFar", decodedSoFar) - .addKeyValue("expectedLength", expectedLength) - .log("Creating retryable exception with offset"); - - return cause != null ? new IOException(fullMessage, cause) : new IOException(fullMessage); - } } diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java deleted file mode 100644 index 6d9a88a8f2ba..000000000000 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTest.java +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.common.policy; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -/** - * Unit tests for parsing helpers used by StorageContentValidationDecoderPolicy. - */ -public class StorageContentValidationDecoderPolicyTest { - - @Test - public void parseRetryStartOffsetFromValidMessage() { - String message - = "Incomplete structured message: decoded 512 of 1081 bytes. RETRY-START-OFFSET=287. Stream ended"; - long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); - assertEquals(287, offset); - } - - @Test - public void parseRetryStartOffsetFromMessageWithLargeOffset() { - String message = "RETRY-START-OFFSET=9999999999"; - long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); - assertEquals(9999999999L, offset); - } - - @Test - public void parseRetryStartOffsetFromMessageWithZeroOffset() { - String message = "Some error. RETRY-START-OFFSET=0. Details"; - long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); - assertEquals(0, offset); - } - - @Test - public void parseRetryStartOffsetReturnsNegativeOneForNullMessage() { - long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(null); - assertEquals(-1, offset); - } - - @Test - public void parseRetryStartOffsetReturnsNegativeOneForMissingToken() { - String message = "Some error without retry offset"; - long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); - assertEquals(-1, offset); - } - - @Test - public void parseRetryStartOffsetReturnsNegativeOneForEmptyMessage() { - long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(""); - assertEquals(-1, offset); - } - - @Test - public void parseRetryStartOffsetReturnsNegativeOneForMalformedToken() { - String message = "RETRY-START-OFFSET=abc"; - long offset = StorageContentValidationDecoderPolicy.parseRetryStartOffset(message); - assertEquals(-1, offset); - } - - @Test - public void parseDecoderOffsetsFromEnrichedMessage() { - String message = "Invalid segment size [decoderOffset=523,lastCompleteSegment=287]"; - long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); - assertArrayEquals(new long[] { 523, 287 }, offsets); - } - - @Test - public void parseDecoderOffsetsWithZeroValues() { - String message = "Header error [decoderOffset=0,lastCompleteSegment=0]"; - long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); - assertArrayEquals(new long[] { 0, 0 }, offsets); - } - - @Test - public void parseDecoderOffsetsWithLargeValues() { - String message = "Error [decoderOffset=9999999999,lastCompleteSegment=8888888888]"; - long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); - assertArrayEquals(new long[] { 9999999999L, 8888888888L }, offsets); - } - - @Test - public void parseDecoderOffsetsReturnsNullForMissingPattern() { - String message = "Error without decoder offset information"; - long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); - assertNull(offsets); - } - - @Test - public void parseDecoderOffsetsReturnsNullForNullMessage() { - long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(null); - assertNull(offsets); - } - - @Test - public void parseDecoderOffsetsReturnsNullForMalformedPattern() { - String message = "[decoderOffset=abc,lastCompleteSegment=xyz]"; - long[] offsets = ContentValidationDecoderUtils.parseDecoderOffsets(message); - assertNull(offsets); - } -} From 5b153a4fe31fbc22a030a68d787af8cce20eebd8 Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Wed, 8 Apr 2026 19:00:36 +0530 Subject: [PATCH 16/31] removing dead code --- .../StorageCrc64Calculator.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java index 119f83c1f546..170de1a9afdb 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java @@ -2426,7 +2426,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { long uSize = length; long uBytes, uStop; - uCrc = ~uCrc; + uCrc = ~uCrc; // Flip all bits of uCrc uStop = uSize - (uSize % 32); if (uStop >= 2 * 32) { @@ -2442,11 +2442,15 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { ByteBuffer buffer = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); for (; pData < pLast; pData += 32) { - long b0 = buffer.getLong(pData) ^ uCrc0; - long b1 = buffer.getLong(pData + 8) ^ uCrc1; - long b2 = buffer.getLong(pData + 16) ^ uCrc2; - long b3 = buffer.getLong(pData + 24) ^ uCrc3; + long b0, b1, b2, b3; + // Load and XOR data with CRC + b0 = buffer.getLong(pData) ^ uCrc0; + b1 = buffer.getLong(pData + 8) ^ uCrc1; + b2 = buffer.getLong(pData + 16) ^ uCrc2; + b3 = buffer.getLong(pData + 24) ^ uCrc3; + + // Unsigned updates using tables and masking uCrc0 = M_U32[7 * 256 + ((int) (b0 & 0xFF))]; b0 >>>= 8; uCrc1 = M_U32[7 * 256 + ((int) (b1 & 0xFF))]; @@ -2516,6 +2520,7 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { uCrc3 ^= M_U32[((int) (b3 & 0xFF))]; } + // Combine CRC values uCrc = 0; uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; @@ -2560,11 +2565,12 @@ public static long compute(byte[] src, int offset, int length, long uCrc) { pData += 32; } + // Process remaining bytes for (uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[pData]) & 0xFF)]; } - return ~uCrc; + return ~uCrc; // Flip all bits of uCrc and return as long } /** From f72cb8b11d70c02264496a63c2709bc50a8927ba Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Fri, 17 Apr 2026 10:11:09 +0530 Subject: [PATCH 17/31] addressing Kyle's review comments --- .../util/ChunkedDownloadUtils.java | 31 +- .../StructuredMessageDecoder.java | 267 +----------------- ...StorageContentValidationDecoderPolicy.java | 11 +- .../StructuredMessageDecoderTests.java | 51 +--- 4 files changed, 31 insertions(+), 329 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java index a5845408a36b..0505b6a3f6f6 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ChunkedDownloadUtils.java @@ -34,7 +34,7 @@ public class ChunkedDownloadUtils { public static Mono> downloadFirstChunk( BlobRange range, ParallelTransferOptions parallelTransferOptions, BlobRequestConditions requestConditions, BiFunction> downloader, boolean eTagLock) { - return downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloader, null, eTagLock, null); + return downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloader, eTagLock, null); } /* @@ -48,22 +48,6 @@ public static Mono> downloader, boolean eTagLock, Context context) { - return downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloader, null, eTagLock, - context); - } - - /* - Has a context value for additional download adjustments and an optional fallback downloader for empty blobs. - - Download the first chunk. Construct a Mono which will emit the total count for calculating the number of chunks, - access conditions containing the etag to lock on, and the response from downloading the first chunk. - */ - @SuppressWarnings("unchecked") - public static Mono> downloadFirstChunk( - BlobRange range, ParallelTransferOptions parallelTransferOptions, BlobRequestConditions requestConditions, - BiFunction> downloader, - BiFunction> emptyBlobDownloader, - boolean eTagLock, Context context) { // We will scope our initial download to either be one chunk or the total size. long initialChunkSize = range.getCount() != null && range.getCount() < parallelTransferOptions.getBlockSizeLong() @@ -85,12 +69,7 @@ public static Mono contextAdjustment = context.getData(Constants.ADJUSTED_BLOB_LENGTH_KEY); @@ -120,9 +99,7 @@ public static Mono> fallbackDownloader - = emptyBlobDownloader != null ? emptyBlobDownloader : downloader; - return fallbackDownloader.apply(new BlobRange(0, 0L), requestConditions) + return downloader.apply(new BlobRange(0, 0L), requestConditions) // Subscribe on boundElastic instead of elastic as elastic is deprecated and boundElastic // provided the same functionality with the added benefit that it won't infinitely create // threads if needed and will instead queue. @@ -166,7 +143,7 @@ public static Flux downloadChunk(Integer chunkNum, BlobDownloadAsyncRespo private static BlobRequestConditions setEtag(BlobRequestConditions requestConditions, String etag) { // We don't want to modify the user's object, so we'll create a duplicate and set the retrieved etag. return new BlobRequestConditions().setIfModifiedSince(requestConditions.getIfModifiedSince()) - .setIfUnmodifiedSince(requestConditions.getIfUnmodifiedSince()) + .setIfUnmodifiedSince(requestConditions.getIfModifiedSince()) .setIfMatch(etag) .setIfNoneMatch(requestConditions.getIfNoneMatch()) .setLeaseId(requestConditions.getLeaseId()); diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 977e4a78f105..eb77efd2011b 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -4,14 +4,11 @@ package com.azure.storage.common.implementation.contentvalidation; import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; @@ -45,7 +42,6 @@ public class StructuredMessageDecoder { // Offset tracking private long messageOffset = 0; // Absolute encoded bytes consumed from the message private long totalDecodedPayloadBytes = 0; // Total decoded (payload) bytes output - private long decodedBytesAtLastCompleteSegment = 0; // Current segment state private int currentSegmentNumber = 0; @@ -55,11 +51,8 @@ public class StructuredMessageDecoder { // CRC validation private long messageCrc64 = 0; - private long messageCrc64AtLastCompleteSegment = 0; private long segmentCrc64 = 0; - private final Map segmentCrcs = new HashMap<>(); private final Map segmentLengths = new HashMap<>(); - private final List completedSegments = new ArrayList<>(); // Smart retry tracking - lastCompleteSegmentStart is the absolute offset where the last // fully completed segment ended. This is the safe retry boundary. @@ -72,8 +65,6 @@ public class StructuredMessageDecoder { * Decode result status codes. */ public enum DecodeStatus { - /** Decoding succeeded, more data may be available */ - SUCCESS, /** Need more bytes to continue (partial header/segment) */ NEED_MORE_BYTES, /** Decoding completed successfully */ @@ -134,19 +125,6 @@ public long getLastCompleteSegmentStart() { return lastCompleteSegmentStart; } - /** - * Returns the canonical absolute byte index (0-based) that should be used to resume a failed/incomplete download. - * This MUST be used directly as the Range header start value: "Range: bytes={retryStartOffset}-" - * - *

This is equivalent to {@link #getLastCompleteSegmentStart()} but provides a clearer semantic name - * for the smart retry use case.

- * - * @return The absolute byte index for the retry start offset. - */ - public long getRetryStartOffset() { - return getLastCompleteSegmentStart(); - } - /** * Gets the current message offset (total bytes consumed from the structured message). * @@ -165,68 +143,6 @@ public long getTotalDecodedPayloadBytes() { return totalDecodedPayloadBytes; } - /** - * Gets the total decoded payload bytes at the last complete segment boundary. - * - * @return The decoded byte count at the last complete segment boundary. - */ - public long getDecodedBytesAtLastCompleteSegment() { - return decodedBytesAtLastCompleteSegment; - } - - /** - * Advances the message offset by the specified number of bytes. - * This should be called after consuming an encoded segment to maintain - * the authoritative encoded offset. - * - * @param bytes The number of bytes to advance. - */ - public void advanceMessageOffset(long bytes) { - long priorOffset = messageOffset; - messageOffset += bytes; - LOGGER.atInfo() - .addKeyValue("priorOffset", priorOffset) - .addKeyValue("bytesAdvanced", bytes) - .addKeyValue("newOffset", messageOffset) - .log("Advanced message offset"); - } - - /** - * Resets the decoder position to the last complete segment boundary. - * This is used during smart retry to ensure the decoder is in sync with - * the data being provided from the retry offset. - */ - public void resetToLastCompleteSegment() { - boolean needsReset = messageOffset != lastCompleteSegmentStart - || pendingBytes.size() > 0 - || currentSegmentContentOffset != 0 - || currentSegmentContentLength != 0 - || currentSegmentNumber != lastCompleteSegmentNumber; - if (needsReset) { - LOGGER.atInfo() - .addKeyValue("fromOffset", messageOffset) - .addKeyValue("toOffset", lastCompleteSegmentStart) - .addKeyValue("currentSegmentNum", currentSegmentNumber) - .addKeyValue("currentSegmentContentOffset", currentSegmentContentOffset) - .addKeyValue("currentSegmentContentLength", currentSegmentContentLength) - .log("Resetting decoder to last complete segment boundary"); - messageOffset = lastCompleteSegmentStart; - totalDecodedPayloadBytes = decodedBytesAtLastCompleteSegment; - messageCrc64 = messageCrc64AtLastCompleteSegment; - // Reset current segment state - next decode will read the segment header - currentSegmentContentOffset = 0; - currentSegmentContentLength = 0; - currentSegmentNumber = lastCompleteSegmentNumber; - segmentCrc64 = 0; - // Clear any pending bytes since we're resetting to a known boundary - pendingBytes.reset(); - } else { - LOGGER.atVerbose() - .addKeyValue("offset", messageOffset) - .log("Decoder already at last complete segment boundary, no reset needed"); - } - } - /** * Converts a ByteBuffer range to hex string for diagnostic purposes. */ @@ -304,41 +220,6 @@ private void appendToPending(ByteBuffer buffer) { } } - /** - * Peeks the next segment length without consuming from the buffer. - * Used by the policy to calculate encoded segment size before slicing. - * - * @param buffer The buffer to peek from. - * @param relativeIndex The position in the buffer to start reading from. - * @return The segment content length, or -1 if not enough bytes. - */ - public long peekNextSegmentLength(ByteBuffer buffer, int relativeIndex) { - // Need at least V1_SEGMENT_HEADER_LENGTH bytes to read segment number (2) + segment size (8) - if (relativeIndex + V1_SEGMENT_HEADER_LENGTH > buffer.limit()) { - return -1; - } - // Segment size is at offset 2 (after segment number which is 2 bytes) - return buffer.getLong(relativeIndex + 2); - } - - /** - * Gets the flags for the current message (needed to determine if CRC is present). - * - * @return The message flags, or null if header not yet read. - */ - public StructuredMessageFlags getFlags() { - return flags; - } - - /** - * Gets the completed segments in decode order. - * - * @return List of completed segment CRCs and lengths. - */ - public List getCompletedSegments() { - return new ArrayList<>(completedSegments); - } - /** * Gets the expected message length from the header. * @@ -348,38 +229,6 @@ public long getMessageLength() { return messageLength; } - /** - * Gets the number of segments from the header. - * - * @return The number of segments, or -1 if header not yet read. - */ - public int getNumSegments() { - return numSegments; - } - - /** - * Checks if the message header has been read. - * - * @return true if header has been read, false otherwise. - */ - public boolean isHeaderRead() { - return messageLength != -1; - } - - /** - * Gets the number of encoded bytes that have been seen but not yet fully - * processed by the decoder (pending bytes). - * - *

This is used by smart-retry logic to determine the absolute encoded - * offset that a retry request should start from while still preserving - * the decoder's buffered state.

- * - * @return The number of pending encoded bytes buffered by the decoder. - */ - public int getPendingEncodedByteCount() { - return pendingBytes.size(); - } - /** * Reads the message header if we have enough bytes. * @@ -393,7 +242,7 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { int available = getAvailableBytes(buffer); if (available < V1_HEADER_LENGTH) { - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("available", available) .addKeyValue("required", V1_HEADER_LENGTH) .addKeyValue("pendingBytes", pendingBytes.size()) @@ -428,7 +277,7 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { messageOffset += V1_HEADER_LENGTH; messageLength = msgLen; - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("messageLength", messageLength) .addKeyValue("numSegments", numSegments) .addKeyValue("flags", flags) @@ -447,7 +296,7 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { private boolean tryReadSegmentHeader(ByteBuffer buffer) { int available = getAvailableBytes(buffer); if (available < V1_SEGMENT_HEADER_LENGTH) { - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("available", available) .addKeyValue("required", V1_SEGMENT_HEADER_LENGTH) .addKeyValue("pendingBytes", pendingBytes.size()) @@ -460,7 +309,7 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { ByteBuffer combined = getCombinedBuffer(buffer); // Log the raw bytes we're about to read - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("decoderOffset", messageOffset) .addKeyValue("bufferPos", combined.position()) .addKeyValue("bufferRemaining", combined.remaining()) @@ -499,7 +348,7 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { segmentCrc64 = 0; } - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("segmentNum", segmentNum) .addKeyValue("segmentLength", segmentSize) .addKeyValue("decoderOffset", messageOffset) @@ -560,7 +409,7 @@ private boolean tryReadSegmentFooter(ByteBuffer buffer) { if (flags == StructuredMessageFlags.STORAGE_CRC64) { int available = getAvailableBytes(buffer); if (available < CRC64_LENGTH) { - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("available", available) .addKeyValue("required", CRC64_LENGTH) .addKeyValue("segmentNum", currentSegmentNumber) @@ -579,18 +428,13 @@ private boolean tryReadSegmentFooter(ByteBuffer buffer) { } consumeBytes(CRC64_LENGTH, buffer); - segmentCrcs.put(currentSegmentNumber, segmentCrc64); - long length = segmentLengths.getOrDefault(currentSegmentNumber, currentSegmentContentLength); - completedSegments.add(new SegmentInfo(segmentCrc64, length)); messageOffset += CRC64_LENGTH; } // Mark that this segment is complete lastCompleteSegmentStart = messageOffset; - decodedBytesAtLastCompleteSegment = totalDecodedPayloadBytes; - messageCrc64AtLastCompleteSegment = messageCrc64; lastCompleteSegmentNumber = currentSegmentNumber; - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("segmentNum", currentSegmentNumber) .addKeyValue("offset", lastCompleteSegmentStart) .addKeyValue("segmentLength", currentSegmentContentLength) @@ -614,7 +458,7 @@ private boolean tryReadMessageFooter(ByteBuffer buffer) { if (flags == StructuredMessageFlags.STORAGE_CRC64) { int available = getAvailableBytes(buffer); if (available < CRC64_LENGTH) { - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("available", available) .addKeyValue("required", CRC64_LENGTH) .log("Not enough bytes for message CRC footer, waiting for more"); @@ -650,7 +494,7 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { ByteArrayOutputStream decodedContent = new ByteArrayOutputStream(); int startPos = buffer.position(); - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("newBytes", buffer.remaining()) .addKeyValue("pendingBytes", pendingBytes.size()) .addKeyValue("decoderOffset", messageOffset) @@ -667,7 +511,7 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { while (messageOffset < messageLength) { // Read segment header only after the *previous* segment is fully complete (including its CRC footer). // Otherwise we would misinterpret segment N's footer bytes as segment N+1's header when the footer - // is split across network buffers (e.g. "Unexpected segment number. Expected: 2, got: 10710"). + // is split across network buffers if (lastCompleteSegmentNumber == currentSegmentNumber && currentSegmentNumber < numSegments) { if (!tryReadSegmentHeader(buffer)) { break; // Need more bytes for segment header @@ -690,7 +534,7 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { // Check if all segments are complete if (currentSegmentNumber == numSegments && messageOffset >= messageLength) { - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("messageOffset", messageOffset) .addKeyValue("messageLength", messageLength) .addKeyValue("totalDecodedPayload", totalDecodedPayloadBytes) @@ -724,74 +568,6 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { } } - /** - * Decodes the structured message from the given buffer up to the specified size. - * This is a convenience method that wraps decodeChunk for backwards compatibility. - * - * @param buffer The buffer containing the structured message. - * @param size The maximum number of bytes to decode. - * @return A ByteBuffer containing the decoded message content. - * @throws IllegalArgumentException if the buffer does not contain a valid structured message. - */ - public ByteBuffer decode(ByteBuffer buffer, int size) { - buffer.order(ByteOrder.LITTLE_ENDIAN); - ByteArrayOutputStream decodedContent = new ByteArrayOutputStream(); - - if (messageOffset == 0) { - if (!tryReadMessageHeader(buffer)) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException( - enrichExceptionMessage("Content not long enough to contain a valid message header."))); - } - } - - while (buffer.hasRemaining() && decodedContent.size() < size) { - // Only read next segment header after previous segment is fully complete (including footer). - if (lastCompleteSegmentNumber == currentSegmentNumber && currentSegmentNumber < numSegments) { - if (!tryReadSegmentHeader(buffer)) { - break; // Need more bytes - } - } - - tryReadSegmentContent(buffer, decodedContent); - - if (currentSegmentContentOffset == currentSegmentContentLength) { - if (!tryReadSegmentFooter(buffer)) { - break; // Need more bytes - } - } - } - - return ByteBuffer.wrap(decodedContent.toByteArray()); - } - - /** - * Decodes the entire structured message from the given buffer. - * - * @param buffer The buffer containing the structured message. - * @return A ByteBuffer containing the decoded message content. - * @throws IllegalArgumentException if the buffer does not contain a valid structured message. - */ - public ByteBuffer decode(ByteBuffer buffer) { - return decode(buffer, buffer.remaining()); - } - - /** - * Finalizes the decoding process and returns any final decoded bytes still buffered internally. - * The policy should aggregate decoded byte counts and perform the final length comparison. - * - * @return A ByteBuffer containing any final decoded bytes, or null if none remain. - * @throws IllegalArgumentException if the encoded message offset doesn't match expected length. - */ - public ByteBuffer finalizeDecoding() { - if (messageOffset != messageLength) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException( - enrichExceptionMessage("Decoded message length does not match expected length. Expected: " - + messageLength + ", but was: " + messageOffset))); - } - // No buffered decoded bytes in current implementation - return null; - } - /** * Checks if decoding is complete. * @@ -801,27 +577,6 @@ public boolean isComplete() { return messageLength != -1 && messageOffset >= messageLength; } - /** - * Represents a completed segment with CRC and length. - */ - public static final class SegmentInfo { - private final long crc64; - private final long length; - - public SegmentInfo(long crc64, long length) { - this.crc64 = crc64; - this.length = length; - } - - public long getCrc64() { - return crc64; - } - - public long getLength() { - return length; - } - } - /** * Enriches an exception message with decoder offset information for debugging and retry. * Format: "original message [decoderOffset=X,lastCompleteSegment=Y]" diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 418c7c249375..d19d6ea4ed0c 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -112,7 +112,7 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode return Flux.empty(); } - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("newBytes", buffer.remaining()) .addKeyValue("decoderOffset", decoder.getMessageOffset()) .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) @@ -122,7 +122,7 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode try { StructuredMessageDecoder.DecodeResult result = decoder.decodeChunk(buffer); - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("status", result.getStatus()) .addKeyValue("bytesConsumed", result.getBytesConsumed()) .addKeyValue("decoderOffset", decoder.getMessageOffset()) @@ -130,7 +130,6 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode .log("Decode chunk result"); switch (result.getStatus()) { - case SUCCESS: case NEED_MORE_BYTES: return emitDecodedPayload(result.getDecodedPayload()); @@ -152,7 +151,7 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode private Flux handleStreamError(Throwable throwable, StructuredMessageDecoder decoder) { if (decoder.isComplete()) { - LOGGER.atInfo().log("Decoder complete; suppressing downstream error and completing successfully"); + LOGGER.atVerbose().log("Decoder complete; suppressing downstream error and completing successfully"); return Flux.empty(); } @@ -161,7 +160,7 @@ private Flux handleStreamError(Throwable throwable, StructuredMessag private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { if (!decoder.isComplete()) { - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("messageOffset", decoder.getMessageOffset()) .addKeyValue("messageLength", decoder.getMessageLength()) .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) @@ -170,7 +169,7 @@ private Mono handleStreamCompletion(StructuredMessageDecoder decoder return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); } - LOGGER.atInfo() + LOGGER.atVerbose() .addKeyValue("messageOffset", decoder.getMessageOffset()) .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) .log("Stream complete and decode finalized successfully"); diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java index 9bdedc7dbe62..28efdcc720cc 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -12,9 +12,6 @@ import java.nio.ByteOrder; import java.util.concurrent.ThreadLocalRandom; -import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; -import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; -import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -49,7 +46,9 @@ public void readsCompleteMessageInSingleChunk() throws IOException { int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - ByteBuffer result = decoder.decode(encodedData); + StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); + assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); + ByteBuffer result = decodeResult.getDecodedPayload(); assertNotNull(result); byte[] decodedData = new byte[result.remaining()]; @@ -140,7 +139,9 @@ public void handlesZeroLengthSegment() throws IOException { int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - ByteBuffer result = decoder.decode(encodedData); + StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); + assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); + ByteBuffer result = decodeResult.getDecodedPayload(); assertNotNull(result); assertEquals(1, result.remaining()); @@ -163,8 +164,8 @@ public void tracksLastCompleteSegmentCorrectly() throws IOException { // Initially lastCompleteSegmentStart should be 0 assertEquals(0, decoder.getLastCompleteSegmentStart()); - // Decode the entire message - decoder.decode(encodedData); + StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); + assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); // After complete decode, lastCompleteSegmentStart should point to end of last segment // (before message footer, if any) @@ -175,38 +176,6 @@ public void tracksLastCompleteSegmentCorrectly() throws IOException { assertTrue(decoder.getLastCompleteSegmentStart() > 0); } - @Test - public void resetToLastCompleteSegmentWorks() throws IOException { - // Test: Verify reset functionality for smart retry - byte[] originalData = new byte[512]; - ThreadLocalRandom.current().nextBytes(originalData); - - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); - int encodedLength = encodedData.remaining(); - byte[] encodedBytes = new byte[encodedLength]; - encodedData.get(encodedBytes); - - // Parse first segment completely, then simulate interruption - // First segment ends after: header(13) + segment_header(10) + content(256) + crc(8) = 287 - int firstSegmentEnd = V1_HEADER_LENGTH + V1_SEGMENT_HEADER_LENGTH + 256 + CRC64_LENGTH; - ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, firstSegmentEnd + 5); // 5 bytes into second segment header - chunk1.order(ByteOrder.LITTLE_ENDIAN); - - StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - decoder.decodeChunk(chunk1); - - // lastCompleteSegmentStart should be at end of first segment - long lastComplete = decoder.getLastCompleteSegmentStart(); - assertTrue(lastComplete > 0); - assertEquals(firstSegmentEnd, lastComplete); - - // Reset to last complete segment - decoder.resetToLastCompleteSegment(); - assertEquals(lastComplete, decoder.getMessageOffset()); - } - @Test public void multipleChunksDecode() throws IOException { // Test: Decode message across multiple small chunks @@ -259,7 +228,9 @@ public void decodeWithNoCrc() throws IOException { int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - ByteBuffer result = decoder.decode(encodedData); + StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); + assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); + ByteBuffer result = decodeResult.getDecodedPayload(); assertNotNull(result); byte[] decodedData = new byte[result.remaining()]; From 20e33039883686401159690f641fcd9ed9c6313f Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Fri, 17 Apr 2026 10:39:38 +0530 Subject: [PATCH 18/31] addressing latest review comments --- .../models/BlobDownloadAsyncResponse.java | 25 ++++++------------- .../blob/specialized/BlobAsyncClientBase.java | 7 ++++++ .../BlobMessageAsyncDecoderDownloadTests.java | 16 ++++++------ .../policy/MockPartialResponsePolicy.java | 2 -- .../StructuredMessageDecoderTests.java | 12 ++------- 5 files changed, 24 insertions(+), 38 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java index 2eebcb44bdcf..c1c62368d093 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadAsyncResponse.java @@ -13,7 +13,6 @@ import com.azure.storage.blob.implementation.accesshelpers.BlobDownloadAsyncResponseConstructorProxy; import com.azure.storage.blob.implementation.models.BlobsDownloadHeaders; import com.azure.storage.blob.implementation.util.ModelHelper; -import com.azure.core.util.logging.ClientLogger; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -30,18 +29,8 @@ public final class BlobDownloadAsyncResponse extends ResponseBase> implements Closeable { - private static final ClientLogger LOGGER = new ClientLogger(BlobDownloadAsyncResponse.class); - static { - BlobDownloadAsyncResponseConstructorProxy - .setAccessor(new BlobDownloadAsyncResponseConstructorProxy.BlobDownloadAsyncResponseConstructorAccessor() { - @Override - public BlobDownloadAsyncResponse create(StreamResponse sourceResponse, - BiFunction> onErrorResume, - DownloadRetryOptions retryOptions) { - return new BlobDownloadAsyncResponse(sourceResponse, onErrorResume, retryOptions); - } - }); + BlobDownloadAsyncResponseConstructorProxy.setAccessor(BlobDownloadAsyncResponse::new); } private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0); @@ -67,6 +56,13 @@ public BlobDownloadAsyncResponse(HttpRequest request, int statusCode, HttpHeader this.retryOptions = null; } + /** + * Constructs a {@link BlobDownloadAsyncResponse}. + * + * @param sourceResponse The initial Stream Response + * @param onErrorResume Function used to resume. + * @param retryOptions Retry options. + */ BlobDownloadAsyncResponse(StreamResponse sourceResponse, BiFunction> onErrorResume, DownloadRetryOptions retryOptions) { super(sourceResponse.getRequest(), sourceResponse.getStatusCode(), sourceResponse.getHeaders(), @@ -99,12 +95,7 @@ private static Flux createResponseFlux(StreamResponse sourceResponse */ public Mono writeValueToAsync(AsynchronousByteChannel channel, ProgressReporter progressReporter) { Objects.requireNonNull(channel, "'channel' must not be null"); - LOGGER.atVerbose() - .addKeyValue("thread", Thread.currentThread().getName()) - .log("BlobDownloadAsyncResponse.writeValueToAsync entry"); if (sourceResponse != null) { - LOGGER.atVerbose() - .log("BlobDownloadAsyncResponse.writeValueToAsync using sourceResponse (IOUtils.transfer)"); return IOUtils.transferStreamResponseToAsynchronousByteChannel(channel, sourceResponse, onErrorResume, progressReporter, retryOptions.getMaxRetryRequests()); } else if (super.getValue() != null) { diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 42122ab4e6b6..c0c59e81defb 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1283,6 +1283,10 @@ Mono downloadStreamWithResponseInternal(BlobRange ran BlobDownloadHeaders blobDownloadHeaders = ModelHelper.populateBlobDownloadHeaders(blobsDownloadHeaders, ModelHelper.getErrorCode(response.getHeaders())); + /* + * If the customer did not specify a count, they are reading to the end of the blob. Extract this value + * from the response for better book-keeping towards the end. + */ long finalCount; long initialOffset = finalRange.getOffset(); if (finalRange.getCount() == null) { @@ -1292,6 +1296,8 @@ Mono downloadStreamWithResponseInternal(BlobRange ran finalCount = finalRange.getCount(); } + // The resume function takes throwable and offset at the destination. + // I.e. offset is relative to the starting point. BiFunction> onDownloadErrorResume = (throwable, offset) -> { if (!(throwable instanceof IOException || throwable instanceof TimeoutException)) { return Mono.error(throwable); @@ -1539,6 +1545,7 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti BlobRequestConditions finalConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); + // Default behavior is not to overwrite Set openOptions = options.getOpenOptions(); if (openOptions == null) { openOptions = DEFAULT_OPEN_OPTIONS_SET; diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java index 2f2b59e0567b..fa5f1292c3e8 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java @@ -252,15 +252,13 @@ public void interruptAndVerifyProperRewind() throws IOException { DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); - StepVerifier.create(downloadClient - .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) - .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) - .doFinally(signalType -> { - System.out.println("[MockPartialResponsePolicy] hits=" + mockPolicy.getHits() + ", triesRemaining=" - + mockPolicy.getTriesRemaining() + ", ranges=" + mockPolicy.getRangeHeaders()); - assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked"); - }) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + StepVerifier + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setResponseChecksumAlgorithm(StorageChecksumAlgorithm.CRC64)) + .doFinally( + signalType -> assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked")) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) .verifyComplete(); diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index ccef4a5505d3..347d3ac11a59 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -94,8 +94,6 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN return Mono.just(response); } hits.incrementAndGet(); - System.out.println("[MockPartialResponsePolicy] invoked. tries=" + remainingTries - + ", maxBytesPerResponse=" + maxBytesPerResponse); Flux limitedBody = limitStreamToBytes(response.getBody(), maxBytesPerResponse); return Mono.just( diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java index 28efdcc720cc..10e7f7ab78ec 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -3,6 +3,7 @@ package com.azure.storage.common.implementation.contentvalidation; +import com.azure.core.util.FluxUtil; import org.junit.jupiter.api.Test; import reactor.core.publisher.Flux; @@ -25,13 +26,7 @@ public class StructuredMessageDecoderTests { private static ByteBuffer collectFlux(Flux flux) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - flux.toIterable().forEach(buf -> { - byte[] bytes = new byte[buf.remaining()]; - buf.get(bytes); - out.write(bytes, 0, bytes.length); - }); - return ByteBuffer.wrap(out.toByteArray()).order(ByteOrder.LITTLE_ENDIAN); + return ByteBuffer.wrap(FluxUtil.collectBytesInByteBufferStream(flux).block()).order(ByteOrder.LITTLE_ENDIAN); } @Test @@ -127,9 +122,6 @@ public void readsSegmentHeaderSplitAcrossChunks() throws IOException { public void handlesZeroLengthSegment() throws IOException { // Test: Zero-length segment should decode correctly // Note: Zero-length segments are valid in the format - byte[] originalData = new byte[0]; - - // For zero-length data, encoder behavior varies - let's test with minimal data byte[] minimalData = new byte[1]; ThreadLocalRandom.current().nextBytes(minimalData); From f9befffcc71bfdfd962125a346cf5904c0ced05b Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Wed, 22 Apr 2026 19:30:14 +0530 Subject: [PATCH 19/31] refactoring based on latest review comments --- .../util/DownloadValidationUtils.java | 51 ----- .../blob/specialized/BlobAsyncClientBase.java | 52 ++--- .../blob/BlobMessageDecoderDownloadTests.java | 6 +- .../common/implementation/Constants.java | 5 - .../ContentValidationModeResolver.java | 22 ++ .../StructuredMessageConstants.java | 2 + .../StructuredMessageDecoder.java | 188 +++--------------- .../policy/ContentValidationDecoderUtils.java | 53 ----- ...StorageContentValidationDecoderPolicy.java | 75 +++---- 9 files changed, 117 insertions(+), 337 deletions(-) delete mode 100644 sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java delete mode 100644 sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java deleted file mode 100644 index d9769a5645f2..000000000000 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/DownloadValidationUtils.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.blob.implementation.util; - -import com.azure.core.util.Context; -import com.azure.storage.common.ContentValidationAlgorithm; -import com.azure.storage.common.implementation.Constants; -import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; - -/** - * Centralizes download content validation decisions based on {@link ContentValidationAlgorithm}. - *

- * Mirrors the pattern established by {@link ContentValidationModeResolver} for uploads. - *

- * RESERVED FOR INTERNAL USE. - */ -public final class DownloadValidationUtils { - - private DownloadValidationUtils() { - } - - /** - * Whether the algorithm requires structured message decoding (CRC64 / AUTO). - */ - public static boolean isStructuredMessageAlgorithm(ContentValidationAlgorithm algorithm) { - return ContentValidationModeResolver.isCrc64OrAuto(algorithm); - } - - /** - * Resolves the effective algorithm, defaulting null to NONE. - */ - public static ContentValidationAlgorithm resolveAlgorithm(ContentValidationAlgorithm algorithm) { - return algorithm != null ? algorithm : ContentValidationAlgorithm.NONE; - } - - /** - * Adds structured message decoding context key when CRC64/AUTO validation is active. - * - * @param context The base context to augment. Null is treated as {@link Context#NONE}. - * @param algorithm The resolved checksum algorithm. - * @return The augmented context. - */ - public static Context applyStructuredMessageContext(Context context, ContentValidationAlgorithm algorithm) { - Context base = context == null ? Context.NONE : context; - if (!isStructuredMessageAlgorithm(algorithm)) { - return base; - } - return base.addData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); - } -} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index b525d2ef9c0d..621aea39723f 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -45,7 +45,6 @@ import com.azure.storage.blob.implementation.util.BlobRequestConditionProperty; import com.azure.storage.blob.implementation.util.BlobSasImplUtil; import com.azure.storage.blob.implementation.util.ChunkedDownloadUtils; -import com.azure.storage.blob.implementation.util.DownloadValidationUtils; import com.azure.storage.blob.implementation.util.ModelHelper; import com.azure.storage.blob.models.AccessTier; import com.azure.storage.blob.models.BlobBeginCopySourceRequestConditions; @@ -1263,11 +1262,13 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ContentValidationAlgorithm contentValidationAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; - final ContentValidationAlgorithm algorithm - = DownloadValidationUtils.resolveAlgorithm(contentValidationAlgorithm); - final boolean isStructuredMessageEnabled = DownloadValidationUtils.isStructuredMessageAlgorithm(algorithm); + ContentValidationModeResolver.validateTransactionalChecksumOptions(getRangeContentMd5, + contentValidationAlgorithm); - final Boolean finalGetMD5 = (!isStructuredMessageEnabled && getRangeContentMd5) ? true : null; + final Boolean getMD5 + = (!ContentValidationModeResolver.isCrc64OrAuto(contentValidationAlgorithm) && getRangeContentMd5) + ? true + : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; @@ -1277,9 +1278,10 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - final Context downloadContext = DownloadValidationUtils.applyStructuredMessageContext(baseContext, algorithm); + final Context downloadContext = ContentValidationModeResolver.addStructuredMessageDecodingToContext(baseContext, + contentValidationAlgorithm); - return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), finalGetMD5, + return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, downloadContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); String eTag = blobsDownloadHeaders.getETag(); @@ -1306,25 +1308,24 @@ Mono downloadStreamWithResponseInternal(BlobRange ran return Mono.error(throwable); } + long newCount = finalCount - offset; + + /* + * It's possible that the network stream will throw an error after emitting all data but before + * completing. Issuing a retry at this stage would leave the download in a bad state with + * incorrect count and offset values. Because we have read the intended amount of data, we can + * ignore the error at the end of the stream. + */ + if (newCount == 0) { + LOGGER.warning("Exception encountered in ReliableDownload after all data read from the network " + + "but before stream signaled completion. Returning success as all data was downloaded. " + + "Exception message: " + throwable.getMessage()); + return Mono.empty(); + } + try { - long newCount = finalCount - offset; - - /* - * It's possible that the network stream will throw an error after emitting all data but - * before completing. Issuing a retry at this stage would leave the download in a bad - * state with incorrect count and offset values. Because we have read the intended amount - * of data, we can ignore the error at the end of the stream. - */ - if (newCount == 0) { - LOGGER.warning( - "Exception encountered in ReliableDownload after all data read from the network " - + "but before stream signaled completion. Returning success as all data was " - + "downloaded. Exception message: " + throwable.getMessage()); - return Mono.empty(); - } - - BlobRange retryRange = new BlobRange(initialOffset + offset, newCount); - return downloadRange(retryRange, finalRequestConditions, eTag, finalGetMD5, downloadContext); + return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, + eTag, getMD5, downloadContext); } catch (Exception e) { return Mono.error(e); } @@ -1332,7 +1333,6 @@ Mono downloadStreamWithResponseInternal(BlobRange ran return BlobDownloadAsyncResponseConstructorProxy.create(response, onDownloadErrorResume, finalOptions); }); - } private Mono downloadRange(BlobRange range, BlobRequestConditions requestConditions, String eTag, diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 01e5085d9c40..2900165c13e7 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -52,7 +52,7 @@ public void setup() { * downloadStreamWithResponse with CRC64 content validation. */ @Test - public void downloadStreamWithResponseContentValidationSync() { + public void downloadStreamWithResponseContentValidation() { byte[] data = getRandomByteArray(10 * 1024 * 1024); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); @@ -70,7 +70,7 @@ public void downloadStreamWithResponseContentValidationSync() { * downloadContentWithResponse with CRC64 content validation. */ @Test - public void downloadContentWithResponseContentValidationSync() { + public void downloadContentWithResponseContentValidation() { byte[] data = getRandomByteArray(10 * 1024 * 1024); bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); @@ -93,7 +93,7 @@ public void downloadContentWithResponseContentValidationSync() { @ParameterizedTest @ValueSource(ints = { 512, 2048 }) @Timeout(value = 5, unit = TimeUnit.MINUTES) - public void downloadToFileWithResponseContentValidationSync(int blockSize) throws IOException { + public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { int payloadSize = (4 * blockSize) + 1; byte[] randomData = getRandomByteArray(payloadSize); bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index 26511db2eda6..c7b4f0198525 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -94,11 +94,6 @@ public final class Constants { public static final String SKIP_ECHO_VALIDATION_KEY = "skipEchoValidation"; - /** - * Context key used to signal that structured message decoding should be applied. - */ - public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; - private Constants() { } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java index fc29330620f7..3c9f958e3ddb 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java @@ -7,6 +7,7 @@ import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER; import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY; import com.azure.core.util.Context; import com.azure.core.util.FluxUtil; @@ -147,6 +148,27 @@ public static boolean isCrc64OrAuto(ContentValidationAlgorithm algorithm) { return algorithm == ContentValidationAlgorithm.CRC64 || algorithm == ContentValidationAlgorithm.AUTO; } + /** + * When the transfer validation mode is {@link ContentValidationAlgorithm#CRC64} or + * {@link ContentValidationAlgorithm#AUTO}, adds + * {@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY} so the HTTP + * pipeline can decode/validate the structured message response. For {@code null} or + * {@link ContentValidationAlgorithm#NONE}, returns the context unchanged (no key added), matching "no + * structured-message validation" for that download. + * + * @param context The base {@link Context}; null is treated as {@link Context#NONE}. + * @param contentValidationAlgorithm The algorithm from download options, or null. + * @return The same context, or a copy with the decoding key set when applicable. + */ + public static Context addStructuredMessageDecodingToContext(Context context, + ContentValidationAlgorithm contentValidationAlgorithm) { + Context base = context == null ? Context.NONE : context; + if (!isCrc64OrAuto(contentValidationAlgorithm)) { + return base; + } + return base.addData(STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + } + /** * Validates that parallel transfer progress reporting is not combined with CRC64/AUTO content validation. * diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java index caddb8104739..8e0ed1ff6e86 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java @@ -52,4 +52,6 @@ public final class StructuredMessageConstants { public static final String USE_CRC64_CHECKSUM_HEADER_CONTEXT = "crc64ChecksumHeaderContext"; public static final String USE_STRUCTURED_MESSAGE_CONTEXT = "structuredMessageChecksumAlgorithm"; + + public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index eb77efd2011b..5eaf8e280650 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -8,8 +8,6 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.HashMap; -import java.util.Map; import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; @@ -25,7 +23,7 @@ * *

Key invariants: *

    - *
  • Never read partial headers - always check buffer remaining >= required bytes
    • + *
  • Never read partial headers - always check buffer remaining >= required bytes
    • *
  • Only advance messageOffset when bytes are fully consumed and validated
    • *
  • lastCompleteSegmentStart always points to a valid segment boundary for retry
    • *
@@ -52,7 +50,6 @@ public class StructuredMessageDecoder { // CRC validation private long messageCrc64 = 0; private long segmentCrc64 = 0; - private final Map segmentLengths = new HashMap<>(); // Smart retry tracking - lastCompleteSegmentStart is the absolute offset where the last // fully completed segment ended. This is the safe retry boundary. @@ -144,22 +141,12 @@ public long getTotalDecodedPayloadBytes() { } /** - * Converts a ByteBuffer range to hex string for diagnostic purposes. + * Gets the expected message length from the header. + * + * @return The message length, or -1 if header not yet read. */ - private static String toHex(ByteBuffer buf, int len) { - int pos = buf.position(); - int peek = Math.min(len, buf.remaining()); - byte[] out = new byte[peek]; - buf.get(out, 0, peek); - buf.position(pos); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < out.length; i++) { - sb.append(String.format("%02X", out[i])); - if (i < out.length - 1) { - sb.append(' '); - } - } - return sb.toString(); + public long getMessageLength() { + return messageLength; } /** @@ -191,20 +178,34 @@ private ByteBuffer getCombinedBuffer(ByteBuffer buffer) { } /** - * Consumes bytes from pending first, then from buffer. - * Updates the buffer's position to reflect bytes consumed. + * When {@code flags} require a segment or message CRC footer, reads 8 bytes, validates against {@code expectedCrc64}, + * and advances {@link #messageOffset}. Returns false if the footer is not yet available. */ + private boolean tryConsumeCrc64Footer(ByteBuffer buffer, long expectedCrc64, String mismatchDetail) { + if (getAvailableBytes(buffer) < CRC64_LENGTH) { + appendToPending(buffer); + return false; + } + ByteBuffer combined = getCombinedBuffer(buffer); + long reportedCrc = combined.getLong(); + if (expectedCrc64 != reportedCrc) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "CRC64 mismatch" + mismatchDetail + ". Expected: " + expectedCrc64 + ", got: " + reportedCrc))); + } + consumeBytes(CRC64_LENGTH, buffer); + messageOffset += CRC64_LENGTH; + return true; + } + private void consumeBytes(int bytesToConsume, ByteBuffer buffer) { int pendingSize = pendingBytes.size(); if (bytesToConsume <= pendingSize) { - // All bytes come from pending - remove from pending byte[] remaining = pendingBytes.toByteArray(); pendingBytes.reset(); if (bytesToConsume < pendingSize) { pendingBytes.write(remaining, bytesToConsume, pendingSize - bytesToConsume); } } else { - // Consume all pending and some from buffer int bytesFromBuffer = bytesToConsume - pendingSize; pendingBytes.reset(); buffer.position(buffer.position() + bytesFromBuffer); @@ -220,15 +221,6 @@ private void appendToPending(ByteBuffer buffer) { } } - /** - * Gets the expected message length from the header. - * - * @return The message length, or -1 if header not yet read. - */ - public long getMessageLength() { - return messageLength; - } - /** * Reads the message header if we have enough bytes. * @@ -237,16 +229,10 @@ public long getMessageLength() { */ private boolean tryReadMessageHeader(ByteBuffer buffer) { if (messageLength != -1) { - return true; // Already read + return true; } - int available = getAvailableBytes(buffer); - if (available < V1_HEADER_LENGTH) { - LOGGER.atVerbose() - .addKeyValue("available", available) - .addKeyValue("required", V1_HEADER_LENGTH) - .addKeyValue("pendingBytes", pendingBytes.size()) - .log("Not enough bytes for message header, waiting for more"); + if (getAvailableBytes(buffer) < V1_HEADER_LENGTH) { appendToPending(buffer); return false; } @@ -277,13 +263,6 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { messageOffset += V1_HEADER_LENGTH; messageLength = msgLen; - LOGGER.atVerbose() - .addKeyValue("messageLength", messageLength) - .addKeyValue("numSegments", numSegments) - .addKeyValue("flags", flags) - .addKeyValue("messageOffset", messageOffset) - .log("Message header read successfully"); - return true; } @@ -294,29 +273,13 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { * @return true if segment header was read, false if more bytes needed. */ private boolean tryReadSegmentHeader(ByteBuffer buffer) { - int available = getAvailableBytes(buffer); - if (available < V1_SEGMENT_HEADER_LENGTH) { - LOGGER.atVerbose() - .addKeyValue("available", available) - .addKeyValue("required", V1_SEGMENT_HEADER_LENGTH) - .addKeyValue("pendingBytes", pendingBytes.size()) - .addKeyValue("decoderOffset", messageOffset) - .log("Not enough bytes for segment header, waiting for more"); + if (getAvailableBytes(buffer) < V1_SEGMENT_HEADER_LENGTH) { appendToPending(buffer); return false; } ByteBuffer combined = getCombinedBuffer(buffer); - // Log the raw bytes we're about to read - LOGGER.atVerbose() - .addKeyValue("decoderOffset", messageOffset) - .addKeyValue("bufferPos", combined.position()) - .addKeyValue("bufferRemaining", combined.remaining()) - .addKeyValue("peek16", toHex(combined, 16)) - .addKeyValue("lastCompleteSegment", lastCompleteSegmentStart) - .log("Decoder about to read segment header"); - int segmentNum = Short.toUnsignedInt(combined.getShort()); long segmentSize = combined.getLong(); @@ -326,34 +289,22 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { "Unexpected segment number. Expected: " + (currentSegmentNumber + 1) + ", got: " + segmentNum))); } - // Validate segment size - must be non-negative and reasonable - // We can't have segments larger than the remaining message length long remainingMessageBytes = messageLength - messageOffset - V1_SEGMENT_HEADER_LENGTH; if (segmentSize < 0 || segmentSize > remainingMessageBytes) { - LOGGER.error("Invalid segment length read: segmentLength={}, decoderOffset={}, lastCompleteSegment={}", - segmentSize, messageOffset, lastCompleteSegmentStart); throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( "Invalid segment size detected: " + segmentSize + " (remaining=" + remainingMessageBytes + ")"))); } - // Consume the bytes and update state consumeBytes(V1_SEGMENT_HEADER_LENGTH, buffer); messageOffset += V1_SEGMENT_HEADER_LENGTH; currentSegmentNumber = segmentNum; currentSegmentContentLength = segmentSize; currentSegmentContentOffset = 0; - segmentLengths.put(currentSegmentNumber, segmentSize); if (flags == StructuredMessageFlags.STORAGE_CRC64) { segmentCrc64 = 0; } - LOGGER.atVerbose() - .addKeyValue("segmentNum", segmentNum) - .addKeyValue("segmentLength", segmentSize) - .addKeyValue("decoderOffset", messageOffset) - .log("Segment header read successfully"); - return true; } @@ -367,12 +318,12 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { private int tryReadSegmentContent(ByteBuffer buffer, ByteArrayOutputStream output) { long remaining = currentSegmentContentLength - currentSegmentContentOffset; if (remaining == 0) { - return 0; // All content read, need to read footer + return 0; } int available = getAvailableBytes(buffer); if (available == 0) { - return 0; // No bytes available + return 0; } int toRead = (int) Math.min(available, remaining); @@ -403,44 +354,18 @@ private int tryReadSegmentContent(ByteBuffer buffer, ByteArrayOutputStream outpu */ private boolean tryReadSegmentFooter(ByteBuffer buffer) { if (currentSegmentContentOffset != currentSegmentContentLength) { - return true; // Content not fully read yet + return true; } if (flags == StructuredMessageFlags.STORAGE_CRC64) { - int available = getAvailableBytes(buffer); - if (available < CRC64_LENGTH) { - LOGGER.atVerbose() - .addKeyValue("available", available) - .addKeyValue("required", CRC64_LENGTH) - .addKeyValue("segmentNum", currentSegmentNumber) - .log("Not enough bytes for segment CRC footer, waiting for more"); - appendToPending(buffer); + if (!tryConsumeCrc64Footer(buffer, segmentCrc64, " in segment " + currentSegmentNumber)) { return false; } - - ByteBuffer combined = getCombinedBuffer(buffer); - long reportedCrc64 = combined.getLong(); - - if (segmentCrc64 != reportedCrc64) { - throw LOGGER.logExceptionAsError( - new IllegalArgumentException(enrichExceptionMessage("CRC64 mismatch detected in segment " - + currentSegmentNumber + ". Expected: " + segmentCrc64 + ", got: " + reportedCrc64))); - } - - consumeBytes(CRC64_LENGTH, buffer); - messageOffset += CRC64_LENGTH; } - // Mark that this segment is complete lastCompleteSegmentStart = messageOffset; lastCompleteSegmentNumber = currentSegmentNumber; - LOGGER.atVerbose() - .addKeyValue("segmentNum", currentSegmentNumber) - .addKeyValue("offset", lastCompleteSegmentStart) - .addKeyValue("segmentLength", currentSegmentContentLength) - .log("Segment complete at byte offset"); - // Check if we need to read message footer if (currentSegmentNumber == numSegments) { return tryReadMessageFooter(buffer); } @@ -456,28 +381,8 @@ private boolean tryReadSegmentFooter(ByteBuffer buffer) { */ private boolean tryReadMessageFooter(ByteBuffer buffer) { if (flags == StructuredMessageFlags.STORAGE_CRC64) { - int available = getAvailableBytes(buffer); - if (available < CRC64_LENGTH) { - LOGGER.atVerbose() - .addKeyValue("available", available) - .addKeyValue("required", CRC64_LENGTH) - .log("Not enough bytes for message CRC footer, waiting for more"); - appendToPending(buffer); - return false; - } - - ByteBuffer combined = getCombinedBuffer(buffer); - long reportedCrc = combined.getLong(); - - if (messageCrc64 != reportedCrc) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( - "CRC64 mismatch detected in message footer. Expected: " + messageCrc64 + ", got: " + reportedCrc))); - } - - consumeBytes(CRC64_LENGTH, buffer); - messageOffset += CRC64_LENGTH; + return tryConsumeCrc64Footer(buffer, messageCrc64, " in message footer"); } - return true; } @@ -494,65 +399,38 @@ public DecodeResult decodeChunk(ByteBuffer buffer) { ByteArrayOutputStream decodedContent = new ByteArrayOutputStream(); int startPos = buffer.position(); - LOGGER.atVerbose() - .addKeyValue("newBytes", buffer.remaining()) - .addKeyValue("pendingBytes", pendingBytes.size()) - .addKeyValue("decoderOffset", messageOffset) - .addKeyValue("lastCompleteSegment", lastCompleteSegmentStart) - .log("Received buffer in decode"); - try { - // Step 1: Read message header if not yet read if (!tryReadMessageHeader(buffer)) { return new DecodeResult(DecodeStatus.NEED_MORE_BYTES, null, 0, "Waiting for message header"); } - // Step 2: Process segments while (messageOffset < messageLength) { - // Read segment header only after the *previous* segment is fully complete (including its CRC footer). - // Otherwise we would misinterpret segment N's footer bytes as segment N+1's header when the footer - // is split across network buffers if (lastCompleteSegmentNumber == currentSegmentNumber && currentSegmentNumber < numSegments) { if (!tryReadSegmentHeader(buffer)) { - break; // Need more bytes for segment header + break; } } - // Read segment content int payloadRead = tryReadSegmentContent(buffer, decodedContent); - // Read segment footer (CRC) if content is complete if (currentSegmentContentOffset == currentSegmentContentLength) { if (!tryReadSegmentFooter(buffer)) { - break; // Need more bytes for segment footer - } - // After reading this segment's footer, if it was the last segment, read message footer. - if (currentSegmentNumber == numSegments && !tryReadMessageFooter(buffer)) { break; } } - // Check if all segments are complete if (currentSegmentNumber == numSegments && messageOffset >= messageLength) { - LOGGER.atVerbose() - .addKeyValue("messageOffset", messageOffset) - .addKeyValue("messageLength", messageLength) - .addKeyValue("totalDecodedPayload", totalDecodedPayloadBytes) - .log("Message decode completed"); - ByteBuffer result = decodedContent.size() > 0 ? ByteBuffer.wrap(decodedContent.toByteArray()) : null; return new DecodeResult(DecodeStatus.COMPLETED, result, buffer.position() - startPos, "Decode completed"); } - // If we couldn't read any bytes and no data available, need more if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { break; } } - // Return any decoded content even if we need more bytes ByteBuffer result = decodedContent.size() > 0 ? ByteBuffer.wrap(decodedContent.toByteArray()) : null; if (messageOffset >= messageLength) { diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java deleted file mode 100644 index d1fa42ba9e6c..000000000000 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/ContentValidationDecoderUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.common.policy; - -import com.azure.core.http.HttpHeaderName; -import com.azure.core.http.HttpHeaders; -import com.azure.core.http.HttpMethod; -import com.azure.core.http.HttpResponse; -import com.azure.core.util.logging.ClientLogger; - -/** - * Utility class for download eligibility checks - * used by {@link StorageContentValidationDecoderPolicy}. - */ -final class ContentValidationDecoderUtils { - private static final ClientLogger LOGGER = new ClientLogger(ContentValidationDecoderUtils.class); - - private ContentValidationDecoderUtils() { - } - - /** - * Checks whether the response represents a successful download (GET with 2xx). - */ - static boolean isDownloadResponse(HttpResponse response) { - return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; - } - - /** - * Extracts Content-Length from the response headers. - * - * @return The content length, or null if absent or unparseable. - */ - static Long getContentLength(HttpHeaders headers) { - String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); - if (value != null) { - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - LOGGER.warning("Invalid content length in response headers: " + value); - } - } - return null; - } - - /** - * Returns {@code true} when the response is a download response with a positive content length, - * making it eligible for structured message decoding. - */ - static boolean isEligibleDownload(HttpResponse response, Long contentLength) { - return isDownloadResponse(response) && contentLength != null && contentLength > 0; - } -} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index cd2e36638ba1..41dcfcbdf785 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -4,13 +4,14 @@ package com.azure.storage.common.policy; import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; import com.azure.core.http.HttpPipelinePosition; import com.azure.core.util.logging.ClientLogger; -import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; import reactor.core.publisher.Flux; @@ -25,10 +26,9 @@ * is {@code CRC64} or {@code AUTO}). * *

The policy is activated by the presence of a boolean context key - * ({@link Constants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY}). It validates per-segment - * CRC64 checksums during decoding. Since this policy is placed at the front of the pipeline - * (before the retry policy), each retry re-enters the policy with a fresh response body, - * so no cross-retry state management is needed.

+ * ({@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY}). It validates per-segment + * CRC64 checksums during decoding. The policy is registered early in the pipeline; each retry re-enters the policy + * with a fresh response body, so no cross-retry state management is needed in the policy.

*/ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); @@ -58,10 +58,9 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN .set(X_MS_STRUCTURED_BODY, StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); return next.process().map(httpResponse -> { - Long contentLength = ContentValidationDecoderUtils.getContentLength(httpResponse.getHeaders()); + Long contentLength = getContentLength(httpResponse.getHeaders()); - if (!ContentValidationDecoderUtils.isEligibleDownload(httpResponse, contentLength)) { - LOGGER.atVerbose().log("Not a download response with content, passing through"); + if (!isEligibleDownload(httpResponse, contentLength)) { return httpResponse; } @@ -71,16 +70,12 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN StructuredMessageDecoder decoder = new StructuredMessageDecoder(expectedLength); Flux decodedStream = decodeStream(httpResponse.getBody(), decoder); - - LOGGER.atVerbose() - .addKeyValue("expectedLength", expectedLength) - .log("Returning DecodedResponse with structured message decoding"); return new DecodedResponse(httpResponse, decodedStream); }); } private boolean shouldApplyDecoding(HttpPipelineCallContext context) { - return context.getData(Constants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) + return context.getData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) .map(value -> value instanceof Boolean && (Boolean) value) .orElse(false); } @@ -94,6 +89,29 @@ private void validateStructuredMessageHeaders(HttpResponse httpResponse) { } } + private static boolean isDownloadResponse(HttpResponse response) { + return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; + } + + /** + * @return The content length, or null if absent or unparseable. + */ + private static Long getContentLength(HttpHeaders headers) { + String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + // Header invalid; treat as not eligible. + } + } + return null; + } + + private static boolean isEligibleDownload(HttpResponse response, Long contentLength) { + return isDownloadResponse(response) && contentLength != null && contentLength > 0; + } + private Flux decodeStream(Flux encodedFlux, StructuredMessageDecoder decoder) { return encodedFlux.concatMap(buffer -> decodeBuffer(buffer, decoder)) .onErrorResume(throwable -> handleStreamError(throwable, decoder)) @@ -102,9 +120,6 @@ private Flux decodeStream(Flux encodedFlux, StructuredMe private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecoder decoder) { if (decoder.isComplete()) { - LOGGER.atVerbose() - .addKeyValue("bufferLength", buffer == null ? "null" : buffer.remaining()) - .log("Decoder already completed; ignoring extra buffer"); return Flux.empty(); } @@ -112,23 +127,8 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode return Flux.empty(); } - LOGGER.atVerbose() - .addKeyValue("newBytes", buffer.remaining()) - .addKeyValue("decoderOffset", decoder.getMessageOffset()) - .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) - .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) - .log("Received buffer in decodeStream"); - try { StructuredMessageDecoder.DecodeResult result = decoder.decodeChunk(buffer); - - LOGGER.atVerbose() - .addKeyValue("status", result.getStatus()) - .addKeyValue("bytesConsumed", result.getBytesConsumed()) - .addKeyValue("decoderOffset", decoder.getMessageOffset()) - .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) - .log("Decode chunk result"); - switch (result.getStatus()) { case NEED_MORE_BYTES: return emitDecodedPayload(result.getDecodedPayload()); @@ -137,7 +137,6 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode return emitDecodedPayload(result.getDecodedPayload()); case INVALID: - LOGGER.error("Invalid data during decode: {}", result.getMessage()); return Flux.error(new IOException("Failed to decode structured message: " + result.getMessage())); default: @@ -151,7 +150,6 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode private Flux handleStreamError(Throwable throwable, StructuredMessageDecoder decoder) { if (decoder.isComplete()) { - LOGGER.atVerbose().log("Decoder complete; suppressing downstream error and completing successfully"); return Flux.empty(); } @@ -160,19 +158,8 @@ private Flux handleStreamError(Throwable throwable, StructuredMessag private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { if (!decoder.isComplete()) { - LOGGER.atVerbose() - .addKeyValue("messageOffset", decoder.getMessageOffset()) - .addKeyValue("messageLength", decoder.getMessageLength()) - .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) - .addKeyValue("lastCompleteSegment", decoder.getLastCompleteSegmentStart()) - .log("Stream ended but decode not finalized"); return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); } - - LOGGER.atVerbose() - .addKeyValue("messageOffset", decoder.getMessageOffset()) - .addKeyValue("totalDecodedPayload", decoder.getTotalDecodedPayloadBytes()) - .log("Stream complete and decode finalized successfully"); return Mono.empty(); } From b3298c22181eaa998befcc5b55539ffb0db2f5ea Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Wed, 22 Apr 2026 20:44:08 +0530 Subject: [PATCH 20/31] refactoring based on latest review comments --- .../blob/specialized/BlobAsyncClientBase.java | 5 +- .../StructuredMessageDecoder.java | 226 ++++++------------ ...StorageContentValidationDecoderPolicy.java | 23 +- .../StructuredMessageDecoderTests.java | 149 ++++++------ 4 files changed, 155 insertions(+), 248 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 621aea39723f..20cf023ac3bc 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1274,11 +1274,12 @@ Mono downloadStreamWithResponseInternal(BlobRange ran = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; - final Context baseContext = context == null + // The first range should eagerly convert headers as they'll be used to create response types. + Context firstRangeContext = context == null ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - final Context downloadContext = ContentValidationModeResolver.addStructuredMessageDecodingToContext(baseContext, + Context downloadContext = ContentValidationModeResolver.addStructuredMessageDecodingToContext(firstRangeContext, contentValidationAlgorithm); return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 5eaf8e280650..5576d887278b 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -21,12 +21,12 @@ * by maintaining a pending buffer and only advancing offsets when complete structures * have been fully read and validated.

* - *

Key invariants: - *

    - *
  • Never read partial headers - always check buffer remaining >= required bytes
    • - *
  • Only advance messageOffset when bytes are fully consumed and validated
    • - *
  • lastCompleteSegmentStart always points to a valid segment boundary for retry
    • - *
+ *

Emission guarantee: payload bytes for a segment are never emitted + * downstream until the segment payload has been fully read and, when CRC64 is enabled, + * its segment CRC footer has been validated. This matches the emission semantics used by + * {@code BlobDecryptionPolicy}/{@code DecryptorV2} (which only emits a decrypted region + * after the GCM tag is verified) and guarantees that no unvalidated bytes are ever + * exposed to callers, even under retries.

*/ public class StructuredMessageDecoder { private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecoder.class); @@ -38,70 +38,25 @@ public class StructuredMessageDecoder { private final long expectedContentLength; // Offset tracking - private long messageOffset = 0; // Absolute encoded bytes consumed from the message - private long totalDecodedPayloadBytes = 0; // Total decoded (payload) bytes output + private long messageOffset = 0; // Current segment state private int currentSegmentNumber = 0; private long currentSegmentContentLength = 0; private long currentSegmentContentOffset = 0; - private int lastCompleteSegmentNumber = 0; + private boolean segmentHeaderRead = false; // CRC validation private long messageCrc64 = 0; private long segmentCrc64 = 0; - // Smart retry tracking - lastCompleteSegmentStart is the absolute offset where the last - // fully completed segment ended. This is the safe retry boundary. - private long lastCompleteSegmentStart = 0; - // Pending buffer for handling partial headers/segments across chunks private final ByteArrayOutputStream pendingBytes = new ByteArrayOutputStream(); - /** - * Decode result status codes. - */ - public enum DecodeStatus { - /** Need more bytes to continue (partial header/segment) */ - NEED_MORE_BYTES, - /** Decoding completed successfully */ - COMPLETED, - /** Invalid data encountered */ - INVALID - } - - /** - * Result of a decode operation. - */ - public static class DecodeResult { - private final DecodeStatus status; - private final ByteBuffer decodedPayload; - private final String message; - private final int bytesConsumed; - - DecodeResult(DecodeStatus status, ByteBuffer decodedPayload, int bytesConsumed, String message) { - this.status = status; - this.decodedPayload = decodedPayload; - this.bytesConsumed = bytesConsumed; - this.message = message; - } - - public DecodeStatus getStatus() { - return status; - } - - public ByteBuffer getDecodedPayload() { - return decodedPayload; - } - - public int getBytesConsumed() { - return bytesConsumed; - } - - public String getMessage() { - return message; - } - } + // Payload bytes accumulated for the current segment. These are held back and NOT + // emitted until the segment CRC footer has been validated, so callers never observe + // bytes that could later fail validation. + private final ByteArrayOutputStream currentSegmentBuffer = new ByteArrayOutputStream(); /** * Constructs a new StructuredMessageDecoder. @@ -112,43 +67,6 @@ public StructuredMessageDecoder(long expectedContentLength) { this.expectedContentLength = expectedContentLength; } - /** - * Gets the byte offset where the last complete segment ended. - * This is used for smart retry to resume from a segment boundary. - * - * @return The byte offset of the last complete segment boundary. - */ - public long getLastCompleteSegmentStart() { - return lastCompleteSegmentStart; - } - - /** - * Gets the current message offset (total bytes consumed from the structured message). - * - * @return The current message offset. - */ - public long getMessageOffset() { - return messageOffset; - } - - /** - * Gets the total decoded payload bytes produced so far. - * - * @return The total decoded payload bytes. - */ - public long getTotalDecodedPayloadBytes() { - return totalDecodedPayloadBytes; - } - - /** - * Gets the expected message length from the header. - * - * @return The message length, or -1 if header not yet read. - */ - public long getMessageLength() { - return messageLength; - } - /** * Gets the total available bytes (pending + buffer remaining). */ @@ -300,6 +218,7 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { currentSegmentNumber = segmentNum; currentSegmentContentLength = segmentSize; currentSegmentContentOffset = 0; + currentSegmentBuffer.reset(); if (flags == StructuredMessageFlags.STORAGE_CRC64) { segmentCrc64 = 0; @@ -309,13 +228,14 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { } /** - * Reads segment content bytes if available. + * Reads segment content bytes if available, accumulating them into the per-segment + * buffer. Bytes remain held in the buffer until the segment's CRC footer is + * validated; they are not returned to the caller here. * * @param buffer The buffer to read from. - * @param output The output stream to write decoded payload to. - * @return The number of payload bytes read, or -1 if more bytes needed for CRC. + * @return The number of payload bytes read into the segment buffer. */ - private int tryReadSegmentContent(ByteBuffer buffer, ByteArrayOutputStream output) { + private int tryReadSegmentContent(ByteBuffer buffer) { long remaining = currentSegmentContentLength - currentSegmentContentOffset; if (remaining == 0) { return 0; @@ -331,7 +251,7 @@ private int tryReadSegmentContent(ByteBuffer buffer, ByteArrayOutputStream outpu byte[] content = new byte[toRead]; combined.get(content); - output.write(content, 0, toRead); + currentSegmentBuffer.write(content, 0, toRead); if (flags == StructuredMessageFlags.STORAGE_CRC64) { segmentCrc64 = StorageCrc64Calculator.compute(content, segmentCrc64); @@ -341,13 +261,15 @@ private int tryReadSegmentContent(ByteBuffer buffer, ByteArrayOutputStream outpu consumeBytes(toRead, buffer); messageOffset += toRead; currentSegmentContentOffset += toRead; - totalDecodedPayloadBytes += toRead; return toRead; } /** - * Reads the segment CRC footer if needed and available. + * Reads the segment CRC footer if needed and available. Does not advance into the + * message footer; callers drive message-footer consumption separately so that + * segment bytes can be flushed as soon as their CRC passes, even if the message + * footer is not yet available in the current chunk. * * @param buffer The buffer to read from. * @return true if footer was read (or not needed), false if more bytes needed. @@ -358,16 +280,7 @@ private boolean tryReadSegmentFooter(ByteBuffer buffer) { } if (flags == StructuredMessageFlags.STORAGE_CRC64) { - if (!tryConsumeCrc64Footer(buffer, segmentCrc64, " in segment " + currentSegmentNumber)) { - return false; - } - } - - lastCompleteSegmentStart = messageOffset; - lastCompleteSegmentNumber = currentSegmentNumber; - - if (currentSegmentNumber == numSegments) { - return tryReadMessageFooter(buffer); + return tryConsumeCrc64Footer(buffer, segmentCrc64, " in segment " + currentSegmentNumber); } return true; @@ -387,63 +300,70 @@ private boolean tryReadMessageFooter(ByteBuffer buffer) { } /** - * Decodes as much as possible from the given buffer. - * This method properly handles partial headers and segments by buffering - * incomplete data and returning NEED_MORE_BYTES when more data is required. + * Decodes as much as possible from the given buffer and returns any fully validated + * payload bytes that are now safe to emit downstream. + * + *

The returned buffer will only ever contain bytes from segments whose CRC (when + * enabled) has already been verified. If no segments have been fully validated by + * this invocation the method returns {@code null}. Callers distinguish "more bytes + * needed" from "stream complete" via {@link #isComplete()}.

* * @param buffer The buffer containing encoded data. - * @return A DecodeResult indicating the outcome and any decoded payload. + * @return Validated payload bytes ready to emit, or {@code null} if none are ready. + * @throws IllegalArgumentException if the input is malformed or a CRC64 check fails. */ - public DecodeResult decodeChunk(ByteBuffer buffer) { + public ByteBuffer decodeChunk(ByteBuffer buffer) { buffer.order(ByteOrder.LITTLE_ENDIAN); - ByteArrayOutputStream decodedContent = new ByteArrayOutputStream(); - int startPos = buffer.position(); - - try { - if (!tryReadMessageHeader(buffer)) { - return new DecodeResult(DecodeStatus.NEED_MORE_BYTES, null, 0, "Waiting for message header"); - } + ByteArrayOutputStream validatedOutput = new ByteArrayOutputStream(); - while (messageOffset < messageLength) { - if (lastCompleteSegmentNumber == currentSegmentNumber && currentSegmentNumber < numSegments) { - if (!tryReadSegmentHeader(buffer)) { - break; - } - } - - int payloadRead = tryReadSegmentContent(buffer, decodedContent); + if (!tryReadMessageHeader(buffer)) { + return emptyOrNull(validatedOutput); + } - if (currentSegmentContentOffset == currentSegmentContentLength) { - if (!tryReadSegmentFooter(buffer)) { + while (messageOffset < messageLength) { + if (!segmentHeaderRead) { + // All segments are done; only the trailing message footer remains. + if (currentSegmentNumber == numSegments) { + if (!tryReadMessageFooter(buffer)) { break; } + break; } - - if (currentSegmentNumber == numSegments && messageOffset >= messageLength) { - ByteBuffer result - = decodedContent.size() > 0 ? ByteBuffer.wrap(decodedContent.toByteArray()) : null; - return new DecodeResult(DecodeStatus.COMPLETED, result, buffer.position() - startPos, - "Decode completed"); - } - - if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { + if (!tryReadSegmentHeader(buffer)) { break; } + segmentHeaderRead = true; } - ByteBuffer result = decodedContent.size() > 0 ? ByteBuffer.wrap(decodedContent.toByteArray()) : null; + int payloadRead = tryReadSegmentContent(buffer); - if (messageOffset >= messageLength) { - return new DecodeResult(DecodeStatus.COMPLETED, result, buffer.position() - startPos, - "Decode completed"); + if (currentSegmentContentOffset == currentSegmentContentLength) { + if (!tryReadSegmentFooter(buffer)) { + break; + } + // Segment is fully validated: safe to release the buffered payload. + try { + currentSegmentBuffer.writeTo(validatedOutput); + } catch (java.io.IOException e) { + // ByteArrayOutputStream.writeTo(ByteArrayOutputStream) cannot throw. + throw LOGGER.logExceptionAsError(new IllegalStateException(e)); + } + currentSegmentBuffer.reset(); + segmentHeaderRead = false; + // Loop continues: either consume the next segment header or the message footer. + } else if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { + break; } + } - return new DecodeResult(DecodeStatus.NEED_MORE_BYTES, result, buffer.position() - startPos, - "Waiting for more data"); + return emptyOrNull(validatedOutput); + } - } catch (IllegalArgumentException e) { - return new DecodeResult(DecodeStatus.INVALID, null, buffer.position() - startPos, e.getMessage()); + private static ByteBuffer emptyOrNull(ByteArrayOutputStream output) { + if (output.size() == 0) { + return null; } + return ByteBuffer.wrap(output.toByteArray()); } /** @@ -456,14 +376,12 @@ public boolean isComplete() { } /** - * Enriches an exception message with decoder offset information for debugging and retry. - * Format: "original message [decoderOffset=X,lastCompleteSegment=Y]" + * Enriches an exception message with decoder offset information for debugging. * * @param message The original exception message. * @return The enriched message with offset information. */ private String enrichExceptionMessage(String message) { - return String.format("%s [decoderOffset=%d,lastCompleteSegment=%d]", message, messageOffset, - lastCompleteSegmentStart); + return String.format("%s [decoderOffset=%d]", message, messageOffset); } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 41dcfcbdf785..4c7465310fad 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -29,6 +29,11 @@ * ({@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY}). It validates per-segment * CRC64 checksums during decoding. The policy is registered early in the pipeline; each retry re-enters the policy * with a fresh response body, so no cross-retry state management is needed in the policy.

+ * + *

Emission guarantee: the policy only forwards payload bytes that the + * {@link StructuredMessageDecoder} has already CRC-validated at the segment boundary. Retries never expose + * partially validated bytes because each attempt creates a fresh decoder and the decoder itself withholds + * bytes until their enclosing segment passes validation.

*/ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); @@ -128,20 +133,10 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode } try { - StructuredMessageDecoder.DecodeResult result = decoder.decodeChunk(buffer); - switch (result.getStatus()) { - case NEED_MORE_BYTES: - return emitDecodedPayload(result.getDecodedPayload()); - - case COMPLETED: - return emitDecodedPayload(result.getDecodedPayload()); - - case INVALID: - return Flux.error(new IOException("Failed to decode structured message: " + result.getMessage())); - - default: - return Flux.error(new IllegalStateException("Unknown decode status: " + result.getStatus())); - } + ByteBuffer validated = decoder.decodeChunk(buffer); + return emitDecodedPayload(validated); + } catch (IllegalArgumentException e) { + return Flux.error(new IOException("Failed to decode structured message: " + e.getMessage(), e)); } catch (Exception e) { LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); return Flux.error(new IOException("Failed to decode structured message chunk: " + e.getMessage(), e)); diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java index 10e7f7ab78ec..64e8a0d25430 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -17,11 +17,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Unit tests for StructuredMessageDecoder with focus on handling partial headers - * and segment splits across chunks. + * Unit tests for StructuredMessageDecoder with focus on the validated-emission guarantee: + * payload bytes for a segment are only returned after the segment's CRC has been verified. */ public class StructuredMessageDecoderTests { @@ -31,7 +32,6 @@ private static ByteBuffer collectFlux(Flux flux) { @Test public void readsCompleteMessageInSingleChunk() throws IOException { - // Test: Complete message in a single ByteBuffer should decode fully byte[] originalData = new byte[1024]; ThreadLocalRandom.current().nextBytes(originalData); @@ -41,20 +41,17 @@ public void readsCompleteMessageInSingleChunk() throws IOException { int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); - assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); - ByteBuffer result = decodeResult.getDecodedPayload(); + ByteBuffer result = decoder.decodeChunk(encodedData); + assertTrue(decoder.isComplete()); assertNotNull(result); byte[] decodedData = new byte[result.remaining()]; result.get(decodedData); assertArrayEquals(originalData, decodedData); - assertTrue(decoder.isComplete()); } @Test public void readsMessageSplitHeaderAcrossChunks() throws IOException { - // Test: Feed header bytes split across two buffers byte[] originalData = new byte[256]; ThreadLocalRandom.current().nextBytes(originalData); @@ -73,20 +70,17 @@ public void readsMessageSplitHeaderAcrossChunks() throws IOException { StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - // First chunk should not throw, should wait for more bytes - StructuredMessageDecoder.DecodeResult result1 = decoder.decodeChunk(chunk1); - assertEquals(StructuredMessageDecoder.DecodeStatus.NEED_MORE_BYTES, result1.getStatus()); + ByteBuffer result1 = decoder.decodeChunk(chunk1); + assertNull(result1); assertFalse(decoder.isComplete()); - // Second chunk should complete the decode - StructuredMessageDecoder.DecodeResult result2 = decoder.decodeChunk(chunk2); - assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, result2.getStatus()); + ByteBuffer result2 = decoder.decodeChunk(chunk2); + assertNotNull(result2); assertTrue(decoder.isComplete()); } @Test public void readsSegmentHeaderSplitAcrossChunks() throws IOException { - // Test: Split the 10-byte segment header across two chunks byte[] originalData = new byte[512]; ThreadLocalRandom.current().nextBytes(originalData); @@ -97,8 +91,7 @@ public void readsSegmentHeaderSplitAcrossChunks() throws IOException { byte[] encodedBytes = new byte[encodedLength]; encodedData.get(encodedBytes); - // Split after message header (13 bytes) + 5 bytes into first segment header - // Segment header is 10 bytes, so split at byte 18 (mid-segment-header) + // Split after message header (13 bytes) + 5 bytes into first segment header. int splitPoint = 18; ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, splitPoint); ByteBuffer chunk2 = ByteBuffer.wrap(encodedBytes, splitPoint, encodedLength - splitPoint); @@ -107,21 +100,18 @@ public void readsSegmentHeaderSplitAcrossChunks() throws IOException { StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - // First chunk should parse header but wait for segment header completion - StructuredMessageDecoder.DecodeResult result1 = decoder.decodeChunk(chunk1); - assertEquals(StructuredMessageDecoder.DecodeStatus.NEED_MORE_BYTES, result1.getStatus()); + ByteBuffer result1 = decoder.decodeChunk(chunk1); + // Only the message header is consumed; segment header is incomplete so nothing validated yet. + assertNull(result1); assertFalse(decoder.isComplete()); - // Second chunk should complete - StructuredMessageDecoder.DecodeResult result2 = decoder.decodeChunk(chunk2); - assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, result2.getStatus()); + ByteBuffer result2 = decoder.decodeChunk(chunk2); + assertNotNull(result2); assertTrue(decoder.isComplete()); } @Test public void handlesZeroLengthSegment() throws IOException { - // Test: Zero-length segment should decode correctly - // Note: Zero-length segments are valid in the format byte[] minimalData = new byte[1]; ThreadLocalRandom.current().nextBytes(minimalData); @@ -131,46 +121,15 @@ public void handlesZeroLengthSegment() throws IOException { int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); - assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); - ByteBuffer result = decodeResult.getDecodedPayload(); + ByteBuffer result = decoder.decodeChunk(encodedData); + assertTrue(decoder.isComplete()); assertNotNull(result); assertEquals(1, result.remaining()); - assertTrue(decoder.isComplete()); - } - - @Test - public void tracksLastCompleteSegmentCorrectly() throws IOException { - // Test: Verify lastCompleteSegmentStart is updated correctly after each segment - byte[] originalData = new byte[1024]; - ThreadLocalRandom.current().nextBytes(originalData); - - StructuredMessageEncoder encoder - = new StructuredMessageEncoder(originalData.length, 256, StructuredMessageFlags.STORAGE_CRC64); - ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); - int encodedLength = encodedData.remaining(); - - StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - - // Initially lastCompleteSegmentStart should be 0 - assertEquals(0, decoder.getLastCompleteSegmentStart()); - - StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); - assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); - - // After complete decode, lastCompleteSegmentStart should point to end of last segment - // (before message footer, if any) - assertTrue(decoder.isComplete()); - // lastCompleteSegmentStart should be <= messageOffset - assertTrue(decoder.getLastCompleteSegmentStart() <= decoder.getMessageOffset()); - // And should be > 0 (we processed at least one segment) - assertTrue(decoder.getLastCompleteSegmentStart() > 0); } @Test public void multipleChunksDecode() throws IOException { - // Test: Decode message across multiple small chunks byte[] originalData = new byte[256]; ThreadLocalRandom.current().nextBytes(originalData); @@ -183,7 +142,6 @@ public void multipleChunksDecode() throws IOException { StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - // Feed in chunks of 32 bytes int chunkSize = 32; ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -192,14 +150,13 @@ public void multipleChunksDecode() throws IOException { ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); chunk.order(ByteOrder.LITTLE_ENDIAN); - StructuredMessageDecoder.DecodeResult result = decoder.decodeChunk(chunk); - if (result.getDecodedPayload() != null && result.getDecodedPayload().hasRemaining()) { - byte[] decoded = new byte[result.getDecodedPayload().remaining()]; - result.getDecodedPayload().get(decoded); + ByteBuffer result = decoder.decodeChunk(chunk); + if (result != null && result.hasRemaining()) { + byte[] decoded = new byte[result.remaining()]; + result.get(decoded); output.write(decoded, 0, decoded.length); } - - if (result.getStatus() == StructuredMessageDecoder.DecodeStatus.COMPLETED) { + if (decoder.isComplete()) { break; } } @@ -210,7 +167,6 @@ public void multipleChunksDecode() throws IOException { @Test public void decodeWithNoCrc() throws IOException { - // Test: Decode message without CRC (NONE flag) byte[] originalData = new byte[256]; ThreadLocalRandom.current().nextBytes(originalData); @@ -220,20 +176,17 @@ public void decodeWithNoCrc() throws IOException { int encodedLength = encodedData.remaining(); StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - StructuredMessageDecoder.DecodeResult decodeResult = decoder.decodeChunk(encodedData); - assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, decodeResult.getStatus()); - ByteBuffer result = decodeResult.getDecodedPayload(); + ByteBuffer result = decoder.decodeChunk(encodedData); + assertTrue(decoder.isComplete()); assertNotNull(result); byte[] decodedData = new byte[result.remaining()]; result.get(decodedData); assertArrayEquals(originalData, decodedData); - assertTrue(decoder.isComplete()); } @Test public void handlesZeroLengthBuffer() throws IOException { - // Test: Decoder should handle zero-length buffers gracefully byte[] originalData = new byte[256]; ThreadLocalRandom.current().nextBytes(originalData); @@ -246,17 +199,57 @@ public void handlesZeroLengthBuffer() throws IOException { StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); - // Feed zero-length buffer first ByteBuffer emptyBuffer = ByteBuffer.allocate(0); - StructuredMessageDecoder.DecodeResult result1 = decoder.decodeChunk(emptyBuffer); - assertEquals(StructuredMessageDecoder.DecodeStatus.NEED_MORE_BYTES, result1.getStatus()); - assertEquals(0, result1.getBytesConsumed()); + ByteBuffer result1 = decoder.decodeChunk(emptyBuffer); + assertNull(result1); - // Then feed actual data ByteBuffer dataBuffer = ByteBuffer.wrap(encodedBytes); dataBuffer.order(ByteOrder.LITTLE_ENDIAN); - StructuredMessageDecoder.DecodeResult result2 = decoder.decodeChunk(dataBuffer); - assertEquals(StructuredMessageDecoder.DecodeStatus.COMPLETED, result2.getStatus()); + ByteBuffer result2 = decoder.decodeChunk(dataBuffer); + assertNotNull(result2); assertTrue(decoder.isComplete()); } + + /** + * Verifies Kyle's emission guarantee (r3120267493): payload bytes for a segment are + * not emitted until the segment's CRC footer is read and validated. When the decoder + * has received the full segment payload but the CRC footer is still incomplete, + * {@code decodeChunk} must return {@code null}, never the in-progress payload bytes. + */ + @Test + public void withholdsPayloadUntilSegmentFooterValidated() throws IOException { + byte[] originalData = new byte[1024]; + ThreadLocalRandom.current().nextBytes(originalData); + + StructuredMessageEncoder encoder + = new StructuredMessageEncoder(originalData.length, 1024, StructuredMessageFlags.STORAGE_CRC64); + ByteBuffer encodedData = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); + int encodedLength = encodedData.remaining(); + byte[] encodedBytes = new byte[encodedLength]; + encodedData.get(encodedBytes); + + // Layout: msgHeader(13) + segHeader(10) + payload(1024) + segCrc(8) + msgCrc(8) = 1063. + // Feed the full payload but stop 1 byte short of completing the SEGMENT CRC footer. + int segCrcAllButLast = 13 + 10 + 1024 + 7; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, segCrcAllButLast); + chunk1.order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer partial = decoder.decodeChunk(chunk1); + assertNull(partial, "Decoder must not emit payload before segment CRC is validated"); + assertFalse(decoder.isComplete()); + + // Feed the remainder; segment CRC completes, payload is released, and message CRC completes. + ByteBuffer chunk2 = ByteBuffer.wrap(encodedBytes, segCrcAllButLast, encodedLength - segCrcAllButLast); + chunk2.order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer emitted = decoder.decodeChunk(chunk2); + assertNotNull(emitted); + assertTrue(decoder.isComplete()); + + byte[] decodedData = new byte[emitted.remaining()]; + emitted.get(decodedData); + assertArrayEquals(originalData, decodedData); + } + } From ed8c3eb76334a2f53968c228ddb7b54294d3d5f6 Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Thu, 23 Apr 2026 22:53:53 +0530 Subject: [PATCH 21/31] refactoring based on latest review comments --- .../blob/specialized/BlobAsyncClientBase.java | 5 +---- .../StructuredMessageDecoder.java | 20 +++++++++++++---- .../common/policy/DecodedResponse.java | 12 +++++++++- ...StorageContentValidationDecoderPolicy.java | 22 ++++++------------- 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index 20cf023ac3bc..d653c2bc52ee 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1265,10 +1265,7 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ContentValidationModeResolver.validateTransactionalChecksumOptions(getRangeContentMd5, contentValidationAlgorithm); - final Boolean getMD5 - = (!ContentValidationModeResolver.isCrc64OrAuto(contentValidationAlgorithm) && getRangeContentMd5) - ? true - : null; + Boolean getMD5 = getRangeContentMd5 ? getRangeContentMd5 : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 5576d887278b..c7de91e169bb 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -175,6 +175,10 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(combined.getShort())); numSegments = Short.toUnsignedInt(combined.getShort()); + if (numSegments < 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Structured message must have at least one segment, got: " + numSegments))); + } // Consume the bytes from pending/buffer consumeBytes(V1_HEADER_LENGTH, buffer); @@ -207,10 +211,14 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { "Unexpected segment number. Expected: " + (currentSegmentNumber + 1) + ", got: " + segmentNum))); } - long remainingMessageBytes = messageLength - messageOffset - V1_SEGMENT_HEADER_LENGTH; - if (segmentSize < 0 || segmentSize > remainingMessageBytes) { + long footerSize = flags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; + long remainingSegmentsAfterThis = (long) numSegments - segmentNum; + long reservedBytes + = footerSize + remainingSegmentsAfterThis * (V1_SEGMENT_HEADER_LENGTH + footerSize) + footerSize; + long maxSegmentSize = messageLength - messageOffset - V1_SEGMENT_HEADER_LENGTH - reservedBytes; + if (segmentSize < 0 || segmentSize > maxSegmentSize) { throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( - "Invalid segment size detected: " + segmentSize + " (remaining=" + remainingMessageBytes + ")"))); + "Invalid segment size detected: " + segmentSize + " (max=" + maxSegmentSize + ")"))); } consumeBytes(V1_SEGMENT_HEADER_LENGTH, buffer); @@ -372,7 +380,11 @@ private static ByteBuffer emptyOrNull(ByteArrayOutputStream output) { * @return true if all expected bytes have been decoded, false otherwise. */ public boolean isComplete() { - return messageLength != -1 && messageOffset >= messageLength; + return messageLength != -1 + && messageOffset >= messageLength + && pendingBytes.size() == 0 + && !segmentHeaderRead + && currentSegmentContentOffset == currentSegmentContentLength; } /** diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java index db8a597bf265..f962e869cd2b 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -3,9 +3,11 @@ package com.azure.storage.common.policy; +import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; import com.azure.core.util.FluxUtil; +import com.azure.storage.common.implementation.Constants; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -37,7 +39,15 @@ public String getHeaderValue(String name) { @Override public HttpHeaders getHeaders() { - return originalResponse.getHeaders(); + HttpHeaders headers = new HttpHeaders(originalResponse.getHeaders()); + String structuredContentLength + = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + if (structuredContentLength != null) { + headers.set(HttpHeaderName.CONTENT_LENGTH, structuredContentLength); + } else { + headers.remove(HttpHeaderName.CONTENT_LENGTH); + } + return headers; } @Override diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 4c7465310fad..d1ad478fde14 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -12,6 +12,7 @@ import com.azure.core.http.policy.HttpPipelinePolicy; import com.azure.core.http.HttpPipelinePosition; import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; import reactor.core.publisher.Flux; @@ -37,9 +38,6 @@ */ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); - private static final HttpHeaderName X_MS_STRUCTURED_BODY = HttpHeaderName.fromString("x-ms-structured-body"); - private static final HttpHeaderName X_MS_STRUCTURED_CONTENT_LENGTH - = HttpHeaderName.fromString("x-ms-structured-content-length"); /** * Creates a new instance of {@link StorageContentValidationDecoderPolicy}. @@ -60,7 +58,8 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN context.getHttpRequest() .getHeaders() - .set(X_MS_STRUCTURED_BODY, StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); + .set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, + StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); return next.process().map(httpResponse -> { Long contentLength = getContentLength(httpResponse.getHeaders()); @@ -86,8 +85,10 @@ private boolean shouldApplyDecoding(HttpPipelineCallContext context) { } private void validateStructuredMessageHeaders(HttpResponse httpResponse) { - String structuredBody = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_BODY); - String structuredContentLength = httpResponse.getHeaders().getValue(X_MS_STRUCTURED_CONTENT_LENGTH); + String structuredBody + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String structuredContentLength + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); if (structuredBody == null || structuredContentLength == null) { throw LOGGER.logExceptionAsError( new IllegalStateException("Structured message was requested but the response did not acknowledge it.")); @@ -119,7 +120,6 @@ private static boolean isEligibleDownload(HttpResponse response, Long contentLen private Flux decodeStream(Flux encodedFlux, StructuredMessageDecoder decoder) { return encodedFlux.concatMap(buffer -> decodeBuffer(buffer, decoder)) - .onErrorResume(throwable -> handleStreamError(throwable, decoder)) .concatWith(Mono.defer(() -> handleStreamCompletion(decoder))); } @@ -143,14 +143,6 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode } } - private Flux handleStreamError(Throwable throwable, StructuredMessageDecoder decoder) { - if (decoder.isComplete()) { - return Flux.empty(); - } - - return Flux.error(throwable); - } - private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { if (!decoder.isComplete()) { return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); From d7b0c5134e88aa722ee1d1c4f4c264f3eaafcdb6 Mon Sep 17 00:00:00 2001 From: Gunjan Singh Date: Tue, 28 Apr 2026 20:40:00 +0530 Subject: [PATCH 22/31] refactoring based on latest review comments from kyle --- .../implementation/util/BuilderHelper.java | 3 +- .../blob/specialized/BlobAsyncClientBase.java | 10 ++-- .../BlobMessageAsyncDecoderDownloadTests.java | 47 +++++++++------- .../blob/BlobMessageDecoderDownloadTests.java | 56 ++++++++++--------- .../com/azure/storage/blob/BlobTestBase.java | 12 ++++ .../common/policy/DecodedResponse.java | 39 +++++-------- ...StorageContentValidationDecoderPolicy.java | 14 +---- 7 files changed, 91 insertions(+), 90 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index 614a320bb270..53fcb67447dc 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -98,8 +98,6 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare // Closest to API goes first, closest to wire goes last. List policies = new ArrayList<>(); - policies.add(new StorageContentValidationDecoderPolicy()); - policies.add(getUserAgentPolicy(configuration, logOptions, clientOptions)); policies.add(new RequestIdPolicy()); @@ -120,6 +118,7 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare policies.add(new MetadataValidationPolicy()); policies.add(new StorageContentValidationPolicy()); + policies.add(new StorageContentValidationDecoderPolicy()); if (storageSharedKeyCredential != null) { policies.add(new StorageSharedKeyCredentialPolicy(storageSharedKeyCredential)); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index d653c2bc52ee..acd575bd475b 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1271,13 +1271,11 @@ Mono downloadStreamWithResponseInternal(BlobRange ran = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; - // The first range should eagerly convert headers as they'll be used to create response types. - Context firstRangeContext = context == null + // Eagerly convert headers for the response types and propagate any structured-message decoding flag. + // The same context is used for the initial range and any retry ranges. + Context downloadContext = ContentValidationModeResolver.addStructuredMessageDecodingToContext(context == null ? new Context("azure-eagerly-convert-headers", true) - : context.addData("azure-eagerly-convert-headers", true); - - Context downloadContext = ContentValidationModeResolver.addStructuredMessageDecodingToContext(firstRangeContext, - contentValidationAlgorithm); + : context.addData("azure-eagerly-convert-headers", true), contentValidationAlgorithm); return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, downloadContext).map(response -> { diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java index cb8890c22617..bdb15fb9f181 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java @@ -3,8 +3,10 @@ package com.azure.storage.blob; +import com.azure.core.http.HttpHeaders; import com.azure.core.test.TestMode; import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; import com.azure.core.util.FluxUtil; import com.azure.storage.blob.models.BlobRange; import com.azure.storage.blob.models.BlobRequestConditions; @@ -36,6 +38,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -65,10 +69,9 @@ public void setup() { public void downloadStreamWithResponseContentValidation() throws IOException { byte[] randomData = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(randomData)).block(); StepVerifier .create(downloadClient @@ -77,6 +80,7 @@ public void downloadStreamWithResponseContentValidation() throws IOException { .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -85,16 +89,17 @@ public void downloadStreamWithResponseContentValidation() throws IOException { @Test public void downloadContentWithResponseContentValidation() { byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(data)).block(); StepVerifier .create(downloadClient.downloadContentWithResponse( new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -106,10 +111,10 @@ public void downloadContentWithResponseContentValidation() { public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { int payloadSize = (4 * blockSize) + 1; byte[] randomData = getRandomByteArray(payloadSize); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(randomData)).block(); Path tempFile = Files.createTempFile("structured-download", ".bin"); Files.deleteIfExists(tempFile); @@ -126,6 +131,7 @@ public void downloadToFileWithResponseContentValidation(int blockSize) throws IO .verify(Duration.ofSeconds(60)); TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } finally { Files.deleteIfExists(tempFile); } @@ -138,10 +144,10 @@ public void downloadToFileWithResponseContentValidation(int blockSize) throws IO public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOException { int dataLength = 10 * 1024 * 1024; byte[] data = getRandomByteArray(dataLength); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(data)).block(); long expectedCrc = StorageCrc64Calculator.compute(data, 0); byte[] expectedCrcBytes = new byte[8]; @@ -159,6 +165,7 @@ public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOExceptio TestUtils.assertArraysEqual(expectedCrcBytes, tuple.getT1().getDeserializedHeaders().getContentCrc64()); }) .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -344,10 +351,10 @@ public void downloadStreamDefaultAlgorithmIsNone() { @Test public void downloadStreamWithAuto() { byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(data)).block(); StepVerifier .create(downloadClient @@ -356,6 +363,7 @@ public void downloadStreamWithAuto() { .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) .assertNext(result -> TestUtils.assertArraysEqual(data, result)) .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -379,16 +387,17 @@ public void downloadContentWithNone() { @Test public void downloadContentWithAuto() { byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(data)).block(); StepVerifier .create(downloadClient.downloadContentWithResponse( new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO))) .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } static boolean isLiveMode() { diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java index 2900165c13e7..ef579945ac68 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java @@ -3,7 +3,9 @@ package com.azure.storage.blob; +import com.azure.core.http.HttpHeaders; import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; import com.azure.core.util.Context; import com.azure.storage.blob.options.BlobDownloadContentOptions; import com.azure.storage.blob.options.BlobDownloadStreamOptions; @@ -13,14 +15,12 @@ import com.azure.storage.blob.specialized.BlobInputStream; import com.azure.storage.common.ParallelTransferOptions; import com.azure.storage.common.ContentValidationAlgorithm; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import reactor.core.publisher.Flux; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -28,9 +28,12 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Sync tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. @@ -39,31 +42,24 @@ @Execution(ExecutionMode.SAME_THREAD) public class BlobMessageDecoderDownloadTests extends BlobTestBase { - private BlobAsyncClient bc; - - @BeforeEach - public void setup() { - String blobName = generateBlobName(); - bc = ccAsync.getBlobAsyncClient(blobName); - bc.upload(Flux.just(ByteBuffer.wrap(new byte[0])), null).block(); - } - /** * downloadStreamWithResponse with CRC64 content validation. */ @Test public void downloadStreamWithResponseContentValidation() { byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - syncClient.downloadStreamWithResponse(outputStream, + client.downloadStreamWithResponse(outputStream, new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, Context.NONE); TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -72,12 +68,13 @@ public void downloadStreamWithResponseContentValidation() { @Test public void downloadContentWithResponseContentValidation() { byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); byte[] result - = syncClient + = client .downloadContentWithResponse( new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, Context.NONE) @@ -85,6 +82,7 @@ public void downloadContentWithResponseContentValidation() { .toBytes(); TestUtils.assertArraysEqual(data, result); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -96,9 +94,10 @@ public void downloadContentWithResponseContentValidation() { public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { int payloadSize = (4 * blockSize) + 1; byte[] randomData = getRandomByteArray(payloadSize); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(randomData)); Path tempFile = Files.createTempFile("structured-download-sync", ".bin"); Files.deleteIfExists(tempFile); @@ -109,8 +108,9 @@ public void downloadToFileWithResponseContentValidation(int blockSize) throws IO .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); try { - assertNotNull(downloadClient.downloadToFileWithResponse(options, null, Context.NONE).getValue()); + assertNotNull(client.downloadToFileWithResponse(options, null, Context.NONE).getValue()); TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } finally { Files.deleteIfExists(tempFile); } @@ -122,11 +122,12 @@ public void downloadToFileWithResponseContentValidation(int blockSize) throws IO @Test public void openInputStreamContentValidation() throws IOException { byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); - try (BlobInputStream blobInputStream = syncClient.openInputStream( + try (BlobInputStream blobInputStream = client.openInputStream( new BlobInputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), Context.NONE)) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -137,6 +138,7 @@ public void openInputStreamContentValidation() throws IOException { } TestUtils.assertArraysEqual(data, baos.toByteArray()); } + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -145,11 +147,12 @@ public void openInputStreamContentValidation() throws IOException { @Test public void openSeekableByteChannelReadContentValidation() throws IOException { byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - BlobClient syncClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl()); + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); - try (SeekableByteChannel channel = syncClient.openSeekableByteChannelRead( + try (SeekableByteChannel channel = client.openSeekableByteChannelRead( new BlobSeekableByteChannelReadOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), Context.NONE).getChannel()) { ByteBuffer buf = ByteBuffer.allocate(data.length + 100); @@ -163,5 +166,6 @@ public void openSeekableByteChannelReadContentValidation() throws IOException { buf.get(result, 0, totalRead); TestUtils.assertArraysEqual(data, result); } + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java index e4e49ff383d9..514ff455fb90 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java @@ -1382,6 +1382,15 @@ public static HttpPipelinePolicy getAddHeadersAndQueryPolicy(Map } protected static boolean hasOnlyStructuredMessageHeaders(List recordedRequestHeaders) { + return hasStructuredMessageRequestHeaders(recordedRequestHeaders, true); + } + + protected static boolean hasOnlyStructuredMessageDownloadHeaders(List recordedRequestHeaders) { + return hasStructuredMessageRequestHeaders(recordedRequestHeaders, false); + } + + private static boolean hasStructuredMessageRequestHeaders(List recordedRequestHeaders, + boolean requireStructuredContentLength) { if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { return false; } @@ -1404,6 +1413,9 @@ protected static boolean hasOnlyStructuredMessageHeaders(List recor if (!StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE.equals(bodyType) || contentCrc64 != null) { return false; } + if (!requireStructuredContentLength) { + return true; + } // Require non-blank content length that parses as non-negative long (same format as policy uses). // Rejects empty string, whitespace, or non-numeric values so we never return true when // structured message was not actually applied. diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java index f962e869cd2b..abc18d20fe8d 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -3,11 +3,9 @@ package com.azure.storage.common.policy; -import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; import com.azure.core.util.FluxUtil; -import com.azure.storage.common.implementation.Constants; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -18,60 +16,49 @@ * Decoded HTTP response that wraps the original response with a decoded body stream. */ class DecodedResponse extends HttpResponse { - private final HttpResponse originalResponse; private final Flux decodedBody; + private final HttpHeaders httpHeaders; + private final int statusCode; - DecodedResponse(HttpResponse originalResponse, Flux decodedBody) { - super(originalResponse.getRequest()); - this.originalResponse = originalResponse; + DecodedResponse(HttpResponse httpResponse, Flux decodedBody) { + super(httpResponse.getRequest()); this.decodedBody = decodedBody; + this.statusCode = httpResponse.getStatusCode(); + this.httpHeaders = httpResponse.getHeaders(); } @Override public int getStatusCode() { - return originalResponse.getStatusCode(); + return statusCode; } @Override public String getHeaderValue(String name) { - return originalResponse.getHeaderValue(name); + return httpHeaders.getValue(name); } @Override public HttpHeaders getHeaders() { - HttpHeaders headers = new HttpHeaders(originalResponse.getHeaders()); - String structuredContentLength - = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); - if (structuredContentLength != null) { - headers.set(HttpHeaderName.CONTENT_LENGTH, structuredContentLength); - } else { - headers.remove(HttpHeaderName.CONTENT_LENGTH); - } - return headers; + return httpHeaders; } @Override public Flux getBody() { - return Flux.using(() -> originalResponse, r -> decodedBody, HttpResponse::close); + return decodedBody; } @Override public Mono getBodyAsByteArray() { - return FluxUtil.collectBytesInByteBufferStream(getBody()); + return FluxUtil.collectBytesInByteBufferStream(decodedBody); } @Override public Mono getBodyAsString() { - return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); + return FluxUtil.collectBytesInByteBufferStream(decodedBody).map(String::new); } @Override public Mono getBodyAsString(Charset charset) { - return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); - } - - @Override - public void close() { - originalResponse.close(); + return FluxUtil.collectBytesInByteBufferStream(decodedBody).map(b -> new String(b, charset)); } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index d1ad478fde14..e1961d9d77e9 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -10,7 +10,6 @@ import com.azure.core.http.HttpPipelineNextPolicy; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; -import com.azure.core.http.HttpPipelinePosition; import com.azure.core.util.logging.ClientLogger; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; @@ -28,13 +27,11 @@ * *

The policy is activated by the presence of a boolean context key * ({@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY}). It validates per-segment - * CRC64 checksums during decoding. The policy is registered early in the pipeline; each retry re-enters the policy - * with a fresh response body, so no cross-retry state management is needed in the policy.

+ * CRC64 checksums during decoding.

* *

Emission guarantee: the policy only forwards payload bytes that the - * {@link StructuredMessageDecoder} has already CRC-validated at the segment boundary. Retries never expose - * partially validated bytes because each attempt creates a fresh decoder and the decoder itself withholds - * bytes until their enclosing segment passes validation.

+ * {@link StructuredMessageDecoder} has already CRC-validated at the segment boundary. Each invocation creates + * a fresh decoder and the decoder itself withholds bytes until their enclosing segment passes validation.

*/ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); @@ -45,11 +42,6 @@ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy public StorageContentValidationDecoderPolicy() { } - @Override - public HttpPipelinePosition getPipelinePosition() { - return HttpPipelinePosition.PER_RETRY; - } - @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { if (!shouldApplyDecoding(context)) { From 73f22e99cefe6cdf5e48a7f4b0a0ed27576da60a Mon Sep 17 00:00:00 2001 From: Isabelle Date: Tue, 28 Apr 2026 12:16:21 -0700 Subject: [PATCH 23/31] expanding test coverage --- ...bContentValidationAsyncDownloadTests.java} | 400 ++++++++-------- ...BlobContentValidationAsyncUploadTests.java | 3 +- .../BlobContentValidationDownloadTests.java | 433 ++++++++++++++++++ .../blob/BlobMessageDecoderDownloadTests.java | 171 ------- .../BlobSeekableByteChannelTests.java | 2 +- 5 files changed, 654 insertions(+), 355 deletions(-) rename sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/{BlobMessageAsyncDecoderDownloadTests.java => BlobContentValidationAsyncDownloadTests.java} (58%) create mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java delete mode 100644 sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java similarity index 58% rename from sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java rename to sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java index bdb15fb9f181..b49108415ce3 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageAsyncDecoderDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java @@ -4,7 +4,7 @@ package com.azure.storage.blob; import com.azure.core.http.HttpHeaders; -import com.azure.core.test.TestMode; +import com.azure.core.http.policy.HttpPipelinePolicy; import com.azure.core.test.utils.TestUtils; import com.azure.core.util.BinaryData; import com.azure.core.util.FluxUtil; @@ -19,30 +19,26 @@ import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.test.shared.extensions.LiveOnly; import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import reactor.core.publisher.Flux; import reactor.test.StepVerifier; import reactor.util.function.Tuples; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -50,35 +46,33 @@ * Async tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. */ -@Execution(ExecutionMode.SAME_THREAD) -public class BlobMessageAsyncDecoderDownloadTests extends BlobTestBase { +public class BlobContentValidationAsyncDownloadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + private final List createdFiles = new ArrayList<>(); - private BlobAsyncClient bc; - - @BeforeEach - public void setup() { - String blobName = generateBlobName(); - bc = ccAsync.getBlobAsyncClient(blobName); - bc.upload(Flux.just(ByteBuffer.wrap(new byte[0])), null).block(); + @AfterEach + public void cleanup() { + createdFiles.forEach(File::delete); } /** * downloadStreamWithResponse with CRC64 content validation. */ @Test - public void downloadStreamWithResponseContentValidation() throws IOException { - byte[] randomData = getRandomByteArray(10 * 1024 * 1024); + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); List recorded = new CopyOnWriteArrayList<>(); BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); - downloadClient.upload(BinaryData.fromBytes(randomData)).block(); + downloadClient.upload(BinaryData.fromBytes(data)).block(); + + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); StepVerifier - .create(downloadClient - .downloadStreamWithResponse( - new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .create(downloadClient.downloadStreamWithResponse(options) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(r -> TestUtils.assertArraysEqual(r, randomData)) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) .verifyComplete(); assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } @@ -88,83 +82,93 @@ public void downloadStreamWithResponseContentValidation() throws IOException { */ @Test public void downloadContentWithResponseContentValidation() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); + byte[] data = getRandomByteArray(TEN_MB); List recorded = new CopyOnWriteArrayList<>(); BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); downloadClient.upload(BinaryData.fromBytes(data)).block(); - StepVerifier - .create(downloadClient.downloadContentWithResponse( - new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) + BlobDownloadContentOptions options + = new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(downloadClient.downloadContentWithResponse(options)) .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) .verifyComplete(); assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** - * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + * downloadToFileWithResponse with CRC64 content validation. */ @ParameterizedTest - @ValueSource(ints = { 512, 2048 }) - @Timeout(value = 5, unit = TimeUnit.MINUTES) - public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { - int payloadSize = (4 * blockSize) + 1; - byte[] randomData = getRandomByteArray(payloadSize); + @ValueSource( + ints = { + 0, // empty file + 20, // small file + 16 * 1024 * 1024, // medium file in several chunks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize) throws IOException { + File file = getRandomFile(fileSize); + file.deleteOnExit(); + createdFiles.add(file); List recorded = new CopyOnWriteArrayList<>(); BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); - downloadClient.upload(BinaryData.fromBytes(randomData)).block(); + downloadClient.uploadFromFile(file.toPath().toString(), true).block(); - Path tempFile = Files.createTempFile("structured-download", ".bin"); - Files.deleteIfExists(tempFile); + File outFile = new File(prefix + ".txt"); + createdFiles.add(outFile); + outFile.deleteOnExit(); + Files.deleteIfExists(outFile.toPath()); - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize); + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(4L * 1024 * 1024); BlobDownloadToFileOptions options - = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); - try { - StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) - .assertNext(r -> assertNotNull(r.getValue())) - .expectComplete() - .verify(Duration.ofSeconds(60)); - - TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); - assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); - } finally { - Files.deleteIfExists(tempFile); - } + StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) + .assertNext(r -> assertNotNull(r.getValue())) + .verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** - * After consuming the response stream with CRC64 validation, ContentCrc64 header is populated. + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). */ - @Test - public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOException { - int dataLength = 10 * 1024 * 1024; - byte[] data = getRandomByteArray(dataLength); + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize) throws IOException { + File file = getRandomFile(fileSize); + file.deleteOnExit(); + createdFiles.add(file); List recorded = new CopyOnWriteArrayList<>(); BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); - downloadClient.upload(BinaryData.fromBytes(data)).block(); + downloadClient.uploadFromFile(file.toPath().toString(), true).block(); - long expectedCrc = StorageCrc64Calculator.compute(data, 0); - byte[] expectedCrcBytes = new byte[8]; - ByteBuffer.wrap(expectedCrcBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(expectedCrc); + File outFile = new File(prefix + ".txt"); + createdFiles.add(outFile); + outFile.deleteOnExit(); + Files.deleteIfExists(outFile.toPath()); - StepVerifier - .create(downloadClient - .downloadStreamWithResponse( - new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes)))) - .assertNext(tuple -> { - TestUtils.assertArraysEqual(data, tuple.getT2()); - assertNotNull(tuple.getT1().getDeserializedHeaders().getContentCrc64(), - "ContentCrc64 should be populated after stream consumption"); - TestUtils.assertArraysEqual(expectedCrcBytes, tuple.getT1().getDeserializedHeaders().getContentCrc64()); - }) + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(4L * 1024 * 1024); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(downloadClient.downloadToFileWithResponse(options)) + .assertNext(r -> assertNotNull(r.getValue())) .verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } @@ -172,90 +176,120 @@ public void structuredMessagePopulatesCrc64DownloadStreaming() throws IOExceptio * Range download without content validation works correctly. */ @Test - public void downloadStreamWithResponseContentValidationRange() throws IOException { + public void downloadStreamWithResponseContentValidationRange() { byte[] randomData = getRandomByteArray(4 * Constants.KB); Flux input = Flux.just(ByteBuffer.wrap(randomData)); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); BlobRange range = new BlobRange(0, 512L); - StepVerifier.create(bc.upload(input, null, true) - .then( - bc.downloadStreamWithResponse(range, (DownloadRetryOptions) null, (BlobRequestConditions) null, false)) + StepVerifier.create(downloadClient.upload(input, null, true) + .then(downloadClient.downloadStreamWithResponse(range, null, null, false)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(r -> { assertNotNull(r); assertEquals(512, r.length); }).verifyComplete(); + assertFalse(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** - * Single interrupt with data intact: fault policy + decoder; structured message retry recovers. + * Default behavior: when no algorithm is specified, default is NONE (no validation). */ @Test - public void interruptWithDataIntact() throws IOException { - final int segmentSize = Constants.KB; - byte[] randomData = getRandomByteArray(4 * segmentSize); - - int interruptPos = segmentSize + (3 * 128) + 10; - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + StepVerifier.create(downloadClient.downloadStreamWithResponse(new BlobDownloadStreamOptions()) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + assertNotNull(result); + assertEquals(data.length, result.length); + }).verifyComplete(); + assertFalse(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(data)).block(); StepVerifier .create(downloadClient - .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) - .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** - * Multiple interrupts with data intact: fault policy + decoder; structured message retry recovers. + * downloadContentWithResponse with NONE: no validation triggered. */ @Test - public void interruptMultipleTimesWithDataIntact() throws IOException { - final int segmentSize = Constants.KB; - byte[] randomData = getRandomByteArray(4 * segmentSize); - - int interruptPos = segmentSize + (3 * 128) + 10; - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, bc.getBlobUrl()); + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + StepVerifier + .create(downloadClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + assertFalse(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); - DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(data)).block(); StepVerifier - .create(downloadClient - .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) - .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) + .create(downloadClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** * Interrupt with proper rewind to segment boundary; verifies retry range headers. */ @Test - public void interruptAndVerifyProperRewind() throws IOException { + public void interruptAndVerifyProperRewind() { final int segmentSize = Constants.KB; byte[] randomData = getRandomByteArray(2 * segmentSize); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; - MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, bc.getBlobUrl()); + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recorded.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + blobClient.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); @@ -272,6 +306,7 @@ public void interruptAndVerifyProperRewind() throws IOException { assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); assertTrue(mockPolicy.getRangeHeaders().size() >= 2, "Expected at least the initial request and one retry with a range header"); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** @@ -279,19 +314,25 @@ public void interruptAndVerifyProperRewind() throws IOException { */ @ParameterizedTest @ValueSource(booleans = { false, true }) - public void interruptAndVerifyProperDecode(boolean multipleInterrupts) throws IOException { + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { final int segmentSize = 128 * Constants.KB; final int dataSize = 4 * Constants.KB; byte[] randomData = getRandomByteArray(dataSize); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; MockPartialResponsePolicy mockPolicy - = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, bc.getBlobUrl()); + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recorded.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; - bc.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); + blobClient.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - BlobAsyncClient downloadClient - = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), bc.getBlobUrl(), mockPolicy); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); @@ -302,105 +343,102 @@ public void interruptAndVerifyProperDecode(boolean multipleInterrupts) throws IO assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); TestUtils.assertArraysEqual(randomData, result); }).verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** - * Older service version throws when structured message validation is requested. + * After consuming the response stream with CRC64 validation, decoded payload preserves the expected CRC64. */ @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - @EnabledIf("isLiveMode") - public void olderServiceVersionThrowsOnStructuredMessage() { - int dataLength = 10 * 1024 * 1024; - byte[] data = getRandomByteArray(dataLength); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); - - BlobClientBuilder builder = new BlobClientBuilder().endpoint(bc.getBlobUrl()) - .credential(ENVIRONMENT.getPrimaryAccount().getCredential()) - .serviceVersion(BlobServiceVersion.V2024_11_04); - instrument(builder); - BlobAsyncClient oldVersionClient = builder.buildAsyncClient(); + public void structuredMessageVerifiesDecodedCrc64DownloadStreaming() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); + downloadClient.upload(BinaryData.fromBytes(data)).block(); + + long expectedCrc = StorageCrc64Calculator.compute(data, 0); StepVerifier - .create(oldVersionClient + .create(downloadClient .downloadStreamWithResponse( - new BlobDownloadStreamOptions().setRange(new BlobRange(0, (long) (10 * 1024 * 1024))) - .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .verifyError(BlobStorageException.class); + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes)))) + .assertNext(tuple -> { + TestUtils.assertArraysEqual(data, tuple.getT2()); + long actualCrc = StorageCrc64Calculator.compute(tuple.getT2(), 0); + assertEquals(expectedCrc, actualCrc); + }) + .verifyComplete(); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** - * Default behavior: when no algorithm is specified, default is NONE (no validation). + * Single interrupt with data intact: fault policy + decoder; structured message retry recovers. */ @Test - public void downloadStreamDefaultAlgorithmIsNone() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + public void interruptWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(4 * segmentSize); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); - StepVerifier.create(bc.downloadStreamWithResponse(new BlobDownloadStreamOptions()) - .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { - assertNotNull(result); - assertEquals(data.length, result.length); - }).verifyComplete(); - } + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recorded.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; - /** - * AUTO on downloadStream resolves to CRC64 behavior. - */ - @Test - public void downloadStreamWithAuto() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); + blobClient.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - List recorded = new CopyOnWriteArrayList<>(); - BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); - downloadClient.upload(BinaryData.fromBytes(data)).block(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); StepVerifier .create(downloadClient - .downloadStreamWithResponse( - new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO)) + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) - .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) .verifyComplete(); assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } /** - * downloadContentWithResponse with NONE: no validation triggered. + * Multiple interrupts with data intact: fault policy + decoder; structured message retry recovers. */ @Test - public void downloadContentWithNone() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - bc.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + public void interruptMultipleTimesWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(4 * segmentSize); + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); - StepVerifier - .create(bc.downloadContentWithResponse( - new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) - .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) - .verifyComplete(); - } + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recorded.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; - /** - * downloadContentWithResponse with AUTO resolves to CRC64 behavior. - */ - @Test - public void downloadContentWithAuto() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); + blobClient.upload(Flux.just(ByteBuffer.wrap(randomData)), null, true).block(); - List recorded = new CopyOnWriteArrayList<>(); - BlobAsyncClient downloadClient = createBlobAsyncClientWithRequestSniffer(recorded); - downloadClient.upload(BinaryData.fromBytes(data)).block(); + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); StepVerifier - .create(downloadClient.downloadContentWithResponse( - new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO))) - .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(randomData, result)) .verifyComplete(); assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); } - static boolean isLiveMode() { - return ENVIRONMENT.getTestMode() == TestMode.LIVE; - } } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java index dc23983506b8..0d9b8e0e45e8 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java @@ -55,7 +55,7 @@ public class BlobContentValidationAsyncUploadTests extends BlobTestBase { private static final int UNDER_4MB = 2 * Constants.MB; private static final long LARGE_UPLOAD_MIN_BYTES = 500L * Constants.MB; - private static final long LARGE_UPLOAD_MAX_BYTES = 1L * Constants.GB; + private static final long LARGE_UPLOAD_MAX_BYTES = Constants.GB; private static final long LARGE_UPLOAD_BLOCK_SIZE_BYTES = 8L * Constants.MB; private static final int LARGE_UPLOAD_MAX_CONCURRENCY = 8; @@ -163,7 +163,6 @@ public void uploadWithoutContentValidation() { /** * Blob parallel upload rejects using both computeMd5 (SDK-computed MD5) and CRC64 (transfer validation checksum algorithm) at once. */ - @SuppressWarnings("deprecation") @Test public void uploadWithComputeMd5AndCrc64Throws() { BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java new file mode 100644 index 000000000000..86b7f116a60d --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java @@ -0,0 +1,433 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.specialized.BlobInputStream; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Stream; + +import static com.azure.storage.blob.specialized.BlobSeekableByteChannelTests.copy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Sync tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobContentValidationDownloadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + private final List createdFiles = new ArrayList<>(); + + @AfterEach + public void cleanup() { + createdFiles.forEach(File::delete); + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + client.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, + Context.NONE); + + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + byte[] result + = client + .downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, Context.NONE) + .getValue() + .toBytes(); + + TestUtils.assertArraysEqual(data, result); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @ParameterizedTest + @ValueSource( + ints = { + 0, // empty file + 20, // small file + 16 * 1024 * 1024, // medium file in several chunks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize) throws IOException { + File file = getRandomFile(fileSize); + file.deleteOnExit(); + createdFiles.add(file); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.uploadFromFile(file.toPath().toString(), true); + + File outFile = new File(prefix + ".txt"); + createdFiles.add(outFile); + outFile.deleteOnExit(); + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(4L * 1024 * 1024); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + assertNotNull(client.downloadToFileWithResponse(options, null, Context.NONE).getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize) throws IOException { + File file = getRandomFile(fileSize); + file.deleteOnExit(); + createdFiles.add(file); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.uploadFromFile(file.toPath().toString(), true); + + File outFile = new File(prefix + ".txt"); + createdFiles.add(outFile); + outFile.deleteOnExit(); + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(4L * 1024 * 1024); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + assertNotNull(client.downloadToFileWithResponse(options, null, Context.NONE).getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * Range download without content validation works correctly. + */ + @Test + public void downloadStreamWithResponseContentValidationRange() { + byte[] randomData = getRandomByteArray(4 * Constants.KB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(randomData)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setRange(new BlobRange(0, 512L)); + client.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertEquals(512, outputStream.toByteArray().length); + assertFalse(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + client.downloadStreamWithResponse(outputStream, new BlobDownloadStreamOptions(), null, Context.NONE); + + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertFalse(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO); + client.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + byte[] result + = client + .downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE), + null, Context.NONE) + .getValue() + .toBytes(); + + TestUtils.assertArraysEqual(data, result); + assertFalse(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + byte[] result + = client + .downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO), + null, Context.NONE) + .getValue() + .toBytes(); + + TestUtils.assertArraysEqual(data, result); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() { + final int segmentSize = Constants.KB; + byte[] randomData = getRandomByteArray(2 * segmentSize); + List recorded = new CopyOnWriteArrayList<>(); + + BlobClient uploadClient = createBlobClientWithRequestSniffer(recorded); + uploadClient.upload(BinaryData.fromBytes(randomData)); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(1, interruptPos, uploadClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recorded.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + uploadClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + TestUtils.assertArraysEqual(randomData, outputStream.toByteArray()); + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] randomData = getRandomByteArray(dataSize); + List recorded = new CopyOnWriteArrayList<>(); + + BlobClient uploadClient = createBlobClientWithRequestSniffer(recorded); + uploadClient.upload(BinaryData.fromBytes(randomData)); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, uploadClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recorded.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + uploadClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + byte[] result = outputStream.toByteArray(); + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(randomData, result); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options + = new BlobInputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobInputStream inputStream = client.openInputStream(options, Context.NONE); + + TestUtils.assertArraysEqual(data, convertInputStreamToByteArray(inputStream)); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamRangeContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + + int start = Constants.MB; + int count = 3 * Constants.MB + 257; + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options = new BlobInputStreamOptions().setRange(new BlobRange(start, (long) count)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setBlockSize(Constants.MB); + BlobInputStream inputStream = client.openInputStream(options, Context.NONE); + + byte[] downloadedRange = convertInputStreamToByteArray(inputStream); + assertEquals(count, downloadedRange.length); + TestUtils.assertArraysEqual(data, start, downloadedRange, 0, count); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + /** + * openSeekableByteChannelRead with CRC64 content validation. + */ + @ParameterizedTest + @MethodSource("channelReadDataSupplier") + public void openSeekableByteChannelReadContentValidation(Integer streamBufferSize, int copyBufferSize, + int dataLength) throws IOException { + byte[] data = getRandomByteArray(dataLength); + + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + client.upload(BinaryData.fromBytes(data)); + + // when: "Channel initialized" + BlobSeekableByteChannelReadOptions options + = new BlobSeekableByteChannelReadOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setReadSizeInBytes(streamBufferSize); + BlobSeekableByteChannelReadResult result = client.openSeekableByteChannelRead(options, Context.NONE); + SeekableByteChannel channel = result.getChannel(); + + // then: "Channel initialized to position zero" + assertEquals(0, channel.position()); + assertNotNull(result.getProperties()); + assertEquals(data.length, result.getProperties().getBlobSize()); + + // when: "read from channel" + ByteArrayOutputStream downloadedData = new ByteArrayOutputStream(); + int copied = copy(channel, downloadedData, copyBufferSize); + + // then: "channel position updated accordingly" + assertEquals(dataLength, copied); + assertEquals(dataLength, channel.position()); + + // and: "expected data downloaded" + TestUtils.assertArraysEqual(data, downloadedData.toByteArray()); + assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); + } + + static Stream channelReadDataSupplier() { + return Stream.of(Arguments.of(50, 40, Constants.KB), Arguments.of(Constants.KB + 50, 40, Constants.KB), + Arguments.of(null, Constants.MB, TEN_MB)); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java deleted file mode 100644 index ef579945ac68..000000000000 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobMessageDecoderDownloadTests.java +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.storage.blob; - -import com.azure.core.http.HttpHeaders; -import com.azure.core.test.utils.TestUtils; -import com.azure.core.util.BinaryData; -import com.azure.core.util.Context; -import com.azure.storage.blob.options.BlobDownloadContentOptions; -import com.azure.storage.blob.options.BlobDownloadStreamOptions; -import com.azure.storage.blob.options.BlobDownloadToFileOptions; -import com.azure.storage.blob.options.BlobInputStreamOptions; -import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; -import com.azure.storage.blob.specialized.BlobInputStream; -import com.azure.storage.common.ParallelTransferOptions; -import com.azure.storage.common.ContentValidationAlgorithm; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.SeekableByteChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Sync tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. - * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. - */ -@Execution(ExecutionMode.SAME_THREAD) -public class BlobMessageDecoderDownloadTests extends BlobTestBase { - - /** - * downloadStreamWithResponse with CRC64 content validation. - */ - @Test - public void downloadStreamWithResponseContentValidation() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - - List recorded = new CopyOnWriteArrayList<>(); - BlobClient client = createBlobClientWithRequestSniffer(recorded); - client.upload(BinaryData.fromBytes(data)); - - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - client.downloadStreamWithResponse(outputStream, - new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, - Context.NONE); - - TestUtils.assertArraysEqual(data, outputStream.toByteArray()); - assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); - } - - /** - * downloadContentWithResponse with CRC64 content validation. - */ - @Test - public void downloadContentWithResponseContentValidation() { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - - List recorded = new CopyOnWriteArrayList<>(); - BlobClient client = createBlobClientWithRequestSniffer(recorded); - client.upload(BinaryData.fromBytes(data)); - - byte[] result - = client - .downloadContentWithResponse( - new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), - null, Context.NONE) - .getValue() - .toBytes(); - - TestUtils.assertArraysEqual(data, result); - assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); - } - - /** - * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). - */ - @ParameterizedTest - @ValueSource(ints = { 512, 2048 }) - @Timeout(value = 5, unit = TimeUnit.MINUTES) - public void downloadToFileWithResponseContentValidation(int blockSize) throws IOException { - int payloadSize = (4 * blockSize) + 1; - byte[] randomData = getRandomByteArray(payloadSize); - - List recorded = new CopyOnWriteArrayList<>(); - BlobClient client = createBlobClientWithRequestSniffer(recorded); - client.upload(BinaryData.fromBytes(randomData)); - - Path tempFile = Files.createTempFile("structured-download-sync", ".bin"); - Files.deleteIfExists(tempFile); - - ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) blockSize); - BlobDownloadToFileOptions options - = new BlobDownloadToFileOptions(tempFile.toString()).setParallelTransferOptions(parallelOptions) - .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); - - try { - assertNotNull(client.downloadToFileWithResponse(options, null, Context.NONE).getValue()); - TestUtils.assertArraysEqual(randomData, Files.readAllBytes(tempFile)); - assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); - } finally { - Files.deleteIfExists(tempFile); - } - } - - /** - * openInputStream with CRC64 content validation. - */ - @Test - public void openInputStreamContentValidation() throws IOException { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - - List recorded = new CopyOnWriteArrayList<>(); - BlobClient client = createBlobClientWithRequestSniffer(recorded); - client.upload(BinaryData.fromBytes(data)); - - try (BlobInputStream blobInputStream = client.openInputStream( - new BlobInputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), - Context.NONE)) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buf = new byte[4096]; - int n; - while ((n = blobInputStream.read(buf)) != -1) { - baos.write(buf, 0, n); - } - TestUtils.assertArraysEqual(data, baos.toByteArray()); - } - assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); - } - - /** - * openSeekableByteChannelRead with CRC64 content validation. - */ - @Test - public void openSeekableByteChannelReadContentValidation() throws IOException { - byte[] data = getRandomByteArray(10 * 1024 * 1024); - - List recorded = new CopyOnWriteArrayList<>(); - BlobClient client = createBlobClientWithRequestSniffer(recorded); - client.upload(BinaryData.fromBytes(data)); - - try (SeekableByteChannel channel = client.openSeekableByteChannelRead( - new BlobSeekableByteChannelReadOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), - Context.NONE).getChannel()) { - ByteBuffer buf = ByteBuffer.allocate(data.length + 100); - int totalRead = 0; - int bytesRead; - while ((bytesRead = channel.read(buf)) > 0) { - totalRead += bytesRead; - } - buf.flip(); - byte[] result = new byte[totalRead]; - buf.get(result, 0, totalRead); - TestUtils.assertArraysEqual(data, result); - } - assertTrue(hasOnlyStructuredMessageDownloadHeaders(recorded)); - } -} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java index 4ff8054894a6..24a9ca7e781a 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java @@ -106,7 +106,7 @@ static Stream channelReadDataSupplier() { * @param copySize Size of array to copy contents with. * @return Total number of bytes read from src. */ - private static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { + public static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { int read; int totalRead = 0; byte[] temp = new byte[copySize]; From c5f3a8c51cd65dc203099455fcb86914e9ef470a Mon Sep 17 00:00:00 2001 From: Isabelle Date: Tue, 28 Apr 2026 14:08:02 -0700 Subject: [PATCH 24/31] removing unused imports --- .../storage/blob/BlobContentValidationAsyncDownloadTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java index b49108415ce3..38a96c2521a9 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java @@ -9,8 +9,6 @@ import com.azure.core.util.BinaryData; import com.azure.core.util.FluxUtil; import com.azure.storage.blob.models.BlobRange; -import com.azure.storage.blob.models.BlobRequestConditions; -import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.DownloadRetryOptions; import com.azure.storage.blob.options.BlobDownloadContentOptions; import com.azure.storage.blob.options.BlobDownloadStreamOptions; From 2d5c9b89dd0d573aa3565c44b58ae3acc18de84c Mon Sep 17 00:00:00 2001 From: Isabelle Date: Tue, 28 Apr 2026 16:11:59 -0700 Subject: [PATCH 25/31] recordings --- sdk/storage/azure-storage-blob/assets.json | 2 +- .../common/policy/StorageContentValidationDecoderPolicy.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index eea1f647e8e6..0c3832771777 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "java", "TagPrefix": "java/storage/azure-storage-blob", - "Tag": "java/storage/azure-storage-blob_4e6c4fe966" + "Tag": "java/storage/azure-storage-blob_1f689f90f0" } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index e1961d9d77e9..442b7197499d 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -92,7 +92,7 @@ private static boolean isDownloadResponse(HttpResponse response) { } /** - * @return The content length, or null if absent or unparseable. + * @return The content length, or null if absent or non-parseable. */ private static Long getContentLength(HttpHeaders headers) { String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); From c1a3f1fbbfea7a46d3b9749a24657b235fe1b624 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Wed, 29 Apr 2026 10:21:55 -0700 Subject: [PATCH 26/31] adding documentation to decoder classes --- .../StructuredMessageDecoder.java | 362 ++++++++++++------ .../common/policy/DecodedResponse.java | 18 +- ...StorageContentValidationDecoderPolicy.java | 84 +++- 3 files changed, 329 insertions(+), 135 deletions(-) diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index c7de91e169bb..3ffb9585cb1f 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -15,47 +15,67 @@ import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; /** - * Decoder for structured messages with support for segmenting and CRC64 checksums. + * Streaming decoder for the storage structured message format used to validate downloaded blob/file/datalake + * content with CRC64 checksums. * - *

This decoder properly handles partial headers and segment splits across HTTP chunks - * by maintaining a pending buffer and only advancing offsets when complete structures - * have been fully read and validated.

+ *

This class owns the actual parsing and CRC validation. The pipeline policy hands it raw {@link ByteBuffer}s as + * they arrive on the wire (via {@link #decodeChunk(ByteBuffer)}); the decoder returns only the payload bytes that + * have already been CRC-validated and tells the policy when the entire message has been consumed + * (via {@link #isComplete()}). Any malformed input or CRC mismatch surfaces as an + * {@link IllegalArgumentException} thrown from {@code decodeChunk} so the policy can translate it into a stream + * error.

* - *

Emission guarantee: payload bytes for a segment are never emitted - * downstream until the segment payload has been fully read and, when CRC64 is enabled, - * its segment CRC footer has been validated. This matches the emission semantics used by - * {@code BlobDecryptionPolicy}/{@code DecryptorV2} (which only emits a decrypted region - * after the GCM tag is verified) and guarantees that no unvalidated bytes are ever - * exposed to callers, even under retries.

+ *

Wire format (V1)

+ * + *

The encoded body has the following layout (all integers little-endian):

+ * 
+ *   |-- message header (13 B) ----------------------------------------|
+ *   |  version (1)  |  total message length (8)  |  flags (2)  |  numSegments (2)  |
+ *
+ *   for each segment in 1..numSegments:
+ *     |-- segment header (10 B) -|
+ *     |  segNum (2)  |  segContentLen (8)  |
+ *     |-- segment payload (segContentLen B) --|
+ *     |-- segment CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ *   |-- message CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ * + *

Emission guarantee

+ * + * Payload bytes for a segment are never emitted to the caller until that segment's CRC64 footer + * has been validated. This matches the emission semantics used by {@code BlobDecryptionPolicy}/{@code DecryptorV2} + * (which only emits a decrypted region after its GCM tag is verified) and ensures that no unvalidated bytes are + * exposed to consumers, even if the connection is later torn down or the download is retried. + * + *

Thread-safety

+ * + *

This class is not thread-safe. A new instance is created for every HTTP response, and the + * reactive operators in the policy ({@code concatMap}) serialize access to the single instance. Retries produce new + * HTTP responses and therefore new decoder instances, so a CRC failure on one attempt cannot pollute another.

*/ public class StructuredMessageDecoder { private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecoder.class); - // Message state private long messageLength = -1; private StructuredMessageFlags flags; private int numSegments = -1; private final long expectedContentLength; - - // Offset tracking + // Number of encoded bytes consumed so far (headers + payloads + footers). private long messageOffset = 0; - - // Current segment state private int currentSegmentNumber = 0; private long currentSegmentContentLength = 0; private long currentSegmentContentOffset = 0; private boolean segmentHeaderRead = false; - - // CRC validation + // Running CRC64 over all payload bytes seen so far (across every segment). private long messageCrc64 = 0; + // Running CRC64 over only the current segment's payload bytes. private long segmentCrc64 = 0; - - // Pending buffer for handling partial headers/segments across chunks + // Holds bytes left over from a previous decodeChunk() call when the current chunk did not contain a full + // header or footer. private final ByteArrayOutputStream pendingBytes = new ByteArrayOutputStream(); - - // Payload bytes accumulated for the current segment. These are held back and NOT - // emitted until the segment CRC footer has been validated, so callers never observe - // bytes that could later fail validation. + // Holds the payload bytes of the segment that is currently being decoded. These bytes are intentionally NOT + // emitted to the caller until the segment's CRC footer has been validated. private final ByteArrayOutputStream currentSegmentBuffer = new ByteArrayOutputStream(); /** @@ -68,101 +88,36 @@ public StructuredMessageDecoder(long expectedContentLength) { } /** - * Gets the total available bytes (pending + buffer remaining). - */ - private int getAvailableBytes(ByteBuffer buffer) { - return pendingBytes.size() + buffer.remaining(); - } - - /** - * Creates a combined buffer from pending bytes and new buffer. - * Returns a new buffer with position=0 and LITTLE_ENDIAN order. - * The original buffer's position is NOT advanced. - */ - private ByteBuffer getCombinedBuffer(ByteBuffer buffer) { - if (pendingBytes.size() == 0) { - ByteBuffer dup = buffer.duplicate(); - dup.order(ByteOrder.LITTLE_ENDIAN); - return dup; - } - - byte[] pending = pendingBytes.toByteArray(); - ByteBuffer combined = ByteBuffer.allocate(pending.length + buffer.remaining()); - combined.order(ByteOrder.LITTLE_ENDIAN); - combined.put(pending); - combined.put(buffer.duplicate()); - combined.flip(); - return combined; - } - - /** - * When {@code flags} require a segment or message CRC footer, reads 8 bytes, validates against {@code expectedCrc64}, - * and advances {@link #messageOffset}. Returns false if the footer is not yet available. - */ - private boolean tryConsumeCrc64Footer(ByteBuffer buffer, long expectedCrc64, String mismatchDetail) { - if (getAvailableBytes(buffer) < CRC64_LENGTH) { - appendToPending(buffer); - return false; - } - ByteBuffer combined = getCombinedBuffer(buffer); - long reportedCrc = combined.getLong(); - if (expectedCrc64 != reportedCrc) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( - "CRC64 mismatch" + mismatchDetail + ". Expected: " + expectedCrc64 + ", got: " + reportedCrc))); - } - consumeBytes(CRC64_LENGTH, buffer); - messageOffset += CRC64_LENGTH; - return true; - } - - private void consumeBytes(int bytesToConsume, ByteBuffer buffer) { - int pendingSize = pendingBytes.size(); - if (bytesToConsume <= pendingSize) { - byte[] remaining = pendingBytes.toByteArray(); - pendingBytes.reset(); - if (bytesToConsume < pendingSize) { - pendingBytes.write(remaining, bytesToConsume, pendingSize - bytesToConsume); - } - } else { - int bytesFromBuffer = bytesToConsume - pendingSize; - pendingBytes.reset(); - buffer.position(buffer.position() + bytesFromBuffer); - } - } - - /** - * Appends remaining buffer bytes to pending for next chunk. - */ - private void appendToPending(ByteBuffer buffer) { - while (buffer.hasRemaining()) { - pendingBytes.write(buffer.get()); - } - } - - /** - * Reads the message header if we have enough bytes. + * Reads the 13-byte message header (version + total length + flags + numSegments) the first time the decoder + * sees enough bytes, and validates each field. Subsequent calls are no-ops. * * @param buffer The buffer to read from. - * @return true if header was successfully read, false if more bytes needed. + * @return true if the header was successfully read (or had already been read on a previous pass); false if more + * bytes are still needed. */ private boolean tryReadMessageHeader(ByteBuffer buffer) { if (messageLength != -1) { + // Header already parsed on a previous chunk; nothing to do. return true; } if (getAvailableBytes(buffer) < V1_HEADER_LENGTH) { + // Not enough bytes for the full header yet; carry over what we have. appendToPending(buffer); return false; } ByteBuffer combined = getCombinedBuffer(buffer); + // Byte 0: protocol version. int messageVersion = Byte.toUnsignedInt(combined.get()); if (messageVersion != DEFAULT_MESSAGE_VERSION) { throw LOGGER.logExceptionAsError(new IllegalArgumentException( enrichExceptionMessage("Unsupported structured message version: " + messageVersion))); } + // Bytes 1-8: total encoded message length. Must be at least the header itself, and must agree with what the + // HTTP layer told us via Content-Length – any disagreement implies a truncated/extended response. long msgLen = combined.getLong(); if (msgLen < V1_HEADER_LENGTH) { throw LOGGER.logExceptionAsError( @@ -173,6 +128,7 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { "Structured message length " + msgLen + " did not match content length " + expectedContentLength))); } + // Bytes 9-10: flags (NONE or STORAGE_CRC64). Bytes 11-12: number of segments. flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(combined.getShort())); numSegments = Short.toUnsignedInt(combined.getShort()); if (numSegments < 1) { @@ -180,7 +136,7 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { enrichExceptionMessage("Structured message must have at least one segment, got: " + numSegments))); } - // Consume the bytes from pending/buffer + // Commit: drop the 13 bytes we just parsed from pending/buffer and record the message length. consumeBytes(V1_HEADER_LENGTH, buffer); messageOffset += V1_HEADER_LENGTH; messageLength = msgLen; @@ -189,10 +145,17 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { } /** - * Reads a segment header if we have enough bytes. + * Reads the 10-byte header for the next segment (segment number + segment payload length) and resets + * per-segment state so {@link #tryReadSegmentContent(ByteBuffer)} can begin filling + * {@link #currentSegmentBuffer}. + * + *

Validates that segments arrive in order and that the declared segment size leaves enough room in the + * remaining message for any subsequent segment headers, payloads, footers, and the trailing message footer – + * this catches malformed/oversized segment lengths up front instead of waiting until we run off the end of the + * stream.

* * @param buffer The buffer to read from. - * @return true if segment header was read, false if more bytes needed. + * @return true if the segment header was read; false if more bytes are needed. */ private boolean tryReadSegmentHeader(ByteBuffer buffer) { if (getAvailableBytes(buffer) < V1_SEGMENT_HEADER_LENGTH) { @@ -202,15 +165,20 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { ByteBuffer combined = getCombinedBuffer(buffer); + // Bytes 0-1: segment number. Bytes 2-9: declared payload length of this segment. int segmentNum = Short.toUnsignedInt(combined.getShort()); long segmentSize = combined.getLong(); - // Validate segment number + // Segments must arrive strictly in order so the running CRC and "segment N follows segment N-1" assumption + // hold. Anything else implies a malformed/reordered response. if (segmentNum != currentSegmentNumber + 1) { throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( "Unexpected segment number. Expected: " + (currentSegmentNumber + 1) + ", got: " + segmentNum))); } + // Compute an upper bound on the legal segment size: whatever is left in the message, minus the bytes that + // MUST still appear after this segment's payload (this segment's footer, the headers/payloads/footers of all + // remaining segments, and the trailing message footer). long footerSize = flags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; long remainingSegmentsAfterThis = (long) numSegments - segmentNum; long reservedBytes @@ -221,6 +189,7 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { "Invalid segment size detected: " + segmentSize + " (max=" + maxSegmentSize + ")"))); } + // Commit: drop the 10 header bytes and set up per-segment state so payload accumulation can start fresh. consumeBytes(V1_SEGMENT_HEADER_LENGTH, buffer); messageOffset += V1_SEGMENT_HEADER_LENGTH; currentSegmentNumber = segmentNum; @@ -229,6 +198,8 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { currentSegmentBuffer.reset(); if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Reset only the per-segment running CRC; the message-wide running CRC keeps accumulating across all + // segments so the final message footer covers the entire payload. segmentCrc64 = 0; } @@ -236,16 +207,22 @@ private boolean tryReadSegmentHeader(ByteBuffer buffer) { } /** - * Reads segment content bytes if available, accumulating them into the per-segment - * buffer. Bytes remain held in the buffer until the segment's CRC footer is - * validated; they are not returned to the caller here. + * Pulls as many payload bytes as possible (bounded by what is still owed for the current segment) from the + * pending+buffer view into {@link #currentSegmentBuffer}, updating the running per-segment and per-message + * CRC64 values along the way. + * + *

Bytes accumulated here are not yet emitted to the caller. They are released only after + * {@link #tryReadSegmentFooter(ByteBuffer)} validates this segment's CRC. This is the mechanism that enforces + * "no unvalidated bytes ever leave the decoder".

* * @param buffer The buffer to read from. - * @return The number of payload bytes read into the segment buffer. + * @return The number of payload bytes read in this call (0 means we either had no bytes available or the + * current segment's payload was already complete). */ private int tryReadSegmentContent(ByteBuffer buffer) { long remaining = currentSegmentContentLength - currentSegmentContentOffset; if (remaining == 0) { + // Segment payload is already complete; nothing to do here. The caller will move on to read the footer. return 0; } @@ -254,14 +231,20 @@ private int tryReadSegmentContent(ByteBuffer buffer) { return 0; } + // Read the minimum of "what's available right now" and "what's still owed for this segment" so we never + // accidentally consume the segment footer here. int toRead = (int) Math.min(available, remaining); ByteBuffer combined = getCombinedBuffer(buffer); + // Materialize the bytes into a fresh array so we can both feed the CRC64 calculator and stash them in the + // per-segment buffer in one pass. byte[] content = new byte[toRead]; combined.get(content); currentSegmentBuffer.write(content, 0, toRead); if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Update both CRCs incrementally: the segment CRC will be checked at the segment footer, and the + // message CRC accumulates across every segment to be checked at the message footer. segmentCrc64 = StorageCrc64Calculator.compute(content, segmentCrc64); messageCrc64 = StorageCrc64Calculator.compute(content, messageCrc64); } @@ -274,16 +257,20 @@ private int tryReadSegmentContent(ByteBuffer buffer) { } /** - * Reads the segment CRC footer if needed and available. Does not advance into the - * message footer; callers drive message-footer consumption separately so that - * segment bytes can be flushed as soon as their CRC passes, even if the message - * footer is not yet available in the current chunk. + * Validates the 8-byte segment CRC64 footer for the segment that has just finished accumulating. Pre-condition: + * {@code currentSegmentContentOffset == currentSegmentContentLength}. + * + *

This step is intentionally separate from reading the message footer: when the CRC matches, we want to be + * able to flush the buffered segment payload to the caller right away – even if the trailing message footer is + * not yet available in the current chunk.

* * @param buffer The buffer to read from. - * @return true if footer was read (or not needed), false if more bytes needed. + * @return true if the footer was successfully read (or no footer is required for this message); false if more + * bytes are still needed. */ private boolean tryReadSegmentFooter(ByteBuffer buffer) { if (currentSegmentContentOffset != currentSegmentContentLength) { + // Segment payload is not complete yet; wait for more content. return true; } @@ -291,14 +278,15 @@ private boolean tryReadSegmentFooter(ByteBuffer buffer) { return tryConsumeCrc64Footer(buffer, segmentCrc64, " in segment " + currentSegmentNumber); } + // No CRC was negotiated, so there is no footer to read; the caller can release the buffered payload. return true; } /** - * Reads the message CRC footer if needed and available. + * Validates the 8-byte message CRC64 footer that follows the last segment. * * @param buffer The buffer to read from. - * @return true if footer was read (or not needed), false if more bytes needed. + * @return true if the footer was successfully read (or none is required); false if more bytes are still needed. */ private boolean tryReadMessageFooter(ByteBuffer buffer) { if (flags == StructuredMessageFlags.STORAGE_CRC64) { @@ -321,45 +309,62 @@ private boolean tryReadMessageFooter(ByteBuffer buffer) { * @throws IllegalArgumentException if the input is malformed or a CRC64 check fails. */ public ByteBuffer decodeChunk(ByteBuffer buffer) { + // Decoder always reads little-endian; force the order on the caller's buffer so all our get() calls match + // the wire format regardless of how the buffer was constructed. buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Output collected during this single invocation. Each segment whose CRC validates in this call is appended + // here and ultimately returned to the policy as one ByteBuffer. ByteArrayOutputStream validatedOutput = new ByteArrayOutputStream(); + // Step 1: parse the message header on the first chunk that has enough bytes for it. If this chunk doesn't, + // bail out early. if (!tryReadMessageHeader(buffer)) { return emptyOrNull(validatedOutput); } + // Step 2: walk forward through the message until we either hit the end (messageOffset == messageLength) or + // we run out of bytes for the current structural element and have to wait for the next chunk. while (messageOffset < messageLength) { if (!segmentHeaderRead) { - // All segments are done; only the trailing message footer remains. + // We are between segments. If every segment has been processed, only the trailing message footer + // can still appear in the stream – read it (or wait for it) and exit. if (currentSegmentNumber == numSegments) { if (!tryReadMessageFooter(buffer)) { break; } break; } + // Otherwise, parse the next segment's header. May return false if it is split across chunks. if (!tryReadSegmentHeader(buffer)) { break; } segmentHeaderRead = true; } + // Drain as many payload bytes as are available into the per-segment buffer. int payloadRead = tryReadSegmentContent(buffer); if (currentSegmentContentOffset == currentSegmentContentLength) { + // Segment payload fully buffered. Validate the CRC footer (if any). When the footer isn't fully + // available yet, break and resume on the next chunk – currentSegmentBuffer keeps its contents so + // we can still emit them on the call where the footer arrives. if (!tryReadSegmentFooter(buffer)) { break; } - // Segment is fully validated: safe to release the buffered payload. + // Segment passed validation: it is now safe to release the buffered payload to the caller. try { currentSegmentBuffer.writeTo(validatedOutput); } catch (java.io.IOException e) { - // ByteArrayOutputStream.writeTo(ByteArrayOutputStream) cannot throw. + // ByteArrayOutputStream.writeTo(ByteArrayOutputStream) does not actually throw, but the + // signature forces us to handle it. throw LOGGER.logExceptionAsError(new IllegalStateException(e)); } currentSegmentBuffer.reset(); segmentHeaderRead = false; - // Loop continues: either consume the next segment header or the message footer. + // Loop continues: either consume the next segment's header or the message footer. } else if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { + // Nothing left to read this pass and the segment is not complete – wait for the next chunk. break; } } @@ -367,6 +372,111 @@ public ByteBuffer decodeChunk(ByteBuffer buffer) { return emptyOrNull(validatedOutput); } + /** + * @return the total number of bytes the decoder can currently see across the carry-over {@link #pendingBytes} + * plus the unread tail of the supplied buffer. Used to decide whether a structural element (header / + * footer) can be parsed in this pass or whether we must defer to the next chunk. + */ + private int getAvailableBytes(ByteBuffer buffer) { + return pendingBytes.size() + buffer.remaining(); + } + + /** + * Returns a single read-only view that logically concatenates {@link #pendingBytes} with the unread tail of + * a buffer. + * + *

The position of the supplied buffer is intentionally not advanced here – reads happen on the + * combined view, and the original buffer's position is moved later by {@link #consumeBytes(int, ByteBuffer)} + * once we know the parse succeeded.

+ * + *

When pendingBytes is empty we avoid the allocation and just return a duplicate of the buffer; + * otherwise we materialize a fresh array of size {@code pending + buffer.remaining()}.

+ */ + private ByteBuffer getCombinedBuffer(ByteBuffer buffer) { + if (pendingBytes.size() == 0) { + ByteBuffer dup = buffer.duplicate(); + dup.order(ByteOrder.LITTLE_ENDIAN); + return dup; + } + + byte[] pending = pendingBytes.toByteArray(); + ByteBuffer combined = ByteBuffer.allocate(pending.length + buffer.remaining()); + combined.order(ByteOrder.LITTLE_ENDIAN); + combined.put(pending); + combined.put(buffer.duplicate()); + combined.flip(); + return combined; + } + + /** + * Consumes the next 8 bytes as a little-endian CRC64 footer, validates it against expectedCrc64, and + * advances {@link #messageOffset}. Used for both segment and message footers. + * + *

If fewer than 8 bytes are available, the remaining buffer bytes are stashed in {@link #pendingBytes} and + * the method returns false so the caller can break out of the decode loop and wait for the next + * chunk. On a CRC mismatch, an {@link IllegalArgumentException} is thrown (the decoder is then discarded by + * the enclosing policy).

+ */ + private boolean tryConsumeCrc64Footer(ByteBuffer buffer, long expectedCrc64, String mismatchDetail) { + if (getAvailableBytes(buffer) < CRC64_LENGTH) { + // Not enough bytes yet for the footer; carry whatever we have over to the next call. + appendToPending(buffer); + return false; + } + ByteBuffer combined = getCombinedBuffer(buffer); + long reportedCrc = combined.getLong(); + if (expectedCrc64 != reportedCrc) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "CRC64 mismatch" + mismatchDetail + ". Expected: " + expectedCrc64 + ", got: " + reportedCrc))); + } + consumeBytes(CRC64_LENGTH, buffer); + messageOffset += CRC64_LENGTH; + return true; + } + + /** + * Drains {@code bytesToConsume} bytes from the logical pending+buffer stream that + * {@link #getCombinedBuffer(ByteBuffer)} produced. + * + *

Bytes are taken from {@link #pendingBytes} first, then from the live buffer. The pending stream is + * reset whenever it is fully drained, and any leftover (when {@code bytesToConsume} was less than what was in + * pending) is rewritten so the carry-over stays compact.

+ */ + private void consumeBytes(int bytesToConsume, ByteBuffer buffer) { + int pendingSize = pendingBytes.size(); + if (bytesToConsume <= pendingSize) { + // The entire consume fits in pending: rewrite whatever survives back into pending after a reset. + byte[] remaining = pendingBytes.toByteArray(); + pendingBytes.reset(); + if (bytesToConsume < pendingSize) { + pendingBytes.write(remaining, bytesToConsume, pendingSize - bytesToConsume); + } + } else { + // Pending is fully drained and the remainder comes from the live buffer; advance its position directly. + int bytesFromBuffer = bytesToConsume - pendingSize; + pendingBytes.reset(); + buffer.position(buffer.position() + bytesFromBuffer); + } + } + + /** + * Stashes everything still unread in the buffer into {@link #pendingBytes} so it can be combined with the + * next chunk on the next call to {@link #decodeChunk(ByteBuffer)}. + * + *

This is only called when the current chunk does not contain enough bytes for the next structural element, + * so the carry-over is always small (bounded by the largest header size, currently 13 bytes).

+ */ + private void appendToPending(ByteBuffer buffer) { + while (buffer.hasRemaining()) { + pendingBytes.write(buffer.get()); + } + } + + /** + * Wraps {@code output} as a {@link ByteBuffer}, or returns {@code null} when no bytes were emitted in this + * pass. The {@code null} return distinguishes "no validated bytes ready in this chunk" (still need more input) + * from "stream complete" (which the caller checks via {@link #isComplete()}). + */ private static ByteBuffer emptyOrNull(ByteArrayOutputStream output) { if (output.size() == 0) { return null; @@ -375,9 +485,20 @@ private static ByteBuffer emptyOrNull(ByteArrayOutputStream output) { } /** - * Checks if decoding is complete. + * Reports whether the decoder has finished consuming the entire structured message and validated everything it + * was supposed to validate. Used by the pipeline policy to distinguish "stream ended cleanly" from "stream was + * truncated". + * + *

The check requires all of:

+ *
    + *
  • The message header has been parsed ({@code messageLength != -1}).
    • + *
  • Every byte of the declared message has been consumed.
    • + *
  • No carry-over bytes remain in pending.
    • + *
  • No segment is currently in progress (no segment header without a matching footer).
    • + *
  • The current segment's payload accumulation is itself complete.
    • + *
* - * @return true if all expected bytes have been decoded, false otherwise. + * @return true if all expected bytes have been decoded and validated; false otherwise. */ public boolean isComplete() { return messageLength != -1 @@ -388,10 +509,11 @@ public boolean isComplete() { } /** - * Enriches an exception message with decoder offset information for debugging. + * Appends the current decoder offset to an exception message so failures can be traced back to a specific + * point in the encoded stream. * * @param message The original exception message. - * @return The enriched message with offset information. + * @return The original message with {@code [decoderOffset=N]} appended. */ private String enrichExceptionMessage(String message) { return String.format("%s [decoderOffset=%d]", message, messageOffset); diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java index abc18d20fe8d..7c70d1aec669 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -13,14 +13,30 @@ import java.nio.charset.Charset; /** - * Decoded HTTP response that wraps the original response with a decoded body stream. + * {@link HttpResponse} wrapper that exposes a decoded body stream while preserving the request, status code, and + * headers of the original response. + * + *

The policy hands this class a Flux that already represents validated, framing-stripped bytes (produced by the + * decoder pipeline). This class's only job is to make that Flux look like the body of the original + * {@link HttpResponse}. Status code, headers, and request remain identical to the underlying response so callers + * cannot distinguish a validated download from a normal one – the validation is transparent.

*/ class DecodedResponse extends HttpResponse { private final Flux decodedBody; private final HttpHeaders httpHeaders; private final int statusCode; + /** + * Wraps {@code httpResponse} with a body backed by {@code decodedBody}. + * + * @param httpResponse The original response from the storage service. Its request, status code, and headers + * are preserved verbatim. + * @param decodedBody The Flux of CRC-validated, framing-stripped payload bytes produced by the decoder + * pipeline. + */ DecodedResponse(HttpResponse httpResponse, Flux decodedBody) { + // Preserve the original request so retry policies, response models, and logging keep their reference chain + // intact. super(httpResponse.getRequest()); this.decodedBody = decodedBody; this.statusCode = httpResponse.getStatusCode(); diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index 442b7197499d..aa9973952345 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -21,17 +21,22 @@ import java.nio.ByteBuffer; /** - * Pipeline policy that decodes structured messages in storage download responses when - * CRC64-based content validation is active (i.e., when {@link com.azure.storage.common.ContentValidationAlgorithm} - * is {@code CRC64} or {@code AUTO}). + * HTTP pipeline policy that decodes the storage structured message body returned for downloads when CRC64 + * content validation is active. * - *

The policy is activated by the presence of a boolean context key - * ({@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY}). It validates per-segment - * CRC64 checksums during decoding.

+ *

The policy decides when to opt in (via the context key), tells the service to + * encode the response (via the request header), constructs the decoder and the wrapper response, and + * translates decoder-level failures (malformed framing, CRC mismatch, premature end-of-stream) into reactive + * {@link IOException} errors.

* - *

Emission guarantee: the policy only forwards payload bytes that the - * {@link StructuredMessageDecoder} has already CRC-validated at the segment boundary. Each invocation creates - * a fresh decoder and the decoder itself withholds bytes until their enclosing segment passes validation.

+ *

This policy uses {@link com.azure.core.http.HttpPipelinePosition#PER_RETRY PER_RETRY} semantics by default, so + * each retry produces a fresh response that this policy wraps with a fresh decoder. A CRC failure on one attempt + * cannot pollute another, and the storage download retry logic ({@code BlobAsyncClientBase.downloadStream...}) can + * resume by reissuing range requests; each new range response is validated end-to-end on its own.

+ * + *

Because the wrapped {@link StructuredMessageDecoder} only releases payload bytes after the corresponding + * segment's CRC has been verified, the {@link DecodedResponse}'s body Flux is guaranteed to contain only validated + * bytes – callers never see a byte that could later fail validation, even when retries are involved.

*/ public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); @@ -44,25 +49,32 @@ public StorageContentValidationDecoderPolicy() { @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Check if the decoding should be applied. if (!shouldApplyDecoding(context)) { return next.process(); } + // Tell the service we want a structured-message body. context.getHttpRequest() .getHeaders() .set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); return next.process().map(httpResponse -> { + // The HTTP Content-Length is the size of the encoded structured message body. We hand it to the + // decoder which cross-checks it against the message header. Long contentLength = getContentLength(httpResponse.getHeaders()); + // Only 2xx GET responses with a positive content length carry a body that we can decode. if (!isEligibleDownload(httpResponse, contentLength)) { return httpResponse; } + // Confirm the service actually honored our structured-body request before we hand the body to the decoder. validateStructuredMessageHeaders(httpResponse); long expectedLength = contentLength; + // Fresh decoder per response so retries each get a clean state machine. StructuredMessageDecoder decoder = new StructuredMessageDecoder(expectedLength); Flux decodedStream = decodeStream(httpResponse.getBody(), decoder); @@ -70,12 +82,21 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN }); } + /** + * @return true when the request carries the boolean opt-in flag set + * by {@code ContentValidationModeResolver.addStructuredMessageDecodingToContext}. + */ private boolean shouldApplyDecoding(HttpPipelineCallContext context) { return context.getData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) .map(value -> value instanceof Boolean && (Boolean) value) .orElse(false); } + /** + * Verifies the response acknowledges the structured-body request: presence of the + * {@code x-ms-structured-body} header and the {@code x-ms-structured-content-length} + * header. If either is missing, the service is sending us a normal body and we must not run the decoder over it. + */ private void validateStructuredMessageHeaders(HttpResponse httpResponse) { String structuredBody = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); @@ -87,12 +108,9 @@ private void validateStructuredMessageHeaders(HttpResponse httpResponse) { } } - private static boolean isDownloadResponse(HttpResponse response) { - return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; - } - /** - * @return The content length, or null if absent or non-parseable. + * Reads {@code Content-Length} as a {@code long}, returning {@code null} when the header is missing or + * unparseable so callers can simply skip decoding for non-bodied responses. */ private static Long getContentLength(HttpHeaders headers) { String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); @@ -106,17 +124,43 @@ private static Long getContentLength(HttpHeaders headers) { return null; } + /** + * @return true for a 2xx response to a GET request, the only response shape that carries a body we + * can decode. 206 (Partial Content) on retried range downloads is included. + */ + private static boolean isDownloadResponse(HttpResponse response) { + return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; + } + + /** + * @return true when the response is one we should decode: a 2xx GET with a positive, parseable + * {@code Content-Length}. + */ private static boolean isEligibleDownload(HttpResponse response, Long contentLength) { return isDownloadResponse(response) && contentLength != null && contentLength > 0; } + /** + * Builds the body-decoding Flux: each upstream {@link ByteBuffer} is fed to the decoder in order + * ({@code concatMap} preserves order and serializes access), and a deferred stream-completion check is + * appended so a truncated body raises an error instead of completing silently. + */ private Flux decodeStream(Flux encodedFlux, StructuredMessageDecoder decoder) { return encodedFlux.concatMap(buffer -> decodeBuffer(buffer, decoder)) .concatWith(Mono.defer(() -> handleStreamCompletion(decoder))); } + /** + * Feeds a single inbound chunk to the decoder and translates its outputs into reactive emissions: + * If the decoder reports validated bytes, emit them downstream. + * If the decoder threw because the input is malformed or a CRC mismatch was detected, surface that as + * an {@link IOException}. + * If the decoder is already complete (e.g., extra trailing bytes after the message footer), drop the + * chunk silently. + */ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecoder decoder) { if (decoder.isComplete()) { + // Decoding finished on a previous chunk; ignore any trailing bytes the transport might still emit. return Flux.empty(); } @@ -130,11 +174,17 @@ private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecode } catch (IllegalArgumentException e) { return Flux.error(new IOException("Failed to decode structured message: " + e.getMessage(), e)); } catch (Exception e) { + // Anything not foreseen by the decoder, log it. LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); return Flux.error(new IOException("Failed to decode structured message chunk: " + e.getMessage(), e)); } } + /** + * Run after the upstream Flux completes. If the decoder is not in a complete state, the response body ended + * before all expected bytes arrived – surface this as an {@link IOException} so callers don't accept a + * truncated payload. + */ private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { if (!decoder.isComplete()) { return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); @@ -142,6 +192,12 @@ private Mono handleStreamCompletion(StructuredMessageDecoder decoder return Mono.empty(); } + /** + * Wraps the decoder output in a Flux. The decoder hands back a freshly-allocated buffer wrapping its own + * internal byte array; we make a defensive copy so downstream consumers that aggregate or keep references to + * the buffer cannot accidentally see the decoder's internal storage if the decoder ever changes to reuse + * arrays. + */ private static Flux emitDecodedPayload(ByteBuffer decodedPayload) { if (decodedPayload == null || !decodedPayload.hasRemaining()) { return Flux.empty(); From 3c77a763d8339280ae6c6dbc0b97e17b5ff8fe06 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Wed, 29 Apr 2026 11:19:01 -0700 Subject: [PATCH 27/31] small fixes and failure path tests --- .../StructuredMessageDecoder.java | 18 +-- .../common/policy/DecodedResponse.java | 7 ++ ...StorageContentValidationDecoderPolicy.java | 15 +-- .../StructuredMessageDecoderTests.java | 105 ++++++++++++++++++ 4 files changed, 125 insertions(+), 20 deletions(-) diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java index 3ffb9585cb1f..7930067a1c5d 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -60,7 +60,7 @@ public class StructuredMessageDecoder { private long messageLength = -1; private StructuredMessageFlags flags; private int numSegments = -1; - private final long expectedContentLength; + private final long expectedEncodedMessageLength; // Number of encoded bytes consumed so far (headers + payloads + footers). private long messageOffset = 0; private int currentSegmentNumber = 0; @@ -81,10 +81,11 @@ public class StructuredMessageDecoder { /** * Constructs a new StructuredMessageDecoder. * - * @param expectedContentLength The expected length of the content to be decoded. + * @param expectedEncodedMessageLength The expected encoded structured-message length (typically HTTP + * {@code Content-Length}). */ - public StructuredMessageDecoder(long expectedContentLength) { - this.expectedContentLength = expectedContentLength; + public StructuredMessageDecoder(long expectedEncodedMessageLength) { + this.expectedEncodedMessageLength = expectedEncodedMessageLength; } /** @@ -123,9 +124,10 @@ private boolean tryReadMessageHeader(ByteBuffer buffer) { throw LOGGER.logExceptionAsError( new IllegalArgumentException(enrichExceptionMessage("Message length too small: " + msgLen))); } - if (msgLen != expectedContentLength) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( - "Structured message length " + msgLen + " did not match content length " + expectedContentLength))); + if (msgLen != expectedEncodedMessageLength) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage("Structured message length " + + msgLen + " did not match content length " + expectedEncodedMessageLength))); } // Bytes 9-10: flags (NONE or STORAGE_CRC64). Bytes 11-12: number of segments. @@ -502,7 +504,7 @@ private static ByteBuffer emptyOrNull(ByteArrayOutputStream output) { */ public boolean isComplete() { return messageLength != -1 - && messageOffset >= messageLength + && messageOffset == messageLength && pendingBytes.size() == 0 && !segmentHeaderRead && currentSegmentContentOffset == currentSegmentContentLength; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java index 7c70d1aec669..989a4dc258df 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -22,6 +22,7 @@ * cannot distinguish a validated download from a normal one – the validation is transparent.

*/ class DecodedResponse extends HttpResponse { + private final HttpResponse originalResponse; private final Flux decodedBody; private final HttpHeaders httpHeaders; private final int statusCode; @@ -38,6 +39,7 @@ class DecodedResponse extends HttpResponse { // Preserve the original request so retry policies, response models, and logging keep their reference chain // intact. super(httpResponse.getRequest()); + this.originalResponse = httpResponse; this.decodedBody = decodedBody; this.statusCode = httpResponse.getStatusCode(); this.httpHeaders = httpResponse.getHeaders(); @@ -77,4 +79,9 @@ public Mono getBodyAsString() { public Mono getBodyAsString(Charset charset) { return FluxUtil.collectBytesInByteBufferStream(decodedBody).map(b -> new String(b, charset)); } + + @Override + public void close() { + originalResponse.close(); + } } diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index aa9973952345..d53c590bc272 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -73,9 +73,8 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN // Confirm the service actually honored our structured-body request before we hand the body to the decoder. validateStructuredMessageHeaders(httpResponse); - long expectedLength = contentLength; // Fresh decoder per response so retries each get a clean state machine. - StructuredMessageDecoder decoder = new StructuredMessageDecoder(expectedLength); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(contentLength); Flux decodedStream = decodeStream(httpResponse.getBody(), decoder); return new DecodedResponse(httpResponse, decodedStream); @@ -193,20 +192,12 @@ private Mono handleStreamCompletion(StructuredMessageDecoder decoder } /** - * Wraps the decoder output in a Flux. The decoder hands back a freshly-allocated buffer wrapping its own - * internal byte array; we make a defensive copy so downstream consumers that aggregate or keep references to - * the buffer cannot accidentally see the decoder's internal storage if the decoder ever changes to reuse - * arrays. + * Wraps decoder output in a Flux. */ private static Flux emitDecodedPayload(ByteBuffer decodedPayload) { if (decodedPayload == null || !decodedPayload.hasRemaining()) { return Flux.empty(); } - - ByteBuffer copy = ByteBuffer.allocate(decodedPayload.remaining()); - copy.put(decodedPayload.duplicate()); - copy.flip(); - - return Flux.just(copy); + return Flux.just(decodedPayload); } } diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java index 64e8a0d25430..faa3d69cde46 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.Arrays; import java.util.concurrent.ThreadLocalRandom; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -18,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -25,11 +27,23 @@ * payload bytes for a segment are only returned after the segment's CRC has been verified. */ public class StructuredMessageDecoderTests { + private static final int MESSAGE_HEADER_LENGTH = 13; + private static final int SEGMENT_HEADER_LENGTH = 10; + private static final int CRC64_LENGTH = 8; private static ByteBuffer collectFlux(Flux flux) { return ByteBuffer.wrap(FluxUtil.collectBytesInByteBufferStream(flux).block()).order(ByteOrder.LITTLE_ENDIAN); } + private static byte[] encode(byte[] originalData, int segmentLength, StructuredMessageFlags flags) + throws IOException { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, segmentLength, flags); + ByteBuffer encoded = collectFlux(encoder.encode(ByteBuffer.wrap(originalData))); + byte[] encodedBytes = new byte[encoded.remaining()]; + encoded.get(encodedBytes); + return encodedBytes; + } + @Test public void readsCompleteMessageInSingleChunk() throws IOException { byte[] originalData = new byte[1024]; @@ -252,4 +266,95 @@ public void withholdsPayloadUntilSegmentFooterValidated() throws IOException { assertArrayEquals(originalData, decodedData); } + @Test + public void throwsOnUnsupportedStructuredMessageVersion() throws IOException { + byte[] data = new byte[64]; + ThreadLocalRandom.current().nextBytes(data); + byte[] encodedBytes = encode(data, 64, StructuredMessageFlags.STORAGE_CRC64); + + // Corrupt message version (byte 0 of the message header). + encodedBytes[0] = (byte) (StructuredMessageConstants.DEFAULT_MESSAGE_VERSION + 1); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN))); + assertTrue(exception.getMessage().contains("Unsupported structured message version")); + } + + @Test + public void throwsOnMessageLengthMismatch() throws IOException { + byte[] data = new byte[128]; + ThreadLocalRandom.current().nextBytes(data); + byte[] encodedBytes = encode(data, 128, StructuredMessageFlags.STORAGE_CRC64); + + // Construct decoder with wrong expected encoded length. + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length + 1); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN))); + assertTrue(exception.getMessage().contains("did not match content length")); + } + + @Test + public void throwsOnUnexpectedSegmentNumber() throws IOException { + byte[] data = new byte[300]; + ThreadLocalRandom.current().nextBytes(data); + byte[] encodedBytes = encode(data, 128, StructuredMessageFlags.STORAGE_CRC64); + + // Corrupt first segment number from 1 to 2 (offset 13 in v1 format). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(MESSAGE_HEADER_LENGTH, (short) 2); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN))); + assertTrue(exception.getMessage().contains("Unexpected segment number")); + } + + @Test + public void throwsOnInvalidSegmentSize() throws IOException { + byte[] data = new byte[256]; + ThreadLocalRandom.current().nextBytes(data); + byte[] encodedBytes = encode(data, 128, StructuredMessageFlags.STORAGE_CRC64); + + // Corrupt first segment size to an impossible value (offsets 15..22 in v1 format). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(MESSAGE_HEADER_LENGTH + 2, Long.MAX_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN))); + assertTrue(exception.getMessage().contains("Invalid segment size detected")); + } + + @Test + public void throwsOnSegmentCrcMismatch() throws IOException { + byte[] data = new byte[512]; + ThreadLocalRandom.current().nextBytes(data); + byte[] encodedBytes = encode(data, 512, StructuredMessageFlags.STORAGE_CRC64); + + // Layout for one-segment message: + // messageHeader(13) + segmentHeader(10) + payload(512) + segmentCrc(8) + messageCrc(8) + int segmentCrcOffset = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + data.length; + encodedBytes[segmentCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in segment")); + } + + @Test + public void throwsOnMessageCrcMismatch() throws IOException { + byte[] data = new byte[512]; + ThreadLocalRandom.current().nextBytes(data); + byte[] encodedBytes = encode(data, 512, StructuredMessageFlags.STORAGE_CRC64); + + int messageCrcOffset = encodedBytes.length - CRC64_LENGTH; + byte[] corrupted = Arrays.copyOf(encodedBytes, encodedBytes.length); + corrupted[messageCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(corrupted.length); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> decoder.decodeChunk(ByteBuffer.wrap(corrupted).order(ByteOrder.LITTLE_ENDIAN))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in message footer")); + } + } From aaa6414e12c87a60fd17f553cc32c8cd134dcf02 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Wed, 29 Apr 2026 11:41:12 -0700 Subject: [PATCH 28/31] addressing context comment --- .../blob/specialized/BlobAsyncClientBase.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index acd575bd475b..b462df20b7b3 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1262,23 +1262,21 @@ Mono downloadStreamWithResponseInternal(BlobRange ran ContentValidationAlgorithm contentValidationAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; - ContentValidationModeResolver.validateTransactionalChecksumOptions(getRangeContentMd5, - contentValidationAlgorithm); - Boolean getMD5 = getRangeContentMd5 ? getRangeContentMd5 : null; - BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; - // Eagerly convert headers for the response types and propagate any structured-message decoding flag. - // The same context is used for the initial range and any retry ranges. - Context downloadContext = ContentValidationModeResolver.addStructuredMessageDecodingToContext(context == null - ? new Context("azure-eagerly-convert-headers", true) - : context.addData("azure-eagerly-convert-headers", true), contentValidationAlgorithm); + context + = ContentValidationModeResolver.addStructuredMessageDecodingToContext(context, contentValidationAlgorithm); + // The first range should eagerly convert headers as they'll be used to create response types. + Context firstRangeContext = context == null + ? new Context("azure-eagerly-convert-headers", true) + : context.addData("azure-eagerly-convert-headers", true); + Context nextRangeContext = context; return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, - downloadContext).map(response -> { + firstRangeContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); String eTag = blobsDownloadHeaders.getETag(); BlobDownloadHeaders blobDownloadHeaders = ModelHelper.populateBlobDownloadHeaders(blobsDownloadHeaders, @@ -1321,7 +1319,7 @@ Mono downloadStreamWithResponseInternal(BlobRange ran try { return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, - eTag, getMD5, downloadContext); + eTag, getMD5, nextRangeContext); } catch (Exception e) { return Mono.error(e); } From e08eaba0f85bb385cb86dd9e7837789ea1e415da Mon Sep 17 00:00:00 2001 From: Isabelle Date: Wed, 29 Apr 2026 12:47:57 -0700 Subject: [PATCH 29/31] analyze error --- .../common/policy/StorageContentValidationDecoderPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java index d53c590bc272..647a95f71e7e 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -109,7 +109,7 @@ private void validateStructuredMessageHeaders(HttpResponse httpResponse) { /** * Reads {@code Content-Length} as a {@code long}, returning {@code null} when the header is missing or - * unparseable so callers can simply skip decoding for non-bodied responses. + * not parseable so callers can simply skip decoding for non-bodied responses. */ private static Long getContentLength(HttpHeaders headers) { String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); From a4393f5299bf6d27190f28109fd8324a874d25a7 Mon Sep 17 00:00:00 2001 From: Isabelle Date: Wed, 29 Apr 2026 21:56:52 -0700 Subject: [PATCH 30/31] removing close override from decodedresponse --- .../com/azure/storage/common/policy/DecodedResponse.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java index 989a4dc258df..df9c0bc88c10 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -79,9 +79,4 @@ public Mono getBodyAsString() { public Mono getBodyAsString(Charset charset) { return FluxUtil.collectBytesInByteBufferStream(decodedBody).map(b -> new String(b, charset)); } - - @Override - public void close() { - originalResponse.close(); - } } From 73c0d2552b3413773c752e71cb4fbd16eceb217b Mon Sep 17 00:00:00 2001 From: Isabelle Date: Wed, 29 Apr 2026 22:26:16 -0700 Subject: [PATCH 31/31] line removal --- .../com/azure/storage/blob/specialized/BlobAsyncClientBase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index b462df20b7b3..a7ae5ce9c373 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -1261,7 +1261,6 @@ Mono downloadStreamWithResponseInternal(BlobRange ran BlobRequestConditions requestConditions, boolean getRangeContentMd5, ContentValidationAlgorithm contentValidationAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; - Boolean getMD5 = getRangeContentMd5 ? getRangeContentMd5 : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions;