From 50ed5eeeed6283b9b09115b028a6a4092c725a91 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Fri, 22 May 2026 14:44:40 -0700 Subject: [PATCH 1/3] Reorganize and fix extensions packaging --- .../azure/storage/blob/_shared/validation.py | 14 ++--- sdk/storage/azure-storage-blob/setup.py | 3 ++ .../azure-storage-extensions/MANIFEST.in | 1 + .../azure-storage-extensions/README.md | 10 ++-- .../storage/extensions/checksums/__init__.py | 9 ++++ .../storage/extensions/checksums/crc64.pyi | 37 +++++++++++++ .../{ => checksums}/crc64/crc64module.c | 0 .../{ => checksums}/crc64/helpers.h | 14 ++--- .../{__init__.py => checksums/py.typed} | 0 sdk/storage/azure-storage-extensions/setup.py | 52 +++++++++++-------- .../tests/test_crc64.py | 22 ++++---- .../filedatalake/_shared/validation.py | 14 ++--- .../azure-storage-file-datalake/setup.py | 3 ++ .../storage/fileshare/_shared/validation.py | 14 ++--- sdk/storage/azure-storage-file-share/setup.py | 3 ++ .../azure/storage/queue/_shared/validation.py | 14 ++--- 16 files changed, 134 insertions(+), 76 deletions(-) create mode 100644 sdk/storage/azure-storage-extensions/MANIFEST.in create mode 100644 sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/__init__.py create mode 100644 sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64.pyi rename sdk/storage/azure-storage-extensions/azure/storage/extensions/{ => checksums}/crc64/crc64module.c (100%) rename sdk/storage/azure-storage-extensions/azure/storage/extensions/{ => checksums}/crc64/helpers.h (99%) rename sdk/storage/azure-storage-extensions/azure/storage/extensions/{__init__.py => checksums/py.typed} (100%) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/validation.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/validation.py index 21d3b081d8cc..7161a31bb15e 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/validation.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/validation.py @@ -20,11 +20,11 @@ def _verify_extensions(module: str) -> None: try: - import azure.storage.extensions # pylint: disable=unused-import + import azure.storage.extensions.checksums # pylint: disable=unused-import except ImportError as exc: raise ValueError( - f"The use of {module} requires the azure-storage-extensions package to be installed. " - f"Please install this package and try again." + f"The use of {module} requires the extra [ext-checksums] to be installed. " + f"Please install this extra and try again." ) from exc @@ -100,13 +100,13 @@ def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes: def calculate_crc64(data: bytes, initial_crc: int) -> int: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(int, crc64.compute(data, initial_crc)) + return checksums.crc64.compute(data, initial_crc) def calculate_crc64_bytes(data: bytes) -> bytes: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(bytes, crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little")) + return checksums.crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little") diff --git a/sdk/storage/azure-storage-blob/setup.py b/sdk/storage/azure-storage-blob/setup.py index 0abb2504e0fb..a2fc64cf3d75 100644 --- a/sdk/storage/azure-storage-blob/setup.py +++ b/sdk/storage/azure-storage-blob/setup.py @@ -88,5 +88,8 @@ "aio": [ "azure-core[aio]>=1.37.0", ], + "ext-checksums": [ + "azure-storage-extensions>=0.1.0,<1.0.0", + ], }, ) diff --git a/sdk/storage/azure-storage-extensions/MANIFEST.in b/sdk/storage/azure-storage-extensions/MANIFEST.in new file mode 100644 index 000000000000..6fd53e3d6d8c --- /dev/null +++ b/sdk/storage/azure-storage-extensions/MANIFEST.in @@ -0,0 +1 @@ +recursive-include azure *.h *.c *.pyi py.typed diff --git a/sdk/storage/azure-storage-extensions/README.md b/sdk/storage/azure-storage-extensions/README.md index 9a93cae2cef6..131ea1a71713 100644 --- a/sdk/storage/azure-storage-extensions/README.md +++ b/sdk/storage/azure-storage-extensions/README.md @@ -12,19 +12,15 @@ This package contains native extension modules that provide performance-critical ## Installation -**Recommended**: Install this package via extras when installing Azure Storage libraries: +Install this package via extras when installing Azure Storage libraries: ```bash -pip install azure-storage-blob[extensions] +pip install azure-storage-blob[ext-checksums] ``` This ensures you get compatible versions of both the SDK and the extensions package. -You can also install it directly if needed: - -```bash -pip install azure-storage-extensions -``` +> ⚠️ Installing `azure-storage-extensions` directly is not recommended. Use the extras syntax above to ensure compatibility. ### Prerequisites * Python 3.9 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md). diff --git a/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/__init__.py b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/__init__.py new file mode 100644 index 000000000000..76370dbcbc6e --- /dev/null +++ b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/__init__.py @@ -0,0 +1,9 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from . import crc64 + +__all__ = ["crc64"] diff --git a/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64.pyi b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64.pyi new file mode 100644 index 000000000000..0644d168341f --- /dev/null +++ b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64.pyi @@ -0,0 +1,37 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +def compute(data: bytes, crc: int, /) -> int: + """Compute Storage CRC64 over given data with given initial CRC64 value. + + :param data: The bytes data to compute the CRC64 over. + :param crc: The initial CRC64 value. + :returns: The computed CRC64 value. + """ + ... + +def concat( + initial_crc_ab: int, + initial_crc_a: int, + final_crc_a: int, + size_a: int, + initial_crc_b: int, + final_crc_b: int, + size_b: int, + /, +) -> int: + """Concatenate two Storage CRC64s together. + + :param initial_crc_ab: The initial CRC64 of the combined data. + :param initial_crc_a: The initial CRC64 of the first data segment. + :param final_crc_a: The final CRC64 of the first data segment. + :param size_a: The size of the first data segment in bytes. + :param initial_crc_b: The initial CRC64 of the second data segment. + :param final_crc_b: The final CRC64 of the second data segment. + :param size_b: The size of the second data segment in bytes. + :returns: The concatenated CRC64 value. + """ + ... diff --git a/sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/crc64module.c b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/crc64module.c similarity index 100% rename from sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/crc64module.c rename to sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/crc64module.c diff --git a/sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/helpers.h b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/helpers.h similarity index 99% rename from sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/helpers.h rename to sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/helpers.h index e4dd83bb49db..25970d64e2dc 100644 --- a/sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/helpers.h +++ b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/helpers.h @@ -3,10 +3,10 @@ #include -const uint64_t poly = 0x9A6C9329AC4BC9B5ULL; +static const uint64_t poly = 0x9A6C9329AC4BC9B5ULL; -const uint64_t m_uComplement = ~0ULL; -const uint64_t m_u1[] = +static const uint64_t m_uComplement = ~0ULL; +static const uint64_t m_u1[] = { 0x0000000000000000ULL, 0x7f6ef0c830358979ULL, 0xfedde190606b12f2ULL, 0x81b31158505e9b8bULL, 0xc962e5739841b68fULL, 0xb60c15bba8743ff6ULL, 0x37bf04e3f82aa47dULL, 0x48d1f42bc81f2d04ULL, @@ -74,7 +74,7 @@ const uint64_t m_u1[] = 0xab69411fbfb21ca3ULL, 0xd407b1d78f8795daULL, 0x55b4a08fdfd90e51ULL, 0x2ada5047efec8728ULL, }; -const uint64_t m_u32[] = +static const uint64_t m_u32[] = { 0x0000000000000000ULL, 0xb8c533c1177eb231ULL, 0x455341d1766af709ULL, 0xfd96721061144538ULL, 0x8aa683a2ecd5ee12ULL, 0x3263b063fbab5c23ULL, 0xcff5c2739abf191bULL, 0x7730f1b28dc1ab2aULL, @@ -597,7 +597,7 @@ const uint64_t m_u32[] = 0x609abef3367d2567ULL, 0xd02690aba479d067ULL, 0x353bc4114ae35c0cULL, 0x8587ea49d8e7a90cULL, }; -const uint64_t m_uX2N[] = +static const uint64_t m_uX2N[] = { 0x0080000000000000ULL, 0x0000800000000000ULL, 0x0000000080000000ULL, 0x9a6c9329ac4bc9b5ULL, 0x10f4bb0f129310d6ULL, 0x70f05dcea2ebd226ULL, 0x311211205672822dULL, 0x2fc297db0f46c96eULL, @@ -622,7 +622,7 @@ const uint64_t m_uX2N[] = // "a" and "b" are represented in "reversed" order -- LSB is x**(XX-1) coefficient, MSB is x^0 coefficient. // "POLY" is represented in the same manner except for omitted x**XX coefficient // -uint64_t MulPolyUnrolled(uint64_t a, uint64_t b) +static uint64_t MulPolyUnrolled(uint64_t a, uint64_t b) { const uint64_t p = poly; const uint64_t p2 = (p >> 1) ^ (p * (p & 1)); @@ -647,7 +647,7 @@ uint64_t MulPolyUnrolled(uint64_t a, uint64_t b) return vr[0] ^ vr[1]; } -uint64_t MulX_N(uint64_t a, uint64_t uSize) +static uint64_t MulX_N(uint64_t a, uint64_t uSize) { uint64_t i = 0; uint64_t r = a; diff --git a/sdk/storage/azure-storage-extensions/azure/storage/extensions/__init__.py b/sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/py.typed similarity index 100% rename from sdk/storage/azure-storage-extensions/azure/storage/extensions/__init__.py rename to sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/py.typed diff --git a/sdk/storage/azure-storage-extensions/setup.py b/sdk/storage/azure-storage-extensions/setup.py index 1c0cfebe2b38..66022de8ec8c 100644 --- a/sdk/storage/azure-storage-extensions/setup.py +++ b/sdk/storage/azure-storage-extensions/setup.py @@ -11,6 +11,7 @@ class bdist_wheel_abi3(bdist_wheel): """Override bdist_wheel tag behavior to add abi3 tag.""" + def get_tag(self): python, abi, plat = super().get_tag() @@ -22,43 +23,48 @@ def get_tag(self): PACKAGE_NAME = "azure-storage-extensions" PACKAGE_PPRINT_NAME = "Azure Storage Extensions" -package_folder_path = PACKAGE_NAME.replace('-', '/') +package_folder_path = PACKAGE_NAME.replace("-", "/") setup( name=PACKAGE_NAME, version="0.1.0", - include_package_data=True, description=PACKAGE_PPRINT_NAME, - long_description=open('README.md', 'r', encoding='utf-8').read(), - long_description_content_type='text/markdown', - license='MIT License', - author='Microsoft Corporation', - author_email='ascl@microsoft.com', - url='https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-extensions', + long_description=open("README.md", "r", encoding="utf-8").read(), + long_description_content_type="text/markdown", + license="MIT", + author="Microsoft Corporation", + author_email="ascl@microsoft.com", + url="https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-extensions", keywords="azure, azure sdk", classifiers=[ - 'Development Status :: 4 - Beta', - 'Programming Language :: Python', + "Development Status :: 4 - Beta", + "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "License :: OSI Approved :: MIT License", ], zip_safe=False, python_requires=">=3.9", - ext_package='azure.storage.extensions', + packages=[ + "azure.storage.extensions.checksums", + ], + package_data={ + "azure.storage.extensions.checksums": ["py.typed"], + }, + ext_package="azure.storage.extensions.checksums", ext_modules=[ Extension( - 'crc64', - [os.path.join(package_folder_path, "crc64", "crc64module.c")], + "crc64", + [os.path.join(package_folder_path, "checksums", "crc64", "crc64module.c")], define_macros=[("Py_LIMITED_API", "3")], - py_limited_api=True + py_limited_api=True, ), ], cmdclass={"bdist_wheel": bdist_wheel_abi3}, -) \ No newline at end of file +) diff --git a/sdk/storage/azure-storage-extensions/tests/test_crc64.py b/sdk/storage/azure-storage-extensions/tests/test_crc64.py index 4202d4c892ac..b0fe4199309b 100644 --- a/sdk/storage/azure-storage-extensions/tests/test_crc64.py +++ b/sdk/storage/azure-storage-extensions/tests/test_crc64.py @@ -7,14 +7,14 @@ import os import pytest -from azure.storage.extensions import crc64 +from azure.storage.extensions import checksums def test_compute_none(): with pytest.raises(TypeError): - crc64.compute(None, 0) + checksums.crc64.compute(None, 0) with pytest.raises(TypeError): - crc64.compute(b'', None) + checksums.crc64.compute(b'', None) @pytest.mark.parametrize("data, initial, expected", [ (b'', 0, 0), @@ -46,7 +46,7 @@ def test_compute_none(): (b'6QYd.$\xd9n\xf5\xc0\xec\x17Y\xcap\xc9D\xf7\x03fn}]\xfc\x18\xb4\xdd\xf0oK\xb6W~\xffs{\xbf\x98\xf9Xs\xb6\xcd\x10\x07Fmbz\x0e\xbbH\x02\xaa\xec\xecr\x12\x95\xf24\xb2\xc3;\x81\xd3\xa5\xdc\xfeM(\xdaVs\x99\x1a\x915q\xc1#\xc2\xe8\xe7\xf6eE\xc7J\xe1dC\x91@\x89\t1\x84\xdeG\xb3\xd6G\x11\xf6\x13\xd6\x85j\xc9q@\xce\xbd\xfc{0\xc9\xbe*\x90\x02U&&F\xe0\xde\xd3 \x02\x1e\xad\xae\xfd5\xea\xb0\xe1XW\x1f\xb0\x13N\xd3\x8c\xae\x07\x928\xf0\x98\x91\xac\xfa\xd1\x9cO\x990\x85\xc3\x91\xbbh\xac\x83\xf59{\xa5\xca\x07\r]\xb1\x80W\xde\xbd\xa6\x08\n!\xe7\x02\xfeK\xe9B\x95\x08l\xa6~\xad\xf1J\x02\r\x01\x1f)&.\x95\x1c\x11\x0b\x15\xb0a\x81V\xa9]\xa0\xed\x01\xc8\xd0%\x02\x14"\x7f\xddI\xd2\x08\xd5"6\xe2y\xf3\xe8\xf6\x0f\xedn\x8e\x96\x8b\xf3{~\xbaf"T\xfc\x17\xd7lP\xab\xbb\xb2\xf5\xf0\xfaF\x13\xaaV>\x896\xc9\x8dBM\xb3\x0eO\x9b\xf9\x7f\xe9LJ\xc6\x8bQ\x81cA\xac\xe1\x82\x10\x9bT\r\xb9\x8au\xbdN\xbd\xa3\x00\x16nz1ZF8R\xa4\x18\xf3\x85\x17\xfa;\xd7k,\to\xebc\xd0\x1f\xd5N\xfbX\x88\xb6\x9e\xf3\x85\x83\xb4\xd9\xd9\xfc<\xe5\x94I\xc8\x19\xfb>\xe2\x97A\xbc\xaf\xa2\xee\xb7\xd7\x93\x08\xd8b\x9b\x9b\x83\xb9\xd0\x93\xbf\xb5\x9d\xbb,\x10\x04|\xfc\xf0\xad\xc7\xb3\xd1X6\xdc\xf53\x0e\xb96I\x1ajj`\xdd\xac:\x06/H\xe3/}\x8e4\xbae\xf8\x0e\x88\xd5\x99\xdf/\x15\xb9C\xb4Yu\xba\x19\xdd;4,\x81l\xb0S\xd7D\x02\x8e\xd5zX\\\x88\xa7\xcb\xe3QX(\xe5\x08\xc9\xf9\xaa\xcc\x85\xbc\xb4l\x84"\x1a\xe4\xc0!\xaa\x1b\x15\xcc%\xe6\xa1\xee\x11V\x8cu\xa6\xd3Xe\x9d\xa4\xd3\xbb\xbaN7\x8b\xf3\x98\xd2\xcd\x99\xfc\xddv<\xad{\xd8\xc5\xc0R\n\x11\xe8L\xab\x19\xbf\xf6w\x88\x18\xb3bg\x15\\0:q\x95\x8b%\xb9;\xadr\x96\x98\xe6\xea?\xb4\xb7\x04c<\t:\x87G\x97%\xde\xd2\x83\xbb\xa7\xe7n\xc6\'WY\xc7yP\xad\x0bC\x7f\xf8\x02c#\xc0\xc8$\x81\xa1\xb6\xbb\xa9\xcb\xcf\xae?Z\x1aP\x1e?"\xa5\xc0\xb8J\xea=\xa7\xc39\xb9}M\x84\xd9mKd\x05\x9f\x1a\x10\x99q\xefji\xbf\xf4\xbc\xdf\xd7\xb2e\xcd\xbc>\x05uL\x05\xac\xbd\xfa\xfc\xbd`R0\t\x19eu\n\xc9Y\xa2e\xfd\x84 >\xb0\xf7\xfb\xbe!\xb0\x80\x89W\nO\x8f\x14)\x00\xc1E\xf5[\xd4\xae\x7f\xbfn\xba\x81}\'\x14M\x9b\xc6\x13\xd6\xcd\x1aph\xd6!-\xe5`\xd2h\x1e\xe6\xa20\xf1\x1eH\xe3\x80\x0e\xde\x06\x93\xdb\x10B\xaa:\xb0N\x81F\x07\xde8z\xb9\xd1KQB_\xf3\xb9\xae\x88\\q%\xc0\x07\xf8;-V\xc0\x9f\x1aX\xc7\xee8\x9a\x1dO\x18\xb3X\x8c\x85\xcf\xf15eg\xa3\xaft:\xbc\x1f\x1bA\xfa/T\xf1\x1c\xe4\xe7\xe7Ot\x06\xcc\xaf\xfe\x166C\xd5x\xf9d%\x9c~q"\x16\xf5\xb8\xb4~UN\x00\x88\xad\xa6]\xb2\x18\x7f7\xa0\xca\x83)QL\xe1\xe1jP\xd2\x7f(\x1fj\xed1W\xe9\xf1\x8c\xa6\xe4f\xbd\xab\x80\xd0\x88\xb7\x9d\xa9\x1e\x9c\xaf"~\x0b\\\x99<\x02r\x15\xfco\x00\xa2,\xa9\xc5Lv\x0ez\x1b\xc8Y\xbe,\xe2%QTUf!\x9b\xee\x11\xa4\xe6\x18\x87\xe8\xb9\xfb\x14\xfb/\xfa9\xd7Ag\x95\x035\xc7\x15EA\x06\x15n+T\xdb\xf6\x05,-;\x80s(\xca}\xd6\xbf{\x87\xe0 XB\xf3r.z\xf512)\xc8\x07\xc7v\xd7S\x10\x80\xc2\xd1\x1e`\xfb\xf2\xf8O=\x08\xc3\xc7\x8fe\xea31HKF\xa6\xc6\x1a\xc75\xbc\xa7\xac\xb5\xc4\xf4\xd3]\xb8\xd85Z\x97\xba\xee\xf9\xf6\xa32-{\xdb\xe6?q\x13:\x93\x18\x91\x00(\x06\x91\xda=\x98E5r\x17\xf5z\x8b\xabt\xe4\xaax\x17C\x14\xf5\xa7\x10 \xa5\x81\x9d\xc0\xe5\x86', 12345, 6065674385535310669), ]) def test_compute(data, initial, expected): - actual = crc64.compute(data, initial) + actual = checksums.crc64.compute(data, initial) assert actual == expected @pytest.mark.parametrize("size", [1024, 4 * 1024 * 1024, 1 * 1024 * 1024 * 1024, 3 * 1024 * 1024 * 1024]) @@ -57,13 +57,13 @@ def test_compute_chunks(size): data += os.urandom(length) chunk_size = 1024 * 1024 - full_crc = crc64.compute(data, 0) + full_crc = checksums.crc64.compute(data, 0) index = 0 crc = 0 while index < len(data): end = index + chunk_size - crc = crc64.compute(data[index:end], crc) + crc = checksums.crc64.compute(data[index:end], crc) index = end assert crc == full_crc @@ -82,7 +82,7 @@ def test_compute_chunks(size): (3705122932895036835, 444701, 17573219681510991482, 603875, 4421337306620606216), ]) def test_concat(crc1, size1, crc2, size2, expected): - actaul = crc64.concat(0, 0, crc1, size1, 0, crc2, size2) + actaul = checksums.crc64.concat(0, 0, crc1, size1, 0, crc2, size2) assert actaul == expected @pytest.mark.parametrize("initial, initial1, crc1, size1, initial2, crc2, size2, expected", [ @@ -99,20 +99,20 @@ def test_concat(crc1, size1, crc2, size2, expected): (889000539881195835, 2971048229276949174, 5346315327374690144, 307387, 1407121768110541356, 10535852615249992663, 741189, 3634018251978804152), ]) def test_concat__with_initials(initial, initial1, crc1, size1, initial2, crc2, size2, expected): - actaul = crc64.concat(initial, initial1, crc1, size1, initial2, crc2, size2) + actaul = checksums.crc64.concat(initial, initial1, crc1, size1, initial2, crc2, size2) assert actaul == expected def test_concat_chunks(): data = os.urandom(4 * 1024 * 1024) chunk_size = 1024 * 1024 - full_crc = crc64.compute(data, 0) + full_crc = checksums.crc64.compute(data, 0) index = 0 crc = 0 while index < len(data): end = index + chunk_size - chunk_crc = crc64.compute(data[index:end], 0) - crc = crc64.concat(0, 0, crc, index, 0, chunk_crc, chunk_size) + chunk_crc = checksums.crc64.compute(data[index:end], 0) + crc = checksums.crc64.concat(0, 0, crc, index, 0, chunk_crc, chunk_size) index = end assert crc == full_crc diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/validation.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/validation.py index 21d3b081d8cc..7161a31bb15e 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/validation.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/validation.py @@ -20,11 +20,11 @@ def _verify_extensions(module: str) -> None: try: - import azure.storage.extensions # pylint: disable=unused-import + import azure.storage.extensions.checksums # pylint: disable=unused-import except ImportError as exc: raise ValueError( - f"The use of {module} requires the azure-storage-extensions package to be installed. " - f"Please install this package and try again." + f"The use of {module} requires the extra [ext-checksums] to be installed. " + f"Please install this extra and try again." ) from exc @@ -100,13 +100,13 @@ def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes: def calculate_crc64(data: bytes, initial_crc: int) -> int: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(int, crc64.compute(data, initial_crc)) + return checksums.crc64.compute(data, initial_crc) def calculate_crc64_bytes(data: bytes) -> bytes: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(bytes, crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little")) + return checksums.crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little") diff --git a/sdk/storage/azure-storage-file-datalake/setup.py b/sdk/storage/azure-storage-file-datalake/setup.py index 69a3c66a2b59..0bb13f7ff7ac 100644 --- a/sdk/storage/azure-storage-file-datalake/setup.py +++ b/sdk/storage/azure-storage-file-datalake/setup.py @@ -87,5 +87,8 @@ "aio": [ "azure-core[aio]>=1.37.0", ], + "ext-checksums": [ + "azure-storage-extensions>=0.1.0,<1.0.0", + ], }, ) diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/validation.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/validation.py index 21d3b081d8cc..7161a31bb15e 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/validation.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/validation.py @@ -20,11 +20,11 @@ def _verify_extensions(module: str) -> None: try: - import azure.storage.extensions # pylint: disable=unused-import + import azure.storage.extensions.checksums # pylint: disable=unused-import except ImportError as exc: raise ValueError( - f"The use of {module} requires the azure-storage-extensions package to be installed. " - f"Please install this package and try again." + f"The use of {module} requires the extra [ext-checksums] to be installed. " + f"Please install this extra and try again." ) from exc @@ -100,13 +100,13 @@ def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes: def calculate_crc64(data: bytes, initial_crc: int) -> int: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(int, crc64.compute(data, initial_crc)) + return checksums.crc64.compute(data, initial_crc) def calculate_crc64_bytes(data: bytes) -> bytes: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(bytes, crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little")) + return checksums.crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little") diff --git a/sdk/storage/azure-storage-file-share/setup.py b/sdk/storage/azure-storage-file-share/setup.py index 2e62c6db6a86..62164f376e45 100644 --- a/sdk/storage/azure-storage-file-share/setup.py +++ b/sdk/storage/azure-storage-file-share/setup.py @@ -75,5 +75,8 @@ "aio": [ "azure-core[aio]>=1.37.0", ], + "ext-checksums": [ + "azure-storage-extensions>=0.1.0,<1.0.0", + ], }, ) diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/validation.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/validation.py index 21d3b081d8cc..7161a31bb15e 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/validation.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/validation.py @@ -20,11 +20,11 @@ def _verify_extensions(module: str) -> None: try: - import azure.storage.extensions # pylint: disable=unused-import + import azure.storage.extensions.checksums # pylint: disable=unused-import except ImportError as exc: raise ValueError( - f"The use of {module} requires the azure-storage-extensions package to be installed. " - f"Please install this package and try again." + f"The use of {module} requires the extra [ext-checksums] to be installed. " + f"Please install this extra and try again." ) from exc @@ -100,13 +100,13 @@ def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes: def calculate_crc64(data: bytes, initial_crc: int) -> int: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(int, crc64.compute(data, initial_crc)) + return checksums.crc64.compute(data, initial_crc) def calculate_crc64_bytes(data: bytes) -> bytes: # Locally import to avoid error if not installed. - from azure.storage.extensions import crc64 + from azure.storage.extensions import checksums - return cast(bytes, crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little")) + return checksums.crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little") From 18684cfebb473b87b538617812c84d543fab1f4d Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Fri, 22 May 2026 15:21:37 -0700 Subject: [PATCH 2/3] Drop Python 3.9 support --- sdk/storage/azure-storage-extensions/README.md | 2 +- sdk/storage/azure-storage-extensions/pyproject.toml | 2 +- sdk/storage/azure-storage-extensions/setup.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-extensions/README.md b/sdk/storage/azure-storage-extensions/README.md index 131ea1a71713..2fb652411738 100644 --- a/sdk/storage/azure-storage-extensions/README.md +++ b/sdk/storage/azure-storage-extensions/README.md @@ -23,7 +23,7 @@ This ensures you get compatible versions of both the SDK and the extensions pack > ⚠️ Installing `azure-storage-extensions` directly is not recommended. Use the extras syntax above to ensure compatibility. ### Prerequisites -* Python 3.9 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md). +* Python 3.10 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md). ### Troubleshooting Installation Issues diff --git a/sdk/storage/azure-storage-extensions/pyproject.toml b/sdk/storage/azure-storage-extensions/pyproject.toml index 3da29b9f50da..f4591da73f1c 100644 --- a/sdk/storage/azure-storage-extensions/pyproject.toml +++ b/sdk/storage/azure-storage-extensions/pyproject.toml @@ -17,7 +17,7 @@ suppressed_skip_warnings = ["*"] requires = ["setuptools", "wheel"] [tool.cibuildwheel] -build = ["cp39*", "pp310*", "pp311*"] +build = ["cp310*", "pp310*", "pp311*"] test-requires = "pytest" test-command = "pytest {project}/tests" test-skip = "*-macosx_arm64" diff --git a/sdk/storage/azure-storage-extensions/setup.py b/sdk/storage/azure-storage-extensions/setup.py index 66022de8ec8c..206e2771a780 100644 --- a/sdk/storage/azure-storage-extensions/setup.py +++ b/sdk/storage/azure-storage-extensions/setup.py @@ -41,7 +41,6 @@ def get_tag(self): "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -50,7 +49,7 @@ def get_tag(self): "License :: OSI Approved :: MIT License", ], zip_safe=False, - python_requires=">=3.9", + python_requires=">=3.10", packages=[ "azure.storage.extensions.checksums", ], From aff5f87c0986e6a1ff2067211296dd699b36d76d Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Fri, 22 May 2026 16:13:40 -0700 Subject: [PATCH 3/3] PR feedback --- sdk/storage/azure-storage-extensions/setup.py | 3 --- sdk/storage/azure-storage-extensions/tests/test_crc64.py | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/sdk/storage/azure-storage-extensions/setup.py b/sdk/storage/azure-storage-extensions/setup.py index 206e2771a780..2d9f1fe5c043 100644 --- a/sdk/storage/azure-storage-extensions/setup.py +++ b/sdk/storage/azure-storage-extensions/setup.py @@ -53,9 +53,6 @@ def get_tag(self): packages=[ "azure.storage.extensions.checksums", ], - package_data={ - "azure.storage.extensions.checksums": ["py.typed"], - }, ext_package="azure.storage.extensions.checksums", ext_modules=[ Extension( diff --git a/sdk/storage/azure-storage-extensions/tests/test_crc64.py b/sdk/storage/azure-storage-extensions/tests/test_crc64.py index b0fe4199309b..30b86004de3c 100644 --- a/sdk/storage/azure-storage-extensions/tests/test_crc64.py +++ b/sdk/storage/azure-storage-extensions/tests/test_crc64.py @@ -82,8 +82,8 @@ def test_compute_chunks(size): (3705122932895036835, 444701, 17573219681510991482, 603875, 4421337306620606216), ]) def test_concat(crc1, size1, crc2, size2, expected): - actaul = checksums.crc64.concat(0, 0, crc1, size1, 0, crc2, size2) - assert actaul == expected + actual = checksums.crc64.concat(0, 0, crc1, size1, 0, crc2, size2) + assert actual == expected @pytest.mark.parametrize("initial, initial1, crc1, size1, initial2, crc2, size2, expected", [ (0, 0, 0, 0, 0, 0, 0, 0), @@ -99,8 +99,8 @@ def test_concat(crc1, size1, crc2, size2, expected): (889000539881195835, 2971048229276949174, 5346315327374690144, 307387, 1407121768110541356, 10535852615249992663, 741189, 3634018251978804152), ]) def test_concat__with_initials(initial, initial1, crc1, size1, initial2, crc2, size2, expected): - actaul = checksums.crc64.concat(initial, initial1, crc1, size1, initial2, crc2, size2) - assert actaul == expected + actual = checksums.crc64.concat(initial, initial1, crc1, size1, initial2, crc2, size2) + assert actual == expected def test_concat_chunks(): data = os.urandom(4 * 1024 * 1024)