[Storage] Refactor extensions packaging + other misc changes

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/validation.py

@@ -20,11 +20,11 @@
 
 def _verify_extensions(module: str) -> None:
     try:
-        import azure.storage.extensions  # pylint: disable=unused-import
+        import azure.storage.extensions.checksums  # pylint: disable=unused-import
     except ImportError as exc:
         raise ValueError(
-            f"The use of {module} requires the azure-storage-extensions package to be installed. "
-            f"Please install this package and try again."
+            f"The use of {module} requires the extra [ext-checksums] to be installed. "
+            f"Please install this extra and try again."
         ) from exc
 
 
@@ -100,13 +100,13 @@ def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes:
 
 def calculate_crc64(data: bytes, initial_crc: int) -> int:
     # Locally import to avoid error if not installed.
-    from azure.storage.extensions import crc64
+    from azure.storage.extensions import checksums
 
-    return cast(int, crc64.compute(data, initial_crc))
+    return checksums.crc64.compute(data, initial_crc)
 
 
 def calculate_crc64_bytes(data: bytes) -> bytes:
     # Locally import to avoid error if not installed.
-    from azure.storage.extensions import crc64
+    from azure.storage.extensions import checksums
 
-    return cast(bytes, crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little"))
+    return checksums.crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little")

sdk/storage/azure-storage-blob/setup.py

@@ -88,5 +88,8 @@
         "aio": [
             "azure-core[aio]>=1.37.0",
         ],
+        "ext-checksums": [
+            "azure-storage-extensions>=0.1.0,<1.0.0",
+        ],
     },
 )

sdk/storage/azure-storage-extensions/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include azure *.h *.c *.pyi py.typed

sdk/storage/azure-storage-extensions/README.md

@@ -12,22 +12,18 @@ This package contains native extension modules that provide performance-critical
 
 ## Installation
 
-**Recommended**: Install this package via extras when installing Azure Storage libraries:
+Install this package via extras when installing Azure Storage libraries:
 
 ```bash
-pip install azure-storage-blob[extensions]
+pip install azure-storage-blob[ext-checksums]
 ```
 
 This ensures you get compatible versions of both the SDK and the extensions package.
 
-You can also install it directly if needed:
-
-```bash
-pip install azure-storage-extensions
-```
+> ⚠️ Installing `azure-storage-extensions` directly is not recommended. Use the extras syntax above to ensure compatibility.
 
 ### Prerequisites
-* Python 3.9 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md).
+* Python 3.10 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md).
 
 ### Troubleshooting Installation Issues
 

sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/__init__.py

@@ -0,0 +1,9 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+from . import crc64
+
+__all__ = ["crc64"]

sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64.pyi

@@ -0,0 +1,37 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+def compute(data: bytes, crc: int, /) -> int:
+    """Compute Storage CRC64 over given data with given initial CRC64 value.
+
+    :param data: The bytes data to compute the CRC64 over.
+    :param crc: The initial CRC64 value.
+    :returns: The computed CRC64 value.
+    """
+    ...
+
+def concat(
+    initial_crc_ab: int,
+    initial_crc_a: int,
+    final_crc_a: int,
+    size_a: int,
+    initial_crc_b: int,
+    final_crc_b: int,
+    size_b: int,
+    /,
+) -> int:
+    """Concatenate two Storage CRC64s together.
+
+    :param initial_crc_ab: The initial CRC64 of the combined data.
+    :param initial_crc_a: The initial CRC64 of the first data segment.
+    :param final_crc_a: The final CRC64 of the first data segment.
+    :param size_a: The size of the first data segment in bytes.
+    :param initial_crc_b: The initial CRC64 of the second data segment.
+    :param final_crc_b: The final CRC64 of the second data segment.
+    :param size_b: The size of the second data segment in bytes.
+    :returns: The concatenated CRC64 value.
+    """
+    ...

sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/helpers.h → sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/helpers.h

@@ -3,10 +3,10 @@
 
 #include <stdint.h>
 
-const uint64_t poly = 0x9A6C9329AC4BC9B5ULL;
+static const uint64_t poly = 0x9A6C9329AC4BC9B5ULL;
 
-const uint64_t m_uComplement = ~0ULL;
-const uint64_t m_u1[] =
+static const uint64_t m_uComplement = ~0ULL;
+static const uint64_t m_u1[] =
 {
     0x0000000000000000ULL, 0x7f6ef0c830358979ULL, 0xfedde190606b12f2ULL, 0x81b31158505e9b8bULL,
     0xc962e5739841b68fULL, 0xb60c15bba8743ff6ULL, 0x37bf04e3f82aa47dULL, 0x48d1f42bc81f2d04ULL,
@@ -74,7 +74,7 @@ const uint64_t m_u1[] =
     0xab69411fbfb21ca3ULL, 0xd407b1d78f8795daULL, 0x55b4a08fdfd90e51ULL, 0x2ada5047efec8728ULL,
 };
 
-const uint64_t m_u32[] =
+static const uint64_t m_u32[] =
 {
     0x0000000000000000ULL, 0xb8c533c1177eb231ULL, 0x455341d1766af709ULL, 0xfd96721061144538ULL,
     0x8aa683a2ecd5ee12ULL, 0x3263b063fbab5c23ULL, 0xcff5c2739abf191bULL, 0x7730f1b28dc1ab2aULL,
@@ -597,7 +597,7 @@ const uint64_t m_u32[] =
     0x609abef3367d2567ULL, 0xd02690aba479d067ULL, 0x353bc4114ae35c0cULL, 0x8587ea49d8e7a90cULL,
 };
 
-const uint64_t m_uX2N[] =
+static const uint64_t m_uX2N[] =
 {
     0x0080000000000000ULL, 0x0000800000000000ULL, 0x0000000080000000ULL, 0x9a6c9329ac4bc9b5ULL,
     0x10f4bb0f129310d6ULL, 0x70f05dcea2ebd226ULL, 0x311211205672822dULL, 0x2fc297db0f46c96eULL,
@@ -622,7 +622,7 @@ const uint64_t m_uX2N[] =
 // "a" and "b" are represented in "reversed" order -- LSB is x**(XX-1) coefficient, MSB is x^0 coefficient.
 // "POLY" is represented in the same manner except for omitted x**XX coefficient
 //
-uint64_t MulPolyUnrolled(uint64_t a, uint64_t b)
+static uint64_t MulPolyUnrolled(uint64_t a, uint64_t b)
 {
     const uint64_t p = poly;
     const uint64_t p2 = (p >> 1) ^ (p * (p & 1));
@@ -647,7 +647,7 @@ uint64_t MulPolyUnrolled(uint64_t a, uint64_t b)
     return vr[0] ^ vr[1];
 }
 
-uint64_t MulX_N(uint64_t a, uint64_t uSize)
+static uint64_t MulX_N(uint64_t a, uint64_t uSize)
 {
     uint64_t i = 0;
     uint64_t r = a;

sdk/storage/azure-storage-extensions/pyproject.toml

@@ -17,7 +17,7 @@ suppressed_skip_warnings = ["*"]
 requires = ["setuptools", "wheel"]
 
 [tool.cibuildwheel]
-build = ["cp39*", "pp310*", "pp311*"]
+build = ["cp310*", "pp310*", "pp311*"]
 test-requires = "pytest"
 test-command = "pytest {project}/tests"
 test-skip = "*-macosx_arm64"

sdk/storage/azure-storage-extensions/setup.py

@@ -11,6 +11,7 @@
 
 class bdist_wheel_abi3(bdist_wheel):
     """Override bdist_wheel tag behavior to add abi3 tag."""
+
     def get_tag(self):
         python, abi, plat = super().get_tag()
 
@@ -22,43 +23,44 @@ def get_tag(self):
 PACKAGE_NAME = "azure-storage-extensions"
 PACKAGE_PPRINT_NAME = "Azure Storage Extensions"
 
-package_folder_path = PACKAGE_NAME.replace('-', '/')
+package_folder_path = PACKAGE_NAME.replace("-", "/")
 
 setup(
     name=PACKAGE_NAME,
     version="0.1.0",
-    include_package_data=True,
     description=PACKAGE_PPRINT_NAME,
-    long_description=open('README.md', 'r', encoding='utf-8').read(),
-    long_description_content_type='text/markdown',
-    license='MIT License',
-    author='Microsoft Corporation',
-    author_email='ascl@microsoft.com',
-    url='https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-extensions',
+    long_description=open("README.md", "r", encoding="utf-8").read(),
+    long_description_content_type="text/markdown",
+    license="MIT",
+    author="Microsoft Corporation",
+    author_email="ascl@microsoft.com",
+    url="https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-extensions",
     keywords="azure, azure sdk",
     classifiers=[
-        'Development Status :: 4 - Beta',
-        'Programming Language :: Python',
+        "Development Status :: 4 - Beta",
+        "Programming Language :: Python",
         "Programming Language :: Python :: 3 :: Only",
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Programming Language :: Python :: 3.12',
-        'Programming Language :: Python :: 3.13',
-        'Programming Language :: Python :: 3.14',
-        'License :: OSI Approved :: MIT License',
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
+        "License :: OSI Approved :: MIT License",
     ],
     zip_safe=False,
-    python_requires=">=3.9",
-    ext_package='azure.storage.extensions',
+    python_requires=">=3.10",
+    packages=[
+        "azure.storage.extensions.checksums",
+    ],
+    ext_package="azure.storage.extensions.checksums",
     ext_modules=[
         Extension(
-            'crc64',
-            [os.path.join(package_folder_path, "crc64", "crc64module.c")],
+            "crc64",
+            [os.path.join(package_folder_path, "checksums", "crc64", "crc64module.c")],
             define_macros=[("Py_LIMITED_API", "3")],
-            py_limited_api=True
+            py_limited_api=True,
         ),
     ],
     cmdclass={"bdist_wheel": bdist_wheel_abi3},
-)
+)

sdk/storage/azure-storage-extensions/tests/test_crc64.py

@@ -7,14 +7,14 @@
 import os
 import pytest
 
-from azure.storage.extensions import crc64
+from azure.storage.extensions import checksums
 
 def test_compute_none():
     with pytest.raises(TypeError):
-        crc64.compute(None, 0)
+        checksums.crc64.compute(None, 0)
 
     with pytest.raises(TypeError):
-        crc64.compute(b'', None)
+        checksums.crc64.compute(b'', None)
 
 @pytest.mark.parametrize("data, initial, expected", [
     (b'', 0, 0),
@@ -46,7 +46,7 @@ def test_compute_none():
     (b'6QYd.$\xd9n\xf5\xc0\xec\x17Y\xcap\xc9D\xf7\x03fn}]\xfc\x18\xb4\xdd\xf0oK\xb6W~\xffs{\xbf\x98\xf9Xs\xb6\xcd\x10\x07Fmbz\x0e\xbbH\x02\xaa\xec\xecr\x12\x95\xf24\xb2\xc3;\x81\xd3\xa5\xdc\xfeM(\xdaVs\x99\x1a\x915q\xc1#\xc2\xe8\xe7\xf6eE\xc7J\xe1dC\x91@\x89\t1\x84\xdeG\xb3\xd6G\x11\xf6\x13\xd6\x85j\xc9q@\xce\xbd\xfc{0\xc9\xbe*\x90\x02U&&F\xe0\xde\xd3 \x02\x1e\xad\xae\xfd5\xea\xb0\xe1XW\x1f\xb0\x13N\xd3\x8c\xae\x07\x928\xf0\x98\x91\xac\xfa\xd1\x9cO\x990\x85\xc3\x91\xbbh\xac\x83\xf59{\xa5\xca\x07\r]\xb1\x80W\xde\xbd\xa6\x08\n!\xe7\x02\xfeK\xe9B\x95\x08l\xa6~\xad\xf1J\x02\r\x01\x1f)&.\x95\x1c\x11\x0b\x15\xb0a\x81V\xa9]\xa0\xed\x01\xc8\xd0%\x02\x14"\x7f\xddI\xd2\x08\xd5"6\xe2y\xf3\xe8\xf6\x0f\xedn\x8e\x96\x8b\xf3{~\xbaf"T\xfc\x17\xd7lP\xab\xbb\xb2\xf5\xf0\xfaF\x13\xaaV>\x896\xc9\x8dBM\xb3\x0eO\x9b\xf9\x7f\xe9LJ\xc6\x8bQ\x81cA\xac\xe1\x82\x10\x9bT\r\xb9\x8au\xbdN\xbd\xa3\x00\x16nz1ZF8R\xa4\x18\xf3\x85\x17\xfa;\xd7k,\to\xebc\xd0\x1f\xd5N\xfbX\x88\xb6\x9e\xf3\x85\x83\xb4\xd9\xd9\xfc<\xe5\x94I\xc8\x19\xfb>\xe2\x97A\xbc\xaf\xa2\xee\xb7\xd7\x93\x08\xd8b\x9b\x9b\x83\xb9\xd0\x93\xbf\xb5\x9d\xbb,\x10\x04|\xfc\xf0\xad\xc7\xb3\xd1X6\xdc\xf53\x0e\xb96I\x1ajj`\xdd\xac:\x06/H\xe3/}\x8e4\xbae\xf8\x0e\x88\xd5\x99\xdf/\x15\xb9C\xb4Yu\xba\x19\xdd;4,\x81l\xb0S\xd7D\x02\x8e\xd5zX\\\x88\xa7\xcb\xe3QX(\xe5\x08\xc9\xf9\xaa\xcc\x85\xbc\xb4l\x84"\x1a\xe4\xc0!\xaa\x1b\x15\xcc%\xe6\xa1\xee\x11V\x8cu\xa6\xd3Xe\x9d\xa4\xd3\xbb\xbaN7\x8b\xf3\x98\xd2\xcd\x99\xfc\xddv<\xad{\xd8\xc5\xc0R\n\x11\xe8L\xab\x19\xbf\xf6w\x88\x18\xb3bg\x15\\0:q\x95\x8b%\xb9;\xadr\x96\x98\xe6\xea?\xb4\xb7\x04c<\t:\x87G\x97%\xde\xd2\x83\xbb\xa7\xe7n\xc6\'WY\xc7yP\xad\x0bC\x7f\xf8\x02c#\xc0\xc8$\x81\xa1\xb6\xbb\xa9\xcb\xcf\xae?Z\x1aP\x1e?"\xa5\xc0\xb8J\xea=\xa7\xc39\xb9}M\x84\xd9mKd\x05\x9f\x1a\x10\x99q\xefji\xbf\xf4\xbc\xdf\xd7\xb2e\xcd\xbc>\x05uL\x05\xac\xbd\xfa\xfc\xbd`R0\t\x19eu\n\xc9Y\xa2e\xfd\x84 >\xb0\xf7\xfb\xbe!\xb0\x80\x89W\nO\x8f\x14)\x00\xc1E\xf5[\xd4\xae\x7f\xbfn\xba\x81}\'\x14M\x9b\xc6\x13\xd6\xcd\x1aph\xd6!-\xe5`\xd2h\x1e\xe6\xa20\xf1\x1eH\xe3\x80\x0e\xde\x06\x93\xdb\x10B\xaa:\xb0N\x81F\x07\xde8z\xb9\xd1KQB_\xf3\xb9\xae\x88\\q%\xc0\x07\xf8;-V\xc0\x9f\x1aX\xc7\xee8\x9a\x1dO\x18\xb3X\x8c\x85\xcf\xf15eg\xa3\xaft:\xbc\x1f\x1bA\xfa/T\xf1\x1c\xe4\xe7\xe7Ot\x06\xcc\xaf\xfe\x166C\xd5x\xf9d%\x9c~q"\x16\xf5\xb8\xb4~UN\x00\x88\xad\xa6]\xb2\x18\x7f7\xa0\xca\x83)QL\xe1\xe1jP\xd2\x7f(\x1fj\xed1W\xe9\xf1\x8c\xa6\xe4f\xbd\xab\x80\xd0\x88\xb7\x9d\xa9\x1e\x9c\xaf"~\x0b\\\x99<\x02r\x15\xfco\x00\xa2,\xa9\xc5Lv\x0ez\x1b\xc8Y\xbe,\xe2%QTUf!\x9b\xee\x11\xa4\xe6\x18\x87\xe8\xb9\xfb\x14\xfb/\xfa9\xd7Ag\x95\x035\xc7\x15EA\x06\x15n+T\xdb\xf6\x05,-;\x80s(\xca}\xd6\xbf{\x87\xe0 XB\xf3r.z\xf512)\xc8\x07\xc7v\xd7S\x10\x80\xc2\xd1\x1e`\xfb\xf2\xf8O=\x08\xc3\xc7\x8fe\xea31HKF\xa6\xc6\x1a\xc75\xbc\xa7\xac\xb5\xc4\xf4\xd3]\xb8\xd85Z\x97\xba\xee\xf9\xf6\xa32-{\xdb\xe6?q\x13:\x93\x18\x91\x00(\x06\x91\xda=\x98E5r\x17\xf5z\x8b\xabt\xe4\xaax\x17C\x14\xf5\xa7\x10 \xa5\x81\x9d\xc0\xe5\x86', 12345, 6065674385535310669),
 ])
 def test_compute(data, initial, expected):
-    actual = crc64.compute(data, initial)
+    actual = checksums.crc64.compute(data, initial)
     assert actual == expected
 
 @pytest.mark.parametrize("size", [1024, 4 * 1024 * 1024, 1 * 1024 * 1024 * 1024, 3 * 1024 * 1024 * 1024])
@@ -57,13 +57,13 @@ def test_compute_chunks(size):
         data += os.urandom(length)
 
     chunk_size = 1024 * 1024
-    full_crc = crc64.compute(data, 0)
+    full_crc = checksums.crc64.compute(data, 0)
 
     index = 0
     crc = 0
     while index < len(data):
         end = index + chunk_size
-        crc = crc64.compute(data[index:end], crc)
+        crc = checksums.crc64.compute(data[index:end], crc)
         index = end
 
     assert crc == full_crc
@@ -82,8 +82,8 @@ def test_compute_chunks(size):
     (3705122932895036835, 444701, 17573219681510991482, 603875, 4421337306620606216),
 ])
 def test_concat(crc1, size1, crc2, size2, expected):
-    actaul = crc64.concat(0, 0, crc1, size1, 0, crc2, size2)
-    assert actaul == expected
+    actual = checksums.crc64.concat(0, 0, crc1, size1, 0, crc2, size2)
+    assert actual == expected
 
 @pytest.mark.parametrize("initial, initial1, crc1, size1, initial2, crc2, size2, expected", [
     (0, 0, 0, 0, 0, 0, 0, 0),
@@ -99,20 +99,20 @@ def test_concat(crc1, size1, crc2, size2, expected):
     (889000539881195835, 2971048229276949174, 5346315327374690144, 307387, 1407121768110541356, 10535852615249992663, 741189, 3634018251978804152),
 ])
 def test_concat__with_initials(initial, initial1, crc1, size1, initial2, crc2, size2, expected):
-    actaul = crc64.concat(initial, initial1, crc1, size1, initial2, crc2, size2)
-    assert actaul == expected
+    actual = checksums.crc64.concat(initial, initial1, crc1, size1, initial2, crc2, size2)
+    assert actual == expected
 
 def test_concat_chunks():
     data = os.urandom(4 * 1024 * 1024)
     chunk_size = 1024 * 1024
-    full_crc = crc64.compute(data, 0)
+    full_crc = checksums.crc64.compute(data, 0)
 
     index = 0
     crc = 0
     while index < len(data):
         end = index + chunk_size
-        chunk_crc = crc64.compute(data[index:end], 0)
-        crc = crc64.concat(0, 0, crc, index, 0, chunk_crc, chunk_size)
+        chunk_crc = checksums.crc64.compute(data[index:end], 0)
+        crc = checksums.crc64.concat(0, 0, crc, index, 0, chunk_crc, chunk_size)
         index = end
 
     assert crc == full_crc

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/validation.py

@@ -20,11 +20,11 @@
 
 def _verify_extensions(module: str) -> None:
     try:
-        import azure.storage.extensions  # pylint: disable=unused-import
+        import azure.storage.extensions.checksums  # pylint: disable=unused-import
     except ImportError as exc:
         raise ValueError(
-            f"The use of {module} requires the azure-storage-extensions package to be installed. "
-            f"Please install this package and try again."
+            f"The use of {module} requires the extra [ext-checksums] to be installed. "
+            f"Please install this extra and try again."
         ) from exc
 
 
@@ -100,13 +100,13 @@ def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes:
 
 def calculate_crc64(data: bytes, initial_crc: int) -> int:
     # Locally import to avoid error if not installed.
-    from azure.storage.extensions import crc64
+    from azure.storage.extensions import checksums
 
-    return cast(int, crc64.compute(data, initial_crc))
+    return checksums.crc64.compute(data, initial_crc)
 
 
 def calculate_crc64_bytes(data: bytes) -> bytes:
     # Locally import to avoid error if not installed.
-    from azure.storage.extensions import crc64
+    from azure.storage.extensions import checksums
 
-    return cast(bytes, crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little"))
+    return checksums.crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little")

sdk/storage/azure-storage-file-datalake/setup.py

@@ -87,5 +87,8 @@
         "aio": [
             "azure-core[aio]>=1.37.0",
         ],
+        "ext-checksums": [
+            "azure-storage-extensions>=0.1.0,<1.0.0",
+        ],
     },
 )