diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index b8b1c451ad10..ba62851d9ccf 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.16.2 (Unreleased) #### Features Added +* Added `GlobalSecondaryIndexDefinition` class and `global_secondary_index_definition` keyword to `create_container`, `create_container_if_not_exists`, and `replace_container` methods for creating Global Secondary Index (GSI) containers. See [PR 47468](https://github.com/Azure/azure-sdk-for-python/pull/47468). #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/__init__.py b/sdk/cosmos/azure-cosmos/azure/cosmos/__init__.py index d7501df99558..5e6720a0726c 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/__init__.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/__init__.py @@ -43,6 +43,7 @@ ) from .partition_key import PartitionKey from .permission import Permission +from ._global_secondary_index import GlobalSecondaryIndexDefinition __all__ = ( "CosmosClient", @@ -66,6 +67,7 @@ "ConnectionRetryPolicy", "ThroughputProperties", "CosmosDict", - "CosmosList" + "CosmosList", + "GlobalSecondaryIndexDefinition" ) __version__ = VERSION diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/_global_secondary_index.py b/sdk/cosmos/azure-cosmos/azure/cosmos/_global_secondary_index.py new file mode 100644 index 000000000000..22d2710b9290 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/_global_secondary_index.py @@ -0,0 +1,123 @@ +# The MIT License (MIT) +# Copyright (c) 2014 Microsoft Corporation + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Global Secondary Index (GSI) container definition.""" + +from typing import Optional + + +class GlobalSecondaryIndexDefinition: + """**provisional** Definition for a Global Secondary Index (GSI) container. + + A GSI container is a derived container built from a source container + using a SQL-like projection query. The GSI definition is immutable after creation. + + .. note:: + A maximum of 5 GSI containers can be created per source container. + All GSI containers must be deleted before deleting the source container. + + :param str source_container_id: The ID of the source container the GSI is derived from. Required. + :param str definition: The SQL-like projection query that defines the GSI. Required. + """ + + def __init__(self, source_container_id: str, definition: str): + if not source_container_id or not source_container_id.strip(): + raise ValueError("source_container_id cannot be None or empty.") + if not definition or not definition.strip(): + raise ValueError("definition cannot be None or empty.") + self._source_container_id = source_container_id + self._definition = definition + self._source_container_rid: Optional[str] = None + self._status: Optional[str] = None + + @property + def source_container_id(self) -> str: + """The ID of the source container. + + :returns: The source container ID. + :rtype: str + """ + return self._source_container_id + + @property + def definition(self) -> str: + """The SQL-like projection query that defines the GSI. + + :returns: The projection query. + :rtype: str + """ + return self._definition + + @property + def source_container_rid(self) -> Optional[str]: + """The server-populated resource ID (_rid) of the source container. Read-only. + + :returns: The source container resource ID, or None if not yet populated. + :rtype: str or None + """ + return self._source_container_rid + + @property + def status(self) -> Optional[str]: + """The GSI build status. Read-only, server-populated. + + Possible values: "Initializing", "InitialBuildAfterCreate", + "InitialBuildAfterRestore", "Active", "DeleteInProgress" + + :returns: The GSI status, or None if not yet populated. + :rtype: str or None + """ + return self._status + + def _to_dict(self) -> dict: + """Serialize to wire format dict. + + :returns: A dictionary representation of the GSI definition. + :rtype: dict + """ + result: dict = { + "sourceCollectionId": self._source_container_id, + "definition": self._definition, + } + if self._source_container_rid is not None: + result["sourceCollectionRid"] = self._source_container_rid + if self._status is not None: + result["status"] = self._status + return result + + @classmethod + def _from_dict(cls, data: Optional[dict]) -> Optional["GlobalSecondaryIndexDefinition"]: + """Deserialize from wire format dict. + + :param dict data: The wire format dictionary. + :returns: A GlobalSecondaryIndexDefinition instance, or None if data is None or invalid. + :rtype: ~azure.cosmos.GlobalSecondaryIndexDefinition or None + """ + if data is None: + return None + source_container_id = data.get("sourceCollectionId") + definition_query = data.get("definition") + if not source_container_id or not definition_query: + return None + instance = cls(source_container_id, definition_query) + instance._source_container_rid = data.get("sourceCollectionRid") # pylint: disable=protected-access + instance._status = data.get("status") # pylint: disable=protected-access + return instance diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py index 35f5af35d624..9eefff7a195c 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/aio/_database.py @@ -40,6 +40,7 @@ from ..documents import IndexingMode from ..partition_key import PartitionKey from .._cosmos_responses import CosmosDict +from .._global_secondary_index import GlobalSecondaryIndexDefinition __all__ = ("DatabaseProxy",) @@ -178,6 +179,7 @@ async def create_container( vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[False] = False, **kwargs: Any ) -> ContainerProxy: @@ -216,6 +218,10 @@ async def create_container( :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :raises ~azure.cosmos.exceptions.CosmosHttpResponseError: The container creation failed. @@ -259,6 +265,7 @@ async def create_container( # pylint: disable=too-many-statements vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[True], **kwargs: Any ) -> tuple[ContainerProxy, CosmosDict]: @@ -297,6 +304,10 @@ async def create_container( # pylint: disable=too-many-statements :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :raises ~azure.cosmos.exceptions.CosmosHttpResponseError: The container creation failed. @@ -365,6 +376,10 @@ async def create_container( # pylint:disable=docstring-should-be-keyword, too-ma :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :raises ~azure.cosmos.exceptions.CosmosHttpResponseError: The container creation failed. @@ -405,6 +420,7 @@ async def create_container( # pylint:disable=docstring-should-be-keyword, too-ma computed_properties = kwargs.pop('computed_properties', None) change_feed_policy = kwargs.pop('change_feed_policy', None) full_text_policy = kwargs.pop('full_text_policy', None) + global_secondary_index_definition = kwargs.pop('global_secondary_index_definition', None) return_properties = kwargs.pop('return_properties', False) session_token = kwargs.get('session_token') @@ -452,6 +468,12 @@ async def create_container( # pylint:disable=docstring-should-be-keyword, too-ma definition["changeFeedPolicy"] = change_feed_policy if full_text_policy is not None: definition["fullTextPolicy"] = full_text_policy + if global_secondary_index_definition is not None: + gsi_dict = (global_secondary_index_definition._to_dict() + if hasattr(global_secondary_index_definition, '_to_dict') + else global_secondary_index_definition) + definition["globalSecondaryIndexDefinition"] = gsi_dict + definition["materializedViewDefinition"] = gsi_dict request_options = _build_options(kwargs) _set_throughput_options(offer=offer_throughput, request_options=request_options) @@ -479,6 +501,7 @@ async def create_container_if_not_exists( vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[False] = False, **kwargs: Any ) -> ContainerProxy: @@ -519,6 +542,10 @@ async def create_container_if_not_exists( :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :raises ~azure.cosmos.exceptions.CosmosHttpResponseError: The container creation failed. @@ -544,6 +571,7 @@ async def create_container_if_not_exists( vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[True], **kwargs: Any ) -> tuple[ContainerProxy, CosmosDict]: @@ -584,6 +612,10 @@ async def create_container_if_not_exists( :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :raises ~azure.cosmos.exceptions.CosmosHttpResponseError: The container creation failed. @@ -636,6 +668,10 @@ async def create_container_if_not_exists( # pylint:disable=docstring-should-be-k :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :raises ~azure.cosmos.exceptions.CosmosHttpResponseError: The container creation failed. @@ -659,6 +695,7 @@ async def create_container_if_not_exists( # pylint:disable=docstring-should-be-k computed_properties = kwargs.pop('computed_properties', None) change_feed_policy = kwargs.pop('change_feed_policy', None) full_text_policy = kwargs.pop('full_text_policy', None) + global_secondary_index_definition = kwargs.pop('global_secondary_index_definition', None) return_properties = kwargs.pop('return_properties', False) session_token = kwargs.get('session_token') @@ -703,6 +740,7 @@ async def create_container_if_not_exists( # pylint:disable=docstring-should-be-k vector_embedding_policy=vector_embedding_policy, change_feed_policy=change_feed_policy, full_text_policy=full_text_policy, + global_secondary_index_definition=global_secondary_index_definition, return_properties=return_properties, **kwargs ) @@ -840,6 +878,7 @@ async def replace_container( analytical_storage_ttl: Optional[int] = None, computed_properties: Optional[list[dict[str, str]]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[False] = False, vector_embedding_policy: Optional[dict[str, Any]] = None, **kwargs: Any @@ -870,6 +909,10 @@ async def replace_container( :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container. @@ -901,6 +944,7 @@ async def replace_container( # pylint:disable=docstring-missing-param analytical_storage_ttl: Optional[int] = None, computed_properties: Optional[list[dict[str, str]]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[True], vector_embedding_policy: Optional[dict[str, Any]] = None, **kwargs: Any @@ -931,6 +975,10 @@ async def replace_container( # pylint:disable=docstring-missing-param :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A tuple of the `ContainerProxy` and CosmosDict with the container properties. @@ -983,6 +1031,10 @@ async def replace_container( # pylint:disable=docstring-should-be-keyword :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container or a tuple of the ContainerProxy @@ -1013,6 +1065,7 @@ async def replace_container( # pylint:disable=docstring-should-be-keyword analytical_storage_ttl = kwargs.pop('analytical_storage_ttl', None) computed_properties = kwargs.pop('computed_properties', None) full_text_policy = kwargs.pop('full_text_policy', None) + global_secondary_index_definition = kwargs.pop('global_secondary_index_definition', None) return_properties = kwargs.pop('return_properties', False) vector_embedding_policy = kwargs.pop('vector_embedding_policy', None) @@ -1055,6 +1108,12 @@ async def replace_container( # pylint:disable=docstring-should-be-keyword }.items() if value is not None } + if global_secondary_index_definition is not None: + gsi_dict = (global_secondary_index_definition._to_dict() + if hasattr(global_secondary_index_definition, '_to_dict') + else global_secondary_index_definition) + parameters["globalSecondaryIndexDefinition"] = gsi_dict + parameters["materializedViewDefinition"] = gsi_dict container_properties = await self.client_connection.ReplaceContainer( container_link, collection=parameters, options=request_options, **kwargs diff --git a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py index 938c74636b06..449eadd94403 100644 --- a/sdk/cosmos/azure-cosmos/azure/cosmos/database.py +++ b/sdk/cosmos/azure-cosmos/azure/cosmos/database.py @@ -38,6 +38,7 @@ from .user import UserProxy from .documents import IndexingMode from ._cosmos_responses import CosmosDict +from ._global_secondary_index import GlobalSecondaryIndexDefinition __all__ = ("DatabaseProxy",) @@ -176,6 +177,7 @@ def create_container( # pylint:disable=docstring-missing-param vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[False] = False, **kwargs: Any ) -> ContainerProxy: @@ -211,6 +213,10 @@ def create_container( # pylint:disable=docstring-missing-param :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container @@ -253,6 +259,7 @@ def create_container( # pylint:disable=docstring-missing-param vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[True], **kwargs: Any ) -> tuple[ContainerProxy, CosmosDict]: @@ -288,6 +295,10 @@ def create_container( # pylint:disable=docstring-missing-param :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A tuple of the `ContainerProxy`and CosmosDict with the container properties. @@ -351,6 +362,10 @@ def create_container( # pylint:disable=docstring-missing-param, too-many-statem :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container or a tuple of the ContainerProxy @@ -389,6 +404,7 @@ def create_container( # pylint:disable=docstring-missing-param, too-many-statem computed_properties = kwargs.pop('computed_properties', None) change_feed_policy = kwargs.pop('change_feed_policy', None) full_text_policy = kwargs.pop('full_text_policy', None) + global_secondary_index_definition = kwargs.pop('global_secondary_index_definition', None) return_properties = kwargs.pop('return_properties', False) session_token = kwargs.get('session_token') @@ -442,6 +458,12 @@ def create_container( # pylint:disable=docstring-missing-param, too-many-statem definition["changeFeedPolicy"] = change_feed_policy if full_text_policy is not None: definition["fullTextPolicy"] = full_text_policy + if global_secondary_index_definition is not None: + gsi_dict = (global_secondary_index_definition._to_dict() + if hasattr(global_secondary_index_definition, '_to_dict') + else global_secondary_index_definition) + definition["globalSecondaryIndexDefinition"] = gsi_dict + definition["materializedViewDefinition"] = gsi_dict request_options = build_options(kwargs) _set_throughput_options(offer=offer_throughput, request_options=request_options) result = self.client_connection.CreateContainer( @@ -470,6 +492,7 @@ def create_container_if_not_exists( # pylint:disable=docstring-missing-param vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[False] = False, **kwargs: Any ) -> ContainerProxy: @@ -507,6 +530,10 @@ def create_container_if_not_exists( # pylint:disable=docstring-missing-param :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container. @@ -533,6 +560,7 @@ def create_container_if_not_exists( # pylint:disable=docstring-missing-param vector_embedding_policy: Optional[dict[str, Any]] = None, change_feed_policy: Optional[dict[str, Any]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[True], **kwargs: Any ) -> tuple[ContainerProxy, CosmosDict]: @@ -570,6 +598,10 @@ def create_container_if_not_exists( # pylint:disable=docstring-missing-param :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A tuple of the `ContainerProxy`and CosmosDict with the container properties. @@ -619,6 +651,10 @@ def create_container_if_not_exists( # pylint:disable=docstring-missing-param, d :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container or a tuple of the ContainerProxy @@ -643,6 +679,7 @@ def create_container_if_not_exists( # pylint:disable=docstring-missing-param, d computed_properties = kwargs.pop('computed_properties', None) change_feed_policy = kwargs.pop('change_feed_policy', None) full_text_policy = kwargs.pop('full_text_policy', None) + global_secondary_index_definition = kwargs.pop('global_secondary_index_definition', None) return_properties = kwargs.pop('return_properties', False) session_token = kwargs.get('session_token') @@ -690,6 +727,7 @@ def create_container_if_not_exists( # pylint:disable=docstring-missing-param, d vector_embedding_policy=vector_embedding_policy, change_feed_policy=change_feed_policy, full_text_policy=full_text_policy, + global_secondary_index_definition=global_secondary_index_definition, return_properties=return_properties, **kwargs ) @@ -892,6 +930,7 @@ def replace_container( # pylint:disable=docstring-missing-param analytical_storage_ttl: Optional[int] = None, computed_properties: Optional[list[dict[str, str]]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[False] = False, vector_embedding_policy: Optional[dict[str, Any]] = None, **kwargs: Any @@ -920,6 +959,10 @@ def replace_container( # pylint:disable=docstring-missing-param :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container. @@ -952,6 +995,7 @@ def replace_container( # pylint:disable=docstring-missing-param analytical_storage_ttl: Optional[int] = None, computed_properties: Optional[list[dict[str, str]]] = None, full_text_policy: Optional[dict[str, Any]] = None, + global_secondary_index_definition: Optional[Union[GlobalSecondaryIndexDefinition, dict[str, Any]]] = None, return_properties: Literal[True], vector_embedding_policy: Optional[dict[str, Any]] = None, **kwargs: Any @@ -980,6 +1024,10 @@ def replace_container( # pylint:disable=docstring-missing-param :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A tuple of the `ContainerProxy`and CosmosDict with the container properties. @@ -1029,6 +1077,10 @@ def replace_container( # pylint:disable=docstring-missing-param, docstring-shou :keyword dict[str, Any] full_text_policy: **provisional** The full text policy for the container. Used to denote the default language to be used for all full text indexes, or to individually assign a language to each full text index path. + :keyword global_secondary_index_definition: **provisional** The global secondary index + definition for the container. + Used to create a GSI container derived from a source container via a SQL projection query. + :paramtype global_secondary_index_definition: ~azure.cosmos.GlobalSecondaryIndexDefinition or dict[str, Any] :keyword bool return_properties: Specifies whether to return either a ContainerProxy or a Tuple of a ContainerProxy and the container properties. :returns: A `ContainerProxy` instance representing the new container or a tuple of the ContainerProxy @@ -1059,6 +1111,7 @@ def replace_container( # pylint:disable=docstring-missing-param, docstring-shou analytical_storage_ttl = kwargs.pop('analytical_storage_ttl', None) computed_properties = kwargs.pop('computed_properties', None) full_text_policy = kwargs.pop('full_text_policy', None) + global_secondary_index_definition = kwargs.pop('global_secondary_index_definition', None) return_properties = kwargs.pop('return_properties', False) vector_embedding_policy = kwargs.pop('vector_embedding_policy', None) @@ -1107,6 +1160,12 @@ def replace_container( # pylint:disable=docstring-missing-param, docstring-shou }.items() if value is not None } + if global_secondary_index_definition is not None: + gsi_dict = (global_secondary_index_definition._to_dict() + if hasattr(global_secondary_index_definition, '_to_dict') + else global_secondary_index_definition) + parameters["globalSecondaryIndexDefinition"] = gsi_dict + parameters["materializedViewDefinition"] = gsi_dict container_properties = self.client_connection.ReplaceContainer( container_link, collection=parameters, options=request_options, **kwargs) diff --git a/sdk/cosmos/azure-cosmos/pytest.ini b/sdk/cosmos/azure-cosmos/pytest.ini index 04146d38d7a9..e7facc2e9ca7 100644 --- a/sdk/cosmos/azure-cosmos/pytest.ini +++ b/sdk/cosmos/azure-cosmos/pytest.ini @@ -21,3 +21,4 @@ markers = cosmosCircuitBreakerMultiRegion: marks tests running on Cosmos DB live account with one write region and multiple read regions and per partition circuit breaker enabled. cosmosPerPartitionAutomaticFailover: marks tests running on Cosmos DB live account with one write region and multiple read regions and per partition automatic failover enabled. semanticReranker: marks tests running on a Cosmos DB live account with semantic reranker enabled. + cosmosGSI: marks tests running on a Cosmos DB live account with Global Secondary Index (GSI) support enabled. diff --git a/sdk/cosmos/azure-cosmos/tests/test_global_secondary_index.py b/sdk/cosmos/azure-cosmos/tests/test_global_secondary_index.py new file mode 100644 index 000000000000..fe6c2674afa5 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/tests/test_global_secondary_index.py @@ -0,0 +1,214 @@ +# The MIT License (MIT) +# Copyright (c) 2014 Microsoft Corporation + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Unit tests for GlobalSecondaryIndexDefinition.""" + +import pytest +from azure.cosmos._global_secondary_index import GlobalSecondaryIndexDefinition + + +class TestGlobalSecondaryIndexDefinition: + """Tests for GlobalSecondaryIndexDefinition class.""" + + def test_constructor_valid(self): + """Test valid construction.""" + gsi = GlobalSecondaryIndexDefinition("source-container", "SELECT c.email, c.name FROM c") + assert gsi.source_container_id == "source-container" + assert gsi.definition == "SELECT c.email, c.name FROM c" + assert gsi.source_container_rid is None + assert gsi.status is None + + def test_constructor_rejects_none_source_container_id(self): + """Test that None source_container_id raises ValueError.""" + with pytest.raises(ValueError, match="source_container_id cannot be None or empty"): + GlobalSecondaryIndexDefinition(None, "SELECT c.email FROM c") + + def test_constructor_rejects_empty_source_container_id(self): + """Test that empty source_container_id raises ValueError.""" + with pytest.raises(ValueError, match="source_container_id cannot be None or empty"): + GlobalSecondaryIndexDefinition("", "SELECT c.email FROM c") + + def test_constructor_rejects_whitespace_source_container_id(self): + """Test that whitespace-only source_container_id raises ValueError.""" + with pytest.raises(ValueError, match="source_container_id cannot be None or empty"): + GlobalSecondaryIndexDefinition(" ", "SELECT c.email FROM c") + + def test_constructor_rejects_none_definition(self): + """Test that None definition raises ValueError.""" + with pytest.raises(ValueError, match="definition cannot be None or empty"): + GlobalSecondaryIndexDefinition("source-container", None) + + def test_constructor_rejects_empty_definition(self): + """Test that empty definition raises ValueError.""" + with pytest.raises(ValueError, match="definition cannot be None or empty"): + GlobalSecondaryIndexDefinition("source-container", "") + + def test_constructor_rejects_whitespace_definition(self): + """Test that whitespace-only definition raises ValueError.""" + with pytest.raises(ValueError, match="definition cannot be None or empty"): + GlobalSecondaryIndexDefinition("source-container", " ") + + def test_to_dict_basic(self): + """Test _to_dict produces correct wire format.""" + gsi = GlobalSecondaryIndexDefinition("my-source", "SELECT c.email FROM c") + result = gsi._to_dict() + assert result == { + "sourceCollectionId": "my-source", + "definition": "SELECT c.email FROM c", + } + + def test_to_dict_with_server_fields(self): + """Test _to_dict includes server-populated fields when present.""" + gsi = GlobalSecondaryIndexDefinition("my-source", "SELECT c.email FROM c") + gsi._source_container_rid = "Z4oBANAPOuI=" + gsi._status = "Active" + result = gsi._to_dict() + assert result == { + "sourceCollectionId": "my-source", + "definition": "SELECT c.email FROM c", + "sourceCollectionRid": "Z4oBANAPOuI=", + "status": "Active", + } + + def test_from_dict_valid(self): + """Test _from_dict correctly deserializes.""" + data = { + "sourceCollectionId": "source-container-id", + "definition": "SELECT c.email, c.name FROM c", + "sourceCollectionRid": "Z4oBANAPOuI=", + "status": "Active", + } + gsi = GlobalSecondaryIndexDefinition._from_dict(data) + assert gsi is not None + assert gsi.source_container_id == "source-container-id" + assert gsi.definition == "SELECT c.email, c.name FROM c" + assert gsi.source_container_rid == "Z4oBANAPOuI=" + assert gsi.status == "Active" + + def test_from_dict_minimal(self): + """Test _from_dict with only required fields.""" + data = { + "sourceCollectionId": "source-container", + "definition": "SELECT c.id FROM c", + } + gsi = GlobalSecondaryIndexDefinition._from_dict(data) + assert gsi is not None + assert gsi.source_container_id == "source-container" + assert gsi.definition == "SELECT c.id FROM c" + assert gsi.source_container_rid is None + assert gsi.status is None + + def test_from_dict_returns_none_for_none_input(self): + """Test _from_dict returns None for None input.""" + assert GlobalSecondaryIndexDefinition._from_dict(None) is None + + def test_from_dict_returns_none_for_missing_source_collection_id(self): + """Test _from_dict returns None when sourceCollectionId is missing.""" + data = {"definition": "SELECT c.email FROM c"} + assert GlobalSecondaryIndexDefinition._from_dict(data) is None + + def test_from_dict_returns_none_for_missing_definition(self): + """Test _from_dict returns None when definition is missing.""" + data = {"sourceCollectionId": "source-container"} + assert GlobalSecondaryIndexDefinition._from_dict(data) is None + + def test_from_dict_returns_none_for_empty_dict(self): + """Test _from_dict returns None for empty dict.""" + assert GlobalSecondaryIndexDefinition._from_dict({}) is None + + def test_roundtrip(self): + """Test serialization roundtrip.""" + original = GlobalSecondaryIndexDefinition("src-container", "SELECT c.name FROM c") + original._source_container_rid = "abc123=" + original._status = "InitialBuildAfterCreate" + + wire_dict = original._to_dict() + restored = GlobalSecondaryIndexDefinition._from_dict(wire_dict) + + assert restored is not None + assert restored.source_container_id == original.source_container_id + assert restored.definition == original.definition + assert restored.source_container_rid == original.source_container_rid + assert restored.status == original.status + + def test_all_status_values_accepted(self): + """Test that all documented status values can be read.""" + statuses = [ + "Initializing", + "InitialBuildAfterCreate", + "InitialBuildAfterRestore", + "Active", + "DeleteInProgress", + ] + for status in statuses: + data = { + "sourceCollectionId": "src", + "definition": "SELECT c.id FROM c", + "status": status, + } + gsi = GlobalSecondaryIndexDefinition._from_dict(data) + assert gsi is not None + assert gsi.status == status + + def test_unknown_status_accepted(self): + """Test that unknown status values are accepted for forward-compatibility.""" + data = { + "sourceCollectionId": "src", + "definition": "SELECT c.id FROM c", + "status": "SomeNewFutureStatus", + } + gsi = GlobalSecondaryIndexDefinition._from_dict(data) + assert gsi is not None + assert gsi.status == "SomeNewFutureStatus" + + def test_properties_are_readonly(self): + """Test that server-populated properties cannot be set directly via public API.""" + gsi = GlobalSecondaryIndexDefinition("src", "SELECT c.id FROM c") + with pytest.raises(AttributeError): + gsi.source_container_id = "new-value" + with pytest.raises(AttributeError): + gsi.definition = "new-value" + with pytest.raises(AttributeError): + gsi.source_container_rid = "new-value" + with pytest.raises(AttributeError): + gsi.status = "new-value" + + def test_import_from_azure_cosmos(self): + """Test that GlobalSecondaryIndexDefinition is importable from azure.cosmos.""" + from azure.cosmos import GlobalSecondaryIndexDefinition as GSI + assert GSI is GlobalSecondaryIndexDefinition + + def test_dual_write_pattern(self): + """Test that dual-write produces both keys in the definition dict.""" + gsi = GlobalSecondaryIndexDefinition("source-container", "SELECT c.email FROM c") + gsi_dict = gsi._to_dict() + + # Simulate the dual-write pattern used in database.py + definition = {} + definition["globalSecondaryIndexDefinition"] = gsi_dict + definition["materializedViewDefinition"] = gsi_dict + + # Both keys must be present with identical content + assert "globalSecondaryIndexDefinition" in definition + assert "materializedViewDefinition" in definition + assert definition["globalSecondaryIndexDefinition"] == definition["materializedViewDefinition"] + assert definition["globalSecondaryIndexDefinition"]["sourceCollectionId"] == "source-container" + assert definition["globalSecondaryIndexDefinition"]["definition"] == "SELECT c.email FROM c" diff --git a/sdk/cosmos/azure-cosmos/tests/test_global_secondary_index_live.py b/sdk/cosmos/azure-cosmos/tests/test_global_secondary_index_live.py new file mode 100644 index 000000000000..f66e1bddda6c --- /dev/null +++ b/sdk/cosmos/azure-cosmos/tests/test_global_secondary_index_live.py @@ -0,0 +1,138 @@ +# The MIT License (MIT) +# Copyright (c) Microsoft Corporation. All rights reserved. + +import os +import unittest +import uuid + +import pytest + +import test_config +from azure.cosmos import CosmosClient, PartitionKey, GlobalSecondaryIndexDefinition + + +@pytest.mark.cosmosGSI +class TestGlobalSecondaryIndexLive(unittest.TestCase): + """Live tests for Global Secondary Index (GSI) container operations. + + These tests require a Cosmos DB account with GSI support enabled. + The account endpoint and key are sourced from Key Vault secrets: + - gsi-pipeline-uri -> GSI_ACCOUNT_HOST + - gsi-pipeline-key -> GSI_ACCOUNT_KEY + """ + client: CosmosClient = None + host = os.getenv('GSI_ACCOUNT_HOST', test_config.TestConfig.host) + masterKey = os.getenv('GSI_ACCOUNT_KEY', test_config.TestConfig.masterKey) + connectionPolicy = test_config.TestConfig.connectionPolicy + + @classmethod + def setUpClass(cls): + if (cls.masterKey == '[YOUR_KEY_HERE]' or + cls.host == '[YOUR_ENDPOINT_HERE]'): + raise Exception( + "You must specify your Azure Cosmos account values for " + "'masterKey' and 'host' at the top of this class to run the " + "tests.") + cls.client = CosmosClient(cls.host, cls.masterKey) + cls.test_db = cls.client.create_database(str(uuid.uuid4())) + + @classmethod + def tearDownClass(cls): + if cls.test_db: + cls.client.delete_database(cls.test_db.id) + + def test_create_gsi_container(self): + """Test creating a GSI container derived from a source container.""" + # Create source container + source_container = self.test_db.create_container( + id="source-container-" + str(uuid.uuid4())[:8], + partition_key=PartitionKey(path="/id") + ) + + # Create GSI container using GlobalSecondaryIndexDefinition + gsi_definition = GlobalSecondaryIndexDefinition( + source_container_id=source_container.id, + definition="SELECT c.id, c.email, c.name FROM c" + ) + gsi_container = self.test_db.create_container( + id="gsi-container-" + str(uuid.uuid4())[:8], + partition_key=PartitionKey(path="/id"), + global_secondary_index_definition=gsi_definition + ) + + # Read back the container properties and verify GSI definition is present + properties = gsi_container.read() + self.assertIn("globalSecondaryIndexDefinition", properties) + gsi_props = properties["globalSecondaryIndexDefinition"] + self.assertEqual(gsi_props["sourceCollectionId"], source_container.id) + self.assertEqual(gsi_props["definition"], "SELECT c.id, c.email, c.name FROM c") + self.assertIn("status", gsi_props) + + # Clean up - delete GSI container first, then source + self.test_db.delete_container(gsi_container.id) + self.test_db.delete_container(source_container.id) + + def test_create_gsi_container_with_dict(self): + """Test creating a GSI container using a raw dict instead of the class.""" + # Create source container + source_container = self.test_db.create_container( + id="source-dict-" + str(uuid.uuid4())[:8], + partition_key=PartitionKey(path="/id") + ) + + # Create GSI container using a dict + gsi_dict = { + "sourceCollectionId": source_container.id, + "definition": "SELECT c.id, c.category FROM c" + } + gsi_container = self.test_db.create_container( + id="gsi-dict-" + str(uuid.uuid4())[:8], + partition_key=PartitionKey(path="/id"), + global_secondary_index_definition=gsi_dict + ) + + # Verify + properties = gsi_container.read() + self.assertIn("globalSecondaryIndexDefinition", properties) + + # Clean up + self.test_db.delete_container(gsi_container.id) + self.test_db.delete_container(source_container.id) + + def test_create_gsi_container_if_not_exists(self): + """Test create_container_if_not_exists with GSI definition.""" + # Create source container + source_container = self.test_db.create_container( + id="source-notexist-" + str(uuid.uuid4())[:8], + partition_key=PartitionKey(path="/id") + ) + + gsi_definition = GlobalSecondaryIndexDefinition( + source_container_id=source_container.id, + definition="SELECT c.id, c.timestamp FROM c" + ) + container_id = "gsi-notexist-" + str(uuid.uuid4())[:8] + + # First call creates + gsi_container = self.test_db.create_container_if_not_exists( + id=container_id, + partition_key=PartitionKey(path="/id"), + global_secondary_index_definition=gsi_definition + ) + self.assertEqual(gsi_container.id, container_id) + + # Second call returns existing + gsi_container_again = self.test_db.create_container_if_not_exists( + id=container_id, + partition_key=PartitionKey(path="/id"), + global_secondary_index_definition=gsi_definition + ) + self.assertEqual(gsi_container_again.id, container_id) + + # Clean up + self.test_db.delete_container(gsi_container.id) + self.test_db.delete_container(source_container.id) + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index 2baba082bb39..d363e4ee4549 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -255,6 +255,18 @@ "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $true; defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true }" } } + }, + { + "GSITestConfig": { + "Ubuntu2404_313_gsi": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL", + "PythonVersion": "3.13", + "CoverageArg": "--disablecov", + "TestSamples": "false", + "TestMarkArgument": "cosmosGSI" + } + } } ] } diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index adfc92fd084d..e9f211c734b5 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -16,3 +16,6 @@ extends: MaxParallel: 8 BuildTargetingString: azure-cosmos ServiceDirectory: cosmos + EnvVars: + GSI_ACCOUNT_HOST: $(gsi-pipeline-uri) + GSI_ACCOUNT_KEY: $(gsi-pipeline-key)