From 8359207bdf2257740d642838367e8dd5f750ef6f Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Fri, 6 Feb 2026 15:30:28 -0800 Subject: [PATCH 01/24] Partial work for CSR over IoT Hub --- .../iot/device/iothub/abstract_clients.py | 4 + .../models/certificate_signing_request.py | 67 ++++++++ .../iot/device/iothub/pipeline/constant.py | 1 + .../device/iothub/pipeline/mqtt_pipeline.py | 40 +++++ .../iothub/pipeline/mqtt_topic_iothub.py | 62 +++++++ .../iothub/pipeline/pipeline_ops_iothub.py | 16 ++ .../pipeline/pipeline_stages_iothub_mqtt.py | 31 ++++ .../azure/iot/device/iothub/sync_clients.py | 39 +++++ samples/cert-mgmt/certificate_issuance.py | 126 +++++++++++++++ samples/cert-mgmt/certificate_management.md | 152 ++++++++++++++++++ 10 files changed, 538 insertions(+) create mode 100644 azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py create mode 100644 samples/cert-mgmt/certificate_issuance.py create mode 100644 samples/cert-mgmt/certificate_management.md diff --git a/azure-iot-device/azure/iot/device/iothub/abstract_clients.py b/azure-iot-device/azure/iot/device/iothub/abstract_clients.py index a867dd614..b607b205d 100644 --- a/azure-iot-device/azure/iot/device/iothub/abstract_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/abstract_clients.py @@ -451,6 +451,10 @@ def patch_twin_reported_properties(self, reported_properties_patch: TwinPatch) - def receive_twin_desired_properties_patch(self) -> TwinPatch: pass + @abc.abstractmethod + def send_certificate_signing_request(self, certificate_signing_request: CertificateSigningRequest) -> CertificateSigningResponse: + pass + @property def connected(self) -> bool: """ diff --git a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py new file mode 100644 index 000000000..ee9579277 --- /dev/null +++ b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py @@ -0,0 +1,67 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""This module contains a class representing messages that are sent or received. +""" +from azure.iot.device import constant +import sys + + +class CertificateSigningRequest(object): + """Represents a message to or from IoTHub + + :ivar device_id: The ID of the device associated with the certificate signing request. + :ivar csr: The base64-encoded certificate signing request. + :ivar replace: Replace any active credential operation for this device. + """ + + def __init__( + self, device_id, csr, replace=None + ): + """ + Initializer for CertificateSigningRequest + + :param str device_id: The ID of the device associated with the certificate signing request. + :param str csr: The base64-encoded certificate signing request. + :param str replace: Replace any active credential operation for this device. + """ + self.device_id = device_id + self.csr = csr + self.replace = replace + + def __str__(self): + return str(self.csr) + + def get_size(self) -> int: + total = 0 + total = total + sum( + sys.getsizeof(v) + for v in self.__dict__.values() + if v is not None + ) + return total + +class CertificateSigningResponse(object): + """Represents a message to or from IoTHub + + :ivar device_id: The ID of the device associated with the certificate signing request. + :ivar csr: The base64-encoded certificate signing request. + :ivar replace: Replace any active credential operation for this device. + """ + + def __init__( + self, correlation_id, certificates + ): + """ + Initializer for CertificateSigningRequest + + :param str device_id: The ID of the device associated with the certificate signing request. + :param str csr: The base64-encoded certificate signing request. + :param str replace: Replace any active credential operation for this device. + """ + self.correlation_id = correlation_id + self.certificates = certificates + + diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/constant.py b/azure-iot-device/azure/iot/device/iothub/pipeline/constant.py index 8cc88bb06..62d49c2d1 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/constant.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/constant.py @@ -8,6 +8,7 @@ # Feature names C2D_MSG = "c2d" +CSR = "csr" INPUT_MSG = "input" METHODS = "methods" TWIN = "twin" diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py index fdea45b99..b24239a45 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py @@ -34,6 +34,7 @@ def __init__(self, pipeline_configuration): self.feature_enabled = { constant.C2D_MSG: False, + constant.CSR: False, constant.INPUT_MSG: False, constant.METHODS: False, constant.TWIN: False, @@ -482,6 +483,45 @@ def on_complete(op, error): ) ) + def send_certificate_signing_request(self, request, callback): + """ + Send a patch for a twin's reported properties to the service. + + :param patch: the reported properties patch to send + :param callback: callback which is called when the request attempt is complete. + + :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.PipelineNotRunning` if the + pipeline has previously been shut down + + The following exceptions are not "raised", but rather returned via the "error" parameter + when invoking "callback": + + :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.NoConnectionError` + :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.ProtocolClientError` + + The following exceptions can be returned via the "error" parameter only if auto-connect + is enabled in the pipeline configuration: + + :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.ConnectionFailedError` + :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.ConnectionDroppedError` + :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.UnauthorizedError` + :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.OperationTimeout` + """ + self._verify_running() + logger.debug("Starting CertificateSigningRequestOperation on the pipeline") + + def on_complete(op, error): + if error: + callback(error=error, twin=None) + else: + callback(twin=op.twin) + + self._pipeline.run_op( + pipeline_ops_iothub.CertificateSigningRequestOperation( + request=request, callback=on_complete + ) + ) + # NOTE: Currently, this operation will retry itself indefinitely in the case of timeout def enable_feature(self, feature_name, callback): """ diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py index 9db638882..aa891e487 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py @@ -165,6 +165,68 @@ def is_method_topic(topic): return False +def get_certificate_signing_response_topic_for_subscribe(): + """ + :return: The topic for Certificate Signing Response messages. It is of the format + "$iothub/credentials/res/#" + """ + return "$iothub/credentials/res/#" + +def get_certificate_signing_request_topic_for_publish(request_id): + """ + :return: The topic for Certificate Signing Request messages. It is of the format + "" + """ + return "$iothub/credentials/POST/?$rid={request_id}&$op=issueCertificate".format( + request_id=request_id + ) + +def is_certificate_signing_response_topic(topic): + """ + Topics for certificate signing responses are of the following format: + "$iothub/credentials/res/{status}/?$rid={request_id}" + + :param str topic: The topic string. + """ + return topic.startswith("$iothub/credentials/res/") + +def get_certificate_signing_response_request_id_from_topic(topic): + """ + Extract the request id from the certificate signing response topic. + Topics for certificate signing responses are of the following format: + "$iothub/credentials/res/{status}/?$rid={request_id}" + + :param str topic: The topic string + """ + if is_certificate_signing_response_topic(topic): + parts = topic.split("/") + if len(parts) == 5: + return urllib.parse.unquote(parts[3]) + + properties = _extract_properties(topic.split("?")[1]) + return properties["rid"] + else: + raise ValueError("topic has incorrect format") + else: + raise ValueError("topic has incorrect format") + +def get_certificate_signing_response_status_from_topic(topic): + """ + Extract the status from the certificate signing response topic. + Topics for certificate signing responses are of the following format: + "$iothub/credentials/res/{status}/?$rid={request_id}" + + :param str topic: The topic string + """ + if is_certificate_signing_response_topic(topic): + parts = topic.split("/") + if len(parts) == 5: + return urllib.parse.unquote(parts[3]) + else: + raise ValueError("topic has incorrect format") + else: + raise ValueError("topic has incorrect format") + def is_twin_response_topic(topic): """Topics for twin responses are of the following format: $iothub/twin/res/{status}/?$rid={rid} diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py index 9556b4569..e4f4b497c 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py @@ -101,3 +101,19 @@ def __init__(self, patch, callback): """ super().__init__(callback=callback) self.patch = patch + +class CertificateSigningRequestOperation(PipelineOperation): + """ + A PipelineOperation object which contains arguments used to send a reported properties patch to the Azure + IoT Hub or Azure IoT Edge Hub service. + """ + + def __init__(self, request, callback): + """ + Initializer for CertificateSigningRequestOperation object + + :param patch: The reported properties patch to send to the service. + :type patch: dict, str, int, float, bool, or None (JSON compatible values) + """ + super().__init__(callback=callback) + self.request = request diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py index 7b9d3dea4..51bde332c 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py @@ -112,6 +112,18 @@ def _run_op(self, op): ) self.send_op_down(worker_op) + elif isinstance(op, pipeline_ops_iothub.CertificateSigningRequestOperation): + # Sending a Method Response gets translated into an MQTT Publish operation + topic = mqtt_topic_iothub.get_certificate_signing_request_topic_for_publish( + op.method_response.request_id, op.method_response.status + ) + payload = json.dumps(op.method_response.payload) + worker_op = op.spawn_worker_op( + worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, topic=topic, payload=payload + ) + self.send_op_down(worker_op) + + elif isinstance(op, pipeline_ops_base.EnableFeatureOperation): # Enabling a feature gets translated into an MQTT subscribe operation topic = self._get_feature_subscription_topic(op.feature_name) @@ -141,6 +153,16 @@ def _run_op(self, op): payload=op.request_body, ) self.send_op_down(worker_op) + elif op.request_type == pipeline_constant.CSR: + topic = mqtt_topic_iothub.get_certificate_signing_request_topic_for_publish( + request_id=op.request_id + ) + worker_op = op.spawn_worker_op( + worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, + topic=topic, + payload=op.request_body, + ) + self.send_op_down(worker_op) else: raise pipeline_exceptions.OperationError( "RequestOperation request_type {} not supported".format(op.request_type) @@ -223,6 +245,15 @@ def _handle_pipeline_event(self, event): ) ) + elif mqtt_topic_iothub.is_certificate_signing_response_topic(topic): + request_id = mqtt_topic_iothub.get_certificate_signing_response_request_id_from_topic(topic) + status_code = int(mqtt_topic_iothub.get_certificate_signing_response_status_from_topic(topic)) + self.send_event_up( + pipeline_events_base.ResponseEvent( + request_id=request_id, status_code=status_code, response_body=event.payload + ) + ) + else: logger.debug("Unknown topic: {} passing up to next handler".format(topic)) self.send_event_up(event) diff --git a/azure-iot-device/azure/iot/device/iothub/sync_clients.py b/azure-iot-device/azure/iot/device/iothub/sync_clients.py index 4088e0b6f..775d2761a 100644 --- a/azure-iot-device/azure/iot/device/iothub/sync_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/sync_clients.py @@ -535,6 +535,45 @@ def receive_twin_desired_properties_patch(self, block=True, timeout=None) -> Twi return None return patch + def send_certificate_signing_request(self, request: CertificateSigningRequest) -> CertificateSigningResponse: + """ + Update reported properties with the Azure IoT Hub or Azure IoT Edge Hub service. + + This is a synchronous call, meaning that this function will not return until the patch + has been sent to the service and acknowledged. + + If the service returns an error on the patch operation, this function will raise the + appropriate error. + + :param reported_properties_patch: Twin Reported Properties patch as a JSON dict + :type reported_properties_patch: dict + + :raises: :class:`azure.iot.device.exceptions.CredentialError` if credentials are invalid + and a connection cannot be established. + :raises: :class:`azure.iot.device.exceptions.ConnectionFailedError` if a establishing a + connection results in failure. + :raises: :class:`azure.iot.device.exceptions.ConnectionDroppedError` if connection is lost + during execution. + :raises: :class:`azure.iot.device.exceptions.OperationTimeout` if connection attempt + times out + :raises: :class:`azure.iot.device.exceptions.NoConnectionError` if the client is not + connected (and there is no auto-connect enabled) + :raises: :class:`azure.iot.device.exceptions.ClientError` if there is an unexpected failure + during execution. + """ + + if not self._mqtt_pipeline.feature_enabled[pipeline_constant.CSR]: + self._enable_feature(pipeline_constant.CSR) + + callback = EventedCallback(return_arg_name="csr") + self._mqtt_pipeline.send_certificate_signing_request( + request=request, callback=callback + ) + certificate_signing_response = handle_result(callback) + + logger.info("Received certificate signing response") + return certificate_signing_response + class IoTHubDeviceClient(GenericIoTHubClient, AbstractIoTHubDeviceClient): """A synchronous device client that connects to an Azure IoT Hub instance.""" diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py new file mode 100644 index 000000000..86a374090 --- /dev/null +++ b/samples/cert-mgmt/certificate_issuance.py @@ -0,0 +1,126 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import asyncio +from azure.iot.device.aio import ProvisioningDeviceClient +import os +from azure.iot.device.aio import IoTHubDeviceClient +from azure.iot.device import Message +import uuid +from azure.iot.device import X509 +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +messages_to_send = 10 +provisioning_host = os.getenv("PROVISIONING_HOST") +id_scope = os.getenv("PROVISIONING_IDSCOPE") +registration_id = os.getenv("PROVISIONING_REGISTRATION_ID") + +dps_x509_cert_file = os.getenv("PROVISIONING_X509_CERT_FILE") +dps_x509_key_file = os.getenv("PROVISIONING_X509_KEY_FILE") + +dps_sas_key = os.getenv("PROVISIONING_SAS_KEY") + +csr_data = os.getenv("PROVISIONING_CSR") +csr_key_file = os.getenv("PROVISIONING_CSR_KEY_FILE") +issued_cert_file = os.getenv("PROVISIONING_ISSUED_CERT_FILE") + +def x509_certificate_list_to_pem(cert_list): + begin_cert_header = "-----BEGIN CERTIFICATE-----\r\n" + end_cert_footer = "\r\n-----END CERTIFICATE-----" + separator = end_cert_footer + "\r\n" + begin_cert_header + # return begin_cert_header + separator.join(cert_list) + end_cert_footer + return begin_cert_header + cert_list[0] + end_cert_footer + +async def main(): + if dps_x509_cert_file is not None and dps_x509_key_file is not None: + print("Using x509 authentication") + dps_x509 = X509( + cert_file=dps_x509_cert_file, + key_file=dps_x509_key_file, + ) + + provisioning_device_client = ProvisioningDeviceClient.create_from_x509_certificate( + provisioning_host=provisioning_host, + registration_id=registration_id, + id_scope=id_scope, + x509=dps_x509, + ) + elif dps_sas_key is not None: + print("Using symmetric-key authentication") + provisioning_device_client = ProvisioningDeviceClient.create_from_symmetric_key( + provisioning_host=provisioning_host, + registration_id=registration_id, + id_scope=id_scope, + symmetric_key=dps_sas_key, + ) + else: + print("Either provide PROVISIONING_X509_CERT_FILE and PROVISIONING_X509_KEY_FILE or PROVISIONING_SAS_KEY") + sys.exit(1) + + # set the CSR on the client + provisioning_device_client.client_certificate_signing_request = csr_data + + registration_result = await provisioning_device_client.register() + + print("The complete registration result is") + print(registration_result.registration_state) + + with open(issued_cert_file, "w") as out_ca_pem: + # Write the issued certificate on the file. + out_ca_pem.write(x509_certificate_list_to_pem(registration_result.registration_state.issued_client_certificate)) + + if registration_result.status == "assigned": + print("Will send telemetry from the provisioned device") + + iot_hub_x509 = X509( + cert_file=issued_cert_file, + key_file=csr_key_file, + ) + + device_client = IoTHubDeviceClient.create_from_x509_certificate( + hostname=registration_result.registration_state.assigned_hub, + device_id=registration_result.registration_state.device_id, + x509=iot_hub_x509, + ) + + # Connect the client. + await device_client.connect() + + async def send_test_message(i): + print("sending message #" + str(i)) + msg = Message("test wind speed " + str(i)) + msg.message_id = uuid.uuid4() + await device_client.send_message(msg) + print("done sending message #" + str(i)) + + # send `messages_to_send` messages in parallel + await asyncio.gather(*[send_test_message(i) for i in range(1, messages_to_send + 1)]) + + # # Get new issued certificate from IoT Hub + # csr_request = CertificateSigningRequest( + # registration_id, + # csr_data, + # None + # ) + + # csr_response = await device_client.send_certificate_signing_request(csr_request) + + # finally, disconnect + await device_client.shutdown() + else: + print("Can not send telemetry from the provisioned device") + + +if __name__ == "__main__": + asyncio.run(main()) + + # If using Python 3.6 or below, use the following code instead of asyncio.run(main()): + # loop = asyncio.get_event_loop() + # loop.run_until_complete(main()) + # loop.close() diff --git a/samples/cert-mgmt/certificate_management.md b/samples/cert-mgmt/certificate_management.md new file mode 100644 index 000000000..dcb9ed218 --- /dev/null +++ b/samples/cert-mgmt/certificate_management.md @@ -0,0 +1,152 @@ +# Azure Device Provisioning Certificate Management + +Azure Device Provisioning Service is capable (through pairing with Azure Device Registry service) of issuing a certificate chain for Azure IoT Hub authentication. +This is done by sending a Certificate Signing Request when the device registration is performed against the Device Provisioning Service. + +This sample shows how to use the Azure IoT Python SDK Device Provisiong Client to perform a registration with Certificate-Signing Request and authenticate against the Azure IoT Hub with the issued certificate chain. + +# Requirements + +- An Azure Device Provisioning Service and Azure IoT Hub configured for CA-based certificate issuance and authentication. + +- Azure Device Provisioning Service with a group or individual enrollment created to support Certificate Management. + +- If using enrollment group, create the [derived certificate](https://learn.microsoft.com/en-us/azure/iot-dps/concepts-x509-attestation) or [symmetric-key](https://learn.microsoft.com/en-us/azure/iot-dps/concepts-symmetric-key-attestation?tabs=linux#group-enrollments-with-symmetric-keys) for attestation. + +# Sample Configuration + +Environment variables are used to configure the `provisioning_client_certificate_issuance.py` sample. + +This is a common set of environment variables that must be defined: + +Linux: + +```bash +export PROVISIONING_HOST="global.azure-devices-provisioning.net" # Or your specific Azure DPS service hostname. +export PROVISIONING_IDSCOPE= +export PROVISIONING_REGISTRATION_ID= # I.e., the ID of the device to be registered. +``` + +Windows: +```powershell +$env:PROVISIONING_HOST="global.azure-devices-provisioning.net" # Or your specific Azure DPS service hostname. +$env:PROVISIONING_IDSCOPE="" +$env:PROVISIONING_REGISTRATION_ID="" # I.e., the ID of the device to be registered. +``` + +If using x509-based attestation, set: + +Linux +```bash +export PROVISIONING_X509_CERT_FILE= +export PROVISIONING_X509_KEY_FILE= +``` + +Windows: +```powershell +$env:PROVISIONING_X509_CERT_FILE="" +$env:PROVISIONING_X509_KEY_FILE="" +``` + +Otherwise, if using symmetric-key attestation, set: + +Linux +```bash +export PROVISIONING_SAS_KEY="" +``` + +Windows: +```powershell +$env:PROVISIONING_SAS_KEY="" +``` + +Finally, set the variables for the certificate-signing request feature. + +Linux +```bash +export PROVISIONING_CSR_KEY_FILE= +export PROVISIONING_CSR= +export PROVISIONING_ISSUED_CERT_FILE= +``` + +Windows: +```powershell +$env:PROVISIONING_CSR_KEY_FILE="" +$env:PROVISIONING_CSR="" +$env:PROVISIONING_ISSUED_CERT_FILE="" +``` + +## Generating a Certificate Key and Certificate-Signing-Request for Testing + +The steps below can be used **for testing only**. + +**Do not use the key or certificate-sigining-request below in production.** + +Linux: +```bash` +export PROVISIONING_REGISTRATION_ID= # If not done already above. +export PROVISIONING_CSR_KEY_FILE=$(pwd)/${PROVISIONING_REGISTRATION_ID}-csr-private-key.pem + +openssl ecparam -name prime256v1 -genkey -noout | openssl pkcs8 -topk8 -nocrypt -out $PROVISIONING_CSR_KEY_FILE + +export PROVISIONING_CSR=$(openssl req -new -key $PROVISIONING_CSR_KEY_FILE -subj "/CN=$PROVISIONING_REGISTRATION_ID" -outform DER | openssl base64 -A) +``` + +Windows: +```powershell +$env:PROVISIONING_REGISTRATION_ID="" # If not done already above. +$env:PROVISIONING_CSR_KEY_FILE="$(pwd)\${env:PROVISIONING_REGISTRATION_ID}-csr-private-key.pem" + +$privateKey = [System.Security.Cryptography.ECDsa]::Create([System.Security.Cryptography.ECCurve]::CreateFromFriendlyName("nistP256")) + +if ($PSVersionTable.PSVersion.Major -lt 7) { + $base64pkcs8PrivateKey = [Convert]::ToBase64String($privateKey.Key.Export([System.Security.Cryptography.CngKeyBlobFormat]::Pkcs8PrivateBlob), 'InsertLineBreaks') +} else { + $base64pkcs8PrivateKey = [Convert]::ToBase64String($privateKey.ExportPkcs8PrivateKey(), 'InsertLineBreaks') +} + +$dn = New-Object System.Security.Cryptography.X509Certificates.X500DistinguishedName("CN=$env:PROVISIONING_REGISTRATION_ID") +$csr = New-Object System.Security.Cryptography.X509Certificates.CertificateRequest($dn, $privateKey, [System.Security.Cryptography.HashAlgorithmName]::SHA256) +$env:PROVISIONING_CSR = [Convert]::ToBase64String($csr.CreateSigningRequest()) + +echo "-----BEGIN PRIVATE KEY-----`n$base64pkcs8PrivateKey`n-----END PRIVATE KEY-----" > $env:PROVISIONING_CSR_KEY_FILE +``` + +# Running the Sample + +```bash +git clone -b feature/dps-csr-preview https://github.com/Azure/azure-iot-sdk-python +cd azure-iot-sdk-python +python3 azure-iot-device/samples/dps-cert-mgmt/provisioning_client_certificate_issuance.py +``` + +Example of sample output: +```bash +Using x509 authentication +The complete registration result is +myDeviceId +myAssignedIoTHub.azure-devices.net +reprovisionedToInitialAssignment +null +Will send telemetry from the provisioned device +sending message #1 +sending message #2 +sending message #3 +sending message #4 +sending message #5 +sending message #6 +sending message #7 +sending message #8 +sending message #9 +sending message #10 +done sending message #1 +done sending message #2 +done sending message #3 +done sending message #4 +done sending message #5 +done sending message #6 +done sending message #7 +done sending message #8 +done sending message #9 +done sending message #10 +``` \ No newline at end of file From 3c65f2691a9f4f510c1a0fbf09d7957530574584 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 6 Feb 2026 23:56:56 +0000 Subject: [PATCH 02/24] Fix documentation --- samples/cert-mgmt/certificate_management.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/cert-mgmt/certificate_management.md b/samples/cert-mgmt/certificate_management.md index dcb9ed218..846c093d2 100644 --- a/samples/cert-mgmt/certificate_management.md +++ b/samples/cert-mgmt/certificate_management.md @@ -83,7 +83,7 @@ The steps below can be used **for testing only**. **Do not use the key or certificate-sigining-request below in production.** Linux: -```bash` +```bash export PROVISIONING_REGISTRATION_ID= # If not done already above. export PROVISIONING_CSR_KEY_FILE=$(pwd)/${PROVISIONING_REGISTRATION_ID}-csr-private-key.pem @@ -149,4 +149,4 @@ done sending message #7 done sending message #8 done sending message #9 done sending message #10 -``` \ No newline at end of file +``` From f4f211582b65239eac14bc1aeec1aec1d22edc23 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 7 Feb 2026 00:00:29 +0000 Subject: [PATCH 03/24] Implement async version of send_certificate_signing_request --- .../iot/device/iothub/aio/async_clients.py | 45 ++++++++++++++++++- .../azure/iot/device/iothub/sync_clients.py | 26 +++++++---- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py index 6eaca9c1a..b10f3818b 100644 --- a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py @@ -16,7 +16,13 @@ AbstractIoTHubDeviceClient, AbstractIoTHubModuleClient, ) -from azure.iot.device.iothub.models import Message, MethodRequest, MethodResponse +from azure.iot.device.iothub.models import ( + Message, + MethodRequest, + MethodResponse, + CertificateSigningRequest, + CertificateSigningResponse, +) from azure.iot.device.iothub.pipeline import constant from azure.iot.device.iothub.pipeline import exceptions as pipeline_exceptions from azure.iot.device import exceptions @@ -520,6 +526,43 @@ async def receive_twin_desired_properties_patch(self) -> TwinPatch: logger.info("twin patch received") return patch + async def send_certificate_signing_request( + self, request: CertificateSigningRequest + ) -> CertificateSigningResponse: + """ + Gets the device or module twin from the Azure IoT Hub or Azure IoT Edge Hub service. + + :returns: Complete Twin as a JSON dict + :rtype: dict + + :raises: :class:`azure.iot.device.exceptions.CredentialError` if credentials are invalid + and a connection cannot be established. + :raises: :class:`azure.iot.device.exceptions.ConnectionFailedError` if a establishing a + connection results in failure. + :raises: :class:`azure.iot.device.exceptions.ConnectionDroppedError` if connection is lost + during execution. + :raises: :class:`azure.iot.device.exceptions.OperationTimeout` if connection attempt + times out + :raises: :class:`azure.iot.device.exceptions.NoConnectionError` if the client is not + connected (and there is no auto-connect enabled) + :raises: :class:`azure.iot.device.exceptions.ClientError` if there is an unexpected failure + during execution. + """ + logger.info("Sending certificate signing request") + + if not self._mqtt_pipeline.feature_enabled[constant.CSR]: + await self._enable_feature(constant.CSR) + + send_certificate_signing_request_async = async_adapter.emulate_async( + self._mqtt_pipeline.send_certificate_signing_request + ) + + callback = async_adapter.AwaitableCallback(return_arg_name="csr") + await send_certificate_signing_request_async(request=request, callback=callback) + certificate_signing_response = await handle_result(callback) + logger.info("Received certificate signing response") + return certificate_signing_response + class IoTHubDeviceClient(GenericIoTHubClient, AbstractIoTHubDeviceClient): """An asynchronous device client that connects to an Azure IoT Hub instance.""" diff --git a/azure-iot-device/azure/iot/device/iothub/sync_clients.py b/azure-iot-device/azure/iot/device/iothub/sync_clients.py index 775d2761a..6085e1835 100644 --- a/azure-iot-device/azure/iot/device/iothub/sync_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/sync_clients.py @@ -15,7 +15,13 @@ AbstractIoTHubDeviceClient, AbstractIoTHubModuleClient, ) -from .models import Message, MethodResponse, MethodRequest +from .models import ( + Message, + MethodResponse, + MethodRequest, + CertificateSigningRequest, + CertificateSigningResponse, +) from .inbox_manager import InboxManager from .sync_inbox import SyncClientInbox, InboxEmpty from . import sync_handler_manager @@ -379,7 +385,9 @@ def receive_method_request( self._enable_feature(pipeline_constant.METHODS) if self._inbox_manager is not None: - method_inbox : Queue[MethodRequest] = self._inbox_manager.get_method_request_inbox(method_name) + method_inbox: Queue[MethodRequest] = self._inbox_manager.get_method_request_inbox( + method_name + ) logger.info("Waiting for method request...") try: @@ -524,7 +532,7 @@ def receive_twin_desired_properties_patch(self, block=True, timeout=None) -> Twi if not self._mqtt_pipeline.feature_enabled[pipeline_constant.TWIN_PATCHES]: self._enable_feature(pipeline_constant.TWIN_PATCHES) if self._inbox_manager is not None: - twin_patch_inbox : Queue[TwinPatch] = self._inbox_manager.get_twin_patch_inbox() + twin_patch_inbox: Queue[TwinPatch] = self._inbox_manager.get_twin_patch_inbox() logger.info("Waiting for twin patches...") try: @@ -535,7 +543,9 @@ def receive_twin_desired_properties_patch(self, block=True, timeout=None) -> Twi return None return patch - def send_certificate_signing_request(self, request: CertificateSigningRequest) -> CertificateSigningResponse: + def send_certificate_signing_request( + self, request: CertificateSigningRequest + ) -> CertificateSigningResponse: """ Update reported properties with the Azure IoT Hub or Azure IoT Edge Hub service. @@ -566,9 +576,7 @@ def send_certificate_signing_request(self, request: CertificateSigningRequest) - self._enable_feature(pipeline_constant.CSR) callback = EventedCallback(return_arg_name="csr") - self._mqtt_pipeline.send_certificate_signing_request( - request=request, callback=callback - ) + self._mqtt_pipeline.send_certificate_signing_request(request=request, callback=callback) certificate_signing_response = handle_result(callback) logger.info("Received certificate signing response") @@ -611,7 +619,7 @@ def receive_message(self, block=True, timeout=None) -> Optional[Message]: if not self._mqtt_pipeline.feature_enabled[pipeline_constant.C2D_MSG]: self._enable_feature(pipeline_constant.C2D_MSG) if self._inbox_manager is not None: - c2d_inbox : Queue[Message] = self._inbox_manager.get_c2d_message_inbox() + c2d_inbox: Queue[Message] = self._inbox_manager.get_c2d_message_inbox() logger.info("Waiting for message from Hub...") try: @@ -743,7 +751,7 @@ def receive_message_on_input( if not self._mqtt_pipeline.feature_enabled[pipeline_constant.INPUT_MSG]: self._enable_feature(pipeline_constant.INPUT_MSG) if self._inbox_manager is not None: - input_inbox : Queue[Message] = self._inbox_manager.get_input_message_inbox(input_name) + input_inbox: Queue[Message] = self._inbox_manager.get_input_message_inbox(input_name) logger.info("Waiting for input message on: " + input_name + "...") try: From bbbea0c89398135d4162cb4076517dc4fd8edb3a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 7 Feb 2026 00:12:20 +0000 Subject: [PATCH 04/24] Delete CSR sample for DPS-only, under azure-iot-device --- .../dps_certificate_management.md | 152 ------------------ ...rovisioning_client_certificate_issuance.py | 116 ------------- 2 files changed, 268 deletions(-) delete mode 100644 azure-iot-device/samples/dps-cert-mgmt/dps_certificate_management.md delete mode 100644 azure-iot-device/samples/dps-cert-mgmt/provisioning_client_certificate_issuance.py diff --git a/azure-iot-device/samples/dps-cert-mgmt/dps_certificate_management.md b/azure-iot-device/samples/dps-cert-mgmt/dps_certificate_management.md deleted file mode 100644 index 889f9ac05..000000000 --- a/azure-iot-device/samples/dps-cert-mgmt/dps_certificate_management.md +++ /dev/null @@ -1,152 +0,0 @@ -# Azure Device Provisioning Certificate Management - -Azure Device Provisioning Service is capable (through pairing with Azure Device Registry service) of issuing a certificate chain for Azure IoT Hub authentication. -This is done by sending a Certificate Signing Request when the device registration is performed against the Device Provisioning Service. - -This sample shows how to use the Azure IoT Python SDK Device Provisiong Client to perform a registration with Certificate-Signing Request and authenticate against the Azure IoT Hub with the issued certificate chain. - -# Requirements - -- An Azure Device Provisioning Service and Azure IoT Hub configured for CA-based certificate issuance and authentication. - -- Azure Device Provisioning Service with a group or individual enrollment created to support Certificate Management. - -- If using enrollment group, create the [derived certificate](https://learn.microsoft.com/en-us/azure/iot-dps/concepts-x509-attestation) or [symmetric-key](https://learn.microsoft.com/en-us/azure/iot-dps/concepts-symmetric-key-attestation?tabs=linux#group-enrollments-with-symmetric-keys) for attestation. - -# Sample Configuration - -Environment variables are used to configure the `provisioning_client_certificate_issuance.py` sample. - -This is a common set of environment variables that must be defined: - -Linux: - -```bash -export PROVISIONING_HOST="global.azure-devices-provisioning.net" # Or your specific Azure DPS service hostname. -export PROVISIONING_IDSCOPE= -export PROVISIONING_REGISTRATION_ID= # I.e., the ID of the device to be registered. -``` - -Windows: -```powershell -$env:$PROVISIONING_HOST="global.azure-devices-provisioning.net" # Or your specific Azure DPS service hostname. -$env:PROVISIONING_IDSCOPE="" -$env:PROVISIONING_REGISTRATION_ID="" # I.e., the ID of the device to be registered. -``` - -If using x509-based attestation, set: - -Linux -```bash -export PROVISIONING_X509_CERT_FILE= -export PROVISIONING_X509_KEY_FILE= -``` - -Windows: -```powershell -$env:PROVISIONING_X509_CERT_FILE="" -$env:PROVISIONING_X509_KEY_FILE="" -``` - -Otherwise, if using symmetric-key attestation, set: - -Linux -```bash -export PROVISIONING_SAS_KEY="" -``` - -Windows: -```powershell -$env:PROVISIONING_SAS_KEY="" -``` - -Finally, set the variables for the certificate-signing request feature. - -Linux -```bash -export PROVISIONING_CSR_KEY_FILE= -export PROVISIONING_CSR= -export PROVISIONING_ISSUED_CERT_FILE= -``` - -Windows: -```powershell -$env:PROVISIONING_CSR_KEY_FILE="" -$env:PROVISIONING_CSR="" -$env:PROVISIONING_ISSUED_CERT_FILE="" -``` - -## Generating a Certificate Key and Certificate-Signing-Request for Testing - -The steps below can be used **for testing only**. - -**Do not use the key or certificate-sigining-request below in production.** - -Linux: -```bash` -export PROVISIONING_REGISTRATION_ID= # If not done already above. -export PROVISIONING_CSR_KEY_FILE=$(pwd)/${PROVISIONING_REGISTRATION_ID}-csr-private-key.pem - -openssl ecparam -name prime256v1 -genkey -noout | openssl pkcs8 -topk8 -nocrypt -out $PROVISIONING_CSR_KEY_FILE - -export PROVISIONING_CSR=$(openssl req -new -key $PROVISIONING_CSR_KEY_FILE -subj "/CN=$PROVISIONING_REGISTRATION_ID" -outform DER | openssl base64 -A) -``` - -Windows: -```powershell -$env:PROVISIONING_REGISTRATION_ID="" # If not done already above. -$env:PROVISIONING_CSR_KEY_FILE="$(pwd)\${env:PROVISIONING_REGISTRATION_ID}-csr-private-key.pem" - -$privateKey = [System.Security.Cryptography.ECDsa]::Create([System.Security.Cryptography.ECCurve]::CreateFromFriendlyName("nistP256")) - -if ($PSVersionTable.PSVersion.Major -lt 7) { - $base64pkcs8PrivateKey = [Convert]::ToBase64String($privateKey.Key.Export([System.Security.Cryptography.CngKeyBlobFormat]::Pkcs8PrivateBlob), 'InsertLineBreaks') -} else { - $base64pkcs8PrivateKey = [Convert]::ToBase64String($privateKey.ExportPkcs8PrivateKey(), 'InsertLineBreaks') -} - -$dn = New-Object System.Security.Cryptography.X509Certificates.X500DistinguishedName("CN=$env:PROVISIONING_REGISTRATION_ID") -$csr = New-Object System.Security.Cryptography.X509Certificates.CertificateRequest($dn, $privateKey, [System.Security.Cryptography.HashAlgorithmName]::SHA256) -$env:PROVISIONING_CSR = [Convert]::ToBase64String($csr.CreateSigningRequest()) - -echo "-----BEGIN PRIVATE KEY-----`n$base64pkcs8PrivateKey`n-----END PRIVATE KEY-----" > $env:PROVISIONING_CSR_KEY_FILE -``` - -# Running the Sample - -```bash -git clone -b feature/dps-csr-preview https://github.com/Azure/azure-iot-sdk-python -cd azure-iot-sdk-python -python3 azure-iot-device/samples/dps-cert-mgmt/provisioning_client_certificate_issuance.py -``` - -Example of sample output: -```bash -Using x509 authentication -The complete registration result is -myDeviceId -myAssignedIoTHub.azure-devices.net -reprovisionedToInitialAssignment -null -Will send telemetry from the provisioned device -sending message #1 -sending message #2 -sending message #3 -sending message #4 -sending message #5 -sending message #6 -sending message #7 -sending message #8 -sending message #9 -sending message #10 -done sending message #1 -done sending message #2 -done sending message #3 -done sending message #4 -done sending message #5 -done sending message #6 -done sending message #7 -done sending message #8 -done sending message #9 -done sending message #10 -``` \ No newline at end of file diff --git a/azure-iot-device/samples/dps-cert-mgmt/provisioning_client_certificate_issuance.py b/azure-iot-device/samples/dps-cert-mgmt/provisioning_client_certificate_issuance.py deleted file mode 100644 index 334699b3d..000000000 --- a/azure-iot-device/samples/dps-cert-mgmt/provisioning_client_certificate_issuance.py +++ /dev/null @@ -1,116 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -import asyncio -from azure.iot.device.aio import ProvisioningDeviceClient -import os -from azure.iot.device.aio import IoTHubDeviceClient -from azure.iot.device import Message -import uuid -from azure.iot.device import X509 -import logging - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -messages_to_send = 10 -provisioning_host = os.getenv("PROVISIONING_HOST") -id_scope = os.getenv("PROVISIONING_IDSCOPE") -registration_id = os.getenv("PROVISIONING_REGISTRATION_ID") - -dps_x509_cert_file = os.getenv("PROVISIONING_X509_CERT_FILE") -dps_x509_key_file = os.getenv("PROVISIONING_X509_KEY_FILE") - -dps_sas_key = os.getenv("PROVISIONING_SAS_KEY") - -csr_data = os.getenv("PROVISIONING_CSR") -csr_key_file = os.getenv("PROVISIONING_CSR_KEY_FILE") -issued_cert_file = os.getenv("PROVISIONING_ISSUED_CERT_FILE") - -def x509_certificate_list_to_pem(cert_list): - begin_cert_header = "-----BEGIN CERTIFICATE-----\r\n" - end_cert_footer = "\r\n-----END CERTIFICATE-----" - separator = end_cert_footer + "\r\n" + begin_cert_header - return begin_cert_header + separator.join(cert_list) + end_cert_footer - -async def main(): - if dps_x509_cert_file is not None and dps_x509_key_file is not None: - print("Using x509 authentication") - dps_x509 = X509( - cert_file=dps_x509_cert_file, - key_file=dps_x509_key_file, - ) - - provisioning_device_client = ProvisioningDeviceClient.create_from_x509_certificate( - provisioning_host=provisioning_host, - registration_id=registration_id, - id_scope=id_scope, - x509=dps_x509, - ) - elif dps_sas_key is not None: - print("Using symmetric-key authentication") - provisioning_device_client = ProvisioningDeviceClient.create_from_symmetric_key( - provisioning_host=provisioning_host, - registration_id=registration_id, - id_scope=id_scope, - symmetric_key=dps_sas_key, - ) - else: - print("Either provide PROVISIONING_X509_CERT_FILE and PROVISIONING_X509_KEY_FILE or PROVISIONING_SAS_KEY") - sys.exit(1) - - # set the CSR on the client - provisioning_device_client.client_certificate_signing_request = csr_data - - registration_result = await provisioning_device_client.register() - - print("The complete registration result is") - print(registration_result.registration_state) - - with open(issued_cert_file, "w") as out_ca_pem: - # Write the issued certificate on the file. - out_ca_pem.write(x509_certificate_list_to_pem(registration_result.registration_state.issued_client_certificate)) - - if registration_result.status == "assigned": - print("Will send telemetry from the provisioned device") - - iot_hub_x509 = X509( - cert_file=issued_cert_file, - key_file=csr_key_file, - ) - - device_client = IoTHubDeviceClient.create_from_x509_certificate( - hostname=registration_result.registration_state.assigned_hub, - device_id=registration_result.registration_state.device_id, - x509=iot_hub_x509, - ) - - # Connect the client. - await device_client.connect() - - async def send_test_message(i): - print("sending message #" + str(i)) - msg = Message("test wind speed " + str(i)) - msg.message_id = uuid.uuid4() - await device_client.send_message(msg) - print("done sending message #" + str(i)) - - # send `messages_to_send` messages in parallel - await asyncio.gather(*[send_test_message(i) for i in range(1, messages_to_send + 1)]) - - # finally, disconnect - await device_client.shutdown() - else: - print("Can not send telemetry from the provisioned device") - - -if __name__ == "__main__": - asyncio.run(main()) - - # If using Python 3.6 or below, use the following code instead of asyncio.run(main()): - # loop = asyncio.get_event_loop() - # loop.run_until_complete(main()) - # loop.close() From 8aab8d4065c2d976c557d487f6c891de93714655 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 7 Feb 2026 01:36:06 +0000 Subject: [PATCH 05/24] Several little fixes --- .../iot/device/iothub/models/__init__.py | 4 +++ .../models/certificate_signing_request.py | 24 +++++++-------- .../pipeline/pipeline_stages_iothub_mqtt.py | 29 ++++++++----------- samples/cert-mgmt/certificate_issuance.py | 29 ++++++++++++------- 4 files changed, 44 insertions(+), 42 deletions(-) diff --git a/azure-iot-device/azure/iot/device/iothub/models/__init__.py b/azure-iot-device/azure/iot/device/iothub/models/__init__.py index e31fb8930..0ccfb9951 100644 --- a/azure-iot-device/azure/iot/device/iothub/models/__init__.py +++ b/azure-iot-device/azure/iot/device/iothub/models/__init__.py @@ -5,3 +5,7 @@ from .message import Message # noqa: F401 from .methods import MethodRequest, MethodResponse # noqa: F401 +from .certificate_signing_request import ( # noqa: F401 + CertificateSigningRequest, + CertificateSigningResponse, +) diff --git a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py index ee9579277..503615099 100644 --- a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py +++ b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py @@ -5,7 +5,6 @@ # -------------------------------------------------------------------------- """This module contains a class representing messages that are sent or received. """ -from azure.iot.device import constant import sys @@ -17,9 +16,7 @@ class CertificateSigningRequest(object): :ivar replace: Replace any active credential operation for this device. """ - def __init__( - self, device_id, csr, replace=None - ): + def __init__(self, request_id, device_id, csr, replace=None): """ Initializer for CertificateSigningRequest @@ -27,6 +24,7 @@ def __init__( :param str csr: The base64-encoded certificate signing request. :param str replace: Replace any active credential operation for this device. """ + self.request_id = request_id self.device_id = device_id self.csr = csr self.replace = replace @@ -36,13 +34,15 @@ def __str__(self): def get_size(self) -> int: total = 0 - total = total + sum( - sys.getsizeof(v) - for v in self.__dict__.values() - if v is not None - ) + total = total + sum(sys.getsizeof(v) for v in self.__dict__.values() if v is not None) return total + def to_json(obj): + data = obj.__dict__.copy() + data.pop("request_id", None) + return data + + class CertificateSigningResponse(object): """Represents a message to or from IoTHub @@ -51,9 +51,7 @@ class CertificateSigningResponse(object): :ivar replace: Replace any active credential operation for this device. """ - def __init__( - self, correlation_id, certificates - ): + def __init__(self, correlation_id, certificates): """ Initializer for CertificateSigningRequest @@ -63,5 +61,3 @@ def __init__( """ self.correlation_id = correlation_id self.certificates = certificates - - diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py index 51bde332c..7f0f02216 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py @@ -115,15 +115,14 @@ def _run_op(self, op): elif isinstance(op, pipeline_ops_iothub.CertificateSigningRequestOperation): # Sending a Method Response gets translated into an MQTT Publish operation topic = mqtt_topic_iothub.get_certificate_signing_request_topic_for_publish( - op.method_response.request_id, op.method_response.status + request_id=op.request.request_id ) - payload = json.dumps(op.method_response.payload) + payload = json.dumps(op.request) worker_op = op.spawn_worker_op( worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, topic=topic, payload=payload ) self.send_op_down(worker_op) - elif isinstance(op, pipeline_ops_base.EnableFeatureOperation): # Enabling a feature gets translated into an MQTT subscribe operation topic = self._get_feature_subscription_topic(op.feature_name) @@ -153,16 +152,6 @@ def _run_op(self, op): payload=op.request_body, ) self.send_op_down(worker_op) - elif op.request_type == pipeline_constant.CSR: - topic = mqtt_topic_iothub.get_certificate_signing_request_topic_for_publish( - request_id=op.request_id - ) - worker_op = op.spawn_worker_op( - worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, - topic=topic, - payload=op.request_body, - ) - self.send_op_down(worker_op) else: raise pipeline_exceptions.OperationError( "RequestOperation request_type {} not supported".format(op.request_type) @@ -178,6 +167,8 @@ def _get_feature_subscription_topic(self, feature): return mqtt_topic_iothub.get_c2d_topic_for_subscribe( self.nucleus.pipeline_configuration.device_id ) + elif feature == pipeline_constant.CSR: + return mqtt_topic_iothub.get_certificate_signing_response_topic_for_subscribe() elif feature == pipeline_constant.INPUT_MSG: return mqtt_topic_iothub.get_input_topic_for_subscribe( self.nucleus.pipeline_configuration.device_id, @@ -225,7 +216,7 @@ def _handle_pipeline_event(self, event): method_received = MethodRequest( request_id=request_id, name=method_name, - payload=json.loads(event.payload.decode("utf-8") or 'null'), + payload=json.loads(event.payload.decode("utf-8") or "null"), ) self.send_event_up(pipeline_events_iothub.MethodRequestEvent(method_received)) @@ -241,13 +232,17 @@ def _handle_pipeline_event(self, event): elif mqtt_topic_iothub.is_twin_desired_property_patch_topic(topic): self.send_event_up( pipeline_events_iothub.TwinDesiredPropertiesPatchEvent( - patch=json.loads(event.payload.decode("utf-8") or 'null') + patch=json.loads(event.payload.decode("utf-8") or "null") ) ) elif mqtt_topic_iothub.is_certificate_signing_response_topic(topic): - request_id = mqtt_topic_iothub.get_certificate_signing_response_request_id_from_topic(topic) - status_code = int(mqtt_topic_iothub.get_certificate_signing_response_status_from_topic(topic)) + request_id = ( + mqtt_topic_iothub.get_certificate_signing_response_request_id_from_topic(topic) + ) + status_code = int( + mqtt_topic_iothub.get_certificate_signing_response_status_from_topic(topic) + ) self.send_event_up( pipeline_events_base.ResponseEvent( request_id=request_id, status_code=status_code, response_body=event.payload diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index 86a374090..43cf5de31 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -10,8 +10,11 @@ from azure.iot.device.aio import IoTHubDeviceClient from azure.iot.device import Message import uuid +import secrets from azure.iot.device import X509 +from azure.iot.device.iothub.models import CertificateSigningRequest import logging +import sys logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -30,13 +33,13 @@ csr_key_file = os.getenv("PROVISIONING_CSR_KEY_FILE") issued_cert_file = os.getenv("PROVISIONING_ISSUED_CERT_FILE") + def x509_certificate_list_to_pem(cert_list): begin_cert_header = "-----BEGIN CERTIFICATE-----\r\n" end_cert_footer = "\r\n-----END CERTIFICATE-----" - separator = end_cert_footer + "\r\n" + begin_cert_header - # return begin_cert_header + separator.join(cert_list) + end_cert_footer return begin_cert_header + cert_list[0] + end_cert_footer + async def main(): if dps_x509_cert_file is not None and dps_x509_key_file is not None: print("Using x509 authentication") @@ -60,7 +63,9 @@ async def main(): symmetric_key=dps_sas_key, ) else: - print("Either provide PROVISIONING_X509_CERT_FILE and PROVISIONING_X509_KEY_FILE or PROVISIONING_SAS_KEY") + print( + "Either provide PROVISIONING_X509_CERT_FILE and PROVISIONING_X509_KEY_FILE or PROVISIONING_SAS_KEY" + ) sys.exit(1) # set the CSR on the client @@ -73,7 +78,11 @@ async def main(): with open(issued_cert_file, "w") as out_ca_pem: # Write the issued certificate on the file. - out_ca_pem.write(x509_certificate_list_to_pem(registration_result.registration_state.issued_client_certificate)) + out_ca_pem.write( + x509_certificate_list_to_pem( + registration_result.registration_state.issued_client_certificate + ) + ) if registration_result.status == "assigned": print("Will send telemetry from the provisioned device") @@ -102,14 +111,12 @@ async def send_test_message(i): # send `messages_to_send` messages in parallel await asyncio.gather(*[send_test_message(i) for i in range(1, messages_to_send + 1)]) - # # Get new issued certificate from IoT Hub - # csr_request = CertificateSigningRequest( - # registration_id, - # csr_data, - # None - # ) + # Get new issued certificate from IoT Hub + csr_request_id = secrets.randbelow(1000) # This range is arbitrary. + csr_request = CertificateSigningRequest(csr_request_id, registration_id, csr_data, None) - # csr_response = await device_client.send_certificate_signing_request(csr_request) + # csr_response = + await device_client.send_certificate_signing_request(csr_request) # finally, disconnect await device_client.shutdown() From 3b89ed9a513c823a496bea1cde7fcbb93e55dbe4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 9 Feb 2026 22:49:23 +0000 Subject: [PATCH 06/24] Partial changes to implement CSR/IoT --- azure-iot-device/azure/iot/device/constant.py | 2 +- .../models/certificate_signing_request.py | 17 +++---- .../device/iothub/pipeline/mqtt_pipeline.py | 4 +- .../iothub/pipeline/mqtt_topic_iothub.py | 9 ++-- .../iothub/pipeline/pipeline_stages_iothub.py | 50 +++++++++++++++++++ .../pipeline/pipeline_stages_iothub_mqtt.py | 3 +- samples/cert-mgmt/certificate_issuance.py | 15 ++++-- 7 files changed, 77 insertions(+), 23 deletions(-) diff --git a/azure-iot-device/azure/iot/device/constant.py b/azure-iot-device/azure/iot/device/constant.py index 13c7ce5cf..58d56a75d 100644 --- a/azure-iot-device/azure/iot/device/constant.py +++ b/azure-iot-device/azure/iot/device/constant.py @@ -9,7 +9,7 @@ VERSION = "2.15.0rc1" IOTHUB_IDENTIFIER = "azure-iot-device-iothub-py" PROVISIONING_IDENTIFIER = "azure-iot-device-provisioning-py" -IOTHUB_API_VERSION = "2019-10-01" +IOTHUB_API_VERSION = "2025-08-01-preview" PROVISIONING_API_VERSION = "2025-07-01-preview" SECURITY_MESSAGE_INTERFACE_ID = "urn:azureiot:Security:SecurityAgent:1" TELEMETRY_MESSAGE_SIZE_LIMIT = 262144 diff --git a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py index 503615099..2ef32f6d5 100644 --- a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py +++ b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py @@ -5,7 +5,6 @@ # -------------------------------------------------------------------------- """This module contains a class representing messages that are sent or received. """ -import sys class CertificateSigningRequest(object): @@ -16,7 +15,7 @@ class CertificateSigningRequest(object): :ivar replace: Replace any active credential operation for this device. """ - def __init__(self, request_id, device_id, csr, replace=None): + def __init__(self, request_id, device_id, csr, replace): """ Initializer for CertificateSigningRequest @@ -25,19 +24,14 @@ def __init__(self, request_id, device_id, csr, replace=None): :param str replace: Replace any active credential operation for this device. """ self.request_id = request_id - self.device_id = device_id + self.id = device_id # TODO(ewertons): do not expose to customer, grab it from the internal state of the client. self.csr = csr self.replace = replace def __str__(self): return str(self.csr) - def get_size(self) -> int: - total = 0 - total = total + sum(sys.getsizeof(v) for v in self.__dict__.values() if v is not None) - return total - - def to_json(obj): + def to_dict(obj): data = obj.__dict__.copy() data.pop("request_id", None) return data @@ -51,7 +45,7 @@ class CertificateSigningResponse(object): :ivar replace: Replace any active credential operation for this device. """ - def __init__(self, correlation_id, certificates): + def __init__(self, request_id, status, certificates): """ Initializer for CertificateSigningRequest @@ -59,5 +53,6 @@ def __init__(self, correlation_id, certificates): :param str csr: The base64-encoded certificate signing request. :param str replace: Replace any active credential operation for this device. """ - self.correlation_id = correlation_id + self.request_id = request_id + self.status = status self.certificates = certificates diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py index b24239a45..fe353183d 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py @@ -512,9 +512,9 @@ def send_certificate_signing_request(self, request, callback): def on_complete(op, error): if error: - callback(error=error, twin=None) + callback(error=error, csr=None) else: - callback(twin=op.twin) + callback(csr=op) self._pipeline.run_op( pipeline_ops_iothub.CertificateSigningRequestOperation( diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py index aa891e487..4cd3736ca 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py @@ -172,15 +172,17 @@ def get_certificate_signing_response_topic_for_subscribe(): """ return "$iothub/credentials/res/#" + def get_certificate_signing_request_topic_for_publish(request_id): """ :return: The topic for Certificate Signing Request messages. It is of the format "" """ - return "$iothub/credentials/POST/?$rid={request_id}&$op=issueCertificate".format( + return "$iothub/credentials/POST/issueCertificate/?$rid={request_id}".format( request_id=request_id ) + def is_certificate_signing_response_topic(topic): """ Topics for certificate signing responses are of the following format: @@ -190,6 +192,7 @@ def is_certificate_signing_response_topic(topic): """ return topic.startswith("$iothub/credentials/res/") + def get_certificate_signing_response_request_id_from_topic(topic): """ Extract the request id from the certificate signing response topic. @@ -201,8 +204,6 @@ def get_certificate_signing_response_request_id_from_topic(topic): if is_certificate_signing_response_topic(topic): parts = topic.split("/") if len(parts) == 5: - return urllib.parse.unquote(parts[3]) - properties = _extract_properties(topic.split("?")[1]) return properties["rid"] else: @@ -210,6 +211,7 @@ def get_certificate_signing_response_request_id_from_topic(topic): else: raise ValueError("topic has incorrect format") + def get_certificate_signing_response_status_from_topic(topic): """ Extract the status from the certificate signing response topic. @@ -227,6 +229,7 @@ def get_certificate_signing_response_status_from_topic(topic): else: raise ValueError("topic has incorrect format") + def is_twin_response_topic(topic): """Topics for twin responses are of the following format: $iothub/twin/res/{status}/?$rid={rid} diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py index da3af4839..30090d3fc 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py @@ -208,3 +208,53 @@ def on_twin_response(op, error): else: super()._run_op(op) + + +class CertificateSigningRequestResponseStage(PipelineStage): + """ + PipelineStage which handles twin operations. In particular, it converts twin GET and PATCH + operations into RequestAndResponseOperation operations. This is done at the IoTHub level because + there is nothing protocol-specific about this code. The protocol-specific implementation + for twin requests and responses is handled inside IoTHubMQTTTranslationStage, when it converts + the RequestOperation to a protocol-specific send operation and when it converts the + protocol-specific receive event into an ResponseEvent event. + """ + + @pipeline_thread.runs_on_pipeline_thread + def _run_op(self, op): + def map_twin_error(error, twin_op): + if error: + return error + elif twin_op.status_code >= 300: + # TODO map error codes to correct exceptions + logger.info("Error {} received from twin operation".format(twin_op.status_code)) + logger.info("response body: {}".format(twin_op.response_body)) + return exceptions.ServiceError( + "twin operation returned status {}".format(twin_op.status_code) + ) + + if isinstance(op, pipeline_ops_iothub.CertificateSigningRequestOperation): + + # Alias to avoid overload within the callback below + # CT-TODO: remove the need for this with better callback semantics + op_waiting_for_response = op + + def on_twin_response(op, error): + logger.debug("{}({}): Got response for GetTwinOperation".format(self.name, op.name)) + error = map_twin_error(error=error, twin_op=op) + if not error: + op_waiting_for_response.twin = json.loads(op.response_body.decode("utf-8")) + op_waiting_for_response.complete(error=error) + + self.send_op_down( + pipeline_ops_base.RequestAndResponseOperation( + request_type=constant.CSR, + method="GET", + resource_location="/", + request_body=" ", + callback=on_twin_response, + ) + ) + + else: + super()._run_op(op) diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py index 7f0f02216..9ae79c736 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py @@ -117,7 +117,8 @@ def _run_op(self, op): topic = mqtt_topic_iothub.get_certificate_signing_request_topic_for_publish( request_id=op.request.request_id ) - payload = json.dumps(op.request) + payload = json.dumps(op.request.to_dict()) + logger.debug("CertificateSigningRequestOperation={}".format(payload)) worker_op = op.spawn_worker_op( worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, topic=topic, payload=payload ) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index 43cf5de31..f74a831f4 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -10,12 +10,17 @@ from azure.iot.device.aio import IoTHubDeviceClient from azure.iot.device import Message import uuid -import secrets from azure.iot.device import X509 from azure.iot.device.iothub.models import CertificateSigningRequest import logging import sys +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s %(thread)s %(funcName)s %(message)s", + filename="certificate_issuance.log", +) + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -112,11 +117,11 @@ async def send_test_message(i): await asyncio.gather(*[send_test_message(i) for i in range(1, messages_to_send + 1)]) # Get new issued certificate from IoT Hub - csr_request_id = secrets.randbelow(1000) # This range is arbitrary. - csr_request = CertificateSigningRequest(csr_request_id, registration_id, csr_data, None) + csr_request_id = uuid.uuid4() # This range is arbitrary. + csr_request = CertificateSigningRequest(csr_request_id, registration_id, csr_data, "*") - # csr_response = - await device_client.send_certificate_signing_request(csr_request) + csr_response = await device_client.send_certificate_signing_request(csr_request) + print("csr_response={}".format(csr_response)) # finally, disconnect await device_client.shutdown() From f13c63bbcd18ee3a30c3706466caf3515b7ba209 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 10 Feb 2026 07:30:49 +0000 Subject: [PATCH 07/24] Add stages, ops, events for CertificateSigningRequest --- .../iot/device/iothub/aio/async_clients.py | 2 +- .../models/certificate_signing_request.py | 9 +-- .../device/iothub/pipeline/mqtt_pipeline.py | 9 ++- .../iothub/pipeline/pipeline_events_iothub.py | 13 ++++ .../iothub/pipeline/pipeline_ops_iothub.py | 3 + .../iothub/pipeline/pipeline_stages_iothub.py | 76 ++++++++++++------- .../pipeline/pipeline_stages_iothub_mqtt.py | 6 +- .../azure/iot/device/iothub/sync_clients.py | 4 +- samples/cert-mgmt/certificate_issuance.py | 3 +- 9 files changed, 83 insertions(+), 42 deletions(-) diff --git a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py index b10f3818b..20155d2b5 100644 --- a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py @@ -557,7 +557,7 @@ async def send_certificate_signing_request( self._mqtt_pipeline.send_certificate_signing_request ) - callback = async_adapter.AwaitableCallback(return_arg_name="csr") + callback = async_adapter.AwaitableCallback(return_arg_name="response") await send_certificate_signing_request_async(request=request, callback=callback) certificate_signing_response = await handle_result(callback) logger.info("Received certificate signing response") diff --git a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py index 2ef32f6d5..47408a298 100644 --- a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py +++ b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py @@ -15,7 +15,7 @@ class CertificateSigningRequest(object): :ivar replace: Replace any active credential operation for this device. """ - def __init__(self, request_id, device_id, csr, replace): + def __init__(self, csr, replace): """ Initializer for CertificateSigningRequest @@ -23,8 +23,7 @@ def __init__(self, request_id, device_id, csr, replace): :param str csr: The base64-encoded certificate signing request. :param str replace: Replace any active credential operation for this device. """ - self.request_id = request_id - self.id = device_id # TODO(ewertons): do not expose to customer, grab it from the internal state of the client. + self.id = None self.csr = csr self.replace = replace @@ -33,7 +32,6 @@ def __str__(self): def to_dict(obj): data = obj.__dict__.copy() - data.pop("request_id", None) return data @@ -45,7 +43,7 @@ class CertificateSigningResponse(object): :ivar replace: Replace any active credential operation for this device. """ - def __init__(self, request_id, status, certificates): + def __init__(self, status, certificates): """ Initializer for CertificateSigningRequest @@ -53,6 +51,5 @@ def __init__(self, request_id, status, certificates): :param str csr: The base64-encoded certificate signing request. :param str replace: Replace any active credential operation for this device. """ - self.request_id = request_id self.status = status self.certificates = certificates diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py index fe353183d..6029a78ac 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py @@ -83,6 +83,11 @@ def __init__(self, pipeline_configuration): # .append_stage(pipeline_stages_base.CoordinateRequestAndResponseStage()) # + # CertificateSigningRequestResponseStage needs to be before IoTHubMQTTTranslationStage + # because that stage operates on ops that CertificateSigningRequestResponseStage produces + # + .append_stage(pipeline_stages_iothub.CertificateSigningRequestResponseStage()) + # # IoTHubMQTTTranslationStage comes here because this is the point where we can translate # all operations directly into MQTT. After this stage, only pipeline_stages_base stages # are allowed because IoTHubMQTTTranslationStage removes all the IoTHub-ness from the ops @@ -512,9 +517,9 @@ def send_certificate_signing_request(self, request, callback): def on_complete(op, error): if error: - callback(error=error, csr=None) + callback(error=error, response=None) else: - callback(csr=op) + callback(response=op.response) self._pipeline.run_op( pipeline_ops_iothub.CertificateSigningRequestOperation( diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py index 9eb847e64..c9673d7bf 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py @@ -58,3 +58,16 @@ class TwinDesiredPropertiesPatchEvent(PipelineEvent): def __init__(self, patch): super().__init__() self.patch = patch + + +class CertificateSigningResponseEvent(PipelineEvent): + """ + A PipelineEvent object which represents an incoming twin desired properties patch. This + object is probably created by some converter stage based on a protocol-specific event. + """ + + def __init__(self, request_id, status_code, payload): + super().__init__() + self.request_id = request_id + self.status_code = status_code + self.payload = payload diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py index e4f4b497c..630040fc8 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py @@ -102,6 +102,7 @@ def __init__(self, patch, callback): super().__init__(callback=callback) self.patch = patch + class CertificateSigningRequestOperation(PipelineOperation): """ A PipelineOperation object which contains arguments used to send a reported properties patch to the Azure @@ -117,3 +118,5 @@ def __init__(self, request, callback): """ super().__init__(callback=callback) self.request = request + self.request_id = None + self.response = None diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py index 30090d3fc..9adb6c4d8 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py @@ -6,6 +6,7 @@ import json import logging +import uuid from azure.iot.device.common.pipeline import ( pipeline_events_base, pipeline_ops_base, @@ -13,6 +14,7 @@ pipeline_thread, ) from azure.iot.device import exceptions +from azure.iot.device.iothub.models import CertificateSigningResponse from . import pipeline_events_iothub, pipeline_ops_iothub from . import constant @@ -220,41 +222,61 @@ class CertificateSigningRequestResponseStage(PipelineStage): protocol-specific receive event into an ResponseEvent event. """ + def __init__(self): + super().__init__() + self.pending_responses = {} + @pipeline_thread.runs_on_pipeline_thread def _run_op(self, op): - def map_twin_error(error, twin_op): - if error: - return error - elif twin_op.status_code >= 300: - # TODO map error codes to correct exceptions - logger.info("Error {} received from twin operation".format(twin_op.status_code)) - logger.info("response body: {}".format(twin_op.response_body)) - return exceptions.ServiceError( - "twin operation returned status {}".format(twin_op.status_code) + if isinstance(op, pipeline_ops_iothub.CertificateSigningRequestOperation): + + op.request.id = self.nucleus.pipeline_configuration.device_id + op.request_id = str(uuid.uuid4()) + + logger.debug( + "{}({}): adding certificate signing request {} for {} to pending list".format( + self.name, op.name, op.request_id, op.request.id ) + ) + self.pending_responses[op.request_id] = op - if isinstance(op, pipeline_ops_iothub.CertificateSigningRequestOperation): + self.send_op_down(op) - # Alias to avoid overload within the callback below - # CT-TODO: remove the need for this with better callback semantics - op_waiting_for_response = op + else: + super()._run_op(op) - def on_twin_response(op, error): - logger.debug("{}({}): Got response for GetTwinOperation".format(self.name, op.name)) - error = map_twin_error(error=error, twin_op=op) - if not error: - op_waiting_for_response.twin = json.loads(op.response_body.decode("utf-8")) - op_waiting_for_response.complete(error=error) + @pipeline_thread.runs_on_pipeline_thread + def _handle_pipeline_event(self, event): + if isinstance(event, pipeline_events_iothub.CertificateSigningResponseEvent): - self.send_op_down( - pipeline_ops_base.RequestAndResponseOperation( - request_type=constant.CSR, - method="GET", - resource_location="/", - request_body=" ", - callback=on_twin_response, + logger.debug( + "{}: Handling certificate signing response event with request_id {}".format( + self.name, event.request_id ) ) + if event.request_id in self.pending_responses: + op = self.pending_responses[event.request_id] + del self.pending_responses[event.request_id] + + logger.debug( + "{}({}): Completing {} request with status {}".format( + self.name, + op.name, + event.request_id, + event.status_code, + ) + ) + + op.response = CertificateSigningResponse(event.status_code, event.payload) + + op.complete() + else: + logger.info( + "{}({}): request_id {} not found in pending list. Nothing to do. Dropping".format( + self.name, event.name, event.request_id + ) + ) + else: - super()._run_op(op) + self.send_event_up(event) diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py index 9ae79c736..abca40789 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py @@ -115,7 +115,7 @@ def _run_op(self, op): elif isinstance(op, pipeline_ops_iothub.CertificateSigningRequestOperation): # Sending a Method Response gets translated into an MQTT Publish operation topic = mqtt_topic_iothub.get_certificate_signing_request_topic_for_publish( - request_id=op.request.request_id + request_id=op.request_id ) payload = json.dumps(op.request.to_dict()) logger.debug("CertificateSigningRequestOperation={}".format(payload)) @@ -245,8 +245,8 @@ def _handle_pipeline_event(self, event): mqtt_topic_iothub.get_certificate_signing_response_status_from_topic(topic) ) self.send_event_up( - pipeline_events_base.ResponseEvent( - request_id=request_id, status_code=status_code, response_body=event.payload + pipeline_events_iothub.CertificateSigningResponseEvent( + request_id=request_id, status_code=status_code, payload=event.payload ) ) diff --git a/azure-iot-device/azure/iot/device/iothub/sync_clients.py b/azure-iot-device/azure/iot/device/iothub/sync_clients.py index 6085e1835..e7a43c79f 100644 --- a/azure-iot-device/azure/iot/device/iothub/sync_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/sync_clients.py @@ -575,7 +575,9 @@ def send_certificate_signing_request( if not self._mqtt_pipeline.feature_enabled[pipeline_constant.CSR]: self._enable_feature(pipeline_constant.CSR) - callback = EventedCallback(return_arg_name="csr") + request.id = None + + callback = EventedCallback(return_arg_name="response") self._mqtt_pipeline.send_certificate_signing_request(request=request, callback=callback) certificate_signing_response = handle_result(callback) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index f74a831f4..04b3349a6 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -117,8 +117,7 @@ async def send_test_message(i): await asyncio.gather(*[send_test_message(i) for i in range(1, messages_to_send + 1)]) # Get new issued certificate from IoT Hub - csr_request_id = uuid.uuid4() # This range is arbitrary. - csr_request = CertificateSigningRequest(csr_request_id, registration_id, csr_data, "*") + csr_request = CertificateSigningRequest(csr_data, "*") csr_response = await device_client.send_certificate_signing_request(csr_request) print("csr_response={}".format(csr_response)) From 8cdb873cb8af24c8c2a5e21409401648f440690f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 10 Feb 2026 09:41:40 +0000 Subject: [PATCH 08/24] Complete CSR functionality --- .../models/certificate_signing_request.py | 4 +-- .../iothub/pipeline/pipeline_stages_iothub.py | 33 +++++++++++++------ .../pipeline/pipeline_stages_iothub_mqtt.py | 20 +++++++++-- samples/cert-mgmt/certificate_issuance.py | 10 ++++-- 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py index 47408a298..9b4adfaad 100644 --- a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py +++ b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py @@ -43,7 +43,7 @@ class CertificateSigningResponse(object): :ivar replace: Replace any active credential operation for this device. """ - def __init__(self, status, certificates): + def __init__(self, status_code, certificates): """ Initializer for CertificateSigningRequest @@ -51,5 +51,5 @@ def __init__(self, status, certificates): :param str csr: The base64-encoded certificate signing request. :param str replace: Replace any active credential operation for this device. """ - self.status = status + self.status_code = status_code self.certificates = certificates diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py index 9adb6c4d8..ed684ef0b 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py @@ -257,20 +257,33 @@ def _handle_pipeline_event(self, event): if event.request_id in self.pending_responses: op = self.pending_responses[event.request_id] - del self.pending_responses[event.request_id] - logger.debug( - "{}({}): Completing {} request with status {}".format( - self.name, - op.name, - event.request_id, - event.status_code, + if event.status_code == 202: + logger.debug( + "{}: Certificate signing request {} accepted (status_code={}, payload={})".format( + self.name, event.request_id, event.status_code, event.payload + ) ) - ) - op.response = CertificateSigningResponse(event.status_code, event.payload) + else: + del self.pending_responses[event.request_id] + + logger.debug( + "{}({}): Completing {} request with status {}".format( + self.name, + op.name, + event.request_id, + event.status_code, + ) + ) + + parsed_payload = json.loads(event.payload.decode("utf-8") or "null") + + op.response = CertificateSigningResponse( + event.status_code, parsed_payload["certificates"] + ) - op.complete() + op.complete() else: logger.info( "{}({}): request_id {} not found in pending list. Nothing to do. Dropping".format( diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py index abca40789..895097074 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py @@ -118,10 +118,26 @@ def _run_op(self, op): request_id=op.request_id ) payload = json.dumps(op.request.to_dict()) - logger.debug("CertificateSigningRequestOperation={}".format(payload)) + + def on_worker_op_complete(op, error): + logger.debug( + "{}: Worker op ({}) for certificate signing request is completing (request_id={}, response={}, error={})".format( + self.name, op.name, op.parent.request_id, op.parent.response, error + ) + ) + + if error is None and op.parent.response is None: + # This is a completion for the MQTT publish, but we must wait until a final response is received + # for the Certificate Signing Request. + op.halt_completion() + worker_op = op.spawn_worker_op( - worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, topic=topic, payload=payload + worker_op_type=pipeline_ops_mqtt.MQTTPublishOperation, + topic=topic, + payload=payload, + callback=on_worker_op_complete, ) + worker_op.parent = op self.send_op_down(worker_op) elif isinstance(op, pipeline_ops_base.EnableFeatureOperation): diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index 04b3349a6..2f67510fd 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -messages_to_send = 10 +messages_to_send = 3 provisioning_host = os.getenv("PROVISIONING_HOST") id_scope = os.getenv("PROVISIONING_IDSCOPE") registration_id = os.getenv("PROVISIONING_REGISTRATION_ID") @@ -120,9 +120,13 @@ async def send_test_message(i): csr_request = CertificateSigningRequest(csr_data, "*") csr_response = await device_client.send_certificate_signing_request(csr_request) - print("csr_response={}".format(csr_response)) + print( + "csr_response=[status={}, certificates={}]".format( + csr_response.status_code, csr_response.certificates + ) + ) - # finally, disconnect + # Now, disconnect and reconnect with the new issued certificate await device_client.shutdown() else: print("Can not send telemetry from the provisioned device") From c5876b6cb8bb98bdb2cd10e5305f81bb6cf1da95 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 10 Feb 2026 21:58:49 +0000 Subject: [PATCH 09/24] Update code documentation for CSR --- .../iot/device/iothub/aio/async_clients.py | 6 +++--- .../models/certificate_signing_request.py | 20 ++++++++----------- .../device/iothub/pipeline/mqtt_pipeline.py | 4 ++-- .../iothub/pipeline/mqtt_topic_iothub.py | 8 ++++---- .../iothub/pipeline/pipeline_events_iothub.py | 4 ++-- .../iothub/pipeline/pipeline_ops_iothub.py | 12 +++++++---- .../iothub/pipeline/pipeline_stages_iothub.py | 18 ++++++++++------- .../pipeline/pipeline_stages_iothub_mqtt.py | 10 ++++++---- .../azure/iot/device/iothub/sync_clients.py | 14 ++++++------- 9 files changed, 51 insertions(+), 45 deletions(-) diff --git a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py index 20155d2b5..9ae61262a 100644 --- a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py @@ -530,10 +530,10 @@ async def send_certificate_signing_request( self, request: CertificateSigningRequest ) -> CertificateSigningResponse: """ - Gets the device or module twin from the Azure IoT Hub or Azure IoT Edge Hub service. + Sends a Certificate Signing Request to Azure IoT Hub. - :returns: Complete Twin as a JSON dict - :rtype: dict + :returns: The certificate issued by Azure IoT Hub for the certificate signing request provided. + :rtype: CertificateSigningResponse :raises: :class:`azure.iot.device.exceptions.CredentialError` if credentials are invalid and a connection cannot be established. diff --git a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py index 9b4adfaad..a17170e93 100644 --- a/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py +++ b/azure-iot-device/azure/iot/device/iothub/models/certificate_signing_request.py @@ -8,9 +8,8 @@ class CertificateSigningRequest(object): - """Represents a message to or from IoTHub + """Represents a Certificate Signing Request message to Azure IoT Hub - :ivar device_id: The ID of the device associated with the certificate signing request. :ivar csr: The base64-encoded certificate signing request. :ivar replace: Replace any active credential operation for this device. """ @@ -19,11 +18,10 @@ def __init__(self, csr, replace): """ Initializer for CertificateSigningRequest - :param str device_id: The ID of the device associated with the certificate signing request. :param str csr: The base64-encoded certificate signing request. :param str replace: Replace any active credential operation for this device. """ - self.id = None + self.id = None # The device id, filled internally by the pipeline configuration. self.csr = csr self.replace = replace @@ -36,20 +34,18 @@ def to_dict(obj): class CertificateSigningResponse(object): - """Represents a message to or from IoTHub + """Represents a Certificate Signing Response message from Azure IoT Hub - :ivar device_id: The ID of the device associated with the certificate signing request. - :ivar csr: The base64-encoded certificate signing request. - :ivar replace: Replace any active credential operation for this device. + :ivar status_code: The result code for the certificate signing request. + :ivar certificates: An array with the base64-encoded issued certificate chain (leaf, intermediate and root, in this order). """ def __init__(self, status_code, certificates): """ - Initializer for CertificateSigningRequest + Initializer for CertificateSigningResponse - :param str device_id: The ID of the device associated with the certificate signing request. - :param str csr: The base64-encoded certificate signing request. - :param str replace: Replace any active credential operation for this device. + :param status_code: The result code for the certificate signing request. + :param certificates: An array with the base64-encoded issued certificate chain (leaf, intermediate and root, in this order). """ self.status_code = status_code self.certificates = certificates diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py index 6029a78ac..19b97c191 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_pipeline.py @@ -490,9 +490,9 @@ def on_complete(op, error): def send_certificate_signing_request(self, request, callback): """ - Send a patch for a twin's reported properties to the service. + Send a certificate signing request to the service. - :param patch: the reported properties patch to send + :param request: the certificate signing request to send :param callback: callback which is called when the request attempt is complete. :raises: :class:`azure.iot.device.iothub.pipeline.exceptions.PipelineNotRunning` if the diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py index 4cd3736ca..f95a42be3 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/mqtt_topic_iothub.py @@ -176,7 +176,7 @@ def get_certificate_signing_response_topic_for_subscribe(): def get_certificate_signing_request_topic_for_publish(request_id): """ :return: The topic for Certificate Signing Request messages. It is of the format - "" + "$iothub/credentials/POST/issueCertificate/?$rid={request_id}" """ return "$iothub/credentials/POST/issueCertificate/?$rid={request_id}".format( request_id=request_id @@ -212,11 +212,11 @@ def get_certificate_signing_response_request_id_from_topic(topic): raise ValueError("topic has incorrect format") -def get_certificate_signing_response_status_from_topic(topic): +def get_certificate_signing_response_status_code_from_topic(topic): """ - Extract the status from the certificate signing response topic. + Extract the status-code from the certificate signing response topic. Topics for certificate signing responses are of the following format: - "$iothub/credentials/res/{status}/?$rid={request_id}" + "$iothub/credentials/res/{status-code}/?$rid={request_id}" :param str topic: The topic string """ diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py index c9673d7bf..706ab5a24 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_events_iothub.py @@ -62,8 +62,8 @@ def __init__(self, patch): class CertificateSigningResponseEvent(PipelineEvent): """ - A PipelineEvent object which represents an incoming twin desired properties patch. This - object is probably created by some converter stage based on a protocol-specific event. + A PipelineEvent object which represents an incoming response for a certificate signing request. This + object is created by a Azure IoT Hub converter stage based on a protocol-specific event. """ def __init__(self, request_id, status_code, payload): diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py index 630040fc8..b06b158dd 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_ops_iothub.py @@ -105,16 +105,20 @@ def __init__(self, patch, callback): class CertificateSigningRequestOperation(PipelineOperation): """ - A PipelineOperation object which contains arguments used to send a reported properties patch to the Azure - IoT Hub or Azure IoT Edge Hub service. + A PipelineOperation object which contains arguments used to send a certificate signing request + to the Azure IoT Hub. """ def __init__(self, request, callback): """ Initializer for CertificateSigningRequestOperation object - :param patch: The reported properties patch to send to the service. - :type patch: dict, str, int, float, bool, or None (JSON compatible values) + :param request: The certificate signing request to send to the service. User-provided. + :type request: CertificateSigningRequest + :param request_id: The id of the certificate signing request to send to the service. Internally-generated. + :type request_id: str + :param response: The certificate signing response received by the service. Generated by this SDK. + :type response: CertificateSigningResponse """ super().__init__(callback=callback) self.request = request diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py index ed684ef0b..4352776ca 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub.py @@ -214,12 +214,16 @@ def on_twin_response(op, error): class CertificateSigningRequestResponseStage(PipelineStage): """ - PipelineStage which handles twin operations. In particular, it converts twin GET and PATCH - operations into RequestAndResponseOperation operations. This is done at the IoTHub level because - there is nothing protocol-specific about this code. The protocol-specific implementation - for twin requests and responses is handled inside IoTHubMQTTTranslationStage, when it converts - the RequestOperation to a protocol-specific send operation and when it converts the - protocol-specific receive event into an ResponseEvent event. + PipelineStage which handles CertificateSigningRequestOperations. + More specifically, it + - generates a request id for it and + - queues it for later correlation with the incoming response for callback invokation, as well as + - properly handle any intermediary responses (202). + This is done at the IoTHub level because there is nothing protocol-specific about this code. + The protocol-specific implementation for certificate signing requests and responses is handled inside + IoTHubMQTTTranslationStage, when it converts the CertificateSigningRequestOperation to a protocol-specific + send operation and when it converts the protocol-specific receive event into individual properties + that are stored back into the CertificateSigningRequestOperation instance. """ def __init__(self): @@ -258,7 +262,7 @@ def _handle_pipeline_event(self, event): if event.request_id in self.pending_responses: op = self.pending_responses[event.request_id] - if event.status_code == 202: + if event.status_code == 202: # Meaning: request accepted. logger.debug( "{}: Certificate signing request {} accepted (status_code={}, payload={})".format( self.name, event.request_id, event.status_code, event.payload diff --git a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py index 895097074..29d49f1f8 100644 --- a/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py +++ b/azure-iot-device/azure/iot/device/iothub/pipeline/pipeline_stages_iothub_mqtt.py @@ -113,7 +113,7 @@ def _run_op(self, op): self.send_op_down(worker_op) elif isinstance(op, pipeline_ops_iothub.CertificateSigningRequestOperation): - # Sending a Method Response gets translated into an MQTT Publish operation + # Sending a Certificate Signing Request gets translated into an MQTT Publish operation topic = mqtt_topic_iothub.get_certificate_signing_request_topic_for_publish( request_id=op.request_id ) @@ -127,8 +127,10 @@ def on_worker_op_complete(op, error): ) if error is None and op.parent.response is None: - # This is a completion for the MQTT publish, but we must wait until a final response is received - # for the Certificate Signing Request. + # This is a completion for the MQTT publish, which would also complete the CertificateSigningRequestOperation. + # However this was triggered by a provisional Certificate Signing response (202 Accepted). + # We must cancel this completion and wait until a final response is received with the actual + # newly-issued certificate. op.halt_completion() worker_op = op.spawn_worker_op( @@ -258,7 +260,7 @@ def _handle_pipeline_event(self, event): mqtt_topic_iothub.get_certificate_signing_response_request_id_from_topic(topic) ) status_code = int( - mqtt_topic_iothub.get_certificate_signing_response_status_from_topic(topic) + mqtt_topic_iothub.get_certificate_signing_response_status_code_from_topic(topic) ) self.send_event_up( pipeline_events_iothub.CertificateSigningResponseEvent( diff --git a/azure-iot-device/azure/iot/device/iothub/sync_clients.py b/azure-iot-device/azure/iot/device/iothub/sync_clients.py index e7a43c79f..10d08263c 100644 --- a/azure-iot-device/azure/iot/device/iothub/sync_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/sync_clients.py @@ -547,16 +547,16 @@ def send_certificate_signing_request( self, request: CertificateSigningRequest ) -> CertificateSigningResponse: """ - Update reported properties with the Azure IoT Hub or Azure IoT Edge Hub service. + Sends a certificate signing request to Azure IoT Hub service. - This is a synchronous call, meaning that this function will not return until the patch - has been sent to the service and acknowledged. + This is a synchronous call, meaning that this function will not return until the response is + received. - If the service returns an error on the patch operation, this function will raise the - appropriate error. + If the service returns an error on the certificate signing operations operation, + this function will raise the appropriate error. - :param reported_properties_patch: Twin Reported Properties patch as a JSON dict - :type reported_properties_patch: dict + :param request: Certificate signing request to be sent + :type request: CertificateSigningRequest :raises: :class:`azure.iot.device.exceptions.CredentialError` if credentials are invalid and a connection cannot be established. From 526fd2c42f7cffd0713cd443f3e645c143aca1ca Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 10 Feb 2026 22:09:24 +0000 Subject: [PATCH 10/24] Cleanup certificate_issuance.py (#1) --- samples/cert-mgmt/certificate_issuance.py | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index 2f67510fd..15b1ae86c 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -39,10 +39,15 @@ issued_cert_file = os.getenv("PROVISIONING_ISSUED_CERT_FILE") -def x509_certificate_list_to_pem(cert_list): +def x509_certificate_to_pem_format(certificate_info): begin_cert_header = "-----BEGIN CERTIFICATE-----\r\n" end_cert_footer = "\r\n-----END CERTIFICATE-----" - return begin_cert_header + cert_list[0] + end_cert_footer + return begin_cert_header + certificate_info + end_cert_footer + + +def write_certificate_data_to_pem_file(certificate_info, certificate_file_path): + with open(certificate_file_path, "w") as out_cert_pem: + out_cert_pem.write(x509_certificate_to_pem_format(certificate_info)) async def main(): @@ -78,16 +83,14 @@ async def main(): registration_result = await provisioning_device_client.register() - print("The complete registration result is") - print(registration_result.registration_state) + print("The complete registration result is {}".format(registration_result.registration_state)) - with open(issued_cert_file, "w") as out_ca_pem: - # Write the issued certificate on the file. - out_ca_pem.write( - x509_certificate_list_to_pem( - registration_result.registration_state.issued_client_certificate - ) - ) + write_certificate_data_to_pem_file( + registration_result.registration_state.issued_client_certificate[ + 0 + ], # Use only leaf-certificate. + issued_cert_file, + ) if registration_result.status == "assigned": print("Will send telemetry from the provisioned device") From 861ccd4082b4fa2dfcd214e2bc9ece88c43de8f1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 10 Feb 2026 23:22:01 +0000 Subject: [PATCH 11/24] Add reconnection to certificate_issuance.py, cleanup --- samples/cert-mgmt/certificate_issuance.py | 108 +++++++++++++++------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index 15b1ae86c..e6e77131c 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -31,12 +31,16 @@ dps_x509_cert_file = os.getenv("PROVISIONING_X509_CERT_FILE") dps_x509_key_file = os.getenv("PROVISIONING_X509_KEY_FILE") - +# Or dps_sas_key = os.getenv("PROVISIONING_SAS_KEY") -csr_data = os.getenv("PROVISIONING_CSR") -csr_key_file = os.getenv("PROVISIONING_CSR_KEY_FILE") -issued_cert_file = os.getenv("PROVISIONING_ISSUED_CERT_FILE") +dps_csr_data = os.getenv("PROVISIONING_CSR") +dps_csr_key_file = os.getenv("PROVISIONING_CSR_KEY_FILE") +dps_issued_cert_file = os.getenv("PROVISIONING_ISSUED_CERT_FILE") + +iothub_csr_data = os.getenv("IOTHUB_CSR") +iothub_csr_key_file = dps_csr_key_file # Must be the same. +iothub_issued_cert_file = os.getenv("IOTHUB_ISSUED_CERT_FILE") def x509_certificate_to_pem_format(certificate_info): @@ -50,6 +54,36 @@ def write_certificate_data_to_pem_file(certificate_info, certificate_file_path): out_cert_pem.write(x509_certificate_to_pem_format(certificate_info)) +async def connect_device_client_and_send_test_messages( + iothub_hostname, device_id, device_certificate, device_private_key +) -> IoTHubDeviceClient: + iot_hub_x509 = X509( + cert_file=device_certificate, + key_file=device_private_key, + ) + + device_client = IoTHubDeviceClient.create_from_x509_certificate( + hostname=iothub_hostname, + device_id=device_id, + x509=iot_hub_x509, + ) + + # Connect the client. + await device_client.connect() + + async def send_test_message(i): + print("sending message #" + str(i)) + msg = Message("test wind speed " + str(i)) + msg.message_id = uuid.uuid4() + await device_client.send_message(msg) + print("done sending message #" + str(i)) + + # send `messages_to_send` messages in parallel + await asyncio.gather(*[send_test_message(i) for i in range(1, messages_to_send + 1)]) + + return device_client + + async def main(): if dps_x509_cert_file is not None and dps_x509_key_file is not None: print("Using x509 authentication") @@ -79,7 +113,7 @@ async def main(): sys.exit(1) # set the CSR on the client - provisioning_device_client.client_certificate_signing_request = csr_data + provisioning_device_client.client_certificate_signing_request = dps_csr_data registration_result = await provisioning_device_client.register() @@ -89,47 +123,57 @@ async def main(): registration_result.registration_state.issued_client_certificate[ 0 ], # Use only leaf-certificate. - issued_cert_file, + dps_issued_cert_file, ) if registration_result.status == "assigned": print("Will send telemetry from the provisioned device") - iot_hub_x509 = X509( - cert_file=issued_cert_file, - key_file=csr_key_file, - ) - - device_client = IoTHubDeviceClient.create_from_x509_certificate( - hostname=registration_result.registration_state.assigned_hub, + device_client = await connect_device_client_and_send_test_messages( + iothub_hostname=registration_result.registration_state.assigned_hub, device_id=registration_result.registration_state.device_id, - x509=iot_hub_x509, + device_certificate=dps_issued_cert_file, + device_private_key=dps_csr_key_file, ) - # Connect the client. - await device_client.connect() + if ( + iothub_csr_data is not None + and iothub_csr_key_file is not None + and iothub_issued_cert_file is not None + ): - async def send_test_message(i): - print("sending message #" + str(i)) - msg = Message("test wind speed " + str(i)) - msg.message_id = uuid.uuid4() - await device_client.send_message(msg) - print("done sending message #" + str(i)) + print("Performing Azure IoT Hub certificate re-issuance") - # send `messages_to_send` messages in parallel - await asyncio.gather(*[send_test_message(i) for i in range(1, messages_to_send + 1)]) + # Get new issued certificate from IoT Hub + csr_request = CertificateSigningRequest(dps_csr_data, "*") - # Get new issued certificate from IoT Hub - csr_request = CertificateSigningRequest(csr_data, "*") + csr_response = await device_client.send_certificate_signing_request(csr_request) + print( + "IoT Hub certificate re-issuance completed. Status-code={}".format( + csr_response.status_code + ) + ) - csr_response = await device_client.send_certificate_signing_request(csr_request) - print( - "csr_response=[status={}, certificates={}]".format( - csr_response.status_code, csr_response.certificates + # Now, disconnect and reconnect with the new issued certificate + print("Reconnecting to Azure IoT Hub with re-issued certificate") + + await device_client.shutdown() + + write_certificate_data_to_pem_file( + csr_response.certificates[0], # Use only leaf-certificate. + iothub_issued_cert_file, ) - ) - # Now, disconnect and reconnect with the new issued certificate + device_client = await connect_device_client_and_send_test_messages( + iothub_hostname=registration_result.registration_state.assigned_hub, + device_id=registration_result.registration_state.device_id, + device_certificate=iothub_issued_cert_file, + device_private_key=iothub_csr_key_file, + ) + else: + print("Skipping Azure IoT Hub certificate re-issuance") + + # Finally, disconnect device client. await device_client.shutdown() else: print("Can not send telemetry from the provisioned device") From 340d97a7ff3149028360a9371435a022c0ef33e0 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 11 Feb 2026 00:20:34 +0000 Subject: [PATCH 12/24] Update CSR sample and its documentation --- samples/cert-mgmt/certificate_issuance.py | 6 +- samples/cert-mgmt/certificate_management.md | 78 ++++++++++++++------- 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index e6e77131c..664930db8 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -136,11 +136,7 @@ async def main(): device_private_key=dps_csr_key_file, ) - if ( - iothub_csr_data is not None - and iothub_csr_key_file is not None - and iothub_issued_cert_file is not None - ): + if iothub_csr_data is not None and iothub_issued_cert_file is not None: print("Performing Azure IoT Hub certificate re-issuance") diff --git a/samples/cert-mgmt/certificate_management.md b/samples/cert-mgmt/certificate_management.md index 846c093d2..ad96c8aaf 100644 --- a/samples/cert-mgmt/certificate_management.md +++ b/samples/cert-mgmt/certificate_management.md @@ -1,9 +1,13 @@ -# Azure Device Provisioning Certificate Management +# Azure Device Provisioning and IoT Hub Certificate Management Azure Device Provisioning Service is capable (through pairing with Azure Device Registry service) of issuing a certificate chain for Azure IoT Hub authentication. This is done by sending a Certificate Signing Request when the device registration is performed against the Device Provisioning Service. -This sample shows how to use the Azure IoT Python SDK Device Provisiong Client to perform a registration with Certificate-Signing Request and authenticate against the Azure IoT Hub with the issued certificate chain. +Azure IoT Hub is also capable of re-issuing a device certificate for a currently-connected device client, which can be used for later authentication to the Azure IoT Hub service. + +This sample shows how to: +- Use the Azure IoT Python SDK Device Provisiong Client to perform a registration with Certificate-Signing Request and authenticate against the Azure IoT Hub with the issued certificate chain +- Use the Azure IoT Hub Device Client to request a new issue of a device certificate and reconnect using it. # Requirements @@ -15,7 +19,7 @@ This sample shows how to use the Azure IoT Python SDK Device Provisiong Client t # Sample Configuration -Environment variables are used to configure the `provisioning_client_certificate_issuance.py` sample. +Environment variables are used to configure the `certificate_issuance.py` sample. This is a common set of environment variables that must be defined: @@ -67,6 +71,7 @@ Linux export PROVISIONING_CSR_KEY_FILE= export PROVISIONING_CSR= export PROVISIONING_ISSUED_CERT_FILE= + ``` Windows: @@ -76,6 +81,23 @@ $env:PROVISIONING_CSR="" $env:PROVISIONING_ISSUED_CERT_FILE="" ``` +That is all needed for the device certificate issuance through the Azure Device Provisioning service. + +If you wish the sample to perform the device certificate re-issuance through the Azure IoT Hub service, also set: + +Linux +```bash +export IOTHUB_CSR= +export IOTHUB_ISSUED_CERT_FILE= + +``` + +Windows: +```powershell +$env:IOTHUB_CSR="" +$env:IOTHUB_ISSUED_CERT_FILE="" +``` + ## Generating a Certificate Key and Certificate-Signing-Request for Testing The steps below can be used **for testing only**. @@ -85,17 +107,22 @@ The steps below can be used **for testing only**. Linux: ```bash export PROVISIONING_REGISTRATION_ID= # If not done already above. -export PROVISIONING_CSR_KEY_FILE=$(pwd)/${PROVISIONING_REGISTRATION_ID}-csr-private-key.pem +export PROVISIONING_CSR_KEY_FILE=$(pwd)/${PROVISIONING_REGISTRATION_ID}-dps-csr-private-key.pem openssl ecparam -name prime256v1 -genkey -noout | openssl pkcs8 -topk8 -nocrypt -out $PROVISIONING_CSR_KEY_FILE export PROVISIONING_CSR=$(openssl req -new -key $PROVISIONING_CSR_KEY_FILE -subj "/CN=$PROVISIONING_REGISTRATION_ID" -outform DER | openssl base64 -A) + +# Ommit these next commands if you do not want to perform certificate re-issuance through Azure IoT Hub: +export IOTHUB_CSR_KEY_FILE=$PROVISIONING_CSR_KEY_FILE # This must be the same. Made explicit here for awareness. +export DEVICE_ID=$PROVISIONING_REGISTRATION_ID # The final device id is the registration id. +export IOTHUB_CSR=$(openssl req -new -key $IOTHUB_CSR_KEY_FILE -subj "/CN=$DEVICE_ID" -outform DER | openssl base64 -A) ``` Windows: ```powershell $env:PROVISIONING_REGISTRATION_ID="" # If not done already above. -$env:PROVISIONING_CSR_KEY_FILE="$(pwd)\${env:PROVISIONING_REGISTRATION_ID}-csr-private-key.pem" +$env:PROVISIONING_CSR_KEY_FILE="$(pwd)\${env:PROVISIONING_REGISTRATION_ID}-dps-csr-private-key.pem" $privateKey = [System.Security.Cryptography.ECDsa]::Create([System.Security.Cryptography.ECCurve]::CreateFromFriendlyName("nistP256")) @@ -106,47 +133,46 @@ if ($PSVersionTable.PSVersion.Major -lt 7) { } $dn = New-Object System.Security.Cryptography.X509Certificates.X500DistinguishedName("CN=$env:PROVISIONING_REGISTRATION_ID") -$csr = New-Object System.Security.Cryptography.X509Certificates.CertificateRequest($dn, $privateKey, [System.Security.Cryptography.HashAlgorithmName]::SHA256) -$env:PROVISIONING_CSR = [Convert]::ToBase64String($csr.CreateSigningRequest()) +$dps_csr = New-Object System.Security.Cryptography.X509Certificates.CertificateRequest($dn, $privateKey, [System.Security.Cryptography.HashAlgorithmName]::SHA256) +$env:PROVISIONING_CSR = [Convert]::ToBase64String($dps_csr.CreateSigningRequest()) echo "-----BEGIN PRIVATE KEY-----`n$base64pkcs8PrivateKey`n-----END PRIVATE KEY-----" > $env:PROVISIONING_CSR_KEY_FILE + +# Ommit these next commands if you do not want to perform certificate re-issuance through Azure IoT Hub. +# Private Key and DN must be the same... +$iothub_csr = New-Object System.Security.Cryptography.X509Certificates.CertificateRequest($dn, $privateKey, [System.Security.Cryptography.HashAlgorithmName]::SHA256) +$env:IOTHUB_CSR = [Convert]::ToBase64String($iothub_csr.CreateSigningRequest()) ``` # Running the Sample ```bash -git clone -b feature/dps-csr-preview https://github.com/Azure/azure-iot-sdk-python +git clone -b feature/iot-csr-preview https://github.com/Azure/azure-iot-sdk-python cd azure-iot-sdk-python -python3 azure-iot-device/samples/dps-cert-mgmt/provisioning_client_certificate_issuance.py +python3 azure-iot-device/samples/dps-cert-mgmt/certificate_issuance.py ``` Example of sample output: ```bash Using x509 authentication -The complete registration result is -myDeviceId -myAssignedIoTHub.azure-devices.net +The complete registration result is myDeviceId +myIoTHub.azure-devices.net reprovisionedToInitialAssignment null Will send telemetry from the provisioned device sending message #1 sending message #2 sending message #3 -sending message #4 -sending message #5 -sending message #6 -sending message #7 -sending message #8 -sending message #9 -sending message #10 done sending message #1 done sending message #2 done sending message #3 -done sending message #4 -done sending message #5 -done sending message #6 -done sending message #7 -done sending message #8 -done sending message #9 -done sending message #10 +Performing Azure IoT Hub certificate re-issuance +IoT Hub certificate re-issuance completed. Status-code=200 +Reconnecting to Azure IoT Hub with re-issued certificate +sending message #1 +sending message #2 +sending message #3 +done sending message #1 +done sending message #2 +done sending message #3 ``` From e63dc1d9e411125e958bb46ec85b2a7f2414c397 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 11 Feb 2026 01:49:29 +0000 Subject: [PATCH 13/24] Fix use of iothub_csr_data variable --- samples/cert-mgmt/certificate_issuance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index 664930db8..bd0ffe478 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -141,7 +141,7 @@ async def main(): print("Performing Azure IoT Hub certificate re-issuance") # Get new issued certificate from IoT Hub - csr_request = CertificateSigningRequest(dps_csr_data, "*") + csr_request = CertificateSigningRequest(iothub_csr_data, "*") csr_response = await device_client.send_certificate_signing_request(csr_request) print( From 76cea5cbf1132de8148a09412a81628c6f8ef15e Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Thu, 5 Mar 2026 11:22:21 -0800 Subject: [PATCH 14/24] Add cert mgmt pipeline yaml --- vsts/dps-e2e-cert-mgmt.yaml | 168 ++++++++++++++++++++++++------------ 1 file changed, 112 insertions(+), 56 deletions(-) diff --git a/vsts/dps-e2e-cert-mgmt.yaml b/vsts/dps-e2e-cert-mgmt.yaml index af0fe5b47..b6ae1aa62 100644 --- a/vsts/dps-e2e-cert-mgmt.yaml +++ b/vsts/dps-e2e-cert-mgmt.yaml @@ -1,59 +1,115 @@ resources: - repo: self #Multi-configuration and multi-agent job options are not exported to YAML. Configure these options using documentation guidance: https://docs.microsoft.com/vsts/pipelines/process/phases -jobs: - -- job: 'Test' - pool: - vmImage: 'Ubuntu 20.04' - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - architecture: 'x64' - - - script: 'python scripts/env_setup.py --no_dev' - displayName: 'Prepare environment (install packages + dev dependencies + test dependencies + tools)' - - - script: 'python -m pip install cryptography' - displayName: 'Install pyca/cryptography for X509 functionality' - - - script: | - cd $(Agent.WorkFolder) - cd .. - touch .rnd - displayName: 'create RANDFILE file (needed to store seed data) separately due to openssl version issues in the pipeline' - - - script: | - cd $(Build.SourcesDirectory)/tests/e2e/provisioning_e2e - pytest --junitxml=junit/dps-cert-mgmt-e2e-test-results.xml - displayName: 'Run Specified E2E Test with env variables' - - env: - IOTHUB_CONNECTION_STRING: $(DPSCERT-MAC-IOTHUB-CONNECTION-STRING) - IOTHUB_EVENTHUB_CONNECTION_STRING: $(DPSCERT-MAC-IOTHUB-EVENTHUB-CONNECTION-STRING) - IOTHUB_CA_ROOT_CERT: $(DPSCERT-MAC-IOTHUB-CA-ROOT-CERT) - IOTHUB_CA_ROOT_CERT_KEY: $(DPSCERT-MAC-IOTHUB-CA-ROOT-CERT-KEY) - STORAGE_CONNECTION_STRING: $(DPSCERT-MAC-STORAGE-CONNECTION-STRING) - - PROVISIONING_DEVICE_ENDPOINT: $(DPSCERT-MAC-DPS-DEVICE-ENDPOINT) - PROVISIONING_SERVICE_CONNECTION_STRING: $(DPSCERT-MAC-DPS-CONNECTION-STRING) - PROVISIONING_DEVICE_IDSCOPE: $(DPSCERT-MAC-DPS-ID-SCOPE) - - PROVISIONING_ROOT_CERT: $(DPSCERT-MAC-IOT-PROVISIONING-ROOT-CERT) - PROVISIONING_ROOT_CERT_KEY: $(DPSCERT-MAC-IOT-PROVISIONING-ROOT-CERT-KEY) - PROVISIONING_ROOT_PASSWORD: $(DPSCERT-MAC-ROOT-CERT-PASSWORD) - PYTHONUNBUFFERED: True - - # Extra variable manually created - CLIENT_CERTIFICATE_AUTHORITY_NAME: $(DPSCERT-MAC-DPS-CLIENT-CERTIFICATE-AUTHORITY-NAME) - DPS_CERT_ISSUANCE_SYM_KEY_AIO: $(DPSCERT-MAC-DPS-CLIENT-CERT-ISSUE-SYM-KEY-ASYNC) - DPS_CERT_ISSUANCE_SYM_KEY_SYNC: $(DPSCERT-MAC-DPS-CLIENT-CERT-ISSUE-SYM-KEY-SYNC) - - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: always() - inputs: - testResultsFiles: '**/dps-cert-mgmt-e2e-test-*.xml' - testRunTitle: 'Publish test results for Python $(python.version)' + +stages: +- stage: setup + jobs: + - job: create_azure_resources + pool: + vmImage: 'windows-latest' + steps: + - checkout: self + submodules: false + fetchDepth: 1 + sparseCheckoutDirectories: testtools/scripts + displayName: "Checkout (scripts only)" + - task: AzureCLI@2 + inputs: + azureSubscription: 'iot hub sdk service connection' + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + Invoke-WebRequest -Uri https://raw.githubusercontent.com/Azure/azure-iot-sdk-c/refs/heads/ewertons/pipelinefixes/testtools/scripts/Azure.Iot.Sdk.Test.psm1 -OutFile ./Azure.Iot.Sdk.Test.psm1 + Import-Module ./Azure.Iot.Sdk.Test.psm1 + + $ResourceGroupName = New-AzureResourceGroupName -OutFile "test_config/azure-resource-group-name.txt" + $TestEnvInfo = New-AzIotTestEnvironment -AzureLocation $env:AZURE_LOCATION -ResourceGroup $ResourceGroupName -EnableFileUpload -IotHubX509ThumbprintDevices 1 -DpsX509IndividualEnrollments 1 + displayName: Create Azure Resources and Test Config + - publish: test_config + artifact: test_config_scripts + displayName: Publish artifact (test config scripts) + condition: always() +# - stage: build_and_tests +# jobs: +# - job: 'Test' +# variables: +# IOTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.IOTHUB_CONNECTION_STRING']] +# EVENTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.EVENTHUB_CONNECTION_STRING']] +# DPS_CONNECTION_STRING: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.DPS_CONNECTION_STRING']] +# DPS_ID_SCOPE: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.DPS_ID_SCOPE']] +# DPS_HOSTNAME: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.DPS_HOSTNAME']] +# DPS_ROOT_CA_CERT_BASE64: $[stageDependencies.setup.create_azure_resources.outputs['testCertificates.DPS_ROOT_CA_CERT_BASE64']] +# DPS_ROOT_CA_PK_BASE64: $[stageDependencies.setup.create_azure_resources.outputs['testCertificates.DPS_ROOT_CA_PK_BASE64']] +# DPS_ROOT_CA_PASSWORD: $[stageDependencies.setup.create_azure_resources.outputs['testCertificates.DPS_ROOT_CA_PASSWORD']] +# pool: +# vmImage: 'ubuntu-22.04' + +# steps: +# - task: UsePythonVersion@0 +# inputs: +# versionSpec: '3.9' +# architecture: 'x64' + +# - script: 'python scripts/env_setup.py --no_dev' +# displayName: 'Prepare environment (install packages + dev dependencies + test dependencies + tools)' + +# - script: 'python -m pip install cryptography' +# displayName: 'Install pyca/cryptography for X509 functionality' + +# - script: | +# cd $(Agent.WorkFolder) +# cd .. +# touch .rnd +# displayName: 'create RANDFILE file (needed to store seed data) separately due to openssl version issues in the pipeline' + +# - script: | +# cd $(Build.SourcesDirectory)/tests/e2e/provisioning_e2e +# pytest --junitxml=junit/dps-e2e-test-results.xml +# displayName: 'Run Specified E2E Test with env variables' + +# env: +# IOTHUB_CONNECTION_STRING: $(IOTHUB_CONNECTION_STRING) +# IOTHUB_EVENTHUB_CONNECTION_STRING: $(EVENTHUB_CONNECTION_STRING) + +# PROVISIONING_DEVICE_ENDPOINT: $(DPS_HOSTNAME) +# PROVISIONING_SERVICE_CONNECTION_STRING: $(DPS_CONNECTION_STRING) +# PROVISIONING_DEVICE_IDSCOPE: $(DPS_ID_SCOPE) + +# PROVISIONING_ROOT_CERT: $(DPS_ROOT_CA_CERT_BASE64) +# PROVISIONING_ROOT_CERT_KEY: $(DPS_ROOT_CA_PK_BASE64) +# PROVISIONING_ROOT_PASSWORD: $(DPS_ROOT_CA_PASSWORD) +# PYTHONUNBUFFERED: True + +# - task: PublishTestResults@2 +# displayName: 'Publish Test Results' +# condition: always() +# inputs: +# testResultsFiles: '**/dps-e2e-test-*.xml' +# testRunTitle: 'Publish test results for Python $(python.version)' +- stage: cleanup + dependsOn: + - setup + # - build_and_tests + condition: always() # Run stage even if the pipeline run is cancelled. + jobs: + - job: destroy_azure_resource_group + condition: always() # Run job even if the pipeline run is cancelled. + pool: + vmImage: 'ubuntu-24.04' + steps: + - checkout: none + - download: current + artifact: test_config_scripts + displayName: Download artifact (test config scripts) + - task: AzureCLI@2 + inputs: + azureSubscription: 'iot hub sdk service connection' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -e + export AZURE_RESOURCE_GROUP=$(cat "$(Pipeline.Workspace)/test_config_scripts/azure-resource-group-name.txt") + az group delete --name $AZURE_RESOURCE_GROUP --yes --no-wait + displayName: Destroy Azure Resource Group + condition: always() # Run step even if the pipeline run is cancelled. \ No newline at end of file From 401577b563041a868f654a8913b967059b65bc1c Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Fri, 6 Mar 2026 14:29:10 -0800 Subject: [PATCH 15/24] Rename certificate_management.md -> readme.md --- samples/cert-mgmt/{certificate_management.md => readme.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename samples/cert-mgmt/{certificate_management.md => readme.md} (100%) diff --git a/samples/cert-mgmt/certificate_management.md b/samples/cert-mgmt/readme.md similarity index 100% rename from samples/cert-mgmt/certificate_management.md rename to samples/cert-mgmt/readme.md From 07179fab32ed0b400b59f85d1172df92b0bb1bdc Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Fri, 6 Mar 2026 22:02:23 -0800 Subject: [PATCH 16/24] Address CR comments (change env var name) --- samples/cert-mgmt/certificate_issuance.py | 4 ++-- samples/cert-mgmt/readme.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index bd0ffe478..33914e03a 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -32,7 +32,7 @@ dps_x509_cert_file = os.getenv("PROVISIONING_X509_CERT_FILE") dps_x509_key_file = os.getenv("PROVISIONING_X509_KEY_FILE") # Or -dps_sas_key = os.getenv("PROVISIONING_SAS_KEY") +dps_sas_key = os.getenv("PROVISIONING_SYMMETRIC_KEY") dps_csr_data = os.getenv("PROVISIONING_CSR") dps_csr_key_file = os.getenv("PROVISIONING_CSR_KEY_FILE") @@ -108,7 +108,7 @@ async def main(): ) else: print( - "Either provide PROVISIONING_X509_CERT_FILE and PROVISIONING_X509_KEY_FILE or PROVISIONING_SAS_KEY" + "Either provide PROVISIONING_X509_CERT_FILE and PROVISIONING_X509_KEY_FILE or PROVISIONING_SYMMETRIC_KEY" ) sys.exit(1) diff --git a/samples/cert-mgmt/readme.md b/samples/cert-mgmt/readme.md index ad96c8aaf..46bdd1325 100644 --- a/samples/cert-mgmt/readme.md +++ b/samples/cert-mgmt/readme.md @@ -56,12 +56,12 @@ Otherwise, if using symmetric-key attestation, set: Linux ```bash -export PROVISIONING_SAS_KEY="" +export PROVISIONING_SYMMETRIC_KEY="" ``` Windows: ```powershell -$env:PROVISIONING_SAS_KEY="" +$env:PROVISIONING_SYMMETRIC_KEY="" ``` Finally, set the variables for the certificate-signing request feature. From 7b7b5f3c80858e7537873bdd00637546227765dd Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Fri, 6 Mar 2026 22:06:03 -0800 Subject: [PATCH 17/24] Add credentialPolicyName on enrollment creation requests --- .../protocol/models/enrollment_group.py | 2 ++ .../protocol/models/individual_enrollment.py | 2 ++ .../tests/test_async_dps_cert_mgmt.py | 27 ++++++++++++++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py b/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py index 65b7c7020..03d623691 100644 --- a/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py +++ b/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py @@ -88,6 +88,7 @@ class EnrollmentGroup(Model): "key": "clientCertificateIssuancePolicy", "type": "ClientCertificateIssuancePolicy", }, + "credential_policy_name": {"key": "credentialPolicyName", "type": "str"}, } def __init__(self, **kwargs): @@ -107,3 +108,4 @@ def __init__(self, **kwargs): self.client_certificate_issuance_policy = kwargs.get( "client_certificate_issuance_policy", None ) + seltf.credential_policy_name = kwargs.get("credential_policy_name", None) diff --git a/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py b/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py index 0b9c78f11..a343335cc 100644 --- a/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py +++ b/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py @@ -99,6 +99,7 @@ class IndividualEnrollment(Model): "key": "clientCertificateIssuancePolicy", "type": "ClientCertificateIssuancePolicy", }, + "credential_policy_name": {"key": "credentialPolicyName", "type": "str"}, } def __init__(self, **kwargs): @@ -121,3 +122,4 @@ def __init__(self, **kwargs): self.client_certificate_issuance_policy = kwargs.get( "client_certificate_issuance_policy", None ) + self.credential_policy_name = kwargs.get("credential_policy_name", None) diff --git a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py index e39b26fc0..c76bafe95 100644 --- a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py +++ b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py @@ -46,6 +46,7 @@ PROVISIONING_HOST = os.getenv("PROVISIONING_DEVICE_ENDPOINT") ID_SCOPE = os.getenv("PROVISIONING_DEVICE_IDSCOPE") CLIENT_CERT_AUTH_NAME = os.getenv("CLIENT_CERTIFICATE_AUTHORITY_NAME") +ADR_CERT_MGMT_POLICY_NAME = os.getenv("ADR_CERT_MGMT_POLICY_NAME") type_to_device_indices = { "individual_with_device_id": [1], @@ -95,6 +96,7 @@ async def test_device_register_with_client_cert_issuance_for_a_symmetric_key_ind registration_id=registration_id, attestation_mechanism=attestation_mechanism, client_ca_name=CLIENT_CERT_AUTH_NAME, + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) symmetric_key = individual_enrollment_record.attestation.symmetric_key.primary_key private_key = create_private_key(key_file) @@ -132,7 +134,11 @@ async def test_device_register_with_client_cert_issuance_for_a_symmetric_key_gro common_device_id = "e2edpsgroupsymmetric" try: attestation_mechanism = models.AttestationMechanism(type="symmetricKey") - eg = create_enrollment_group(group_id=group_id, attestation_mechanism=attestation_mechanism) + eg = create_enrollment_group( + group_id=group_id, + attestation_mechanism=attestation_mechanism, + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, + ) master_key = eg.attestation.symmetric_key.primary_key count = 0 for index in devices_indices: @@ -187,6 +193,7 @@ async def test_device_register_with_device_id_for_a_x509_individual_enrollment(p attestation_mechanism=attestation_mechanism, device_id=device_id, client_ca_name=CLIENT_CERT_AUTH_NAME, + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) registration_id = individual_enrollment_record.registration_id @@ -233,6 +240,7 @@ async def test_device_register_with_no_device_id_for_a_x509_individual_enrollmen registration_id=registration_id, attestation_mechanism=attestation_mechanism, client_ca_name=CLIENT_CERT_AUTH_NAME, + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) registration_id = individual_enrollment_record.registration_id @@ -288,7 +296,11 @@ async def test_group_of_devices_register_with_no_device_id_for_a_x509_intermedia primary_cert=intermediate_cert_content, ) attestation_mechanism = models.AttestationMechanism(type="x509", x509=x509) - create_enrollment_group(group_id=group_id, attestation_mechanism=attestation_mechanism) + create_enrollment_group( + group_id=group_id, + attestation_mechanism=attestation_mechanism, + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME + ) count = 0 common_device_key_input_file = "demoCA/private/device_key" common_device_cert_input_file = "demoCA/newcerts/device_cert" @@ -359,7 +371,11 @@ async def test_group_of_devices_register_with_no_device_id_for_a_x509_ca_authent DPS_GROUP_CA_CERT = os.getenv("PROVISIONING_ROOT_CERT") x509 = create_x509_ca_refs(primary_ref=DPS_GROUP_CA_CERT) attestation_mechanism = models.AttestationMechanism(type="x509", x509=x509) - create_enrollment_group(group_id=group_id, attestation_mechanism=attestation_mechanism) + create_enrollment_group( + group_id=group_id, + attestation_mechanism=attestation_mechanism, + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME + ) count = 0 intermediate_cert_filename = "demoCA/newcerts/intermediate_cert.pem" common_device_key_input_file = "demoCA/private/device_key" @@ -441,6 +457,7 @@ def create_individual_enrollment( attestation_mechanism, device_id=None, client_ca_name=None, + credential_policy_name=None, ): reprovision_policy = models.ReprovisionPolicy(migrate_device_data=True) client_certificate_issuance_policy = None @@ -455,6 +472,7 @@ def create_individual_enrollment( reprovision_policy=reprovision_policy, device_id=device_id, client_certificate_issuance_policy=client_certificate_issuance_policy, + credential_policy_name=credential_policy_name, ) return service_client.create_or_update_individual_enrollment(individual_provisioning_model) @@ -537,7 +555,7 @@ async def register_via_symmetric_key(registration_id, symmetric_key, protocol, c return await provisioning_device_client.register() -def create_enrollment_group(group_id, attestation_mechanism): +def create_enrollment_group(group_id, attestation_mechanism, client_ca_name=None, credential_policy_name=None): reprovision_policy = models.ReprovisionPolicy(migrate_device_data=True) client_certificate_issuance_policy = models.ClientCertificateIssuancePolicy( @@ -548,6 +566,7 @@ def create_enrollment_group(group_id, attestation_mechanism): attestation=attestation_mechanism, reprovision_policy=reprovision_policy, client_certificate_issuance_policy=client_certificate_issuance_policy, + credential_policy_name=credential_policy_name, ) return service_client.create_or_update_enrollment_group(enrollment_group_provisioning_model) From 158ee6e6b713428718b428fbe7b2bdfb84196efa Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Fri, 6 Mar 2026 22:36:56 -0800 Subject: [PATCH 18/24] Remove previous ClientCertificateIssuancePolicy construct --- .../protocol/models/__init__.py | 2 -- .../client_certificate_issuance_policy.py | 31 ------------------- .../protocol/models/enrollment_group.py | 9 +----- .../protocol/models/individual_enrollment.py | 7 ----- .../test_async_certificate_enrollments.py | 7 ----- .../tests/test_async_dps_cert_mgmt.py | 18 ++--------- .../tests/test_async_symmetric_enrollments.py | 6 ---- 7 files changed, 3 insertions(+), 77 deletions(-) delete mode 100644 dev_utils/dev_utils/provisioningservice/protocol/models/client_certificate_issuance_policy.py diff --git a/dev_utils/dev_utils/provisioningservice/protocol/models/__init__.py b/dev_utils/dev_utils/provisioningservice/protocol/models/__init__.py index 476329b0d..080b6b62c 100644 --- a/dev_utils/dev_utils/provisioningservice/protocol/models/__init__.py +++ b/dev_utils/dev_utils/provisioningservice/protocol/models/__init__.py @@ -27,7 +27,6 @@ from .custom_allocation_definition import CustomAllocationDefinition from .individual_enrollment import IndividualEnrollment from .enrollment_group import EnrollmentGroup - from .client_certificate_issuance_policy import ClientCertificateIssuancePolicy except (ImportError) as e: print(e) @@ -51,5 +50,4 @@ "CustomAllocationDefinition", "IndividualEnrollment", "EnrollmentGroup", - "ClientCertificateIssuancePolicy", ] diff --git a/dev_utils/dev_utils/provisioningservice/protocol/models/client_certificate_issuance_policy.py b/dev_utils/dev_utils/provisioningservice/protocol/models/client_certificate_issuance_policy.py deleted file mode 100644 index d054b44f5..000000000 --- a/dev_utils/dev_utils/provisioningservice/protocol/models/client_certificate_issuance_policy.py +++ /dev/null @@ -1,31 +0,0 @@ -# coding=utf-8 -# -------------------------------------------------------------------------- -# Code generated by Microsoft (R) AutoRest Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is -# regenerated. -# -------------------------------------------------------------------------- - -from msrest.serialization import Model - - -class ClientCertificateIssuancePolicy(Model): - """The device enrollment record. - - Variables are only populated by the server, and will be ignored when - sending a request. - - All required parameters must be populated in order to send to Azure. - - param certificateAuthorityName: The certificate authority name - :type certificateAuthorityName: str - """ - - _validation = {"certificate_authority_name": {"required": True}} - - _attribute_map = { - "certificate_authority_name": {"key": "certificateAuthorityName", "type": "str"}, - } - - def __init__(self, **kwargs): - super(ClientCertificateIssuancePolicy, self).__init__(**kwargs) - self.certificate_authority_name = kwargs.get("certificate_authority_name", None) diff --git a/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py b/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py index 03d623691..69914c6f7 100644 --- a/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py +++ b/dev_utils/dev_utils/provisioningservice/protocol/models/enrollment_group.py @@ -84,10 +84,6 @@ class EnrollmentGroup(Model): "key": "customAllocationDefinition", "type": "CustomAllocationDefinition", }, - "client_certificate_issuance_policy": { - "key": "clientCertificateIssuancePolicy", - "type": "ClientCertificateIssuancePolicy", - }, "credential_policy_name": {"key": "credentialPolicyName", "type": "str"}, } @@ -105,7 +101,4 @@ def __init__(self, **kwargs): self.allocation_policy = kwargs.get("allocation_policy", None) self.iot_hubs = kwargs.get("iot_hubs", None) self.custom_allocation_definition = kwargs.get("custom_allocation_definition", None) - self.client_certificate_issuance_policy = kwargs.get( - "client_certificate_issuance_policy", None - ) - seltf.credential_policy_name = kwargs.get("credential_policy_name", None) + self.credential_policy_name = kwargs.get("credential_policy_name", None) diff --git a/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py b/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py index a343335cc..e733014fd 100644 --- a/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py +++ b/dev_utils/dev_utils/provisioningservice/protocol/models/individual_enrollment.py @@ -95,10 +95,6 @@ class IndividualEnrollment(Model): "key": "customAllocationDefinition", "type": "CustomAllocationDefinition", }, - "client_certificate_issuance_policy": { - "key": "clientCertificateIssuancePolicy", - "type": "ClientCertificateIssuancePolicy", - }, "credential_policy_name": {"key": "credentialPolicyName", "type": "str"}, } @@ -119,7 +115,4 @@ def __init__(self, **kwargs): self.allocation_policy = kwargs.get("allocation_policy", None) self.iot_hubs = kwargs.get("iot_hubs", None) self.custom_allocation_definition = kwargs.get("custom_allocation_definition", None) - self.client_certificate_issuance_policy = kwargs.get( - "client_certificate_issuance_policy", None - ) self.credential_policy_name = kwargs.get("credential_policy_name", None) diff --git a/tests/e2e/provisioning_e2e/tests/test_async_certificate_enrollments.py b/tests/e2e/provisioning_e2e/tests/test_async_certificate_enrollments.py index f2c6dcb35..1d26f31c1 100644 --- a/tests/e2e/provisioning_e2e/tests/test_async_certificate_enrollments.py +++ b/tests/e2e/provisioning_e2e/tests/test_async_certificate_enrollments.py @@ -287,18 +287,11 @@ def create_individual_enrollment_with_x509_client_certs( ) attestation_mechanism = models.AttestationMechanism(type="x509", x509=x509) - client_certificate_issuance_policy = None - if client_ca_name: - client_certificate_issuance_policy = models.ClientCertificateIssuancePolicy( - certificate_authority_name=client_ca_name - ) - individual_provisioning_model = models.IndividualEnrollment( attestation=attestation_mechanism, registration_id=registration_id, reprovision_policy=reprovision_policy, device_id=device_id, - client_certificate_issuance_policy=client_certificate_issuance_policy, ) return service_client.create_or_update_individual_enrollment(individual_provisioning_model) diff --git a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py index c76bafe95..77593b35c 100644 --- a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py +++ b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py @@ -45,7 +45,6 @@ PROVISIONING_HOST = os.getenv("PROVISIONING_DEVICE_ENDPOINT") ID_SCOPE = os.getenv("PROVISIONING_DEVICE_IDSCOPE") -CLIENT_CERT_AUTH_NAME = os.getenv("CLIENT_CERTIFICATE_AUTHORITY_NAME") ADR_CERT_MGMT_POLICY_NAME = os.getenv("ADR_CERT_MGMT_POLICY_NAME") type_to_device_indices = { @@ -95,7 +94,6 @@ async def test_device_register_with_client_cert_issuance_for_a_symmetric_key_ind individual_enrollment_record = create_individual_enrollment( registration_id=registration_id, attestation_mechanism=attestation_mechanism, - client_ca_name=CLIENT_CERT_AUTH_NAME, credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) symmetric_key = individual_enrollment_record.attestation.symmetric_key.primary_key @@ -192,7 +190,6 @@ async def test_device_register_with_device_id_for_a_x509_individual_enrollment(p registration_id=registration_id, attestation_mechanism=attestation_mechanism, device_id=device_id, - client_ca_name=CLIENT_CERT_AUTH_NAME, credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) registration_id = individual_enrollment_record.registration_id @@ -239,7 +236,6 @@ async def test_device_register_with_no_device_id_for_a_x509_individual_enrollmen individual_enrollment_record = create_individual_enrollment( registration_id=registration_id, attestation_mechanism=attestation_mechanism, - client_ca_name=CLIENT_CERT_AUTH_NAME, credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) @@ -456,22 +452,15 @@ def create_individual_enrollment( registration_id, attestation_mechanism, device_id=None, - client_ca_name=None, credential_policy_name=None, ): reprovision_policy = models.ReprovisionPolicy(migrate_device_data=True) - client_certificate_issuance_policy = None - if client_ca_name: - client_certificate_issuance_policy = models.ClientCertificateIssuancePolicy( - certificate_authority_name=client_ca_name - ) individual_provisioning_model = models.IndividualEnrollment( attestation=attestation_mechanism, registration_id=registration_id, reprovision_policy=reprovision_policy, device_id=device_id, - client_certificate_issuance_policy=client_certificate_issuance_policy, credential_policy_name=credential_policy_name, ) @@ -555,17 +544,14 @@ async def register_via_symmetric_key(registration_id, symmetric_key, protocol, c return await provisioning_device_client.register() -def create_enrollment_group(group_id, attestation_mechanism, client_ca_name=None, credential_policy_name=None): +def create_enrollment_group(group_id, attestation_mechanism, credential_policy_name=None): reprovision_policy = models.ReprovisionPolicy(migrate_device_data=True) - client_certificate_issuance_policy = models.ClientCertificateIssuancePolicy( - certificate_authority_name=CLIENT_CERT_AUTH_NAME - ) + enrollment_group_provisioning_model = models.EnrollmentGroup( enrollment_group_id=group_id, attestation=attestation_mechanism, reprovision_policy=reprovision_policy, - client_certificate_issuance_policy=client_certificate_issuance_policy, credential_policy_name=credential_policy_name, ) return service_client.create_or_update_enrollment_group(enrollment_group_provisioning_model) diff --git a/tests/e2e/provisioning_e2e/tests/test_async_symmetric_enrollments.py b/tests/e2e/provisioning_e2e/tests/test_async_symmetric_enrollments.py index fbd3de29a..236b8b026 100644 --- a/tests/e2e/provisioning_e2e/tests/test_async_symmetric_enrollments.py +++ b/tests/e2e/provisioning_e2e/tests/test_async_symmetric_enrollments.py @@ -90,17 +90,11 @@ def create_individual_enrollment(registration_id, device_id=None, client_ca_name """ reprovision_policy = models.ReprovisionPolicy(migrate_device_data=True) attestation_mechanism = models.AttestationMechanism(type="symmetricKey") - client_certificate_issuance_policy = None - if client_ca_name: - client_certificate_issuance_policy = models.ClientCertificateIssuancePolicy( - certificate_authority_name=client_ca_name - ) individual_provisioning_model = models.IndividualEnrollment( attestation=attestation_mechanism, registration_id=registration_id, device_id=device_id, reprovision_policy=reprovision_policy, - client_certificate_issuance_policy=client_certificate_issuance_policy, ) return service_client.create_or_update_individual_enrollment(individual_provisioning_model) From e1beadc21687a9459a534ec536c85cd5de5643f3 Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Fri, 6 Mar 2026 23:17:36 -0800 Subject: [PATCH 19/24] Add env var generation cmdlet, re-enable build and tests --- vsts/dps-e2e-cert-mgmt.yaml | 89 +++++++++++++++---------------------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/vsts/dps-e2e-cert-mgmt.yaml b/vsts/dps-e2e-cert-mgmt.yaml index b6ae1aa62..87bf32ee7 100644 --- a/vsts/dps-e2e-cert-mgmt.yaml +++ b/vsts/dps-e2e-cert-mgmt.yaml @@ -24,73 +24,58 @@ stages: Import-Module ./Azure.Iot.Sdk.Test.psm1 $ResourceGroupName = New-AzureResourceGroupName -OutFile "test_config/azure-resource-group-name.txt" - $TestEnvInfo = New-AzIotTestEnvironment -AzureLocation $env:AZURE_LOCATION -ResourceGroup $ResourceGroupName -EnableFileUpload -IotHubX509ThumbprintDevices 1 -DpsX509IndividualEnrollments 1 + $TestEnvInfo = New-AzIotTestEnvironment -AzureLocation $env:AZURE_LOCATION -ResourceGroup $ResourceGroupName -EnableFileUpload -IotHubX509ThumbprintDevices 1 -DpsX509IndividualEnrollments 1 -EnableCertificateManagement + New-AzIotPythonSDKE2ETestConfig -TestEnvInfo $TestEnvInfo -Target bash -OutFile test_config/set_e2e_test_env_vars.sh displayName: Create Azure Resources and Test Config - publish: test_config artifact: test_config_scripts displayName: Publish artifact (test config scripts) condition: always() -# - stage: build_and_tests -# jobs: -# - job: 'Test' -# variables: -# IOTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.IOTHUB_CONNECTION_STRING']] -# EVENTHUB_CONNECTION_STRING: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.EVENTHUB_CONNECTION_STRING']] -# DPS_CONNECTION_STRING: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.DPS_CONNECTION_STRING']] -# DPS_ID_SCOPE: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.DPS_ID_SCOPE']] -# DPS_HOSTNAME: $[stageDependencies.setup.create_azure_resources.outputs['createAzureResources.DPS_HOSTNAME']] -# DPS_ROOT_CA_CERT_BASE64: $[stageDependencies.setup.create_azure_resources.outputs['testCertificates.DPS_ROOT_CA_CERT_BASE64']] -# DPS_ROOT_CA_PK_BASE64: $[stageDependencies.setup.create_azure_resources.outputs['testCertificates.DPS_ROOT_CA_PK_BASE64']] -# DPS_ROOT_CA_PASSWORD: $[stageDependencies.setup.create_azure_resources.outputs['testCertificates.DPS_ROOT_CA_PASSWORD']] -# pool: -# vmImage: 'ubuntu-22.04' - -# steps: -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: '3.9' -# architecture: 'x64' - -# - script: 'python scripts/env_setup.py --no_dev' -# displayName: 'Prepare environment (install packages + dev dependencies + test dependencies + tools)' +- stage: build_and_tests + jobs: + - job: 'Test' + pool: + vmImage: 'ubuntu-22.04' -# - script: 'python -m pip install cryptography' -# displayName: 'Install pyca/cryptography for X509 functionality' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + architecture: 'x64' -# - script: | -# cd $(Agent.WorkFolder) -# cd .. -# touch .rnd -# displayName: 'create RANDFILE file (needed to store seed data) separately due to openssl version issues in the pipeline' + - download: current + artifact: test_config_scripts + displayName: Download artifact (test config scripts) -# - script: | -# cd $(Build.SourcesDirectory)/tests/e2e/provisioning_e2e -# pytest --junitxml=junit/dps-e2e-test-results.xml -# displayName: 'Run Specified E2E Test with env variables' + - script: 'python scripts/env_setup.py --no_dev' + displayName: 'Prepare environment (install packages + dev dependencies + test dependencies + tools)' -# env: -# IOTHUB_CONNECTION_STRING: $(IOTHUB_CONNECTION_STRING) -# IOTHUB_EVENTHUB_CONNECTION_STRING: $(EVENTHUB_CONNECTION_STRING) + - script: 'python -m pip install cryptography' + displayName: 'Install pyca/cryptography for X509 functionality' -# PROVISIONING_DEVICE_ENDPOINT: $(DPS_HOSTNAME) -# PROVISIONING_SERVICE_CONNECTION_STRING: $(DPS_CONNECTION_STRING) -# PROVISIONING_DEVICE_IDSCOPE: $(DPS_ID_SCOPE) + - script: | + cd $(Agent.WorkFolder) + cd .. + touch .rnd + displayName: 'create RANDFILE file (needed to store seed data) separately due to openssl version issues in the pipeline' -# PROVISIONING_ROOT_CERT: $(DPS_ROOT_CA_CERT_BASE64) -# PROVISIONING_ROOT_CERT_KEY: $(DPS_ROOT_CA_PK_BASE64) -# PROVISIONING_ROOT_PASSWORD: $(DPS_ROOT_CA_PASSWORD) -# PYTHONUNBUFFERED: True + - script: | + set -e + source "$(Pipeline.Workspace)/test_config_scripts/set_e2e_test_env_vars.sh" + cd $(Build.SourcesDirectory)/tests/e2e/provisioning_e2e + pytest --junitxml=junit/dps-e2e-test-results.xml + displayName: 'Run Specified E2E Test with env variables' -# - task: PublishTestResults@2 -# displayName: 'Publish Test Results' -# condition: always() -# inputs: -# testResultsFiles: '**/dps-e2e-test-*.xml' -# testRunTitle: 'Publish test results for Python $(python.version)' + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + condition: always() + inputs: + testResultsFiles: '**/dps-e2e-test-*.xml' + testRunTitle: 'Publish test results for Python $(python.version)' - stage: cleanup dependsOn: - setup - # - build_and_tests + - build_and_tests condition: always() # Run stage even if the pipeline run is cancelled. jobs: - job: destroy_azure_resource_group From f831984fb74bbe36e2d9a112b002b7f4758fd1fe Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Sun, 8 Mar 2026 00:52:49 -0800 Subject: [PATCH 20/24] Pass private key password only if defined as a non-empty string --- scripts/create_x509_chain_crypto.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/create_x509_chain_crypto.py b/scripts/create_x509_chain_crypto.py index 7927d778a..5a4390d4f 100644 --- a/scripts/create_x509_chain_crypto.py +++ b/scripts/create_x509_chain_crypto.py @@ -454,8 +454,14 @@ def call_intermediate_cert_and_device_cert_creation_from_pipeline( key_pem_data = str(base64.b64decode(ca_key), "ascii") out_ca_key.write(key_pem_data) encoded_key_pem_data = str.encode(key_pem_data) + + if ca_password is not None and ca_password != "": + password_bytes = str.encode(ca_password) + else: + password_bytes = None + root_private_key = serialization.load_pem_private_key( - encoded_key_pem_data, password=str.encode(ca_password), backend=default_backend() + encoded_key_pem_data, password=password_bytes, backend=default_backend() ) if os.path.exists(in_key_file_path): From 7da5ed436b24bc49eace1c00aa2af40028468c35 Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Sun, 8 Mar 2026 16:52:43 -0700 Subject: [PATCH 21/24] Adjust Dps service API version for cert mgmt (2021-10-01) --- dev_utils/dev_utils/provisioningservice/protocol/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils/dev_utils/provisioningservice/protocol/version.py b/dev_utils/dev_utils/provisioningservice/protocol/version.py index e559f14ac..b784b7625 100644 --- a/dev_utils/dev_utils/provisioningservice/protocol/version.py +++ b/dev_utils/dev_utils/provisioningservice/protocol/version.py @@ -5,4 +5,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "2021-11-01-preview" +VERSION = "2021-10-01" From b578023e3b0eb20d22777379ed9e1b89b3af4c5a Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Sun, 8 Mar 2026 22:55:26 -0700 Subject: [PATCH 22/24] Adjust Dps service API version for cert mgmt (try 2025-07-01-preview) --- dev_utils/dev_utils/provisioningservice/protocol/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils/dev_utils/provisioningservice/protocol/version.py b/dev_utils/dev_utils/provisioningservice/protocol/version.py index b784b7625..9945d9870 100644 --- a/dev_utils/dev_utils/provisioningservice/protocol/version.py +++ b/dev_utils/dev_utils/provisioningservice/protocol/version.py @@ -5,4 +5,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "2021-10-01" +VERSION = "2025-07-01-preview" From a171acb08d39852d9924343ebe26c9dee86f72c2 Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Tue, 10 Mar 2026 00:51:36 -0700 Subject: [PATCH 23/24] Add e2e tests for certificate signing request --- samples/cert-mgmt/certificate_issuance.py | 5 - scripts/create_x509_chain_crypto.py | 51 +- .../tests/test_async_dps_cert_mgmt.py | 496 ++++++------------ 3 files changed, 201 insertions(+), 351 deletions(-) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index 33914e03a..f516e29d7 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -177,8 +177,3 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) - - # If using Python 3.6 or below, use the following code instead of asyncio.run(main()): - # loop = asyncio.get_event_loop() - # loop.run_until_complete(main()) - # loop.close() diff --git a/scripts/create_x509_chain_crypto.py b/scripts/create_x509_chain_crypto.py index 5a4390d4f..ccbb86e66 100644 --- a/scripts/create_x509_chain_crypto.py +++ b/scripts/create_x509_chain_crypto.py @@ -1,7 +1,7 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric import ec, rsa from cryptography.x509.oid import NameOID from datetime import datetime, timedelta import uuid @@ -51,6 +51,37 @@ def create_private_key(key_file, password=None, key_size=4096): return private_key +def create_ec_private_key(key_file, password=None, curve=None): + """ + Create an EC private key and write it to a file in PKCS8 PEM format. + Equivalent to: openssl ecparam -name prime256v1 -genkey -noout | openssl pkcs8 -topk8 -nocrypt -out + :param key_file: The file to store the key. + :param password: Optional password for the key. If None, the key is unencrypted. + :param curve: The elliptic curve to use. Defaults to SECP256R1 (prime256v1). + :return: The private key. + """ + if password: + encrypt_algo = serialization.BestAvailableEncryption(str.encode(password)) + else: + encrypt_algo = serialization.NoEncryption() + + if curve is None: + curve = ec.SECP256R1() + + private_key = ec.generate_private_key(curve, backend=default_backend()) + + with open(key_file, "wb") as f: + f.write( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=encrypt_algo, + ) + ) + + return private_key + + def create_certificate_chain( common_name, ca_password, @@ -376,18 +407,12 @@ def delete_directories_certs_created_from_pipeline(): shutil.rmtree(dirPath) except Exception: print("Error while deleting directory") - if os.path.exists("out_ca_cert.pem"): - os.remove("out_ca_cert.pem") - else: - print("The file does not exist") - if os.path.exists("out_ca_key.pem"): - os.remove("out_ca_key.pem") - else: - print("The file does not exist") - if os.path.exists(".rnd"): - os.remove(".rnd") - else: - print("The file does not exist") + + for f in ["out_ca_cert.pem", "out_ca_key.pem", "ca_cert.pem", "ca_key.pem", ".rnd"]: + if os.path.exists(f): + os.remove(f) + else: + print(f"The file {f} does not exist") def before_cert_creation_from_pipeline(): diff --git a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py index 77593b35c..65b4ed4b3 100644 --- a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py +++ b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py @@ -7,6 +7,7 @@ from provisioning_e2e.service_helper import Helper, connection_string_to_hostname from azure.iot.device.aio import ProvisioningDeviceClient, IoTHubDeviceClient from azure.iot.device.common import X509 +from azure.iot.device.iothub.models import CertificateSigningRequest from dev_utils.provisioningservice.protocol import models from dev_utils.provisioningservice.client import ProvisioningServiceClient @@ -25,17 +26,17 @@ before_cert_creation_from_pipeline, call_intermediate_cert_and_device_cert_creation_from_pipeline, delete_directories_certs_created_from_pipeline, - create_private_key, + create_ec_private_key, create_csr, ) pytestmark = pytest.mark.asyncio logging.basicConfig(level=logging.DEBUG) -intermediate_common_name = "e2edpshomenumdps" -intermediate_password = "revelio" -device_common_name = "e2edpslocomotor" + str(uuid.uuid4()) -device_password = "mortis" +intermediate_common_name = "e2edpscsrintcn" +intermediate_password = "password123" +device_common_name = "e2edpscsr" + str(uuid.uuid4()) +device_password = "password123" service_client = ProvisioningServiceClient.create_from_connection_string( os.getenv("PROVISIONING_SERVICE_CONNECTION_STRING") @@ -48,11 +49,8 @@ ADR_CERT_MGMT_POLICY_NAME = os.getenv("ADR_CERT_MGMT_POLICY_NAME") type_to_device_indices = { - "individual_with_device_id": [1], - "individual_no_device_id": [2], - "group_intermediate": [3, 4, 5], - "group_ca": [6, 7, 8], - "group_symmetric": [9, 10, 11], + "group_intermediate": [3, 4], + "group_symmetric": [6, 7], } @@ -76,211 +74,22 @@ def after_module(): request.addfinalizer(after_module) +# TODO : Don't do mqttws as it conflicts with SAME cert problem, Need complete set of new certs with mqtts @pytest.mark.it( - "A device requests a client cert by sending a certificate signing request " - "while being provisioned to the linked IoTHub with the device_id equal to the registration_id" - "of the individual enrollment that has been created with a symmetric key authentication" -) -@pytest.mark.parametrize("protocol", ["mqtt", "mqttws"]) -async def test_device_register_with_client_cert_issuance_for_a_symmetric_key_individual_enrollment( - protocol, -): - registration_id = "e2e-dps-locomotor" + str(uuid.uuid4()) - key_file = "key.pem" - csr_file = "request.pem" - issued_cert_file = "cert.pem" - try: - attestation_mechanism = models.AttestationMechanism(type="symmetricKey") - individual_enrollment_record = create_individual_enrollment( - registration_id=registration_id, - attestation_mechanism=attestation_mechanism, - credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, - ) - symmetric_key = individual_enrollment_record.attestation.symmetric_key.primary_key - private_key = create_private_key(key_file) - create_csr(private_key, csr_file, registration_id) - - registration_result = await register_via_symmetric_key( - registration_id, symmetric_key, protocol, csr_file=csr_file - ) - - assert_device_provisioned( - device_id=registration_id, registration_result=registration_result - ) - await connect_device_with_operational_cert( - registration_result=registration_result, - issued_cert_file=issued_cert_file, - key_file=key_file, - ) - device_registry_helper.try_delete_device(registration_id) - finally: - service_client.delete_individual_enrollment_by_param(registration_id) - delete_client_certs(key_file, csr_file, issued_cert_file) - - -@pytest.mark.it( - "A group of devices request client certs by sending certificate signing requests while being provisioned" - " to the linked IoTHub inside a group enrollment that has been created with a symmetric key authentication" + "A group of devices get provisioned to the linked IoTHub with device_ids equal to the individual registration_ids inside a group enrollment that has been created with intermediate X509 authentication" ) @pytest.mark.parametrize("protocol", ["mqtt"]) -async def test_device_register_with_client_cert_issuance_for_a_symmetric_key_group_enrollment( +@pytest.mark.timeout(120) +@pytest.mark.skipif( + ADR_CERT_MGMT_POLICY_NAME is None, + reason="Deployment with ADR cert management policy is required to run this test", +) +async def test_group_of_devices_register_with_no_device_id_for_a_x509_intermediate_authentication_group_enrollment( protocol, ): - group_id = "e2e-symmetric-group" + str(uuid.uuid4()) - devices_indices = type_to_device_indices.get("group_symmetric") - device_count_in_group = len(devices_indices) - common_device_id = "e2edpsgroupsymmetric" - try: - attestation_mechanism = models.AttestationMechanism(type="symmetricKey") - eg = create_enrollment_group( - group_id=group_id, - attestation_mechanism=attestation_mechanism, - credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, - ) - master_key = eg.attestation.symmetric_key.primary_key - count = 0 - for index in devices_indices: - count = count + 1 - device_id = common_device_id + str(index) - device_key = derive_device_key(device_id, master_key) - - key_file = "key" + str(index) + ".pem" - csr_file = "request" + str(index) + ".pem" - - private_key = create_private_key(key_file) - create_csr(private_key, csr_file, device_id) - registration_result = await register_via_symmetric_key( - registration_id=device_id, - symmetric_key=device_key, - protocol=protocol, - csr_file=csr_file, - ) - - assert_device_provisioned(device_id=device_id, registration_result=registration_result) - issued_cert_file = "cert" + str(index) + ".pem" - await connect_device_with_operational_cert( - registration_result=registration_result, - issued_cert_file=issued_cert_file, - key_file=key_file, - ) - assert count == device_count_in_group - device_registry_helper.try_delete_device(device_id) - finally: - for index in devices_indices: - key_file = "key" + str(index) + ".pem" - csr_file = "request" + str(index) + ".pem" - issued_cert_file = "cert" + str(index) + ".pem" - delete_client_certs(key_file, csr_file, issued_cert_file) - service_client.delete_enrollment_group_by_param(group_id) - - -@pytest.mark.it( - "A device gets provisioned to the linked IoTHub with the user supplied device_id different from the registration_id of the individual enrollment that has been created with a selfsigned X509 authentication" -) -@pytest.mark.parametrize("protocol", ["mqtt", "mqttws"]) -async def test_device_register_with_device_id_for_a_x509_individual_enrollment(protocol): - device_id = "e2edpsthunderbolt" - device_index = type_to_device_indices.get("individual_with_device_id")[0] - registration_id = device_common_name + str(device_index) - try: - cert_content = read_cert_content_from_file(device_index=device_index) - x509 = create_x509_client_or_sign_certs(is_client=True, primary_cert=cert_content) - attestation_mechanism = models.AttestationMechanism(type="x509", x509=x509) - individual_enrollment_record = create_individual_enrollment( - registration_id=registration_id, - attestation_mechanism=attestation_mechanism, - device_id=device_id, - credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, - ) - registration_id = individual_enrollment_record.registration_id - - device_cert_file = "demoCA/newcerts/device_cert" + str(device_index) + ".pem" - device_key_file = "demoCA/private/device_key" + str(device_index) + ".pem" - - key_file = "key.pem" - csr_file = "request.pem" - - private_key = create_private_key(key_file) - create_csr(private_key, csr_file, registration_id) - - registration_result = await register_via_x509( - registration_id, device_cert_file, device_key_file, protocol, csr_file=csr_file - ) - - assert device_id != registration_id - assert_device_provisioned(device_id=device_id, registration_result=registration_result) - issued_cert_file = "cert.pem" - await connect_device_with_operational_cert( - registration_result=registration_result, - issued_cert_file=issued_cert_file, - key_file=key_file, - ) - device_registry_helper.try_delete_device(device_id) - finally: - delete_client_certs(key_file, csr_file, issued_cert_file) - service_client.delete_individual_enrollment_by_param(registration_id) - - -@pytest.mark.it( - "A device gets provisioned to the linked IoTHub with device_id equal to the registration_id of the " - "individual enrollment that has been created with a selfsigned X509 authentication" -) -@pytest.mark.parametrize("protocol", ["mqtt", "mqttws"]) -async def test_device_register_with_no_device_id_for_a_x509_individual_enrollment(protocol): - device_index = type_to_device_indices.get("individual_no_device_id")[0] - registration_id = device_common_name + str(device_index) - try: - cert_content = read_cert_content_from_file(device_index=device_index) - x509 = create_x509_client_or_sign_certs(is_client=True, primary_cert=cert_content) - attestation_mechanism = models.AttestationMechanism(type="x509", x509=x509) - individual_enrollment_record = create_individual_enrollment( - registration_id=registration_id, - attestation_mechanism=attestation_mechanism, - credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, - ) - - registration_id = individual_enrollment_record.registration_id - - device_cert_file = "demoCA/newcerts/device_cert" + str(device_index) + ".pem" - device_key_file = "demoCA/private/device_key" + str(device_index) + ".pem" - - key_file = "key.pem" - csr_file = "request.pem" - - private_key = create_private_key(key_file) - create_csr(private_key, csr_file, registration_id) - - registration_result = await register_via_x509( - registration_id, device_cert_file, device_key_file, protocol, csr_file=csr_file - ) - - assert_device_provisioned( - device_id=registration_id, registration_result=registration_result - ) - - issued_cert_file = "cert.pem" - - await connect_device_with_operational_cert( - registration_result=registration_result, - issued_cert_file=issued_cert_file, - key_file=key_file, - ) - device_registry_helper.try_delete_device(registration_id) - finally: - delete_client_certs(key_file, csr_file, issued_cert_file) - service_client.delete_individual_enrollment_by_param(registration_id) - - -@pytest.mark.it( - "A group of devices get provisioned to the linked IoTHub with device_ids equal to the individual registration_ids " - "inside a group enrollment that has been created with intermediate X509 authentication" -) -async def test_group_of_devices_register_with_no_device_id_for_a_x509_intermediate_authentication_group_enrollment(): - protocol = "mqtt" - group_id = "e2e-intermediate-durmstrang" + str(uuid.uuid4()) - common_device_id = "e2edpsinterdevice" + group_id = "e2e-intermediate-csr-" + str(uuid.uuid4()) + common_device_id = "e2edpscsrdevice" devices_indices = type_to_device_indices.get("group_intermediate") - device_count_in_group = len(devices_indices) try: intermediate_cert_filename = "demoCA/newcerts/intermediate_cert.pem" @@ -288,22 +97,24 @@ async def test_group_of_devices_register_with_no_device_id_for_a_x509_intermedia intermediate_cert_content = intermediate_pem.read() x509 = create_x509_client_or_sign_certs( - is_client=False, - primary_cert=intermediate_cert_content, + is_client=False, primary_cert=intermediate_cert_content ) - attestation_mechanism = models.AttestationMechanism(type="x509", x509=x509) + create_enrollment_group( group_id=group_id, - attestation_mechanism=attestation_mechanism, - credential_policy_name=ADR_CERT_MGMT_POLICY_NAME + attestation_mechanism=models.AttestationMechanism(type="x509", x509=x509), + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) + count = 0 common_device_key_input_file = "demoCA/private/device_key" common_device_cert_input_file = "demoCA/newcerts/device_cert" common_device_inter_cert_chain_file = "demoCA/newcerts/out_inter_device_chain_cert" + for index in devices_indices: count = count + 1 device_id = common_device_id + str(index) + device_key_input_file = common_device_key_input_file + str(index) + ".pem" device_cert_input_file = common_device_cert_input_file + str(index) + ".pem" device_inter_cert_chain_file = common_device_inter_cert_chain_file + str(index) + ".pem" @@ -313,119 +124,132 @@ async def test_group_of_devices_register_with_no_device_id_for_a_x509_intermedia with open(fname) as infile: outfile.write(infile.read()) - key_file = "key" + str(index) + ".pem" - csr_file = "request" + str(index) + ".pem" - - private_key = create_private_key(key_file) - create_csr(private_key, csr_file, device_id) + csr_key_file = "csr_key_" + str(index) + ".pem" + dps_csr_file = "dps_csr_" + str(index) + ".pem" + csr_private_key = create_ec_private_key(csr_key_file) + create_csr(csr_private_key, dps_csr_file, device_id) registration_result = await register_via_x509( registration_id=device_id, device_cert_file=device_inter_cert_chain_file, device_key_file=device_key_input_file, protocol=protocol, - csr_file=csr_file, + csr_file=dps_csr_file, ) assert_device_provisioned(device_id=device_id, registration_result=registration_result) - print("device was provisioned") - print(device_id) - issued_cert_file = "cert" + str(index) + ".pem" - await connect_device_with_operational_cert( + device_client = await connect_device_with_issued_certificate( + registration_result=registration_result, key_file=csr_key_file + ) + + iot_csr_file = "iot_hub_csr_" + str(index) + ".pem" + create_csr( + csr_private_key, iot_csr_file, registration_result.registration_state.device_id + ) + + iot_hub_csr_request = CertificateSigningRequest(read_csr_from_file(iot_csr_file), "*") + csr_response = await device_client.send_certificate_signing_request(iot_hub_csr_request) + assert csr_response.status_code == 200 + assert len(csr_response.certificates) == 3 # leaf, intermediate and root certs + + await device_client.disconnect() + + device_client = await connect_device_with_issued_certificate( registration_result=registration_result, - issued_cert_file=issued_cert_file, - key_file=key_file, + key_file=csr_key_file, + iot_hub_csr_response=csr_response, ) + + await device_client.disconnect() + device_registry_helper.try_delete_device(device_id) + delete_client_certs(csr_key_file, dps_csr_file, iot_csr_file) - assert count == device_count_in_group + assert count == len( + devices_indices + ) # Verify that all devices in the group were provisioned. finally: - for index in devices_indices: - key_file = "key" + str(index) + ".pem" - csr_file = "request" + str(index) + ".pem" - issued_cert_file = "cert" + str(index) + ".pem" - delete_client_certs(key_file, csr_file, issued_cert_file) - service_client.delete_enrollment_group_by_param(group_id) -@pytest.mark.skip( - reason="The enrollment is never properly created on the pipeline and it is always created without any CA reference and eventually the registration fails" -) @pytest.mark.it( - "A group of devices get provisioned to the linked IoTHub with device_ids equal to the individual registration_ids inside a group enrollment that has been created with an already uploaded ca cert X509 authentication" + "A group of devices request client certs by sending certificate signing requests while being provisioned" + " to the linked IoTHub inside a group enrollment that has been created with a symmetric key authentication" +) +@pytest.mark.parametrize("protocol", ["mqtt"]) +@pytest.mark.timeout(120) +@pytest.mark.skipif( + ADR_CERT_MGMT_POLICY_NAME is None, + reason="Deployment with ADR cert management policy is required to run this test", ) -async def test_group_of_devices_register_with_no_device_id_for_a_x509_ca_authentication_group_enrollment(): - protocol = "mqtt" - group_id = "e2e-ca-ilvermorny" + str(uuid.uuid4()) - common_device_id = "e2edpscadevice" - devices_indices = type_to_device_indices.get("group_ca") - device_count_in_group = len(devices_indices) +async def test_device_register_with_client_cert_issuance_for_a_symmetric_key_group_enrollment( + protocol, +): + group_id = "e2e-symmetric-csr-" + str(uuid.uuid4()) + common_device_id = "e2edpscsrskdev" + devices_indices = type_to_device_indices.get("group_symmetric") + try: - DPS_GROUP_CA_CERT = os.getenv("PROVISIONING_ROOT_CERT") - x509 = create_x509_ca_refs(primary_ref=DPS_GROUP_CA_CERT) - attestation_mechanism = models.AttestationMechanism(type="x509", x509=x509) - create_enrollment_group( + eg = create_enrollment_group( group_id=group_id, - attestation_mechanism=attestation_mechanism, - credential_policy_name=ADR_CERT_MGMT_POLICY_NAME + attestation_mechanism=models.AttestationMechanism(type="symmetricKey"), + credential_policy_name=ADR_CERT_MGMT_POLICY_NAME, ) + count = 0 - intermediate_cert_filename = "demoCA/newcerts/intermediate_cert.pem" - common_device_key_input_file = "demoCA/private/device_key" - common_device_cert_input_file = "demoCA/newcerts/device_cert" - common_device_inter_cert_chain_file = "demoCA/newcerts/out_inter_device_chain_cert" for index in devices_indices: count = count + 1 device_id = common_device_id + str(index) - device_key_input_file = common_device_key_input_file + str(index) + ".pem" - device_cert_input_file = common_device_cert_input_file + str(index) + ".pem" - device_inter_cert_chain_file = common_device_inter_cert_chain_file + str(index) + ".pem" - filenames = [device_cert_input_file, intermediate_cert_filename] - with open(device_inter_cert_chain_file, "w") as outfile: - for fname in filenames: - with open(fname) as infile: - logging.debug("Filename is {}".format(fname)) - content = infile.read() - logging.debug(content) - outfile.write(content) + device_key = derive_device_key(device_id, eg.attestation.symmetric_key.primary_key) - key_file = "key" + str(index) + ".pem" - csr_file = "request" + str(index) + ".pem" + csr_key_file = "csr_key_" + str(index) + ".pem" + dps_csr_file = "dps_csr_" + str(index) + ".pem" + csr_private_key = create_ec_private_key(csr_key_file) + create_csr(csr_private_key, dps_csr_file, device_id) - private_key = create_private_key(key_file) - create_csr(private_key, csr_file, device_id) - - registration_result = await register_via_x509( + registration_result = await register_via_symmetric_key( registration_id=device_id, - device_cert_file=device_inter_cert_chain_file, - device_key_file=device_key_input_file, + symmetric_key=device_key, protocol=protocol, - csr_file=csr_file, + csr_file=dps_csr_file, ) assert_device_provisioned(device_id=device_id, registration_result=registration_result) - print("device was provisioned for ca") - print(device_id) - issued_cert_file = "cert" + str(index) + ".pem" + device_client = await connect_device_with_issued_certificate( + registration_result=registration_result, key_file=csr_key_file + ) + + iot_csr_file = "iot_hub_csr_" + str(index) + ".pem" + create_csr( + csr_private_key, iot_csr_file, registration_result.registration_state.device_id + ) + + iot_hub_csr_request = CertificateSigningRequest(read_csr_from_file(iot_csr_file), "*") + csr_response = await device_client.send_certificate_signing_request(iot_hub_csr_request) + assert csr_response.status_code == 200 + assert len(csr_response.certificates) == 3 # leaf, intermediate and root certs - await connect_device_with_operational_cert( + await device_client.disconnect() + + device_client = await connect_device_with_issued_certificate( registration_result=registration_result, - issued_cert_file=issued_cert_file, - key_file=key_file, + key_file=csr_key_file, + iot_hub_csr_response=csr_response, ) + + await device_client.disconnect() + device_registry_helper.try_delete_device(device_id) + delete_client_certs(csr_key_file, dps_csr_file, iot_csr_file) + + assert count == len( + devices_indices + ) # Verify that all devices in the group were provisioned. - assert count == device_count_in_group finally: - for index in devices_indices: - key_file = "key" + str(index) + ".pem" - csr_file = "request" + str(index) + ".pem" - issued_cert_file = "cert" + str(index) + ".pem" - delete_client_certs(key_file, csr_file, issued_cert_file) service_client.delete_enrollment_group_by_param(group_id) @@ -441,30 +265,14 @@ def assert_device_provisioned(device_id, registration_result): device = device_registry_helper.get_device(device_id) assert device is not None - assert device.authentication.type == "selfSigned" + assert device.authentication.type == "certificateAuthority" print("assertions") - print(device_id) + print(device_id) # leaf, intermediate, root print(device.authentication.type) assert device.device_id == device_id - - -def create_individual_enrollment( - registration_id, - attestation_mechanism, - device_id=None, - credential_policy_name=None, -): - reprovision_policy = models.ReprovisionPolicy(migrate_device_data=True) - - individual_provisioning_model = models.IndividualEnrollment( - attestation=attestation_mechanism, - registration_id=registration_id, - reprovision_policy=reprovision_policy, - device_id=device_id, - credential_policy_name=credential_policy_name, - ) - - return service_client.create_or_update_individual_enrollment(individual_provisioning_model) + assert ( + len(registration_result.registration_state.issued_client_certificate) == 3 + ) # leaf, intermediate, root def create_x509_client_or_sign_certs(is_client, primary_cert, secondary_cert=None): @@ -481,31 +289,33 @@ def create_x509_client_or_sign_certs(is_client, primary_cert, secondary_cert=Non return x509_attestation -def create_x509_ca_refs(primary_ref, secondary_ref=None): - ca_refs = models.X509CAReferences(primary=primary_ref, secondary=secondary_ref) - x509_attestation = models.X509Attestation(ca_references=ca_refs) - return x509_attestation +def delete_client_certs(*args): + for cert_file in args: + if os.path.exists(cert_file): + os.remove(cert_file) -def read_cert_content_from_file(device_index): - device_cert_input_file = "demoCA/newcerts/device_cert" + str(device_index) + ".pem" - with open(device_cert_input_file, "r") as in_device_cert: - device_cert_content = in_device_cert.read() - return device_cert_content +def strip_csr_headers(csr_data): + # Strip PEM header/footer and whitespace to get raw base64 content + csr_b64 = ( + csr_data.replace("-----BEGIN CERTIFICATE REQUEST-----", "") + .replace("-----END CERTIFICATE REQUEST-----", "") + .replace("\n", "") + .strip() + ) + return csr_b64 -def delete_client_certs(key_file, csr_file, issued_cert_file): - if os.path.exists(key_file): - os.remove(key_file) - if os.path.exists(csr_file): - os.remove(csr_file) - if os.path.exists(issued_cert_file): - os.remove(issued_cert_file) +def read_csr_from_file(csr_file): + with open(csr_file, "r") as csr: + csr_data = csr.read() + return strip_csr_headers(csr_data) async def register_via_x509( registration_id, device_cert_file, device_key_file, protocol, csr_file=None ): + print("registering device {}".format(registration_id)) x509 = X509(cert_file=device_cert_file, key_file=device_key_file, pass_phrase=device_password) protocol_boolean_mapping = {"mqtt": False, "mqttws": True} provisioning_device_client = ProvisioningDeviceClient.create_from_x509_certificate( @@ -517,10 +327,7 @@ async def register_via_x509( ) if csr_file: - with open(csr_file, "r") as csr: - csr_data = csr.read() - # Set the CSR on the client to send it to DPS - provisioning_device_client.client_certificate_signing_request = str(csr_data) + provisioning_device_client.client_certificate_signing_request = read_csr_from_file(csr_file) return await provisioning_device_client.register() @@ -540,7 +347,9 @@ async def register_via_symmetric_key(registration_id, symmetric_key, protocol, c with open(csr_file, "r") as csr: csr_data = csr.read() # Set the CSR on the client to send it to DPS - provisioning_device_client.client_certificate_signing_request = str(csr_data) + provisioning_device_client.client_certificate_signing_request = strip_csr_headers( + csr_data + ) return await provisioning_device_client.register() @@ -571,15 +380,32 @@ def derive_device_key(device_id, group_symmetric_key): return device_key_encoded.decode("utf-8") -async def connect_device_with_operational_cert(registration_result, issued_cert_file, key_file): +def add_certificate_headers(cert_data): + return "-----BEGIN CERTIFICATE-----\r\n" + cert_data + "\r\n-----END CERTIFICATE-----" + + +async def connect_device_with_issued_certificate( + registration_result, key_file, iot_hub_csr_response=None +): + issued_leaf_cert_file = ( + "issued_cert_" + registration_result.registration_state.device_id + ".pem" + ) - with open(issued_cert_file, "w") as out_ca_pem: + with open(issued_leaf_cert_file, "w") as out_ca_pem: # Write the issued certificate on the file. This forms the certificate portion of the X509 object. - cert_data = registration_result.registration_state.issued_client_certificate - out_ca_pem.write(cert_data) + if iot_hub_csr_response: + cert_b64_data = iot_hub_csr_response.certificates[ + 0 + ] # use the certificate issued by IoT Hub in response to the CSR request + else: + cert_b64_data = registration_result.registration_state.issued_client_certificate[ + 0 + ] # use only leaf certificate. + + out_ca_pem.write(add_certificate_headers(cert_b64_data)) x509 = X509( - cert_file=issued_cert_file, + cert_file=issued_leaf_cert_file, key_file=key_file, ) @@ -590,6 +416,10 @@ async def connect_device_with_operational_cert(registration_result, issued_cert_ ) # Connect the client. await device_client.connect() + + delete_client_certs(issued_leaf_cert_file) + # Assert that this X509 was able to connect. assert device_client.connected - await device_client.disconnect() + + return device_client From bde4ea9e552c54f933633fdb47bfcfca14f65df5 Mon Sep 17 00:00:00 2001 From: Ewerton Scaboro da Silva Date: Tue, 10 Mar 2026 01:39:50 -0700 Subject: [PATCH 24/24] device_client.send_certificate_signing_request to take args directly --- .../azure/iot/device/iothub/abstract_clients.py | 5 ++++- .../azure/iot/device/iothub/aio/async_clients.py | 8 ++++++-- .../azure/iot/device/iothub/models/__init__.py | 5 +---- .../azure/iot/device/iothub/sync_clients.py | 10 +++++----- samples/cert-mgmt/certificate_issuance.py | 8 ++++---- .../tests/test_async_dps_cert_mgmt.py | 12 +++++++----- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/azure-iot-device/azure/iot/device/iothub/abstract_clients.py b/azure-iot-device/azure/iot/device/iothub/abstract_clients.py index b607b205d..f962b6a36 100644 --- a/azure-iot-device/azure/iot/device/iothub/abstract_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/abstract_clients.py @@ -18,6 +18,7 @@ from azure.iot.device.common.auth import sastoken as st from azure.iot.device.iothub import client_event from azure.iot.device.iothub.models import Message, MethodRequest, MethodResponse +from azure.iot.device.iothub.models import CertificateSigningResponse from azure.iot.device.common.models import X509 from azure.iot.device import exceptions from azure.iot.device.common import auth, handle_exceptions @@ -452,7 +453,9 @@ def receive_twin_desired_properties_patch(self) -> TwinPatch: pass @abc.abstractmethod - def send_certificate_signing_request(self, certificate_signing_request: CertificateSigningRequest) -> CertificateSigningResponse: + def send_certificate_signing_request( + self, csr: str, replace: str + ) -> CertificateSigningResponse: pass @property diff --git a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py index 9ae61262a..81dad2d14 100644 --- a/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/aio/async_clients.py @@ -20,9 +20,9 @@ Message, MethodRequest, MethodResponse, - CertificateSigningRequest, CertificateSigningResponse, ) +from azure.iot.device.iothub.models.certificate_signing_request import CertificateSigningRequest from azure.iot.device.iothub.pipeline import constant from azure.iot.device.iothub.pipeline import exceptions as pipeline_exceptions from azure.iot.device import exceptions @@ -527,11 +527,13 @@ async def receive_twin_desired_properties_patch(self) -> TwinPatch: return patch async def send_certificate_signing_request( - self, request: CertificateSigningRequest + self, csr: str, replace: str ) -> CertificateSigningResponse: """ Sends a Certificate Signing Request to Azure IoT Hub. + :param str csr: The base64-encoded certificate signing request. + :param str replace: Replace any active credential operation for this device. :returns: The certificate issued by Azure IoT Hub for the certificate signing request provided. :rtype: CertificateSigningResponse @@ -553,6 +555,8 @@ async def send_certificate_signing_request( if not self._mqtt_pipeline.feature_enabled[constant.CSR]: await self._enable_feature(constant.CSR) + request = CertificateSigningRequest(csr=csr, replace=replace) + send_certificate_signing_request_async = async_adapter.emulate_async( self._mqtt_pipeline.send_certificate_signing_request ) diff --git a/azure-iot-device/azure/iot/device/iothub/models/__init__.py b/azure-iot-device/azure/iot/device/iothub/models/__init__.py index 0ccfb9951..77ac8a5fb 100644 --- a/azure-iot-device/azure/iot/device/iothub/models/__init__.py +++ b/azure-iot-device/azure/iot/device/iothub/models/__init__.py @@ -5,7 +5,4 @@ from .message import Message # noqa: F401 from .methods import MethodRequest, MethodResponse # noqa: F401 -from .certificate_signing_request import ( # noqa: F401 - CertificateSigningRequest, - CertificateSigningResponse, -) +from .certificate_signing_request import CertificateSigningResponse # noqa: F401 diff --git a/azure-iot-device/azure/iot/device/iothub/sync_clients.py b/azure-iot-device/azure/iot/device/iothub/sync_clients.py index 10d08263c..6cc024b7b 100644 --- a/azure-iot-device/azure/iot/device/iothub/sync_clients.py +++ b/azure-iot-device/azure/iot/device/iothub/sync_clients.py @@ -19,9 +19,9 @@ Message, MethodResponse, MethodRequest, - CertificateSigningRequest, CertificateSigningResponse, ) +from .models.certificate_signing_request import CertificateSigningRequest from .inbox_manager import InboxManager from .sync_inbox import SyncClientInbox, InboxEmpty from . import sync_handler_manager @@ -544,7 +544,7 @@ def receive_twin_desired_properties_patch(self, block=True, timeout=None) -> Twi return patch def send_certificate_signing_request( - self, request: CertificateSigningRequest + self, csr: str, replace: str ) -> CertificateSigningResponse: """ Sends a certificate signing request to Azure IoT Hub service. @@ -555,8 +555,8 @@ def send_certificate_signing_request( If the service returns an error on the certificate signing operations operation, this function will raise the appropriate error. - :param request: Certificate signing request to be sent - :type request: CertificateSigningRequest + :param str csr: The base64-encoded certificate signing request. + :param str replace: Replace any active credential operation for this device. :raises: :class:`azure.iot.device.exceptions.CredentialError` if credentials are invalid and a connection cannot be established. @@ -575,7 +575,7 @@ def send_certificate_signing_request( if not self._mqtt_pipeline.feature_enabled[pipeline_constant.CSR]: self._enable_feature(pipeline_constant.CSR) - request.id = None + request = CertificateSigningRequest(csr=csr, replace=replace) callback = EventedCallback(return_arg_name="response") self._mqtt_pipeline.send_certificate_signing_request(request=request, callback=callback) diff --git a/samples/cert-mgmt/certificate_issuance.py b/samples/cert-mgmt/certificate_issuance.py index f516e29d7..e7335d8fe 100644 --- a/samples/cert-mgmt/certificate_issuance.py +++ b/samples/cert-mgmt/certificate_issuance.py @@ -11,7 +11,7 @@ from azure.iot.device import Message import uuid from azure.iot.device import X509 -from azure.iot.device.iothub.models import CertificateSigningRequest + import logging import sys @@ -141,9 +141,9 @@ async def main(): print("Performing Azure IoT Hub certificate re-issuance") # Get new issued certificate from IoT Hub - csr_request = CertificateSigningRequest(iothub_csr_data, "*") - - csr_response = await device_client.send_certificate_signing_request(csr_request) + csr_response = await device_client.send_certificate_signing_request( + iothub_csr_data, "*" + ) print( "IoT Hub certificate re-issuance completed. Status-code={}".format( csr_response.status_code diff --git a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py index 65b4ed4b3..19760ab11 100644 --- a/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py +++ b/tests/e2e/provisioning_e2e/tests/test_async_dps_cert_mgmt.py @@ -7,7 +7,7 @@ from provisioning_e2e.service_helper import Helper, connection_string_to_hostname from azure.iot.device.aio import ProvisioningDeviceClient, IoTHubDeviceClient from azure.iot.device.common import X509 -from azure.iot.device.iothub.models import CertificateSigningRequest + from dev_utils.provisioningservice.protocol import models from dev_utils.provisioningservice.client import ProvisioningServiceClient @@ -148,8 +148,9 @@ async def test_group_of_devices_register_with_no_device_id_for_a_x509_intermedia csr_private_key, iot_csr_file, registration_result.registration_state.device_id ) - iot_hub_csr_request = CertificateSigningRequest(read_csr_from_file(iot_csr_file), "*") - csr_response = await device_client.send_certificate_signing_request(iot_hub_csr_request) + csr_response = await device_client.send_certificate_signing_request( + read_csr_from_file(iot_csr_file), "*" + ) assert csr_response.status_code == 200 assert len(csr_response.certificates) == 3 # leaf, intermediate and root certs @@ -227,8 +228,9 @@ async def test_device_register_with_client_cert_issuance_for_a_symmetric_key_gro csr_private_key, iot_csr_file, registration_result.registration_state.device_id ) - iot_hub_csr_request = CertificateSigningRequest(read_csr_from_file(iot_csr_file), "*") - csr_response = await device_client.send_certificate_signing_request(iot_hub_csr_request) + csr_response = await device_client.send_certificate_signing_request( + read_csr_from_file(iot_csr_file), "*" + ) assert csr_response.status_code == 200 assert len(csr_response.certificates) == 3 # leaf, intermediate and root certs