From aaf4cf9e73bb0b905133bf13d9fb621992e71534 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 27 Feb 2026 16:23:06 +0100 Subject: [PATCH 01/31] fixing last line duplication in csv apis via recipe --- python-lib/rest_api_recipe_session.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python-lib/rest_api_recipe_session.py b/python-lib/rest_api_recipe_session.py index 8a474ba..a7d5427 100644 --- a/python-lib/rest_api_recipe_session.py +++ b/python-lib/rest_api_recipe_session.py @@ -126,6 +126,7 @@ def retrieve_next_page(self, is_raw_output): page_rows.append(base_row) else: json_response = decode_csv_data(json_response) + is_api_returning_dict = False for row in json_response: base_row = copy.deepcopy(metadata) base_row.update(parse_keys_for_json(row)) From ad19d07f6d1a40edd5d188d0cfa54363b8cacb2e Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 27 Feb 2026 16:23:14 +0100 Subject: [PATCH 02/31] changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a815c09..7f8531f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Version 1.2.7](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.7) - Feature - 2026-02-18 - Detecting dialect for better csv decoding +- Fixing duplication of last line in csv APIs using the recipe ## [Version 1.2.6](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.6) - Feature - 2025-09-24 From 3996901629ae399365dc8815b3711743abcafcae Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 5 Mar 2026 09:56:08 +0100 Subject: [PATCH 03/31] dumping in text mode as last resort --- python-lib/rest_api_recipe_session.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/python-lib/rest_api_recipe_session.py b/python-lib/rest_api_recipe_session.py index a7d5427..fada80a 100644 --- a/python-lib/rest_api_recipe_session.py +++ b/python-lib/rest_api_recipe_session.py @@ -1,7 +1,7 @@ from dataikuapi.utils import DataikuException from rest_api_client import RestAPIClient from safe_logger import SafeLogger -from dku_utils import parse_keys_for_json, get_value_from_path, decode_csv_data, de_NaN +from dku_utils import parse_keys_for_json, get_value_from_path, decode_csv_data, de_NaN, decode_bytes from dku_constants import DKUConstants import copy import json @@ -108,6 +108,7 @@ def retrieve_next_page(self, is_raw_output): # Todo: check api_response key is free and add something overwise base_row = copy.deepcopy(metadata) if is_raw_output: + assert_json(json_response) if is_error_message(json_response): base_row.update(parse_keys_for_json(json_response)) else: @@ -125,9 +126,18 @@ def retrieve_next_page(self, is_raw_output): base_row.update(self.initial_parameter_columns) page_rows.append(base_row) else: - json_response = decode_csv_data(json_response) + decoded_csv_data = decode_csv_data(json_response) is_api_returning_dict = False - for row in json_response: + if not decoded_csv_data and json_response: + logger.warning("Data is not in CSV format. Dumping it in text mode.") + decoded_csv_data = [ + { + DKUConstants.API_RESPONSE_KEY: "{}".format( + decode_bytes(json_response) + ) + } + ] + for row in decoded_csv_data: base_row = copy.deepcopy(metadata) base_row.update(parse_keys_for_json(row)) base_row.update(self.initial_parameter_columns) @@ -182,3 +192,9 @@ def is_error_message(jsons_response): return True else: return False + + +def assert_json(variable_to_check): + if isinstance(variable_to_check, dict) or isinstance(variable_to_check, list): + return + raise Exception("Returned data is not JSON format. Try again with 'Raw JSON output' un-checked.") From 1a4d9fcef954bf43c6da29a97246f7e6f2712ca4 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 5 Mar 2026 09:56:20 +0100 Subject: [PATCH 04/31] beta2 --- python-lib/dku_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index d16088e..fab7777 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -2,6 +2,6 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"] FORM_DATA_BODY_FORMAT = "FORM_DATA" - PLUGIN_VERSION = "1.2.7-beta.1" + PLUGIN_VERSION = "1.2.7-beta.2" RAW_BODY_FORMAT = "RAW" REPONSE_ERROR_KEY = "dku_error" From 7ecb1c9ab4cfa9e89834470a0bb786859740b26c Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 5 Mar 2026 13:50:23 +0100 Subject: [PATCH 05/31] adding mTLS --- python-lib/dku_utils.py | 3 ++- python-lib/rest_api_client.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index 3b65fa4..b16a8f8 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -39,7 +39,8 @@ def get_endpoint_parameters(configuration): "requests_per_minute", "pagination_type", "next_page_url_key", "is_next_page_url_relative", "next_page_url_base", - "top_key", "skip_key", "maximum_number_rows" + "top_key", "skip_key", "maximum_number_rows", + "use_mtls", "mtls_certificate_path", "mtls_key_path", ] parameters = { endpoint_parameter: configuration.get(endpoint_parameter) for endpoint_parameter in endpoint_parameters if configuration.get(endpoint_parameter) is not None diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 170ad41..57a542e 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -59,6 +59,14 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ self.requests_kwargs.update({"verify": False}) else: self.requests_kwargs.update({"verify": True}) + if endpoint.get("use_mtls", False): + mtls_certificate_path = endpoint.get("mtls_certificate_path") + mtls_key_path = endpoint.get("mtls_key_path") + self.requests_kwargs.update( + { + "cert": (mtls_certificate_path, mtls_key_path) + } + ) self.redirect_auth_header = endpoint.get("redirect_auth_header", False) self.timeout = endpoint.get("timeout", -1) if self.timeout > 0: From 157117bf021c1481a6007bbc9b7dd8a151c58d66 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 5 Mar 2026 13:50:32 +0100 Subject: [PATCH 06/31] update recipe for mTLS --- custom-recipes/api-connect/recipe.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index 5842cd0..ee84f57 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -291,6 +291,27 @@ "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'", "defaultValue": false }, + { + "name": "use_mtls", + "label": "Use mTLS", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" + }, { "name": "redirect_auth_header", "label": "Redirect authorization header", From d5a85bfa9c44e33231a071dd1c77d0eb65f8094c Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 5 Mar 2026 13:50:41 +0100 Subject: [PATCH 07/31] upadte connector for mTLS --- .../api-connect_dataset/connector.json | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index 754c507..ee987bc 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -238,6 +238,27 @@ "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'", "defaultValue": false }, + { + "name": "use_mtls", + "label": " ", + "description": "Use mTLS", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" + }, { "name": "redirect_auth_header", "label": " ", From 4d5c5799747e75436cc7476a3d0e1a12e80ccfa5 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 5 Mar 2026 13:51:07 +0100 Subject: [PATCH 08/31] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f8531f..f048b01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Version 1.2.7](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.7) - Feature - 2026-02-18 - Detecting dialect for better csv decoding +- Adding mutual TLS authentication - Fixing duplication of last line in csv APIs using the recipe ## [Version 1.2.6](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.6) - Feature - 2025-09-24 From af9ff5d9c6622d2f8714327321abeb191c7655ff Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 11 Mar 2026 17:00:13 +0100 Subject: [PATCH 09/31] add option overide for csv decoding --- custom-recipes/api-connect/recipe.json | 75 +++++++++++++++++++ .../api-connect_dataset/connector.json | 75 +++++++++++++++++++ .../api-connect_dataset/connector.py | 3 +- python-lib/dku_utils.py | 69 ++++++++++++++--- python-lib/rest_api_recipe_session.py | 5 +- 5 files changed, 214 insertions(+), 13 deletions(-) diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index ee84f57..d590d92 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -312,6 +312,81 @@ "type": "STRING", "visibilityCondition": "model.use_mtls==true" }, + { + "name": "force_csv_parameters", + "label": "Force CSV parameters", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "csv_delimiter", + "label": "Delimiter", + "description": "", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_doublequote", + "label": "Double quote", + "description": "", + "type": "SELECT", + "selectChoices":[ + {"value": null, "label": "Auto detect"}, + {"value": "double_quote", "label": "Double quote"}, + {"value": "not_double_quote", "label": "No double quote"} + ], + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_escapechar", + "label": "Escape char", + "description": "", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_lineterminator", + "label": "Line terminator", + "description": "", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_quotechar", + "label": "Quote char", + "description": "", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_quoting", + "label": "Quote", + "description": "", + "type": "SELECT", + "selectChoices":[ + {"value": null, "label": "Auto detect"}, + {"value": 0, "label": "Minimal"}, + {"value": 1, "label": "All"}, + {"value": 2, "label": "Non numeric"}, + {"value": 3, "label": "None"}, + {"value": 4, "label": "Strings"}, + {"value": 5, "label": "Not null"} + ], + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_skipinitialspace", + "label": "Skip initial space", + "description": "", + "type": "SELECT", + "selectChoices":[ + {"value": null, "label": "Auto detect"}, + {"value": "skip", "label": "Skip"}, + {"value": "not_skip", "label": "Do not skip"} + ], + "visibilityCondition": "model.force_csv_parameters==true" + }, { "name": "redirect_auth_header", "label": "Redirect authorization header", diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index ee987bc..0dac288 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -259,6 +259,81 @@ "type": "STRING", "visibilityCondition": "model.use_mtls==true" }, + { + "name": "force_csv_parameters", + "label": " ", + "description": "Force CSV parameters", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "csv_delimiter", + "label": " ", + "description": "Delimiter", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_doublequote", + "label": " ", + "description": "Double quote", + "type": "SELECT", + "selectChoices":[ + {"value": null, "label": "Auto detect"}, + {"value": "double_quote", "label": "Double quote"}, + {"value": "not_double_quote", "label": "No double quote"} + ], + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_escapechar", + "label": " ", + "description": "Escape char", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_lineterminator", + "label": " ", + "description": "Line terminator", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_quotechar", + "label": " ", + "description": "Quote char", + "type": "STRING", + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_quoting", + "label": " ", + "description": "Quote", + "type": "SELECT", + "selectChoices":[ + {"value": null, "label": "Auto detect"}, + {"value": 0, "label": "Minimal"}, + {"value": 1, "label": "All"}, + {"value": 2, "label": "Non numeric"}, + {"value": 3, "label": "None"}, + {"value": 4, "label": "Strings"}, + {"value": 5, "label": "Not null"} + ], + "visibilityCondition": "model.force_csv_parameters==true" + }, + { + "name": "csv_skipinitialspace", + "label": " ", + "description": "Skip initial space", + "type": "SELECT", + "selectChoices":[ + {"value": null, "label": "Auto detect"}, + {"value": "skip", "label": "Skip"}, + {"value": "not_skip", "label": "Do not skip"} + ], + "visibilityCondition": "model.force_csv_parameters==true" + }, { "name": "redirect_auth_header", "label": " ", diff --git a/python-connectors/api-connect_dataset/connector.py b/python-connectors/api-connect_dataset/connector.py index fc155de..1873a6a 100644 --- a/python-connectors/api-connect_dataset/connector.py +++ b/python-connectors/api-connect_dataset/connector.py @@ -33,6 +33,7 @@ def __init__(self, config, plugin_config): self.raw_output = endpoint_parameters.get("raw_output", None) self.maximum_number_rows = config.get("maximum_number_rows", -1) self.display_metadata = config.get("display_metadata", False) + self.csv_configuration = config def get_read_schema(self): # In this example, we don't specify a schema here, so DSS will infer the schema @@ -60,7 +61,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, record_count += 1 yield self.format_output(data, metadata) else: - csv_data = decode_csv_data(data) + csv_data = decode_csv_data(data, self.csv_configuration) if csv_data: record_count += len(csv_data) for row in csv_data: diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index b16a8f8..22ebc9d 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -41,6 +41,7 @@ def get_endpoint_parameters(configuration): "next_page_url_key", "is_next_page_url_relative", "next_page_url_base", "top_key", "skip_key", "maximum_number_rows", "use_mtls", "mtls_certificate_path", "mtls_key_path", + "force_csv_parameters", "csv_delimiter" ] parameters = { endpoint_parameter: configuration.get(endpoint_parameter) for endpoint_parameter in endpoint_parameters if configuration.get(endpoint_parameter) is not None @@ -167,7 +168,7 @@ def xml_to_json(content): return json_response -def decode_csv_data(data): +def decode_csv_data(data, csv_configuation): import csv import io json_data = None @@ -189,19 +190,67 @@ def decode_csv_data(data): ) except Exception as error: logger.error("Could not sniff csv dialect. Error={}".format(error)) - dialect = "excel" - try: - reader = csv.DictReader( - io.StringIO(data), - dialect=dialect - ) - json_data = list(reader) - except Exception as error: - logger.error("Could not extract csv data. Error={}. Trying method 2.".format(error)) + # dialect = "excel" + dialect = csv.Dialect() + dialect.delimiter = ',' + dialect.quotechar = '"' + dialect.doublequote = True + dialect.skipinitialspace = False + dialect.lineterminator = '\r\n' + dialect.quoting = 0 + dialect = update_csv_dialect(csv_configuation, dialect) + if not csv_configuation.get("force_csv_parameters", False): + # For back compatibility reason, if csv params are not forced, + # we try the old method first. + try: + reader = csv.DictReader( + io.StringIO(data), + dialect=dialect + ) + json_data = list(reader) + except Exception as error: + logger.error("Could not extract csv data. Error={}. Trying method 2.".format(error)) + json_data = decode_csv_data_m2(data, dialect) + else: + logger.error("CSV parameters are forced, trying method 2") json_data = decode_csv_data_m2(data, dialect) return json_data +def update_csv_dialect(config, input_dialect): + if config.get("force_csv_parameters", False): + logger.info("Updating csv parameters with ") + csv_delimiter = config.get("csv_delimiter") + if csv_delimiter: + input_dialect.delimiter = csv_delimiter + logger.info("delimiter={}".format(csv_delimiter)) + csv_doublequote = config.get("csv_doublequote", None) + if csv_doublequote: + input_dialect.doublequote = csv_doublequote == "double_quote" + logger.info("doublequote={}".format(input_dialect.doublequote)) + csv_escapechar = config.get("csv_escapechar", "") + if csv_escapechar: + input_dialect.escapechar = csv_escapechar + logger.info("escapechar={}".format(csv_escapechar)) + csv_lineterminator = config.get("csv_lineterminator", "") + if csv_lineterminator: + input_dialect.lineterminator = csv_lineterminator + logger.info("lineterminator={}".format(csv_lineterminator)) + csv_quotechar = config.get("csv_quotechar", "") + if csv_quotechar: + input_dialect.quotechar = csv_quotechar + logger.info("quotechar={}".format(csv_quotechar)) + csv_quoting = config.get("csv_quoting", None) + if csv_quoting is not None: + input_dialect.quoting = csv_quoting + logger.info("quoting={}".format(csv_quoting)) + csv_skipinitialspace = config.get("csv_skipinitialspace", None) + if csv_skipinitialspace: + input_dialect.skipinitialspace = csv_skipinitialspace == "skip" + logger.info("skipinitialspace={}".format(input_dialect.skipinitialspace)) + return input_dialect + + def decode_csv_data_m2(data, dialect): import csv json_data = None diff --git a/python-lib/rest_api_recipe_session.py b/python-lib/rest_api_recipe_session.py index fada80a..5df182d 100644 --- a/python-lib/rest_api_recipe_session.py +++ b/python-lib/rest_api_recipe_session.py @@ -29,6 +29,7 @@ def __init__(self, custom_key_values, credential_parameters, secure_credentials, self.is_row_limit = (self.maximum_number_rows > 0) self.behaviour_when_error = behaviour_when_error or "add-error-column" self.can_raise = self.behaviour_when_error == "raise" + self.csv_configuration = endpoint_parameters @staticmethod def get_column_to_parameter_dict(parameter_columns, parameter_renamings): @@ -126,7 +127,7 @@ def retrieve_next_page(self, is_raw_output): base_row.update(self.initial_parameter_columns) page_rows.append(base_row) else: - decoded_csv_data = decode_csv_data(json_response) + decoded_csv_data = decode_csv_data(json_response, self.csv_configuration) is_api_returning_dict = False if not decoded_csv_data and json_response: logger.warning("Data is not in CSV format. Dumping it in text mode.") @@ -151,7 +152,7 @@ def format_page_rows(self, data_rows, is_raw_output, metadata=None): page_rows = [] metadata = metadata or {} if type(data_rows) in [str, bytes]: - data_rows = decode_csv_data(data_rows) + data_rows = decode_csv_data(data_rows, self.csv_configuration) if type(data_rows) in [list]: for data_row in data_rows: base_row = copy.deepcopy(self.initial_parameter_columns) From 34f3125e344658531fa64ba64fb417bd76335b5d Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 11 Mar 2026 17:00:20 +0100 Subject: [PATCH 10/31] beta.3 --- python-lib/dku_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index fab7777..8bf962b 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -2,6 +2,6 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"] FORM_DATA_BODY_FORMAT = "FORM_DATA" - PLUGIN_VERSION = "1.2.7-beta.2" + PLUGIN_VERSION = "1.2.7-beta.3" RAW_BODY_FORMAT = "RAW" REPONSE_ERROR_KEY = "dku_error" From 9fe53a8d54e946c20614199baeb5da9567bed93d Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 12 Mar 2026 19:06:28 +0100 Subject: [PATCH 11/31] removing mtls from recipe and connector --- custom-recipes/api-connect/recipe.json | 21 ------------------- .../api-connect_dataset/connector.json | 21 ------------------- 2 files changed, 42 deletions(-) diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index d590d92..cc0fac3 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -291,27 +291,6 @@ "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'", "defaultValue": false }, - { - "name": "use_mtls", - "label": "Use mTLS", - "description": "", - "type": "BOOLEAN", - "defaultValue": false - }, - { - "name": "mtls_certificate_path", - "label": "Path to certificate", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, - { - "name": "mtls_key_path", - "label": "Path to key", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, { "name": "force_csv_parameters", "label": "Force CSV parameters", diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index 0dac288..2f34370 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -238,27 +238,6 @@ "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'", "defaultValue": false }, - { - "name": "use_mtls", - "label": " ", - "description": "Use mTLS", - "type": "BOOLEAN", - "defaultValue": false - }, - { - "name": "mtls_certificate_path", - "label": "Path to certificate", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, - { - "name": "mtls_key_path", - "label": "Path to key", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, { "name": "force_csv_parameters", "label": " ", From 3bf56eaeed9d42a993fea49bec565d67d447c264 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 12 Mar 2026 19:06:59 +0100 Subject: [PATCH 12/31] Moving mtls to secure and normal presets --- parameter-sets/credential/parameter-set.json | 21 +++++++++++++++++++ .../secure-basic/parameter-set.json | 21 +++++++++++++++++++ .../secure-oauth/parameter-set.json | 21 +++++++++++++++++++ python-lib/rest_api_client.py | 14 ++++++++++--- tests/python/integration/test_scenario.py | 4 ++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/parameter-sets/credential/parameter-set.json b/parameter-sets/credential/parameter-set.json index 4196159..63f0e15 100644 --- a/parameter-sets/credential/parameter-set.json +++ b/parameter-sets/credential/parameter-set.json @@ -119,6 +119,27 @@ "label": "User key/values", "description": "User defined keys/values that can be used later in url, query string...", "type": "KEY_VALUE_LIST" + }, + { + "name": "use_mtls", + "label": " ", + "description": "Use mTLS", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" } ] } diff --git a/parameter-sets/secure-basic/parameter-set.json b/parameter-sets/secure-basic/parameter-set.json index 623d24d..e3181da 100644 --- a/parameter-sets/secure-basic/parameter-set.json +++ b/parameter-sets/secure-basic/parameter-set.json @@ -38,6 +38,27 @@ "label": "NTLM" } ] + }, + { + "name": "use_mtls", + "label": " ", + "description": "Use mTLS", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" } ] } diff --git a/parameter-sets/secure-oauth/parameter-set.json b/parameter-sets/secure-oauth/parameter-set.json index 464f9ac..5be17e7 100644 --- a/parameter-sets/secure-oauth/parameter-set.json +++ b/parameter-sets/secure-oauth/parameter-set.json @@ -47,6 +47,27 @@ "label": "Domain", "description": "", "type": "STRING" + }, + { + "name": "use_mtls", + "label": " ", + "description": "Use mTLS", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "", + "type": "STRING", + "visibilityCondition": "model.use_mtls==true" } ] } diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 57a542e..1ee4519 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -59,9 +59,17 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ self.requests_kwargs.update({"verify": False}) else: self.requests_kwargs.update({"verify": True}) - if endpoint.get("use_mtls", False): - mtls_certificate_path = endpoint.get("mtls_certificate_path") - mtls_key_path = endpoint.get("mtls_key_path") + if credential.get("use_mtls", False): + mtls_certificate_path = credential.get("mtls_certificate_path") + mtls_key_path = credential.get("mtls_key_path") + self.requests_kwargs.update( + { + "cert": (mtls_certificate_path, mtls_key_path) + } + ) + if secure_credentials.get("use_mtls", False): + mtls_certificate_path = secure_credentials.get("mtls_certificate_path") + mtls_key_path = secure_credentials.get("mtls_key_path") self.requests_kwargs.update( { "cert": (mtls_certificate_path, mtls_key_path) diff --git a/tests/python/integration/test_scenario.py b/tests/python/integration/test_scenario.py index 63c41af..315b6fc 100644 --- a/tests/python/integration/test_scenario.py +++ b/tests/python/integration/test_scenario.py @@ -57,3 +57,7 @@ def test_run_api_connect_xml_handling(user_dss_clients): def test_run_api_connect_parameters_renaming(user_dss_clients): dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="COLUMNPARAMETERRENAMING") + + +def test_run_api_connect_mtls(user_dss_clients): + dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="MTLS") From 1875efa601e47c793e5490147554e84ac27c6b5d Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 13 Mar 2026 14:48:47 +0100 Subject: [PATCH 13/31] adding warnings --- parameter-sets/credential/parameter-set.json | 4 ++-- parameter-sets/secure-basic/parameter-set.json | 10 ++++++++-- parameter-sets/secure-oauth/parameter-set.json | 10 ++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/parameter-sets/credential/parameter-set.json b/parameter-sets/credential/parameter-set.json index 63f0e15..3035645 100644 --- a/parameter-sets/credential/parameter-set.json +++ b/parameter-sets/credential/parameter-set.json @@ -122,8 +122,8 @@ }, { "name": "use_mtls", - "label": " ", - "description": "Use mTLS", + "label": "Use mTLS", + "description": "", "type": "BOOLEAN", "defaultValue": false }, diff --git a/parameter-sets/secure-basic/parameter-set.json b/parameter-sets/secure-basic/parameter-set.json index e3181da..d54f19d 100644 --- a/parameter-sets/secure-basic/parameter-set.json +++ b/parameter-sets/secure-basic/parameter-set.json @@ -41,11 +41,17 @@ }, { "name": "use_mtls", - "label": " ", - "description": "Use mTLS", + "label": "Use mTLS", + "description": "", "type": "BOOLEAN", "defaultValue": false }, + { + "type": "SEPARATOR", + "label": "Warning", + "description": "Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.", + "visibilityCondition": "model.use_mtls==true" + }, { "name": "mtls_certificate_path", "label": "Path to certificate", diff --git a/parameter-sets/secure-oauth/parameter-set.json b/parameter-sets/secure-oauth/parameter-set.json index 5be17e7..79da6d0 100644 --- a/parameter-sets/secure-oauth/parameter-set.json +++ b/parameter-sets/secure-oauth/parameter-set.json @@ -50,11 +50,17 @@ }, { "name": "use_mtls", - "label": " ", - "description": "Use mTLS", + "label": "Use mTLS", + "description": "", "type": "BOOLEAN", "defaultValue": false }, + { + "type": "SEPARATOR", + "label": "Warning", + "description": "1 - Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.\n2 - Because the OAuth flow is not controled by the plugin, mTLS cannot be used for the retrieving the access token itself.", + "visibilityCondition": "model.use_mtls==true" + }, { "name": "mtls_certificate_path", "label": "Path to certificate", From bd623021c9406070101b9ff728dc991293c0da0d Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 13 Mar 2026 16:14:50 +0100 Subject: [PATCH 14/31] beta.4 --- python-lib/dku_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index 8bf962b..b20ccde 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -2,6 +2,6 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"] FORM_DATA_BODY_FORMAT = "FORM_DATA" - PLUGIN_VERSION = "1.2.7-beta.3" + PLUGIN_VERSION = "1.2.7-beta.4" RAW_BODY_FORMAT = "RAW" REPONSE_ERROR_KEY = "dku_error" From 4cf349067e2cb627474611bfc521973d2cdbf95f Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 16 Mar 2026 17:12:45 +0100 Subject: [PATCH 15/31] update tags and category --- plugin.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugin.json b/plugin.json index afb4273..88ad213 100644 --- a/plugin.json +++ b/plugin.json @@ -6,8 +6,7 @@ "description": "Retrieve data from any REST API", "author": "Dataiku (Alex Bourret)", "icon": "icon-rocket", - "category": "Connect", - "tags": ["API", "Recipe", "Dataset"], + "tags": ["Connector"], "url": "https://www.dataiku.com/product/plugins/api-connect/", "licenseInfo": "Apache Software License", "recipesCategory": "visual" From c5a0c20662b8bc3a7ca61301fe8b72f41bfdb3ed Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Tue, 24 Mar 2026 16:21:57 +0100 Subject: [PATCH 16/31] removing beta tag --- python-lib/dku_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index b20ccde..3736450 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -2,6 +2,6 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"] FORM_DATA_BODY_FORMAT = "FORM_DATA" - PLUGIN_VERSION = "1.2.7-beta.4" + PLUGIN_VERSION = "1.2.7" RAW_BODY_FORMAT = "RAW" REPONSE_ERROR_KEY = "dku_error" From 38af12d03dcbcd39cf02acf41d9adb0382df6c00 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 2 Apr 2026 12:20:20 +0200 Subject: [PATCH 17/31] storing key+certificats as passwords --- parameter-sets/credential/parameter-set.json | 8 ++++---- parameter-sets/secure-basic/parameter-set.json | 8 ++++---- parameter-sets/secure-oauth/parameter-set.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/parameter-sets/credential/parameter-set.json b/parameter-sets/credential/parameter-set.json index 3035645..b5202c1 100644 --- a/parameter-sets/credential/parameter-set.json +++ b/parameter-sets/credential/parameter-set.json @@ -130,15 +130,15 @@ { "name": "mtls_certificate_path", "label": "Path to certificate", - "description": "", - "type": "STRING", + "description": "or full certificate from -----BEGIN to END CERTIFICATE-----", + "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" }, { "name": "mtls_key_path", "label": "Path to key", - "description": "", - "type": "STRING", + "description": "or full key from -----BEGIN to END PRIVATE KEY-----", + "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" } ] diff --git a/parameter-sets/secure-basic/parameter-set.json b/parameter-sets/secure-basic/parameter-set.json index d54f19d..6c1fb58 100644 --- a/parameter-sets/secure-basic/parameter-set.json +++ b/parameter-sets/secure-basic/parameter-set.json @@ -55,15 +55,15 @@ { "name": "mtls_certificate_path", "label": "Path to certificate", - "description": "", - "type": "STRING", + "description": "or full certificate from -----BEGIN to END CERTIFICATE-----", + "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" }, { "name": "mtls_key_path", "label": "Path to key", - "description": "", - "type": "STRING", + "description": "or full key from -----BEGIN to END PRIVATE KEY-----", + "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" } ] diff --git a/parameter-sets/secure-oauth/parameter-set.json b/parameter-sets/secure-oauth/parameter-set.json index 79da6d0..947f096 100644 --- a/parameter-sets/secure-oauth/parameter-set.json +++ b/parameter-sets/secure-oauth/parameter-set.json @@ -64,15 +64,15 @@ { "name": "mtls_certificate_path", "label": "Path to certificate", - "description": "", - "type": "STRING", + "description": "or full certificate from -----BEGIN to END CERTIFICATE-----", + "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" }, { "name": "mtls_key_path", "label": "Path to key", - "description": "", - "type": "STRING", + "description": "or full key from -----BEGIN to END PRIVATE KEY-----", + "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" } ] From 6252b88cd2594758fc87340ec2d5caba72d40f04 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 2 Apr 2026 12:20:54 +0200 Subject: [PATCH 18/31] using temp file for using key+certificats stored in presets --- python-lib/rest_api_client.py | 40 +++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 1ee4519..49ffd08 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -1,6 +1,7 @@ import requests import time import copy +import tempfile from pagination import Pagination from safe_logger import SafeLogger from loop_detector import LoopDetector @@ -184,14 +185,35 @@ def request(self, method, url, can_raise_exeption=True, **kwargs): def request_with_redirect_retry(self, method, url, **kwargs): # In case of redirection to another domain, the authorization header is not kept # If redirect_auth_header is true, another attempt is made with initial headers to the redirected url - response = self.session.request(method, url, **kwargs) + response = self.request_with_cert(method, url, **kwargs) if self.redirect_auth_header and not response.url.startswith(url): redirection_kwargs = copy.deepcopy(kwargs) redirection_kwargs.pop("params", None) # params are contained in the redirected url logger.warning("Redirection ! Accessing endpoint {} with initial authorization headers".format(response.url)) - response = self.session.request(method, response.url, **redirection_kwargs) + response = self.request_with_cert(method, response.url, **redirection_kwargs) return response + def request_with_cert(self, method, url, **kwargs): + cert = kwargs.get("cert", None) + if cert and len(cert) == 2: + if cert[0].startswith("-----BEGIN CERTIFICATE") and cert[1].startswith("-----BEGIN PRIVATE KEY"): + logger.info("mTLS certificate and key are strings") + response = None + with tempfile.NamedTemporaryFile(mode="w", suffix=".crt") as tmp_certificate: + with tempfile.NamedTemporaryFile(mode="w", suffix=".key") as tmp_key: + tmp_certificate.write( + normalize_key(cert[0]) + ) + tmp_certificate.seek(0) + tmp_key.write( + normalize_key(cert[1]) + ) + tmp_key.seek(0) + kwargs["cert"] = (tmp_certificate.name, tmp_key.name) + response = self.session.request(method, url, **kwargs) + return response + return self.session.request(method, url, **kwargs) + def paginated_api_call(self, can_raise_exeption=True): if self.pagination.params_must_be_blanked: self.requests_kwargs["params"] = {} @@ -278,3 +300,17 @@ def get_headers(response): if isinstance(response, requests.Response): return response.headers return None + + +def normalize_key(key): + tempo_text = str(key) + tempo_text = tempo_text.replace("BEGIN CERTIFICATE", "BEGINCERTIFICATE") + tempo_text = tempo_text.replace("END CERTIFICATE", "ENDCERTIFICATE") + tempo_text = tempo_text.replace("-----BEGIN PRIVATE KEY-----", "-----BEGINPRIVATEKEY-----") + tempo_text = tempo_text.replace("-----END PRIVATE KEY-----", "-----ENDPRIVATEKEY-----") + tempo_text = tempo_text.replace(" ", "\n") + tempo_text = tempo_text.replace("BEGINCERTIFICATE", "BEGIN CERTIFICATE") + tempo_text = tempo_text.replace("ENDCERTIFICATE", "END CERTIFICATE") + tempo_text = tempo_text.replace("-----BEGINPRIVATEKEY-----", "-----BEGIN PRIVATE KEY-----") + tempo_text = tempo_text.replace("-----ENDPRIVATEKEY-----", "-----END PRIVATE KEY-----") + return tempo_text From 734dd6c041e734092c9060bbc80294253f276ba7 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 2 Apr 2026 12:21:29 +0200 Subject: [PATCH 19/31] adding mtls keys to data that should not be displayed --- python-lib/dku_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index 3736450..ffab18d 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -1,6 +1,6 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" - FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"] + FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token", "mtls_key_path", "mtls_certificate_path"] FORM_DATA_BODY_FORMAT = "FORM_DATA" PLUGIN_VERSION = "1.2.7" RAW_BODY_FORMAT = "RAW" From 987c597f1620e453a40e8d3fc1e05d66777c6839 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 2 Apr 2026 13:56:47 +0200 Subject: [PATCH 20/31] beta.5 --- python-lib/dku_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index ffab18d..7e007f8 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -2,6 +2,6 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token", "mtls_key_path", "mtls_certificate_path"] FORM_DATA_BODY_FORMAT = "FORM_DATA" - PLUGIN_VERSION = "1.2.7" + PLUGIN_VERSION = "1.2.7-beta.5" RAW_BODY_FORMAT = "RAW" REPONSE_ERROR_KEY = "dku_error" From 5dc9987bbacb21f12f5e216f1824de455724c3e3 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 15 Apr 2026 10:10:25 +0200 Subject: [PATCH 21/31] removing json assertion, dumping response if json.dumps fails --- python-lib/rest_api_recipe_session.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/python-lib/rest_api_recipe_session.py b/python-lib/rest_api_recipe_session.py index fada80a..8c82f78 100644 --- a/python-lib/rest_api_recipe_session.py +++ b/python-lib/rest_api_recipe_session.py @@ -108,13 +108,17 @@ def retrieve_next_page(self, is_raw_output): # Todo: check api_response key is free and add something overwise base_row = copy.deepcopy(metadata) if is_raw_output: - assert_json(json_response) if is_error_message(json_response): base_row.update(parse_keys_for_json(json_response)) else: - base_row.update({ - DKUConstants.API_RESPONSE_KEY: json.dumps(json_response) - }) + try: + base_row.update({ + DKUConstants.API_RESPONSE_KEY: json.dumps(json_response) + }) + except Exception: + base_row.update({ + DKUConstants.API_RESPONSE_KEY: decode_bytes(json_response) + }) else: if isinstance(json_response, dict): base_row.update(parse_keys_for_json(json_response)) @@ -192,9 +196,3 @@ def is_error_message(jsons_response): return True else: return False - - -def assert_json(variable_to_check): - if isinstance(variable_to_check, dict) or isinstance(variable_to_check, list): - return - raise Exception("Returned data is not JSON format. Try again with 'Raw JSON output' un-checked.") From c1e81b0be4a426d3dcc1117902e84873df1d0e9b Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 15 Apr 2026 10:10:37 +0200 Subject: [PATCH 22/31] refacto --- python-connectors/api-connect_dataset/connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-connectors/api-connect_dataset/connector.py b/python-connectors/api-connect_dataset/connector.py index fc155de..59d382f 100644 --- a/python-connectors/api-connect_dataset/connector.py +++ b/python-connectors/api-connect_dataset/connector.py @@ -68,7 +68,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, else: record_count += 1 yield { - DKUConstants.API_RESPONSE_KEY: "{}".format(decode_bytes(data)) + DKUConstants.API_RESPONSE_KEY: decode_bytes(data) } if is_records_limit and record_count >= records_limit: break From 7bc0c4932e36f5dbf32f0b448a892f326af3a6b9 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 15 Apr 2026 10:10:46 +0200 Subject: [PATCH 23/31] changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f8531f..5248b19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Detecting dialect for better csv decoding - Fixing duplication of last line in csv APIs using the recipe +- Dumping API's response as a last resort ## [Version 1.2.6](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.6) - Feature - 2025-09-24 From 581a267a0dc3f936b096bbff2bdc2544fd164c08 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 16 Apr 2026 13:38:35 +0200 Subject: [PATCH 24/31] Clarify the UI --- parameter-sets/credential/parameter-set.json | 4 ++-- parameter-sets/secure-basic/parameter-set.json | 4 ++-- parameter-sets/secure-oauth/parameter-set.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parameter-sets/credential/parameter-set.json b/parameter-sets/credential/parameter-set.json index b5202c1..0c027f6 100644 --- a/parameter-sets/credential/parameter-set.json +++ b/parameter-sets/credential/parameter-set.json @@ -130,14 +130,14 @@ { "name": "mtls_certificate_path", "label": "Path to certificate", - "description": "or full certificate from -----BEGIN to END CERTIFICATE-----", + "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----", "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" }, { "name": "mtls_key_path", "label": "Path to key", - "description": "or full key from -----BEGIN to END PRIVATE KEY-----", + "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----", "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" } diff --git a/parameter-sets/secure-basic/parameter-set.json b/parameter-sets/secure-basic/parameter-set.json index 6c1fb58..a6b6809 100644 --- a/parameter-sets/secure-basic/parameter-set.json +++ b/parameter-sets/secure-basic/parameter-set.json @@ -55,14 +55,14 @@ { "name": "mtls_certificate_path", "label": "Path to certificate", - "description": "or full certificate from -----BEGIN to END CERTIFICATE-----", + "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----", "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" }, { "name": "mtls_key_path", "label": "Path to key", - "description": "or full key from -----BEGIN to END PRIVATE KEY-----", + "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----", "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" } diff --git a/parameter-sets/secure-oauth/parameter-set.json b/parameter-sets/secure-oauth/parameter-set.json index 947f096..72c6437 100644 --- a/parameter-sets/secure-oauth/parameter-set.json +++ b/parameter-sets/secure-oauth/parameter-set.json @@ -64,14 +64,14 @@ { "name": "mtls_certificate_path", "label": "Path to certificate", - "description": "or full certificate from -----BEGIN to END CERTIFICATE-----", + "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----", "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" }, { "name": "mtls_key_path", "label": "Path to key", - "description": "or full key from -----BEGIN to END PRIVATE KEY-----", + "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----", "type": "PASSWORD", "visibilityCondition": "model.use_mtls==true" } From 1cfed4416c356fbd379b65d9f28b480f48ec5912 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 16 Apr 2026 13:38:53 +0200 Subject: [PATCH 25/31] Add normalization for RSA keys --- python-lib/rest_api_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 49ffd08..9609151 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -308,7 +308,11 @@ def normalize_key(key): tempo_text = tempo_text.replace("END CERTIFICATE", "ENDCERTIFICATE") tempo_text = tempo_text.replace("-----BEGIN PRIVATE KEY-----", "-----BEGINPRIVATEKEY-----") tempo_text = tempo_text.replace("-----END PRIVATE KEY-----", "-----ENDPRIVATEKEY-----") + tempo_text = tempo_text.replace("BEGIN RSA PRIVATE KEY", "BEGINRSAPRIVATEKEY") + tempo_text = tempo_text.replace("END RSA PRIVATE KEY", "ENDRSAPRIVATEKEY") tempo_text = tempo_text.replace(" ", "\n") + tempo_text = tempo_text.replace("BEGINRSAPRIVATEKEY", "BEGIN RSA PRIVATE KEY") + tempo_text = tempo_text.replace("ENDRSAPRIVATEKEY", "END RSA PRIVATE KEY") tempo_text = tempo_text.replace("BEGINCERTIFICATE", "BEGIN CERTIFICATE") tempo_text = tempo_text.replace("ENDCERTIFICATE", "END CERTIFICATE") tempo_text = tempo_text.replace("-----BEGINPRIVATEKEY-----", "-----BEGIN PRIVATE KEY-----") From 59597ce669c0c8cbb66c5281196942cbea77f9f0 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 16 Apr 2026 14:00:27 +0200 Subject: [PATCH 26/31] fix to accept RSA keys --- python-lib/rest_api_client.py | 38 +++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 9609151..12ff0a2 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -196,7 +196,7 @@ def request_with_redirect_retry(self, method, url, **kwargs): def request_with_cert(self, method, url, **kwargs): cert = kwargs.get("cert", None) if cert and len(cert) == 2: - if cert[0].startswith("-----BEGIN CERTIFICATE") and cert[1].startswith("-----BEGIN PRIVATE KEY"): + if cert[0].startswith("-----BEGIN CERTIFICATE") and cert[1].startswith("-----BEGIN "): logger.info("mTLS certificate and key are strings") response = None with tempfile.NamedTemporaryFile(mode="w", suffix=".crt") as tmp_certificate: @@ -303,18 +303,30 @@ def get_headers(response): def normalize_key(key): + PROTECTED_EXPRESSIONS = [ + "BEGIN CERTIFICATE", "END CERTIFICATE", + "BEGIN PRIVATE KEY", "END PRIVATE KEY", + "BEGIN RSA PRIVATE KEY", "END RSA PRIVATE KEY" + ] tempo_text = str(key) - tempo_text = tempo_text.replace("BEGIN CERTIFICATE", "BEGINCERTIFICATE") - tempo_text = tempo_text.replace("END CERTIFICATE", "ENDCERTIFICATE") - tempo_text = tempo_text.replace("-----BEGIN PRIVATE KEY-----", "-----BEGINPRIVATEKEY-----") - tempo_text = tempo_text.replace("-----END PRIVATE KEY-----", "-----ENDPRIVATEKEY-----") - tempo_text = tempo_text.replace("BEGIN RSA PRIVATE KEY", "BEGINRSAPRIVATEKEY") - tempo_text = tempo_text.replace("END RSA PRIVATE KEY", "ENDRSAPRIVATEKEY") + for expression_to_protect in PROTECTED_EXPRESSIONS: + protected_form = expression_to_protect.replace(" ", "") + tempo_text = tempo_text.replace(expression_to_protect, protected_form) tempo_text = tempo_text.replace(" ", "\n") - tempo_text = tempo_text.replace("BEGINRSAPRIVATEKEY", "BEGIN RSA PRIVATE KEY") - tempo_text = tempo_text.replace("ENDRSAPRIVATEKEY", "END RSA PRIVATE KEY") - tempo_text = tempo_text.replace("BEGINCERTIFICATE", "BEGIN CERTIFICATE") - tempo_text = tempo_text.replace("ENDCERTIFICATE", "END CERTIFICATE") - tempo_text = tempo_text.replace("-----BEGINPRIVATEKEY-----", "-----BEGIN PRIVATE KEY-----") - tempo_text = tempo_text.replace("-----ENDPRIVATEKEY-----", "-----END PRIVATE KEY-----") + for expression_to_protect in PROTECTED_EXPRESSIONS: + protected_form = expression_to_protect.replace(" ", "") + tempo_text = tempo_text.replace(protected_form, expression_to_protect) + # tempo_text = tempo_text.replace("BEGIN CERTIFICATE", "BEGINCERTIFICATE") + # tempo_text = tempo_text.replace("END CERTIFICATE", "ENDCERTIFICATE") + # tempo_text = tempo_text.replace("-----BEGIN PRIVATE KEY-----", "-----BEGINPRIVATEKEY-----") + # tempo_text = tempo_text.replace("-----END PRIVATE KEY-----", "-----ENDPRIVATEKEY-----") + # tempo_text = tempo_text.replace("BEGIN RSA PRIVATE KEY", "BEGINRSAPRIVATEKEY") + # tempo_text = tempo_text.replace("END RSA PRIVATE KEY", "ENDRSAPRIVATEKEY") + # tempo_text = tempo_text.replace(" ", "\n") + # tempo_text = tempo_text.replace("BEGINRSAPRIVATEKEY", "BEGIN RSA PRIVATE KEY") + # tempo_text = tempo_text.replace("ENDRSAPRIVATEKEY", "END RSA PRIVATE KEY") + # tempo_text = tempo_text.replace("BEGINCERTIFICATE", "BEGIN CERTIFICATE") + # tempo_text = tempo_text.replace("ENDCERTIFICATE", "END CERTIFICATE") + # tempo_text = tempo_text.replace("-----BEGINPRIVATEKEY-----", "-----BEGIN PRIVATE KEY-----") + # tempo_text = tempo_text.replace("-----ENDPRIVATEKEY-----", "-----END PRIVATE KEY-----") return tempo_text From b807d8573bbfd2684f6a7f0916dea91720704bc3 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 16 Apr 2026 14:01:18 +0200 Subject: [PATCH 27/31] removing comments --- python-lib/rest_api_client.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 12ff0a2..cb4e824 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -316,17 +316,4 @@ def normalize_key(key): for expression_to_protect in PROTECTED_EXPRESSIONS: protected_form = expression_to_protect.replace(" ", "") tempo_text = tempo_text.replace(protected_form, expression_to_protect) - # tempo_text = tempo_text.replace("BEGIN CERTIFICATE", "BEGINCERTIFICATE") - # tempo_text = tempo_text.replace("END CERTIFICATE", "ENDCERTIFICATE") - # tempo_text = tempo_text.replace("-----BEGIN PRIVATE KEY-----", "-----BEGINPRIVATEKEY-----") - # tempo_text = tempo_text.replace("-----END PRIVATE KEY-----", "-----ENDPRIVATEKEY-----") - # tempo_text = tempo_text.replace("BEGIN RSA PRIVATE KEY", "BEGINRSAPRIVATEKEY") - # tempo_text = tempo_text.replace("END RSA PRIVATE KEY", "ENDRSAPRIVATEKEY") - # tempo_text = tempo_text.replace(" ", "\n") - # tempo_text = tempo_text.replace("BEGINRSAPRIVATEKEY", "BEGIN RSA PRIVATE KEY") - # tempo_text = tempo_text.replace("ENDRSAPRIVATEKEY", "END RSA PRIVATE KEY") - # tempo_text = tempo_text.replace("BEGINCERTIFICATE", "BEGIN CERTIFICATE") - # tempo_text = tempo_text.replace("ENDCERTIFICATE", "END CERTIFICATE") - # tempo_text = tempo_text.replace("-----BEGINPRIVATEKEY-----", "-----BEGIN PRIVATE KEY-----") - # tempo_text = tempo_text.replace("-----ENDPRIVATEKEY-----", "-----END PRIVATE KEY-----") return tempo_text From 98d8ccd3e0ed104b2e73acf9d46fafa9076bff65 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 7 May 2026 14:38:48 +0200 Subject: [PATCH 28/31] simplify --- python-lib/rest_api_recipe_session.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python-lib/rest_api_recipe_session.py b/python-lib/rest_api_recipe_session.py index 8c82f78..239d5bb 100644 --- a/python-lib/rest_api_recipe_session.py +++ b/python-lib/rest_api_recipe_session.py @@ -136,9 +136,7 @@ def retrieve_next_page(self, is_raw_output): logger.warning("Data is not in CSV format. Dumping it in text mode.") decoded_csv_data = [ { - DKUConstants.API_RESPONSE_KEY: "{}".format( - decode_bytes(json_response) - ) + DKUConstants.API_RESPONSE_KEY: decode_bytes(json_response) } ] for row in decoded_csv_data: From 185915b1c04171be1ce89103c60ba5512db8403c Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 13 May 2026 13:34:59 +0200 Subject: [PATCH 29/31] fix get_endpoint_parameters --- python-lib/dku_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index 22ebc9d..e9b7087 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -41,7 +41,9 @@ def get_endpoint_parameters(configuration): "next_page_url_key", "is_next_page_url_relative", "next_page_url_base", "top_key", "skip_key", "maximum_number_rows", "use_mtls", "mtls_certificate_path", "mtls_key_path", - "force_csv_parameters", "csv_delimiter" + "force_csv_parameters", "csv_delimiter", + "csv_doublequote", "csv_escapechar", "csv_lineterminator", + "csv_quotechar", "csv_quoting", "csv_skipinitialspace" ] parameters = { endpoint_parameter: configuration.get(endpoint_parameter) for endpoint_parameter in endpoint_parameters if configuration.get(endpoint_parameter) is not None From bba57519ba028c4378e95f9c8c95acd87fd39314 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 13 May 2026 14:09:12 +0200 Subject: [PATCH 30/31] UI: moving above the csv parameters names --- .../api-connect_dataset/connector.json | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index 0dac288..4921bc8 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -268,15 +268,15 @@ }, { "name": "csv_delimiter", - "label": " ", - "description": "Delimiter", + "label": "Delimiter", + "description": "", "type": "STRING", "visibilityCondition": "model.force_csv_parameters==true" }, { "name": "csv_doublequote", - "label": " ", - "description": "Double quote", + "label": "Double quote", + "description": "", "type": "SELECT", "selectChoices":[ {"value": null, "label": "Auto detect"}, @@ -287,29 +287,29 @@ }, { "name": "csv_escapechar", - "label": " ", - "description": "Escape char", + "label": "Escape char", + "description": "", "type": "STRING", "visibilityCondition": "model.force_csv_parameters==true" }, { "name": "csv_lineterminator", - "label": " ", - "description": "Line terminator", + "label": "Line terminator", + "description": "", "type": "STRING", "visibilityCondition": "model.force_csv_parameters==true" }, { "name": "csv_quotechar", - "label": " ", - "description": "Quote char", + "label": "Quote char", + "description": "", "type": "STRING", "visibilityCondition": "model.force_csv_parameters==true" }, { "name": "csv_quoting", - "label": " ", - "description": "Quote", + "label": "Quote", + "description": "", "type": "SELECT", "selectChoices":[ {"value": null, "label": "Auto detect"}, @@ -324,8 +324,8 @@ }, { "name": "csv_skipinitialspace", - "label": " ", - "description": "Skip initial space", + "label": "Skip initial space", + "description": "", "type": "SELECT", "selectChoices":[ {"value": null, "label": "Auto detect"}, From a39cb711d702953ff02dc4a26a9be6b64167c886 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 18 May 2026 09:19:45 +0200 Subject: [PATCH 31/31] fix log --- python-lib/dku_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index e9b7087..d9f59a6 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -184,7 +184,7 @@ def decode_csv_data(data, csv_configuation): dialect.delimiter, dialect.doublequote, dialect.escapechar, - dialect.lineterminator, + dialect.lineterminator.encode(encoding="utf-8"), dialect.quotechar, dialect.quoting, dialect.skipinitialspace