Apply csv and timezone handling changes to 10.4

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "v10.4.x" ]
     paths-ignore:
       - '.github/workflows/pylint.yml'
       - '.github/workflows/release.yml'
@@ -11,7 +11,7 @@ on:
       - 'README.md'
       - 'DEVELOPMENT.md'
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "v10.4.x" ]
     paths-ignore:
       - '.github/workflows/pylint.yml'
       - '.github/workflows/release.yml'
@@ -27,7 +27,6 @@ jobs:
       matrix:
         python-version: ${{ fromJSON(vars.CI_PYTHON_VERSIONS) }}
         # build for (last 5) major supported versions to ensure compatibility to some degree, but really recommended, linted and deployed is only the latest
-        # due to csv-writing issues with versions below 3.12, versions before 3.12 are not supported
 
 
     steps:
@@ -43,3 +42,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install .
+        pip install pytest
+
+    - name: Run tests
+      run: |
+        pytest src/cadenzaanalytics/tests/ -v

.github/workflows/pylint.yml

@@ -2,7 +2,7 @@ name: Pylint
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "v10.4.x" ]
     paths-ignore:
       - '.github/workflows/ci.yml'
       - '.github/workflows/release.yml'
@@ -11,7 +11,7 @@ on:
       - 'README.md'
       - 'DEVELOPMENT.md'
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "v10.4.x" ]
     paths-ignore:
       - '.github/workflows/ci.yml'
       - '.github/workflows/release.yml'

CHANGELOG.md

@@ -5,6 +5,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Changed
+- Improved type annotations for mappings and parameter values
+- Uses custom csv handling instead of pandas csv handling to fix various edge cases
+- Minimum required python version reduced to 3.11
+- values of data type `ZONED_DATE_TIME` are now received as pandas Timestamps and normalized to UTC
+
+### Fixed
+- the `basic-extension` example enrichment now actually enriches the data
+
 ## 10.4.0 - 2025-12-05
 ### Added
 - New visual response of type text (TextResponse)

examples/enrichment/extension/example_extensions.py

@@ -1,14 +1,12 @@
 """Example module for running a disy Cadenza analytics extension that
  will execute a basic enrichment."""
 
-import pandas as pd
-
 import cadenzaanalytics as ca
 
 
 def enrichment_basic_analytics_function(request: ca.AnalyticsRequest):
     # pylint: disable=unused-argument
-    df_data = pd.DataFrame()
+    df_data = request["table"].data
     df_data["new_value"] = "value"
 
     result_metadata = [

pyproject.toml

@@ -22,14 +22,15 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.12"
+python = "^3.10"
 Flask = "3.1.2"
 Werkzeug = "3.1.4"
 Flask-Cors = "6.0.1"
 requests-toolbelt = "1.0.0"
 pandas = " ^2.0.2"
 chardet = "5.2.0"
 Shapely = "2.1.2"
+pytest = "9.0.2"
 
 [project]
 name = "cadenzaanalytics"

src/cadenzaanalytics/cadenza_analytics_extension.py

@@ -3,13 +3,9 @@
 invoked via HTTP POST on the relative path."""
 import json
 import logging
-from io import StringIO
 from typing import Callable, List, Optional
 
-import numpy as np
-import pandas as pd
 from flask import Response, request
-from shapely import from_wkt
 
 from cadenzaanalytics.data.analytics_extension import AnalyticsExtension
 from cadenzaanalytics.data.extension_type import ExtensionType
@@ -21,6 +17,7 @@
 from cadenzaanalytics.request.request_metadata import RequestMetadata
 from cadenzaanalytics.request.request_table import RequestTable
 from cadenzaanalytics.response.extension_response import ExtensionResponse
+from cadenzaanalytics.util.csv import from_cadenza_csv
 
 
 logger = logging.getLogger('cadenzaanalytics')
@@ -151,51 +148,26 @@ def _get_request_data(self, multipart_request) -> AnalyticsRequest:
         if len(metadata.columns) > 0:
             has_data = True
             type_mapping = {}
-            na_values_mapping = {}
             datetime_columns = []
             geometry_columns = []
 
             for column in metadata.columns:
                 if column.data_type == DataType.ZONEDDATETIME:
                     datetime_columns.append(column.name)
-                    # must be empty list, otherwise pd.read_csv interprets empty strings as NA which
-                    # is rejected by the parse_dates mechanism before it reaches the _parse_datetime function
-                    na_values_mapping[column.name] = []
-                elif column.data_type == DataType.STRING:
-                    # only empty strings must be considered as NA
-                    # unfortunately there does not seem to be a way to interpret empty quotes as empty string
-                    # and unquoted as None
-                    na_values_mapping[column.name] = ['']
-                else:
-                    # pandas default list of NA values, mostly relevant for numeric columns
-                    na_values_mapping[column.name] = ['-1.#IND', '1.#QNAN', '1.#IND', '-1.#QNAN', '#N/A N/A',
-                                                      '#N/A', 'N/A', 'n/a', 'NA', '<NA>', '#NA', 'NULL', 'null',
-                                                      'NaN', '-NaN', 'nan', '-nan', 'None', '']
-
-                if column.data_type == DataType.GEOMETRY:
+                elif column.data_type == DataType.GEOMETRY:
                     geometry_columns.append(column.name)
 
                 type_mapping[column.name] = column.data_type.pandas_type()
 
-            csv_data = StringIO(self._get_from_request(multipart_request, 'data'))
-            # read_csv cannot distinguish None from empty strings
-            df_data = pd.read_csv(
+            csv_data = self._get_from_request(multipart_request, 'data')
+            # Use custom parser that properly handles quoted vs unquoted values
+            df_data = from_cadenza_csv(
                 csv_data,
-                sep=';',
-                dtype=type_mapping,
-                parse_dates=datetime_columns,
-                date_format='ISO8601',
-                na_values=na_values_mapping,
-                keep_default_na=False,
+                type_mapping=type_mapping,
+                datetime_columns=datetime_columns,
+                geometry_columns=geometry_columns
             )
 
-            # Parse WKT geometries into shapely geometry objects using vectorized from_wkt
-            for gcol in geometry_columns:
-                values = df_data[gcol].to_numpy()
-                # from_wkt handles None values; replace empty strings with None
-                values = np.where((values == '') | pd.isna(values), None, values)
-                df_data[gcol] = from_wkt(values, on_invalid='warn')
-
             logger.debug('Received data:\n%s', df_data.head())
         else:
             has_data = False

src/cadenzaanalytics/data/parameter.py

@@ -1,8 +1,9 @@
-from typing import List, Any, Optional
+from typing import List, Optional
 
 from cadenzaanalytics.data.geometry_type import GeometryType
 from cadenzaanalytics.data.data_object import DataObject
 from cadenzaanalytics.data.parameter_type import ParameterType
+from cadenzaanalytics.data.parameter_value_type import ParameterValueType
 
 
 # pylint: disable=too-many-instance-attributes
@@ -31,7 +32,7 @@ def __init__(self, *,
                  geometry_types: Optional[List[GeometryType]] = None,
                  options: Optional[List[str]] = None,
                  required: bool = False,
-                 default_value: Any = None,
+                 default_value: Optional[ParameterValueType] = None,
                  requested_srs: Optional[str] = None) -> None:
         """Initialize a Parameter.
 
@@ -51,7 +52,7 @@ def __init__(self, *,
             Whether the parameter is required, by default False.
             For parameter type boolean, required=True makes submitting the value
             True mandatory.
-        default_value : Any, optional
+        default_value : Optional[ParameterValueType], optional
             Default value if the user doesn't provide one.
         requested_srs : Optional[str], optional
             Requested spatial reference system for GEOMETRY parameters.

src/cadenzaanalytics/data/parameter_value.py

@@ -5,6 +5,7 @@
 from cadenzaanalytics.data.data_type import DataType
 from cadenzaanalytics.data.geometry_type import GeometryType
 from cadenzaanalytics.data.data_object import DataObject
+from cadenzaanalytics.data.parameter_value_type import ParameterValueType
 
 
 class ParameterValue(DataObject):
@@ -92,12 +93,12 @@ def data_type(self) -> DataType:
         return self._data_type
 
     @property
-    def value(self) -> Any:
+    def value(self) -> Optional[ParameterValueType]:
         """Get the typed value of the parameter.
 
         Returns
         -------
-        Any
+        Optional[ParameterValueType]
             The value of the parameter, typed according to the data type.
         """
         return self._value
@@ -125,7 +126,7 @@ def srs(self) -> Optional[str]:
         return self._srs
 
 
-    def _parse_value(self, value: Any, data_type: DataType) -> Any:
+    def _parse_value(self, value: Any, data_type: DataType) -> Optional[ParameterValueType]:
         """Parse and convert a parameter value according to its data type.
 
         Parameters
@@ -137,7 +138,7 @@ def _parse_value(self, value: Any, data_type: DataType) -> Any:
 
         Returns
         -------
-        Any
+        Optional[ParameterValueType]
             The parsed value with appropriate type, or None if input is None.
         """
         if value is None:

src/cadenzaanalytics/data/parameter_value_type.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Union
+
+from shapely.geometry.base import BaseGeometry
+
+# A typed value for parameters: numbers, strings, boolean, shapely geometries, and datetime
+ParameterValueType = Union[int, float, str, bool, datetime, BaseGeometry]

src/cadenzaanalytics/request/analytics_request.py

@@ -5,7 +5,7 @@
 from cadenzaanalytics.request.request_table import RequestTable
 
 
-class AnalyticsRequest(collections.abc.Mapping):
+class AnalyticsRequest(collections.abc.Mapping[str, RequestTable]):
     """Represents an incoming analytics request from Cadenza.
 
     Provides access to request parameters and data tables. Supports dict-like

src/cadenzaanalytics/request/request_metadata.py

@@ -6,7 +6,7 @@
 
 
 # pylint: disable=protected-access
-class RequestMetadata(collections.abc.Mapping):
+class RequestMetadata(collections.abc.Mapping[str, ColumnMetadata]):
     """Metadata describing the columns in a request table.
 
     Provides access to column metadata by name and groupings by attribute group.

src/cadenzaanalytics/request/request_parameter.py

@@ -1,11 +1,12 @@
 import collections
-from typing import Iterator, List, Optional, Any
+from typing import Iterator, List, Optional
 
 from cadenzaanalytics.data.parameter_value import ParameterValue
 from cadenzaanalytics.request.view_parameter import ViewParameter
+from cadenzaanalytics.data.parameter_value_type import ParameterValueType
 
 
-class RequestParameter(collections.abc.Mapping):
+class RequestParameter(collections.abc.Mapping[str, ParameterValueType]):
     """Provides access to parameters from an analytics request.
 
     Supports dict-like access to parameter values via `params["name"]` syntax.
@@ -41,7 +42,7 @@ def view(self) -> ViewParameter:
             device_pixel_ratio=device_pixel_ratio
         )
 
-    def __getitem__(self, name: str) -> Any:
+    def __getitem__(self, name: str) -> Optional[ParameterValueType]:
         parameter = self._get_parameter(name)
         if parameter is not None:
             return parameter.value
@@ -90,7 +91,7 @@ def _get_parameter(self, name: str) -> Optional[ParameterValue]:
         return None
 
 
-    def _get_parameter_value(self, name: str) -> Any:
+    def _get_parameter_value(self, name: str) -> Optional[ParameterValueType]:
         """Returns a specific parameter value.
 
         Parameters
@@ -100,7 +101,7 @@ def _get_parameter_value(self, name: str) -> Any:
 
         Returns
         -------
-        Any
+        Optional[ParameterValueType]
             The value of the parameter if found, else None.
         """
 

src/cadenzaanalytics/response/csv_response.py

@@ -1,6 +1,3 @@
-import csv
-import re
-import sys
 from typing import List, Optional
 import logging
 
@@ -12,6 +9,7 @@
 from cadenzaanalytics.request.request_table import RequestTable
 from cadenzaanalytics.response.extension_data_response import ExtensionDataResponse
 from cadenzaanalytics.response.missing_metadata_strategy import MissingMetadataStrategy
+from cadenzaanalytics.util import to_cadenza_csv
 
 logger = logging.getLogger('cadenzaanalytics')
 
@@ -107,41 +105,15 @@ def get_response(self, request_table: Optional[RequestTable] = None) -> Response
         leftover_metadata_column_names = self._apply_missing_metadata_strategy()
         self._validate_response(leftover_metadata_column_names)
 
-        python_3_12 = (3, 12)
-        if sys.version_info >= python_3_12 and len(self._data.columns) > 1:
-            # The quoting strategies QUOTE_NOTNULL or QUOTE_NULL would fail with the csv writer
-            # error "single empty field record must be quoted"
-            # if there is only one column and if there is any null-ish value available.
-            # Also refer to https://github.com/pandas-dev/pandas/issues/59116
-            # Thus we can only use this strategy if there is more than one column, else fallback to
-            # the fallback approach that always quotes and then removes quotes again.
-            # The limitation to python 3.12 comes from the option QUOTE_NOTNULL only becoming available on that version.
-            csv_data = self._data.to_csv(
-                sep=';',
-                encoding='utf-8',
-                quoting=csv.QUOTE_NOTNULL,
-                index=False,
-                na_rep=None,  # missing/None/Null values are sent without quotes
-                quotechar='"',
-                lineterminator='\r\n',
-                date_format='%Y-%m-%dT%H:%M:%SZ')
-        else:
-            # info: this approach cannot distinguish empty strings from NULL
-            csv_data = self._data.to_csv(
-                sep=';',
-                encoding='utf-8',
-                quoting=csv.QUOTE_ALL,
-                index=False,
-                quotechar='"',
-                lineterminator='\r\n',
-                date_format='%Y-%m-%dT%H:%M:%SZ')
-            # Needed to make sure to send NULL/None values (unquoted empty content) and not empty strings
-            # (quoted empty content)
-            # as empty strings would only be valid for DataType.STRING and cause errors for other DataTypes.
-            # regex searches and replaces double quotes that are surrounded by separators
-            # (start file, end file, semicolon or newline)
-            # this way double-quotes that represent a single escaped quote character within a string value are retained
-            csv_data = re.sub(r'(^|;|\r\n)""(?=;|\r\n|$)', r'\1', csv_data)
+        datetime_columns = [c.name for c in self._column_meta_data if c.data_type == DataType.ZONEDDATETIME]
+        geometry_columns = [c.name for c in self._column_meta_data if c.data_type == DataType.GEOMETRY]
+        float_columns = [c.name for c in self._column_meta_data if c.data_type == DataType.FLOAT64]
+        int_columns = [c.name for c in self._column_meta_data if c.data_type == DataType.INT64]
+        csv_data = to_cadenza_csv(self._data,
+                                  datetime_columns=datetime_columns,
+                                  geometry_columns=geometry_columns,
+                                  float_columns=float_columns,
+                                  int_columns=int_columns)
         return self._create_response(csv_data, self._column_meta_data)
 
 

src/cadenzaanalytics/tests/__init__.py

@@ -0,0 +1 @@
+"""Test suite for cadenzaanalytics package."""