chore: use pydantic as schema provider

config/config.schema.json

@@ -0,0 +1,83 @@
+{
+  "additionalProperties": false,
+  "properties": {
+    "countries": {
+      "default": [
+        "BE"
+      ],
+      "description": "List of countries to retrieve OSM data for",
+      "items": {
+        "type": "string"
+      },
+      "minItems": 1,
+      "type": "array"
+    },
+    "retrieve": {
+      "additionalProperties": false,
+      "properties": {
+        "source": {
+          "default": "geofabrik",
+          "description": "Retrieval backend for OSM data",
+          "enum": [
+            "geofabrik",
+            "overpass"
+          ],
+          "type": "string"
+        },
+        "primary_name": {
+          "default": "power",
+          "description": "Primary OSM feature to retrieve (e.g., 'power')",
+          "type": "string"
+        },
+        "features": {
+          "default": [
+            "substation",
+            "line"
+          ],
+          "description": "OSM features to retrieve for each country",
+          "items": {
+            "type": "string"
+          },
+          "minItems": 1,
+          "type": "array"
+        },
+        "force_redownload": {
+          "default": false,
+          "description": "Force refresh of cached data in earth-osm",
+          "type": "boolean"
+        },
+        "mp": {
+          "default": true,
+          "description": "Enable multiprocessing in earth-osm",
+          "type": "boolean"
+        },
+        "stream_backend": {
+          "default": true,
+          "description": "Enable streaming backend in earth-osm",
+          "type": "boolean"
+        },
+        "cache_primary": {
+          "default": false,
+          "description": "Enable caching of primary feature data in earth-osm",
+          "type": "boolean"
+        },
+        "target_date": {
+          "anyOf": [
+            {
+              "format": "date-time",
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Optional historical date for data retrieval in ISO 8601 datetime format"
+        }
+      },
+      "description": "Configuration for OSM data retrieval using earth-osm"
+    }
+  },
+  "title": "ConfigSchema",
+  "type": "object"
+}

config/config.yaml

@@ -1,15 +1,16 @@
+%YAML 1.1
+---
+# yaml-language-server: $schema=./config.schema.json
 countries:
-  - benin
-  - togo
-
+- BE
 retrieve:
   source: geofabrik
   primary_name: power
   features:
-    - substation
-    - line
+  - substation
+  - line
   force_redownload: false
   mp: true
   stream_backend: true
   cache_primary: false
-  target_date: null
+  target_date:

pixi.toml

@@ -13,7 +13,8 @@ clio-tools = ">=2026.03.30"
 conda = ">=25.0.0"
 ipdb = ">=0.13.13"
 ipykernel = ">=6.29.5"
-jsonschema = ">=4.0.0"
+pydantic = ">=2.0"
+"ruamel.yaml" = ">=0.18"
 mypy = ">=1.15.0"
 pytest = ">=8.3.5"
 python = ">=3.12"
@@ -25,3 +26,4 @@ earth-osm = ">=3.0.2"
 
 [tasks]
 test-integration = {cmd = "pytest tests/integration_test.py"}
+generate-config = {cmd = "python workflow/scripts/_schema.py"}

tests/integration/Snakefile

@@ -2,37 +2,20 @@
 configfile: workflow.source_path("./test_config.yaml")
 
 
-# Import the module and configure it.
-# `snakefile:` specifies the module. It can use file paths and special github(...) / gitlab(...) markers
-# `config`: specifies the module configuration.
-# `pathvars:` helps you re-wire where the module places files.
 module grid_builder:
     pathvars:
-        # Redirect OSM outputs
-        osm_out="resources/module/resources/automatic/osm/out",
-        # Redirect intermediate files that are internal to the module
-        logs="resources/module/logs",
-        resources="resources/module/resources",
-        results="resources/module/results",
+        resources="resources/grid-builder",
+        logs="logs/grid-builder",
     snakefile:
         "../../workflow/Snakefile"
     config:
         config["grid_builder"]
 
 
-# rename all module rules with a prefix, to avoid naming conflicts.
 use rule * from grid_builder as grid_builder_*
 
 
-# Request OSM retrieval outputs from the module
 rule all:
     default_target: True
     input:
-        expand(
-            "resources/module/resources/automatic/osm/out/{country}_{feature}.{ext}",
-            country="belgium",
-            feature=["substation", "line"],
-            ext=["csv", "geojson"],
-        ),
-    message:
-        "Retrieve OSM grid data by country and feature."
+        rules.grid_builder_retrieve_osm_all.input,

tests/integration/test_config.yaml

@@ -1,6 +1,6 @@
 grid_builder:
   countries:
-    - belgium
+    - benin
 
   retrieve:
     source: geofabrik
@@ -12,4 +12,4 @@ grid_builder:
     mp: true
     stream_backend: true
     cache_primary: false
-    target_date: null
+    target_date:

tests/integration_test.py

@@ -31,7 +31,7 @@ def test_interface_file(module_path):
         "LICENSE",
         "README.md",
         "config/config.yaml",
-        "workflow/internal/config.schema.yaml",
+        "config/config.schema.json",
         "tests/integration/Snakefile",
     ],
 )
@@ -40,14 +40,6 @@ def test_standard_file_existance(module_path, file):
     assert Path(module_path / file).exists()
 
 
-def test_snakemake_all_failure(module_path):
-    """The snakemake 'all' rule should return an error by default."""
-    process = subprocess.run(
-        "snakemake --cores 1", shell=True, cwd=module_path, capture_output=True
-    )
-    assert "INVALID (missing locally)" in str(process.stderr)
-
-
 def test_snakemake_integration_testing(module_path):
     """Run a light-weight test simulating someone using this module."""
     assert subprocess.run(

workflow/Snakefile

@@ -1,47 +1,35 @@
+import sys
+
 import yaml
+from snakemake.utils import min_version
 
-from snakemake.utils import min_version, validate
+sys.path.insert(0, workflow.basedir)
+from scripts._schema import validate_config
 
 min_version("9.19")
 
 
 # Define pathvars to expose OSM retrieval outputs for downstream use.
 pathvars:
     # OSM retrieval outputs by country and feature
-    osm_out="<resources>/automatic/osm/out",
+    osm_out="<resources>/osm/out",
 
 
-# Load the example configuration. This will be overridden by users.
+# Default configuration file generated from pydantic schema.
 configfile: workflow.source_path("../config/config.yaml")
 
 
-# Validate the configuration using the schema file.
-validate(config, workflow.source_path("internal/config.schema.yaml"))
+config = validate_config(config)
 
 # Load internal settings separately so users cannot modify them.
 with open(workflow.source_path("internal/settings.yaml"), "r") as f:
     internal = yaml.safe_load(f)
 
 
-# Add all your includes here.
 include: "rules/retrieve.smk"
 
 
-# Add additional files to be delivered alongside the workflow here.
-# This is needed e.g. for python code files that are never
-# explicitly used as a script or input in a snakemake rule.
-# e.g.: workflow.source_path("scripts/_utils.py")
-
-
 rule all:
     default_target: True
-    output:
-        "INVALID",
-    log:
-        stderr="<logs>/all.stderr",
-    conda:
-        "envs/shell.yaml"
-    message:
-        "ERROR: Invalid `rule all:` call"
-    shell:
-        'echo "This workflow must be called as a snakemake module." > {log.stderr}'
+    input:
+        rules.retrieve_osm_all.input,

workflow/rules/retrieve.smk

@@ -2,15 +2,16 @@
 #
 # SPDX-License-Identifier: MIT
 
+
 rule retrieve_osm:
     output:
         csv=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.csv",
+            "<resources>/osm/out/{country}_{feature}.csv",
             country="{country}",
             feature=config["retrieve"]["features"],
         ),
         geojson=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.geojson",
+            "<resources>/osm/out/{country}_{feature}.geojson",
             country="{country}",
             feature=config["retrieve"]["features"],
         ),
@@ -36,12 +37,12 @@ rule retrieve_osm:
 rule retrieve_osm_all:
     input:
         csv=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.csv",
+            "<resources>/osm/out/{country}_{feature}.csv",
             country=config["countries"],
             feature=config["retrieve"]["features"],
         ),
         geojson=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.geojson",
+            "<resources>/osm/out/{country}_{feature}.geojson",
             country=config["countries"],
             feature=config["retrieve"]["features"],
-        )
+        ),

workflow/scripts/_helpers.py

@@ -17,8 +17,9 @@ def mock_snakemake(
     submodule_dir="workflow/submodules/pypsa-eur",
     **wildcards,
 ):
-    """
-    This function is expected to be executed from the 'scripts'-directory of '
+    """Mock a Snakemake object for testing scripts outside of Snakemake.
+
+    This function is expected to be executed from the 'scripts'-directory of
     the snakemake project. It returns a snakemake.script.Snakemake object,
     based on the Snakefile.
 
@@ -155,4 +156,4 @@ def make_accessable(*ios):
     finally:
         if user_in_script_dir:
             os.chdir(script_dir)
-    return snakemake
+    return snakemake