Basic and advanced gap-filling via standalone tclean library

.github/workflows/pr-ci.yml

@@ -23,6 +23,10 @@ jobs:
         id: tests
         run: pixi run test-integration
         continue-on-error: true
+      - name: Run unit tests
+        id: unit-tests
+        run: pixi run test-unit
+        continue-on-error: true
       - name: Run snakemake linting
         id: linting
         run: |
@@ -37,6 +41,6 @@ jobs:
           path: tests/integration/resources/module/logs
           if-no-files-found: ignore
           retention-days: 30
-      - name: Fail if integration or linting failed
-        if: ${{ steps.tests.outcome == 'failure' || steps.linting.outcome == 'failure' }}
+      - name: Fail if tests or linting failed
+        if: ${{ steps.tests.outcome == 'failure' || steps.unit-tests.outcome == 'failure' || steps.linting.outcome == 'failure' }}
         run: exit 1

.gitignore

@@ -28,3 +28,9 @@ gurobi.log
 **/results/*
 **/temp/*
 **/tmp/*
+
+#ignore ENTSOE token
+resources/user/token_entsoe.txt
+
+#dev workspace
+**/dev/

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
       # Repo quality
       - id: check-added-large-files
         args: [--enforce-all]
-        exclude: ^pixi\.lock$
+        exclude: ^(pixi\.lock|figures/readme_cleaning_timeline\.png)$
       - id: forbid-submodules
       - id: check-case-conflict
       - id: check-illegal-windows-names
@@ -43,6 +43,8 @@ repos:
     hooks:
       - id: codespell
         files: .*\.(py|smk|md)$|^Snakefile$
+        args:
+        - --ignore-words-list=ND
 
 ci:  # https://pre-commit.ci/
   autoupdate_schedule: quarterly

CITATION.cff

@@ -10,3 +10,6 @@ authors:
   - given-names: "Jann"
     family-names: "Launer"
     email: "j.a.c.launer@tudelft.nl"
+  - given-names: "Dano"
+    family-names: "Hawkins"
+    email: "d.d.a.hawkins@tudelft.nl"

INTERFACE.yaml

@@ -5,26 +5,35 @@ pathvars:
   snakemake_defaults:
     logs:
       default: "<logs>"
-      description: location of snakemake log files.
+      description: "Location of Snakemake log files."
     resources:
       default: "<resources>"
-      description: "location of module resource (input) files."
+      description: "Location of module resource (input) files."
     results:
       default: "<results>"
-      description: "location of module result (output) files."
+      description: "Location of module result (output) files."
+
   user_resources:
     token_entsoe:
       default: "<resources>/user/token_entsoe.txt"
       description: "User token for ENTSO-E API access."
+
     shapes:
       default: "<resources>/user/{shape}/shapes.parquet"
       description: >
-        Shapes to aggregate the output data to.
-        IMPORTANT: they must add up to the full spatial extent of the country.
-        The module's processing requires it.
+        Target shapes used to spatially aggregate electricity demand.
+        The supplied shapes must collectively cover the full spatial extent
+        of the corresponding country.
+
+    external_profiles:
+      default: "<resources>/user/external_profiles"
+      description: >
+        Directory of user-provided .csv to feed into advanced tclean gap-filling.
+
   results:
     demand_electricity_MW:
       default: "<results>/{shape}/demand_electricity_MW.parquet"
-      description: Electricity demand profile per custom shape in units of MW.
+      description: "Hourly electricity demand per custom shape, in MW."
+
 wildcards:
-  shape: Name of the custom shape is a wildcard to allow multiple outputs based on different shapes.
+  shape: "Name of the custom target shape set. This wildcard allows multiple outputs for different shape definitions."