diff --git a/app/CHANGELOG.md b/app/CHANGELOG.md index 8a431628d..fb790136c 100644 --- a/app/CHANGELOG.md +++ b/app/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Added +- Allow user to generate custom timeseries (currently only heat demand) from eesyplan functions [[#511](https://github.com/open-plan-tool/gui/pull/511)] ### Fixed - Fix issues with read and edit rights on shared projects [[#478](https://github.com/open-plan-tool/gui/pull/478)] diff --git a/app/projects/forms.py b/app/projects/forms.py index 8680d5343..6cf1faddd 100644 --- a/app/projects/forms.py +++ b/app/projects/forms.py @@ -1,47 +1,36 @@ +import json import logging -import pickle import os -import json -import io -import csv -from django.db.models import Q -from django.utils.html import format_html -from django.utils.safestring import mark_safe -from openpyxl import load_workbook -import numpy as np +import pickle -from crispy_forms.bootstrap import AppendedText, PrependedText, FormActions +import numpy as np from crispy_forms.helper import FormHelper -from crispy_forms.layout import ( - Submit, - Layout, - Row, - Column, - Field, - Fieldset, - ButtonHolder, -) +from crispy_forms.layout import Submit +from dashboard.helpers import KPI_PARAMETERS_ASSETS from django import forms -from django.forms import ModelForm +from django.conf import settings as django_settings from django.core.exceptions import ValidationError -from django.core.validators import MaxValueValidator, MinValueValidator +from django.db.models import Q +from django.forms import ModelForm +from django.utils.html import format_html +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ -from django.conf import settings as django_settings -from projects.models import * -from projects.constants import MAP_EPA_MVS, RENEWABLE_ASSETS, CURRENCY_SYMBOLS -from dashboard.helpers import KPI_PARAMETERS_ASSETS, KPIFinder +from projects.constants import ( + ASSET_TO_TIMESERIES_ASSET_TYPE, + CURRENCY_SYMBOLS, + RENEWABLE_ASSETS, +) from projects.helpers import ( - parameters_helper, PARAMETERS, - DualNumberField, - parse_input_timeseries, - TimeseriesField, + TS_MANUAL_TYPE, TS_SELECT_TYPE, TS_UPLOAD_TYPE, - TS_MANUAL_TYPE, + DualNumberField, + TimeseriesField, + parameters_helper, ) -from projects.constants import ASSET_TO_TIMESERIES_ASSET_TYPE +from projects.models import * def gettext_variables(some_string, lang="de"): @@ -109,9 +98,7 @@ def set_parameter_info(param_name, field, parameters=PARAMETERS): unit = PARAMETERS[param_name][":Unit:"] verbose = PARAMETERS[param_name]["verbose"] default_value = PARAMETERS[param_name][":Default:"] - if unit == "None" or unit == "": - unit = None - elif unit == "Factor": + if unit == "None" or unit == "" or unit == "Factor": unit = None if verbose == "None": verbose = None @@ -138,7 +125,7 @@ class OpenPlanModelForm(ModelForm): """Class to automatize the assignation and translation of the labels, help_text and units""" def __init__(self, *args, **kwargs): - super(OpenPlanModelForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) for fieldname, field in self.fields.items(): set_parameter_info(fieldname, field) @@ -151,7 +138,7 @@ class OpenPlanForm(forms.Form): """Class to automatize the assignation and translation of the labels, help_text and units""" def __init__(self, *args, **kwargs): - super(OpenPlanForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) for fieldname, field in self.fields.items(): set_parameter_info(fieldname, field) @@ -168,7 +155,7 @@ class Meta: exclude = ["date_created", "date_updated", "economic_data", "user", "viewers"] def __init__(self, *args, **kwargs): - super(ProjectDetailForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) for field in self.fields.values(): field.disabled = True @@ -179,7 +166,7 @@ class Meta: fields = "__all__" def __init__(self, *args, **kwargs): - super(EconomicDataDetailForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) for field in self.fields.values(): field.disabled = True @@ -331,7 +318,7 @@ class ProjectCreateForm(OpenPlanForm): # Render form def __init__(self, *args, **kwargs): - super(ProjectCreateForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.helper = FormHelper() self.helper.form_id = "project_form_id" # self.helper.form_class = 'blueForm' @@ -712,7 +699,7 @@ def __init__(self, *args, **kwargs): proj_id = kwargs.pop("proj_id", None) scenario_id = kwargs.pop("scenario_id", None) view_only = kwargs.pop("view_only", False) - self.existing_asset = kwargs.get("instance", None) + self.existing_asset = kwargs.get("instance") # get the connections with busses self.input_output_mapping = kwargs.pop("input_output_mapping", None) @@ -891,12 +878,9 @@ def clean_input_timeseries_old(self): if timeseries_file is not None: input_timeseries_values = parse_input_timeseries(timeseries_file) # TODO here list the possible options - else: - # set the previous timeseries from the asset if any - if self.is_input_timeseries_empty() is False: - input_timeseries_values = ( - self.existing_asset.input_timeseries_values - ) + # set the previous timeseries from the asset if any + elif self.is_input_timeseries_empty() is False: + input_timeseries_values = self.existing_asset.input_timeseries_values return input_timeseries_values except json.decoder.JSONDecodeError as ex: raise ValidationError( @@ -1015,9 +999,16 @@ def assign_timeseries_from_input(self, input_timeseries): ts_asset_type = ASSET_TO_TIMESERIES_ASSET_TYPE.get(asset_type_name) if input_timeseries["input_method"]["type"] == TS_MANUAL_TYPE: - timeseries_name = f"constant value = {timeseries_values[0]}" - timeseries_values = timeseries_values - ts_default_settings["ts_type"] = "scalar" + if len(timeseries_values) == 1: + timeseries_name = f"constant value = {timeseries_values[0]}" + ts_default_settings["ts_type"] = "scalar" + else: + timeseries_name = f"Created timeseries ({self.asset_type_name})" + generation_parameters = input_timeseries["input_method"].get( + "generation_parameters" + ) + if generation_parameters: + ts_default_settings["generation_parameters"] = generation_parameters timeseries, created = Timeseries.objects.get_or_create( values=timeseries_values, @@ -1159,7 +1150,7 @@ class Meta: class StorageForm(AssetCreateForm): def __init__(self, *args, **kwargs): asset_type_name = kwargs.pop("asset_type", None) - super(StorageForm, self).__init__(*args, asset_type="capacity", **kwargs) + super().__init__(*args, asset_type="capacity", **kwargs) self.fields["dispatchable"].widget = forms.HiddenInput() self.initial["dispatchable"] = True @@ -1226,3 +1217,188 @@ class Meta: }, ) } + + +class CreatePVProductionTimeseriesForm(OpenPlanForm): + mounting_type_choices = ( + ("fix_tilt", _("Fix Tilt")), + ("fix_tilt_two_dir", _("Fix Tilt Two Directions Back To Back")), + ("tracker", _("Tracker")), + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: these parameters would not be manual inputs but come from weather data, I assume? check with Markus + + # direct_irradiation_horizontal = + # diffuse_irradiation_horizontal = + azimuth = forms.FloatField( + label=_("Azimuth"), + widget=forms.NumberInput( + attrs={ + "placeholder": _("e.g. 180"), + "data-bs-toggle": "tooltip", + "title": _( + "For fix tilt: Azimuth angle of the module orientation in degrees (North is 0°, East is 90°...); For tracker: Azimuth angle of the rotation-axis for tracking systems" + ), + } + ), + ) + + tilt = forms.FloatField( + label=_("Tilt"), + widget=forms.NumberInput( + attrs={ + "placeholder": _("e.g. 180"), + "data-bs-toggle": "tooltip", + "title": _("Tilt angle in degrees (0° is horizontal, 90° is vertical)"), + } + ), + ) + + system_efficiency = forms.FloatField( + label=_("System Efficiency"), + widget=forms.NumberInput( + attrs={ + "placeholder": _("e.g. 0.8"), + "data-bs-toggle": "tooltip", + "title": _( + "Performance ratio of the total PV-System (usually around 0.8)" + ), + } + ), + ) + + gcr = forms.FloatField( + label=_("Ground Coverage Ratio"), + widget=forms.NumberInput( + attrs={ + "data-bs-toggle": "tooltip", + "title": _( + "Ground Coverage Ratio (Ratio of the module area to the ground area of the module field), only needed for tracker" + ), + } + ), + required=False, + ) + + mounting_type = forms.ChoiceField( + choices=mounting_type_choices, + label=_("Mounting Type"), + widget=forms.Select( + attrs={ + "data-bs-toggle": "tooltip", + "title": _( + "Static systems, east-west like system or 1-axis tracking system" + ), + } + ), + ) + albedo = forms.FloatField( + label=_("Albedo"), + widget=forms.NumberInput( + attrs={ + "data-bs-toggle": "tooltip", + "title": _("Reflection fraction of sunlight in the surrounding area"), + } + ), + ) + + # TODO: Add validation that checks e.g. that this field is only filled in if tracker is selected + max_angle = forms.FloatField( + label=_("Max. tilt angle"), + widget=forms.NumberInput( + attrs={ + "data-bs-toggle": "tooltip", + "title": _( + "Maximum tilt angle for tracking systems. This value is only used for 'tracker' systems" + ), + } + ), + required=False, + ) + + +class CreateHeatDemandForm(OpenPlanForm): + profile_type_choices = ( + ("EFH", "Single-family house"), + ("MFH", "Apartment building"), + ("GHD", "Commerce/Services general"), + ("GMF", "Household-like business enterprises"), + ("GGA", "Restaurants"), + ("GBH", "Retail and wholesale"), + ("GMK", "Metal and automotive"), + ("GBH", "Accommodation"), + ("GKO", "Local authorities, credit institutions and insurance companies"), + ("GBD", "Other operational services"), + ("GWA", "Laundries, dry cleaning"), + ("GGB", "Horticulture"), + ("GBA", "Bakery"), + ("GPD", "Paper and printing"), + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + outdoor_temperature = DualNumberField( + label=_("Outdoor Temperature"), + param_name="outdoor_temperature", + ) + + profile_type = forms.ChoiceField( + choices=profile_type_choices, + label=_("Profile Type"), + widget=forms.Select( + attrs={ + "data-bs-toggle": "tooltip", + "title": _("Select from one of the available BDEW heat profiles"), + } + ), + ) + + annual_heat_demand = forms.FloatField( + label=_("Annual Heat Demand"), + widget=forms.NumberInput( + attrs={ + "placeholder": _("e.g. 1000"), + "data-bs-toggle": "tooltip", + "title": _("Total heat demand in the chosen timeperiod"), + } + ), + ) + + # TODO: here also check the validation of when the field is required + building_year = forms.FloatField( + label=_("Building Year"), + widget=forms.NumberInput( + attrs={ + "placeholder": _("e.g. 1970"), + "data-bs-toggle": "tooltip", + "title": _( + "Only for residential buildings, used for estimating insulation" + ), + } + ), + required=False, + ) + + wind_class = forms.ChoiceField( + label=_("Wind class"), + choices=(("Windy", _("Windy")), ("Not windy", _("Not Windy"))), + widget=forms.Select( + attrs={ + "data-bs-toggle": "tooltip", + "title": _( + "Windy for exposed buildings on free fields, near coast or high ground. Not windy for unexposed buildings in villages/cities" + ), + } + ), + ) + + +CUSTOM_TIMESERIES_FORMS = { + # TODO: re-enable PV timeseries creation when weather data handling is settled + # "pv_plant": CreatePVProductionTimeseriesForm, + "heat_demand": CreateHeatDemandForm, +} diff --git a/app/projects/helpers.py b/app/projects/helpers.py index e2fe35748..9a2369cb8 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -1,19 +1,19 @@ +import csv +import io import json import logging -import os -import io -import csv -from openpyxl import load_workbook + +from dashboard.helpers import KPIFinder from django import forms from django.core.exceptions import ValidationError -from django.utils.translation import gettext_lazy as _ from django.utils.html import html_safe - +from django.utils.translation import gettext_lazy as _ from epa.settings import RESOURCES_DIR -from projects.dtos import convert_to_dto -from projects.models import Timeseries, AssetType +from openpyxl import load_workbook + from projects.constants import MAP_MVS_EPA -from dashboard.helpers import KPIFinder +from projects.dtos import convert_to_dto +from projects.models import Timeseries TS_SELECT_TYPE = "select" TS_UPLOAD_TYPE = "upload" @@ -199,7 +199,7 @@ def __init__(self, **kwargs): } ), } - super(DualInputWidget, self).__init__(widgets=widgets, **kwargs) + super().__init__(widgets=widgets, **kwargs) def use_required_attribute(self, initial): # overwrite the method of the Widget class of the django.form.widgets module @@ -320,6 +320,8 @@ def set_widget_error(self): class TimeseriesInputWidget(forms.MultiWidget): template_name = "asset/timeseries_input.html" + # TODO: currently hardcoded instead of taken from CUSTOM_FORM_ASSETS to avoid circular import with forms + custom_form_assets = ["heat_demand"] # class Media: # # TODO: currently not loading the content as not within head @@ -330,6 +332,7 @@ def __init__(self, select_widget, **kwargs): self.default = kwargs.pop("default", None) self.param_name = kwargs.pop("param_name", None) + self.asset_type = kwargs.pop("asset_type", None) select_widget.attrs.update( { "class": "form-select", @@ -354,7 +357,7 @@ def __init__(self, select_widget, **kwargs): ), } - super(TimeseriesInputWidget, self).__init__(widgets=widgets, **kwargs) + super().__init__(widgets=widgets, **kwargs) def use_required_attribute(self, initial): # overwrite the method of the Widget class of the django.form.widgets module @@ -369,11 +372,8 @@ def decompress(self, value): logging.error("The value of timeseries index is not an integer") ts_qs = Timeseries.objects.filter(id=value) if ts_qs.exists(): - ts_name = ts_qs.values_list("name", flat=True).get() - if "constant value = " in ts_name: - scalar_value = float(ts_name.replace("constant value = ", "")) - else: - scalar_value = None + ts = ts_qs.get() + scalar_value = ts.values[0] if ts.ts_type == "scalar" else None answer = [scalar_value, value, ""] return answer @@ -393,6 +393,8 @@ def get_context(self, name, value, attrs): active = "select" # default ctx["active_tab"] = active + ctx["asset_type"] = self.asset_type + ctx["custom_form_assets"] = self.custom_form_assets return ctx @@ -422,7 +424,10 @@ def __init__( self.max = kwargs.pop("max", None) select_widget = fields[2].widget kwargs["widget"] = TimeseriesInputWidget( - default=default, param_name=param_name, select_widget=select_widget + default=default, + param_name=param_name, + asset_type=asset_type, + select_widget=select_widget, ) super().__init__(fields=fields, require_all_fields=False, **kwargs) self.label = label @@ -455,17 +460,26 @@ def clean(self, values): input_dict = dict(type=TS_SELECT_TYPE, extra_info=timeseries_id) elif scalar_value != "": - # check the input string is a number or a list + # check the input string is a number, a list, or a + # {"values": [...], "generation_parameters": {...}} payload + generation_parameters = None try: answer = [float(scalar_value.replace(",", "."))] except ValueError: try: - answer = json.loads(scalar_value) - if not isinstance(answer, list): + parsed = json.loads(scalar_value) + if isinstance(parsed, list): + answer = parsed + elif isinstance(parsed, dict) and "values" in parsed: + answer = parsed["values"] + generation_parameters = parsed.get("generation_parameters") + else: scalar_value = "" except json.decoder.JSONDecodeError: scalar_value = "" - input_dict = dict(type=TS_MANUAL_TYPE) + input_dict = dict( + type=TS_MANUAL_TYPE, generation_parameters=generation_parameters + ) elif scalar_value == "": self.set_widget_error() raise ValidationError( @@ -567,11 +581,10 @@ def parse_csv_timeseries(file_str): delimiter = "," elif not has_timestamp: raise ValidationError(msg) - else: - # safe to assume decimal comma in single-column case - if comma_per_line and all(c <= 1 for c in comma_per_line): - is_comma_decimal = True - delimiter = ";" + # safe to assume decimal comma in single-column case + elif comma_per_line and all(c <= 1 for c in comma_per_line): + is_comma_decimal = True + delimiter = ";" # check for number of columns, throw error if more then 2 if any(len(line.split(delimiter)) > 2 for line in lines if line.strip()): @@ -595,11 +608,8 @@ def parse_csv_timeseries(file_str): value = value.strip() # --- decimal normalization --- - if is_comma_decimal: + if is_comma_decimal or ("," in value and "." not in value): value = value.replace(",", ".") - else: - if "," in value and "." not in value: - value = value.replace(",", ".") if value.isalpha(): # catch if there is a header, then the file cannot be parsed raise ValidationError(msg) @@ -623,7 +633,7 @@ def parse_xlsx_timeseries(file_buffer): if n_col > 1: col_idx = 1 - for j in range(0, worksheet.max_row): + for j in range(worksheet.max_row): try: timeseries_values.append( float(worksheet.cell(row=j + 1, column=col_idx + 1).value) diff --git a/app/projects/migrations/0029_timeseries_description_and_more.py b/app/projects/migrations/0029_timeseries_description_and_more.py new file mode 100644 index 000000000..edb0bf47d --- /dev/null +++ b/app/projects/migrations/0029_timeseries_description_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.13 on 2026-08-26 18:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0028_sensitivityanalysis_server_simulation_server'), + ] + + operations = [ + migrations.AddField( + model_name='timeseries', + name='description', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='timeseries', + name='generation_parameters', + field=models.JSONField(blank=True, null=True), + ), + ] diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index e6b4d4ef8..a25955dae 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -1,43 +1,42 @@ import datetime import json import logging +import tempfile import uuid from datetime import timedelta -import pandas as pd from pathlib import Path -import numpy as np -import tempfile -from oemof.datapackage.datapackage import building, export_dp_to_json - +import numpy as np import oemof.thermal.compression_heatpumps_and_chillers as cmpr_hp_chiller +import pandas as pd from django.conf import settings +from django.contrib.postgres.fields import ArrayField from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.forms.models import model_to_dict -from django.contrib.postgres.fields import ArrayField from django.utils.translation import gettext_lazy as _ +from oemof.datapackage.datapackage import export_dp_to_json +from users.models import CustomUser + from projects.constants import ( ASSET_CATEGORY, ASSET_TYPE, + BOOL_CHOICES, + COP_MODES, COUNTRY, CURRENCY, ENERGY_VECTOR, - COP_MODES, FLOW_DIRECTION, MVS_TYPE, - SIMULATION_STATUS, - SIMULATION_SERVERS, PENDING, + SIMULATION_SERVERS, + SIMULATION_STATUS, + TIMESERIES_ASSET_TYPES, + TIMESERIES_CATEGORIES, + TIMESERIES_UNITS, TRUE_FALSE_CHOICES, - BOOL_CHOICES, USER_RATING, - TIMESERIES_UNITS, - TIMESERIES_CATEGORIES, - TIMESERIES_TYPES, - TIMESERIES_ASSET_TYPES, ) -from users.models import CustomUser class Feedback(models.Model): @@ -141,12 +140,11 @@ def add_viewer_if_not_exist(self, email=None, share_rights=""): viewers = Viewer.objects.filter(user=user, share_rights=share_rights) if viewers.exists(): viewer = viewers.get() + elif user == self.user: + viewer = None + message = _("You cannot share a project with yourself") else: - if user == self.user: - viewer = None - message = _("You cannot share a project with yourself") - else: - viewer = Viewer.objects.create(user=user, share_rights=share_rights) + viewer = Viewer.objects.create(user=user, share_rights=share_rights) if viewer not in self.viewers.all() and viewer is not None: self.viewers.add(viewer) @@ -154,19 +152,18 @@ def add_viewer_if_not_exist(self, email=None, share_rights=""): message = _( f"'{email}' belongs to a valid user, they will be able to {share_rights} the project '{self.name}'" ) - else: - if viewer is not None: - if viewer.share_rights != share_rights: - success = True - message = _( - f"The share rights of the user registered under {email} for the project '{self.name}' have been changed from '{viewer.share_rights}' to '{share_rights}'" - ) - viewer.share_rights = share_rights - viewer.save() - else: - message = _( - f"The user registered under {email} for the project '{self.name}' already have '{share_rights}' access" - ) + elif viewer is not None: + if viewer.share_rights != share_rights: + success = True + message = _( + f"The share rights of the user registered under {email} for the project '{self.name}' have been changed from '{viewer.share_rights}' to '{share_rights}'" + ) + viewer.share_rights = share_rights + viewer.save() + else: + message = _( + f"The user registered under {email} for the project '{self.name}' already have '{share_rights}' access" + ) else: message = ( @@ -478,7 +475,7 @@ def clean_dir_str(name): resource_metadata["schema"].update(schema) datapackage_metadata_dict["resources"].append(resource_metadata) - out_path = data_folder / f"project.csv" + out_path = data_folder / "project.csv" Path(out_path).parent.mkdir(parents=True, exist_ok=True) df = pd.DataFrame([proj_dp]) df.drop_duplicates("name").to_csv(out_path, index=False) @@ -541,7 +538,7 @@ def clean_dir_str(name): # Save all unique busses to a elements resource if bus_resource_records: resource_metadata = { - "path": f"data/elements/bus.csv", + "path": "data/elements/bus.csv", "profile": "tabular-data-resource", "name": "bus", "format": "csv", @@ -558,7 +555,7 @@ def clean_dir_str(name): resource_metadata["schema"].update(schema) datapackage_metadata_dict["resources"].append(resource_metadata) - out_path = elements_folder / f"bus.csv" + out_path = elements_folder / "bus.csv" Path(out_path).parent.mkdir(parents=True, exist_ok=True) df_bus = pd.DataFrame(bus_resource_records) df_bus.drop_duplicates("name").to_csv(out_path, index=False) @@ -566,7 +563,7 @@ def clean_dir_str(name): # Save all profiles to a sequences resource if profile_resource_records: resource_metadata = { - "path": f"data/sequences/profiles.csv", + "path": "data/sequences/profiles.csv", "profile": "tabular-data-resource", "name": "profiles", "format": "csv", @@ -579,13 +576,13 @@ def clean_dir_str(name): "missingValues": [""], }, } - for k in profile_resource_records.keys(): + for k in profile_resource_records: resource_metadata["schema"]["fields"].append( {"name": k, "type": "number", "format": "default"} ) datapackage_metadata_dict["resources"].append(resource_metadata) - out_path = sequences_folder / f"profiles.csv" + out_path = sequences_folder / "profiles.csv" Path(out_path).parent.mkdir(parents=True, exist_ok=True) # add timestamps to the profiles profile_resource_records["timeindex"] = self.get_timestamps() @@ -646,6 +643,14 @@ class Timeseries(models.Model): blank=True, null=True, ) + generation_parameters = models.JSONField( + blank=True, + null=True, + ) + description = models.TextField( + blank=True, + null=True, + ) # TODO user or scenario can be both null only if open_source attribute is True --> by way of saving # TODO if the timeseries is open_source and the user is deleted, the timeseries user should just be set to null, diff --git a/app/projects/urls.py b/app/projects/urls.py index ad7edbbe9..1e8d58871 100644 --- a/app/projects/urls.py +++ b/app/projects/urls.py @@ -1,4 +1,5 @@ from django.urls import path, re_path + from .views import * urlpatterns = [ @@ -240,6 +241,16 @@ asset_cops_create_or_update, name="asset_cops_create_or_update", ), + re_path( + r"^asset/get_timeseries_create_form/(?P\d+)/(?P[\w-]+)?$", + get_timeseries_create_form, + name="get_timeseries_create_form", + ), + re_path( + r"^asset/custom_timeseries_create/(?P\d+)/(?P[\w-]+)?(/(?P[0-9a-f-]+))?$", + custom_timeseries_create, + name="custom_timeseries_create", + ), # ParameterChangeTracker (track of simulated scenario changes) path( "reset_scenario_changes/", diff --git a/app/projects/views.py b/app/projects/views.py index a89396b2f..30ccbf510 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -1,101 +1,93 @@ # from bootstrap_modal_forms.generic import BSModalCreateView +import datetime import tempfile -from pathlib import Path +import traceback import zipfile +from pathlib import Path - +from dashboard.helpers import fetch_user_projects +from dashboard.models import FancyResults +from django.contrib import messages from django.contrib.auth.decorators import login_required -import datetime -from django.http import ( - HttpResponseForbidden, - JsonResponse, - HttpResponseRedirect, - HttpResponse, -) +from django.core.exceptions import PermissionDenied +from django.db.models import Q +from django.http import HttpResponse, HttpResponseRedirect, JsonResponse from django.http.response import Http404 -from django.template.loader import get_template -from django.utils.translation import gettext_lazy as _ -from django.utils.safestring import mark_safe # from django.shortcuts import * -from django.shortcuts import get_object_or_404, render, redirect +from django.shortcuts import get_object_or_404, redirect, render +from django.template.loader import get_template from django.urls import reverse -from django.core.exceptions import PermissionDenied +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ from django.views.decorators.http import require_http_methods -from django.contrib import messages -from django.template.loader import get_template - -from jsonview.decorators import json_view -from django.db.models import Q - -from oemof.datapackage.datapackage import export_dp_to_json - from epa.settings import ( + EZP_GET_URL, MVS_GET_URL, MVS_LP_FILE_URL, MVS_SA_GET_URL, - EZP_GET_URL, SHOW_EZP, ) -from .forms import * -from .requests import ( - mvs_simulation_request, - fetch_mvs_simulation_results, - ezp_simulation_request, - fetch_ezp_simulation_results, - mvs_sensitivity_analysis_request, - fetch_mvs_sa_results, - parse_mvs_results, - parse_ezp_results, +from jsonview.decorators import json_view +from oemof.datapackage.datapackage import export_dp_to_json + +from projects.decorators import ( + user_has_edit_rights, + user_has_read_rights, + user_is_owner, ) +from projects.helpers import PARAMETERS, format_scenario_for_mvs from projects.models import ( - Project, - EconomicData, - Comment, - ConnectionLink, - AssetType, - UseCase, - Scenario, - Simulation, - ParameterChangeTracker, - AssetChangeTracker, - SensitivityAnalysis, Asset, + AssetChangeTracker, + AssetType, Bus, + Comment, + ConnectionLink, COPCalculator, - Timeseries, + EconomicData, + MaxEmissionConstraint, MinDOAConstraint, MinRenewableConstraint, - MaxEmissionConstraint, NZEConstraint, + ParameterChangeTracker, + Project, + Scenario, + SensitivityAnalysis, + Simulation, + Timeseries, + UseCase, ) -from projects.decorators import ( - user_is_owner, - user_has_read_rights, - user_has_edit_rights, + +from .constants import DONE, ERROR, MAX_STEP, MODIFIED, PENDING, STEP_LIST +from .forms import * +from .requests import ( + ezp_simulation_request, + fetch_ezp_simulation_results, + fetch_mvs_sa_results, + fetch_mvs_simulation_results, + mvs_sensitivity_analysis_request, + mvs_simulation_request, + parse_ezp_results, + parse_mvs_results, ) -from dashboard.models import FancyResults from .scenario_topology_helpers import ( - handle_storage_unit_form_post, - handle_bus_form_post, - handle_asset_form_post, - load_scenario_topology_from_db, NodeObject, - update_deleted_objects_from_database, - duplicate_scenario_objects, duplicate_scenario_connections, - load_scenario_from_dict, + duplicate_scenario_objects, + handle_asset_form_post, + handle_bus_form_post, + handle_storage_unit_form_post, load_project_from_dict, + load_scenario_from_dict, + load_scenario_topology_from_db, + update_deleted_objects_from_database, ) -from projects.helpers import format_scenario_for_mvs, PARAMETERS -from dashboard.helpers import fetch_user_projects -from .constants import DONE, PENDING, ERROR, MODIFIED, STEP_LIST, MAX_STEP from .services import ( excuses_design_under_development, - send_feedback_email, get_selected_scenarios_in_cache, + send_feedback_email, ) -import traceback logger = logging.getLogger(__name__) @@ -201,7 +193,7 @@ def user_feedback(request): body = f"Feedback form for OpenPlan Tool online api\n\nReceived Feedback\n-----------------\n\nTopic: {feedback.subject}\nContent: {feedback.feedback}\n\nInformation about sender\n------------------------\nName: {feedback.name}\n E-mail Address: {feedback.email}" try: send_feedback_email(subject, body) - messages.success(request, f"Thank you for your feedback.") + messages.success(request, "Thank you for your feedback.") except Exception as e: messages.success(request, e) return HttpResponseRedirect(reverse("project_search")) @@ -287,7 +279,7 @@ def ajax_project_viewers_form(request): @user_has_read_rights def project_detail(request, proj_id): project = get_object_or_404(Project, pk=proj_id) - logger.info(f"Populating project and economic details in forms.") + logger.info("Populating project and economic details in forms.") project_form = ProjectDetailForm(None, instance=project) economic_data_form = EconomicDataDetailForm(None, instance=project.economic_data) @@ -304,7 +296,7 @@ def project_create(request): if request.POST: form = ProjectCreateForm(request.POST) if form.is_valid(): - logger.info(f"Creating new project with economic data.") + logger.info("Creating new project with economic data.") economic_data = EconomicData.objects.create( duration=form.cleaned_data["duration"], currency=form.cleaned_data["currency"], @@ -389,7 +381,7 @@ def project_update(request, proj_id): ) if project_form.is_valid() and economic_data_form.is_valid(): - logger.info(f"Updating project with economic data...") + logger.info("Updating project with economic data...") project_form.save() economic_data_form.save() @@ -875,11 +867,11 @@ def scenario_create_topology(request, proj_id, scen_id, step_id=2, max_step=3): "solar_thermal_plant": _("Solar Thermal Plant"), }, "conversion": { - "transformer_station_in": _("Transformer Station (in)"), # - "transformer_station_out": _("Transformer Station (out)"), # - "storage_charge_controller_in": _("Storage Charge Controller (in)"), # - "storage_charge_controller_out": _("Storage Charge Controller (out)"), # - "solar_inverter": _("Solar Inverter"), # + "transformer_station_in": _("Transformer Station (in)"), + "transformer_station_out": _("Transformer Station (out)"), + "storage_charge_controller_in": _("Storage Charge Controller (in)"), + "storage_charge_controller_out": _("Storage Charge Controller (out)"), + "solar_inverter": _("Solar Inverter"), "diesel_generator": _("Diesel Generator"), "fuel_cell": _(" Fuel Cell"), "gas_boiler": _("Gas Boiler"), @@ -907,7 +899,7 @@ def scenario_create_topology(request, proj_id, scen_id, step_id=2, max_step=3): "bus-h2": _("Hydrogen Bus"), }, } - group_names = {group: _(group) for group in components.keys()} + group_names = {group: _(group) for group in components} # TODO: if the scenario exists, load it, otherwise default form @@ -1084,7 +1076,7 @@ def scenario_review(request, proj_id, scen_id, step_id=4, max_step=MAX_STEP): scenario = get_object_or_404(Scenario, pk=scen_id) if request.method == "GET": - html_template = f"scenario/simulation/no-status.html" + html_template = "scenario/simulation/no-status.html" context = { "scenario": scenario, "scen_id": scen_id, @@ -1120,11 +1112,7 @@ def scenario_review(request, proj_id, scen_id, step_id=4, max_step=MAX_STEP): "rating": simulation.user_rating, "sim_server": simulation.server, "mvs_token": simulation.mvs_token, - "mvs_version": ( - simulation.mvs_version - if simulation.mvs_version - else "undefined" - ), + "mvs_version": (simulation.mvs_version or "undefined"), } ) if simulation.status == DONE: @@ -1566,7 +1554,12 @@ def get_timeseries(request, ts_id=None): if request.method == "GET": if ts_id is not None: ts = Timeseries.objects.get(id=ts_id) - return JsonResponse({"values": ts.get_values}) + return JsonResponse( + { + "values": ts.get_values, + "generation_parameters": ts.generation_parameters, + } + ) @json_view @@ -1709,6 +1702,8 @@ def get_asset_create_form(request, scen_id=0, asset_type_name="", asset_uuid=Non ) input_timeseries_data = "" + # these are the assets for which a function to create a custom timeseries is available via eesyplan + custom_form_assets = CUSTOM_TIMESERIES_FORMS.keys() context = { "form": form, "asset_type_name": asset_type_name, @@ -1716,6 +1711,7 @@ def get_asset_create_form(request, scen_id=0, asset_type_name="", asset_uuid=Non "input_timeseries_timestamps": json.dumps( scenario.get_timestamps(json_format=True) ), + "custom_form_assets": custom_form_assets, } return render(request, "asset/asset_create_form.html", context) @@ -1794,6 +1790,88 @@ def asset_connection_ports_info(request, asset_type_name=None): return answer +@login_required +@require_http_methods(["GET"]) +def get_timeseries_create_form(request, scen_id=0, asset_type_name=""): + if asset_type_name not in CUSTOM_TIMESERIES_FORMS: + logger.error( + "The given asset type does not have a custom timeseries creation form" + ) + raise Http404() + form = CUSTOM_TIMESERIES_FORMS[asset_type_name] + context = {"form": form} + + return render(request, "asset/asset_subform.html", context) + + +@login_required +@require_http_methods(["POST"]) +def custom_timeseries_create(request, scen_id=0, asset_type_name="", asset_uuid=None): + from oemof.eesyplan.importer.create_timeseries_pv import ( + create_pv_production_timeseries, + ) + from oemof.eesyplan.importer.heat_demand import create_heat_demand + + if asset_uuid: + existing_asset = get_object_or_404(Asset, unique_id=asset_uuid) + custom_form = CUSTOM_TIMESERIES_FORMS[asset_type_name] + form = custom_form(request.POST) + + custom_timeseries_functions = { + "pv_plant": create_pv_production_timeseries, + "heat_demand": create_heat_demand, + } + + scenario = get_object_or_404(Scenario, id=scen_id) + if form.is_valid(): + try: + # TODO: calculate from relevant function + # for pv timeseries, add lat/lon to the params dict + # should be able to just pass the validated form as dict as **form + cleaned_data = form.cleaned_data + # outdoor_temperature is excluded from the saved generation parameters: + # it can itself be a full timeseries and we don't want to save a whole + # other timeseries as metadata + generation_parameters = { + key: value + for key, value in cleaned_data.items() + if key != "outdoor_temperature" + } + + outdoor_temperature = cleaned_data.get("outdoor_temperature") + if outdoor_temperature is not None and not isinstance( + outdoor_temperature, list + ): + cleaned_data["outdoor_temperature"] = [ + outdoor_temperature + ] * scenario.get_num_timesteps + + custom_ts_fun = custom_timeseries_functions[asset_type_name] + timeseries = custom_ts_fun(**cleaned_data) + + return JsonResponse( + { + "success": True, + "timeseries": timeseries.values.tolist(), + "generation_parameters": generation_parameters, + }, + status=200, + ) + except Exception: + logger.exception( + "Failed to compute custom timeseries for asset type %s", + asset_type_name, + ) + return JsonResponse({"success": False}, status=422) + + logger.warning("The submitted asset has erroneous field values.") + + form_html = get_template("asset/asset_subform.html") + return JsonResponse( + {"success": False, "form_html": form_html.render({"form": form})}, status=422 + ) + + @login_required @require_http_methods(["GET"]) def get_asset_cops_form(request, scen_id=0, asset_type_name="", asset_uuid=None): @@ -1805,7 +1883,7 @@ def get_asset_cops_form(request, scen_id=0, asset_type_name="", asset_uuid=None) opts["instance"] = existing_cop.get() context = {"form": COPCalculatorForm(**opts)} - return render(request, "asset/asset_cops_form.html", context) + return render(request, "asset/asset_subform.html", context) @login_required @@ -1841,9 +1919,9 @@ def asset_cops_create_or_update( except: return JsonResponse({"success": False, "cop_id": cop.id}, status=422) - logger.warning(f"The submitted asset has erroneous field values.") + logger.warning("The submitted asset has erroneous field values.") - form_html = get_template("asset/asset_cops_form.html") + form_html = get_template("asset/asset_subform.html") return JsonResponse( {"success": False, "form_html": form_html.render({"form": form})}, status=422 ) diff --git a/app/static/css/main.css b/app/static/css/main.css index ea7f80ac3..897f6aea7 100644 --- a/app/static/css/main.css +++ b/app/static/css/main.css @@ -1428,9 +1428,13 @@ form .btn { .modal .modal-body { padding: 3rem; } -.modal .modal-addendum { - padding-right: 3rem; - padding-left: 3rem; } +.modal #form-createTS .modal-addendum { + display: flex; + flex-wrap: wrap; + gap: 1rem; } + .modal #form-createTS .modal-addendum .form-group { + flex: 1 1 220px; + margin-bottom: 0; } .system-design-error .modal-body { display: flex; @@ -4598,5 +4602,3 @@ nav.navbar { height: 100%; background-color: #E3EAEE; z-index: 1; } - -/*# sourceMappingURL=main.css.map */ diff --git a/app/static/js/grid_model_topology.js b/app/static/js/grid_model_topology.js index 3d2c04d10..8bbf9365f 100644 --- a/app/static/js/grid_model_topology.js +++ b/app/static/js/grid_model_topology.js @@ -721,7 +721,78 @@ function updateInputTimeseries(){ // COP calculation from temperature -function toggle_cop_modal(event){ +function toggle_sub_modal(){ + // get the parameters which uniquely identify the asset + const assetTypeName = guiModalDOM.getAttribute("data-node-type"); + const topologyNodeId = guiModalDOM.getAttribute("data-node-topo-id"); // e.g. 'node-2' + + let getUrl = createTimeseriesGetUrl + assetTypeName; + if (nodesToDB.has(topologyNodeId)) + getUrl; + + fetch(getUrl).then(response => response.text()).then(formContent => { + // assign the content of the form to the form tag of the modal + guiModalDOM.querySelector('form .modal-addendum').innerHTML = formContent; + // enable Bootstrap tooltips (help text icons) + $('[data-bs-toggle="tooltip"]').tooltip(); + }).catch(error => { + console.error(error); + }); +} + +function computeCustomTimeseries(event){ + + // get the parameters which uniquely identify the asset + const assetTypeName = guiModalDOM.getAttribute("data-node-type"); + const topologyNodeId = guiModalDOM.getAttribute("data-node-topo-id"); // e.g. 'node-2' + + const form = event.target.closest('.modal-content').querySelector('#timeseriesForm'); + const formData = new FormData(); + // because we can't have nested forms in the html, we construct the form data manually from the fields here instead of relying on the form tag + // TODO: if some of the custom forms rely on more than input, select, need to adapt + form.querySelectorAll('input, select').forEach(el => { + formData.append(el.name, el.value); + }); + + let postUrl = createTimeseriesPostUrl + assetTypeName; + if (nodesToDB.has(topologyNodeId)) + postUrl += "/" + nodesToDB.get(topologyNodeId).uid; + + const createTsDOM = event.target.closest('#form-createTS'); + const paramName = createTsDOM.dataset.paramName; + + fetch(postUrl, { + method: 'POST', + headers: {'X-CSRFToken': csrfToken}, + body: formData, + }).then(response => response.json()).then(jsonRes => { + if (jsonRes.success) { + // show the computed timeseries in the usual field in the modal (same as select/upload do), + // and store it (together with the parameters used to generate it) as the manual/scalar + // value so it gets saved with the asset + const tsValues = jsonRes.timeseries; + const generationParameters = jsonRes.generation_parameters; + // clear the other subfields first: the scalar field's onchange handler + // (initTimeseriesManualValue) re-triggers the select field's own change handler + // with whatever it currently holds, which would otherwise re-fetch and clobber + // the value we're about to set here with a stale, previously-selected timeseries + document.getElementById('id_' + paramName + '_1').value = ''; + document.getElementById('id_' + paramName + '_2').value = ''; + const scalarInput = document.getElementById('id_' + paramName + '_0'); + scalarInput.value = JSON.stringify({values: tsValues, generation_parameters: generationParameters}); + scalarInput.dispatchEvent(new Event('change')); + plotTimeseriesInputTrace(tsValues, paramName); + showGenerationParameters(generationParameters, paramName); + } else { + form.innerHTML = jsonRes.form_html; + } + }).catch(error => { + console.error(error); + alert(error.message); + }); +} + +function toggle_cop_modal(){ // get the parameters which uniquely identify the asset const assetTypeName = guiModalDOM.getAttribute("data-node-type"); const topologyNodeId = guiModalDOM.getAttribute("data-node-topo-id"); // e.g. 'node-2' @@ -767,8 +838,7 @@ function computeCOP(event){ efficiencyDOM = guiModalDOM.querySelector('input[name="efficiency_scalar"]'); if(efficiencyDOM){ - efficiencyDOM.value = jsonRes.cops; - efficiencyDOM.dispatchEvent(new Event('change')); + efficiencyDOM.value = jsonRes.cops; efficiencyDOM.dispatchEvent(new Event('change')); } copDOM = guiModalDOM.querySelector('input[name="copId"]'); if(copDOM){ diff --git a/app/static/js/traceplot.js b/app/static/js/traceplot.js index cc0ae18b1..02b5a288b 100644 --- a/app/static/js/traceplot.js +++ b/app/static/js/traceplot.js @@ -84,10 +84,25 @@ function getTimeseriesValues(ts_id, param_name=""){ else { plotTimeseriesInputTrace(ts_values, param_name=param_name); } + showGenerationParameters(data["generation_parameters"], param_name); console.log("retrieved values", ts_values); }); } +/* Show the parameters a timeseries was generated from (if any) in a small + 
 block under its trace plot, e.g. templates/asset/timeseries_input.html */
+function showGenerationParameters(generationParameters, param_name=""){
+    const el = document.getElementById(param_name + "_generation_parameters");
+    if (!el) return;
+    if (generationParameters && Object.keys(generationParameters).length > 0){
+        el.textContent = JSON.stringify(generationParameters, null, 2);
+        el.style.display = "block";
+    } else {
+        el.textContent = "";
+        el.style.display = "none";
+    }
+}
+
 
 function getConstantTimeseriesId(value){
     //findtsGetUrl is defined in scenario_step2.html
diff --git a/app/static/scss/components/_modals.scss b/app/static/scss/components/_modals.scss
index d0f2530d1..0fdab04ff 100644
--- a/app/static/scss/components/_modals.scss
+++ b/app/static/scss/components/_modals.scss
@@ -12,9 +12,15 @@
     padding: 3rem;
   }
 
-  .modal-addendum {
-    padding-right: 3rem;
-    padding-left: 3rem;
+  #form-createTS .modal-addendum {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 1rem;
+
+    .form-group {
+      flex: 1 1 220px;
+      margin-bottom: 0;
+    }
   }
 }
 
diff --git a/app/templates/asset/asset_cops_form.html b/app/templates/asset/asset_subform.html
similarity index 100%
rename from app/templates/asset/asset_cops_form.html
rename to app/templates/asset/asset_subform.html
diff --git a/app/templates/asset/timeseries_input.html b/app/templates/asset/timeseries_input.html
index 6b15476a7..ae7455c4d 100644
--- a/app/templates/asset/timeseries_input.html
+++ b/app/templates/asset/timeseries_input.html
@@ -15,24 +15,41 @@
 
 
 

-	{% spaceless %}{% for widget in widget.subwidgets %}
-	{{ widget.id }}
-	{% if 'scalar' in widget.name %}
+	{% spaceless %}{% for subwidget in widget.subwidgets %}
+	{{ subwidget.id }}
+	{% if 'scalar' in subwidget.name %}
 
 	

-		{% include widget.template_name %}
+		{% include subwidget.template_name with widget=subwidget %}
 	

-	{% elif 'select' in widget.name %}
+	{% elif 'select' in subwidget.name %}
 	

-		{% include widget.template_name %}
+		{% include subwidget.template_name with widget=subwidget %}
 	

 	{% else %}
 	

-		{% include widget.template_name %}
+
+		{% include subwidget.template_name with widget=subwidget %}
+		{% if asset_type in custom_form_assets %}
+		
+		

+			

+				

+			

+			
+		

+		{% endif %}
 	

 	{% endif %}
 	{% endfor %}{% endspaceless %}
 

 

+
 {% url 'get_timeseries' as ts_url %}
 {{ ts_url|json_script:"tsUrl" }}
diff --git a/app/templates/scenario/scenario_step2.html b/app/templates/scenario/scenario_step2.html
index c31cc66b8..ba15296fa 100644
--- a/app/templates/scenario/scenario_step2.html
+++ b/app/templates/scenario/scenario_step2.html
@@ -141,6 +141,8 @@ 
{{ group_names|get_item:group_name|title }}

     const assetPortInfoUrl = `{% url 'asset_connection_ports_info' %}`;
     const postAssetFormUrl = `{% url 'asset_create_or_update' scenario.id %}`;
     const scenarioBelongsToUser = {% if scenario.project.user == request.user %}true{% else %}false{% endif %};
+    const createTimeseriesGetUrl = `{% url 'get_timeseries_create_form' scenario.id %}`;
+    const createTimeseriesPostUrl = `{% url 'custom_timeseries_create' scenario.id %}`;
     const copGetUrl = `{% url 'get_asset_cops_form' scenario.id %}`;
     const copPostUrl = `{% url 'asset_cops_create_or_update' scenario.id %}`;
     const tsGetUrl = `{% url 'get_timeseries' %}`;