From ddc7ff7a956ec792eb28f6aaf93c33c847cfc885 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Mar 2025 11:56:00 +0100 Subject: [PATCH 01/25] feat: add state-of-charge sensor to flex-model Signed-off-by: Victor Garcia Reolid --- flexmeasures/api/v3_0/sensors.py | 1 + flexmeasures/cli/data_add.py | 12 ++++ flexmeasures/data/models/planning/storage.py | 68 ++++++++++++++----- .../data/schemas/scheduling/storage.py | 14 ++++ flexmeasures/data/tests/conftest.py | 33 +++++++-- .../data/tests/test_scheduling_sequential.py | 12 +++- .../tests/test_scheduling_simultaneous.py | 2 +- 7 files changed, 114 insertions(+), 28 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 695a992155..200362ebb4 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -472,6 +472,7 @@ def trigger_schedule( "duration": "PT24H", "flex-model": { "soc-at-start": "12.1 kWh", + "state-of-charge" : {"sensor" : 24}, "soc-targets": [ { "value": "25 kWh", diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 3993b7b74e..d8e836344a 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1219,6 +1219,14 @@ def create_schedule(ctx): required=True, help="State of charge (e.g 32.8%, or 0.328) at the start of the schedule.", ) +@click.option( + "--state-of-charge", + "state_of_charge", + type=QuantityField("%", validate=validate.Range(min=0, max=1)), + help="State of charge sensor.", + required=False, + default=None, +) @click.option( "--soc-target", "soc_target_strings", @@ -1365,6 +1373,7 @@ def add_schedule_for_storage( # noqa C901 soc_max: ur.Quantity | None = None, roundtrip_efficiency: ur.Quantity | None = None, storage_efficiency: ur.Quantity | Sensor | None = None, + state_of_charge: ur.Quantity | Sensor | None = None, as_job: bool = False, ): """Create a new schedule for a storage asset. @@ -1443,6 +1452,9 @@ def add_schedule_for_storage( # noqa C901 }, ) + if state_of_charge is not None: + scheduling_kwargs["flex_model"]["state-of-charge"] = state_of_charge + quantity_or_sensor_vars = { "flex_model": { "charging-efficiency": charging_efficiency, diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 5b104615af..5be4af1901 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -34,6 +34,10 @@ from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution from flexmeasures.utils.unit_utils import ur, convert_units +from flexmeasures.utils.calculations import ( + integrate_time_series, +) + class MetaStorageScheduler(Scheduler): """This class defines the constraints of a schedule for a storage device from the @@ -1030,6 +1034,28 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: for sensor in sensors } + flex_model = self.flex_model + + if not isinstance(self.flex_model, list): + flex_model["sensor"] = sensors[0] + flex_model = [flex_model] + + soc_schedule = { + flex_model_d["state_of_charge"]: integrate_time_series( + series=ems_schedule[d], + initial_stock=soc_at_start[d], + up_efficiency=device_constraints[d]["derivative up efficiency"].fillna( + 1 + ), + down_efficiency=device_constraints[d][ + "derivative down efficiency" + ].fillna(1), + storage_efficiency=device_constraints[d]["efficiency"].fillna(1), + ) + for d, flex_model_d in enumerate(flex_model) + if isinstance(flex_model_d.get("state_of_charge", None), Sensor) + } + # Resample each device schedule to the resolution of the device's power sensor if self.resolution is None: storage_schedule = { @@ -1045,26 +1071,32 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: sensor: storage_schedule[sensor].round(self.round_to_decimals) for sensor in sensors } - if self.return_multiple: - return [ - { - "name": "storage_schedule", - "sensor": sensor, - "data": storage_schedule[sensor], - } - for sensor in sensors - ] + [ - { - "name": "commitment_costs", - "data": { - c.name: costs - for c, costs in zip( - commitments, model.commitment_costs.values() - ) + return ( + [ + { + "name": "storage_schedule", + "sensor": sensor, + "data": storage_schedule[sensor], + } + for sensor in sensors + ] + + [ + { + "name": "commitment_costs", + "data": { + c.name: costs + for c, costs in zip( + commitments, model.commitment_costs.values() + ) + }, }, - }, - ] + ] + + [ + {"name": "state_of_charge", "data": soc, "sensor": sensor} + for sensor, soc in soc_schedule.items() + ] + ) else: return storage_schedule[sensors[0]] diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index a9dbe61914..320afeaf2e 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -123,6 +123,13 @@ class StorageFlexModelSchema(Schema): required=False, ) + state_of_charge = VariableQuantityField( + to_unit="MWh", + default_src_unit="dimensionless", + data_key="state-of-charge", + required=False, + ) + charging_efficiency = VariableQuantityField( "%", data_key="charging-efficiency", required=False ) @@ -216,6 +223,13 @@ def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs f"Target datetime exceeds {max_server_datetime}. Maximum scheduling horizon is {max_server_horizon}." ) + @validates("state_of_charge") + def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | ur.Quantity): + if isinstance(state_of_charge, ur.Quantity): + raise ValidationError( + "The `state-of-charge` field can only be a Sensor. In the future, state-of-charge will absorve soc-at-start field." + ) + @validates("storage_efficiency") def validate_storage_efficiency_resolution(self, unit: Sensor | ur.Quantity): if ( diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 358636dd50..a7cb7b68cd 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -320,10 +320,11 @@ def smart_building(app, fresh_db, smart_building_types): fresh_db.session.add_all(assets) fresh_db.session.flush() - sensors = [] + power_sensors = [] + soc_sensors = [] - # Add power sensor for asset in assets: + # Add power sensor sensor = Sensor( name="power", unit="MW", @@ -335,12 +336,31 @@ def smart_building(app, fresh_db, smart_building_types): generic_asset=asset, timezone="Europe/Amsterdam", ) - sensors.append(sensor) + power_sensors.append(sensor) - fresh_db.session.add_all(sensors) + # Add SOC sensors + sensor = Sensor( + "state of charge", + unit="MWh", + event_resolution=( + timedelta(hours=1) + if asset.name == "Test Battery 1h" + else timedelta(minutes=15) + ), + generic_asset=asset, + timezone="Europe/Amsterdam", + ) + soc_sensors.append(sensor) + + fresh_db.session.add_all(power_sensors) + fresh_db.session.add_all(soc_sensors) fresh_db.session.flush() asset_names = [asset.name for asset in assets] - return dict(zip(asset_names, assets)), dict(zip(asset_names, sensors)) + return ( + dict(zip(asset_names, assets)), + dict(zip(asset_names, power_sensors)), + dict(zip(asset_names, soc_sensors)), + ) @pytest.fixture @@ -351,7 +371,7 @@ def flex_description_sequential( Specifically, the main flex model is deserialized, while the sensors' individual flex models are still serialized. """ - assets, sensors = smart_building + assets, sensors, soc_sensors = smart_building flex_model = [ { @@ -398,6 +418,7 @@ def flex_description_sequential( "value": 0.094, } # 6 kWh discharge ], + "state-of-charge": {"sensor": soc_sensors["Test Battery"].id}, }, }, ] diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index ad78c09f58..850d531018 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -16,7 +16,10 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil We verify that the pipeline creates the right number of jobs (two), corresponding to the inflexible devices, and an extra one which corresponds to the success callback job. """ - assets, sensors = smart_building + assets, sensors, soc_sensors = smart_building + + assert len(soc_sensors["Test Battery"].search_beliefs()) == 0 + queue = app.queues["scheduling"] start = pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam") end = pd.Timestamp("2015-01-04").tz_localize("Europe/Amsterdam") @@ -84,7 +87,7 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil # Work on jobs queued_jobs[0].perform() - work_on_rq(queue) + work_on_rq(queue, handle_scheduling_exception) # Check that the jobs completed successfully assert queued_jobs[0].get_status() == "finished" @@ -135,6 +138,9 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil ), f"Battery cost should be -4.415 €, got {battery_costs} €" assert total_cost == -2.1775, f"Total cost should be -2.1775 €, got {total_cost} €" + # Check that the SOC data is saved + assert len(soc_sensors["Test Battery"].search_beliefs()) == 97 + def test_create_sequential_jobs_fallback( db, app, flex_description_sequential, smart_building @@ -144,7 +150,7 @@ def test_create_sequential_jobs_fallback( Checks execution of a sequential scheduling job, where 1 of the subjobs is set up to fail and trigger its fallback. The deferred subjobs should still succeed after the fallback succeeds, even though the first subjob fails. """ - assets, sensors = smart_building + assets, sensors, _ = smart_building queue = app.queues["scheduling"] start = pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam") diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py index 2294f8f6e2..2b47940dce 100644 --- a/flexmeasures/data/tests/test_scheduling_simultaneous.py +++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py @@ -10,7 +10,7 @@ def test_create_simultaneous_jobs( db, app, flex_description_sequential, smart_building, use_heterogeneous_resolutions ): - assets, sensors = smart_building + assets, sensors, _ = smart_building queue = app.queues["scheduling"] start = pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam") end = pd.Timestamp("2015-01-04").tz_localize("Europe/Amsterdam") From 2bce555da60fab9917aba3fbdc0c1f21df55fb9d Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 09:31:03 +0200 Subject: [PATCH 02/25] add entries to .gitignore Signed-off-by: Victor Garcia Reolid --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 68ab1c2c98..fd99a8e898 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,7 @@ poetry.lock .gitconfig.* /postgres-data -coverage.lcov \ No newline at end of file +coverage.lcov +venv* +logs/ +*.dump \ No newline at end of file From 01bd0476f03da0b3c82a8a7dce61c220ee8dc8a7 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 10:17:08 +0200 Subject: [PATCH 03/25] use SensorIdField Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index d8e836344a..f4d7031891 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1222,7 +1222,7 @@ def create_schedule(ctx): @click.option( "--state-of-charge", "state_of_charge", - type=QuantityField("%", validate=validate.Range(min=0, max=1)), + type=SensorIdField("%"), help="State of charge sensor.", required=False, default=None, From b4d45fe52b6e335b52d0dcc61b62438aee0eb17f Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 10:20:04 +0200 Subject: [PATCH 04/25] remove default src unit Signed-off-by: Victor Garcia Reolid --- .gitignore | 3 ++- flexmeasures/data/schemas/scheduling/storage.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index fd99a8e898..9f11435290 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,5 @@ poetry.lock coverage.lcov venv* logs/ -*.dump \ No newline at end of file +*.dump +iframe_figures/ \ No newline at end of file diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 320afeaf2e..570f9b20f8 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -125,7 +125,6 @@ class StorageFlexModelSchema(Schema): state_of_charge = VariableQuantityField( to_unit="MWh", - default_src_unit="dimensionless", data_key="state-of-charge", required=False, ) From de9fae4668da3203313cb93515aba38b5c596df5 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 10:22:06 +0200 Subject: [PATCH 05/25] check Sensor explicitly Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/scheduling/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 570f9b20f8..da0db355b4 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -224,7 +224,7 @@ def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs @validates("state_of_charge") def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | ur.Quantity): - if isinstance(state_of_charge, ur.Quantity): + if not isinstance(state_of_charge, Sensor): raise ValidationError( "The `state-of-charge` field can only be a Sensor. In the future, state-of-charge will absorve soc-at-start field." ) From d8894924023d372fcff556aa90824e31d2c256d3 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 10:29:29 +0200 Subject: [PATCH 06/25] fix comment Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/scheduling/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index da0db355b4..61955b8648 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -226,7 +226,7 @@ def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | ur.Quantity): if not isinstance(state_of_charge, Sensor): raise ValidationError( - "The `state-of-charge` field can only be a Sensor. In the future, state-of-charge will absorve soc-at-start field." + "The `state-of-charge` field can only be a Sensor. In the future, the state-of-charge field will replace soc-at-start field." ) @validates("storage_efficiency") From 635dc5933a6a162365cdf72a632bcac23cb86bf3 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 11:02:09 +0200 Subject: [PATCH 07/25] round soc schedule Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 5be4af1901..759d25a4da 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1071,6 +1071,11 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: sensor: storage_schedule[sensor].round(self.round_to_decimals) for sensor in sensors } + soc_schedule = { + sensor: soc_schedule[sensor].round(self.round_to_decimals) + for sensor in sensors + } + if self.return_multiple: return ( [ From d47dbbff5f963fc9ecb7f77570e87a0590221a6c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 11:02:29 +0200 Subject: [PATCH 08/25] fix: use kwarg for unit Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index f4d7031891..fa6f208178 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1222,7 +1222,7 @@ def create_schedule(ctx): @click.option( "--state-of-charge", "state_of_charge", - type=SensorIdField("%"), + type=SensorIdField(unit="%"), help="State of charge sensor.", required=False, default=None, From 67d19f00a7c2040e977a5ebbc797ddd0bf73b14b Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 11:03:03 +0200 Subject: [PATCH 09/25] convert SOC Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 23 ++++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 759d25a4da..b251a86ec3 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1041,16 +1041,21 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: flex_model = [flex_model] soc_schedule = { - flex_model_d["state_of_charge"]: integrate_time_series( - series=ems_schedule[d], - initial_stock=soc_at_start[d], - up_efficiency=device_constraints[d]["derivative up efficiency"].fillna( - 1 + flex_model_d["state_of_charge"]: convert_units( + integrate_time_series( + series=ems_schedule[d], + initial_stock=soc_at_start[d], + up_efficiency=device_constraints[d][ + "derivative up efficiency" + ].fillna(1), + down_efficiency=device_constraints[d][ + "derivative down efficiency" + ].fillna(1), + storage_efficiency=device_constraints[d]["efficiency"].fillna(1), ), - down_efficiency=device_constraints[d][ - "derivative down efficiency" - ].fillna(1), - storage_efficiency=device_constraints[d]["efficiency"].fillna(1), + "MWh", + sensors[d].unit, + event_resolution=sensors[d].event_resolution, ) for d, flex_model_d in enumerate(flex_model) if isinstance(flex_model_d.get("state_of_charge", None), Sensor) From 25b97c9f0f5812f9b01d5e0a5442d8e845f44fe0 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 11:03:51 +0200 Subject: [PATCH 10/25] add traceback message Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/services/scheduling.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index ba2cdb9348..4a746b5d0a 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -12,6 +12,7 @@ from typing import Callable, Type import inspect from copy import deepcopy +from traceback import print_tb from flask import current_app @@ -646,8 +647,8 @@ def handle_scheduling_exception(job, exc_type, exc_value, traceback): click.echo( "HANDLING RQ SCHEDULING WORKER EXCEPTION: %s:%s\n" % (exc_type, exc_value) ) - # from traceback import print_tb - # print_tb(traceback) + + print_tb(traceback) job.meta["exception"] = exc_value job.save_meta() From ebe25b30c437908a2baaff1c5509d8af88793ff9 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 11:04:10 +0200 Subject: [PATCH 11/25] use instantaneous resolution sensor Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/tests/conftest.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index a7cb7b68cd..2478ac35d1 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -342,11 +342,7 @@ def smart_building(app, fresh_db, smart_building_types): sensor = Sensor( "state of charge", unit="MWh", - event_resolution=( - timedelta(hours=1) - if asset.name == "Test Battery 1h" - else timedelta(minutes=15) - ), + event_resolution=timedelta(hours=0), generic_asset=asset, timezone="Europe/Amsterdam", ) From 7accf4726ac37f6edb216f4372a559d1c2606535 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 12:25:46 +0200 Subject: [PATCH 12/25] fix unit conversion Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 7 ++++--- .../data/tests/test_scheduling_sequential.py | 20 ++++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index b251a86ec3..6798ad8218 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1053,9 +1053,10 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: ].fillna(1), storage_efficiency=device_constraints[d]["efficiency"].fillna(1), ), - "MWh", - sensors[d].unit, + from_unit="MWh", + to_unit=flex_model_d["state_of_charge"].unit, event_resolution=sensors[d].event_resolution, + capacity=flex_model_d["soc_max"], ) for d, flex_model_d in enumerate(flex_model) if isinstance(flex_model_d.get("state_of_charge", None), Sensor) @@ -1078,7 +1079,7 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: } soc_schedule = { sensor: soc_schedule[sensor].round(self.round_to_decimals) - for sensor in sensors + for sensor in soc_schedule.keys() } if self.return_multiple: diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index 850d531018..4ab33d3145 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -1,3 +1,4 @@ +from datetime import timedelta from unittest.mock import patch from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException @@ -7,6 +8,7 @@ from flexmeasures.data.tests.utils import work_on_rq from flexmeasures.data.services.scheduling import handle_scheduling_exception from flexmeasures.data.models.time_series import Sensor +from flexmeasures.utils.calculations import integrate_time_series def test_create_sequential_jobs(db, app, flex_description_sequential, smart_building): @@ -139,7 +141,23 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil assert total_cost == -2.1775, f"Total cost should be -2.1775 €, got {total_cost} €" # Check that the SOC data is saved - assert len(soc_sensors["Test Battery"].search_beliefs()) == 97 + soc_schedule = ( + soc_sensors["Test Battery"] + .search_beliefs(resolution=timedelta(0)) + .reset_index() + ) + power_schedule = sensors["Test Battery"].search_beliefs().reset_index() + + power_schedule = pd.Series( + power_schedule.event_value.tolist(), + index=pd.DatetimeIndex(power_schedule.event_start.tolist(), freq="15min"), + ) + soc_schedule_from_power = integrate_time_series( + -power_schedule, + 0.1, + decimal_precision=6, + ) + assert all(soc_schedule.event_value.values == soc_schedule_from_power.values) def test_create_sequential_jobs_fallback( From f4201d177bb093ce52abb60272ae3bad441f0db0 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 12:31:39 +0200 Subject: [PATCH 13/25] move default fillna(1) to integrate_time_series Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 8 ++------ flexmeasures/utils/calculations.py | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 6798ad8218..eaf7aca87d 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1045,12 +1045,8 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: integrate_time_series( series=ems_schedule[d], initial_stock=soc_at_start[d], - up_efficiency=device_constraints[d][ - "derivative up efficiency" - ].fillna(1), - down_efficiency=device_constraints[d][ - "derivative down efficiency" - ].fillna(1), + up_efficiency=device_constraints[d]["derivative up efficiency"], + down_efficiency=device_constraints[d]["derivative down efficiency"], storage_efficiency=device_constraints[d]["efficiency"].fillna(1), ), from_unit="MWh", diff --git a/flexmeasures/utils/calculations.py b/flexmeasures/utils/calculations.py index 9b626ef0a8..0e0ec05eac 100644 --- a/flexmeasures/utils/calculations.py +++ b/flexmeasures/utils/calculations.py @@ -141,13 +141,14 @@ def integrate_time_series( if isinstance(storage_efficiency, pd.Series) else pd.Series(storage_efficiency, index=series.index) ) + storage_efficiency = storage_efficiency.fillna(1) # Convert from flow to stock change, applying conversion efficiencies stock_change = pd.Series(data=np.NaN, index=series.index) stock_change.loc[series > 0] = ( series[series > 0] * ( - up_efficiency[series > 0] + up_efficiency[series > 0].fillna(1) if isinstance(up_efficiency, pd.Series) else up_efficiency ) @@ -156,7 +157,7 @@ def integrate_time_series( stock_change.loc[series <= 0] = ( series[series <= 0] / ( - down_efficiency[series <= 0] + down_efficiency[series <= 0].fillna(1) if isinstance(down_efficiency, pd.Series) else down_efficiency ) From 0597d5f326b5d848289e94c932dc5208d6ef4665 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 12:39:27 +0200 Subject: [PATCH 14/25] check state-of-charge sensor resolution in schema Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/scheduling/storage.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 61955b8648..fe04e6a67a 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta from flask import current_app from marshmallow import ( @@ -226,7 +226,12 @@ def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | ur.Quantity): if not isinstance(state_of_charge, Sensor): raise ValidationError( - "The `state-of-charge` field can only be a Sensor. In the future, the state-of-charge field will replace soc-at-start field." + "The `state-of-charge` field can only be a Sensor. In the future, the state-of-charge field will replace soc-at-start field." + ) + + if state_of_charge.event_resolution != timedelta(0): + raise ValidationError( + "The field `state-of-charge` is points to a sensor with a non-instantaneous event resolution. Please, use an instantaneous sensor." ) @validates("storage_efficiency") From 91ee91b6d68cd6b4cd113c7e85ab727ad7c8cc66 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Apr 2025 12:45:16 +0200 Subject: [PATCH 15/25] typo Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/scheduling/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index fe04e6a67a..3d2deddb2a 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -231,7 +231,7 @@ def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | ur.Quanti if state_of_charge.event_resolution != timedelta(0): raise ValidationError( - "The field `state-of-charge` is points to a sensor with a non-instantaneous event resolution. Please, use an instantaneous sensor." + "The field `state-of-charge` points to a sensor with a non-instantaneous event resolution. Please, use an instantaneous sensor." ) @validates("storage_efficiency") From af8357a0ab443ff734d9e0d70eb82600cd89e056 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 2 Apr 2025 00:20:08 +0200 Subject: [PATCH 16/25] Update flexmeasures/cli/data_add.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Victor --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index fa6f208178..12b19624e8 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1222,7 +1222,7 @@ def create_schedule(ctx): @click.option( "--state-of-charge", "state_of_charge", - type=SensorIdField(unit="%"), + type=SensorIdField(unit="MWh"), help="State of charge sensor.", required=False, default=None, From 2c9ce167d8643659d5a6ad2303cc6f0706282a40 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 2 Apr 2025 00:20:24 +0200 Subject: [PATCH 17/25] Update flexmeasures/data/models/planning/storage.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Victor --- flexmeasures/data/models/planning/storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index eaf7aca87d..2c94743761 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1051,7 +1051,6 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: ), from_unit="MWh", to_unit=flex_model_d["state_of_charge"].unit, - event_resolution=sensors[d].event_resolution, capacity=flex_model_d["soc_max"], ) for d, flex_model_d in enumerate(flex_model) From 1949187ece706532cac123d391887363c4494c32 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 2 Apr 2025 00:26:14 +0200 Subject: [PATCH 18/25] remove unnecesarry fillna Signed-off-by: Victor Garcia Reolid --- flexmeasures/utils/calculations.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/utils/calculations.py b/flexmeasures/utils/calculations.py index 0e0ec05eac..9b626ef0a8 100644 --- a/flexmeasures/utils/calculations.py +++ b/flexmeasures/utils/calculations.py @@ -141,14 +141,13 @@ def integrate_time_series( if isinstance(storage_efficiency, pd.Series) else pd.Series(storage_efficiency, index=series.index) ) - storage_efficiency = storage_efficiency.fillna(1) # Convert from flow to stock change, applying conversion efficiencies stock_change = pd.Series(data=np.NaN, index=series.index) stock_change.loc[series > 0] = ( series[series > 0] * ( - up_efficiency[series > 0].fillna(1) + up_efficiency[series > 0] if isinstance(up_efficiency, pd.Series) else up_efficiency ) @@ -157,7 +156,7 @@ def integrate_time_series( stock_change.loc[series <= 0] = ( series[series <= 0] / ( - down_efficiency[series <= 0].fillna(1) + down_efficiency[series <= 0] if isinstance(down_efficiency, pd.Series) else down_efficiency ) From fb746d8d0d58c35c1b5efdb46003c6a1914bd8e6 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 2 Apr 2025 00:28:20 +0200 Subject: [PATCH 19/25] Update flexmeasures/data/models/planning/storage.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Victor --- flexmeasures/data/models/planning/storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 2c94743761..3967ed2e7b 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1051,7 +1051,6 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: ), from_unit="MWh", to_unit=flex_model_d["state_of_charge"].unit, - capacity=flex_model_d["soc_max"], ) for d, flex_model_d in enumerate(flex_model) if isinstance(flex_model_d.get("state_of_charge", None), Sensor) From c75519e0e11b58397c609f6333d15ff6f0687501 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 2 Apr 2025 00:29:15 +0200 Subject: [PATCH 20/25] Update flexmeasures/data/schemas/scheduling/storage.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Victor --- flexmeasures/data/schemas/scheduling/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 3d2deddb2a..fd7e427b3c 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -223,7 +223,7 @@ def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs ) @validates("state_of_charge") - def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | ur.Quantity): + def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | list[dict] | ur.Quantity): if not isinstance(state_of_charge, Sensor): raise ValidationError( "The `state-of-charge` field can only be a Sensor. In the future, the state-of-charge field will replace soc-at-start field." From d5ab2948940039f33e7067871d5abb148513744e Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 2 Apr 2025 00:38:28 +0200 Subject: [PATCH 21/25] apply pre-commit Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/scheduling/storage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index fd7e427b3c..8f8414ffe5 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -223,7 +223,9 @@ def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs ) @validates("state_of_charge") - def validate_state_of_charge_is_sensor(self, state_of_charge: Sensor | list[dict] | ur.Quantity): + def validate_state_of_charge_is_sensor( + self, state_of_charge: Sensor | list[dict] | ur.Quantity + ): if not isinstance(state_of_charge, Sensor): raise ValidationError( "The `state-of-charge` field can only be a Sensor. In the future, the state-of-charge field will replace soc-at-start field." From c9070224a0191f5988f98b6cb5c4637917da96d5 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 2 Apr 2025 10:51:15 +0200 Subject: [PATCH 22/25] fix potential bug Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 12b19624e8..71a284a6ab 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1373,7 +1373,7 @@ def add_schedule_for_storage( # noqa C901 soc_max: ur.Quantity | None = None, roundtrip_efficiency: ur.Quantity | None = None, storage_efficiency: ur.Quantity | Sensor | None = None, - state_of_charge: ur.Quantity | Sensor | None = None, + state_of_charge: Sensor | None = None, as_job: bool = False, ): """Create a new schedule for a storage asset. @@ -1453,7 +1453,7 @@ def add_schedule_for_storage( # noqa C901 ) if state_of_charge is not None: - scheduling_kwargs["flex_model"]["state-of-charge"] = state_of_charge + scheduling_kwargs["flex_model"]["state-of-charge"] = state_of_charge.id quantity_or_sensor_vars = { "flex_model": { From 3ba54f5a29a3662f23b2a0f47b9c56fa3880717d Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 2 Apr 2025 11:04:12 +0200 Subject: [PATCH 23/25] fix state of charge Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 71a284a6ab..cd15881819 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1453,7 +1453,9 @@ def add_schedule_for_storage( # noqa C901 ) if state_of_charge is not None: - scheduling_kwargs["flex_model"]["state-of-charge"] = state_of_charge.id + scheduling_kwargs["flex_model"]["state-of-charge"] = { + "sensor": state_of_charge.id + } quantity_or_sensor_vars = { "flex_model": { From e0ca0d94a600d03f0e3e1875b4467f50e25ecca5 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 2 Apr 2025 11:17:02 +0200 Subject: [PATCH 24/25] add changelog entry Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index c67347376e..c4f7e2c26f 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -10,6 +10,8 @@ v0.26.0 | May XX, 2025 New features ------------- +* Support saving the storage schedule SOC using the ``flex-model`` field ``state-of-charge`` to the ``flex-model``. + Infrastructure / Support ---------------------- From f6bde4d7bea3fd79217aa3ec3a746b0189441648 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 2 Apr 2025 11:28:59 +0200 Subject: [PATCH 25/25] add PR number Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index c4f7e2c26f..57f7f242df 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -10,7 +10,7 @@ v0.26.0 | May XX, 2025 New features ------------- -* Support saving the storage schedule SOC using the ``flex-model`` field ``state-of-charge`` to the ``flex-model``. +* Support saving the storage schedule SOC using the ``flex-model`` field ``state-of-charge`` to the ``flex-model`` [see `PR #1392 `_] Infrastructure / Support ----------------------