From f8f44d560eea24eafdd12abeddf88106bcf68b57 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 27 Dec 2024 14:20:18 +0100
Subject: [PATCH 001/151] refactor: separate classes for flow and stock
 commitments

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py |  8 ++++
 flexmeasures/data/models/planning/storage.py  | 39 +++++++++++--------
 2 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index fa22338600..002ae2a44e 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -306,6 +306,14 @@ def to_frame(self) -> pd.DataFrame:
         )
 
 
+class FlowCommitment(Commitment):
+    pass
+
+
+class StockCommitment(Commitment):
+    pass
+
+
 """
 Deprecations
 """
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 0561a54f7f..8bb43c39bd 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -11,7 +11,11 @@
 
 
 from flexmeasures import Sensor
-from flexmeasures.data.models.planning import Commitment, Scheduler, SchedulerOutputType
+from flexmeasures.data.models.planning import (
+    FlowCommitment,
+    Scheduler,
+    SchedulerOutputType,
+)
 from flexmeasures.data.models.planning.linear_optimization import device_scheduler
 from flexmeasures.data.models.planning.utils import (
     get_prices,
@@ -231,7 +235,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         )
 
         # Set up commitments to optimise for
-        commitments = []
+        flow_commitments = []
+        stock_commitments = []
 
         index = initialize_index(start, end, self.resolution)
         commitment_quantities = initialize_series(0, start, end, self.resolution)
@@ -249,14 +254,14 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         )
 
         # Set up commitments DataFrame
-        commitment = Commitment(
+        flow_commitment = FlowCommitment(
             name="energy",
             quantity=commitment_quantities,
             upwards_deviation_price=commitment_upwards_deviation_price,
             downwards_deviation_price=commitment_downwards_deviation_price,
             index=index,
         )
-        commitments.append(commitment)
+        flow_commitments.append(flow_commitment)
 
         # Set up peak commitments
         if self.flex_context.get("ems_peak_consumption_price", None) is not None:
@@ -293,7 +298,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame
-            commitment = Commitment(
+            flow_commitment = FlowCommitment(
                 name="consumption peak",
                 quantity=ems_peak_consumption,
                 # positive price because breaching in the upwards (consumption) direction is penalized
@@ -301,7 +306,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            commitments.append(commitment)
+            flow_commitments.append(flow_commitment)
         if self.flex_context.get("ems_peak_production_price", None) is not None:
             ems_peak_production = get_continuous_series_sensor_or_quantity(
                 variable_quantity=self.flex_context.get("ems_peak_production_in_mw"),
@@ -336,7 +341,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame
-            commitment = Commitment(
+            flow_commitment = FlowCommitment(
                 name="production peak",
                 quantity=-ems_peak_production,  # production is negative quantity
                 # negative price because peaking in the downwards (production) direction is penalized
@@ -344,7 +349,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            commitments.append(commitment)
+            flow_commitments.append(flow_commitment)
 
         # Set up capacity breach commitments and EMS capacity constraints
         ems_consumption_breach_price = self.flex_context.get(
@@ -381,7 +386,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame to penalize any breach
-            commitment = Commitment(
+            flow_commitment = FlowCommitment(
                 name="any consumption breach",
                 quantity=ems_consumption_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
@@ -389,17 +394,17 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            commitments.append(commitment)
+            flow_commitments.append(flow_commitment)
 
             # Set up commitments DataFrame to penalize each breach
-            commitment = Commitment(
+            flow_commitment = FlowCommitment(
                 name="all consumption breaches",
                 quantity=ems_consumption_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
                 upwards_deviation_price=ems_consumption_breach_price,
                 index=index,
             )
-            commitments.append(commitment)
+            flow_commitments.append(flow_commitment)
 
             # Take the physical capacity as a hard constraint
             ems_constraints["derivative max"] = ems_power_capacity_in_mw
@@ -430,7 +435,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame to penalize any breach
-            commitment = Commitment(
+            flow_commitment = FlowCommitment(
                 name="any production breach",
                 quantity=ems_production_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
@@ -438,17 +443,17 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            commitments.append(commitment)
+            flow_commitments.append(flow_commitment)
 
             # Set up commitments DataFrame to penalize each breach
-            commitment = Commitment(
+            flow_commitment = FlowCommitment(
                 name="all production breaches",
                 quantity=ems_production_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
                 downwards_deviation_price=-ems_production_breach_price,
                 index=index,
             )
-            commitments.append(commitment)
+            flow_commitments.append(flow_commitment)
 
             # Take the physical capacity as a hard constraint
             ems_constraints["derivative min"] = -ems_power_capacity_in_mw
@@ -663,7 +668,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             soc_at_start,
             device_constraints,
             ems_constraints,
-            commitments,
+            flow_commitments,
         )
 
     def persist_flex_model(self):

From 96f040cd9e048eedb71639e3ad9e2e5cf4520456 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 27 Dec 2024 14:22:59 +0100
Subject: [PATCH 002/151] feat: set up stock commitment for breaching
 soc-minima and soc-maxima

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 66 ++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 8bb43c39bd..0a2ecdff84 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -15,6 +15,7 @@
     FlowCommitment,
     Scheduler,
     SchedulerOutputType,
+    StockCommitment,
 )
 from flexmeasures.data.models.planning.linear_optimization import device_scheduler
 from flexmeasures.data.models.planning.utils import (
@@ -486,6 +487,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 as_instantaneous_events=True,
                 boundary_policy="first",
             )
+            # todo: check flex-model for soc_minima_breach_price and soc_maxima_breach_price fields; if these are defined, create a StockCommitment using both prices (if only 1 price is given, still create the commitment, but only penalize one direction)
         if isinstance(soc_minima, Sensor):
             soc_minima = get_continuous_series_sensor_or_quantity(
                 variable_quantity=soc_minima,
@@ -497,6 +499,38 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 as_instantaneous_events=True,
                 boundary_policy="max",
             )
+            if self.flex_model.get("soc_minima_breach_price", None) is not None:
+                soc_minima_breach_price = self.flex_context.get(
+                    "soc_minima_breach_price"
+                )
+                soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_minima_breach_price,
+                    actuator=sensor,
+                    unit=(
+                        soc_minima_breach_price.unit
+                        if isinstance(soc_minima_breach_price, Sensor)
+                        else (
+                            soc_minima_breach_price[0]["value"].units
+                            if isinstance(soc_minima_breach_price, list)
+                            else str(soc_minima_breach_price.units)
+                        )
+                    ),
+                    query_window=(start, end),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    fallback_attribute="soc-minima-breach-price",
+                    fill_sides=True,
+                )
+                # Set up commitments DataFrame
+                stock_commitment = StockCommitment(
+                    name="soc minima",
+                    quantity=soc_minima,
+                    # negative price because breaching in the downwards (shortage) direction is penalized
+                    downwards_deviation_price=-soc_minima_breach_price,
+                    _type="any",
+                    index=index,
+                )
+                stock_commitments.append(stock_commitment)
         if isinstance(soc_maxima, Sensor):
             soc_maxima = get_continuous_series_sensor_or_quantity(
                 variable_quantity=soc_maxima,
@@ -508,6 +542,38 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 as_instantaneous_events=True,
                 boundary_policy="min",
             )
+            if self.flex_model.get("soc_maxima_breach_price", None) is not None:
+                soc_maxima_breach_price = self.flex_context.get(
+                    "soc_maxima_breach_price"
+                )
+                soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_maxima_breach_price,
+                    actuator=sensor,
+                    unit=(
+                        soc_maxima_breach_price.unit
+                        if isinstance(soc_maxima_breach_price, Sensor)
+                        else (
+                            soc_maxima_breach_price[0]["value"].units
+                            if isinstance(soc_maxima_breach_price, list)
+                            else str(soc_maxima_breach_price.units)
+                        )
+                    ),
+                    query_window=(start, end),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    fallback_attribute="soc-maxima-breach-price",
+                    fill_sides=True,
+                )
+                # Set up commitments DataFrame
+                stock_commitment = StockCommitment(
+                    name="soc maxima",
+                    quantity=soc_maxima,
+                    # positive price because breaching in the upwards (surplus) direction is penalized
+                    upwards_deviation_price=soc_maxima_breach_price,
+                    _type="any",
+                    index=index,
+                )
+                stock_commitments.append(stock_commitment)
 
         device_constraints[0] = add_storage_constraints(
             start,

From e893cfe8522a501ff039f70e42fc7a8484e6a9d0 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 27 Dec 2024 14:32:38 +0100
Subject: [PATCH 003/151] fix: remove hard constraints when moving to soft
 constraint

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 0a2ecdff84..ffcf57c467 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -531,6 +531,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     index=index,
                 )
                 stock_commitments.append(stock_commitment)
+
+                # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint
+                soc_minima = None
+
         if isinstance(soc_maxima, Sensor):
             soc_maxima = get_continuous_series_sensor_or_quantity(
                 variable_quantity=soc_maxima,
@@ -575,6 +579,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 stock_commitments.append(stock_commitment)
 
+                # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
+                soc_maxima = None
+
         device_constraints[0] = add_storage_constraints(
             start,
             end,

From 67921ddead3efb80e8c8d9e192cb8bcf2b775449 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sat, 28 Dec 2024 23:37:32 +0100
Subject: [PATCH 004/151] feat: take into account StockCommitment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py |  1 +
 .../models/planning/linear_optimization.py    | 60 ++++++++++++++++++-
 flexmeasures/data/models/planning/storage.py  | 41 +++++++------
 3 files changed, 80 insertions(+), 22 deletions(-)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index 002ae2a44e..d2d4be7e78 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -301,6 +301,7 @@ def to_frame(self) -> pd.DataFrame:
                 self.upwards_deviation_price,
                 self.downwards_deviation_price,
                 self.group,
+                pd.Series(self.__class__, index=self.index, name="class"),
             ],
             axis=1,
         )
diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index 450a2eb951..ffaa3327cb 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -59,11 +59,12 @@ def device_scheduler(  # noqa C901
     :param ems_constraints:     EMS constraints are on an EMS level. Handled constraints (listed by column name):
                                     derivative max: maximum flow
                                     derivative min: minimum flow
-    :param commitments:         Commitments are on an EMS level. Handled parameters (listed by column name):
+    :param commitments:         Commitments are on an EMS level by default. Handled parameters (listed by column name):
                                     quantity:                   for example, 5.5
                                     downwards deviation price:  10.1
                                     upwards deviation price:    10.2
                                     group:                      1 (defaults to the enumerate time step j)
+                                    device:                     0 (corresponds to device d; if not set, commitment is on an EMS level)
     :param initial_stock:       initial stock for each device. Use a list with the same number of devices as device_constraints,
                                 or use a single value to set the initial stock to be the same for all devices.
 
@@ -443,12 +444,66 @@ def device_down_derivative_sign(m, d, j):
     def ems_derivative_bounds(m, j):
         return m.ems_derivative_min[j], sum(m.ems_power[:, j]), m.ems_derivative_max[j]
 
+    def device_stock_commitment_equalities(m, c, j, d):
+        """Couple device stocks to each commitment."""
+        if "d" not in commitments[c] or commitments[c]["d"] != d:
+            # Commitment c does not concern device d
+            return
+        if commitments[c]["class"] == "FlowCommitment":
+            raise NotImplementedError(
+                "FlowCommitment on a device level has not been implemented. Please file a GitHub ticket explaining your use case."
+            )
+        if isinstance(initial_stock, list):
+            initial_stock_d = initial_stock[d]
+        else:
+            initial_stock_d = initial_stock
+
+        stock_changes = [
+            (
+                m.device_power_down[d, k] / m.device_derivative_down_efficiency[d, k]
+                + m.device_power_up[d, k] * m.device_derivative_up_efficiency[d, k]
+                + m.stock_delta[d, k]
+            )
+            for k in range(0, j + 1)
+        ]
+        efficiencies = [m.device_efficiency[d, k] for k in range(0, j + 1)]
+        return (
+            (
+                0
+                if len(commitments[c]) == 1
+                or "upwards deviation price" in commitments[c].columns
+                else None
+            ),
+            # 0 if "upwards deviation price" in commitments[c].columns else None,  # todo: possible simplification
+            m.commitment_quantity[c]
+            - [
+                stock - initial_stock_d
+                for stock in apply_stock_changes_and_losses(
+                    initial_stock_d, stock_changes, efficiencies
+                )
+            ][-1],
+            (
+                0
+                if len(commitments[c]) == 1
+                or "downwards deviation price" in commitments[c].columns
+                else None
+            ),
+            # 0 if "downwards deviation price" in commitments[c].columns else None,  # todo: possible simplification
+        )
+
     def ems_flow_commitment_equalities(m, c, j):
         """Couple EMS flows (sum over devices) to each commitment.
 
         - Creates an inequality for one-sided commitments.
         - Creates an equality for two-sided commitments and for groups of size 1.
         """
+        if "d" in commitments[c]:
+            # Commitment c does not concern EMS
+            return
+        if commitments[c]["class"] == "StockCommitment":
+            raise NotImplementedError(
+                "StockCommitment on an EMS level has not been implemented. Please file a GitHub ticket explaining your use case."
+            )
         return (
             (
                 0
@@ -498,6 +553,9 @@ def device_derivative_equalities(m, d, j):
     model.ems_power_commitment_equalities = Constraint(
         model.cj, rule=ems_flow_commitment_equalities
     )
+    model.device_energy_commitment_equalities = Constraint(
+        model.cj, model.d, rule=device_stock_commitment_equalities
+    )
     model.device_power_equalities = Constraint(
         model.d, model.j, rule=device_derivative_equalities
     )
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index ffcf57c467..129d1d3dd4 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -236,8 +236,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         )
 
         # Set up commitments to optimise for
-        flow_commitments = []
-        stock_commitments = []
+        commitments = []
 
         index = initialize_index(start, end, self.resolution)
         commitment_quantities = initialize_series(0, start, end, self.resolution)
@@ -255,14 +254,14 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         )
 
         # Set up commitments DataFrame
-        flow_commitment = FlowCommitment(
+        commitment = FlowCommitment(
             name="energy",
             quantity=commitment_quantities,
             upwards_deviation_price=commitment_upwards_deviation_price,
             downwards_deviation_price=commitment_downwards_deviation_price,
             index=index,
         )
-        flow_commitments.append(flow_commitment)
+        commitments.append(commitment)
 
         # Set up peak commitments
         if self.flex_context.get("ems_peak_consumption_price", None) is not None:
@@ -299,7 +298,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame
-            flow_commitment = FlowCommitment(
+            commitment = FlowCommitment(
                 name="consumption peak",
                 quantity=ems_peak_consumption,
                 # positive price because breaching in the upwards (consumption) direction is penalized
@@ -307,7 +306,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            flow_commitments.append(flow_commitment)
+            commitments.append(commitment)
         if self.flex_context.get("ems_peak_production_price", None) is not None:
             ems_peak_production = get_continuous_series_sensor_or_quantity(
                 variable_quantity=self.flex_context.get("ems_peak_production_in_mw"),
@@ -342,7 +341,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame
-            flow_commitment = FlowCommitment(
+            commitment = FlowCommitment(
                 name="production peak",
                 quantity=-ems_peak_production,  # production is negative quantity
                 # negative price because peaking in the downwards (production) direction is penalized
@@ -350,7 +349,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            flow_commitments.append(flow_commitment)
+            commitments.append(commitment)
 
         # Set up capacity breach commitments and EMS capacity constraints
         ems_consumption_breach_price = self.flex_context.get(
@@ -387,7 +386,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame to penalize any breach
-            flow_commitment = FlowCommitment(
+            commitment = FlowCommitment(
                 name="any consumption breach",
                 quantity=ems_consumption_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
@@ -395,17 +394,17 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            flow_commitments.append(flow_commitment)
+            commitments.append(commitment)
 
             # Set up commitments DataFrame to penalize each breach
-            flow_commitment = FlowCommitment(
+            commitment = FlowCommitment(
                 name="all consumption breaches",
                 quantity=ems_consumption_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
                 upwards_deviation_price=ems_consumption_breach_price,
                 index=index,
             )
-            flow_commitments.append(flow_commitment)
+            commitments.append(commitment)
 
             # Take the physical capacity as a hard constraint
             ems_constraints["derivative max"] = ems_power_capacity_in_mw
@@ -436,7 +435,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
             # Set up commitments DataFrame to penalize any breach
-            flow_commitment = FlowCommitment(
+            commitment = FlowCommitment(
                 name="any production breach",
                 quantity=ems_production_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
@@ -444,17 +443,17 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 _type="any",
                 index=index,
             )
-            flow_commitments.append(flow_commitment)
+            commitments.append(commitment)
 
             # Set up commitments DataFrame to penalize each breach
-            flow_commitment = FlowCommitment(
+            commitment = FlowCommitment(
                 name="all production breaches",
                 quantity=ems_production_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
                 downwards_deviation_price=-ems_production_breach_price,
                 index=index,
             )
-            flow_commitments.append(flow_commitment)
+            commitments.append(commitment)
 
             # Take the physical capacity as a hard constraint
             ems_constraints["derivative min"] = -ems_power_capacity_in_mw
@@ -522,7 +521,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
-                stock_commitment = StockCommitment(
+                commitment = StockCommitment(
                     name="soc minima",
                     quantity=soc_minima,
                     # negative price because breaching in the downwards (shortage) direction is penalized
@@ -530,7 +529,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     _type="any",
                     index=index,
                 )
-                stock_commitments.append(stock_commitment)
+                commitments.append(commitment)
 
                 # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint
                 soc_minima = None
@@ -569,7 +568,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
-                stock_commitment = StockCommitment(
+                commitment = StockCommitment(
                     name="soc maxima",
                     quantity=soc_maxima,
                     # positive price because breaching in the upwards (surplus) direction is penalized
@@ -577,7 +576,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     _type="any",
                     index=index,
                 )
-                stock_commitments.append(stock_commitment)
+                commitments.append(commitment)
 
                 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
                 soc_maxima = None
@@ -741,7 +740,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             soc_at_start,
             device_constraints,
             ems_constraints,
-            flow_commitments,
+            commitments,
         )
 
     def persist_flex_model(self):

From aae0bbf49d07ec7aa58d8a327e44bfe4bff4b8a4 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sat, 28 Dec 2024 23:41:56 +0100
Subject: [PATCH 005/151] fix: apply StockCommitment to device 0

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py | 6 ++++++
 flexmeasures/data/models/planning/storage.py  | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index d2d4be7e78..a9b2f8daa0 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -214,6 +214,8 @@ class Commitment:
     ----------
     name:
         Name of the commitment.
+    device:
+        Device to which the commitment pertains. If None, the commitment pertains to the EMS.
     index:
         Pandas DatetimeIndex defining the time slots to which the commitment applies.
         The index is shared by the group, quantity. upwards_deviation_price and downwards_deviation_price Pandas Series.
@@ -234,6 +236,7 @@ class Commitment:
     """
 
     name: str
+    device: pd.Series = None
     index: pd.DatetimeIndex = field(repr=False, default=None)
     _type: str = field(repr=False, default="each")
     group: pd.Series = field(init=False)
@@ -262,6 +265,8 @@ def __post_init__(self):
                 )
 
         # Force type conversion of repr fields to pd.Series
+        if not isinstance(self.device, pd.Series):
+            self.device = pd.Series(self.device, index=self.index)
         if not isinstance(self.quantity, pd.Series):
             self.quantity = pd.Series(self.quantity, index=self.index)
         if not isinstance(self.upwards_deviation_price, pd.Series):
@@ -284,6 +289,7 @@ def __post_init__(self):
             raise ValueError('Commitment `_type` must be "any" or "each".')
 
         # Name the Series as expected by our device scheduler
+        self.device = self.device.rename("device")
         self.quantity = self.quantity.rename("quantity")
         self.upwards_deviation_price = self.upwards_deviation_price.rename(
             "upwards deviation price"
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 129d1d3dd4..a82d1ce021 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -528,6 +528,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     downwards_deviation_price=-soc_minima_breach_price,
                     _type="any",
                     index=index,
+                    device=0,
                 )
                 commitments.append(commitment)
 
@@ -575,6 +576,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     upwards_deviation_price=soc_maxima_breach_price,
                     _type="any",
                     index=index,
+                    device=0,
                 )
                 commitments.append(commitment)
 

From 9eb4a36f67d8f2915b03253318ba3a1c80512af3 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sat, 28 Dec 2024 23:44:10 +0100
Subject: [PATCH 006/151] fix: comparison and skipping of constraint

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/linear_optimization.py        | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index ffaa3327cb..e53088c3ba 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -448,8 +448,8 @@ def device_stock_commitment_equalities(m, c, j, d):
         """Couple device stocks to each commitment."""
         if "d" not in commitments[c] or commitments[c]["d"] != d:
             # Commitment c does not concern device d
-            return
-        if commitments[c]["class"] == "FlowCommitment":
+            return Constraint.Skip
+        if not all(cl.__name__ == "StockCommitment" for cl in commitments[c]["class"]):
             raise NotImplementedError(
                 "FlowCommitment on a device level has not been implemented. Please file a GitHub ticket explaining your use case."
             )
@@ -499,8 +499,10 @@ def ems_flow_commitment_equalities(m, c, j):
         """
         if "d" in commitments[c]:
             # Commitment c does not concern EMS
-            return
-        if commitments[c]["class"] == "StockCommitment":
+            return Constraint.Skip
+        if "class" in commitments[c].columns and not all(
+            cl.__name__ == "FlowCommitment" for cl in commitments[c]["class"]
+        ):
             raise NotImplementedError(
                 "StockCommitment on an EMS level has not been implemented. Please file a GitHub ticket explaining your use case."
             )

From 0957f1670ddc109bdcf4df29b7c413675795e143 Mon Sep 17 00:00:00 2001
From: Tammevesky <tammeveski7@gmail.com>
Date: Tue, 31 Dec 2024 10:29:37 +0100
Subject: [PATCH 007/151] assign deviations from stock position to variable

---
 flexmeasures/data/models/planning/linear_optimization.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index e53088c3ba..136cbb3aec 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -476,6 +476,8 @@ def device_stock_commitment_equalities(m, c, j, d):
             ),
             # 0 if "upwards deviation price" in commitments[c].columns else None,  # todo: possible simplification
             m.commitment_quantity[c]
+            + m.commitment_downwards_deviation[c]
+            + m.commitment_upwards_deviation[c]
             - [
                 stock - initial_stock_d
                 for stock in apply_stock_changes_and_losses(

From 19d6406326de25e05b7605fb45f5d3b8c97b022e Mon Sep 17 00:00:00 2001
From: Tammevesky <tammeveski7@gmail.com>
Date: Fri, 3 Jan 2025 14:13:38 +0100
Subject: [PATCH 008/151] fix: minor fixes to StockCommitment handling

---
 flexmeasures/data/models/planning/linear_optimization.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index 136cbb3aec..f2fe95fba4 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -446,7 +446,10 @@ def ems_derivative_bounds(m, j):
 
     def device_stock_commitment_equalities(m, c, j, d):
         """Couple device stocks to each commitment."""
-        if "d" not in commitments[c] or commitments[c]["d"] != d:
+        if (
+            "device" not in commitments[c].columns
+            or (commitments[c]["device"] != d).all()
+        ):
             # Commitment c does not concern device d
             return Constraint.Skip
         if not all(cl.__name__ == "StockCommitment" for cl in commitments[c]["class"]):
@@ -499,7 +502,7 @@ def ems_flow_commitment_equalities(m, c, j):
         - Creates an inequality for one-sided commitments.
         - Creates an equality for two-sided commitments and for groups of size 1.
         """
-        if "d" in commitments[c]:
+        if "device" in commitments[c].columns:
             # Commitment c does not concern EMS
             return Constraint.Skip
         if "class" in commitments[c].columns and not all(

From 6edaac57bdee76262d002cf36563bb7c7b66c06d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 13 Jan 2025 16:01:19 +0100
Subject: [PATCH 009/151] feat: expose new field flex-model fields

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/storage.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py
index b8775c6a06..5563d9fe57 100644
--- a/flexmeasures/data/schemas/scheduling/storage.py
+++ b/flexmeasures/data/schemas/scheduling/storage.py
@@ -150,6 +150,21 @@ class StorageFlexModelSchema(Schema):
         validate=validate.Length(min=1),
     )
 
+    soc_minima_breach_price = VariableQuantityField(
+        "/MWh",
+        data_key="soc-minima-breach-price",
+        required=False,
+        value_validator=validate.Range(min=0),
+        default=None,
+    )
+    soc_maxima_breach_price = VariableQuantityField(
+        "/MWh",
+        data_key="soc-maxima-breach-price",
+        required=False,
+        value_validator=validate.Range(min=0),
+        default=None,
+    )
+
     def __init__(
         self,
         start: datetime,

From 1be3431d866f221a1c50383da0857b220ca4924e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 13 Jan 2025 16:08:51 +0100
Subject: [PATCH 010/151] feat: move price fields to flex-context

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py     |  4 ++--
 flexmeasures/data/schemas/scheduling/__init__.py | 16 ++++++++++++++++
 flexmeasures/data/schemas/scheduling/storage.py  | 15 ---------------
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 8d148591a2..3480994444 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -502,7 +502,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 as_instantaneous_events=True,
                 resolve_overlaps="max",
             )
-            if self.flex_model.get("soc_minima_breach_price", None) is not None:
+            if self.flex_context.get("soc_minima_breach_price", None) is not None:
                 soc_minima_breach_price = self.flex_context.get(
                     "soc_minima_breach_price"
                 )
@@ -550,7 +550,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 as_instantaneous_events=True,
                 resolve_overlaps="min",
             )
-            if self.flex_model.get("soc_maxima_breach_price", None) is not None:
+            if self.flex_context.get("soc_maxima_breach_price", None) is not None:
                 soc_maxima_breach_price = self.flex_context.get(
                     "soc_maxima_breach_price"
                 )
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 942c0571a4..560737c3f7 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -15,6 +15,22 @@ class FlexContextSchema(Schema):
     This schema lists fields that can be used to describe sensors in the optimised portfolio
     """
 
+    # Device commitments
+    soc_minima_breach_price = VariableQuantityField(
+        "/MWh",
+        data_key="soc-minima-breach-price",
+        required=False,
+        value_validator=validate.Range(min=0),
+        default=None,
+    )
+    soc_maxima_breach_price = VariableQuantityField(
+        "/MWh",
+        data_key="soc-maxima-breach-price",
+        required=False,
+        value_validator=validate.Range(min=0),
+        default=None,
+    )
+
     # Energy commitments
     ems_power_capacity_in_mw = VariableQuantityField(
         "MW",
diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py
index 5563d9fe57..b8775c6a06 100644
--- a/flexmeasures/data/schemas/scheduling/storage.py
+++ b/flexmeasures/data/schemas/scheduling/storage.py
@@ -150,21 +150,6 @@ class StorageFlexModelSchema(Schema):
         validate=validate.Length(min=1),
     )
 
-    soc_minima_breach_price = VariableQuantityField(
-        "/MWh",
-        data_key="soc-minima-breach-price",
-        required=False,
-        value_validator=validate.Range(min=0),
-        default=None,
-    )
-    soc_maxima_breach_price = VariableQuantityField(
-        "/MWh",
-        data_key="soc-maxima-breach-price",
-        required=False,
-        value_validator=validate.Range(min=0),
-        default=None,
-    )
-
     def __init__(
         self,
         start: datetime,

From e87a1dc4dfa7b11abe246668712108046c982be5 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 13 Jan 2025 16:10:15 +0100
Subject: [PATCH 011/151] feat: check compatibility of price units

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 560737c3f7..56af3c0b4b 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -144,6 +144,8 @@ def check_prices(self, data: dict, **kwargs):
         if any(
             field_map[field] in data
             for field in (
+                "soc-minima-breach-price",
+                "soc-maxima-breach-price",
                 "site-consumption-breach-price",
                 "site-production-breach-price",
                 "site-peak-consumption-price",

From 25d21e0e97c749eed808315e835ee51da06002bf Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 13 Jan 2025 16:17:07 +0100
Subject: [PATCH 012/151] fix: param explanation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 3480994444..8e98e173f8 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -1115,7 +1115,7 @@ def add_storage_constraints(
 
     :param start:                       Start of the schedule.
     :param end:                         End of the schedule.
-    :param resolution:                  Timedelta used to resample the forecasts to the resolution of the schedule.
+    :param resolution:                  Timedelta used to resample the constraints to the resolution of the schedule.
     :param soc_at_start:                State of charge at the start time.
     :param soc_targets:                 Exact targets for the state of charge at each time.
     :param soc_maxima:                  Maximum state of charge at each time.

From b17ca057e62e4f1971c1f8662c2e8d11536e9095 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 13 Jan 2025 16:24:48 +0100
Subject: [PATCH 013/151] docs: add note regarding how the SoC breach price
 depends on the sensor resolution

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index d849134efb..e43f8b587e 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -117,6 +117,8 @@ For more details on the possible formats for field values, see :ref:`variable_qu
 
 .. [#production] Example: with a connection capacity (``site-power-capacity``) of 1 MVA (apparent power) and a production capacity (``site-production-capacity``) of 400 kW (active power), the scheduler will make sure that the grid inflow doesn't exceed 400 kW.
 
+.. [#soc_breach_prices] The SoC breach prices (e.g. 5 EUR/kWh) to use for the schedule is applied over each time step equal to the sensor resolution. For example, a SoC breach price of 5 EUR/kWh per hour, for scheduling a 5-minute resolution sensor, should be passed as a SoC breach price of :math:`5*60/5 = 12` EUR/kWh.
+
 .. note:: If no (symmetric, consumption and production) site capacity is defined (also not as defaults), the scheduler will not enforce any bound on the site power.
           The flexible device can still have its own power limit defined in its flex-model.
 

From e619a9627498c5cdebca6decad8547b3d9164ef1 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 13 Jan 2025 16:54:04 +0100
Subject: [PATCH 014/151] docs: add documentation for new fields

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index e43f8b587e..c69defc798 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -104,6 +104,12 @@ For more details on the possible formats for field values, see :ref:`variable_qu
    * - ``site-peak-production-price``
      - ``"260 EUR/MWh"``
      - Production peaks above the ``site-peak-production`` are penalized against this per-kW price. [#penalty_field]_
+   * - ``soc-minima-breach-price``
+     - ``"5 EUR/kWh"``
+     - Penalty for not meeting ``soc-minima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
+   * - ``soc-maxima-breach-price``
+     - ``"5 EUR/kWh"``
+     - Penalty for not meeting ``soc-maxima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
 
 .. [#old_sensor_field] The old field only accepted an integer (sensor ID).
 
@@ -117,7 +123,7 @@ For more details on the possible formats for field values, see :ref:`variable_qu
 
 .. [#production] Example: with a connection capacity (``site-power-capacity``) of 1 MVA (apparent power) and a production capacity (``site-production-capacity``) of 400 kW (active power), the scheduler will make sure that the grid inflow doesn't exceed 400 kW.
 
-.. [#soc_breach_prices] The SoC breach prices (e.g. 5 EUR/kWh) to use for the schedule is applied over each time step equal to the sensor resolution. For example, a SoC breach price of 5 EUR/kWh per hour, for scheduling a 5-minute resolution sensor, should be passed as a SoC breach price of :math:`5*60/5 = 12` EUR/kWh.
+.. [#soc_breach_prices] The SoC breach prices (e.g. 5 EUR/kWh) to use for the schedule are applied over each time step equal to the sensor resolution. For example, a SoC breach price of 5 EUR/kWh per hour, for scheduling a 5-minute resolution sensor, should be passed as a SoC breach price of :math:`5*60/5 = 12` EUR/kWh.
 
 .. note:: If no (symmetric, consumption and production) site capacity is defined (also not as defaults), the scheduler will not enforce any bound on the site power.
           The flexible device can still have its own power limit defined in its flex-model.

From 569e2e540d4f5517432997e693ae929434dcca3d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 14 Jan 2025 18:29:11 +0100
Subject: [PATCH 015/151] fix: example calculation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index c69defc798..61bf6baa55 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -105,10 +105,10 @@ For more details on the possible formats for field values, see :ref:`variable_qu
      - ``"260 EUR/MWh"``
      - Production peaks above the ``site-peak-production`` are penalized against this per-kW price. [#penalty_field]_
    * - ``soc-minima-breach-price``
-     - ``"5 EUR/kWh"``
+     - ``"6 EUR/kWh"``
      - Penalty for not meeting ``soc-minima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
    * - ``soc-maxima-breach-price``
-     - ``"5 EUR/kWh"``
+     - ``"6 EUR/kWh"``
      - Penalty for not meeting ``soc-maxima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
 
 .. [#old_sensor_field] The old field only accepted an integer (sensor ID).
@@ -123,7 +123,7 @@ For more details on the possible formats for field values, see :ref:`variable_qu
 
 .. [#production] Example: with a connection capacity (``site-power-capacity``) of 1 MVA (apparent power) and a production capacity (``site-production-capacity``) of 400 kW (active power), the scheduler will make sure that the grid inflow doesn't exceed 400 kW.
 
-.. [#soc_breach_prices] The SoC breach prices (e.g. 5 EUR/kWh) to use for the schedule are applied over each time step equal to the sensor resolution. For example, a SoC breach price of 5 EUR/kWh per hour, for scheduling a 5-minute resolution sensor, should be passed as a SoC breach price of :math:`5*60/5 = 12` EUR/kWh.
+.. [#soc_breach_prices] The SoC breach prices (e.g. 6 EUR/kWh) to use for the schedule are applied over each time step equal to the sensor resolution. For example, a SoC breach price of 6 EUR/kWh per hour, for scheduling a 5-minute resolution sensor, should be passed as a SoC breach price of :math:`6*5/60 = 0.50` EUR/kWh.
 
 .. note:: If no (symmetric, consumption and production) site capacity is defined (also not as defaults), the scheduler will not enforce any bound on the site power.
           The flexible device can still have its own power limit defined in its flex-model.

From 92318a1a8a8eb317fe0875ead2d3a97772588ab1 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sat, 18 Jan 2025 13:53:05 +0100
Subject: [PATCH 016/151] fix: wrong indentation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 136 +++++++++----------
 1 file changed, 66 insertions(+), 70 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 8e98e173f8..b67c13b5e3 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -502,42 +502,40 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 as_instantaneous_events=True,
                 resolve_overlaps="max",
             )
-            if self.flex_context.get("soc_minima_breach_price", None) is not None:
-                soc_minima_breach_price = self.flex_context.get(
-                    "soc_minima_breach_price"
-                )
-                soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_minima_breach_price,
-                    actuator=sensor,
-                    unit=(
-                        soc_minima_breach_price.unit
-                        if isinstance(soc_minima_breach_price, Sensor)
-                        else (
-                            soc_minima_breach_price[0]["value"].units
-                            if isinstance(soc_minima_breach_price, list)
-                            else str(soc_minima_breach_price.units)
-                        )
-                    ),
-                    query_window=(start, end),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-minima-breach-price",
-                    fill_sides=True,
-                )
-                # Set up commitments DataFrame
-                commitment = StockCommitment(
-                    name="soc minima",
-                    quantity=soc_minima,
-                    # negative price because breaching in the downwards (shortage) direction is penalized
-                    downwards_deviation_price=-soc_minima_breach_price,
-                    _type="any",
-                    index=index,
-                    device=0,
-                )
-                commitments.append(commitment)
+        if self.flex_context.get("soc_minima_breach_price", None) is not None:
+            soc_minima_breach_price = self.flex_context.get("soc_minima_breach_price")
+            soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
+                variable_quantity=soc_minima_breach_price,
+                actuator=sensor,
+                unit=(
+                    soc_minima_breach_price.unit
+                    if isinstance(soc_minima_breach_price, Sensor)
+                    else (
+                        soc_minima_breach_price[0]["value"].units
+                        if isinstance(soc_minima_breach_price, list)
+                        else str(soc_minima_breach_price.units)
+                    )
+                ),
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+                fallback_attribute="soc-minima-breach-price",
+                fill_sides=True,
+            )
+            # Set up commitments DataFrame
+            commitment = StockCommitment(
+                name="soc minima",
+                quantity=soc_minima,
+                # negative price because breaching in the downwards (shortage) direction is penalized
+                downwards_deviation_price=-soc_minima_breach_price,
+                _type="any",
+                index=index,
+                device=0,
+            )
+            commitments.append(commitment)
 
-                # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_minima = None
+            # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint
+            soc_minima = None
 
         if isinstance(soc_maxima, Sensor):
             soc_maxima = get_continuous_series_sensor_or_quantity(
@@ -550,42 +548,40 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 as_instantaneous_events=True,
                 resolve_overlaps="min",
             )
-            if self.flex_context.get("soc_maxima_breach_price", None) is not None:
-                soc_maxima_breach_price = self.flex_context.get(
-                    "soc_maxima_breach_price"
-                )
-                soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_maxima_breach_price,
-                    actuator=sensor,
-                    unit=(
-                        soc_maxima_breach_price.unit
-                        if isinstance(soc_maxima_breach_price, Sensor)
-                        else (
-                            soc_maxima_breach_price[0]["value"].units
-                            if isinstance(soc_maxima_breach_price, list)
-                            else str(soc_maxima_breach_price.units)
-                        )
-                    ),
-                    query_window=(start, end),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-maxima-breach-price",
-                    fill_sides=True,
-                )
-                # Set up commitments DataFrame
-                commitment = StockCommitment(
-                    name="soc maxima",
-                    quantity=soc_maxima,
-                    # positive price because breaching in the upwards (surplus) direction is penalized
-                    upwards_deviation_price=soc_maxima_breach_price,
-                    _type="any",
-                    index=index,
-                    device=0,
-                )
-                commitments.append(commitment)
+        if self.flex_context.get("soc_maxima_breach_price", None) is not None:
+            soc_maxima_breach_price = self.flex_context.get("soc_maxima_breach_price")
+            soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
+                variable_quantity=soc_maxima_breach_price,
+                actuator=sensor,
+                unit=(
+                    soc_maxima_breach_price.unit
+                    if isinstance(soc_maxima_breach_price, Sensor)
+                    else (
+                        soc_maxima_breach_price[0]["value"].units
+                        if isinstance(soc_maxima_breach_price, list)
+                        else str(soc_maxima_breach_price.units)
+                    )
+                ),
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+                fallback_attribute="soc-maxima-breach-price",
+                fill_sides=True,
+            )
+            # Set up commitments DataFrame
+            commitment = StockCommitment(
+                name="soc maxima",
+                quantity=soc_maxima,
+                # positive price because breaching in the upwards (surplus) direction is penalized
+                upwards_deviation_price=soc_maxima_breach_price,
+                _type="any",
+                index=index,
+                device=0,
+            )
+            commitments.append(commitment)
 
-                # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_maxima = None
+            # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
+            soc_maxima = None
 
         device_constraints[0] = add_storage_constraints(
             start,

From c191a775004d4676b9c69ae135b0222d4d91f88e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 21 Jan 2025 11:59:26 +0100
Subject: [PATCH 017/151] fix: return NaN quantity in the correct units in case
 of a missing fallback attribute

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index c14da577f1..fdb375dcf7 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -332,7 +332,7 @@ def idle_after_reaching_target(
 
 def get_quantity_from_attribute(
     entity: Asset | Sensor,
-    attribute: str,
+    attribute: str | None,
     unit: str | ur.Quantity,
 ) -> ur.Quantity:
     """Get the value (in the given unit) of a quantity stored as an entity attribute.
@@ -342,6 +342,9 @@ def get_quantity_from_attribute(
     :param unit:        The unit in which the value should be returned.
     :return:            The retrieved quantity or the provided default.
     """
+    if attribute is None:
+        return np.nan * ur.Quantity(unit)  # at least return result in the desired unit
+
     # Get the default value from the entity attribute
     value: str | float | int = entity.get_attribute(attribute, np.nan)
 

From c60b808482a08cee515fdf2363dd9bc9dc72c128 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 25 Feb 2025 17:57:56 +0100
Subject: [PATCH 018/151] dev: test StockCommitment in simultaneous scheduling

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index fe02eefbdc..5026b3b927 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -25,6 +25,7 @@
     get_sensors_from_db,
 )
 from flexmeasures.data.models.planning.utils import initialize_series, initialize_df
+from flexmeasures.data.models.planning import StockCommitment
 from flexmeasures.data.schemas.sensors import TimedEventSchema
 from flexmeasures.utils.calculations import (
     apply_stock_changes_and_losses,
@@ -2633,9 +2634,8 @@ def initialize_combined_commitments(num_devices: int):
             stock_commitment["downwards deviation price"] = -soc_target_penalty
             stock_commitment["upwards deviation price"] = soc_target_penalty
             stock_commitment["group"] = list(range(len(stock_commitment)))
-            # todo: amend test for https://github.com/FlexMeasures/flexmeasures/pull/1300
-            # stock_commitment["device"] = 0
-            # stock_commitment["class"] = StockCommitment
+            stock_commitment["device"] = 0
+            stock_commitment["class"] = StockCommitment
             commitments.append(stock_commitment)
 
         return commitments

From ec63bb6143ea532543e7e259de01b232e7a9545d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 10:01:09 +0100
Subject: [PATCH 019/151] fix: merge conflicts relating to soc_maxima/minima
 for multiple devices

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 5ef2976ef9..67a3169797 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -553,7 +553,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 # Set up commitments DataFrame
                 commitment = StockCommitment(
                     name="soc minima",
-                    quantity=soc_minima,
+                    quantity=soc_minima[d],
                     # negative price because breaching in the downwards (shortage) direction is penalized
                     downwards_deviation_price=-soc_minima_breach_price,
                     _type="any",
@@ -563,7 +563,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 commitments.append(commitment)
 
                 # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_minima = None
+                soc_minima[d] = None
 
             if isinstance(soc_maxima[d], Sensor):
                 soc_maxima[d] = get_continuous_series_sensor_or_quantity(
@@ -601,7 +601,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 # Set up commitments DataFrame
                 commitment = StockCommitment(
                     name="soc maxima",
-                    quantity=soc_maxima,
+                    quantity=soc_maxima[d],
                     # positive price because breaching in the upwards (surplus) direction is penalized
                     upwards_deviation_price=soc_maxima_breach_price,
                     _type="any",
@@ -611,7 +611,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 commitments.append(commitment)
 
                 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_maxima = None
+                soc_maxima[d] = None
 
             device_constraints[d] = add_storage_constraints(
                 start,

From 6f8e68993b0af20d74e6b9f439cd035867a7e9e9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 13:56:13 +0100
Subject: [PATCH 020/151] fix: failing test (also due to a mistake in resolving
 merge conflicts)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 67a3169797..c6a4e893de 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -558,7 +558,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     downwards_deviation_price=-soc_minima_breach_price,
                     _type="any",
                     index=index,
-                    device=0,
+                    device=d,
                 )
                 commitments.append(commitment)
 
@@ -606,7 +606,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     upwards_deviation_price=soc_maxima_breach_price,
                     _type="any",
                     index=index,
-                    device=0,
+                    device=d,
                 )
                 commitments.append(commitment)
 

From 875e95342b0984de148594c250ac206a38dcae4e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 13:57:03 +0100
Subject: [PATCH 021/151] docs: note cheap times relevant in test

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index db6ea61d15..783a38b0f5 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2564,10 +2564,10 @@ def test_multiple_devices_simultaneous_scheduler():
 
     market_prices = [
         0.8598,
-        1.4613,
+        1.4613,  # cheap from 01:00 to 02:00
         2430.3887,
         3000.1779,
-        18.6619,
+        18.6619,  # cheap from 04:00 to 0:500
         369.3274,
         169.8719,
         174.2279,

From cec296649ff519acb87dc89a57eb19a5265a63cb Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 14:14:21 +0100
Subject: [PATCH 022/151] feat: test SoC relaxation in sequential scheduling

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 38 +++++++++++++++----
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 783a38b0f5..1b44505002 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2801,6 +2801,7 @@ def test_multiple_devices_sequential_scheduler():
         229.8395,
         216.5779,
     ]
+    soc_target_penalty = 10000
 
     def initialize_device_constraints(
         num_devices: int,
@@ -2832,8 +2833,28 @@ def initialize_device_constraints(
 
     def initialize_device_commitments(num_devices: int):
         commitments = []
-        for _ in range(num_devices):
-            commitment = initialize_df(
+
+        # Model energy contract for the site
+        commitment = initialize_df(
+            [
+                "quantity",
+                "downwards deviation price",
+                "upwards deviation price",
+                "group",
+            ],
+            start,
+            end,
+            resolution,
+        )
+        commitment["quantity"] = 0
+        commitment["downwards deviation price"] = market_prices
+        commitment["upwards deviation price"] = market_prices
+        commitment["group"] = list(range(len(commitment)))
+        commitments.append(commitment)
+
+        # Model penalties for demand unmet per device
+        for i in range(num_devices):
+            stock_commitment = initialize_df(
                 [
                     "quantity",
                     "downwards deviation price",
@@ -2844,11 +2865,14 @@ def initialize_device_commitments(num_devices: int):
                 end,
                 resolution,
             )
-            commitment["quantity"] = 0
-            commitment["downwards deviation price"] = market_prices
-            commitment["upwards deviation price"] = market_prices
-            commitment["group"] = list(range(len(commitment)))
-            commitments.append(commitment)
+            stock_commitment["quantity"] = target_value[i] - soc_at_start[i]
+            stock_commitment["downwards deviation price"] = -soc_target_penalty
+            stock_commitment["upwards deviation price"] = soc_target_penalty
+            stock_commitment["group"] = list(range(len(stock_commitment)))
+            stock_commitment["device"] = i
+            stock_commitment["class"] = StockCommitment
+            commitments.append(stock_commitment)
+
         return commitments
 
     def initialize_ems_constraints():

From 200b59861080dec8df68ccc58da346053d1a1d6f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 14:22:43 +0100
Subject: [PATCH 023/151] refactor: enumerate using d rather than i

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 82 +++++++++----------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 1b44505002..3e6bc398c0 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2594,14 +2594,14 @@ def initialize_combined_constraints(
         num_devices: int, max_derivative: float = 1, min_derivative: float = 0
     ):
         device_constraints = []
-        for i in range(num_devices):
+        for d in range(num_devices):
             constraints = initialize_df(
                 StorageScheduler.COLUMNS, start, end, resolution
             )
-            start_time = pd.Timestamp(start_datetime[i]) - timedelta(hours=1)
-            target_time = pd.Timestamp(target_datetime[i])
-            constraints["max"] = soc_max[i] - soc_at_start[i]
-            constraints["min"] = soc_min[i] - soc_at_start[i]
+            start_time = pd.Timestamp(start_datetime[d]) - timedelta(hours=1)
+            target_time = pd.Timestamp(target_datetime[d])
+            constraints["max"] = soc_max[d] - soc_at_start[d]
+            constraints["min"] = soc_min[d] - soc_at_start[d]
             constraints["derivative max"] = max_derivative
             constraints["derivative min"] = min_derivative
             constraints.loc[
@@ -2637,7 +2637,7 @@ def initialize_combined_commitments(num_devices: int):
             energy_commitment["group"] = list(range(len(energy_commitment)))
             commitments.append(energy_commitment)
 
-        for i in range(num_devices):
+        for d in range(num_devices):
             stock_commitment = initialize_df(
                 [
                     "quantity",
@@ -2649,11 +2649,11 @@ def initialize_combined_commitments(num_devices: int):
                 end,
                 resolution,
             )
-            stock_commitment["quantity"] = target_value[i] - soc_at_start[i]
+            stock_commitment["quantity"] = target_value[d] - soc_at_start[d]
             stock_commitment["downwards deviation price"] = -soc_target_penalty
             stock_commitment["upwards deviation price"] = soc_target_penalty
             stock_commitment["group"] = list(range(len(stock_commitment)))
-            stock_commitment["device"] = 0
+            stock_commitment["device"] = d
             stock_commitment["class"] = StockCommitment
             commitments.append(stock_commitment)
 
@@ -2677,17 +2677,17 @@ def initialize_combined_commitments(num_devices: int):
 
     schedules = [
         initialize_series(
-            data=[model.ems_power[i, j].value for j in model.j],
+            data=[model.ems_power[d, j].value for j in model.j],
             start=start,
             end=end,
             resolution=resolution,
         )
-        for i in range(num_devices)
+        for d in range(num_devices)
     ]
 
     individual_costs = [
-        (i, sum(schedule[j] * market_prices[j] for j in range(len(market_prices))))
-        for i, schedule in enumerate(schedules)
+        (d, sum(schedule[j] * market_prices[j] for j in range(len(market_prices))))
+        for d, schedule in enumerate(schedules)
     ]
 
     # Expected results with no unmet demand
@@ -2700,13 +2700,13 @@ def initialize_combined_commitments(num_devices: int):
 
     # Assertions
     assert all(
-        np.isclose(schedule, expected_schedules[i]).all()
-        for i, schedule in enumerate(schedules)
+        np.isclose(schedule, expected_schedules[d]).all()
+        for d, schedule in enumerate(schedules)
     ), "Schedules mismatch: Device schedules do not match the expected schedules."
 
     assert all(
-        device == i and pytest.approx(cost, 0.01) == expected_individual_costs[i][1]
-        for i, (device, cost) in enumerate(individual_costs)
+        device == d and pytest.approx(cost, 0.01) == expected_individual_costs[d][1]
+        for d, (device, cost) in enumerate(individual_costs)
     ), "Individual costs mismatch: Costs for one or more devices are not calculated as expected."
 
     # Test case 2: With lower EMS capacity and unmet demand
@@ -2726,17 +2726,17 @@ def initialize_combined_commitments(num_devices: int):
 
     schedules = [
         initialize_series(
-            data=[model.ems_power[i, j].value for j in model.j],
+            data=[model.ems_power[d, j].value for j in model.j],
             start=start,
             end=end,
             resolution=resolution,
         )
-        for i in range(num_devices)
+        for d in range(num_devices)
     ]
 
     individual_costs = [
-        (i, sum(schedule[j] * market_prices[j] for j in range(len(market_prices))))
-        for i, schedule in enumerate(schedules)
+        (d, sum(schedule[j] * market_prices[j] for j in range(len(market_prices))))
+        for d, schedule in enumerate(schedules)
     ]
 
     # Expected results with unmet demand
@@ -2752,13 +2752,13 @@ def initialize_combined_commitments(num_devices: int):
 
     # Assertions
     assert all(
-        np.isclose(schedule, expected_schedules[i]).all()
-        for i, schedule in enumerate(schedules)
+        np.isclose(schedule, expected_schedules[d]).all()
+        for d, schedule in enumerate(schedules)
     ), "Schedules mismatch: Device schedules do not match the expected schedules."
 
     assert all(
-        device == i and pytest.approx(cost, 0.01) == expected_individual_costs[i][1]
-        for i, (device, cost) in enumerate(individual_costs)
+        device == d and pytest.approx(cost, 0.01) == expected_individual_costs[d][1]
+        for d, (device, cost) in enumerate(individual_costs)
     ), "Individual costs mismatch: Costs for one or more devices are not calculated as expected."
 
 
@@ -2813,18 +2813,18 @@ def initialize_device_constraints(
         start_datetime: list[str],
     ):
         device_constraints = []
-        for i in range(num_devices):
+        for d in range(num_devices):
             constraints = initialize_df(
                 StorageScheduler.COLUMNS, start, end, resolution
             )
 
-            start_time = pd.Timestamp(start_datetime[i]) - timedelta(hours=1)
+            start_time = pd.Timestamp(start_datetime[d]) - timedelta(hours=1)
 
-            constraints["max"] = soc_max[i] - soc_at_start[i]
-            constraints["min"] = soc_min[i] - soc_at_start[i]
+            constraints["max"] = soc_max[d] - soc_at_start[d]
+            constraints["min"] = soc_min[d] - soc_at_start[d]
             constraints["derivative max"] = 1
             constraints["derivative min"] = 0
-            constraints["min"][target_datetime[i]] = target_value[i] - soc_at_start[i]
+            constraints["min"][target_datetime[d]] = target_value[d] - soc_at_start[d]
             constraints.loc[
                 :start_time, ["max", "min", "derivative max", "derivative min"]
             ] = 0
@@ -2853,7 +2853,7 @@ def initialize_device_commitments(num_devices: int):
         commitments.append(commitment)
 
         # Model penalties for demand unmet per device
-        for i in range(num_devices):
+        for d in range(num_devices):
             stock_commitment = initialize_df(
                 [
                     "quantity",
@@ -2865,11 +2865,11 @@ def initialize_device_commitments(num_devices: int):
                 end,
                 resolution,
             )
-            stock_commitment["quantity"] = target_value[i] - soc_at_start[i]
+            stock_commitment["quantity"] = target_value[d] - soc_at_start[d]
             stock_commitment["downwards deviation price"] = -soc_target_penalty
             stock_commitment["upwards deviation price"] = soc_target_penalty
             stock_commitment["group"] = list(range(len(stock_commitment)))
-            stock_commitment["device"] = i
+            stock_commitment["device"] = d
             stock_commitment["class"] = StockCommitment
             commitments.append(stock_commitment)
 
@@ -2904,13 +2904,13 @@ def run_sequential_scheduler():
         combined_schedule = [0] * len(market_prices)
         unmet_targets = []
 
-        for i in range(num_devices):
-            initial_stock = soc_at_start[i]
+        for d in range(num_devices):
+            initial_stock = soc_at_start[d]
 
             _, _, results, model = device_scheduler(
-                device_constraints=[device_constraints[i]],
+                device_constraints=[device_constraints[d]],
                 ems_constraints=ems_constraints,
-                commitments=[commitments[i]],
+                commitments=[commitments[d]],
                 initial_stock=initial_stock,
             )
 
@@ -2934,8 +2934,8 @@ def run_sequential_scheduler():
             total_costs.append(total_cost)
 
             final_soc = initial_stock + sum(schedule)
-            if final_soc < target_value[i]:
-                unmet_targets.append((i, final_soc))
+            if final_soc < target_value[d]:
+                unmet_targets.append((d, final_soc))
 
         return (
             all_schedules,
@@ -2956,12 +2956,12 @@ def run_sequential_scheduler():
     expected_costs = [(0, 1.46), (1, 2430.39)]
 
     assert all(
-        np.isclose(schedules[i], expected_schedules[i]).all()
-        for i in range(len(schedules))
+        np.isclose(schedules[d], expected_schedules[d]).all()
+        for d in range(len(schedules))
     ), "Schedules do not match expected values."
 
     assert all(
-        pytest.approx(costs[i], 0.01) == expected_costs[i][1] for i in range(len(costs))
+        pytest.approx(costs[d], 0.01) == expected_costs[d][1] for d in range(len(costs))
     ), "Costs do not match expected values."
 
     assert total_cost_all_devices == sum(

From 3d1c99910c2fdb27ab6d416dad96044c925d86d8 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 14:42:25 +0100
Subject: [PATCH 024/151] docs: clarifying comment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 3e6bc398c0..fb0cda3ae5 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2925,6 +2925,7 @@ def run_sequential_scheduler():
             for j in range(len(schedule)):
                 combined_schedule[j] += schedule[j]
 
+            # Determine new headroom
             ems_constraints["derivative max"] -= schedule
             ems_constraints["derivative min"] -= schedule
 

From 8830b8dcdef839d315bc41126e1728b66ea684a8 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 15:38:39 +0100
Subject: [PATCH 025/151] refactor: move reused functions to utils

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 52 +++++++----------
 flexmeasures/data/models/planning/utils.py    | 57 +++++++++++++++++++
 2 files changed, 77 insertions(+), 32 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index fb0cda3ae5..4fd71e168e 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -24,7 +24,12 @@
     check_constraints,
     get_sensors_from_db,
 )
-from flexmeasures.data.models.planning.utils import initialize_series, initialize_df
+from flexmeasures.data.models.planning.utils import (
+    initialize_device_commitment,
+    initialize_df,
+    initialize_energy_commitment,
+    initialize_series,
+)
 from flexmeasures.data.models.planning import StockCommitment
 from flexmeasures.data.schemas.sensors import TimedEventSchema
 from flexmeasures.utils.calculations import (
@@ -2834,43 +2839,26 @@ def initialize_device_constraints(
     def initialize_device_commitments(num_devices: int):
         commitments = []
 
-        # Model energy contract for the site
-        commitment = initialize_df(
-            [
-                "quantity",
-                "downwards deviation price",
-                "upwards deviation price",
-                "group",
-            ],
-            start,
-            end,
-            resolution,
+        commitment = initialize_energy_commitment(
+            start=start,
+            end=end,
+            resolution=resolution,
+            market_prices=market_prices,
         )
-        commitment["quantity"] = 0
-        commitment["downwards deviation price"] = market_prices
-        commitment["upwards deviation price"] = market_prices
-        commitment["group"] = list(range(len(commitment)))
         commitments.append(commitment)
 
         # Model penalties for demand unmet per device
         for d in range(num_devices):
-            stock_commitment = initialize_df(
-                [
-                    "quantity",
-                    "downwards deviation price",
-                    "upwards deviation price",
-                    "group",
-                ],
-                start,
-                end,
-                resolution,
+            stock_commitment = initialize_device_commitment(
+                start=start,
+                end=end,
+                resolution=resolution,
+                device=d,
+                target_datetime=target_datetime[d],
+                target_value=target_value[d],
+                soc_at_start=soc_at_start[d],
+                soc_target_penalty=soc_target_penalty,
             )
-            stock_commitment["quantity"] = target_value[d] - soc_at_start[d]
-            stock_commitment["downwards deviation price"] = -soc_target_penalty
-            stock_commitment["upwards deviation price"] = soc_target_penalty
-            stock_commitment["group"] = list(range(len(stock_commitment)))
-            stock_commitment["device"] = d
-            stock_commitment["class"] = StockCommitment
             commitments.append(stock_commitment)
 
         return commitments
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index c0a4e0a36b..658c27da28 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -17,6 +17,7 @@
     UnknownPricesException,
 )
 from flexmeasures import Asset
+from flexmeasures.data.models.planning import StockCommitment
 from flexmeasures.data.queries.utils import simplify_index
 
 from flexmeasures.utils.flexmeasures_inflection import capitalize, pluralize
@@ -568,3 +569,59 @@ def nanmin_of_series_and_value(s: pd.Series, value: float | pd.Series) -> pd.Ser
         # [right]: datetime64[ns, UTC]
         value = value.tz_convert("UTC")
     return s.fillna(value).clip(upper=value)
+
+
+def initialize_energy_commitment(
+    start: pd.Timestamp,
+    end: pd.Timestamp,
+    resolution: timedelta,
+    market_prices: list[float],
+) -> pd.DataFrame:
+    """Model energy contract for the site."""
+    commitment = initialize_df(
+        columns=[
+            "quantity",
+            "downwards deviation price",
+            "upwards deviation price",
+            "group",
+        ],
+        start=start,
+        end=end,
+        resolution=resolution,
+    )
+    commitment["quantity"] = 0
+    commitment["downwards deviation price"] = market_prices
+    commitment["upwards deviation price"] = market_prices
+    commitment["group"] = list(range(len(commitment)))
+    return commitment
+
+
+def initialize_device_commitment(
+    start: pd.Timestamp,
+    end: pd.Timestamp,
+    resolution: timedelta,
+    device: int,
+    target_datetime: str,
+    target_value: float,
+    soc_at_start: float,
+    soc_target_penalty: float,
+) -> pd.DataFrame:
+    """Model energy contract for the site."""
+    stock_commitment = initialize_df(
+        columns=[
+            "quantity",
+            "downwards deviation price",
+            "upwards deviation price",
+            "group",
+        ],
+        start=start,
+        end=end,
+        resolution=resolution,
+    )
+    stock_commitment.loc[target_datetime, "quantity"] = target_value - soc_at_start
+    stock_commitment["downwards deviation price"] = -soc_target_penalty
+    stock_commitment["upwards deviation price"] = soc_target_penalty
+    stock_commitment["group"] = list(range(len(stock_commitment)))
+    stock_commitment["device"] = device
+    stock_commitment["class"] = StockCommitment
+    return stock_commitment

From 10d46c6202bb7d02c6acc629709dd69aa4b8047e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 16:24:44 +0100
Subject: [PATCH 026/151] fix: Pyomo deprecation

WARNING  pyomo.core.base.param:deprecation.py:238 DEPRECATED: Param 'commitment_quantity' declared with an implicit domain of 'Any'. The default domain for Param objects is 'Any'. However, we will be changing that default to 'Reals' in the future. If you really intend the domain of this Param to be 'Any', you can suppress this warning by explicitly specifying 'within=Any' to the Param constructor.  (deprecated in 5.6.9, will be removed in (or after) 6.0)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/linear_optimization.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index 716273e87f..f1d40fa6c7 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -248,7 +248,10 @@ def price_up_select(m, c):
         return price
 
     def commitment_quantity_select(m, c):
-        return commitments[c]["quantity"].iloc[0]
+        quantity = commitments[c]["quantity"].iloc[0]
+        if np.isnan(quantity):
+            return 0
+        return quantity
 
     def device_max_select(m, d, j):
         min_v = device_constraints[d]["min"].iloc[j]
@@ -341,7 +344,9 @@ def device_stock_delta(m, d, j):
 
     model.up_price = Param(model.c, initialize=price_up_select)
     model.down_price = Param(model.c, initialize=price_down_select)
-    model.commitment_quantity = Param(model.c, initialize=commitment_quantity_select)
+    model.commitment_quantity = Param(
+        model.c, domain=Reals, initialize=commitment_quantity_select
+    )
     model.device_max = Param(model.d, model.j, initialize=device_max_select)
     model.device_min = Param(model.d, model.j, initialize=device_min_select)
     model.device_derivative_max = Param(

From ad100be9d54032d1f4b967b7ed1ead7e9142116b Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 16:26:05 +0100
Subject: [PATCH 027/151] fix: Pandas FutureWarning

FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
    ].fillna(0)
Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/linear_optimization.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index f1d40fa6c7..819cbb162e 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -212,9 +212,9 @@ def convert_commitments_to_subcommitments(
         if "stock delta" not in device_constraints[d].columns:
             device_constraints[d]["stock delta"] = 0
         else:
-            device_constraints[d]["stock delta"] = device_constraints[d][
-                "stock delta"
-            ].fillna(0)
+            device_constraints[d]["stock delta"] = (
+                device_constraints[d]["stock delta"].astype(float).fillna(0)
+            )
 
     # Add indices for devices (d), datetimes (j) and commitments (c)
     model.d = RangeSet(0, len(device_constraints) - 1, doc="Set of devices")

From 79172a1001b1cc15fa2461995e0dfb1b55604e72 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 17:32:00 +0100
Subject: [PATCH 028/151] refactor: util function to compute stock change

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../models/planning/linear_optimization.py    | 49 +++++++------------
 1 file changed, 19 insertions(+), 30 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index 819cbb162e..44e96e3bd8 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -386,10 +386,12 @@ def device_stock_delta(m, d, j):
         # bounds=[None, 1000],
     )
 
-    # Add constraints as a tuple of (lower bound, value, upper bound)
-    def device_bounds(m, d, j):
-        """Apply conversion efficiencies to conversion from flow to stock change and vice versa,
-        and apply storage efficiencies to stock levels from one datetime to the next."""
+    def _get_stock_change(m, d, j):
+        """Determine final stock change of device d until time j.
+
+        Apply conversion efficiencies to conversion from flow to stock change and vice versa,
+        and apply storage efficiencies to stock levels from one datetime to the next.
+        """
         if isinstance(initial_stock, list):
             # No initial stock defined for inflexible device
             initial_stock_d = initial_stock[d] if d < len(initial_stock) else 0
@@ -405,14 +407,20 @@ def device_bounds(m, d, j):
             for k in range(0, j + 1)
         ]
         efficiencies = [m.device_efficiency[d, k] for k in range(0, j + 1)]
+        final_stock_change = [
+            stock - initial_stock_d
+            for stock in apply_stock_changes_and_losses(
+                initial_stock_d, stock_changes, efficiencies
+            )
+        ][-1]
+        return final_stock_change
+
+    # Add constraints as a tuple of (lower bound, value, upper bound)
+    def device_bounds(m, d, j):
+        """Constraints on the device's stock."""
         return (
             m.device_min[d, j],
-            [
-                stock - initial_stock_d
-                for stock in apply_stock_changes_and_losses(
-                    initial_stock_d, stock_changes, efficiencies
-                )
-            ][-1],
+            _get_stock_change(m, d, j),
             m.device_max[d, j],
         )
 
@@ -462,20 +470,6 @@ def device_stock_commitment_equalities(m, c, j, d):
             raise NotImplementedError(
                 "FlowCommitment on a device level has not been implemented. Please file a GitHub ticket explaining your use case."
             )
-        if isinstance(initial_stock, list):
-            initial_stock_d = initial_stock[d]
-        else:
-            initial_stock_d = initial_stock
-
-        stock_changes = [
-            (
-                m.device_power_down[d, k] / m.device_derivative_down_efficiency[d, k]
-                + m.device_power_up[d, k] * m.device_derivative_up_efficiency[d, k]
-                + m.stock_delta[d, k]
-            )
-            for k in range(0, j + 1)
-        ]
-        efficiencies = [m.device_efficiency[d, k] for k in range(0, j + 1)]
         return (
             (
                 0
@@ -487,12 +481,7 @@ def device_stock_commitment_equalities(m, c, j, d):
             m.commitment_quantity[c]
             + m.commitment_downwards_deviation[c]
             + m.commitment_upwards_deviation[c]
-            - [
-                stock - initial_stock_d
-                for stock in apply_stock_changes_and_losses(
-                    initial_stock_d, stock_changes, efficiencies
-                )
-            ][-1],
+            - _get_stock_change(m, d, j),
             (
                 0
                 if len(commitments[c]) == 1

From cc72bf831ebb882f0eecfc0d55fe1ccfac833f0b Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 17:49:24 +0100
Subject: [PATCH 029/151] refactor: move reused functions to utils

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 55 +++++++------------
 1 file changed, 20 insertions(+), 35 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 4fd71e168e..9f552f783b 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -30,7 +30,6 @@
     initialize_energy_commitment,
     initialize_series,
 )
-from flexmeasures.data.models.planning import StockCommitment
 from flexmeasures.data.schemas.sensors import TimedEventSchema
 from flexmeasures.utils.calculations import (
     apply_stock_changes_and_losses,
@@ -2624,43 +2623,29 @@ def initialize_combined_constraints(
 
     def initialize_combined_commitments(num_devices: int):
         commitments = []
-        for _ in range(num_devices):
-            energy_commitment = initialize_df(
-                [
-                    "quantity",
-                    "downwards deviation price",
-                    "upwards deviation price",
-                    "group",
-                ],
-                start,
-                end,
-                resolution,
-            )
-            energy_commitment["quantity"] = 0
-            energy_commitment["downwards deviation price"] = market_prices
-            energy_commitment["upwards deviation price"] = market_prices
-            energy_commitment["group"] = list(range(len(energy_commitment)))
-            commitments.append(energy_commitment)
 
+        # Model energy contract for the site
+        energy_commitment = initialize_energy_commitment(
+            start=start,
+            end=end,
+            resolution=resolution,
+            market_prices=market_prices,
+        )
+        commitments.append(energy_commitment)
+
+        # Model penalties for demand unmet per device
         for d in range(num_devices):
-            stock_commitment = initialize_df(
-                [
-                    "quantity",
-                    "downwards deviation price",
-                    "upwards deviation price",
-                    "group",
-                ],
-                start,
-                end,
-                resolution,
+            device_commitment = initialize_device_commitment(
+                start=start,
+                end=end,
+                resolution=resolution,
+                device=d,
+                target_datetime=target_datetime[d],
+                target_value=target_value[d],
+                soc_at_start=soc_at_start[d],
+                soc_target_penalty=soc_target_penalty,
             )
-            stock_commitment["quantity"] = target_value[d] - soc_at_start[d]
-            stock_commitment["downwards deviation price"] = -soc_target_penalty
-            stock_commitment["upwards deviation price"] = soc_target_penalty
-            stock_commitment["group"] = list(range(len(stock_commitment)))
-            stock_commitment["device"] = d
-            stock_commitment["class"] = StockCommitment
-            commitments.append(stock_commitment)
+            commitments.append(device_commitment)
 
         return commitments
 

From bbeba98c26c55a3684e97c0a37a8fa132d65b174 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 18:03:58 +0100
Subject: [PATCH 030/151] fix: ensure type of commitment is known

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/linear_optimization.py     | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index 44e96e3bd8..f2b62e939c 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -22,7 +22,7 @@
 from pyomo.environ import value
 from pyomo.opt import SolverFactory, SolverResults
 
-from flexmeasures.data.models.planning import Commitment
+from flexmeasures.data.models.planning import Commitment, FlowCommitment
 from flexmeasures.data.models.planning.utils import initialize_series, initialize_df
 from flexmeasures.utils.calculations import apply_stock_changes_and_losses
 
@@ -166,6 +166,12 @@ def convert_commitments_to_subcommitments(
         commitment_mapping = {}
         sub_commitments = []
         for c, df in enumerate(dfs):
+            # Make sure each commitment has "device" (default NaN) and "class" (default FlowCommitment) columns
+            if "device" not in df.columns:
+                df["device"] = np.nan
+            if "class" not in df.columns:
+                df["class"] = FlowCommitment
+
             df["j"] = range(len(df.index))
             groups = list(df["group"].unique())
             for group in groups:
@@ -497,7 +503,10 @@ def ems_flow_commitment_equalities(m, c, j):
         - Creates an inequality for one-sided commitments.
         - Creates an equality for two-sided commitments and for groups of size 1.
         """
-        if "device" in commitments[c].columns:
+        if (
+            "device" in commitments[c].columns
+            and not pd.isnull(commitments[c]["device"]).all()
+        ):
             # Commitment c does not concern EMS
             return Constraint.Skip
         if "class" in commitments[c].columns and not all(

From 9c38cbf545a2f2589683466121cc0ffd8883f525 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 18:07:38 +0100
Subject: [PATCH 031/151] fix: test case

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 9f552f783b..88dab32a84 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2762,15 +2762,15 @@ def test_multiple_devices_sequential_scheduler():
     soc_min = [0] * 2
 
     start_datetime = ["2023-01-01 01:00:00"] * 2
-    target_datetime = ["2023-01-01 04:00:00", "2023-01-01 03:00:00"]
+    target_datetime = ["2023-01-01 05:00:00", "2023-01-01 03:00:00"]
     target_value = [1] * 2
 
     market_prices = [
         0.8598,
-        1.4613,
+        1.4613,  # cheap from 01:00 to 02:00
         2430.3887,
         3000.1779,
-        18.6619,
+        18.6619,  # cheap from 04:00 to 0:500
         369.3274,
         169.8719,
         174.2279,
@@ -2925,7 +2925,7 @@ def run_sequential_scheduler():
 
     expected_schedules = [
         [0, 1] + [0] * 22,
-        [0, 0, 1] + [0] * 21,
+        [0, 0, 0, 0, 1] + [0] * 19,
     ]
     expected_costs = [(0, 1.46), (1, 2430.39)]
 

From 118b7594e5ef934851e300b31643b7fdf3cb0890 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 21:35:39 +0100
Subject: [PATCH 032/151] refactor: use loc

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 88dab32a84..0eccc10970 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2814,7 +2814,9 @@ def initialize_device_constraints(
             constraints["min"] = soc_min[d] - soc_at_start[d]
             constraints["derivative max"] = 1
             constraints["derivative min"] = 0
-            constraints["min"][target_datetime[d]] = target_value[d] - soc_at_start[d]
+            constraints.loc[target_datetime[d], "min"] = (
+                target_value[d] - soc_at_start[d]
+            )
             constraints.loc[
                 :start_time, ["max", "min", "derivative max", "derivative min"]
             ] = 0

From 852ccedb03306e90b944c0dd723bb17557fa9226 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 21:38:37 +0100
Subject: [PATCH 033/151] docs: clarifying comment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 0eccc10970..94bc45d909 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2882,6 +2882,7 @@ def run_sequential_scheduler():
         for d in range(num_devices):
             initial_stock = soc_at_start[d]
 
+            # Compute the schedule for device d
             _, _, results, model = device_scheduler(
                 device_constraints=[device_constraints[d]],
                 ems_constraints=ems_constraints,

From 7fa10cb0ca2840ee6669335ee804f6beca12cf3c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 21:39:28 +0100
Subject: [PATCH 034/151] feat: check termination condition of solver

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 94bc45d909..93013fffe9 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -1115,6 +1115,7 @@ def test_numerical_errors(app_with_each_solver, setup_planning_test_data, db):
             for soc_at_start_d in soc_at_start
         ],
     )
+    assert results.solver.termination_condition == "optimal"
 
     assert device_constraints[0]["equals"].max() > device_constraints[0]["max"].max()
     assert device_constraints[0]["equals"].min() < device_constraints[0]["min"].min()
@@ -2343,7 +2344,7 @@ def run_scheduler(device_constraints):
             commitments=commitments,
             initial_stock=soc_at_start,
         )
-        print(results.solver.termination_condition)
+        assert results.solver.termination_condition == "optimal"
 
         schedule = initialize_series(
             data=[model.ems_power[0, j].value for j in model.j],
@@ -2475,7 +2476,7 @@ def run_scheduler():
             commitments=commitments,
             initial_stock=soc_at_start,
         )
-        print(results.solver.termination_condition)
+        assert results.solver.termination_condition == "optimal"
 
         schedule = initialize_series(
             data=[model.ems_power[0, j].value for j in model.j],
@@ -2664,6 +2665,7 @@ def initialize_combined_commitments(num_devices: int):
         commitments=commitments,
         initial_stock=initial_stocks,
     )
+    assert results.solver.termination_condition == "optimal"
 
     schedules = [
         initialize_series(
@@ -2713,6 +2715,7 @@ def initialize_combined_commitments(num_devices: int):
         commitments=commitments,
         initial_stock=initial_stocks,
     )
+    assert results.solver.termination_condition == "optimal"
 
     schedules = [
         initialize_series(
@@ -2889,6 +2892,7 @@ def run_sequential_scheduler():
                 commitments=[commitments[d]],
                 initial_stock=initial_stock,
             )
+            assert results.solver.termination_condition == "optimal"
 
             schedule = initialize_series(
                 data=[model.ems_power[0, j].value for j in model.j],

From fb05ba15ff6c79632b3bd2b15954c2ba07bb48cd Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 21:54:31 +0100
Subject: [PATCH 035/151] fix(docs): correct docstring

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 658c27da28..1c5453225c 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -606,7 +606,7 @@ def initialize_device_commitment(
     soc_at_start: float,
     soc_target_penalty: float,
 ) -> pd.DataFrame:
-    """Model energy contract for the site."""
+    """Model penalties for demand unmet per device."""
     stock_commitment = initialize_df(
         columns=[
             "quantity",

From 6ad0a4e9db10a1f6af6bf69e4df35ba61d25bcfc Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 6 Mar 2025 21:56:20 +0100
Subject: [PATCH 036/151] fix: pass the right commitments

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 50 ++++++++-----------
 1 file changed, 21 insertions(+), 29 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 93013fffe9..fc05a250e9 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2826,33 +2826,6 @@ def initialize_device_constraints(
             device_constraints.append(constraints)
         return device_constraints
 
-    def initialize_device_commitments(num_devices: int):
-        commitments = []
-
-        commitment = initialize_energy_commitment(
-            start=start,
-            end=end,
-            resolution=resolution,
-            market_prices=market_prices,
-        )
-        commitments.append(commitment)
-
-        # Model penalties for demand unmet per device
-        for d in range(num_devices):
-            stock_commitment = initialize_device_commitment(
-                start=start,
-                end=end,
-                resolution=resolution,
-                device=d,
-                target_datetime=target_datetime[d],
-                target_value=target_value[d],
-                soc_at_start=soc_at_start[d],
-                soc_target_penalty=soc_target_penalty,
-            )
-            commitments.append(stock_commitment)
-
-        return commitments
-
     def initialize_ems_constraints():
         ems_constraints = initialize_df(
             StorageScheduler.COLUMNS, start, end, resolution
@@ -2873,7 +2846,14 @@ def run_sequential_scheduler():
             target_value=target_value,
             start_datetime=start_datetime,
         )
-        commitments = initialize_device_commitments(num_devices)
+
+        # Model energy contract for the site
+        energy_commitment = initialize_energy_commitment(
+            start=start,
+            end=end,
+            resolution=resolution,
+            market_prices=market_prices,
+        )
 
         ems_constraints = initialize_ems_constraints()
 
@@ -2885,11 +2865,23 @@ def run_sequential_scheduler():
         for d in range(num_devices):
             initial_stock = soc_at_start[d]
 
+            # Model penalties for demand unmet per device
+            device_commitment = initialize_device_commitment(
+                start=start,
+                end=end,
+                resolution=resolution,
+                device=d,
+                target_datetime=target_datetime[d],
+                target_value=target_value[d],
+                soc_at_start=soc_at_start[d],
+                soc_target_penalty=soc_target_penalty,
+            )
+
             # Compute the schedule for device d
             _, _, results, model = device_scheduler(
                 device_constraints=[device_constraints[d]],
                 ems_constraints=ems_constraints,
-                commitments=[commitments[d]],
+                commitments=[energy_commitment, device_commitment],
                 initial_stock=initial_stock,
             )
             assert results.solver.termination_condition == "optimal"

From 2ff191f7d4a7a5b5bb585c6728a00da992c8fb10 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 7 Mar 2025 15:39:19 +0100
Subject: [PATCH 037/151] fix: correct device number in case of sequential
 scheduling

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py         | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index fc05a250e9..3c01b241bd 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2817,9 +2817,9 @@ def initialize_device_constraints(
             constraints["min"] = soc_min[d] - soc_at_start[d]
             constraints["derivative max"] = 1
             constraints["derivative min"] = 0
-            constraints.loc[target_datetime[d], "min"] = (
-                target_value[d] - soc_at_start[d]
-            )
+            # constraints.loc[target_datetime[d], "min"] = (
+            #     target_value[d] - soc_at_start[d]
+            # )
             constraints.loc[
                 :start_time, ["max", "min", "derivative max", "derivative min"]
             ] = 0
@@ -2870,7 +2870,7 @@ def run_sequential_scheduler():
                 start=start,
                 end=end,
                 resolution=resolution,
-                device=d,
+                device=0,
                 target_datetime=target_datetime[d],
                 target_value=target_value[d],
                 soc_at_start=soc_at_start[d],
@@ -2928,6 +2928,9 @@ def run_sequential_scheduler():
     ]
     expected_costs = [(0, 1.46), (1, 2430.39)]
 
+    for d, schedule in enumerate(schedules):
+        print(schedule)
+        print(expected_schedules[d])
     assert all(
         np.isclose(schedules[d], expected_schedules[d]).all()
         for d in range(len(schedules))

From 82d0a40c55f9bfde619855f56dcb8582959ba57d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 7 Mar 2025 17:06:06 +0100
Subject: [PATCH 038/151] dev: ugly fix

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/linear_optimization.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index f2b62e939c..be13f3817e 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -256,7 +256,7 @@ def price_up_select(m, c):
     def commitment_quantity_select(m, c):
         quantity = commitments[c]["quantity"].iloc[0]
         if np.isnan(quantity):
-            return 0
+            return -infinity
         return quantity
 
     def device_max_select(m, d, j):
@@ -467,8 +467,10 @@ def ems_derivative_bounds(m, j):
     def device_stock_commitment_equalities(m, c, j, d):
         """Couple device stocks to each commitment."""
         if (
-            "device" not in commitments[c].columns
+            "device"
+            not in commitments[c].columns  # "device" not in commitments[28].columns
             or (commitments[c]["device"] != d).all()
+            or m.commitment_quantity[c] == -infinity
         ):
             # Commitment c does not concern device d
             return Constraint.Skip
@@ -506,7 +508,7 @@ def ems_flow_commitment_equalities(m, c, j):
         if (
             "device" in commitments[c].columns
             and not pd.isnull(commitments[c]["device"]).all()
-        ):
+        ) or m.commitment_quantity[c] == -infinity:
             # Commitment c does not concern EMS
             return Constraint.Skip
         if "class" in commitments[c].columns and not all(

From debf61a20f85e129f6810fbfe2f2c23a90595f7f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 7 Mar 2025 17:14:20 +0100
Subject: [PATCH 039/151] fix: revise tests

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 46 ++++++++++---------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 3c01b241bd..dacc4c8442 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2564,7 +2564,7 @@ def test_multiple_devices_simultaneous_scheduler():
     soc_at_start = [0] * num_devices
     soc_max, soc_min = [1] * num_devices, [0] * num_devices
     start_datetime = ["2020-01-01 01:00:00"] * num_devices
-    target_datetime = ["2020-01-01 04:00:00", "2020-01-01 03:00:00"]
+    target_datetime = ["2020-01-01 06:00:00", "2020-01-01 03:00:00"]
     target_value = [1] * num_devices
 
     market_prices = [
@@ -2604,7 +2604,6 @@ def initialize_combined_constraints(
                 StorageScheduler.COLUMNS, start, end, resolution
             )
             start_time = pd.Timestamp(start_datetime[d]) - timedelta(hours=1)
-            target_time = pd.Timestamp(target_datetime[d])
             constraints["max"] = soc_max[d] - soc_at_start[d]
             constraints["min"] = soc_min[d] - soc_at_start[d]
             constraints["derivative max"] = max_derivative
@@ -2612,12 +2611,6 @@ def initialize_combined_constraints(
             constraints.loc[
                 :start_time, ["max", "min", "derivative max", "derivative min"]
             ] = 0
-            constraints.loc[
-                target_time + resolution :, ["derivative max", "derivative min"]
-            ] = 0
-            constraints.loc[target_time + resolution :, ["max", "min"]] = (
-                constraints.loc[target_time, ["max", "min"]].values
-            )
             device_constraints.append(constraints)
 
         return device_constraints
@@ -2684,8 +2677,10 @@ def initialize_combined_commitments(num_devices: int):
 
     # Expected results with no unmet demand
     expected_schedules = [
-        [0] * 4 + [1] + [0] * 19,
-        [0, 1] + [0] * 22,
+        [0] * 4
+        + [1]
+        + [0] * 19,  # the first EV leaves later, and takes the second-cheapest slot
+        [0, 1] + [0] * 22,  # the second EV leaves earlier, and gets the cheapest slot
     ]
     total_expected_demand = np.array(expected_schedules).sum()
     expected_individual_costs = [(0, 18.66), (1, 1.46)]
@@ -2703,11 +2698,13 @@ def initialize_combined_commitments(num_devices: int):
 
     # Test case 2: With lower EMS capacity and unmet demand
     ems_constraints = initialize_df(StorageScheduler.COLUMNS, start, end, resolution)
-    ems_constraints["derivative max"] = 0.4
+    ems_constraints["derivative max"] = 0.25
     ems_constraints["derivative min"] = 0
 
     device_constraints = initialize_combined_constraints(
-        num_devices, max_derivative=0.4, min_derivative=0
+        num_devices,
+        max_derivative=0.25,
+        min_derivative=0,  # todo: change does not seem required
     )
     _, _, results, model = device_scheduler(
         device_constraints=device_constraints,
@@ -2732,16 +2729,19 @@ def initialize_combined_commitments(num_devices: int):
         for d, schedule in enumerate(schedules)
     ]
 
-    # Expected results with unmet demand
+    # Expected results with unfair unmet demand and unfair costs
     expected_schedules = [
-        [0, 0.4, 0, 0, 0.4] + [0] * 19,
-        [0, 0, 0.4, 0.4] + [0] * 20,
+        [0, 0.25, 0, 0, 0.25, 0.25, 0.25]
+        + [0] * 17,  # the first EV leaves later, and takes the four cheapest slots
+        [0, 0, 0.25, 0.25]
+        + [0]
+        * 20,  # the second EV leaves earlier, and takes the remaining (expensive) slots
     ]
     total_expected_demand_unmet = (
         total_expected_demand - np.array(expected_schedules).sum()
     )
     assert total_expected_demand_unmet > 0
-    expected_individual_costs = [(0, 8.05), (1, 2172.23)]
+    expected_individual_costs = [(0, 139.83), (1, 1357.64)]
 
     # Assertions
     assert all(
@@ -2765,7 +2765,10 @@ def test_multiple_devices_sequential_scheduler():
     soc_min = [0] * 2
 
     start_datetime = ["2023-01-01 01:00:00"] * 2
-    target_datetime = ["2023-01-01 05:00:00", "2023-01-01 03:00:00"]
+    target_datetime = [
+        "2023-01-01 06:00:00",
+        "2023-01-01 03:00:00",
+    ]  # todo: problem with interpreting datetime of soc-target?
     target_value = [1] * 2
 
     market_prices = [
@@ -2923,14 +2926,13 @@ def run_sequential_scheduler():
     )
 
     expected_schedules = [
-        [0, 1] + [0] * 22,
-        [0, 0, 0, 0, 1] + [0] * 19,
+        [0, 1] + [0] * 22,  # the first EV leaves later, but takes the cheapest slot
+        [0, 0, 1]
+        + [0]
+        * 21,  # the second EV leaves earlier, and gets the only (expensive) slot left
     ]
     expected_costs = [(0, 1.46), (1, 2430.39)]
 
-    for d, schedule in enumerate(schedules):
-        print(schedule)
-        print(expected_schedules[d])
     assert all(
         np.isclose(schedules[d], expected_schedules[d]).all()
         for d in range(len(schedules))

From 88b7e4dfe396dd6d9031b78aa7fc3c664d5e77d0 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 7 Mar 2025 17:45:06 +0100
Subject: [PATCH 040/151] fix: use temporary variables

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 26 ++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index c6a4e893de..ca0b3a7cf0 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -551,9 +551,20 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
+                # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet
+                soc_minima_d = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_minima[d],
+                    actuator=sensor_d,
+                    unit="MWh",
+                    query_window=(start, end),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    as_instantaneous_events=True,
+                    resolve_overlaps="max",
+                )
                 commitment = StockCommitment(
                     name="soc minima",
-                    quantity=soc_minima[d],
+                    quantity=soc_minima_d,
                     # negative price because breaching in the downwards (shortage) direction is penalized
                     downwards_deviation_price=-soc_minima_breach_price,
                     _type="any",
@@ -599,9 +610,20 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
+                # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet
+                soc_maxima_d = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_maxima[d],
+                    actuator=sensor_d,
+                    unit="MWh",
+                    query_window=(start, end),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    as_instantaneous_events=True,
+                    resolve_overlaps="min",
+                )
                 commitment = StockCommitment(
                     name="soc maxima",
-                    quantity=soc_maxima[d],
+                    quantity=soc_maxima_d,
                     # positive price because breaching in the upwards (surplus) direction is penalized
                     upwards_deviation_price=soc_maxima_breach_price,
                     _type="any",

From d6f9c1b29e3e8ec3fb3fcf86a9ac72eb3837a59c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 10 Mar 2025 16:49:40 +0100
Subject: [PATCH 041/151] feat: swap assert statements to check total costs
 first and distribution of costs second

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/tests/test_scheduling_simultaneous.py         | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py
index 2294f8f6e2..a0782b55c2 100644
--- a/flexmeasures/data/tests/test_scheduling_simultaneous.py
+++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py
@@ -101,19 +101,17 @@ def test_create_simultaneous_jobs(
     total_cost = ev_costs + battery_costs
 
     # Define expected costs based on resolution
+    expected_total_cost = -3.3525
     expected_ev_costs = 2.1625
     expected_battery_costs = -5.515
-    expected_total_cost = -3.3525
 
     # Assert costs
+    assert (
+        round(total_cost, 4) == expected_total_cost
+    ), f"Total cost should be {expected_total_cost} €, got {total_cost} €"
     assert (
         round(ev_costs, 4) == expected_ev_costs
     ), f"EV cost should be {expected_ev_costs} €, got {ev_costs} €"
-
     assert (
         round(battery_costs, 4) == expected_battery_costs
     ), f"Battery cost should be {expected_battery_costs} €, got {battery_costs} €"
-
-    assert (
-        round(total_cost, 4) == expected_total_cost
-    ), f"Total cost should be {expected_total_cost} €, got {total_cost} €"

From 34eb6e9c2c9b14349eae82837d7ada8ef194fffa Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Feb 2025 22:12:45 +0100
Subject: [PATCH 042/151] dev: fix broken downgrade

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/migrations/versions/cb8df44ebda5_flexcontext_field.py  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flexmeasures/data/migrations/versions/cb8df44ebda5_flexcontext_field.py b/flexmeasures/data/migrations/versions/cb8df44ebda5_flexcontext_field.py
index 65b88d5a5f..1483491ebc 100644
--- a/flexmeasures/data/migrations/versions/cb8df44ebda5_flexcontext_field.py
+++ b/flexmeasures/data/migrations/versions/cb8df44ebda5_flexcontext_field.py
@@ -320,6 +320,8 @@ def downgrade():
 
     for row in results:
         asset_id, flex_context, attributes_data = row
+        if flex_context is None:
+            flex_context = {}
 
         consumption_price_as_str, consumption_price_sensor_id = get_price_info(
             flex_context.get("consumption-price")

From 8c54094b8e4cd8d8d4a62c2d34c2173adf26c8e8 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 12 Mar 2025 14:38:03 +0100
Subject: [PATCH 043/151] fix: only add storage constraints in case we know a
 soc_at_start (some device don't deal with SoC at all)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 29 ++++++++++++--------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index ca0b3a7cf0..07e9a85b34 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -635,17 +635,18 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
                 soc_maxima[d] = None
 
-            device_constraints[d] = add_storage_constraints(
-                start,
-                end,
-                resolution,
-                soc_at_start[d],
-                soc_targets[d],
-                soc_maxima[d],
-                soc_minima[d],
-                soc_max[d],
-                soc_min[d],
-            )
+            if soc_at_start[d] is not None:
+                device_constraints[d] = add_storage_constraints(
+                    start,
+                    end,
+                    resolution,
+                    soc_at_start[d],
+                    soc_targets[d],
+                    soc_maxima[d],
+                    soc_minima[d],
+                    soc_max[d],
+                    soc_min[d],
+                )
 
             power_capacity_in_mw[d] = get_continuous_series_sensor_or_quantity(
                 variable_quantity=power_capacity_in_mw[d],
@@ -1134,7 +1135,11 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType:
             ems_constraints,
             commitments=commitments,
             initial_stock=[
-                soc_at_start_d * (timedelta(hours=1) / resolution)
+                (
+                    soc_at_start_d * (timedelta(hours=1) / resolution)
+                    if soc_at_start_d is not None
+                    else 0
+                )
                 for soc_at_start_d in soc_at_start
             ],
         )

From 1d0471d36b955e6661d46a9c76df301b641f0e8f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 12 Mar 2025 14:40:34 +0100
Subject: [PATCH 044/151] feat: optional soc-at-start

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py
index a9dbe61914..77da9f3f22 100644
--- a/flexmeasures/data/schemas/scheduling/storage.py
+++ b/flexmeasures/data/schemas/scheduling/storage.py
@@ -55,7 +55,7 @@ class StorageFlexModelSchema(Schema):
     """
 
     soc_at_start = QuantityField(
-        required=True,
+        required=False,
         to_unit="MWh",
         default_src_unit="dimensionless",  # placeholder, overridden in __init__
         return_magnitude=True,

From 5571805b58288f55dbd153533cdc4818174fb5d0 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 12 Mar 2025 14:43:09 +0100
Subject: [PATCH 045/151] dev: add todo

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 07e9a85b34..7a2854f85c 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -875,6 +875,7 @@ def deserialize_flex_config(self):
             # Extend schedule period in case a target exceeds its end
             self.possibly_extend_end(soc_targets=self.flex_model.get("soc_targets"))
         elif isinstance(self.flex_model, list):
+            # todo: ensure_soc_min_max in case the device is a storage (see line 847)
             self.flex_model = MultiSensorFlexModelSchema(many=True).load(
                 self.flex_model
             )

From f9758fea0f7b230560b54c84936edff59541e57b Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 12 Mar 2025 15:18:13 +0100
Subject: [PATCH 046/151] refactor: prefer kwargs

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 7a2854f85c..f203f3411b 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -1132,8 +1132,8 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType:
         ) = self._prepare(skip_validation=skip_validation)
 
         ems_schedule, expected_costs, scheduler_results, model = device_scheduler(
-            device_constraints,
-            ems_constraints,
+            device_constraints=device_constraints,
+            ems_constraints=ems_constraints,
             commitments=commitments,
             initial_stock=[
                 (

From 21b17ec1f1155f5448c82b007f891fc0036971f4 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 12 Mar 2025 16:49:55 +0100
Subject: [PATCH 047/151] feat: allow passing capacity sensors with negative
 values, which we'll clip to 0, because capacities are by definition strictly
 non-negative (this is useful for modelling PV curtailment, where PV output
 may show small negative values during the night)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 3 +++
 flexmeasures/data/models/planning/utils.py   | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index f203f3411b..f992dc57bb 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -655,6 +655,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
+                min_value=0,  # capacities are positive by definition
                 resolve_overlaps="min",
             )
 
@@ -672,6 +673,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     beliefs_before=belief_time,
                     fallback_attribute="production_capacity",
                     max_value=power_capacity_in_mw[d],
+                    min_value=0,  # capacities are positive by definition
                     resolve_overlaps="min",
                 )
             if sensor_d.get_attribute("is_strictly_non_negative"):
@@ -686,6 +688,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                         resolution=resolution,
                         beliefs_before=belief_time,
                         fallback_attribute="consumption_capacity",
+                        min_value=0,  # capacities are positive by definition
                         max_value=power_capacity_in_mw[d],
                         resolve_overlaps="min",
                     )
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 1c5453225c..be33987194 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -510,13 +510,14 @@ def get_continuous_series_sensor_or_quantity(
     resolution: timedelta,
     beliefs_before: datetime | None = None,
     fallback_attribute: str | None = None,
+    min_value: float | int = np.nan,
     max_value: float | int | pd.Series = np.nan,
     as_instantaneous_events: bool = False,
     resolve_overlaps: str = "first",
     fill_sides: bool = False,
 ) -> pd.Series:
     """Creates a time series from a sensor, time series specification, or quantity within a specified window,
-    falling back to a given `fallback_attribute` and making sure no values exceed `max_value`.
+    falling back to a given `fallback_attribute` and making sure values stay within the domain [min_value, max_value].
 
     :param variable_quantity:       A sensor recording the data, a time series specification or a fixed quantity.
     :param actuator:                The actuator from which relevant defaults are retrieved.
@@ -525,6 +526,7 @@ def get_continuous_series_sensor_or_quantity(
     :param resolution:              The resolution or time interval for the data.
     :param beliefs_before:          Timestamp for prior beliefs or knowledge.
     :param fallback_attribute:      Attribute serving as a fallback default in case no quantity or sensor is given.
+    :param min_value:               Minimum value.
     :param max_value:               Maximum value (also replacing NaN values).
     :param as_instantaneous_events: optionally, convert to instantaneous events, in which case the passed resolution is
                                     interpreted as the desired frequency of the data.
@@ -557,6 +559,9 @@ def get_continuous_series_sensor_or_quantity(
     # Apply upper limit
     time_series = nanmin_of_series_and_value(time_series, max_value)
 
+    # Apply lower limit
+    time_series = time_series.clip(lower=min_value)
+
     return time_series
 
 

From 7026eee7683919ba526297d0b78cc4202835e926 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 10:30:25 +0100
Subject: [PATCH 048/151] refactor: field guaranteed to exist at this point

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index f992dc57bb..e9ec47d4ab 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -529,9 +529,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     resolve_overlaps="max",
                 )
             if self.flex_context.get("soc_minima_breach_price", None) is not None:
-                soc_minima_breach_price = self.flex_context.get(
-                    "soc_minima_breach_price"
-                )
+                soc_minima_breach_price = self.flex_context["soc_minima_breach_price"]
                 soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima_breach_price,
                     actuator=asset,
@@ -588,9 +586,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     resolve_overlaps="min",
                 )
             if self.flex_context.get("soc_maxima_breach_price", None) is not None:
-                soc_maxima_breach_price = self.flex_context.get(
-                    "soc_maxima_breach_price"
-                )
+                soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"]
                 soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima_breach_price,
                     actuator=asset,

From 8cdf2c800efd661e484de452214e06df13e12694 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 10:57:21 +0100
Subject: [PATCH 049/151] refactor: get unit from variable quantity

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 89 ++++----------------
 1 file changed, 17 insertions(+), 72 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index e9ec47d4ab..b694913cc6 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -158,15 +158,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             up_deviation_prices = get_continuous_series_sensor_or_quantity(
                 variable_quantity=consumption_price,
                 actuator=asset,
-                unit=(
-                    consumption_price.unit
-                    if isinstance(consumption_price, Sensor)
-                    else (
-                        consumption_price[0]["value"].units
-                        if isinstance(consumption_price, list)
-                        else str(consumption_price.units)
-                    )
-                ),
+                unit=get_unit(consumption_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -185,15 +177,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             down_deviation_prices = get_continuous_series_sensor_or_quantity(
                 variable_quantity=production_price,
                 actuator=asset,
-                unit=(
-                    production_price.unit
-                    if isinstance(production_price, Sensor)
-                    else (
-                        production_price[0]["value"].units
-                        if isinstance(production_price, list)
-                        else str(production_price.units)
-                    )
-                ),
+                unit=get_unit(production_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -304,15 +288,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_peak_consumption_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_peak_consumption_price,
                 actuator=asset,
-                unit=(
-                    ems_peak_consumption_price.unit
-                    if isinstance(ems_peak_consumption_price, Sensor)
-                    else (
-                        ems_peak_consumption_price[0]["value"].units
-                        if isinstance(ems_peak_consumption_price, list)
-                        else str(ems_peak_consumption_price.units)
-                    )
-                ),
+                unit=get_unit(ems_peak_consumption_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -348,15 +324,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_peak_production_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_peak_production_price,
                 actuator=asset,
-                unit=(
-                    ems_peak_production_price.unit
-                    if isinstance(ems_peak_production_price, Sensor)
-                    else (
-                        ems_peak_production_price[0]["value"].units
-                        if isinstance(ems_peak_production_price, list)
-                        else str(ems_peak_production_price.units)
-                    )
-                ),
+                unit=get_unit(ems_peak_production_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -393,15 +361,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_consumption_breach_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_consumption_breach_price,
                 actuator=asset,
-                unit=(
-                    ems_consumption_breach_price.unit
-                    if isinstance(ems_consumption_breach_price, Sensor)
-                    else (
-                        ems_consumption_breach_price[0]["value"].units
-                        if isinstance(ems_consumption_breach_price, list)
-                        else str(ems_consumption_breach_price.units)
-                    )
-                ),
+                unit=get_unit(ems_consumption_breach_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -442,15 +402,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_production_breach_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_production_breach_price,
                 actuator=asset,
-                unit=(
-                    ems_production_breach_price.unit
-                    if isinstance(ems_production_breach_price, Sensor)
-                    else (
-                        ems_production_breach_price[0]["value"].units
-                        if isinstance(ems_production_breach_price, list)
-                        else str(ems_production_breach_price.units)
-                    )
-                ),
+                unit=get_unit(ems_production_breach_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -533,15 +485,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima_breach_price,
                     actuator=asset,
-                    unit=(
-                        soc_minima_breach_price.unit
-                        if isinstance(soc_minima_breach_price, Sensor)
-                        else (
-                            soc_minima_breach_price[0]["value"].units
-                            if isinstance(soc_minima_breach_price, list)
-                            else str(soc_minima_breach_price.units)
-                        )
-                    ),
+                    unit=get_unit(soc_minima_breach_price),
                     query_window=(start, end),
                     resolution=resolution,
                     beliefs_before=belief_time,
@@ -590,15 +534,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima_breach_price,
                     actuator=asset,
-                    unit=(
-                        soc_maxima_breach_price.unit
-                        if isinstance(soc_maxima_breach_price, Sensor)
-                        else (
-                            soc_maxima_breach_price[0]["value"].units
-                            if isinstance(soc_maxima_breach_price, list)
-                            else str(soc_maxima_breach_price.units)
-                        )
-                    ),
+                    unit=get_unit(soc_maxima_breach_price),
                     query_window=(start, end),
                     resolution=resolution,
                     beliefs_before=belief_time,
@@ -1625,6 +1561,15 @@ def validate_constraint(
     return constraint_violations
 
 
+def get_unit(variable_quantity: Sensor | list[dict] | ur.Quantity) -> str:
+    """Obtain the unit from a variable quantity."""
+    if isinstance(variable_quantity, Sensor):
+        return variable_quantity.unit
+    if isinstance(variable_quantity, list):
+        return variable_quantity[0]["value"].units
+    return str(variable_quantity.units)
+
+
 def prepend_serie(serie: pd.Series, value) -> pd.Series:
     """Prepend a value to a time series series
 

From ac784e86cba0e2e584bb5c01bcac3c901ffe09d2 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 11:49:09 +0100
Subject: [PATCH 050/151] feat: add test case for a time series specification
 of prices with different monetary units (not allowed)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/tests/test_scheduling.py     | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py
index 09bf1b1a4b..ad8032ee69 100644
--- a/flexmeasures/data/schemas/tests/test_scheduling.py
+++ b/flexmeasures/data/schemas/tests/test_scheduling.py
@@ -257,6 +257,25 @@ def load_schema():
             },
             {"site-peak-production-price": "Must be greater than or equal to 0."},
         ),
+        (
+            {
+                "site-consumption-breach-price": [
+                    {
+                        "value": "1 KRW/MWh",
+                        "start": "2025-03-16T00:00+01",
+                        "duration": "P1D",
+                    },
+                    {
+                        "value": "1 KRW/MW",
+                        "start": "2025-03-16T00:00+01",
+                        "duration": "P1D",
+                    },
+                ],
+            },
+            {
+                "site-consumption-breach-price": "Segments of a time series must share the same unit."
+            },
+        ),
     ],
 )
 def test_flex_context_schema(db, app, setup_site_capacity_sensor, flex_context, fails):

From 127b701231b03a3ceafc715c820a85bf53ecee14 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 12:10:11 +0100
Subject: [PATCH 051/151] refactor: let NotImplementedError reference field
 name rather than variable name

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 331f33b8dd..06123e698d 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -183,8 +183,10 @@ def _try_to_convert_price_units(self, data):
         previous_field_name = None
         for field in self.declared_fields:
             if field[-5:] == "price" and field in data:
-                price_unit = self._get_variable_quantity_unit(field, data[field])
                 price_field = self.declared_fields[field]
+                price_unit = self._get_variable_quantity_unit(
+                    price_field.data_key, data[field]
+                )
                 currency_unit = price_unit.split("/")[0]
 
                 if previous_currency_unit is None:
@@ -216,7 +218,7 @@ def _try_to_convert_price_units(self, data):
         return data
 
     def _get_variable_quantity_unit(
-        self, field: str, variable_quantity: ur.Quantity | list[dict | Sensor]
+        self, field_name: str, variable_quantity: ur.Quantity | list[dict | Sensor]
     ) -> str:
         """Gets the unit from the variable quantity."""
         if isinstance(variable_quantity, ur.Quantity):
@@ -227,7 +229,6 @@ def _get_variable_quantity_unit(
                 str(variable_quantity[j]["value"].units) == unit
                 for j in range(len(variable_quantity))
             ):
-                field_name = self.declared_fields[field].data_key
                 raise ValidationError(
                     "Segments of a time series must share the same unit.",
                     field_name=field_name,
@@ -236,7 +237,7 @@ def _get_variable_quantity_unit(
             unit = variable_quantity.unit
         else:
             raise NotImplementedError(
-                f"Unexpected type '{type(variable_quantity)}' for '{field}': {variable_quantity}."
+                f"Unexpected type '{type(variable_quantity)}' for variable_quantity describing '{field_name}': {variable_quantity}."
             )
         return unit
 

From ebc07dbb90793f05cf558684900df1e931bb0dd7 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 12:13:22 +0100
Subject: [PATCH 052/151] refactor: move util function from schema to field

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py       | 28 +------------------
 flexmeasures/data/schemas/sensors.py          | 24 ++++++++++++++++
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 06123e698d..a5000d2bb0 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -184,9 +184,7 @@ def _try_to_convert_price_units(self, data):
         for field in self.declared_fields:
             if field[-5:] == "price" and field in data:
                 price_field = self.declared_fields[field]
-                price_unit = self._get_variable_quantity_unit(
-                    price_field.data_key, data[field]
-                )
+                price_unit = price_field._get_unit(price_field.data_key, data[field])
                 currency_unit = price_unit.split("/")[0]
 
                 if previous_currency_unit is None:
@@ -217,30 +215,6 @@ def _try_to_convert_price_units(self, data):
                     )
         return data
 
-    def _get_variable_quantity_unit(
-        self, field_name: str, variable_quantity: ur.Quantity | list[dict | Sensor]
-    ) -> str:
-        """Gets the unit from the variable quantity."""
-        if isinstance(variable_quantity, ur.Quantity):
-            unit = str(variable_quantity.units)
-        elif isinstance(variable_quantity, list):
-            unit = str(variable_quantity[0]["value"].units)
-            if not all(
-                str(variable_quantity[j]["value"].units) == unit
-                for j in range(len(variable_quantity))
-            ):
-                raise ValidationError(
-                    "Segments of a time series must share the same unit.",
-                    field_name=field_name,
-                )
-        elif isinstance(variable_quantity, Sensor):
-            unit = variable_quantity.unit
-        else:
-            raise NotImplementedError(
-                f"Unexpected type '{type(variable_quantity)}' for variable_quantity describing '{field_name}': {variable_quantity}."
-            )
-        return unit
-
 
 class DBFlexContextSchema(FlexContextSchema):
 
diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py
index 27e2543beb..9731ffe875 100644
--- a/flexmeasures/data/schemas/sensors.py
+++ b/flexmeasures/data/schemas/sensors.py
@@ -375,6 +375,30 @@ def convert(self, value, param, ctx, **kwargs):
 
         return super().convert(_value, param, ctx, **kwargs)
 
+    def _get_unit(
+        self, field_name: str, variable_quantity: ur.Quantity | list[dict | Sensor]
+    ) -> str:
+        """Gets the unit from the variable quantity."""
+        if isinstance(variable_quantity, ur.Quantity):
+            unit = str(variable_quantity.units)
+        elif isinstance(variable_quantity, list):
+            unit = str(variable_quantity[0]["value"].units)
+            if not all(
+                str(variable_quantity[j]["value"].units) == unit
+                for j in range(len(variable_quantity))
+            ):
+                raise ValidationError(
+                    "Segments of a time series must share the same unit.",
+                    field_name=field_name,
+                )
+        elif isinstance(variable_quantity, Sensor):
+            unit = variable_quantity.unit
+        else:
+            raise NotImplementedError(
+                f"Unexpected type '{type(variable_quantity)}' for variable_quantity describing '{field_name}': {variable_quantity}."
+            )
+        return unit
+
 
 class RepurposeValidatorToIgnoreSensorsAndLists(validate.Validator):
     """Validator that executes another validator (the one you initialize it with) only on non-Sensor and non-list values."""

From 327296385fd35825e0d8ae98404060d99268bf7b Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 12:14:08 +0100
Subject: [PATCH 053/151] refactor: remove redundant function argument

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 2 +-
 flexmeasures/data/schemas/sensors.py             | 8 +++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index a5000d2bb0..bec495b61c 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -184,7 +184,7 @@ def _try_to_convert_price_units(self, data):
         for field in self.declared_fields:
             if field[-5:] == "price" and field in data:
                 price_field = self.declared_fields[field]
-                price_unit = price_field._get_unit(price_field.data_key, data[field])
+                price_unit = price_field._get_unit(data[field])
                 currency_unit = price_unit.split("/")[0]
 
                 if previous_currency_unit is None:
diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py
index 9731ffe875..5b80fa9a8f 100644
--- a/flexmeasures/data/schemas/sensors.py
+++ b/flexmeasures/data/schemas/sensors.py
@@ -375,9 +375,7 @@ def convert(self, value, param, ctx, **kwargs):
 
         return super().convert(_value, param, ctx, **kwargs)
 
-    def _get_unit(
-        self, field_name: str, variable_quantity: ur.Quantity | list[dict | Sensor]
-    ) -> str:
+    def _get_unit(self, variable_quantity: ur.Quantity | list[dict | Sensor]) -> str:
         """Gets the unit from the variable quantity."""
         if isinstance(variable_quantity, ur.Quantity):
             unit = str(variable_quantity.units)
@@ -389,13 +387,13 @@ def _get_unit(
             ):
                 raise ValidationError(
                     "Segments of a time series must share the same unit.",
-                    field_name=field_name,
+                    field_name=self.data_key,
                 )
         elif isinstance(variable_quantity, Sensor):
             unit = variable_quantity.unit
         else:
             raise NotImplementedError(
-                f"Unexpected type '{type(variable_quantity)}' for variable_quantity describing '{field_name}': {variable_quantity}."
+                f"Unexpected type '{type(variable_quantity)}' for variable_quantity describing '{self.data_key}': {variable_quantity}."
             )
         return unit
 

From 597c36de6727bbf1d3af4bc1737133632d07b109 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 12:16:32 +0100
Subject: [PATCH 054/151] refactor: spell correct util function

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 22 ++++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index b694913cc6..42f3022085 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -1397,9 +1397,9 @@ def validate_storage_constraints(
     ##########################################
 
     _constraints["factor_w_wh(t)"] = resolution / timedelta(hours=1)
-    _constraints["min(t-1)"] = prepend_serie(_constraints["min(t)"], soc_min)
-    _constraints["equals(t-1)"] = prepend_serie(_constraints["equals(t)"], soc_at_start)
-    _constraints["max(t-1)"] = prepend_serie(_constraints["max(t)"], soc_max)
+    _constraints["min(t-1)"] = prepend_series(_constraints["min(t)"], soc_min)
+    _constraints["equals(t-1)"] = prepend_series(_constraints["equals(t)"], soc_at_start)
+    _constraints["max(t-1)"] = prepend_series(_constraints["max(t)"], soc_max)
 
     # 1) equals(t) - equals(t-1) <= derivative_max(t)
     constraint_violations += validate_constraint(
@@ -1570,19 +1570,19 @@ def get_unit(variable_quantity: Sensor | list[dict] | ur.Quantity) -> str:
     return str(variable_quantity.units)
 
 
-def prepend_serie(serie: pd.Series, value) -> pd.Series:
-    """Prepend a value to a time series series
+def prepend_series(series: pd.Series, value) -> pd.Series:
+    """Prepend a value to a time series
 
-    :param serie: serie containing the timed values
+    :param series: series containing the timed values
     :param value: value to place in the first position
     """
     # extend max
-    serie = serie.copy()
-    # insert `value` at time `serie.index[0] - resolution` which creates a new entry at the end of the series
-    serie[serie.index[0] - serie.index.freq] = value
+    series = series.copy()
+    # insert `value` at time `series.index[0] - resolution` which creates a new entry at the end of the series
+    series[series.index[0] - series.index.freq] = value
     # sort index to keep the time ordering
-    serie = serie.sort_index()
-    return serie.shift(1)
+    series = series.sort_index()
+    return series.shift(1)
 
 
 #####################

From 42d9c867488c9fbb4f6ddb7d4bc7980815b77b38 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 12:27:22 +0100
Subject: [PATCH 055/151] refactor: reuse existing superior util function to
 get the unit from a variable quantity

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 45 ++++++++++++--------
 flexmeasures/data/schemas/sensors.py         |  2 +-
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 42f3022085..487395b287 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -158,7 +158,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             up_deviation_prices = get_continuous_series_sensor_or_quantity(
                 variable_quantity=consumption_price,
                 actuator=asset,
-                unit=get_unit(consumption_price),
+                unit=FlexContextSchema()
+                .declared_fields["consumption_price"]
+                ._get_unit(consumption_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -177,7 +179,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             down_deviation_prices = get_continuous_series_sensor_or_quantity(
                 variable_quantity=production_price,
                 actuator=asset,
-                unit=get_unit(production_price),
+                unit=FlexContextSchema()
+                .declared_fields["production_price"]
+                ._get_unit(production_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -288,7 +292,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_peak_consumption_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_peak_consumption_price,
                 actuator=asset,
-                unit=get_unit(ems_peak_consumption_price),
+                unit=FlexContextSchema()
+                .declared_fields["ems_peak_consumption_price"]
+                ._get_unit(ems_peak_consumption_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -324,7 +330,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_peak_production_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_peak_production_price,
                 actuator=asset,
-                unit=get_unit(ems_peak_production_price),
+                unit=FlexContextSchema()
+                .declared_fields["ems_peak_production_price"]
+                ._get_unit(ems_peak_production_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -361,7 +369,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_consumption_breach_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_consumption_breach_price,
                 actuator=asset,
-                unit=get_unit(ems_consumption_breach_price),
+                unit=FlexContextSchema()
+                .declared_fields["ems_consumption_breach_price"]
+                ._get_unit(ems_consumption_breach_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -402,7 +412,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_production_breach_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_production_breach_price,
                 actuator=asset,
-                unit=get_unit(ems_production_breach_price),
+                unit=FlexContextSchema()
+                .declared_fields["ems_production_breach_price"]
+                ._get_unit(ems_production_breach_price),
                 query_window=(start, end),
                 resolution=resolution,
                 beliefs_before=belief_time,
@@ -485,7 +497,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima_breach_price,
                     actuator=asset,
-                    unit=get_unit(soc_minima_breach_price),
+                    unit=FlexContextSchema()
+                    .declared_fields["soc_minima_breach_price"]
+                    ._get_unit(soc_minima_breach_price),
                     query_window=(start, end),
                     resolution=resolution,
                     beliefs_before=belief_time,
@@ -534,7 +548,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima_breach_price,
                     actuator=asset,
-                    unit=get_unit(soc_maxima_breach_price),
+                    unit=FlexContextSchema()
+                    .declared_fields["soc_maxima_breach_price"]
+                    ._get_unit(soc_maxima_breach_price),
                     query_window=(start, end),
                     resolution=resolution,
                     beliefs_before=belief_time,
@@ -1398,7 +1414,9 @@ def validate_storage_constraints(
 
     _constraints["factor_w_wh(t)"] = resolution / timedelta(hours=1)
     _constraints["min(t-1)"] = prepend_series(_constraints["min(t)"], soc_min)
-    _constraints["equals(t-1)"] = prepend_series(_constraints["equals(t)"], soc_at_start)
+    _constraints["equals(t-1)"] = prepend_series(
+        _constraints["equals(t)"], soc_at_start
+    )
     _constraints["max(t-1)"] = prepend_series(_constraints["max(t)"], soc_max)
 
     # 1) equals(t) - equals(t-1) <= derivative_max(t)
@@ -1561,15 +1579,6 @@ def validate_constraint(
     return constraint_violations
 
 
-def get_unit(variable_quantity: Sensor | list[dict] | ur.Quantity) -> str:
-    """Obtain the unit from a variable quantity."""
-    if isinstance(variable_quantity, Sensor):
-        return variable_quantity.unit
-    if isinstance(variable_quantity, list):
-        return variable_quantity[0]["value"].units
-    return str(variable_quantity.units)
-
-
 def prepend_series(series: pd.Series, value) -> pd.Series:
     """Prepend a value to a time series
 
diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py
index 5b80fa9a8f..eaabf5d729 100644
--- a/flexmeasures/data/schemas/sensors.py
+++ b/flexmeasures/data/schemas/sensors.py
@@ -376,7 +376,7 @@ def convert(self, value, param, ctx, **kwargs):
         return super().convert(_value, param, ctx, **kwargs)
 
     def _get_unit(self, variable_quantity: ur.Quantity | list[dict | Sensor]) -> str:
-        """Gets the unit from the variable quantity."""
+        """Obtain the unit from the variable quantity."""
         if isinstance(variable_quantity, ur.Quantity):
             unit = str(variable_quantity.units)
         elif isinstance(variable_quantity, list):

From a73b32c85d072807b6db17eb1333db97457ec833 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 16 Mar 2025 12:31:56 +0100
Subject: [PATCH 056/151] fix: remove dev comment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/linear_optimization.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index be13f3817e..a64369c66d 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -467,8 +467,7 @@ def ems_derivative_bounds(m, j):
     def device_stock_commitment_equalities(m, c, j, d):
         """Couple device stocks to each commitment."""
         if (
-            "device"
-            not in commitments[c].columns  # "device" not in commitments[28].columns
+            "device" not in commitments[c].columns
             or (commitments[c]["device"] != d).all()
             or m.commitment_quantity[c] == -infinity
         ):

From e6b4e8fa00b105418142eb5e2f0082319c921c29 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 09:13:53 +0100
Subject: [PATCH 057/151] refactor: vectorize operation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/linear_optimization.py   | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index a64369c66d..0c0f6f9b1d 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -473,7 +473,11 @@ def device_stock_commitment_equalities(m, c, j, d):
         ):
             # Commitment c does not concern device d
             return Constraint.Skip
-        if not all(cl.__name__ == "StockCommitment" for cl in commitments[c]["class"]):
+        if (
+            not commitments[c]["class"]
+            .apply(lambda cl: cl.__name__ == "StockCommitment")
+            .all()
+        ):
             raise NotImplementedError(
                 "FlowCommitment on a device level has not been implemented. Please file a GitHub ticket explaining your use case."
             )
@@ -510,8 +514,13 @@ def ems_flow_commitment_equalities(m, c, j):
         ) or m.commitment_quantity[c] == -infinity:
             # Commitment c does not concern EMS
             return Constraint.Skip
-        if "class" in commitments[c].columns and not all(
-            cl.__name__ == "FlowCommitment" for cl in commitments[c]["class"]
+        if (
+            "class" in commitments[c].columns
+            and not (
+                commitments[c]["class"].apply(
+                    lambda cl: cl.__name__ == "FlowCommitment"
+                )
+            ).all()
         ):
             raise NotImplementedError(
                 "StockCommitment on an EMS level has not been implemented. Please file a GitHub ticket explaining your use case."

From 7779f8bfbdaffeab6f9b859db48a1e2016833bcf Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 09:15:55 +0100
Subject: [PATCH 058/151] refactor: compare class rather than name

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/linear_optimization.py  | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index 0c0f6f9b1d..a245ca02b6 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -22,7 +22,11 @@
 from pyomo.environ import value
 from pyomo.opt import SolverFactory, SolverResults
 
-from flexmeasures.data.models.planning import Commitment, FlowCommitment
+from flexmeasures.data.models.planning import (
+    Commitment,
+    FlowCommitment,
+    StockCommitment,
+)
 from flexmeasures.data.models.planning.utils import initialize_series, initialize_df
 from flexmeasures.utils.calculations import apply_stock_changes_and_losses
 
@@ -473,11 +477,7 @@ def device_stock_commitment_equalities(m, c, j, d):
         ):
             # Commitment c does not concern device d
             return Constraint.Skip
-        if (
-            not commitments[c]["class"]
-            .apply(lambda cl: cl.__name__ == "StockCommitment")
-            .all()
-        ):
+        if not commitments[c]["class"].apply(lambda cl: cl == StockCommitment).all():
             raise NotImplementedError(
                 "FlowCommitment on a device level has not been implemented. Please file a GitHub ticket explaining your use case."
             )
@@ -517,9 +517,7 @@ def ems_flow_commitment_equalities(m, c, j):
         if (
             "class" in commitments[c].columns
             and not (
-                commitments[c]["class"].apply(
-                    lambda cl: cl.__name__ == "FlowCommitment"
-                )
+                commitments[c]["class"].apply(lambda cl: cl == FlowCommitment)
             ).all()
         ):
             raise NotImplementedError(

From 499ef7e9d876de4c143cb55f2079572adda248fd Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 09:22:14 +0100
Subject: [PATCH 059/151] docs: API changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index 4a9e0dcb87..411cedd3a2 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -5,6 +5,14 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace.
 
+v3.0-22 | 2025-03-17
+""""""""""""""""""""
+
+- Introduce new price fields in the ``flex-context`` in order to relax SoC constraints in the ``device-model``:
+
+  - ``soc-minima-breach-price``: if set, the ``soc-minima`` are used as a soft constraint, and not meeting the minima is penalized according to this per-kWh price. The price is applied to each breach that occurs given the resolution of the scheduled power sensor.
+  - ``soc-maxima-breach-price``: if set, the ``soc-maxima`` are used as a soft constraint, and not meeting the maxima is penalized according to this per-kWh price. The price is applied to each breach that occurs given the resolution of the scheduled power sensor.
+
 v3.0-22 | 2024-12-27
 """"""""""""""""""""
 

From 51f72b05f8b144e901d2b2487956c641a2686897 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 09:24:34 +0100
Subject: [PATCH 060/151] docs: main changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 9c96e2d83d..e098ab5dad 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -11,6 +11,7 @@ v0.25.0 | February XX, 2025
 
 New features
 -------------
+* Allow relaxing SoC minima and maxima, by setting penalties for not meeting these constraints, using two new ``flex-context`` fields [see `PR #1300 <https://github.com/FlexMeasures/flexmeasures/pull/1300>`_]
 * Added form modal to edit an asset's ``flex_context`` [see `PR #1320 <https://github.com/FlexMeasures/flexmeasures/pull/1320>`_]
 * Better y-axis titles for charts that show multiple sensors with a shared unit [see `PR #1346 <https://github.com/FlexMeasures/flexmeasures/pull/1346>`_]
 * Add CLI command ``flexmeasures jobs save-last-failed`` for saving the last failed jobs [see `PR #1342 <https://www.github.com/FlexMeasures/flexmeasures/pull/1342>`_ and `PR #1359 <https://github.com/FlexMeasures/flexmeasures/pull/1359>`_]

From 8952242e43233cca8b87e6eb994e87447ae7ec90 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 12:26:16 +0100
Subject: [PATCH 061/151] feat: add support for device flow commitment in
 device_scheduler

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../models/planning/linear_optimization.py    | 26 ++++++---
 flexmeasures/data/models/planning/storage.py  | 54 ++++++++++---------
 2 files changed, 49 insertions(+), 31 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index a245ca02b6..f9fe8eedf1 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -477,10 +477,25 @@ def device_stock_commitment_equalities(m, c, j, d):
         ):
             # Commitment c does not concern device d
             return Constraint.Skip
-        if not commitments[c]["class"].apply(lambda cl: cl == StockCommitment).all():
-            raise NotImplementedError(
-                "FlowCommitment on a device level has not been implemented. Please file a GitHub ticket explaining your use case."
+
+        # Determine center part of the lhs <= center part <= rhs constraint
+        commitment_class = commitments[c]["class"].iloc[j]
+        if commitment_class == StockCommitment:
+            center_part = (
+                m.commitment_quantity[c]
+                + m.commitment_downwards_deviation[c]
+                + m.commitment_upwards_deviation[c]
+                - m.ems_power[d, j]
             )
+        elif commitment_class == FlowCommitment:
+            center_part = (
+                m.commitment_quantity[c]
+                + m.commitment_downwards_deviation[c]
+                + m.commitment_upwards_deviation[c]
+                - _get_stock_change(m, d, j)
+            )
+        else:
+            raise NotImplementedError(f"Unknown commitment class '{commitment_class}'")
         return (
             (
                 0
@@ -489,10 +504,7 @@ def device_stock_commitment_equalities(m, c, j, d):
                 else None
             ),
             # 0 if "upwards deviation price" in commitments[c].columns else None,  # todo: possible simplification
-            m.commitment_quantity[c]
-            + m.commitment_downwards_deviation[c]
-            + m.commitment_upwards_deviation[c]
-            - _get_stock_change(m, d, j),
+            center_part,
             (
                 0
                 if len(commitments[c]) == 1
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 487395b287..e5001fa0dc 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -494,18 +494,21 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
             if self.flex_context.get("soc_minima_breach_price", None) is not None:
                 soc_minima_breach_price = self.flex_context["soc_minima_breach_price"]
-                soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_minima_breach_price,
-                    actuator=asset,
-                    unit=FlexContextSchema()
-                    .declared_fields["soc_minima_breach_price"]
-                    ._get_unit(soc_minima_breach_price),
-                    query_window=(start, end),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-minima-breach-price",
-                    fill_sides=True,
-                )
+                soc_minima_breach_price = (
+                    get_continuous_series_sensor_or_quantity(
+                        variable_quantity=soc_minima_breach_price,
+                        actuator=asset,
+                        unit=FlexContextSchema()
+                        .declared_fields["soc_minima_breach_price"]
+                        ._get_unit(soc_minima_breach_price),
+                        query_window=(start, end),
+                        resolution=resolution,
+                        beliefs_before=belief_time,
+                        fallback_attribute="soc-minima-breach-price",
+                        fill_sides=True,
+                    )
+                    + "*h",
+                )  # e.g. from EUR/(kWh*h) to EUR/kWh
                 # Set up commitments DataFrame
                 # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_minima_d = get_continuous_series_sensor_or_quantity(
@@ -545,18 +548,21 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
             if self.flex_context.get("soc_maxima_breach_price", None) is not None:
                 soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"]
-                soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_maxima_breach_price,
-                    actuator=asset,
-                    unit=FlexContextSchema()
-                    .declared_fields["soc_maxima_breach_price"]
-                    ._get_unit(soc_maxima_breach_price),
-                    query_window=(start, end),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-maxima-breach-price",
-                    fill_sides=True,
-                )
+                soc_maxima_breach_price = (
+                    get_continuous_series_sensor_or_quantity(
+                        variable_quantity=soc_maxima_breach_price,
+                        actuator=asset,
+                        unit=FlexContextSchema()
+                        .declared_fields["soc_maxima_breach_price"]
+                        ._get_unit(soc_maxima_breach_price),
+                        query_window=(start, end),
+                        resolution=resolution,
+                        beliefs_before=belief_time,
+                        fallback_attribute="soc-maxima-breach-price",
+                        fill_sides=True,
+                    )
+                    + "*h",
+                )  # e.g. from EUR/(kWh*h) to EUR/kWh
                 # Set up commitments DataFrame
                 # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_maxima_d = get_continuous_series_sensor_or_quantity(

From 9a2a1a7adfa033ce62983593e3f94932ba3d08f8 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 14:34:15 +0100
Subject: [PATCH 062/151] feat: prefer curtailing later

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 26 ++++++++++++++++++-
 .../data/schemas/scheduling/storage.py        | 11 +++++---
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index e5001fa0dc..dd879447e2 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -119,6 +119,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         prefer_charging_sooner = [
             flex_model_d.get("prefer_charging_sooner") for flex_model_d in flex_model
         ]
+        prefer_curtailing_later = [
+            flex_model_d.get("prefer_curtailing_sooner") for flex_model_d in flex_model
+        ]
         soc_gain = [flex_model_d.get("soc_gain") for flex_model_d in flex_model]
         soc_usage = [flex_model_d.get("soc_usage") for flex_model_d in flex_model]
         consumption_capacity = [
@@ -201,7 +204,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         end = pd.Timestamp(end).tz_convert("UTC")
 
         # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
-        # We penalise the future with at most 1 per thousand times the price spread.
+        # We penalise future consumption and reward future production with at most 1 per thousand times the energy price spread.
         # todo: move to flow or stock commitment per device
         if any(prefer_charging_sooner):
             up_deviation_prices = add_tiny_price_slope(
@@ -449,6 +452,27 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             # Take the contracted capacity as a hard constraint
             ems_constraints["derivative min"] = ems_production_capacity
 
+        # Flow commitments per device
+
+        # Add tiny price slope to prefer curtailing later rather than now.
+        # We penalise the future with at most 1 per thousand times the energy price spread.
+        for d, prefer_curtailing_later_d in enumerate(prefer_curtailing_later):
+            if prefer_curtailing_later_d:
+                tiny_price_slope = (
+                    add_tiny_price_slope(up_deviation_prices, "event_value")
+                    - up_deviation_prices
+                )
+                commitment = FlowCommitment(
+                    name=f"prefer curtailing device {d} later",
+                    quantity=0,
+                    # Prefer curtailing consumption later by making later consumption more expensive
+                    upwards_deviation_price=tiny_price_slope,
+                    # Prefer curtailing production later by making later production more expensive
+                    downwards_deviation_price=-tiny_price_slope,
+                    index=index,
+                )
+                commitments.append(commitment)
+
         # Set up device constraints: scheduled flexible devices for this EMS (from index 0 to D-1), plus the forecasted inflexible devices (at indices D to n).
         device_constraints = [
             initialize_df(StorageScheduler.COLUMNS, start, end, resolution)
diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py
index 77da9f3f22..cb4a9b8592 100644
--- a/flexmeasures/data/schemas/scheduling/storage.py
+++ b/flexmeasures/data/schemas/scheduling/storage.py
@@ -89,6 +89,14 @@ class StorageFlexModelSchema(Schema):
         "MW", data_key="production-capacity", required=False
     )
 
+    # Activation prices
+    prefer_curtailing_later = fields.Bool(
+        data_key="prefer-curtailing-later", load_default=True
+    )
+    prefer_charging_sooner = fields.Bool(
+        data_key="prefer-charging-sooner", load_default=True
+    )
+
     # Timezone placeholders for the soc_maxima, soc_minima and soc_targets fields are overridden in __init__
     soc_maxima = VariableQuantityField(
         to_unit="MWh",
@@ -135,9 +143,6 @@ class StorageFlexModelSchema(Schema):
     )
 
     storage_efficiency = VariableQuantityField("%", data_key="storage-efficiency")
-    prefer_charging_sooner = fields.Bool(
-        data_key="prefer-charging-sooner", load_default=True
-    )
 
     soc_gain = fields.List(
         VariableQuantityField("MW"),

From 381d6178932c6bfe1fe9d5a2de74314dca17fb2e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 14:54:51 +0100
Subject: [PATCH 063/151] fix: support for device flow commitments in
 StorageScheduler

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/linear_optimization.py       | 11 +++++------
 flexmeasures/data/models/planning/storage.py          |  2 +-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index f9fe8eedf1..0dcd0d6d87 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -479,23 +479,22 @@ def device_stock_commitment_equalities(m, c, j, d):
             return Constraint.Skip
 
         # Determine center part of the lhs <= center part <= rhs constraint
-        commitment_class = commitments[c]["class"].iloc[j]
-        if commitment_class == StockCommitment:
+        if commitments[c]["class"].apply(lambda cl: cl == StockCommitment).all():
             center_part = (
                 m.commitment_quantity[c]
                 + m.commitment_downwards_deviation[c]
                 + m.commitment_upwards_deviation[c]
-                - m.ems_power[d, j]
+                - _get_stock_change(m, d, j)
             )
-        elif commitment_class == FlowCommitment:
+        elif commitments[c]["class"].apply(lambda cl: cl == FlowCommitment).all():
             center_part = (
                 m.commitment_quantity[c]
                 + m.commitment_downwards_deviation[c]
                 + m.commitment_upwards_deviation[c]
-                - _get_stock_change(m, d, j)
+                - m.ems_power[d, j]
             )
         else:
-            raise NotImplementedError(f"Unknown commitment class '{commitment_class}'")
+            raise NotImplementedError("Unknown commitment class")
         return (
             (
                 0
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index dd879447e2..603e69ed08 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -464,12 +464,12 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 commitment = FlowCommitment(
                     name=f"prefer curtailing device {d} later",
-                    quantity=0,
                     # Prefer curtailing consumption later by making later consumption more expensive
                     upwards_deviation_price=tiny_price_slope,
                     # Prefer curtailing production later by making later production more expensive
                     downwards_deviation_price=-tiny_price_slope,
                     index=index,
+                    device=d,
                 )
                 commitments.append(commitment)
 

From 28f56060cac7eef19f6565b5aa4de75cf407030f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 17 Mar 2025 15:01:04 +0100
Subject: [PATCH 064/151] docs: document new flex-model field

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index 753c2627dd..8b5b052a39 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -212,7 +212,10 @@ For more details on the possible formats for field values, see :ref:`variable_qu
      - This can encode losses over time, so each time step the energy is held longer leads to higher losses (defaults to 100%). Also read [#storage_efficiency]_ about applying this value per time step across longer time spans.
    * - ``prefer-charging-sooner``
      - ``True``
-     - Tie-breaking policy to apply if conditions are stable (defaults to True, which also signals a preference to discharge later). Boolean option only.
+     - Tie-breaking policy to apply if conditions are stable, which signals a preference to charge sooner rather than later (defaults to True). It also signals a preference to discharge later. Boolean option only.
+   * - ``prefer-curtailing-later``
+     - ``True``
+     - Tie-breaking policy to apply if conditions are stable, which signals a preference to curtail both consumption and production later, whichever is applicable (defaults to True). Boolean option only.
    * - ``power-capacity``
      - ``"50kW"``
      - Device-level power constraint. How much power can be applied to this asset (defaults to the Sensor attribute ``capacity_in_mw``). [#minimum_overlap]_

From 1839647ee4106da17629845137709dedd4be0194 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:00:30 +0100
Subject: [PATCH 065/151] refactor: simplify get arguments

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 603e69ed08..2bc1887496 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -277,7 +277,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         commitments.append(commitment)
 
         # Set up peak commitments
-        if self.flex_context.get("ems_peak_consumption_price", None) is not None:
+        if self.flex_context.get("ems_peak_consumption_price") is not None:
             ems_peak_consumption = get_continuous_series_sensor_or_quantity(
                 variable_quantity=self.flex_context.get("ems_peak_consumption_in_mw"),
                 actuator=asset,
@@ -315,7 +315,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 index=index,
             )
             commitments.append(commitment)
-        if self.flex_context.get("ems_peak_production_price", None) is not None:
+        if self.flex_context.get("ems_peak_production_price") is not None:
             ems_peak_production = get_continuous_series_sensor_or_quantity(
                 variable_quantity=self.flex_context.get("ems_peak_production_in_mw"),
                 actuator=asset,
@@ -516,7 +516,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     as_instantaneous_events=True,
                     resolve_overlaps="max",
                 )
-            if self.flex_context.get("soc_minima_breach_price", None) is not None:
+            if self.flex_context.get("soc_minima_breach_price") is not None:
                 soc_minima_breach_price = self.flex_context["soc_minima_breach_price"]
                 soc_minima_breach_price = (
                     get_continuous_series_sensor_or_quantity(
@@ -570,7 +570,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     as_instantaneous_events=True,
                     resolve_overlaps="min",
                 )
-            if self.flex_context.get("soc_maxima_breach_price", None) is not None:
+            if self.flex_context.get("soc_maxima_breach_price") is not None:
                 soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"]
                 soc_maxima_breach_price = (
                     get_continuous_series_sensor_or_quantity(
@@ -925,8 +925,8 @@ def get_min_max_targets(
     def get_min_max_soc_on_sensor(
         self, adjust_unit: bool = False, deserialized_names: bool = True
     ) -> tuple[float | None, float | None]:
-        soc_min_sensor = self.sensor.get_attribute("min_soc_in_mwh", None)
-        soc_max_sensor = self.sensor.get_attribute("max_soc_in_mwh", None)
+        soc_min_sensor: float | None = self.sensor.get_attribute("min_soc_in_mwh")
+        soc_max_sensor: float | None = self.sensor.get_attribute("max_soc_in_mwh")
         soc_unit_label = "soc_unit" if deserialized_names else "soc-unit"
         if adjust_unit:
             if soc_min_sensor and self.flex_model.get(soc_unit_label) == "kWh":

From 837a9f435b38eb51375510fd044f6543a34ce980 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:07:20 +0100
Subject: [PATCH 066/151] revert: revisions accidentally committed in
 8952242e43233cca8b87e6eb994e87447ae7ec90

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 54 +++++++++-----------
 1 file changed, 24 insertions(+), 30 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 2bc1887496..523ecf2d32 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -518,21 +518,18 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
             if self.flex_context.get("soc_minima_breach_price") is not None:
                 soc_minima_breach_price = self.flex_context["soc_minima_breach_price"]
-                soc_minima_breach_price = (
-                    get_continuous_series_sensor_or_quantity(
-                        variable_quantity=soc_minima_breach_price,
-                        actuator=asset,
-                        unit=FlexContextSchema()
-                        .declared_fields["soc_minima_breach_price"]
-                        ._get_unit(soc_minima_breach_price),
-                        query_window=(start, end),
-                        resolution=resolution,
-                        beliefs_before=belief_time,
-                        fallback_attribute="soc-minima-breach-price",
-                        fill_sides=True,
-                    )
-                    + "*h",
-                )  # e.g. from EUR/(kWh*h) to EUR/kWh
+                soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_minima_breach_price,
+                    actuator=asset,
+                    unit=FlexContextSchema()
+                    .declared_fields["soc_minima_breach_price"]
+                    ._get_unit(soc_minima_breach_price),
+                    query_window=(start, end),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    fallback_attribute="soc-minima-breach-price",
+                    fill_sides=True,
+                )
                 # Set up commitments DataFrame
                 # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_minima_d = get_continuous_series_sensor_or_quantity(
@@ -572,21 +569,18 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
             if self.flex_context.get("soc_maxima_breach_price") is not None:
                 soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"]
-                soc_maxima_breach_price = (
-                    get_continuous_series_sensor_or_quantity(
-                        variable_quantity=soc_maxima_breach_price,
-                        actuator=asset,
-                        unit=FlexContextSchema()
-                        .declared_fields["soc_maxima_breach_price"]
-                        ._get_unit(soc_maxima_breach_price),
-                        query_window=(start, end),
-                        resolution=resolution,
-                        beliefs_before=belief_time,
-                        fallback_attribute="soc-maxima-breach-price",
-                        fill_sides=True,
-                    )
-                    + "*h",
-                )  # e.g. from EUR/(kWh*h) to EUR/kWh
+                soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_maxima_breach_price,
+                    actuator=asset,
+                    unit=FlexContextSchema()
+                    .declared_fields["soc_maxima_breach_price"]
+                    ._get_unit(soc_maxima_breach_price),
+                    query_window=(start, end),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    fallback_attribute="soc-maxima-breach-price",
+                    fill_sides=True,
+                )
                 # Set up commitments DataFrame
                 # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_maxima_d = get_continuous_series_sensor_or_quantity(

From c90c3fdcb6512f734f7f76a2bd0b86bb78a09ec2 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:18:17 +0100
Subject: [PATCH 067/151] docs: comment another section of flex-context prices

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 6c7e138bbe..fa0d1bc93b 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -71,6 +71,7 @@ class FlexContextSchema(Schema):
         return_magnitude=False,
     )
 
+    # Capacity breach commitments
     ems_production_capacity_in_mw = VariableQuantityField(
         "MW",
         required=False,

From 0e2651dfe111ac000d377f645a18c44050ca889c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:31:38 +0100
Subject: [PATCH 068/151] dev: add undocumented relax-soc-constraints field

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py         | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index fa0d1bc93b..eebb08e268 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -47,6 +47,10 @@ class FlexContextSchema(Schema):
         value_validator=validate.Range(min=0),
         default=None,
     )
+    # Dev field
+    relax_soc_constraints = fields.Bool(
+        data_key="relax-soc-constraints", load_default=False
+    )
 
     # Energy commitments
     ems_power_capacity_in_mw = VariableQuantityField(
@@ -136,6 +140,19 @@ class FlexContextSchema(Schema):
         SensorIdField(), data_key="inflexible-device-sensors"
     )
 
+    @validates_schema
+    def process_relax_soc_constraints(self, data: dict, **kwargs):
+        """Fill in default soc breach prices when asked to relax SoC constraints.
+
+        todo: this assumes EUR currency is used for all prices
+        """
+        if data["relax_soc_constraints"]:
+            if data.get("soc_minima_breach_price") is None:
+                data["soc_minima_breach_price"] = ur.Quantity("1000 EUR/kWh")
+            if data.get("soc_maxima_breach_price") is None:
+                data["soc_maxima_breach_price"] = ur.Quantity("1000 EUR/kWh")
+        return data
+
     @validates_schema
     def check_prices(self, data: dict, **kwargs):
         """Check assumptions about prices.

From c187f8111faf8a034de69d7c89d9af7cde6c42b1 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:44:12 +0100
Subject: [PATCH 069/151] fix: use of temporary variables started in
 88b7e4dfe396dd6d9031b78aa7fc3c664d5e77d0

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 24 ++++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 523ecf2d32..edac4b5d53 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -506,7 +506,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 # todo: check flex-model for soc_minima_breach_price and soc_maxima_breach_price fields; if these are defined, create a StockCommitment using both prices (if only 1 price is given, still create the commitment, but only penalize one direction)
             if isinstance(soc_minima[d], Sensor):
-                soc_minima[d] = get_continuous_series_sensor_or_quantity(
+                soc_minima_d = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -531,8 +531,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
-                # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet
-                soc_minima_d = get_continuous_series_sensor_or_quantity(
+                # soc_minima_d_temp is a temp variable because add_storage_constraints can't deal with Series yet
+                soc_minima_d_temp = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -544,7 +544,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 commitment = StockCommitment(
                     name="soc minima",
-                    quantity=soc_minima_d,
+                    quantity=soc_minima_d_temp,
                     # negative price because breaching in the downwards (shortage) direction is penalized
                     downwards_deviation_price=-soc_minima_breach_price,
                     _type="any",
@@ -554,10 +554,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 commitments.append(commitment)
 
                 # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_minima[d] = None
+                soc_minima_d = None
 
             if isinstance(soc_maxima[d], Sensor):
-                soc_maxima[d] = get_continuous_series_sensor_or_quantity(
+                soc_maxima_d = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -582,8 +582,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
-                # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet
-                soc_maxima_d = get_continuous_series_sensor_or_quantity(
+                # soc_maxima_d_temp is a temp variable because add_storage_constraints can't deal with Series yet
+                soc_maxima_d_temp = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -595,7 +595,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 commitment = StockCommitment(
                     name="soc maxima",
-                    quantity=soc_maxima_d,
+                    quantity=soc_maxima_d_temp,
                     # positive price because breaching in the upwards (surplus) direction is penalized
                     upwards_deviation_price=soc_maxima_breach_price,
                     _type="any",
@@ -605,7 +605,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 commitments.append(commitment)
 
                 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_maxima[d] = None
+                soc_maxima_d = None
 
             if soc_at_start[d] is not None:
                 device_constraints[d] = add_storage_constraints(
@@ -614,8 +614,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     resolution,
                     soc_at_start[d],
                     soc_targets[d],
-                    soc_maxima[d],
-                    soc_minima[d],
+                    soc_maxima_d,
+                    soc_minima_d,
                     soc_max[d],
                     soc_min[d],
                 )

From 7c164b23fa03edb577f75ba2f521ec530c17d73f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:46:48 +0100
Subject: [PATCH 070/151] Revert "fix: use of temporary variables started in
 88b7e4dfe396dd6d9031b78aa7fc3c664d5e77d0"

This reverts commit c187f8111faf8a034de69d7c89d9af7cde6c42b1.
---
 flexmeasures/data/models/planning/storage.py | 24 ++++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index edac4b5d53..523ecf2d32 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -506,7 +506,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 # todo: check flex-model for soc_minima_breach_price and soc_maxima_breach_price fields; if these are defined, create a StockCommitment using both prices (if only 1 price is given, still create the commitment, but only penalize one direction)
             if isinstance(soc_minima[d], Sensor):
-                soc_minima_d = get_continuous_series_sensor_or_quantity(
+                soc_minima[d] = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -531,8 +531,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
-                # soc_minima_d_temp is a temp variable because add_storage_constraints can't deal with Series yet
-                soc_minima_d_temp = get_continuous_series_sensor_or_quantity(
+                # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet
+                soc_minima_d = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -544,7 +544,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 commitment = StockCommitment(
                     name="soc minima",
-                    quantity=soc_minima_d_temp,
+                    quantity=soc_minima_d,
                     # negative price because breaching in the downwards (shortage) direction is penalized
                     downwards_deviation_price=-soc_minima_breach_price,
                     _type="any",
@@ -554,10 +554,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 commitments.append(commitment)
 
                 # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_minima_d = None
+                soc_minima[d] = None
 
             if isinstance(soc_maxima[d], Sensor):
-                soc_maxima_d = get_continuous_series_sensor_or_quantity(
+                soc_maxima[d] = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -582,8 +582,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     fill_sides=True,
                 )
                 # Set up commitments DataFrame
-                # soc_maxima_d_temp is a temp variable because add_storage_constraints can't deal with Series yet
-                soc_maxima_d_temp = get_continuous_series_sensor_or_quantity(
+                # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet
+                soc_maxima_d = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima[d],
                     actuator=sensor_d,
                     unit="MWh",
@@ -595,7 +595,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 commitment = StockCommitment(
                     name="soc maxima",
-                    quantity=soc_maxima_d_temp,
+                    quantity=soc_maxima_d,
                     # positive price because breaching in the upwards (surplus) direction is penalized
                     upwards_deviation_price=soc_maxima_breach_price,
                     _type="any",
@@ -605,7 +605,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 commitments.append(commitment)
 
                 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint
-                soc_maxima_d = None
+                soc_maxima[d] = None
 
             if soc_at_start[d] is not None:
                 device_constraints[d] = add_storage_constraints(
@@ -614,8 +614,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     resolution,
                     soc_at_start[d],
                     soc_targets[d],
-                    soc_maxima_d,
-                    soc_minima_d,
+                    soc_maxima[d],
+                    soc_minima[d],
                     soc_max[d],
                     soc_min[d],
                 )

From 5c0c02df9dfe941b047b964b29f8a77f542e53d7 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:47:39 +0100
Subject: [PATCH 071/151] fix: make util function robust against redundancy

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index be33987194..9ba53ab032 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -503,7 +503,7 @@ def process_time_series_segments(
 
 
 def get_continuous_series_sensor_or_quantity(
-    variable_quantity: Sensor | list[dict] | ur.Quantity | None,
+    variable_quantity: Sensor | list[dict] | ur.Quantity | pd.Series | None,
     actuator: Sensor | Asset,
     unit: ur.Quantity | str,
     query_window: tuple[datetime, datetime],
@@ -538,6 +538,8 @@ def get_continuous_series_sensor_or_quantity(
                                     - The last available value serves as a naive forecast.
     :returns:                       time series data with missing values handled based on the chosen method.
     """
+    if isinstance(variable_quantity, pd.Series):
+        return variable_quantity
     if variable_quantity is None:
         variable_quantity = get_quantity_from_attribute(
             entity=actuator,

From fd617d95f9e5da9cd33ecf218af37be100f16406 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Mar 2025 17:52:15 +0100
Subject: [PATCH 072/151] fix: apply soc-breach commitments to each timeslot

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 523ecf2d32..c65b9ec5cd 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -547,7 +547,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     quantity=soc_minima_d,
                     # negative price because breaching in the downwards (shortage) direction is penalized
                     downwards_deviation_price=-soc_minima_breach_price,
-                    _type="any",
                     index=index,
                     device=d,
                 )
@@ -598,7 +597,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     quantity=soc_maxima_d,
                     # positive price because breaching in the upwards (surplus) direction is penalized
                     upwards_deviation_price=soc_maxima_breach_price,
-                    _type="any",
                     index=index,
                     device=d,
                 )

From 96e3c8f6c3a1c7ecbba5a9acc4962d5200b4af27 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 10:02:31 +0100
Subject: [PATCH 073/151] refactor: split up lines (yielding smaller git diffs,
 which are faster to review)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index 82e8523424..0466eb8953 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -42,7 +42,11 @@ Fields can have fixed values, but some fields can also point to sensors, so they
 The full list of flex-context fields follows below.
 For more details on the possible formats for field values, see :ref:`variable_quantities`.
 
-Where should you set these fields? Within requests to the API or the data model. If they are not sent in via the API (the endpoint triggering schedule computation), the scheduler will look them up on the `flex-context` field of the asset. For consumption price, production price and inflexible devices, the scheduler will also search if parent assets have set them (it should probably do the same for other flex context fields ― work in progress). Finally, some of these values will default to attributes, for legacy reasons.
+Where should you set these fields?
+Within requests to the API or the data model.
+If they are not sent in via the API (the endpoint triggering schedule computation), the scheduler will look them up on the `flex-context` field of the asset.
+For consumption price, production price and inflexible devices, the scheduler will also search if parent assets have set them (it should probably do the same for other flex context fields ― work in progress).
+Finally, some of these values will default to attributes, for legacy reasons.
 
 
 

From 727d1f32a79bc83d38f63f45fce6fbfce8ed6873 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 10:02:50 +0100
Subject: [PATCH 074/151] docs: clarify comment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index 0466eb8953..ed3fe65d70 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -43,7 +43,7 @@ The full list of flex-context fields follows below.
 For more details on the possible formats for field values, see :ref:`variable_quantities`.
 
 Where should you set these fields?
-Within requests to the API or the data model.
+Within requests to the API or by editing the relevant asset in the UI.
 If they are not sent in via the API (the endpoint triggering schedule computation), the scheduler will look them up on the `flex-context` field of the asset.
 For consumption price, production price and inflexible devices, the scheduler will also search if parent assets have set them (it should probably do the same for other flex context fields ― work in progress).
 Finally, some of these values will default to attributes, for legacy reasons.

From 3feed8eb99f839626ab8d0084eb6b442deb2118f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 10:03:24 +0100
Subject: [PATCH 075/151] fix: remove outdated comment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index ed3fe65d70..b6e91ad8e4 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -46,7 +46,6 @@ Where should you set these fields?
 Within requests to the API or by editing the relevant asset in the UI.
 If they are not sent in via the API (the endpoint triggering schedule computation), the scheduler will look them up on the `flex-context` field of the asset.
 For consumption price, production price and inflexible devices, the scheduler will also search if parent assets have set them (it should probably do the same for other flex context fields ― work in progress).
-Finally, some of these values will default to attributes, for legacy reasons.
 
 
 

From 58f5bbad8926390983f24f9aa7baae45bdabd38c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 10:05:51 +0100
Subject: [PATCH 076/151] docs: improve legibility (and split lines)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index b6e91ad8e4..6f2aa93b1e 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -148,7 +148,9 @@ The storage scheduler is suitable for batteries and :abbr:`EV (electric vehicle)
 The process scheduler is suitable for shiftable, breakable and inflexible loads, and is automatically selected for asset types ``"process"`` and ``"load"``.
 
 
-We describe the respective flex models below. At the moment, they have to be sent through the API (the endpoint to trigger schedule computation, or using the FlexMeasures client) or the CLI (the command to add schedules). We will soon work on the possibility to store (a subset of) these fields on the data model and edit them in the UI.
+We describe the respective flex models below.
+At the moment, they have to be sent through the API (the endpoint to trigger schedule computation, or using the FlexMeasures client) or through the CLI (the command to add schedules).
+We will soon work on the possibility to store (a subset of) these fields on the data model and edit them in the UI.
 
 
 Storage

From b4cc42e9b8882b7674e41b1d7f31534637b84718 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 10:07:56 +0100
Subject: [PATCH 077/151] docs: no over-capitalization in titles

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/views/asset-data.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst
index ee7bbc65d2..72968bd56a 100644
--- a/documentation/views/asset-data.rst
+++ b/documentation/views/asset-data.rst
@@ -26,7 +26,7 @@ The asset page as data dashboard
 The data charts are maybe the most interesting feature - they form your own data dashboard. When the most interesting sensors are shown, the replay button on the right creates a very meaningful dynamic insight!
 
 
-Sensors to show on Graph
+Sensors to show on a graph
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Use the "Add Graph" button to create graphs. For each graph, you can select one or more sensors, from all available sensors associated with the asset, including public sensors, and add them to your plot.  
@@ -42,7 +42,7 @@ Finally, it is possible to set custom titles for any sensor graph by clicking on
 Internally, the asset has a `sensors_to_show`` field, which controls which sensor data appears in the plot. This can also be set by a script. Accepted formats are simple lists of sensor IDs (e.g. `[2, [5,6]]` or a more expressive format (e.g. `[{"title": "Power", "sensor": 2}, {"title": "Costs", "sensors": [5,6] }`). 
 
 
-Editing Asset FlexContext 
+Editing an asset's flex-context
 =========================
 
 |
@@ -59,14 +59,14 @@ Overview
 * **Left Panel:** Displays a list of currently configured fields.
 * **Right Panel:** Shows details of the selected field and provides a form to modify its value.
 
-Adding a Field
+Adding a field
 --------------
 
 1.  **Select Field:** Choose the desired field from the dropdown menu in the top right corner of the modal.
 2.  **Add Field:** Click the "Add Field" button next to the dropdown.
 3.  The field will be added to the list in the left panel.
 
-Setting a Field Value
+Setting a field value
 ----------------------
 
 1.  **Select Field(if it is not selected yet):** Click on the field in the left panel.

From 37db6279a01d9fb43726efb2c4e64740fa2b3798 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 10:09:21 +0100
Subject: [PATCH 078/151] docs: missing space

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/views/asset-data.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst
index 72968bd56a..7b9750312c 100644
--- a/documentation/views/asset-data.rst
+++ b/documentation/views/asset-data.rst
@@ -69,7 +69,7 @@ Adding a field
 Setting a field value
 ----------------------
 
-1.  **Select Field(if it is not selected yet):** Click on the field in the left panel.
+1.  **Select Field (if it is not selected yet):** Click on the field in the left panel.
 2.  **Set Value:** In the right panel, use the provided form to set the field's value.
 
     * Some fields may only accept a sensor value.

From f92ccddeabbc8a77b2fece5e2664bf6518e0a358 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 10:28:01 +0100
Subject: [PATCH 079/151] fix: missing column in commitment frame for device
 flow commitments

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index a9b2f8daa0..84d52fc4cb 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -303,6 +303,7 @@ def to_frame(self) -> pd.DataFrame:
         """Contains all info apart from the name."""
         return pd.concat(
             [
+                self.device,
                 self.quantity,
                 self.upwards_deviation_price,
                 self.downwards_deviation_price,

From c7937055ca0157e0bb642488bdeeea3f681b1a9a Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Mar 2025 12:36:40 +0100
Subject: [PATCH 080/151] fix: stock commitment quantities are with respect to
 soc at start

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index c65b9ec5cd..955213fa6d 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -544,7 +544,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 commitment = StockCommitment(
                     name="soc minima",
-                    quantity=soc_minima_d,
+                    quantity=soc_minima_d - soc_at_start[d],
                     # negative price because breaching in the downwards (shortage) direction is penalized
                     downwards_deviation_price=-soc_minima_breach_price,
                     index=index,
@@ -594,7 +594,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 )
                 commitment = StockCommitment(
                     name="soc maxima",
-                    quantity=soc_maxima_d,
+                    quantity=soc_maxima_d - soc_at_start[d],
                     # positive price because breaching in the upwards (surplus) direction is penalized
                     upwards_deviation_price=soc_maxima_breach_price,
                     index=index,

From 361b3e7c1e8bf4d68da10b731ebcd889805f8b91 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 21 Mar 2025 09:43:47 +0100
Subject: [PATCH 081/151] fix: shift commitment quantities

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 955213fa6d..6321e74c8b 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -541,7 +541,11 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
                     resolve_overlaps="max",
-                )
+                ).shift(-1)
+                # shift soc minima by one resolution (they define a state at a certain time,
+                # while the commitment defines what the total stock should be at the end of a time slot,
+                # where the time slot is indexed by its starting time)
+
                 commitment = StockCommitment(
                     name="soc minima",
                     quantity=soc_minima_d - soc_at_start[d],
@@ -591,7 +595,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
                     resolve_overlaps="min",
-                )
+                ).shift(-1)
+                # shift soc maxima by one resolution (they define a state at a certain time,
+                # while the commitment defines what the total stock should be at the end of a time slot,
+                # where the time slot is indexed by its starting time)
                 commitment = StockCommitment(
                     name="soc maxima",
                     quantity=soc_maxima_d - soc_at_start[d],

From a61cc22b304bf0485083c486b463824297e7a052 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 21 Mar 2025 10:42:08 +0100
Subject: [PATCH 082/151] fix: skip setup of device stock commitment in case of
 no commitment quantity

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 6321e74c8b..ffd06395f0 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -516,7 +516,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     as_instantaneous_events=True,
                     resolve_overlaps="max",
                 )
-            if self.flex_context.get("soc_minima_breach_price") is not None:
+            if (
+                self.flex_context.get("soc_minima_breach_price") is not None
+                and soc_minima[d] is not None
+            ):
                 soc_minima_breach_price = self.flex_context["soc_minima_breach_price"]
                 soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima_breach_price,
@@ -570,7 +573,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     as_instantaneous_events=True,
                     resolve_overlaps="min",
                 )
-            if self.flex_context.get("soc_maxima_breach_price") is not None:
+            if (
+                self.flex_context.get("soc_maxima_breach_price") is not None
+                and soc_maxima[d] is not None
+            ):
                 soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"]
                 soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima_breach_price,

From 7211d0a48be982913c88f0a80a72d72946204475 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 21 Mar 2025 10:43:13 +0100
Subject: [PATCH 083/151] fix: take into account difference between commitment
 resolution and actuator resolution

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index ffd06395f0..71246a35a2 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -544,14 +544,16 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
                     resolve_overlaps="max",
-                ).shift(-1)
+                ).shift(-1) * (timedelta(hours=1) / resolution) - soc_at_start[d] * (
+                    timedelta(hours=1) / resolution
+                )
                 # shift soc minima by one resolution (they define a state at a certain time,
                 # while the commitment defines what the total stock should be at the end of a time slot,
                 # where the time slot is indexed by its starting time)
 
                 commitment = StockCommitment(
                     name="soc minima",
-                    quantity=soc_minima_d - soc_at_start[d],
+                    quantity=soc_minima_d,
                     # negative price because breaching in the downwards (shortage) direction is penalized
                     downwards_deviation_price=-soc_minima_breach_price,
                     index=index,
@@ -601,13 +603,15 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
                     resolve_overlaps="min",
-                ).shift(-1)
+                ).shift(-1) * (timedelta(hours=1) / resolution) - soc_at_start[d] * (
+                    timedelta(hours=1) / resolution
+                )
                 # shift soc maxima by one resolution (they define a state at a certain time,
                 # while the commitment defines what the total stock should be at the end of a time slot,
                 # where the time slot is indexed by its starting time)
                 commitment = StockCommitment(
                     name="soc maxima",
-                    quantity=soc_maxima_d - soc_at_start[d],
+                    quantity=soc_maxima_d,
                     # positive price because breaching in the upwards (surplus) direction is penalized
                     upwards_deviation_price=soc_maxima_breach_price,
                     index=index,

From 124fc3d79b3ea333bd0e4b188d3cf61685adb093 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 21 Mar 2025 11:30:21 +0100
Subject: [PATCH 084/151] fix: avoid adjusting the timing of SoC time series
 segments

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 9ba53ab032..71cb39aa1f 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -429,7 +429,7 @@ def get_series_from_quantity_or_sensor(
             index=index,
             variable_quantity=variable_quantity,
             unit=unit,
-            resolution=resolution,
+            resolution=resolution if not as_instantaneous_events else timedelta(0),
             resolve_overlaps=resolve_overlaps,
             fill_sides=fill_sides,
         )

From 036f54de7c3024e1d62d32b7d92829b3b97c22b9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 25 Mar 2025 18:37:48 +0100
Subject: [PATCH 085/151] feat: move to a price unit that make costs
 independent of sensor resolution: EUR/(kWh*h)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py     |  6 ++++--
 flexmeasures/data/models/planning/utils.py       | 15 ++++++++++++---
 flexmeasures/data/schemas/scheduling/__init__.py |  4 ++--
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 4502870f5b..338268a50a 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -499,7 +499,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     actuator=asset,
                     unit=FlexContextSchema()
                     .declared_fields["soc_minima_breach_price"]
-                    ._get_unit(soc_minima_breach_price),
+                    ._get_unit(soc_minima_breach_price)
+                    + "*h",
                     query_window=(start, end),
                     resolution=resolution,
                     beliefs_before=belief_time,
@@ -558,7 +559,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     actuator=asset,
                     unit=FlexContextSchema()
                     .declared_fields["soc_maxima_breach_price"]
-                    ._get_unit(soc_maxima_breach_price),
+                    ._get_unit(soc_maxima_breach_price)
+                    + "*h",
                     query_window=(start, end),
                     resolution=resolution,
                     beliefs_before=belief_time,
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index c1a9ed6a90..15e322e2b8 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -354,7 +354,12 @@ def get_series_from_quantity_or_sensor(
         if np.isnan(variable_quantity.magnitude):
             magnitude = np.nan
         else:
-            magnitude = variable_quantity.to(unit).magnitude
+            magnitude = convert_units(
+                variable_quantity.magnitude,
+                str(variable_quantity.units),
+                unit,
+                resolution,
+            )
         time_series = pd.Series(magnitude, index=index, name="event_value")
     elif isinstance(variable_quantity, Sensor):
         bdf: tb.BeliefsDataFrame = TimedBelief.search(
@@ -370,7 +375,9 @@ def get_series_from_quantity_or_sensor(
         if as_instantaneous_events:
             bdf = bdf.resample_events(timedelta(0), boundary_policy=resolve_overlaps)
         time_series = simplify_index(bdf).reindex(index).squeeze()
-        time_series = convert_units(time_series, variable_quantity.unit, unit)
+        time_series = convert_units(
+            time_series, variable_quantity.unit, unit, resolution
+        )
     elif isinstance(variable_quantity, list):
         time_series = process_time_series_segments(
             index=index,
@@ -425,7 +432,9 @@ def process_time_series_segments(
             if np.isnan(value.magnitude):
                 value = np.nan
             else:
-                value = value.to(unit).magnitude
+                value = convert_units(
+                    value.magnitude, str(value.units), unit, resolution
+                )
         start = event["start"]
         end = event["end"]
         # Assign the value to the corresponding segment in the DataFrame
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index eebb08e268..af56f24c60 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -34,14 +34,14 @@ class FlexContextSchema(Schema):
 
     # Device commitments
     soc_minima_breach_price = VariableQuantityField(
-        "/MWh",
+        "/(MWh*h)",
         data_key="soc-minima-breach-price",
         required=False,
         value_validator=validate.Range(min=0),
         default=None,
     )
     soc_maxima_breach_price = VariableQuantityField(
-        "/MWh",
+        "/(MWh*h)",
         data_key="soc-maxima-breach-price",
         required=False,
         value_validator=validate.Range(min=0),

From af8b57bc03d11d1f9953a6292b6836a869b8d972 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 25 Mar 2025 18:46:41 +0100
Subject: [PATCH 086/151] docs: update note on soc breach prices

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index 353ad3ce86..493dcc2d06 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -112,10 +112,10 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
      - ``"260 EUR/MWh"``
      - Production peaks above the ``site-peak-production`` are penalized against this per-kW price. [#penalty_field]_
    * - ``soc-minima-breach-price``
-     - ``"6 EUR/kWh"``
+     - ``"2 EUR/kWh/min"``
      - Penalty for not meeting ``soc-minima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
    * - ``soc-maxima-breach-price``
-     - ``"6 EUR/kWh"``
+     - ``"2 EUR/kWh/min"``
      - Penalty for not meeting ``soc-maxima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
 
 .. [#old_sensor_field] The old field only accepted an integer (sensor ID).
@@ -130,7 +130,7 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
 
 .. [#production] Example: with a connection capacity (``site-power-capacity``) of 1 MVA (apparent power) and a production capacity (``site-production-capacity``) of 400 kW (active power), the scheduler will make sure that the grid inflow doesn't exceed 400 kW.
 
-.. [#soc_breach_prices] The SoC breach prices (e.g. 6 EUR/kWh) to use for the schedule are applied over each time step equal to the sensor resolution. For example, a SoC breach price of 6 EUR/kWh per hour, for scheduling a 5-minute resolution sensor, should be passed as a SoC breach price of :math:`6*5/60 = 0.50` EUR/kWh.
+.. [#soc_breach_prices] The SoC breach prices (e.g. 2 EUR/kWh/min) to use for the schedule are applied over each time step equal to the sensor resolution. For example, a SoC breach price of 2 EUR/kWh/min, for scheduling a 5-minute resolution sensor, will be applied as a SoC breach price of 10 EUR/kWh for breaches measured every 5 minutes.
 
 .. note:: If no (symmetric, consumption and production) site capacity is defined (also not as defaults), the scheduler will not enforce any bound on the site power.
           The flexible device can still have its own power limit defined in its flex-model.

From 3d3af94d2a2e482cd7d8c199abe62ea4f792d213 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 09:02:09 +0100
Subject: [PATCH 087/151] docs: fix punctuation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index 84d52fc4cb..0b86dd28a8 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -218,7 +218,7 @@ class Commitment:
         Device to which the commitment pertains. If None, the commitment pertains to the EMS.
     index:
         Pandas DatetimeIndex defining the time slots to which the commitment applies.
-        The index is shared by the group, quantity. upwards_deviation_price and downwards_deviation_price Pandas Series.
+        The index is shared by the group, quantity, upwards_deviation_price and downwards_deviation_price Pandas Series.
     _type:
         'any' or 'each'. Any deviation is penalized via 1 group, whereas each deviation is penalized via n groups.
     group:

From 41ecc75396565458a7e2f911de60275fb1563ec5 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 09:09:53 +0100
Subject: [PATCH 088/151] style: improve IDE tooltip hints with changing to a
 different docstring format

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py | 41 ++++++++-----------
 1 file changed, 17 insertions(+), 24 deletions(-)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index 0b86dd28a8..50d73089d5 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -209,30 +209,23 @@ def deserialize_flex_config(self):
 @dataclass
 class Commitment:
     """Contractual commitment specifying prices for deviating from a given position.
-    ::
-    Parameters
-    ----------
-    name:
-        Name of the commitment.
-    device:
-        Device to which the commitment pertains. If None, the commitment pertains to the EMS.
-    index:
-        Pandas DatetimeIndex defining the time slots to which the commitment applies.
-        The index is shared by the group, quantity, upwards_deviation_price and downwards_deviation_price Pandas Series.
-    _type:
-        'any' or 'each'. Any deviation is penalized via 1 group, whereas each deviation is penalized via n groups.
-    group:
-        Each time slot is assigned to a group. Deviations are determined for each group.
-        The deviation of a group is determined by the time slot with the maximum deviation within that group.
-    quantity:
-        The deviation for each group is determined with respect to this quantity.
-        Can be initialized with a constant value, but always returns a Pandas Series (see also the `index` parameter).
-    upwards_deviation_price:
-        The deviation in the upwards direction is priced against this price. Use a positive price to set a penalty.
-        Can be initialized with a constant value, but always returns a Pandas Series (see also the `index` parameter).
-    downwards_deviation_price:
-        The deviation in the downwards direction is priced against this price. Use a negative price to set a penalty.
-        Can be initialized with a constant value, but always returns a Pandas Series (see also the `index` parameter).
+
+    Attributes:
+        name:       Name of the commitment.
+        device:     Device to which the commitment pertains. If None, the commitment pertains to the EMS.
+        index:      Pandas DatetimeIndex defining the time slots to which the commitment applies.
+                    The index is shared by the group, quantity, upwards_deviation_price and downwards_deviation_price Pandas Series.
+        _type:      'any' or 'each'. Any deviation is penalized via 1 group, whereas each deviation is penalized via n groups.
+        group:      Each time slot is assigned to a group. Deviations are determined for each group.
+                    The deviation of a group is determined by the time slot with the maximum deviation within that group.
+        quantity:   The deviation for each group is determined with respect to this quantity.
+                    Can be initialized with a constant value, but always returns a Pandas Series (see also the `index` parameter).
+        upwards_deviation_price:
+                    The deviation in the upwards direction is priced against this price. Use a positive price to set a penalty.
+                    Can be initialized with a constant value, but always returns a Pandas Series (see also the `index` parameter).
+        downwards_deviation_price:
+                    The deviation in the downwards direction is priced against this price. Use a negative price to set a penalty.
+                    Can be initialized with a constant value, but always returns a Pandas Series (see also the `index` parameter).
     """
 
     name: str

From b6b2ce36c3ccc5224f15de9dd916b16ed203524d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 09:56:26 +0100
Subject: [PATCH 089/151] docs: clarify semantics of index in FlowCommitment
 and StockCommitment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index 50d73089d5..e5336b00fa 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -308,10 +308,14 @@ def to_frame(self) -> pd.DataFrame:
 
 
 class FlowCommitment(Commitment):
+    """NB index contains event start, while quantity applies to average flow between event start and end."""
+
     pass
 
 
 class StockCommitment(Commitment):
+    """NB index contains event start, while quantity applies to stock at event end."""
+
     pass
 
 

From f446ee6e8dac1cebda0ec827d6cccc392cfde80d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 17:47:29 +0100
Subject: [PATCH 090/151] fix: EUR/MWh/h should be interpreted as EUR/(MWh*h)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index af56f24c60..aa6c803d0b 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -215,20 +215,24 @@ def _try_to_convert_price_units(self, data):
                 currency_unit = price_unit.split("/")[0]
 
                 if previous_currency_unit is None:
-                    previous_currency_unit = currency_unit
+                    previous_currency_unit = str(
+                        ur.Quantity(currency_unit).to_base_units().units
+                    )
                     previous_field_name = price_field.data_key
-                elif units_are_convertible(currency_unit, previous_currency_unit):
+                if units_are_convertible(currency_unit, previous_currency_unit):
                     # Make sure all compatible currency units are on the same scale (e.g. not kEUR mixed with EUR)
                     if currency_unit != previous_currency_unit:
-                        denominator_unit = price_unit.split("/")[1]
+                        denominator_unit = str(
+                            ur.Unit(currency_unit) / ur.Unit(price_unit)
+                        )
                         if isinstance(data[field], ur.Quantity):
                             data[field] = data[field].to(
-                                f"{previous_currency_unit}/{denominator_unit}"
+                                f"{previous_currency_unit}/({denominator_unit})"
                             )
                         elif isinstance(data[field], list):
                             for j in range(len(data[field])):
                                 data[field][j]["value"] = data[field][j]["value"].to(
-                                    f"{previous_currency_unit}/{denominator_unit}"
+                                    f"{previous_currency_unit}/({denominator_unit})"
                                 )
                         elif isinstance(data[field], Sensor):
                             raise ValidationError(

From ddfeb97016444cc381ec1fbb8c74ca5b6ff65e18 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 17:54:19 +0100
Subject: [PATCH 091/151] fix: deal with soc constraints set at the end of the
 scheduling horizon

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 338268a50a..341d0e4416 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -513,17 +513,18 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     variable_quantity=soc_minima[d],
                     actuator=sensor_d,
                     unit="MWh",
-                    query_window=(start, end),
+                    query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
                     resolve_overlaps="max",
-                ).shift(-1) * (timedelta(hours=1) / resolution) - soc_at_start[d] * (
-                    timedelta(hours=1) / resolution
                 )
                 # shift soc minima by one resolution (they define a state at a certain time,
                 # while the commitment defines what the total stock should be at the end of a time slot,
                 # where the time slot is indexed by its starting time)
+                soc_minima_d = soc_minima_d.shift(-1, freq=resolution) * (
+                    timedelta(hours=1) / resolution
+                ) - soc_at_start[d] * (timedelta(hours=1) / resolution)
 
                 commitment = StockCommitment(
                     name="soc minima",
@@ -573,17 +574,19 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     variable_quantity=soc_maxima[d],
                     actuator=sensor_d,
                     unit="MWh",
-                    query_window=(start, end),
+                    query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
                     resolve_overlaps="min",
-                ).shift(-1) * (timedelta(hours=1) / resolution) - soc_at_start[d] * (
-                    timedelta(hours=1) / resolution
                 )
                 # shift soc maxima by one resolution (they define a state at a certain time,
                 # while the commitment defines what the total stock should be at the end of a time slot,
                 # where the time slot is indexed by its starting time)
+                soc_maxima_d = soc_maxima_d.shift(-1, freq=resolution) * (
+                    timedelta(hours=1) / resolution
+                ) - soc_at_start[d] * (timedelta(hours=1) / resolution)
+
                 commitment = StockCommitment(
                     name="soc maxima",
                     quantity=soc_maxima_d,

From 10418ce995d87e54d6404019a793e613b3fff7e3 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 18:00:03 +0100
Subject: [PATCH 092/151] feat: test soc-minima-breach-price field

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_storage.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index 39a9595caa..84e9d8b200 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -36,7 +36,7 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
             "soc-min": "0 MWh",
             "soc-max": "1 MWh",
             "power-capacity": "1 MW",
-            "soc-targets": [
+            "soc-minima": [
                 {
                     "datetime": "2015-01-02T00:00:00+01:00",
                     "value": "1 MWh",
@@ -78,6 +78,7 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
                 }
                 for i in production_prices.index
             ],
+            "soc-minima-breach-price": "100 EUR/kWh/min",  # high breach price (to mimic a hard constraint)
         },
         return_multiple=True,
     )

From c7110bec86c4a664e32607fb4e76e93b93fa866d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 18:02:04 +0100
Subject: [PATCH 093/151] feat: scheduler also returns the units of schedules
 (better safe than sorry)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 341d0e4416..972739e1a0 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -1154,6 +1154,7 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType:
                     "name": "storage_schedule",
                     "sensor": sensor,
                     "data": storage_schedule[sensor],
+                    "unit": sensor.unit,
                 }
                 for sensor in sensors
             ] + [

From a3263dd506ab5ff2fbcd4d167ca3db56f554bc9e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 18:04:36 +0100
Subject: [PATCH 094/151] refactor: clarify purpose of variable by renaming

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index aa6c803d0b..c7460ba8db 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -206,7 +206,7 @@ def check_prices(self, data: dict, **kwargs):
     def _try_to_convert_price_units(self, data):
         """Convert price units to the same unit and scale if they can (incl. same currency)."""
 
-        previous_currency_unit = None
+        shared_currency_unit = None
         previous_field_name = None
         for field in self.declared_fields:
             if field[-5:] == "price" and field in data:
@@ -214,25 +214,25 @@ def _try_to_convert_price_units(self, data):
                 price_unit = price_field._get_unit(data[field])
                 currency_unit = price_unit.split("/")[0]
 
-                if previous_currency_unit is None:
-                    previous_currency_unit = str(
+                if shared_currency_unit is None:
+                    shared_currency_unit = str(
                         ur.Quantity(currency_unit).to_base_units().units
                     )
                     previous_field_name = price_field.data_key
-                if units_are_convertible(currency_unit, previous_currency_unit):
+                if units_are_convertible(currency_unit, shared_currency_unit):
                     # Make sure all compatible currency units are on the same scale (e.g. not kEUR mixed with EUR)
-                    if currency_unit != previous_currency_unit:
+                    if currency_unit != shared_currency_unit:
                         denominator_unit = str(
                             ur.Unit(currency_unit) / ur.Unit(price_unit)
                         )
                         if isinstance(data[field], ur.Quantity):
                             data[field] = data[field].to(
-                                f"{previous_currency_unit}/({denominator_unit})"
+                                f"{shared_currency_unit}/({denominator_unit})"
                             )
                         elif isinstance(data[field], list):
                             for j in range(len(data[field])):
                                 data[field][j]["value"] = data[field][j]["value"].to(
-                                    f"{previous_currency_unit}/({denominator_unit})"
+                                    f"{shared_currency_unit}/({denominator_unit})"
                                 )
                         elif isinstance(data[field], Sensor):
                             raise ValidationError(
@@ -241,7 +241,7 @@ def _try_to_convert_price_units(self, data):
                 else:
                     field_name = price_field.data_key
                     raise ValidationError(
-                        f"Prices must share the same monetary unit. '{field_name}' uses '{currency_unit}', but '{previous_field_name}' used '{previous_currency_unit}'.",
+                        f"Prices must share the same monetary unit. '{field_name}' uses '{currency_unit}', but '{previous_field_name}' used '{shared_currency_unit}'.",
                         field_name=field_name,
                     )
         return data

From 75f5007a4fa80201c4d2aa101db78417324e7150 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Mar 2025 23:26:24 +0100
Subject: [PATCH 095/151] feat: return costs unit

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py            | 1 +
 flexmeasures/data/models/planning/tests/test_storage.py | 2 ++
 flexmeasures/data/schemas/scheduling/__init__.py        | 1 +
 3 files changed, 4 insertions(+)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 972739e1a0..cee1530bdf 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -1166,6 +1166,7 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType:
                             commitments, model.commitment_costs.values()
                         )
                     },
+                    "unit": self.flex_context["shared_currency_unit"],
                 },
             ]
         else:
diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index 84e9d8b200..4d01590059 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -86,6 +86,8 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
 
     schedule = results[0]["data"]
     costs = results[1]["data"]
+    costs_unit = results[1]["unit"]
+    assert costs_unit == "EUR"
 
     # Check if constraints were met
     check_constraints(battery, schedule, soc_at_start)
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index c7460ba8db..bafae85f0d 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -244,6 +244,7 @@ def _try_to_convert_price_units(self, data):
                         f"Prices must share the same monetary unit. '{field_name}' uses '{currency_unit}', but '{previous_field_name}' used '{shared_currency_unit}'.",
                         field_name=field_name,
                     )
+        data["shared_currency_unit"] = shared_currency_unit
         return data
 
 

From 0e574b19ec8801a0e0fc33fc0cdc2cdeb7b41880 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 27 Mar 2025 14:09:44 +0100
Subject: [PATCH 096/151] fix: alignment of SoC constraints

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/conftest.py                           |  4 +++-
 flexmeasures/data/models/planning/storage.py       | 14 +++++++-------
 .../data/models/planning/tests/test_solver.py      |  6 +++---
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py
index 75a8c75622..b444ac100b 100644
--- a/flexmeasures/conftest.py
+++ b/flexmeasures/conftest.py
@@ -1424,7 +1424,9 @@ def soc_sensors(db, add_battery_assets, setup_sources) -> tuple:
         source=setup_sources["Seita"],
     )
 
-    yield soc_maxima, soc_minima, soc_targets, values
+    soc_schedule = pd.Series(data=values, index=time_slots)
+
+    yield soc_maxima, soc_minima, soc_targets, soc_schedule
 
 
 @pytest.fixture(scope="module")
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index cee1530bdf..6d3c5cbb65 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -471,7 +471,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     variable_quantity=soc_targets[d],
                     actuator=sensor_d,
                     unit="MWh",
-                    query_window=(start, end),
+                    query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
@@ -483,7 +483,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     variable_quantity=soc_minima[d],
                     actuator=sensor_d,
                     unit="MWh",
-                    query_window=(start, end),
+                    query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
@@ -501,12 +501,12 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     .declared_fields["soc_minima_breach_price"]
                     ._get_unit(soc_minima_breach_price)
                     + "*h",
-                    query_window=(start, end),
+                    query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
                     fallback_attribute="soc-minima-breach-price",
                     fill_sides=True,
-                )
+                ).shift(-1, freq=resolution)
                 # Set up commitments DataFrame
                 # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_minima_d = get_continuous_series_sensor_or_quantity(
@@ -544,7 +544,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     variable_quantity=soc_maxima[d],
                     actuator=sensor_d,
                     unit="MWh",
-                    query_window=(start, end),
+                    query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
                     as_instantaneous_events=True,
@@ -562,12 +562,12 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     .declared_fields["soc_maxima_breach_price"]
                     ._get_unit(soc_maxima_breach_price)
                     + "*h",
-                    query_window=(start, end),
+                    query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
                     fallback_attribute="soc-maxima-breach-price",
                     fill_sides=True,
-                )
+                ).shift(-1, freq=resolution)
                 # Set up commitments DataFrame
                 # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_maxima_d = get_continuous_series_sensor_or_quantity(
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 7177c698f9..8e38d34648 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2035,7 +2035,7 @@ def test_soc_maxima_minima_targets(db, add_battery_assets, soc_sensors):
     power = add_battery_assets["Test battery with dynamic power capacity"].sensors[0]
     epex_da = get_test_sensor(db)
 
-    soc_maxima, soc_minima, soc_targets, values = soc_sensors
+    soc_maxima, soc_minima, soc_targets, expected_soc_schedule = soc_sensors
 
     tz = pytz.timezone("Europe/Amsterdam")
     start = tz.localize(datetime(2015, 1, 1))
@@ -2078,7 +2078,7 @@ def compute_schedule(flex_model):
     soc = check_constraints(power, schedule, soc_at_start)
 
     # soc targets are achieved
-    assert all(abs(soc[9:].values - values[:-1]) < 1e-5)
+    assert all(abs(soc[8:].values - expected_soc_schedule) < 1e-5)
 
     # remove soc-targets and use soc-maxima and soc-minima
     del flex_model["soc-targets"]
@@ -2091,7 +2091,7 @@ def compute_schedule(flex_model):
     # soc-maxima and soc-minima constraints are respected
     # this yields the same results as with the SOC targets
     # because soc-maxima = soc-minima = soc-targets
-    assert all(abs(soc[9:].values - values[:-1]) < 1e-5)
+    assert all(abs(soc[8:].values - expected_soc_schedule) < 1e-5)
 
 
 @pytest.mark.parametrize("unit", [None, "MWh", "kWh"])

From 9e7fa456256753904de95735178d5c792fe1c543 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 28 Mar 2025 09:15:04 +0100
Subject: [PATCH 097/151] fix: compute currency unit by dividing over (1 over)
 the expected denominator

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index bafae85f0d..a8f2914e25 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -212,7 +212,11 @@ def _try_to_convert_price_units(self, data):
             if field[-5:] == "price" and field in data:
                 price_field = self.declared_fields[field]
                 price_unit = price_field._get_unit(data[field])
-                currency_unit = price_unit.split("/")[0]
+                currency_unit = str(
+                    (
+                        ur.Quantity(price_unit) / ur.Quantity(f"1{price_field.to_unit}")
+                    ).units
+                )
 
                 if shared_currency_unit is None:
                     shared_currency_unit = str(

From b0e2b454388ae952228d766d4d60fcbfce0bfe62 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 28 Mar 2025 13:34:04 +0100
Subject: [PATCH 098/151] fix: adapt test

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 8e38d34648..af88b15ae2 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2004,8 +2004,9 @@ def test_add_storage_constraint_from_sensor(
     scheduler_info = scheduler._prepare()
     storage_constraints = scheduler_info[5][0]
 
-    expected_target_start = pd.Timedelta(expected_start) + start
-    expected_target_end = pd.Timedelta(expected_end) + start
+    # Start (date) + start (time) - resolution (due to device_constraints indexing states by the start of their preceding time slot)
+    expected_target_start = start + pd.Timedelta(expected_start) - resolution
+    expected_target_end = start + pd.Timedelta(expected_end) - resolution
     expected_soc_target_value = 0.5 * timedelta(hours=1) / resolution
 
     # convert dates from UTC to local time (Europe/Amsterdam)

From 60a773c310ff594d4800cd8fa7308dea1b303666 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 28 Mar 2025 13:55:54 +0100
Subject: [PATCH 099/151] feat: expand test

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index af88b15ae2..babfea57a0 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -1927,13 +1927,14 @@ def test_battery_storage_efficiency_sensor(
 
 
 @pytest.mark.parametrize(
-    "sensor_name, expected_start, expected_end",
+    "sensor_name, expected_start, expected_end, n_constraints",
     [
         # A value defined in a coarser resolution is upsampled to match the power sensor resolution.
         (
             "soc-targets (1h)",
             "14:00:00",
             "15:00:00",
+            5,
         ),
         # A value defined in a finer resolution is downsampled to match the power sensor resolution.
         # Only a single value coincides with the power sensor resolution.
@@ -1941,6 +1942,7 @@ def test_battery_storage_efficiency_sensor(
             "soc-targets (5min)",
             "14:00:00",
             "14:00:00",  # not "14:15:00"
+            1,
             marks=pytest.mark.xfail(
                 reason="timely-beliefs doesn't yet make it possible to resample to a certain frequency and event resolution simultaneously"
             ),
@@ -1950,12 +1952,14 @@ def test_battery_storage_efficiency_sensor(
             "soc-targets (15min)",
             "14:00:00",
             "14:15:00",
+            2,
         ),
         # For an instantaneous sensor, the value is set to the interval containing the instantaneous event.
         (
             "soc-targets (instantaneous)",
             "14:00:00",
             "14:00:00",
+            1,
         ),
         # This is an event at 14:05:00 with a duration of 15min.
         # This constraint should span the intervals from 14:00 to 14:15 and from 14:15 to 14:30, but we are not reindexing properly.
@@ -1963,6 +1967,7 @@ def test_battery_storage_efficiency_sensor(
             "soc-targets (15min lagged)",
             "14:00:00",
             "14:15:00",
+            1,
             marks=pytest.mark.xfail(
                 reason="we should re-index the series so that values of the original index that overlap are used."
             ),
@@ -1975,6 +1980,7 @@ def test_add_storage_constraint_from_sensor(
     sensor_name,
     expected_start,
     expected_end,
+    n_constraints,
     db,
 ):
     """
@@ -1986,13 +1992,14 @@ def test_add_storage_constraint_from_sensor(
     end = tz.localize(datetime(2015, 1, 2))
     resolution = timedelta(minutes=15)
     soc_targets = add_soc_targets[sensor_name]
+    soc_at_start = 0
 
     flex_model = {
         "soc-max": 2,
         "soc-min": 0,
         "roundtrip-efficiency": 1,
         "production-capacity": "0kW",
-        "soc-at-start": 0,
+        "soc-at-start": soc_at_start,
     }
 
     flex_model["soc-targets"] = {"sensor": soc_targets.id}
@@ -2024,6 +2031,19 @@ def test_add_storage_constraint_from_sensor(
         == expected_soc_target_value
     )
 
+    # Check the resulting schedule against the constraints
+    consumption_schedule = scheduler.compute()
+    soc_schedule = integrate_time_series(
+        consumption_schedule, soc_at_start, decimal_precision=6
+    )
+    comparison_df = pd.concat([equals, soc_schedule], axis=1).dropna()
+    assert (
+        len(comparison_df)
+    ) == n_constraints, f"we expect {n_constraints} device constraints"
+    assert all(
+        comparison_df.iloc[:, 0] == comparison_df.iloc[:, 1] * 4
+    ), "the device constraint values should be 1/4th of the actual SoC values, due to the 15-minute resolution of the battery's power sensor"
+
 
 def test_soc_maxima_minima_targets(db, add_battery_assets, soc_sensors):
     """

From 999d76a1a7f0e2bb07b074219e659c99e8d68cdf Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 28 Mar 2025 14:07:46 +0100
Subject: [PATCH 100/151] fix: previous expansion of test (still required a
 shift, which became evident after adding another soc constraint)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py         | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index babfea57a0..fb563f020c 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -2003,6 +2003,12 @@ def test_add_storage_constraint_from_sensor(
     }
 
     flex_model["soc-targets"] = {"sensor": soc_targets.id}
+    flex_model["soc-maxima"] = [
+        {
+            "datetime": "2015-01-01T13:45:00+01:00",
+            "value": "0.4 MWh",
+        }
+    ]
 
     scheduler: Scheduler = StorageScheduler(
         battery, start, end, resolution, flex_model=flex_model
@@ -2036,7 +2042,10 @@ def test_add_storage_constraint_from_sensor(
     soc_schedule = integrate_time_series(
         consumption_schedule, soc_at_start, decimal_precision=6
     )
-    comparison_df = pd.concat([equals, soc_schedule], axis=1).dropna()
+    # Note the equality constraints are shifted back to account for how they define the index to denote
+    # the start of the event that ends in the given equality state, whereas the index of the soc_schedule
+    # denotes the exact time of the given SoC state
+    comparison_df = pd.concat([equals.shift(1), soc_schedule], axis=1).dropna()
     assert (
         len(comparison_df)
     ) == n_constraints, f"we expect {n_constraints} device constraints"

From ca45058d73908fede3b59894d93210ab5c455754 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 31 Mar 2025 12:50:03 +0200
Subject: [PATCH 101/151] fix: resolve pet peeve: -0.0 after integration

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/utils/calculations.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/utils/calculations.py b/flexmeasures/utils/calculations.py
index 9b626ef0a8..a6ff2c798a 100644
--- a/flexmeasures/utils/calculations.py
+++ b/flexmeasures/utils/calculations.py
@@ -174,4 +174,5 @@ def integrate_time_series(
     )
     if decimal_precision is not None:
         stocks = stocks.round(decimal_precision)
+        stocks = stocks.mask(stocks == -0.0, 0.0)
     return stocks

From 2fe1b1d94671797a83f1383bd64a7181eb4553fe Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 31 Mar 2025 16:29:22 +0200
Subject: [PATCH 102/151] feat: relax consumption-capacity and
 production-capacity using a FlowCommitment per device

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  |  89 ++++++++++--
 .../models/planning/tests/test_storage.py     | 135 +++++++++++++++++-
 .../data/schemas/scheduling/__init__.py       |  14 ++
 3 files changed, 224 insertions(+), 14 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 6d3c5cbb65..8a2a6d9379 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -623,13 +623,13 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 min_value=0,  # capacities are positive by definition
                 resolve_overlaps="min",
             )
+            device_constraints[d]["derivative max"] = power_capacity_in_mw[d]
+            device_constraints[d]["derivative min"] = -power_capacity_in_mw[d]
 
             if sensor_d.get_attribute("is_strictly_non_positive"):
                 device_constraints[d]["derivative min"] = 0
             else:
-                device_constraints[d]["derivative min"] = (
-                    -1
-                ) * get_continuous_series_sensor_or_quantity(
+                production_capacity_d = get_continuous_series_sensor_or_quantity(
                     variable_quantity=production_capacity[d],
                     actuator=sensor_d,
                     unit="MW",
@@ -641,23 +641,86 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     min_value=0,  # capacities are positive by definition
                     resolve_overlaps="min",
                 )
+                if (
+                    self.flex_context.get("production_breach_price") is not None
+                    and production_capacity[d] is not None
+                ):
+                    # consumption-capacity will become a soft constraint
+                    production_breach_price = self.flex_context[
+                        "production_breach_price"
+                    ]
+                    production_breach_price = get_continuous_series_sensor_or_quantity(
+                        variable_quantity=production_breach_price,
+                        actuator=asset,
+                        unit=FlexContextSchema()
+                        .declared_fields["production_breach_price"]
+                        ._get_unit(production_breach_price),
+                        query_window=(start, end),
+                        resolution=resolution,
+                        beliefs_before=belief_time,
+                        fallback_attribute="production-breach-price",
+                        fill_sides=True,
+                    )
+                    # Set up commitments DataFrame
+                    commitment = FlowCommitment(
+                        name=f"production breaches device {d}",
+                        quantity=-production_capacity_d,
+                        # negative price because breaching in the downwards (shortage) direction is penalized
+                        downwards_deviation_price=-production_breach_price,
+                        index=index,
+                        device=d,
+                    )
+                    commitments.append(commitment)
+                else:
+                    # consumption-capacity will become a hard constraint
+                    device_constraints[d]["derivative min"] = -production_capacity_d
             if sensor_d.get_attribute("is_strictly_non_negative"):
                 device_constraints[d]["derivative max"] = 0
             else:
-                device_constraints[d]["derivative max"] = (
-                    get_continuous_series_sensor_or_quantity(
-                        variable_quantity=consumption_capacity[d],
-                        actuator=sensor_d,
-                        unit="MW",
+                consumption_capacity_d = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=consumption_capacity[d],
+                    actuator=sensor_d,
+                    unit="MW",
+                    query_window=(start, end),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    fallback_attribute="consumption_capacity",
+                    min_value=0,  # capacities are positive by definition
+                    max_value=power_capacity_in_mw[d],
+                    resolve_overlaps="min",
+                )
+                if (
+                    self.flex_context.get("consumption_breach_price") is not None
+                    and consumption_capacity[d] is not None
+                ):
+                    # consumption-capacity will become a soft constraint
+                    consumption_breach_price = self.flex_context[
+                        "consumption_breach_price"
+                    ]
+                    consumption_breach_price = get_continuous_series_sensor_or_quantity(
+                        variable_quantity=consumption_breach_price,
+                        actuator=asset,
+                        unit=FlexContextSchema()
+                        .declared_fields["consumption_breach_price"]
+                        ._get_unit(consumption_breach_price),
                         query_window=(start, end),
                         resolution=resolution,
                         beliefs_before=belief_time,
-                        fallback_attribute="consumption_capacity",
-                        min_value=0,  # capacities are positive by definition
-                        max_value=power_capacity_in_mw[d],
-                        resolve_overlaps="min",
+                        fallback_attribute="consumption-breach-price",
+                        fill_sides=True,
                     )
-                )
+                    # Set up commitments DataFrame
+                    commitment = FlowCommitment(
+                        name=f"consumption breaches device {d}",
+                        quantity=consumption_capacity_d,
+                        upwards_deviation_price=consumption_breach_price,
+                        index=index,
+                        device=d,
+                    )
+                    commitments.append(commitment)
+                else:
+                    # consumption-capacity will become a hard constraint
+                    device_constraints[d]["derivative max"] = consumption_capacity_d
 
             all_stock_delta = []
 
diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index 4d01590059..bac98ec079 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -35,7 +35,7 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
             "soc-at-start": f"{soc_at_start} MWh",
             "soc-min": "0 MWh",
             "soc-max": "1 MWh",
-            "power-capacity": "1 MW",
+            "power-capacity": "1 MVA",
             "soc-minima": [
                 {
                     "datetime": "2015-01-02T00:00:00+01:00",
@@ -111,3 +111,136 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
     np.testing.assert_almost_equal(costs["consumption peak"], 260 / 1000 * (25 - 20))
     # No production peak
     np.testing.assert_almost_equal(costs["production peak"], 0)
+
+
+def test_battery_relaxation(add_battery_assets, db):
+    _, battery = get_sensors_from_db(
+        db, add_battery_assets, battery_name="Test battery"
+    )
+    tz = pytz.timezone("Europe/Amsterdam")
+    start = tz.localize(datetime(2015, 1, 1))
+    end = tz.localize(datetime(2015, 1, 2))
+    resolution = timedelta(minutes=15)
+    soc_at_start = 0.4
+    index = initialize_index(start=start, end=end, resolution=resolution)
+    consumption_prices = pd.Series(100, index=index)
+    # Introduce arbitrage opportunity
+    consumption_prices["2015-01-01T16:00:00+01:00":"2015-01-01T17:00:00+01:00"] = (
+        0  # cheap energy
+    )
+    consumption_prices["2015-01-01T17:00:00+01:00":"2015-01-01T18:00:00+01:00"] = (
+        1000  # expensive energy
+    )
+    production_prices = consumption_prices - 10
+    device_power_breach_price = 100
+
+    # Set up consumption/production capacity as a time series
+    # i.e. it takes 16 hours to go from 0.4 to 0.8 MWh
+    consumption_capacity_in_mw = 0.025
+    consumption_capacity = pd.Series(consumption_capacity_in_mw, index=index)
+    consumption_capacity["2015-01-01T12:00:00+01:00":"2015-01-01T18:00:00+01:00"] = (
+        0  # no charging
+    )
+    production_capacity_in_mw = consumption_capacity
+
+    scheduler: Scheduler = StorageScheduler(
+        battery,
+        start,
+        end,
+        resolution,
+        flex_model={
+            "soc-at-start": f"{soc_at_start} MWh",
+            "soc-min": "0 MWh",
+            "soc-max": "1 MWh",
+            "power-capacity": f"{consumption_capacity_in_mw} MVA",
+            "consumption-capacity": [
+                {
+                    "start": i.isoformat(),
+                    "duration": "PT1H",
+                    "value": f"{consumption_capacity[i]} MW",
+                }
+                for i in consumption_capacity.index
+            ],
+            "production-capacity": [
+                {
+                    "start": i.isoformat(),
+                    "duration": "PT1H",
+                    "value": f"{production_capacity_in_mw[i]} MW",
+                }
+                for i in production_capacity_in_mw.index
+            ],
+            "soc-minima": [
+                {
+                    "start": "2015-01-01T12:00:00+01:00",
+                    "end": "2015-01-01T18:00:00+01:00",
+                    # "duration": "PT6H",  # todo: fails validation, which points to a bug
+                    "value": "0.8 MWh",
+                }
+            ],
+            "prefer-charging-sooner": False,
+        },
+        flex_context={
+            "consumption-price": [
+                {
+                    "start": i.isoformat(),
+                    "duration": "PT1H",
+                    "value": f"{consumption_prices[i]} EUR/MWh",
+                }
+                for i in consumption_prices.index
+            ],
+            "production-price": [
+                {
+                    "start": i.isoformat(),
+                    "duration": "PT1H",
+                    "value": f"{production_prices[i]} EUR/MWh",
+                }
+                for i in production_prices.index
+            ],
+            "site-power-capacity": "2 MW",  # should be big enough to avoid any infeasibilities
+            # "site-consumption-capacity": "1 kW",  # we'll need to breach this to reach the target
+            "site-consumption-breach-price": "1000 EUR/kW",
+            "site-production-breach-price": "1000 EUR/kW",
+            "site-peak-consumption": "20 kW",
+            "site-peak-production": "20 kW",
+            "site-peak-consumption-price": "260 EUR/MW",
+            # The following is a constant price, but this checks currency conversion in case a later price field is
+            # set to a time series specs (i.e. a list of dicts, where each dict represents a time slot)
+            "site-peak-production-price": [
+                {
+                    "start": i.isoformat(),
+                    "duration": "PT1H",
+                    "value": "260 EUR/MW",
+                }
+                for i in production_prices.index
+            ],
+            "soc-minima-breach-price": "100 EUR/kWh/min",  # high breach price (to mimic a hard constraint)
+            "consumption-breach-price": f"{device_power_breach_price} EUR/kW",  # lower breach price (thus prioritizing minimizing soc breaches)
+            "production-breach-price": f"{device_power_breach_price} EUR/kW",  # lower breach price (thus prioritizing minimizing soc breaches)
+        },
+        return_multiple=True,
+    )
+    results = scheduler.compute()
+
+    schedule = results[0]["data"]
+    costs = results[1]["data"]
+    costs_unit = results[1]["unit"]
+    assert costs_unit == "EUR"
+
+    # Check if constraints were met
+    check_constraints(battery, schedule, soc_at_start)
+
+    # Check for constant charging profile until 4 PM
+    np.testing.assert_allclose(
+        schedule[:"2015-01-01T15:45:00+01:00"], consumption_capacity_in_mw
+    )
+
+    # Check for standing idle from 4 PM to 6 PM
+    np.testing.assert_allclose(
+        schedule["2015-01-01T16:00:00+01:00":"2015-01-01T17:45:00+01:00"], 0
+    )
+
+    # Check costs are correct
+    np.testing.assert_almost_equal(
+        costs["consumption breaches device 0"],
+        device_power_breach_price * consumption_capacity_in_mw * 1000 * 4 * 4,
+    )  # 100 EUR/kWh/min * mean(0.15 MWh) * 1000 kW/MW * 4 hours * 4 15min/hour
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index a8f2914e25..71d214c2cb 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -33,6 +33,20 @@ class FlexContextSchema(Schema):
     """This schema defines fields that provide context to the portfolio to be optimized."""
 
     # Device commitments
+    consumption_breach_price = VariableQuantityField(
+        "/MW",
+        data_key="consumption-breach-price",
+        required=False,
+        value_validator=validate.Range(min=0),
+        default=None,
+    )
+    production_breach_price = VariableQuantityField(
+        "/MW",
+        data_key="production-breach-price",
+        required=False,
+        value_validator=validate.Range(min=0),
+        default=None,
+    )
     soc_minima_breach_price = VariableQuantityField(
         "/(MWh*h)",
         data_key="soc-minima-breach-price",

From 3f442e96fa3368e45d38a3a3b35f67b19a0162e8 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 10:55:56 +0200
Subject: [PATCH 103/151] fix: correct inline comment

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index a31b2a2280..47c67fa066 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -668,7 +668,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     commitment = FlowCommitment(
                         name=f"production breaches device {d}",
                         quantity=-production_capacity_d,
-                        # negative price because breaching in the downwards (shortage) direction is penalized
+                        # negative price because breaching in the downwards (production) direction is penalized
                         downwards_deviation_price=-production_breach_price,
                         index=index,
                         device=d,

From bddf629b49d15cd35799e76a705e81a82cba2ee5 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 11:01:20 +0200
Subject: [PATCH 104/151] dev: field to switch on device capacity constraint
 relaxation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 71d214c2cb..defe7139a7 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -65,6 +65,9 @@ class FlexContextSchema(Schema):
     relax_soc_constraints = fields.Bool(
         data_key="relax-soc-constraints", load_default=False
     )
+    relax_capacity_constraints = fields.Bool(
+        data_key="relax-capacity-constraints", load_default=False
+    )
 
     # Energy commitments
     ems_power_capacity_in_mw = VariableQuantityField(
@@ -167,6 +170,19 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
                 data["soc_maxima_breach_price"] = ur.Quantity("1000 EUR/kWh")
         return data
 
+    @validates_schema
+    def process_relax_capacity_constraints(self, data: dict, **kwargs):
+        """Fill in default capacity breach prices when asked to relax capacity constraints.
+
+        todo: this assumes EUR currency is used for all prices
+        """
+        if data["relax_capacity_constraints"]:
+            if data.get("consumption_breach_price") is None:
+                data["consumption_breach_price"] = ur.Quantity("10 EUR/kW")
+            if data.get("production_breach_price") is None:
+                data["production_breach_price"] = ur.Quantity("10 EUR/kW")
+        return data
+
     @validates_schema
     def check_prices(self, data: dict, **kwargs):
         """Check assumptions about prices.

From a1169a24bacc3b037b3173db03d8049d7da07a28 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 16:32:08 +0200
Subject: [PATCH 105/151] fix: default prices should use the same denominator
 as defined in the field

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py       | 42 +++++++++++++++++--
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index defe7139a7..91ca0fb906 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -163,11 +163,28 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
 
         todo: this assumes EUR currency is used for all prices
         """
+        default_soc_breach_price = "1000 EUR/(kWh*h)"
         if data["relax_soc_constraints"]:
             if data.get("soc_minima_breach_price") is None:
-                data["soc_minima_breach_price"] = ur.Quantity("1000 EUR/kWh")
+                # use the same denominator as defined in the field
+                data["soc_minima_breach_price"] = ur.Quantity(
+                    default_soc_breach_price
+                ).to(
+                    "EUR/"
+                    + self.declared_fields["soc_minima_breach_price"].to_unit.split(
+                        "/"
+                    )[-1]
+                )
             if data.get("soc_maxima_breach_price") is None:
-                data["soc_maxima_breach_price"] = ur.Quantity("1000 EUR/kWh")
+                # use the same denominator as defined in the field
+                data["soc_maxima_breach_price"] = ur.Quantity(
+                    default_soc_breach_price
+                ).to(
+                    "EUR/"
+                    + self.declared_fields["soc_maxima_breach_price"].to_unit.split(
+                        "/"
+                    )[-1]
+                )
         return data
 
     @validates_schema
@@ -176,11 +193,28 @@ def process_relax_capacity_constraints(self, data: dict, **kwargs):
 
         todo: this assumes EUR currency is used for all prices
         """
+        default_capacity_breach_price = "100 EUR/kW"
         if data["relax_capacity_constraints"]:
             if data.get("consumption_breach_price") is None:
-                data["consumption_breach_price"] = ur.Quantity("10 EUR/kW")
+                # use the same denominator as defined in the field
+                data["consumption_breach_price"] = ur.Quantity(
+                    default_capacity_breach_price
+                ).to(
+                    "EUR/"
+                    + self.declared_fields["consumption_breach_price"].to_unit.split(
+                        "/"
+                    )[-1]
+                )
             if data.get("production_breach_price") is None:
-                data["production_breach_price"] = ur.Quantity("10 EUR/kW")
+                # use the same denominator as defined in the field
+                data["production_breach_price"] = ur.Quantity(
+                    default_capacity_breach_price
+                ).to(
+                    "EUR/"
+                    + self.declared_fields["production_breach_price"].to_unit.split(
+                        "/"
+                    )[-1]
+                )
         return data
 
     @validates_schema

From fd139dd83c6b567d127229cccd2c96339b401478 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 18:06:41 +0200
Subject: [PATCH 106/151] fix: implement todos to not rely on a currency
 assumption

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py       | 22 ++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 91ca0fb906..3d2e7454d9 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -157,11 +157,10 @@ class FlexContextSchema(Schema):
         SensorIdField(), data_key="inflexible-device-sensors"
     )
 
-    @validates_schema
     def process_relax_soc_constraints(self, data: dict, **kwargs):
         """Fill in default soc breach prices when asked to relax SoC constraints.
 
-        todo: this assumes EUR currency is used for all prices
+        This relies on the check_prices validator to run first.
         """
         default_soc_breach_price = "1000 EUR/(kWh*h)"
         if data["relax_soc_constraints"]:
@@ -170,7 +169,8 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
                 data["soc_minima_breach_price"] = ur.Quantity(
                     default_soc_breach_price
                 ).to(
-                    "EUR/"
+                    data["shared_currency_unit"]
+                    + "/"
                     + self.declared_fields["soc_minima_breach_price"].to_unit.split(
                         "/"
                     )[-1]
@@ -180,18 +180,18 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
                 data["soc_maxima_breach_price"] = ur.Quantity(
                     default_soc_breach_price
                 ).to(
-                    "EUR/"
+                    data["shared_currency_unit"]
+                    + "/"
                     + self.declared_fields["soc_maxima_breach_price"].to_unit.split(
                         "/"
                     )[-1]
                 )
         return data
 
-    @validates_schema
     def process_relax_capacity_constraints(self, data: dict, **kwargs):
         """Fill in default capacity breach prices when asked to relax capacity constraints.
 
-        todo: this assumes EUR currency is used for all prices
+        This relies on the check_prices validator to run first.
         """
         default_capacity_breach_price = "100 EUR/kW"
         if data["relax_capacity_constraints"]:
@@ -200,7 +200,8 @@ def process_relax_capacity_constraints(self, data: dict, **kwargs):
                 data["consumption_breach_price"] = ur.Quantity(
                     default_capacity_breach_price
                 ).to(
-                    "EUR/"
+                    data["shared_currency_unit"]
+                    + "/"
                     + self.declared_fields["consumption_breach_price"].to_unit.split(
                         "/"
                     )[-1]
@@ -210,7 +211,8 @@ def process_relax_capacity_constraints(self, data: dict, **kwargs):
                 data["production_breach_price"] = ur.Quantity(
                     default_capacity_breach_price
                 ).to(
-                    "EUR/"
+                    data["shared_currency_unit"]
+                    + "/"
                     + self.declared_fields["production_breach_price"].to_unit.split(
                         "/"
                     )[-1]
@@ -265,6 +267,10 @@ def check_prices(self, data: dict, **kwargs):
         # All prices must share the same unit
         data = self._try_to_convert_price_units(data)
 
+        # Call schema validators that must come after this one, because they rely on data["shared_currency_unit"]
+        self.process_relax_soc_constraints(data)
+        self.process_relax_capacity_constraints(data)
+
         return data
 
     def _try_to_convert_price_units(self, data):

From bd4b35d60228032b62b1ae28bae07b2ebc60e8ce Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 2 Apr 2025 10:31:10 +0200
Subject: [PATCH 107/151] fix: last 3 prices were NaN after resampling from 1
 hour to 15 minutes

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/tests/test_scheduling_simultaneous.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py
index 0ed3239c48..bdb3566dee 100644
--- a/flexmeasures/data/tests/test_scheduling_simultaneous.py
+++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py
@@ -83,7 +83,9 @@ def test_create_simultaneous_jobs(
     prices.index = prices.index.tz_convert("Europe/Amsterdam")
 
     # Resample prices to match power resolution
-    prices = prices.resample("15min").ffill()
+    prices = prices.reindex(
+        battery_power.index,
+    ).ffill()
 
     start_charging = start + pd.Timedelta(hours=8)
     end_charging = start + pd.Timedelta(hours=10) - sensors["Test EV"].event_resolution

From 22ac6a7d5c164193f49577a4cdb677aada73ef40 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 2 Apr 2025 10:34:33 +0200
Subject: [PATCH 108/151] refactor: check total costs first, then its breakdown

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/tests/test_scheduling_simultaneous.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py
index bdb3566dee..fca84edc58 100644
--- a/flexmeasures/data/tests/test_scheduling_simultaneous.py
+++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py
@@ -90,7 +90,7 @@ def test_create_simultaneous_jobs(
     start_charging = start + pd.Timedelta(hours=8)
     end_charging = start + pd.Timedelta(hours=10) - sensors["Test EV"].event_resolution
 
-    # Assertions
+    # Check schedules
     assert (ev_power.loc[start_charging:end_charging] != -0.005).values.any()  # 5 kW
     assert (
         battery_power.loc[start_charging:end_charging] != 0.005
@@ -107,7 +107,11 @@ def test_create_simultaneous_jobs(
     expected_ev_costs = 2.1625
     expected_battery_costs = -5.515
 
-    # Assert costs
+    # Check costs
+    assert (
+        round(total_cost, 4) == expected_total_cost
+    ), f"Total cost should be {expected_total_cost} €, got {total_cost} €"
+
     assert (
         round(total_cost, 4) == expected_total_cost
     ), f"Total cost should be {expected_total_cost} €, got {total_cost} €"

From 1ce92536982b2ac1b340f01178248a3c27ee3126 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 2 Apr 2025 10:35:08 +0200
Subject: [PATCH 109/151] fix: update expected costs

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/tests/test_scheduling_simultaneous.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py
index fca84edc58..96a0bc7743 100644
--- a/flexmeasures/data/tests/test_scheduling_simultaneous.py
+++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py
@@ -103,9 +103,9 @@ def test_create_simultaneous_jobs(
     total_cost = ev_costs + battery_costs
 
     # Define expected costs based on resolution
-    expected_total_cost = -3.3525
-    expected_ev_costs = 2.1625
-    expected_battery_costs = -5.515
+    expected_ev_costs = 2.3125
+    expected_battery_costs = -5.59
+    expected_total_cost = -3.2775
 
     # Check costs
     assert (

From b5152031f9bde0e7ac04de7a4bdebf6fd45a8307 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 2 Apr 2025 10:37:14 +0200
Subject: [PATCH 110/151] fix: grammar

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/tests/test_scheduling_simultaneous.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py
index 96a0bc7743..69166bb97e 100644
--- a/flexmeasures/data/tests/test_scheduling_simultaneous.py
+++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py
@@ -110,14 +110,15 @@ def test_create_simultaneous_jobs(
     # Check costs
     assert (
         round(total_cost, 4) == expected_total_cost
-    ), f"Total cost should be {expected_total_cost} €, got {total_cost} €"
+    ), f"Total costs should be {expected_total_cost} €, got {total_cost} €"
 
     assert (
         round(total_cost, 4) == expected_total_cost
     ), f"Total cost should be {expected_total_cost} €, got {total_cost} €"
     assert (
         round(ev_costs, 4) == expected_ev_costs
-    ), f"EV cost should be {expected_ev_costs} €, got {ev_costs} €"
+    ), f"EV costs should be {expected_ev_costs} €, got {ev_costs} €"
+
     assert (
         round(battery_costs, 4) == expected_battery_costs
-    ), f"Battery cost should be {expected_battery_costs} €, got {battery_costs} €"
+    ), f"Battery costs should be {expected_battery_costs} €, got {battery_costs} €"

From f142322a1586a28b52b3429738ba27aa80e3cd27 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 2 Apr 2025 10:39:06 +0200
Subject: [PATCH 111/151] =?UTF-8?q?fix:=20=E2=82=AC=20currency=20symbols?=
 =?UTF-8?q?=20preceeds=20the=20value?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/tests/test_scheduling_simultaneous.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py
index 69166bb97e..8d75c04f58 100644
--- a/flexmeasures/data/tests/test_scheduling_simultaneous.py
+++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py
@@ -110,15 +110,15 @@ def test_create_simultaneous_jobs(
     # Check costs
     assert (
         round(total_cost, 4) == expected_total_cost
-    ), f"Total costs should be {expected_total_cost} €, got {total_cost} €"
+    ), f"Total costs should be €{expected_total_cost}, got €{total_cost}"
 
     assert (
         round(total_cost, 4) == expected_total_cost
     ), f"Total cost should be {expected_total_cost} €, got {total_cost} €"
     assert (
         round(ev_costs, 4) == expected_ev_costs
-    ), f"EV costs should be {expected_ev_costs} €, got {ev_costs} €"
+    ), f"EV costs should be €{expected_ev_costs}, got €{ev_costs}"
 
     assert (
         round(battery_costs, 4) == expected_battery_costs
-    ), f"Battery costs should be {expected_battery_costs} €, got {battery_costs} €"
+    ), f"Battery costs should be €{expected_battery_costs}, got €{battery_costs}"

From 0f5376343dc14cb3a586ad277f2c4a4016d42ea4 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 2 Apr 2025 10:45:44 +0200
Subject: [PATCH 112/151] fix: downsample rather than upsample to compute costs

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../tests/test_scheduling_simultaneous.py     | 25 +++++++------------
 1 file changed, 9 insertions(+), 16 deletions(-)

diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py
index 8d75c04f58..19a3feec47 100644
--- a/flexmeasures/data/tests/test_scheduling_simultaneous.py
+++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py
@@ -70,6 +70,14 @@ def test_create_simultaneous_jobs(
         assert len(battery_power) == 96
     assert battery_power.sources.unique()[0].model == "StorageScheduler"
     battery_power = battery_power.droplevel([1, 2, 3])
+    start_charging = start + pd.Timedelta(hours=8)
+    end_charging = start + pd.Timedelta(hours=10) - sensors["Test EV"].event_resolution
+
+    # Check schedules
+    assert (ev_power.loc[start_charging:end_charging] != -0.005).values.any()  # 5 kW
+    assert (
+        battery_power.loc[start_charging:end_charging] != 0.005
+    ).values.any()  # 5 kW
 
     # Get price data
     price_sensor_id = flex_description_sequential["flex_context"][
@@ -82,23 +90,8 @@ def test_create_simultaneous_jobs(
     prices = prices.droplevel([1, 2, 3])
     prices.index = prices.index.tz_convert("Europe/Amsterdam")
 
-    # Resample prices to match power resolution
-    prices = prices.reindex(
-        battery_power.index,
-    ).ffill()
-
-    start_charging = start + pd.Timedelta(hours=8)
-    end_charging = start + pd.Timedelta(hours=10) - sensors["Test EV"].event_resolution
-
-    # Check schedules
-    assert (ev_power.loc[start_charging:end_charging] != -0.005).values.any()  # 5 kW
-    assert (
-        battery_power.loc[start_charging:end_charging] != 0.005
-    ).values.any()  # 5 kW
-
     # Calculate costs
-    ev_resolution = sensors["Test EV"].event_resolution.total_seconds() / 3600
-    ev_costs = (-ev_power * prices * ev_resolution).sum().item()
+    ev_costs = (-ev_power.resample("1h").mean() * prices).sum().item()
     battery_costs = (-battery_power.resample("1h").mean() * prices).sum().item()
     total_cost = ev_costs + battery_costs
 

From 67b5d80ea1a6ad97d2f206cd7d73759f5211dbc5 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 18:35:15 +0200
Subject: [PATCH 113/151] docs: main changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index af2ef60b22..8efd0afede 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -11,6 +11,7 @@ New features
 -------------
 
 * Support saving the storage schedule SOC using the ``flex-model`` field ``state-of-charge`` to the ``flex-model`` [see `PR #1392 <https://github.com/FlexMeasures/flexmeasures/pull/1392>`_]
+* Allow relaxing device-level power constraints, by setting penalties for not meeting these constraints, using two new ``flex-context`` fields [see `PR #1405 <https://github.com/FlexMeasures/flexmeasures/pull/1405>`_]
 * Save changes in asset flex-context form right away [see `PR #1390 <https://github.com/FlexMeasures/flexmeasures/pull/1390>`_]
 * Introduced new page to create sensor for a parent asset [see `PR #1394 <https://github.com/FlexMeasures/flexmeasures/pull/1394>`_]
 * Marker clusters on the dashboard map expand in a tree to show the hierarchical relationship of the assets they represent [see `PR #1410 <https://github.com/FlexMeasures/flexmeasures/pull/1410>`_]

From 40d309c8c6982ad4d78310a5f830e29199816f39 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 18:38:13 +0200
Subject: [PATCH 114/151] docs: API changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index 411cedd3a2..ce753f587b 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -5,6 +5,15 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace.
 
+v3.0-23 | 2025-04-08
+""""""""""""""""""""
+
+- Introduce new price fields in the ``flex-context`` in order to relax device-level power constraints in the ``device-model``:
+
+  - ``consumption-breach-price``: if set, the ``consumption-capacity`` is used as a soft constraint.
+  - ``production-breach-price``: if set, the ``production-capacity`` is used as a soft constraint.
+  - In both cases, breaching the capacity is penalized according to this per-kW price. The price is applied to each breach that occurs given the resolution of the scheduled power sensor.
+
 v3.0-22 | 2025-03-17
 """"""""""""""""""""
 

From 6fae8043a6723cee7aa3f2d498712b92329aea78 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 18:39:47 +0200
Subject: [PATCH 115/151] fix: correct units in changelog documentation of
 soc-breach-prices

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index ce753f587b..01d2518aec 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -19,8 +19,9 @@ v3.0-22 | 2025-03-17
 
 - Introduce new price fields in the ``flex-context`` in order to relax SoC constraints in the ``device-model``:
 
-  - ``soc-minima-breach-price``: if set, the ``soc-minima`` are used as a soft constraint, and not meeting the minima is penalized according to this per-kWh price. The price is applied to each breach that occurs given the resolution of the scheduled power sensor.
-  - ``soc-maxima-breach-price``: if set, the ``soc-maxima`` are used as a soft constraint, and not meeting the maxima is penalized according to this per-kWh price. The price is applied to each breach that occurs given the resolution of the scheduled power sensor.
+  - ``soc-minima-breach-price``: if set, the ``soc-minima`` are used as a soft constraint.
+  - ``soc-maxima-breach-price``: if set, the ``soc-maxima`` are used as a soft constraint.
+  - In both cases, not meeting the constraint is penalized according to this per-kWh-per-h price. That means the costs increase linearly the longer the state of charge breaches the constraint.
 
 v3.0-22 | 2024-12-27
 """"""""""""""""""""

From 7fe04553464ea4da0d3c508bb2496aa3c26e3260 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 8 Apr 2025 18:43:22 +0200
Subject: [PATCH 116/151] fix: correct units in changelog documentation of
 device-level power-breach-prices

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index 01d2518aec..69fdbde117 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -12,7 +12,7 @@ v3.0-23 | 2025-04-08
 
   - ``consumption-breach-price``: if set, the ``consumption-capacity`` is used as a soft constraint.
   - ``production-breach-price``: if set, the ``production-capacity`` is used as a soft constraint.
-  - In both cases, breaching the capacity is penalized according to this per-kW price. The price is applied to each breach that occurs given the resolution of the scheduled power sensor.
+  - In both cases, breaching the capacity is penalized according to this per-kW-per-h price (i.e. a per-kWh price). That means the costs increase linearly with the duration of the breach.
 
 v3.0-22 | 2025-03-17
 """"""""""""""""""""

From b5a9764612455d39e4fdaee8cb2fc9b141fd78ba Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Apr 2025 20:05:48 +0200
Subject: [PATCH 117/151] docs: fix typo

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/scheduling/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 3d2e7454d9..4260446c3c 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -61,7 +61,7 @@ class FlexContextSchema(Schema):
         value_validator=validate.Range(min=0),
         default=None,
     )
-    # Dev field
+    # Dev fields
     relax_soc_constraints = fields.Bool(
         data_key="relax-soc-constraints", load_default=False
     )

From 7cf7dda62551ebc34ee3ae072b5e43a9fb2ddd9a Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 14:02:39 +0200
Subject: [PATCH 118/151] feat: apply breach prices to both breach capacity and
 breach effort

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst              |   3 +-
 .../models/planning/linear_optimization.py    |  18 ++-
 flexmeasures/data/models/planning/storage.py  | 107 +++++++++++++-----
 .../models/planning/tests/test_storage.py     |  11 +-
 4 files changed, 98 insertions(+), 41 deletions(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index 69fdbde117..cdce919456 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -12,7 +12,8 @@ v3.0-23 | 2025-04-08
 
   - ``consumption-breach-price``: if set, the ``consumption-capacity`` is used as a soft constraint.
   - ``production-breach-price``: if set, the ``production-capacity`` is used as a soft constraint.
-  - In both cases, breaching the capacity is penalized according to this per-kW-per-h price (i.e. a per-kWh price). That means the costs increase linearly with the duration of the breach.
+  - In both cases, the price is applied both to the largest breach in the planning window (as a per-kW price) and to each breach that occurs (as a per-kW price per hour).
+    That means both high breaches and long breaches are penalized.
 
 v3.0-22 | 2025-03-17
 """"""""""""""""""""
diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
index 052ee80c28..11d9eb0ca0 100644
--- a/flexmeasures/data/models/planning/linear_optimization.py
+++ b/flexmeasures/data/models/planning/linear_optimization.py
@@ -182,10 +182,6 @@ def convert_commitments_to_subcommitments(
                 sub_commitment = df[df["group"] == group].drop(columns=["group"])
 
                 # Catch non-uniqueness
-                if len(sub_commitment["quantity"].unique()) > 1:
-                    raise ValueError(
-                        "Commitment groups cannot have non-unique quantities."
-                    )
                 if len(sub_commitment["upwards deviation price"].unique()) > 1:
                     raise ValueError(
                         "Commitment groups cannot have non-unique upwards deviation prices."
@@ -257,8 +253,8 @@ def price_up_select(m, c):
             return 0
         return price
 
-    def commitment_quantity_select(m, c):
-        quantity = commitments[c]["quantity"].iloc[0]
+    def commitment_quantity_select(m, c, j):
+        quantity = commitments[c][commitments[c]["j"] == j]["quantity"].values[0]
         if np.isnan(quantity):
             return -infinity
         return quantity
@@ -355,7 +351,7 @@ def device_stock_delta(m, d, j):
     model.up_price = Param(model.c, initialize=price_up_select)
     model.down_price = Param(model.c, initialize=price_down_select)
     model.commitment_quantity = Param(
-        model.c, domain=Reals, initialize=commitment_quantity_select
+        model.cj, domain=Reals, initialize=commitment_quantity_select
     )
     model.device_max = Param(model.d, model.j, initialize=device_max_select)
     model.device_min = Param(model.d, model.j, initialize=device_min_select)
@@ -473,14 +469,14 @@ def device_stock_commitment_equalities(m, c, j, d):
         if (
             "device" not in commitments[c].columns
             or (commitments[c]["device"] != d).all()
-            or m.commitment_quantity[c] == -infinity
+            or m.commitment_quantity[c, j] == -infinity
         ):
             # Commitment c does not concern device d
             return Constraint.Skip
 
         # Determine center part of the lhs <= center part <= rhs constraint
         center_part = (
-            m.commitment_quantity[c]
+            m.commitment_quantity[c, j]
             + m.commitment_downwards_deviation[c]
             + m.commitment_upwards_deviation[c]
         )
@@ -505,7 +501,7 @@ def ems_flow_commitment_equalities(m, c, j):
         if (
             "device" in commitments[c].columns
             and not pd.isnull(commitments[c]["device"]).all()
-        ) or m.commitment_quantity[c] == -infinity:
+        ) or m.commitment_quantity[c, j] == -infinity:
             # Commitment c does not concern EMS
             return Constraint.Skip
         if (
@@ -525,7 +521,7 @@ def ems_flow_commitment_equalities(m, c, j):
                 else None
             ),
             # 0 if "upwards deviation price" in commitments[c].columns else None,  # todo: possible simplification
-            m.commitment_quantity[c]
+            m.commitment_quantity[c, j]
             + m.commitment_downwards_deviation[c]
             + m.commitment_upwards_deviation[c]
             - sum(m.ems_power[:, j]),
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 1bf4acdc13..87f32b4ee6 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -652,24 +652,52 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     production_breach_price = self.flex_context[
                         "production_breach_price"
                     ]
-                    production_breach_price = get_continuous_series_sensor_or_quantity(
-                        variable_quantity=production_breach_price,
-                        actuator=asset,
-                        unit=FlexContextSchema()
-                        .declared_fields["production_breach_price"]
-                        ._get_unit(production_breach_price),
-                        query_window=(start, end),
-                        resolution=resolution,
-                        beliefs_before=belief_time,
-                        fallback_attribute="production-breach-price",
-                        fill_sides=True,
+                    any_production_breach_price = (
+                        get_continuous_series_sensor_or_quantity(
+                            variable_quantity=production_breach_price,
+                            actuator=asset,
+                            unit=FlexContextSchema()
+                            .declared_fields["production_breach_price"]
+                            ._get_unit(production_breach_price),
+                            query_window=(start, end),
+                            resolution=resolution,
+                            beliefs_before=belief_time,
+                            fallback_attribute="production-breach-price",
+                            fill_sides=True,
+                        )
+                    )
+                    all_production_breach_price = (
+                        get_continuous_series_sensor_or_quantity(
+                            variable_quantity=production_breach_price,
+                            actuator=asset,
+                            unit=FlexContextSchema()
+                            .declared_fields["production_breach_price"]
+                            ._get_unit(production_breach_price)
+                            + "*h",
+                            query_window=(start, end),
+                            resolution=resolution,
+                            beliefs_before=belief_time,
+                            fallback_attribute="production-breach-price",
+                            fill_sides=True,
+                        )
                     )
                     # Set up commitments DataFrame
                     commitment = FlowCommitment(
-                        name=f"production breaches device {d}",
+                        name=f"any production breach device {d}",
+                        quantity=-production_capacity_d,
+                        # negative price because breaching in the downwards (production) direction is penalized
+                        downwards_deviation_price=-any_production_breach_price,
+                        index=index,
+                        _type="any",
+                        device=d,
+                    )
+                    commitments.append(commitment)
+
+                    commitment = FlowCommitment(
+                        name=f"all production breaches device {d}",
                         quantity=-production_capacity_d,
                         # negative price because breaching in the downwards (production) direction is penalized
-                        downwards_deviation_price=-production_breach_price,
+                        downwards_deviation_price=-all_production_breach_price,
                         index=index,
                         device=d,
                     )
@@ -700,23 +728,50 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     consumption_breach_price = self.flex_context[
                         "consumption_breach_price"
                     ]
-                    consumption_breach_price = get_continuous_series_sensor_or_quantity(
-                        variable_quantity=consumption_breach_price,
-                        actuator=asset,
-                        unit=FlexContextSchema()
-                        .declared_fields["consumption_breach_price"]
-                        ._get_unit(consumption_breach_price),
-                        query_window=(start, end),
-                        resolution=resolution,
-                        beliefs_before=belief_time,
-                        fallback_attribute="consumption-breach-price",
-                        fill_sides=True,
+                    any_consumption_breach_price = (
+                        get_continuous_series_sensor_or_quantity(
+                            variable_quantity=consumption_breach_price,
+                            actuator=asset,
+                            unit=FlexContextSchema()
+                            .declared_fields["consumption_breach_price"]
+                            ._get_unit(consumption_breach_price),
+                            query_window=(start, end),
+                            resolution=resolution,
+                            beliefs_before=belief_time,
+                            fallback_attribute="consumption-breach-price",
+                            fill_sides=True,
+                        )
+                    )
+                    all_consumption_breach_price = (
+                        get_continuous_series_sensor_or_quantity(
+                            variable_quantity=consumption_breach_price,
+                            actuator=asset,
+                            unit=FlexContextSchema()
+                            .declared_fields["consumption_breach_price"]
+                            ._get_unit(consumption_breach_price)
+                            + "*h",
+                            query_window=(start, end),
+                            resolution=resolution,
+                            beliefs_before=belief_time,
+                            fallback_attribute="consumption-breach-price",
+                            fill_sides=True,
+                        )
                     )
                     # Set up commitments DataFrame
                     commitment = FlowCommitment(
-                        name=f"consumption breaches device {d}",
+                        name=f"any consumption breach device {d}",
+                        quantity=consumption_capacity_d,
+                        upwards_deviation_price=any_consumption_breach_price,
+                        index=index,
+                        _type="any",
+                        device=d,
+                    )
+                    commitments.append(commitment)
+
+                    commitment = FlowCommitment(
+                        name=f"all consumption breaches device {d}",
                         quantity=consumption_capacity_d,
-                        upwards_deviation_price=consumption_breach_price,
+                        upwards_deviation_price=all_consumption_breach_price,
                         index=index,
                         device=d,
                     )
diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index bac98ec079..dd3c49747d 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -241,6 +241,11 @@ def test_battery_relaxation(add_battery_assets, db):
 
     # Check costs are correct
     np.testing.assert_almost_equal(
-        costs["consumption breaches device 0"],
-        device_power_breach_price * consumption_capacity_in_mw * 1000 * 4 * 4,
-    )  # 100 EUR/kWh/min * mean(0.15 MWh) * 1000 kW/MW * 4 hours * 4 15min/hour
+        costs["any consumption breach device 0"],
+        device_power_breach_price * consumption_capacity_in_mw * 1000,
+    )  # 100 EUR/kW * 0.025 MW * 1000 kW/MW
+
+    np.testing.assert_almost_equal(
+        costs["all consumption breaches device 0"],
+        device_power_breach_price * consumption_capacity_in_mw * 1000 * 4,
+    )  # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours

From c2740897fe2340bbf6cd87a487fe1d2a33394731 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 15:39:15 +0200
Subject: [PATCH 119/151] feat: apply breach prices to both SoC breach capacity
 and SoC breach effort

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst              |  3 +-
 flexmeasures/data/models/planning/storage.py  | 58 +++++++++++++++++--
 .../models/planning/tests/test_storage.py     |  4 +-
 .../data/schemas/scheduling/__init__.py       |  6 +-
 4 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index cdce919456..d287b9e646 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -22,7 +22,8 @@ v3.0-22 | 2025-03-17
 
   - ``soc-minima-breach-price``: if set, the ``soc-minima`` are used as a soft constraint.
   - ``soc-maxima-breach-price``: if set, the ``soc-maxima`` are used as a soft constraint.
-  - In both cases, not meeting the constraint is penalized according to this per-kWh-per-h price. That means the costs increase linearly the longer the state of charge breaches the constraint.
+  - In both cases, the price is applied both to the largest breach in the planning window (as a per-kWh price) and to each breach that occurs (as a per-kWh price per hour).
+    That means both high breaches and long breaches are penalized.
 
 v3.0-22 | 2024-12-27
 """"""""""""""""""""
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 87f32b4ee6..a921ad6e71 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -497,7 +497,19 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 and soc_minima[d] is not None
             ):
                 soc_minima_breach_price = self.flex_context["soc_minima_breach_price"]
-                soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
+                any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_minima_breach_price,
+                    actuator=asset,
+                    unit=FlexContextSchema()
+                    .declared_fields["soc_minima_breach_price"]
+                    ._get_unit(soc_minima_breach_price),
+                    query_window=(start + resolution, end + resolution),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    fallback_attribute="soc-minima-breach-price",
+                    fill_sides=True,
+                ).shift(-1, freq=resolution)
+                all_soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_minima_breach_price,
                     actuator=asset,
                     unit=FlexContextSchema()
@@ -530,10 +542,21 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 ) - soc_at_start[d] * (timedelta(hours=1) / resolution)
 
                 commitment = StockCommitment(
-                    name="soc minima",
+                    name="any soc minima",
+                    quantity=soc_minima_d,
+                    # negative price because breaching in the downwards (shortage) direction is penalized
+                    downwards_deviation_price=-any_soc_minima_breach_price,
+                    index=index,
+                    _type="any",
+                    device=d,
+                )
+                commitments.append(commitment)
+
+                commitment = StockCommitment(
+                    name="all soc minima",
                     quantity=soc_minima_d,
                     # negative price because breaching in the downwards (shortage) direction is penalized
-                    downwards_deviation_price=-soc_minima_breach_price,
+                    downwards_deviation_price=-all_soc_minima_breach_price,
                     index=index,
                     device=d,
                 )
@@ -558,7 +581,19 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 and soc_maxima[d] is not None
             ):
                 soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"]
-                soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
+                any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
+                    variable_quantity=soc_maxima_breach_price,
+                    actuator=asset,
+                    unit=FlexContextSchema()
+                    .declared_fields["soc_maxima_breach_price"]
+                    ._get_unit(soc_maxima_breach_price),
+                    query_window=(start + resolution, end + resolution),
+                    resolution=resolution,
+                    beliefs_before=belief_time,
+                    fallback_attribute="soc-maxima-breach-price",
+                    fill_sides=True,
+                ).shift(-1, freq=resolution)
+                all_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
                     variable_quantity=soc_maxima_breach_price,
                     actuator=asset,
                     unit=FlexContextSchema()
@@ -591,10 +626,21 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 ) - soc_at_start[d] * (timedelta(hours=1) / resolution)
 
                 commitment = StockCommitment(
-                    name="soc maxima",
+                    name="any soc maxima",
+                    quantity=soc_maxima_d,
+                    # positive price because breaching in the upwards (surplus) direction is penalized
+                    upwards_deviation_price=any_soc_maxima_breach_price,
+                    index=index,
+                    _type="any",
+                    device=d,
+                )
+                commitments.append(commitment)
+
+                commitment = StockCommitment(
+                    name="all soc maxima",
                     quantity=soc_maxima_d,
                     # positive price because breaching in the upwards (surplus) direction is penalized
-                    upwards_deviation_price=soc_maxima_breach_price,
+                    upwards_deviation_price=all_soc_maxima_breach_price,
                     index=index,
                     device=d,
                 )
diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index dd3c49747d..075c2f9964 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -78,7 +78,7 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
                 }
                 for i in production_prices.index
             ],
-            "soc-minima-breach-price": "100 EUR/kWh/min",  # high breach price (to mimic a hard constraint)
+            "soc-minima-breach-price": "6000 EUR/kWh",  # high breach price (to mimic a hard constraint)
         },
         return_multiple=True,
     )
@@ -213,7 +213,7 @@ def test_battery_relaxation(add_battery_assets, db):
                 }
                 for i in production_prices.index
             ],
-            "soc-minima-breach-price": "100 EUR/kWh/min",  # high breach price (to mimic a hard constraint)
+            "soc-minima-breach-price": "6000 EUR/kWh",  # high breach price (to mimic a hard constraint)
             "consumption-breach-price": f"{device_power_breach_price} EUR/kW",  # lower breach price (thus prioritizing minimizing soc breaches)
             "production-breach-price": f"{device_power_breach_price} EUR/kW",  # lower breach price (thus prioritizing minimizing soc breaches)
         },
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 4260446c3c..03ac9b3f7c 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -48,14 +48,14 @@ class FlexContextSchema(Schema):
         default=None,
     )
     soc_minima_breach_price = VariableQuantityField(
-        "/(MWh*h)",
+        "/MWh",
         data_key="soc-minima-breach-price",
         required=False,
         value_validator=validate.Range(min=0),
         default=None,
     )
     soc_maxima_breach_price = VariableQuantityField(
-        "/(MWh*h)",
+        "/MWh",
         data_key="soc-maxima-breach-price",
         required=False,
         value_validator=validate.Range(min=0),
@@ -162,7 +162,7 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
 
         This relies on the check_prices validator to run first.
         """
-        default_soc_breach_price = "1000 EUR/(kWh*h)"
+        default_soc_breach_price = "1000 EUR/kWh"
         if data["relax_soc_constraints"]:
             if data.get("soc_minima_breach_price") is None:
                 # use the same denominator as defined in the field

From b3dc2c951e44dde48b1a6abb56a1ea4854942202 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 15:49:07 +0200
Subject: [PATCH 120/151] docs: add hint at explanation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index a921ad6e71..b7b6e96051 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -515,7 +515,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     unit=FlexContextSchema()
                     .declared_fields["soc_minima_breach_price"]
                     ._get_unit(soc_minima_breach_price)
-                    + "*h",
+                    + "*h",  # from EUR/MWh² to EUR/MWh/resolution
                     query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
@@ -599,7 +599,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     unit=FlexContextSchema()
                     .declared_fields["soc_maxima_breach_price"]
                     ._get_unit(soc_maxima_breach_price)
-                    + "*h",
+                    + "*h",  # from EUR/MWh² to EUR/MWh/resolution
                     query_window=(start + resolution, end + resolution),
                     resolution=resolution,
                     beliefs_before=belief_time,
@@ -719,7 +719,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                             unit=FlexContextSchema()
                             .declared_fields["production_breach_price"]
                             ._get_unit(production_breach_price)
-                            + "*h",
+                            + "*h",  # from EUR/MWh to EUR/MW/resolution
                             query_window=(start, end),
                             resolution=resolution,
                             beliefs_before=belief_time,
@@ -795,7 +795,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                             unit=FlexContextSchema()
                             .declared_fields["consumption_breach_price"]
                             ._get_unit(consumption_breach_price)
-                            + "*h",
+                            + "*h",  # from EUR/MWh to EUR/MW/resolution
                             query_window=(start, end),
                             resolution=resolution,
                             beliefs_before=belief_time,

From bcfb357f29c259321ccf39e752ce0a479be9f640 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 15:51:40 +0200
Subject: [PATCH 121/151] feat: apply breach prices to both site breach
 capacity and site breach effort

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst             |  8 +++--
 flexmeasures/data/models/planning/storage.py | 36 ++++++++++++++++----
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index d287b9e646..703e56f241 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -35,8 +35,12 @@ v3.0-21 | 2024-12-16
 
 - Introduce new fields for defining capacity contracts and peak contracts in the ``flex-context``, used for scheduling against multiple contractual commitments simultaneously:
 
-  - ``site-consumption-breach-price``: if set, the ``site-consumption-capacity`` is used as a soft constraint, and breaching it is penalized according to this per-kW price. The price is applied both to the largest breach in the planning window and to each breach that occurs.
-  - ``site-production-breach-price``: if set, the ``site-production-capacity`` is used as a soft constraint, and breaching it is penalized according to this per-kW price. The price is applied both to the largest breach in the planning window and to each breach that occurs.
+  - ``site-consumption-breach-price``: if set, the ``site-consumption-capacity`` is used as a soft constraint.
+    The price is applied both to the largest breach in the planning window (as a per-kW price) and to each breach that occurs (as a per-kW price per hour).
+    That means both high breaches and long breaches are penalized.
+  - ``site-production-breach-price``: if set, the ``site-production-capacity`` is used as a soft constraint.
+    The price is applied both to the largest breach in the planning window (as a per-kW price) and to each breach that occurs (as a per-kW price per hour).
+    That means both high breaches and long breaches are penalized.
   - ``site-peak-consumption-price``: consumption peaks above the ``site-peak-consumption`` are penalized against this per-kW price.
   - ``site-peak-production-price``: production peaks above the ``site-peak-production`` are penalized against this per-kW price.
   - ``site-peak-consumption``: current peak consumption; costs from peaks below it are considered sunk costs.
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index b7b6e96051..ff082ae856 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -347,7 +347,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         if ems_consumption_breach_price is not None:
 
             # Convert to Series
-            ems_consumption_breach_price = get_continuous_series_sensor_or_quantity(
+            any_ems_consumption_breach_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_consumption_breach_price,
                 actuator=asset,
                 unit=FlexContextSchema()
@@ -358,13 +358,25 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 beliefs_before=belief_time,
                 fill_sides=True,
             )
+            all_ems_consumption_breach_price = get_continuous_series_sensor_or_quantity(
+                variable_quantity=ems_consumption_breach_price,
+                actuator=asset,
+                unit=FlexContextSchema()
+                .declared_fields["ems_consumption_breach_price"]
+                ._get_unit(ems_consumption_breach_price)
+                + "*h",  # from EUR/MWh to EUR/MW/resolution
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+                fill_sides=True,
+            )
 
             # Set up commitments DataFrame to penalize any breach
             commitment = FlowCommitment(
                 name="any consumption breach",
                 quantity=ems_consumption_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
-                upwards_deviation_price=ems_consumption_breach_price,
+                upwards_deviation_price=any_ems_consumption_breach_price,
                 _type="any",
                 index=index,
             )
@@ -375,7 +387,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 name="all consumption breaches",
                 quantity=ems_consumption_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
-                upwards_deviation_price=ems_consumption_breach_price,
+                upwards_deviation_price=all_ems_consumption_breach_price,
                 index=index,
             )
             commitments.append(commitment)
@@ -389,7 +401,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         if ems_production_breach_price is not None:
 
             # Convert to Series
-            ems_production_breach_price = get_continuous_series_sensor_or_quantity(
+            any_ems_production_breach_price = get_continuous_series_sensor_or_quantity(
                 variable_quantity=ems_production_breach_price,
                 actuator=asset,
                 unit=FlexContextSchema()
@@ -400,13 +412,25 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 beliefs_before=belief_time,
                 fill_sides=True,
             )
+            all_ems_production_breach_price = get_continuous_series_sensor_or_quantity(
+                variable_quantity=ems_production_breach_price,
+                actuator=asset,
+                unit=FlexContextSchema()
+                .declared_fields["ems_production_breach_price"]
+                ._get_unit(ems_production_breach_price)
+                + "*h",  # from EUR/MWh to EUR/MW/resolution
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+                fill_sides=True,
+            )
 
             # Set up commitments DataFrame to penalize any breach
             commitment = FlowCommitment(
                 name="any production breach",
                 quantity=ems_production_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
-                downwards_deviation_price=-ems_production_breach_price,
+                downwards_deviation_price=-any_ems_production_breach_price,
                 _type="any",
                 index=index,
             )
@@ -417,7 +441,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 name="all production breaches",
                 quantity=ems_production_capacity,
                 # positive price because breaching in the upwards (consumption) direction is penalized
-                downwards_deviation_price=-ems_production_breach_price,
+                downwards_deviation_price=-all_ems_production_breach_price,
                 index=index,
             )
             commitments.append(commitment)

From 12e5cd63c12a5a1f647fd0f8ead479c8da5c6c60 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 16:30:16 +0200
Subject: [PATCH 122/151] docs: update explanation of breach prices

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/api/change_log.rst      |  8 ++++----
 documentation/features/scheduling.rst | 19 ++++++++++---------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
index 703e56f241..184a3a9795 100644
--- a/documentation/api/change_log.rst
+++ b/documentation/api/change_log.rst
@@ -12,7 +12,7 @@ v3.0-23 | 2025-04-08
 
   - ``consumption-breach-price``: if set, the ``consumption-capacity`` is used as a soft constraint.
   - ``production-breach-price``: if set, the ``production-capacity`` is used as a soft constraint.
-  - In both cases, the price is applied both to the largest breach in the planning window (as a per-kW price) and to each breach that occurs (as a per-kW price per hour).
+  - In both cases, the price is applied both to (the height of) the highest breach in the planning window (as a per-kW price) and to (the area of) each breach that occurs (as a per-kW price per hour).
     That means both high breaches and long breaches are penalized.
 
 v3.0-22 | 2025-03-17
@@ -22,7 +22,7 @@ v3.0-22 | 2025-03-17
 
   - ``soc-minima-breach-price``: if set, the ``soc-minima`` are used as a soft constraint.
   - ``soc-maxima-breach-price``: if set, the ``soc-maxima`` are used as a soft constraint.
-  - In both cases, the price is applied both to the largest breach in the planning window (as a per-kWh price) and to each breach that occurs (as a per-kWh price per hour).
+  - In both cases, the price is applied both to (the height of) the highest breach in the planning window (as a per-kWh price) and to (the area of) each breach that occurs (as a per-kWh price per hour).
     That means both high breaches and long breaches are penalized.
 
 v3.0-22 | 2024-12-27
@@ -36,10 +36,10 @@ v3.0-21 | 2024-12-16
 - Introduce new fields for defining capacity contracts and peak contracts in the ``flex-context``, used for scheduling against multiple contractual commitments simultaneously:
 
   - ``site-consumption-breach-price``: if set, the ``site-consumption-capacity`` is used as a soft constraint.
-    The price is applied both to the largest breach in the planning window (as a per-kW price) and to each breach that occurs (as a per-kW price per hour).
+    The price is applied both to (the height of) the highest breach in the planning window (as a per-kW price) and to (the area of) each breach that occurs (as a per-kW price per hour).
     That means both high breaches and long breaches are penalized.
   - ``site-production-breach-price``: if set, the ``site-production-capacity`` is used as a soft constraint.
-    The price is applied both to the largest breach in the planning window (as a per-kW price) and to each breach that occurs (as a per-kW price per hour).
+    The price is applied both to (the height of) the highest breach in the planning window (as a per-kW price) and to (the area of) each breach that occurs (as a per-kW price per hour).
     That means both high breaches and long breaches are penalized.
   - ``site-peak-consumption-price``: consumption peaks above the ``site-peak-consumption`` are penalized against this per-kW price.
   - ``site-peak-production-price``: production peaks above the ``site-peak-production`` are penalized against this per-kW price.
diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index 705734aaf0..289959c9ec 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -84,8 +84,7 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
    * - ``site-consumption-breach-price``
      - ``"1000 EUR/kW"``
      - The price of breaching the ``site-consumption-capacity``, useful to treat ``site-consumption-capacity`` as a soft constraint but still make the scheduler attempt to respect it.
-       Can be (a sensor recording) contractual penalties, but also a theoretical penalty just to allow the scheduler to breach the consumption capacity, while influencing how badly breaches should be avoided.
-       The price is applied both to the largest breach in the planning window and to each breach that occurs. [#penalty_field]_
+       Can be (a sensor recording) contractual penalties, but also a theoretical penalty just to allow the scheduler to breach the consumption capacity, while influencing how badly breaches should be avoided. [#penalty_field]_ [#breach_field]_
    * - ``site-production-capacity``
      - ``"0kW"``
      - Maximum production power at the grid connection point.
@@ -95,8 +94,7 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
    * - ``site-production-breach-price``
      - ``"1000 EUR/kW"``
      - The price of breaching the ``site-production-capacity``, useful to treat ``site-production-capacity`` as a soft constraint but still make the scheduler attempt to respect it.
-       Can be (a sensor recording) contractual penalties, but also a theoretical penalty just to allow the scheduler to breach the production capacity, while influencing how badly breaches should be avoided.
-       The price is applied both to the largest breach in the planning window and to each breach that occurs. [#penalty_field]_
+       Can be (a sensor recording) contractual penalties, but also a theoretical penalty just to allow the scheduler to breach the production capacity, while influencing how badly breaches should be avoided. [#penalty_field]_ [#breach_field]_
    * - ``site-peak-consumption``
      - ``{"sensor": 7}``
      - Current peak consumption.
@@ -112,11 +110,11 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
      - ``"260 EUR/MWh"``
      - Production peaks above the ``site-peak-production`` are penalized against this per-kW price. [#penalty_field]_
    * - ``soc-minima-breach-price``
-     - ``"2 EUR/kWh/min"``
-     - Penalty for not meeting ``soc-minima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
+     - ``"120 EUR/kWh"``
+     - Penalty for not meeting ``soc-minima`` defined in the flex-model. [#penalty_field]_ [#breach_field]_
    * - ``soc-maxima-breach-price``
-     - ``"2 EUR/kWh/min"``
-     - Penalty for not meeting ``soc-maxima`` defined in the flex-model. [#penalty_field]_ [#soc_breach_prices]_
+     - ``"120 EUR/kWh"``
+     - Penalty for not meeting ``soc-maxima`` defined in the flex-model. [#penalty_field]_ [#breach_field]_
 
 .. [#old_sensor_field] The old field only accepted an integer (sensor ID).
 
@@ -130,7 +128,10 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
 
 .. [#production] Example: with a connection capacity (``site-power-capacity``) of 1 MVA (apparent power) and a production capacity (``site-production-capacity``) of 400 kW (active power), the scheduler will make sure that the grid inflow doesn't exceed 400 kW.
 
-.. [#soc_breach_prices] The SoC breach prices (e.g. 2 EUR/kWh/min) to use for the schedule are applied over each time step equal to the sensor resolution. For example, a SoC breach price of 2 EUR/kWh/min, for scheduling a 5-minute resolution sensor, will be applied as a SoC breach price of 10 EUR/kWh for breaches measured every 5 minutes.
+.. [#breach_field] Breach prices are applied both to (the height of) the highest breach in the planning window and to (the area of) each breach that occurs.
+                   That means both high breaches and long breaches are penalized.
+                   For example, a :abbr:`SoC (state of charge)` breach price of 120 EUR/kWh is applied as a breach price of 120 EUR/kWh on the height of the highest breach, and as a breach price of 120 EUR/kWh/h on the area (kWh*h) of each breach.
+                   For a 5-minute resolution sensor, this would amount to applying a SoC breach price of 10 EUR/kWh for breaches measured every 5 minutes (in addition to the 120 EUR/kWh applied to the highest breach only).
 
 .. note:: If no (symmetric, consumption and production) site capacity is defined (also not as defaults), the scheduler will not enforce any bound on the site power.
           The flexible device can still have its own power limit defined in its flex-model.

From 0d1e032ac26dda71c4aab4d31a11bb24688a3778 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 16:35:35 +0200
Subject: [PATCH 123/151] docs: add documentation for the two new breach-prices

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/features/scheduling.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst
index 289959c9ec..c0c78cffdc 100644
--- a/documentation/features/scheduling.rst
+++ b/documentation/features/scheduling.rst
@@ -115,6 +115,12 @@ And if the asset belongs to a larger system (a hierarchy of assets), the schedul
    * - ``soc-maxima-breach-price``
      - ``"120 EUR/kWh"``
      - Penalty for not meeting ``soc-maxima`` defined in the flex-model. [#penalty_field]_ [#breach_field]_
+   * - ``consumption-breach-price``
+     - ``"10 EUR/kW"``
+     - The price of breaching the ``consumption-capacity`` in the flex-model, useful to treat ``consumption-capacity`` as a soft constraint but still make the scheduler attempt to respect it. [#penalty_field]_ [#breach_field]_
+   * - ``production-breach-price``
+     - ``"10 EUR/kW"``
+     - The price of breaching the ``production-capacity`` in the flex-model, useful to treat ``production-capacity`` as a soft constraint but still make the scheduler attempt to respect it. [#penalty_field]_ [#breach_field]_
 
 .. [#old_sensor_field] The old field only accepted an integer (sensor ID).
 

From 2a14a92be7ee8874a5f789b9ac4eefba14042cc9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 16:38:27 +0200
Subject: [PATCH 124/151] fix: check on costs of "all consumption breaches"

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_storage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index 075c2f9964..747f0b3841 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -100,9 +100,9 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
     np.testing.assert_almost_equal(costs["energy"], 100 * (1 - 0.4))
     # 24000 EUR for any 24 kW consumption breach priced at 1000 EUR/kW
     np.testing.assert_almost_equal(costs["any consumption breach"], 1000 * (25 - 1))
-    # 24000 EUR for each 24 kW consumption breach per 15 minutes priced at 1000 EUR/kW
+    # 24000 EUR for each 24 kW consumption breach per hour priced at 1000 EUR/kWh
     np.testing.assert_almost_equal(
-        costs["all consumption breaches"], 1000 * (25 - 1) * 96
+        costs["all consumption breaches"], 1000 * (25 - 1) * 96 / 4
     )
     # No production breaches
     np.testing.assert_almost_equal(costs["any production breach"], 0)

From b04ae93dc4c8e09dcde232afa89ead1b2cc81689 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Apr 2025 16:42:58 +0200
Subject: [PATCH 125/151] fix: move changelog item for SoC relaxation to the
 correct release version

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/changelog.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index e52b210b14..8fa9e04973 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -10,7 +10,8 @@ v0.26.0 | May XX, 2025
 New features
 -------------
 
-* Support saving the storage schedule SOC using the ``flex-model`` field ``state-of-charge`` to the ``flex-model`` [see `PR #1392 <https://github.com/FlexMeasures/flexmeasures/pull/1392>`_ and `PR #1406 <https://github.com/FlexMeasures/flexmeasures/pull/1406>`_]
+* Support saving the storage schedule :abbr:`SoC (state of charge)` using the ``flex-model`` field ``state-of-charge`` to the ``flex-model`` [see `PR #1392 <https://github.com/FlexMeasures/flexmeasures/pull/1392>`_ and `PR #1406 <https://github.com/FlexMeasures/flexmeasures/pull/1406>`_]
+* Allow relaxing :abbr:`SoC (state of charge)` minima and maxima, by setting penalties for not meeting these constraints, using two new ``flex-context`` fields [see `PR #1300 <https://github.com/FlexMeasures/flexmeasures/pull/1300>`_]
 * Allow relaxing device-level power constraints, by setting penalties for not meeting these constraints, using two new ``flex-context`` fields [see `PR #1405 <https://github.com/FlexMeasures/flexmeasures/pull/1405>`_]
 * Save changes in asset flex-context form right away [see `PR #1390 <https://github.com/FlexMeasures/flexmeasures/pull/1390>`_]
 * Extending sensor CRUD functionality to the UI [see `PR #1394 <https://github.com/FlexMeasures/flexmeasures/pull/1394>`_ and `PR #1413 <https://github.com/FlexMeasures/flexmeasures/pull/1413>`_]
@@ -35,7 +36,6 @@ v0.25.0 | April 01, 2025
 
 New features
 -------------
-* Allow relaxing SoC minima and maxima, by setting penalties for not meeting these constraints, using two new ``flex-context`` fields [see `PR #1300 <https://github.com/FlexMeasures/flexmeasures/pull/1300>`_]
 * Added form modal to edit an asset's ``flex_context`` [see `PR #1320 <https://github.com/FlexMeasures/flexmeasures/pull/1320>`_, `PR #1365 <https://github.com/FlexMeasures/flexmeasures/pull/1365>`_ and `PR #1364 <https://github.com/FlexMeasures/flexmeasures/pull/1364>`_]
 * Improve asset status page - distinguish better by data source type [see `PR #1022 <https://github.com/FlexMeasures/flexmeasures/pull/1022/>`_]
 * Better y-axis titles for charts that show multiple sensors with a shared unit [see `PR #1346 <https://github.com/FlexMeasures/flexmeasures/pull/1346>`_]
@@ -405,7 +405,7 @@ v0.16.1 | October 2, 2023
 
 Bugfixes
 -----------
-* Fix infeasible problem due to incorrect parsing of soc units of the ``soc-minima`` and ``soc-maxima`` fields within the ``flex-model`` field [see `PR #864 <https://github.com/FlexMeasures/flexmeasures/pull/864>`_]
+* Fix infeasible problem due to incorrect parsing of :abbr:`SoC (state of charge)` units of the ``soc-minima`` and ``soc-maxima`` fields within the ``flex-model`` field [see `PR #864 <https://github.com/FlexMeasures/flexmeasures/pull/864>`_]
 
 
 v0.16.0 | September 27, 2023
@@ -439,7 +439,7 @@ v0.15.2 | October 2, 2023
 
 Bugfixes
 -----------
-* Fix infeasible problem due to incorrect parsing of soc units of the ``soc-minima`` and ``soc-maxima`` fields within the ``flex-model`` field [see `PR #864 <https://github.com/FlexMeasures/flexmeasures/pull/864>`_]
+* Fix infeasible problem due to incorrect parsing of :abbr:`SoC (state of charge)` units of the ``soc-minima`` and ``soc-maxima`` fields within the ``flex-model`` field [see `PR #864 <https://github.com/FlexMeasures/flexmeasures/pull/864>`_]
 
 
 v0.15.1 | August 28, 2023
@@ -501,7 +501,7 @@ v0.14.3 | October 2, 2023
 
 Bugfixes
 -----------
-* Fix infeasible problem due to incorrect parsing of soc units of the ``soc-minima`` and ``soc-maxima`` fields within the ``flex-model`` field [see `PR #864 <https://github.com/FlexMeasures/flexmeasures/pull/864>`_]
+* Fix infeasible problem due to incorrect parsing of :abbr:`SoC (state of charge)` units of the ``soc-minima`` and ``soc-maxima`` fields within the ``flex-model`` field [see `PR #864 <https://github.com/FlexMeasures/flexmeasures/pull/864>`_]
 
 
 v0.14.2 | July 25, 2023

From 0369bec37b47c50a53b3b307c3198f1f0bae238c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 18 Apr 2025 15:09:16 +0200
Subject: [PATCH 126/151] docs: fix sign explanation in comments

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index ff082ae856..3fab171dec 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -429,7 +429,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             commitment = FlowCommitment(
                 name="any production breach",
                 quantity=ems_production_capacity,
-                # positive price because breaching in the upwards (consumption) direction is penalized
+                # negative price because breaching in the downwards (production) direction is penalized
                 downwards_deviation_price=-any_ems_production_breach_price,
                 _type="any",
                 index=index,
@@ -440,7 +440,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             commitment = FlowCommitment(
                 name="all production breaches",
                 quantity=ems_production_capacity,
-                # positive price because breaching in the upwards (consumption) direction is penalized
+                # negative price because breaching in the downwards (production) direction is penalized
                 downwards_deviation_price=-all_ems_production_breach_price,
                 index=index,
             )

From 1275fdb2c590256421377d5e28bec72d0a5834cb Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 18 Apr 2025 15:12:25 +0200
Subject: [PATCH 127/151] refactor: simplify default price declarations

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py       | 20 ++++++-------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 03ac9b3f7c..b455627c80 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -162,13 +162,11 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
 
         This relies on the check_prices validator to run first.
         """
-        default_soc_breach_price = "1000 EUR/kWh"
+        default_soc_breach_price = ur.Quantity("1000 EUR/kWh")
         if data["relax_soc_constraints"]:
             if data.get("soc_minima_breach_price") is None:
                 # use the same denominator as defined in the field
-                data["soc_minima_breach_price"] = ur.Quantity(
-                    default_soc_breach_price
-                ).to(
+                data["soc_minima_breach_price"] = default_soc_breach_price.to(
                     data["shared_currency_unit"]
                     + "/"
                     + self.declared_fields["soc_minima_breach_price"].to_unit.split(
@@ -177,9 +175,7 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
                 )
             if data.get("soc_maxima_breach_price") is None:
                 # use the same denominator as defined in the field
-                data["soc_maxima_breach_price"] = ur.Quantity(
-                    default_soc_breach_price
-                ).to(
+                data["soc_maxima_breach_price"] = default_soc_breach_price.to(
                     data["shared_currency_unit"]
                     + "/"
                     + self.declared_fields["soc_maxima_breach_price"].to_unit.split(
@@ -193,13 +189,11 @@ def process_relax_capacity_constraints(self, data: dict, **kwargs):
 
         This relies on the check_prices validator to run first.
         """
-        default_capacity_breach_price = "100 EUR/kW"
+        default_capacity_breach_price = ur.Quantity("100 EUR/kW")
         if data["relax_capacity_constraints"]:
             if data.get("consumption_breach_price") is None:
                 # use the same denominator as defined in the field
-                data["consumption_breach_price"] = ur.Quantity(
-                    default_capacity_breach_price
-                ).to(
+                data["consumption_breach_price"] = default_capacity_breach_price.to(
                     data["shared_currency_unit"]
                     + "/"
                     + self.declared_fields["consumption_breach_price"].to_unit.split(
@@ -208,9 +202,7 @@ def process_relax_capacity_constraints(self, data: dict, **kwargs):
                 )
             if data.get("production_breach_price") is None:
                 # use the same denominator as defined in the field
-                data["production_breach_price"] = ur.Quantity(
-                    default_capacity_breach_price
-                ).to(
+                data["production_breach_price"] = default_capacity_breach_price.to(
                     data["shared_currency_unit"]
                     + "/"
                     + self.declared_fields["production_breach_price"].to_unit.split(

From 23e25b42a54af1b8d796efd2af235f4cc2ca711f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 18 Apr 2025 15:17:03 +0200
Subject: [PATCH 128/151] refactor: merge logic using for-loop

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py       | 52 ++++++-------------
 1 file changed, 16 insertions(+), 36 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index b455627c80..31263e097d 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -164,24 +164,14 @@ def process_relax_soc_constraints(self, data: dict, **kwargs):
         """
         default_soc_breach_price = ur.Quantity("1000 EUR/kWh")
         if data["relax_soc_constraints"]:
-            if data.get("soc_minima_breach_price") is None:
-                # use the same denominator as defined in the field
-                data["soc_minima_breach_price"] = default_soc_breach_price.to(
-                    data["shared_currency_unit"]
-                    + "/"
-                    + self.declared_fields["soc_minima_breach_price"].to_unit.split(
-                        "/"
-                    )[-1]
-                )
-            if data.get("soc_maxima_breach_price") is None:
-                # use the same denominator as defined in the field
-                data["soc_maxima_breach_price"] = default_soc_breach_price.to(
-                    data["shared_currency_unit"]
-                    + "/"
-                    + self.declared_fields["soc_maxima_breach_price"].to_unit.split(
-                        "/"
-                    )[-1]
-                )
+            for field in ("soc_minima_breach_price", "soc_maxima_breach_price"):
+                if data.get(field) is None:
+                    # use the same denominator as defined in the field
+                    data[field] = default_soc_breach_price.to(
+                        data["shared_currency_unit"]
+                        + "/"
+                        + self.declared_fields[field].to_unit.split("/")[-1]
+                    )
         return data
 
     def process_relax_capacity_constraints(self, data: dict, **kwargs):
@@ -191,24 +181,14 @@ def process_relax_capacity_constraints(self, data: dict, **kwargs):
         """
         default_capacity_breach_price = ur.Quantity("100 EUR/kW")
         if data["relax_capacity_constraints"]:
-            if data.get("consumption_breach_price") is None:
-                # use the same denominator as defined in the field
-                data["consumption_breach_price"] = default_capacity_breach_price.to(
-                    data["shared_currency_unit"]
-                    + "/"
-                    + self.declared_fields["consumption_breach_price"].to_unit.split(
-                        "/"
-                    )[-1]
-                )
-            if data.get("production_breach_price") is None:
-                # use the same denominator as defined in the field
-                data["production_breach_price"] = default_capacity_breach_price.to(
-                    data["shared_currency_unit"]
-                    + "/"
-                    + self.declared_fields["production_breach_price"].to_unit.split(
-                        "/"
-                    )[-1]
-                )
+            for field in ("consumption_breach_price", "production_breach_price"):
+                if data.get(field) is None:
+                    # use the same denominator as defined in the field
+                    data[field] = default_capacity_breach_price.to(
+                        data["shared_currency_unit"]
+                        + "/"
+                        + self.declared_fields[field].to_unit.split("/")[-1]
+                    )
         return data
 
     @validates_schema

From 7faf8a0fbad85fc228570f2cdd9527d3adbf508f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 18 Apr 2025 15:30:08 +0200
Subject: [PATCH 129/151] refactor: move setting default prices into a single
 function

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py       | 60 +++++++++----------
 1 file changed, 27 insertions(+), 33 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 31263e097d..4dc080a958 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -157,38 +157,20 @@ class FlexContextSchema(Schema):
         SensorIdField(), data_key="inflexible-device-sensors"
     )
 
-    def process_relax_soc_constraints(self, data: dict, **kwargs):
-        """Fill in default soc breach prices when asked to relax SoC constraints.
+    def set_default_breach_prices(
+        self, data: dict, fields: list[str], price: ur.Quantity
+    ):
+        """Fill in default breach prices.
 
-        This relies on the check_prices validator to run first.
+        This relies on _try_to_convert_price_units to run first, setting a shared currency unit.
         """
-        default_soc_breach_price = ur.Quantity("1000 EUR/kWh")
-        if data["relax_soc_constraints"]:
-            for field in ("soc_minima_breach_price", "soc_maxima_breach_price"):
-                if data.get(field) is None:
-                    # use the same denominator as defined in the field
-                    data[field] = default_soc_breach_price.to(
-                        data["shared_currency_unit"]
-                        + "/"
-                        + self.declared_fields[field].to_unit.split("/")[-1]
-                    )
-        return data
-
-    def process_relax_capacity_constraints(self, data: dict, **kwargs):
-        """Fill in default capacity breach prices when asked to relax capacity constraints.
-
-        This relies on the check_prices validator to run first.
-        """
-        default_capacity_breach_price = ur.Quantity("100 EUR/kW")
-        if data["relax_capacity_constraints"]:
-            for field in ("consumption_breach_price", "production_breach_price"):
-                if data.get(field) is None:
-                    # use the same denominator as defined in the field
-                    data[field] = default_capacity_breach_price.to(
-                        data["shared_currency_unit"]
-                        + "/"
-                        + self.declared_fields[field].to_unit.split("/")[-1]
-                    )
+        for field in fields:
+            # use the same denominator as defined in the field
+            data[field] = price.to(
+                data["shared_currency_unit"]
+                + "/"
+                + self.declared_fields[field].to_unit.split("/")[-1]
+            )
         return data
 
     @validates_schema
@@ -239,9 +221,21 @@ def check_prices(self, data: dict, **kwargs):
         # All prices must share the same unit
         data = self._try_to_convert_price_units(data)
 
-        # Call schema validators that must come after this one, because they rely on data["shared_currency_unit"]
-        self.process_relax_soc_constraints(data)
-        self.process_relax_capacity_constraints(data)
+        # Fill in default soc breach prices when asked to relax SoC constraints.
+        if data["relax_soc_constraints"]:
+            self.set_default_breach_prices(
+                data,
+                fields=["soc_minima_breach_price", "soc_maxima_breach_price"],
+                price=ur.Quantity("1000 EUR/kWh"),
+            )
+
+        # Fill in default capacity breach prices when asked to relax capacity constraints.
+        if data["relax_capacity_constraints"]:
+            self.set_default_breach_prices(
+                data,
+                fields=["consumption_breach_price", "production_breach_price"],
+                price=ur.Quantity("100 EUR/kW"),
+            )
 
         return data
 

From 8aee391cb7166f47cfe29d9add1283094fd2f363 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 18 Apr 2025 15:44:48 +0200
Subject: [PATCH 130/151] refactor: series_to_ts_specs util function

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../models/planning/tests/test_storage.py     | 79 ++++---------------
 .../data/models/planning/tests/utils.py       | 11 +++
 2 files changed, 25 insertions(+), 65 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index 747f0b3841..d970e2fc4f 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -11,6 +11,7 @@
 from flexmeasures.data.models.planning.tests.utils import (
     check_constraints,
     get_sensors_from_db,
+    series_to_ts_specs,
 )
 
 
@@ -45,22 +46,8 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
             "prefer-charging-sooner": False,
         },
         flex_context={
-            "consumption-price": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": f"{consumption_prices[i]} EUR/MWh",
-                }
-                for i in consumption_prices.index
-            ],
-            "production-price": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": f"{production_prices[i]} EUR/MWh",
-                }
-                for i in production_prices.index
-            ],
+            "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"),
+            "production-price": series_to_ts_specs(production_prices, unit="EUR/MWh"),
             "site-power-capacity": "2 MW",  # should be big enough to avoid any infeasibilities
             "site-consumption-capacity": "1 kW",  # we'll need to breach this to reach the target
             "site-consumption-breach-price": "1000 EUR/kW",
@@ -70,14 +57,9 @@ def test_battery_solver_multi_commitment(add_battery_assets, db):
             "site-peak-consumption-price": "260 EUR/MW",
             # The following is a constant price, but this checks currency conversion in case a later price field is
             # set to a time series specs (i.e. a list of dicts, where each dict represents a time slot)
-            "site-peak-production-price": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": "260 EUR/MW",
-                }
-                for i in production_prices.index
-            ],
+            "site-peak-production-price": series_to_ts_specs(
+                pd.Series(260, production_prices.index), unit="EUR/MW"
+            ),
             "soc-minima-breach-price": "6000 EUR/kWh",  # high breach price (to mimic a hard constraint)
         },
         return_multiple=True,
@@ -141,7 +123,7 @@ def test_battery_relaxation(add_battery_assets, db):
     consumption_capacity["2015-01-01T12:00:00+01:00":"2015-01-01T18:00:00+01:00"] = (
         0  # no charging
     )
-    production_capacity_in_mw = consumption_capacity
+    production_capacity = consumption_capacity
 
     scheduler: Scheduler = StorageScheduler(
         battery,
@@ -153,22 +135,8 @@ def test_battery_relaxation(add_battery_assets, db):
             "soc-min": "0 MWh",
             "soc-max": "1 MWh",
             "power-capacity": f"{consumption_capacity_in_mw} MVA",
-            "consumption-capacity": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": f"{consumption_capacity[i]} MW",
-                }
-                for i in consumption_capacity.index
-            ],
-            "production-capacity": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": f"{production_capacity_in_mw[i]} MW",
-                }
-                for i in production_capacity_in_mw.index
-            ],
+            "consumption-capacity": series_to_ts_specs(consumption_capacity, unit="MW"),
+            "production-capacity": series_to_ts_specs(production_capacity, unit="MW"),
             "soc-minima": [
                 {
                     "start": "2015-01-01T12:00:00+01:00",
@@ -180,22 +148,8 @@ def test_battery_relaxation(add_battery_assets, db):
             "prefer-charging-sooner": False,
         },
         flex_context={
-            "consumption-price": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": f"{consumption_prices[i]} EUR/MWh",
-                }
-                for i in consumption_prices.index
-            ],
-            "production-price": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": f"{production_prices[i]} EUR/MWh",
-                }
-                for i in production_prices.index
-            ],
+            "consumption-price": series_to_ts_specs(consumption_prices, unit="EUR/MWh"),
+            "production-price": series_to_ts_specs(production_prices, unit="EUR/MWh"),
             "site-power-capacity": "2 MW",  # should be big enough to avoid any infeasibilities
             # "site-consumption-capacity": "1 kW",  # we'll need to breach this to reach the target
             "site-consumption-breach-price": "1000 EUR/kW",
@@ -205,14 +159,9 @@ def test_battery_relaxation(add_battery_assets, db):
             "site-peak-consumption-price": "260 EUR/MW",
             # The following is a constant price, but this checks currency conversion in case a later price field is
             # set to a time series specs (i.e. a list of dicts, where each dict represents a time slot)
-            "site-peak-production-price": [
-                {
-                    "start": i.isoformat(),
-                    "duration": "PT1H",
-                    "value": "260 EUR/MW",
-                }
-                for i in production_prices.index
-            ],
+            "site-peak-production-price": series_to_ts_specs(
+                pd.Series(260, production_prices.index), unit="EUR/MW"
+            ),
             "soc-minima-breach-price": "6000 EUR/kWh",  # high breach price (to mimic a hard constraint)
             "consumption-breach-price": f"{device_power_breach_price} EUR/kW",  # lower breach price (thus prioritizing minimizing soc breaches)
             "production-breach-price": f"{device_power_breach_price} EUR/kW",  # lower breach price (thus prioritizing minimizing soc breaches)
diff --git a/flexmeasures/data/models/planning/tests/utils.py b/flexmeasures/data/models/planning/tests/utils.py
index 9c270dd9ec..33086ef48f 100644
--- a/flexmeasures/data/models/planning/tests/utils.py
+++ b/flexmeasures/data/models/planning/tests/utils.py
@@ -6,6 +6,17 @@
 from flexmeasures.utils.unit_utils import ur
 
 
+def series_to_ts_specs(s: pd.Series, unit: str) -> list[dict]:
+    return [
+        {
+            "start": i.isoformat(),
+            "duration": pd.to_timedelta(s.index.freq).isoformat(),
+            "value": f"{s[i]} {unit}",
+        }
+        for i in s.index
+    ]
+
+
 def check_constraints(
     sensor: Sensor,
     schedule: pd.Series,

From 1d25d2e19e9bae4796e729aaef691a758679c0e6 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 18 Apr 2025 15:45:38 +0200
Subject: [PATCH 131/151] docs: add clarifying note to docstring of new util
 function

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/models/planning/tests/utils.py b/flexmeasures/data/models/planning/tests/utils.py
index 33086ef48f..cd6f6c68aa 100644
--- a/flexmeasures/data/models/planning/tests/utils.py
+++ b/flexmeasures/data/models/planning/tests/utils.py
@@ -7,6 +7,7 @@
 
 
 def series_to_ts_specs(s: pd.Series, unit: str) -> list[dict]:
+    """Assumes the series frequency should be used as the event resolution."""
     return [
         {
             "start": i.isoformat(),

From e9cf3d0860602570b9155f46579564f57a40b64e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 22:32:27 +0200
Subject: [PATCH 132/151] refactor: prepare to use new
 FlexContextTimeSeriesSchema for loading time series data

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py |  2 +
 flexmeasures/data/models/planning/storage.py  | 13 +++++
 .../data/schemas/scheduling/__init__.py       | 56 ++++++++++++++++++-
 flexmeasures/data/services/scheduling.py      |  4 ++
 4 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index e5336b00fa..bdaec4b83c 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -74,6 +74,7 @@ def __init__(
         flex_model: dict | None = None,
         flex_context: dict | None = None,
         return_multiple: bool = False,
+        load_time_series: bool = True,
     ):
         """
         Initialize a new Scheduler.
@@ -126,6 +127,7 @@ def __init__(
             self.info = dict(scheduler=self.__class__.__name__)
 
         self.return_multiple = return_multiple
+        self.load_time_series = load_time_series
 
     def compute_schedule(self) -> pd.Series | None:
         """
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 3fab171dec..28a7ab5c81 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -32,6 +32,7 @@
 from flexmeasures.data.schemas.scheduling.storage import StorageFlexModelSchema
 from flexmeasures.data.schemas.scheduling import (
     FlexContextSchema,
+    FlexContextTimeSeriesSchema,
     MultiSensorFlexModelSchema,
 )
 from flexmeasures.utils.calculations import (
@@ -986,6 +987,14 @@ def persist_flex_model(self):
                 "soc_in_mwh", self.flex_model["soc_at_start"]
             )
 
+    def _load_time_series(self, asset):
+        self.flex_context = FlexContextTimeSeriesSchema(
+            asset=asset,
+            query_window=(self.start, self.end),
+            resolution=self.resolution,
+            belief_time=self.belief_time,
+        ).load(self.flex_context)
+
     def deserialize_flex_config(self):
         """
         Deserialize storage flex model and the flex context against schemas.
@@ -1010,6 +1019,10 @@ def deserialize_flex_config(self):
             {**db_flex_context, **self.flex_context}
         )
 
+        # Load time series from flex-context
+        if self.load_time_series:
+            self._load_time_series(asset)
+
         if isinstance(self.flex_model, dict):
             # Check state of charge.
             # Preferably, a starting soc is given.
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 4dc080a958..3b24d07e56 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from datetime import datetime, timedelta
+
 from marshmallow import (
     Schema,
     fields,
@@ -10,7 +12,7 @@
     post_dump,
 )
 
-from flexmeasures import Sensor
+from flexmeasures import Asset, Sensor
 from flexmeasures.data.schemas.generic_assets import GenericAssetIdField
 from flexmeasures.data.schemas.sensors import (
     VariableQuantityField,
@@ -32,6 +34,12 @@
 class FlexContextSchema(Schema):
     """This schema defines fields that provide context to the portfolio to be optimized."""
 
+    shared_currency_unit = fields.String(
+        data_key="shared-currency-unit",
+        required=False,
+        load_default="EUR",
+    )
+
     # Device commitments
     consumption_breach_price = VariableQuantityField(
         "/MW",
@@ -157,6 +165,25 @@ class FlexContextSchema(Schema):
         SensorIdField(), data_key="inflexible-device-sensors"
     )
 
+    def __init__(
+        self,
+        asset: Asset | None = None,
+        load_time_series: bool = False,
+        query_window: tuple[datetime, datetime] | None = None,
+        resolution: timedelta | None = None,
+        belief_time: datetime | None = None,
+        *args,
+        **kwargs,
+    ):
+        if load_time_series and asset is None:
+            raise NotImplementedError("Cannot load time series from an unknown asset.")
+        self.asset = asset
+        self.load_time_series = load_time_series
+        self.query_window = query_window
+        self.resolution = resolution
+        self.belief_time = belief_time
+        super().__init__(*args, **kwargs)
+
     def set_default_breach_prices(
         self, data: dict, fields: list[str], price: ur.Quantity
     ):
@@ -180,6 +207,8 @@ def check_prices(self, data: dict, **kwargs):
         1. The flex-context must contain at most 1 consumption price and at most 1 production price field.
         2. All prices must share the same currency.
         """
+        if self.load_time_series:
+            return data
 
         # The flex-context must contain at most 1 consumption price and at most 1 production price field
         if "consumption_price_sensor" in data and "consumption_price" in data:
@@ -284,7 +313,8 @@ def _try_to_convert_price_units(self, data):
                         f"Prices must share the same monetary unit. '{field_name}' uses '{currency_unit}', but '{previous_field_name}' used '{shared_currency_unit}'.",
                         field_name=field_name,
                     )
-        data["shared_currency_unit"] = shared_currency_unit
+        if shared_currency_unit is not None:
+            data["shared_currency_unit"] = shared_currency_unit
         return data
 
 
@@ -408,6 +438,28 @@ def _forbid_fixed_prices(self, data: dict, **kwargs):
             )
 
 
+def passthrough_deserializer():
+    return lambda value, attr, data, **kwargs: value
+
+
+class FlexContextTimeSeriesSchema(FlexContextSchema):
+    """Schema for loading time series data for each VariableQuantityField in an already deserialized flex-context."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.load_time_series = True
+        for field_var, field in self.declared_fields.items():
+            if isinstance(field, VariableQuantityField) and field_var in (
+                # "consumption_breach_price",
+            ):
+                field.load_time_series = True
+            else:
+                # Skip deserialization
+                field._deserialize = passthrough_deserializer()
+            field.data_key = field_var
+            setattr(self, field_var, field)
+
+
 class MultiSensorFlexModelSchema(Schema):
     """
 
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index fe607ed4c8..25d68cfc0a 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -165,6 +165,7 @@ def trigger_optional_fallback(job, connection, type, value, traceback):
 def create_scheduling_job(
     asset_or_sensor: Asset | Sensor | None = None,
     sensor: Sensor | None = None,
+    load_time_series: bool = False,
     job_id: str | None = None,
     enqueue: bool = True,
     requeue: bool = False,
@@ -189,6 +190,7 @@ def create_scheduling_job(
 
     Arguments:
     :param asset_or_sensor:         Asset or sensor for which the schedule is computed.
+    :param load_time_series:        If True, also loads data when the job gets created. todo: does this have a practical use case?
     :param job_id:                  Optionally, set a job id explicitly.
     :param enqueue:                 If True, enqueues the job in case it is new.
     :param requeue:                 If True, requeues the job in case it is not new and had previously failed
@@ -214,6 +216,7 @@ def create_scheduling_job(
     else:
         scheduler_class: Type[Scheduler] = find_scheduler_class(asset_or_sensor)
 
+    scheduler_kwargs["load_time_series"] = load_time_series
     scheduler = get_scheduler_instance(
         scheduler_class=scheduler_class,
         asset_or_sensor=asset_or_sensor,
@@ -519,6 +522,7 @@ def make_schedule(
     if belief_time is None:
         belief_time = server_now()
 
+    scheduler_kwargs["load_time_series"] = True
     scheduler_params = dict(
         start=start,
         end=end,

From ea79830a9ff6655b7e49f76e22c56932cb4a2ca7 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 23:01:46 +0200
Subject: [PATCH 133/151] refactor: move consumption_breach_price loading to
 schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 31 +---------
 .../data/schemas/scheduling/__init__.py       | 57 ++++++++++++++-----
 flexmeasures/data/schemas/sensors.py          | 54 ++++++++++++++----
 3 files changed, 89 insertions(+), 53 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 28a7ab5c81..cbed2c9070 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -799,35 +799,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     consumption_breach_price = self.flex_context[
                         "consumption_breach_price"
                     ]
-                    any_consumption_breach_price = (
-                        get_continuous_series_sensor_or_quantity(
-                            variable_quantity=consumption_breach_price,
-                            actuator=asset,
-                            unit=FlexContextSchema()
-                            .declared_fields["consumption_breach_price"]
-                            ._get_unit(consumption_breach_price),
-                            query_window=(start, end),
-                            resolution=resolution,
-                            beliefs_before=belief_time,
-                            fallback_attribute="consumption-breach-price",
-                            fill_sides=True,
-                        )
-                    )
+                    any_consumption_breach_price = consumption_breach_price
                     all_consumption_breach_price = (
-                        get_continuous_series_sensor_or_quantity(
-                            variable_quantity=consumption_breach_price,
-                            actuator=asset,
-                            unit=FlexContextSchema()
-                            .declared_fields["consumption_breach_price"]
-                            ._get_unit(consumption_breach_price)
-                            + "*h",  # from EUR/MWh to EUR/MW/resolution
-                            query_window=(start, end),
-                            resolution=resolution,
-                            beliefs_before=belief_time,
-                            fallback_attribute="consumption-breach-price",
-                            fill_sides=True,
-                        )
-                    )
+                        consumption_breach_price * resolution / timedelta(hours=1)
+                    )  # from EUR/MWh to EUR/MW/resolution
                     # Set up commitments DataFrame
                     commitment = FlowCommitment(
                         name=f"any consumption breach device {d}",
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 3b24d07e56..9dc3cc39dc 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -11,6 +11,7 @@
     pre_load,
     post_dump,
 )
+import pandas as pd
 
 from flexmeasures import Asset, Sensor
 from flexmeasures.data.schemas.generic_assets import GenericAssetIdField
@@ -31,6 +32,33 @@
 )
 
 
+def series_range_validator(min=None, max=None):
+    range_validator = validate.Range(min=min, max=max)
+
+    def _validate_series(value):
+        if isinstance(value, pd.Series):
+            invalid_mask = pd.Series([False] * len(value), index=value.index)
+
+            if min is not None:
+                invalid_mask |= value < min
+            if max is not None:
+                invalid_mask |= value > max
+
+            if invalid_mask.any():
+                invalid_indexes = value.index[invalid_mask].tolist()
+                invalid_values = value[invalid_mask].tolist()
+                raise ValidationError(
+                    f"Series contains values outside the allowed range (min={min}, max={max}).\n"
+                    f"Invalid entries:\n"
+                    f"Indexes: {invalid_indexes}\n"
+                    f"Values: {invalid_values}"
+                )
+        else:
+            range_validator(value)
+
+    return _validate_series
+
+
 class FlexContextSchema(Schema):
     """This schema defines fields that provide context to the portfolio to be optimized."""
 
@@ -45,28 +73,29 @@ class FlexContextSchema(Schema):
         "/MW",
         data_key="consumption-breach-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
+        fill_sides=True,
     )
     production_breach_price = VariableQuantityField(
         "/MW",
         data_key="production-breach-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
     )
     soc_minima_breach_price = VariableQuantityField(
         "/MWh",
         data_key="soc-minima-breach-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
     )
     soc_maxima_breach_price = VariableQuantityField(
         "/MWh",
         data_key="soc-maxima-breach-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
     )
     # Dev fields
@@ -82,7 +111,7 @@ class FlexContextSchema(Schema):
         "MW",
         required=False,
         data_key="site-power-capacity",
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
     )
     # todo: deprecated since flexmeasures==0.23
     consumption_price_sensor = SensorIdField(data_key="consumption-price-sensor")
@@ -105,26 +134,26 @@ class FlexContextSchema(Schema):
         "MW",
         required=False,
         data_key="site-production-capacity",
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
     )
     ems_consumption_capacity_in_mw = VariableQuantityField(
         "MW",
         required=False,
         data_key="site-consumption-capacity",
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
     )
     ems_consumption_breach_price = VariableQuantityField(
         "/MW",
         data_key="site-consumption-breach-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
     )
     ems_production_breach_price = VariableQuantityField(
         "/MW",
         data_key="site-production-breach-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
     )
 
@@ -133,14 +162,14 @@ class FlexContextSchema(Schema):
         "MW",
         required=False,
         data_key="site-peak-consumption",
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default="0 kW",
     )
     ems_peak_consumption_price = VariableQuantityField(
         "/MW",
         data_key="site-peak-consumption-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
     )
 
@@ -149,14 +178,14 @@ class FlexContextSchema(Schema):
         "MW",
         required=False,
         data_key="site-peak-production",
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default="0 kW",
     )
     ems_peak_production_price = VariableQuantityField(
         "/MW",
         data_key="site-peak-production-price",
         required=False,
-        value_validator=validate.Range(min=0),
+        value_validator=series_range_validator(min=0),
         default=None,
     )
     # todo: group by month start (MS), something like a commitment resolution, or a list of datetimes representing splits of the commitments
@@ -450,7 +479,7 @@ def __init__(self, *args, **kwargs):
         self.load_time_series = True
         for field_var, field in self.declared_fields.items():
             if isinstance(field, VariableQuantityField) and field_var in (
-                # "consumption_breach_price",
+                "consumption_breach_price",
             ):
                 field.load_time_series = True
             else:
diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py
index eaabf5d729..6bac0a82cc 100644
--- a/flexmeasures/data/schemas/sensors.py
+++ b/flexmeasures/data/schemas/sensors.py
@@ -20,6 +20,9 @@
 
 from flexmeasures.data import ma, db
 from flexmeasures.data.models.generic_assets import GenericAsset
+from flexmeasures.data.models.planning.utils import (
+    get_continuous_series_sensor_or_quantity,
+)
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.schemas.utils import (
     FMValidationError,
@@ -239,6 +242,9 @@ def __init__(
         default_src_unit: str | None = None,
         return_magnitude: bool = False,
         timezone: str | None = None,
+        fill_sides: bool = False,
+        add_resolution: bool = False,
+        resolve_overlaps: str = "first",
         value_validator: Validator | None = None,
         **kwargs,
     ):
@@ -274,6 +280,9 @@ def __init__(
             value_validator = RepurposeValidatorToIgnoreSensorsAndLists(value_validator)
             self.validators.insert(0, value_validator)
         self.timezone = timezone
+        self.fill_sides = fill_sides
+        self.add_resolution = add_resolution
+        self.resolve_overlaps = resolve_overlaps
         self.value_validator = value_validator
         if to_unit.startswith("/") and len(to_unit) < 2:
             raise ValueError(
@@ -286,24 +295,47 @@ def __init__(
             default_src_unit = "dimensionless"
         self.default_src_unit = default_src_unit
         self.return_magnitude = return_magnitude
+        self.load_time_series = False
 
     @with_appcontext_if_needed()
     def _deserialize(
         self, value: dict[str, int] | list[dict] | str, attr, obj, **kwargs
     ) -> Sensor | list[dict] | ur.Quantity:
 
-        if isinstance(value, dict):
-            return self._deserialize_dict(value)
-        elif isinstance(value, list):
-            return self._deserialize_list(value)
-        elif isinstance(value, str):
-            return self._deserialize_str(value)
-        elif isinstance(value, numbers.Real) and self.default_src_unit is not None:
-            return self._deserialize_numeric(value, attr, obj, **kwargs)
-        else:
-            raise FMValidationError(
-                f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported."
+        if not self.load_time_series:
+            if isinstance(value, dict):
+                value = self._deserialize_dict(value)
+            elif isinstance(value, list):
+                value = self._deserialize_list(value)
+            elif isinstance(value, str):
+                value = self._deserialize_str(value)
+            elif isinstance(value, numbers.Real) and self.default_src_unit is not None:
+                value = self._deserialize_numeric(value, attr, obj, **kwargs)
+            else:
+                raise FMValidationError(
+                    f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported."
+                )
+        # The schema can be initialized to load time series, rather than just the Sensor, time series specs or Quantity
+        # if hasattr(self.parent, "load_time_series") and self.parent.load_time_series:
+        if self.load_time_series:
+            query_window = self.parent.query_window
+            resolution = self.parent.resolution
+            if self.add_resolution:
+                query_window = (
+                    query_window[0] + resolution,
+                    query_window[1] + resolution,
+                )
+            return get_continuous_series_sensor_or_quantity(
+                variable_quantity=value,
+                actuator=self.parent.asset,
+                unit=self._get_unit(value) if self.to_unit[0] == "/" else self.to_unit,
+                query_window=query_window,
+                resolution=resolution,
+                beliefs_before=self.parent.belief_time,
+                fill_sides=self.fill_sides,
+                resolve_overlaps=self.resolve_overlaps,
             )
+        return value
 
     def _deserialize_dict(self, value: dict[str, int]) -> Sensor:
         """Deserialize a sensor reference to a Sensor."""

From df24453a6cbcd7ab00aebb6b44ddf26ac8e98058 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 23:03:56 +0200
Subject: [PATCH 134/151] refactor: move production_breach_price loading to
 schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 31 ++-----------------
 .../data/schemas/scheduling/__init__.py       |  2 ++
 2 files changed, 5 insertions(+), 28 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index cbed2c9070..a6e1b96dc2 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -723,35 +723,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                     production_breach_price = self.flex_context[
                         "production_breach_price"
                     ]
-                    any_production_breach_price = (
-                        get_continuous_series_sensor_or_quantity(
-                            variable_quantity=production_breach_price,
-                            actuator=asset,
-                            unit=FlexContextSchema()
-                            .declared_fields["production_breach_price"]
-                            ._get_unit(production_breach_price),
-                            query_window=(start, end),
-                            resolution=resolution,
-                            beliefs_before=belief_time,
-                            fallback_attribute="production-breach-price",
-                            fill_sides=True,
-                        )
-                    )
+                    any_production_breach_price = production_breach_price
                     all_production_breach_price = (
-                        get_continuous_series_sensor_or_quantity(
-                            variable_quantity=production_breach_price,
-                            actuator=asset,
-                            unit=FlexContextSchema()
-                            .declared_fields["production_breach_price"]
-                            ._get_unit(production_breach_price)
-                            + "*h",  # from EUR/MWh to EUR/MW/resolution
-                            query_window=(start, end),
-                            resolution=resolution,
-                            beliefs_before=belief_time,
-                            fallback_attribute="production-breach-price",
-                            fill_sides=True,
-                        )
-                    )
+                        production_breach_price * resolution / timedelta(hours=1)
+                    )  # from EUR/MWh to EUR/MW/resolution
                     # Set up commitments DataFrame
                     commitment = FlowCommitment(
                         name=f"any production breach device {d}",
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 9dc3cc39dc..0ee7c8a546 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -83,6 +83,7 @@ class FlexContextSchema(Schema):
         required=False,
         value_validator=series_range_validator(min=0),
         default=None,
+        fill_sides=True,
     )
     soc_minima_breach_price = VariableQuantityField(
         "/MWh",
@@ -480,6 +481,7 @@ def __init__(self, *args, **kwargs):
         for field_var, field in self.declared_fields.items():
             if isinstance(field, VariableQuantityField) and field_var in (
                 "consumption_breach_price",
+                "production_breach_price",
             ):
                 field.load_time_series = True
             else:

From 3ba3af01ef67840215f766848475ec88c2b3c97d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 23:08:17 +0200
Subject: [PATCH 135/151] refactor: move soc_maxima_breach_price loading to
 schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 33 +++++--------------
 .../data/schemas/scheduling/__init__.py       |  3 ++
 2 files changed, 11 insertions(+), 25 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index a6e1b96dc2..09e3bcd2d1 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -606,31 +606,14 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 and soc_maxima[d] is not None
             ):
                 soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"]
-                any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_maxima_breach_price,
-                    actuator=asset,
-                    unit=FlexContextSchema()
-                    .declared_fields["soc_maxima_breach_price"]
-                    ._get_unit(soc_maxima_breach_price),
-                    query_window=(start + resolution, end + resolution),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-maxima-breach-price",
-                    fill_sides=True,
-                ).shift(-1, freq=resolution)
-                all_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_maxima_breach_price,
-                    actuator=asset,
-                    unit=FlexContextSchema()
-                    .declared_fields["soc_maxima_breach_price"]
-                    ._get_unit(soc_maxima_breach_price)
-                    + "*h",  # from EUR/MWh² to EUR/MWh/resolution
-                    query_window=(start + resolution, end + resolution),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-maxima-breach-price",
-                    fill_sides=True,
-                ).shift(-1, freq=resolution)
+                any_soc_maxima_breach_price = soc_maxima_breach_price.shift(
+                    -1, freq=resolution
+                )
+                all_soc_maxima_breach_price = (
+                    soc_maxima_breach_price.shift(-1, freq=resolution)
+                    * resolution
+                    / timedelta(hours=1)
+                )  # from EUR/MWh² to EUR/MWh/resolution
                 # Set up commitments DataFrame
                 # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_maxima_d = get_continuous_series_sensor_or_quantity(
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 0ee7c8a546..745c3403c5 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -98,6 +98,8 @@ class FlexContextSchema(Schema):
         required=False,
         value_validator=series_range_validator(min=0),
         default=None,
+        add_resolution=True,
+        fill_sides=True,
     )
     # Dev fields
     relax_soc_constraints = fields.Bool(
@@ -482,6 +484,7 @@ def __init__(self, *args, **kwargs):
             if isinstance(field, VariableQuantityField) and field_var in (
                 "consumption_breach_price",
                 "production_breach_price",
+                "soc_maxima_breach_price",
             ):
                 field.load_time_series = True
             else:

From e3c4a51a8a1da3d7a0c0992f51cc817764667cf7 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 23:10:13 +0200
Subject: [PATCH 136/151] refactor: move soc_minima_breach_price loading to
 schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 33 +++++--------------
 .../data/schemas/scheduling/__init__.py       |  3 ++
 2 files changed, 11 insertions(+), 25 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 09e3bcd2d1..189bca4c01 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -522,31 +522,14 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 and soc_minima[d] is not None
             ):
                 soc_minima_breach_price = self.flex_context["soc_minima_breach_price"]
-                any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_minima_breach_price,
-                    actuator=asset,
-                    unit=FlexContextSchema()
-                    .declared_fields["soc_minima_breach_price"]
-                    ._get_unit(soc_minima_breach_price),
-                    query_window=(start + resolution, end + resolution),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-minima-breach-price",
-                    fill_sides=True,
-                ).shift(-1, freq=resolution)
-                all_soc_minima_breach_price = get_continuous_series_sensor_or_quantity(
-                    variable_quantity=soc_minima_breach_price,
-                    actuator=asset,
-                    unit=FlexContextSchema()
-                    .declared_fields["soc_minima_breach_price"]
-                    ._get_unit(soc_minima_breach_price)
-                    + "*h",  # from EUR/MWh² to EUR/MWh/resolution
-                    query_window=(start + resolution, end + resolution),
-                    resolution=resolution,
-                    beliefs_before=belief_time,
-                    fallback_attribute="soc-minima-breach-price",
-                    fill_sides=True,
-                ).shift(-1, freq=resolution)
+                any_soc_minima_breach_price = soc_minima_breach_price.shift(
+                    -1, freq=resolution
+                )
+                all_soc_minima_breach_price = (
+                    soc_minima_breach_price.shift(-1, freq=resolution)
+                    * resolution
+                    / timedelta(hours=1)
+                )  # from EUR/MWh² to EUR/MWh/resolution
                 # Set up commitments DataFrame
                 # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet
                 soc_minima_d = get_continuous_series_sensor_or_quantity(
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 745c3403c5..3e0903c646 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -91,6 +91,8 @@ class FlexContextSchema(Schema):
         required=False,
         value_validator=series_range_validator(min=0),
         default=None,
+        add_resolution=True,
+        fill_sides=True,
     )
     soc_maxima_breach_price = VariableQuantityField(
         "/MWh",
@@ -484,6 +486,7 @@ def __init__(self, *args, **kwargs):
             if isinstance(field, VariableQuantityField) and field_var in (
                 "consumption_breach_price",
                 "production_breach_price",
+                "soc_minima_breach_price",
                 "soc_maxima_breach_price",
             ):
                 field.load_time_series = True

From 1e01ee37b3d67dd44d140219304d7aa0de4d59f5 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 23:13:40 +0200
Subject: [PATCH 137/151] refactor: move ems_consumption_breach_price and
 ems_production_breach_price loading to schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 54 +++----------------
 .../data/schemas/scheduling/__init__.py       |  4 ++
 2 files changed, 12 insertions(+), 46 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 189bca4c01..00011c4d2e 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -348,29 +348,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         if ems_consumption_breach_price is not None:
 
             # Convert to Series
-            any_ems_consumption_breach_price = get_continuous_series_sensor_or_quantity(
-                variable_quantity=ems_consumption_breach_price,
-                actuator=asset,
-                unit=FlexContextSchema()
-                .declared_fields["ems_consumption_breach_price"]
-                ._get_unit(ems_consumption_breach_price),
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                fill_sides=True,
-            )
-            all_ems_consumption_breach_price = get_continuous_series_sensor_or_quantity(
-                variable_quantity=ems_consumption_breach_price,
-                actuator=asset,
-                unit=FlexContextSchema()
-                .declared_fields["ems_consumption_breach_price"]
-                ._get_unit(ems_consumption_breach_price)
-                + "*h",  # from EUR/MWh to EUR/MW/resolution
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                fill_sides=True,
-            )
+            any_ems_consumption_breach_price = ems_consumption_breach_price
+            all_ems_consumption_breach_price = (
+                ems_consumption_breach_price * resolution / timedelta(hours=1)
+            )  # from EUR/MWh to EUR/MW/resolution
 
             # Set up commitments DataFrame to penalize any breach
             commitment = FlowCommitment(
@@ -402,29 +383,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         if ems_production_breach_price is not None:
 
             # Convert to Series
-            any_ems_production_breach_price = get_continuous_series_sensor_or_quantity(
-                variable_quantity=ems_production_breach_price,
-                actuator=asset,
-                unit=FlexContextSchema()
-                .declared_fields["ems_production_breach_price"]
-                ._get_unit(ems_production_breach_price),
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                fill_sides=True,
-            )
-            all_ems_production_breach_price = get_continuous_series_sensor_or_quantity(
-                variable_quantity=ems_production_breach_price,
-                actuator=asset,
-                unit=FlexContextSchema()
-                .declared_fields["ems_production_breach_price"]
-                ._get_unit(ems_production_breach_price)
-                + "*h",  # from EUR/MWh to EUR/MW/resolution
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                fill_sides=True,
-            )
+            any_ems_production_breach_price = ems_production_breach_price
+            all_ems_production_breach_price = (
+                ems_production_breach_price * resolution / timedelta(hours=1)
+            )  # from EUR/MWh to EUR/MW/resolution
 
             # Set up commitments DataFrame to penalize any breach
             commitment = FlowCommitment(
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 3e0903c646..e629e844fe 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -153,6 +153,7 @@ class FlexContextSchema(Schema):
         required=False,
         value_validator=series_range_validator(min=0),
         default=None,
+        fill_sides=True,
     )
     ems_production_breach_price = VariableQuantityField(
         "/MW",
@@ -160,6 +161,7 @@ class FlexContextSchema(Schema):
         required=False,
         value_validator=series_range_validator(min=0),
         default=None,
+        fill_sides=True,
     )
 
     # Peak consumption commitment
@@ -488,6 +490,8 @@ def __init__(self, *args, **kwargs):
                 "production_breach_price",
                 "soc_minima_breach_price",
                 "soc_maxima_breach_price",
+                "ems_consumption_breach_price",
+                "ems_production_breach_price",
             ):
                 field.load_time_series = True
             else:

From e521c7bd6965d23728149a71be23811d83ed7e7c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 23:21:07 +0200
Subject: [PATCH 138/151] refactor: move ems_peak_consumption_price and
 ems_peak_production_price loading to schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 22 -------------------
 .../data/schemas/scheduling/__init__.py       |  4 ++++
 2 files changed, 4 insertions(+), 22 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 00011c4d2e..beef773906 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -274,17 +274,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_peak_consumption_price = self.flex_context.get(
                 "ems_peak_consumption_price"
             )
-            ems_peak_consumption_price = get_continuous_series_sensor_or_quantity(
-                variable_quantity=ems_peak_consumption_price,
-                actuator=asset,
-                unit=FlexContextSchema()
-                .declared_fields["ems_peak_consumption_price"]
-                ._get_unit(ems_peak_consumption_price),
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                fill_sides=True,
-            )
 
             # Set up commitments DataFrame
             commitment = FlowCommitment(
@@ -310,17 +299,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             ems_peak_production_price = self.flex_context.get(
                 "ems_peak_production_price"
             )
-            ems_peak_production_price = get_continuous_series_sensor_or_quantity(
-                variable_quantity=ems_peak_production_price,
-                actuator=asset,
-                unit=FlexContextSchema()
-                .declared_fields["ems_peak_production_price"]
-                ._get_unit(ems_peak_production_price),
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                fill_sides=True,
-            )
 
             # Set up commitments DataFrame
             commitment = FlowCommitment(
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index e629e844fe..82d6beaa1c 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -178,6 +178,7 @@ class FlexContextSchema(Schema):
         required=False,
         value_validator=series_range_validator(min=0),
         default=None,
+        fill_sides=True,
     )
 
     # Peak production commitment
@@ -194,6 +195,7 @@ class FlexContextSchema(Schema):
         required=False,
         value_validator=series_range_validator(min=0),
         default=None,
+        fill_sides=True,
     )
     # todo: group by month start (MS), something like a commitment resolution, or a list of datetimes representing splits of the commitments
 
@@ -492,6 +494,8 @@ def __init__(self, *args, **kwargs):
                 "soc_maxima_breach_price",
                 "ems_consumption_breach_price",
                 "ems_production_breach_price",
+                "ems_peak_consumption_price",
+                "ems_peak_production_price",
             ):
                 field.load_time_series = True
             else:

From de936c73e625deaac54923d132ea0f59804f4c9b Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Apr 2025 23:22:46 +0200
Subject: [PATCH 139/151] refactor: move consumption_price and production_price
 loading to schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 24 ++-----------------
 .../data/schemas/scheduling/__init__.py       |  4 ++++
 2 files changed, 6 insertions(+), 22 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index beef773906..caca130560 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -161,29 +161,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         power_capacity_in_mw = self._get_device_power_capacity(flex_model, sensors)
 
         # Check for known prices or price forecasts
-        up_deviation_prices = get_continuous_series_sensor_or_quantity(
-            variable_quantity=consumption_price,
-            actuator=asset,
-            unit=FlexContextSchema()
-            .declared_fields["consumption_price"]
-            ._get_unit(consumption_price),
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            fill_sides=True,
-        ).to_frame(name="event_value")
+        up_deviation_prices = consumption_price.to_frame(name="event_value")
         ensure_prices_are_not_empty(up_deviation_prices, consumption_price)
-        down_deviation_prices = get_continuous_series_sensor_or_quantity(
-            variable_quantity=production_price,
-            actuator=asset,
-            unit=FlexContextSchema()
-            .declared_fields["production_price"]
-            ._get_unit(production_price),
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            fill_sides=True,
-        ).to_frame(name="event_value")
+        down_deviation_prices = production_price.to_frame(name="event_value")
         ensure_prices_are_not_empty(down_deviation_prices, production_price)
 
         start = pd.Timestamp(start).tz_convert("UTC")
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 82d6beaa1c..53f2588183 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -126,12 +126,14 @@ class FlexContextSchema(Schema):
         required=False,
         data_key="consumption-price",
         return_magnitude=False,
+        fill_sides=True,
     )
     production_price = VariableQuantityField(
         "/MWh",
         required=False,
         data_key="production-price",
         return_magnitude=False,
+        fill_sides=True,
     )
 
     # Capacity breach commitments
@@ -496,6 +498,8 @@ def __init__(self, *args, **kwargs):
                 "ems_production_breach_price",
                 "ems_peak_consumption_price",
                 "ems_peak_production_price",
+                "consumption_price",
+                "production_price",
             ):
                 field.load_time_series = True
             else:

From f09bd7fecd07403cae21ccc81d85b06e377b1dcc Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 15:11:19 +0200
Subject: [PATCH 140/151] fix: type annotation

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/schemas/sensors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py
index 6bac0a82cc..d29121c6c7 100644
--- a/flexmeasures/data/schemas/sensors.py
+++ b/flexmeasures/data/schemas/sensors.py
@@ -407,7 +407,7 @@ def convert(self, value, param, ctx, **kwargs):
 
         return super().convert(_value, param, ctx, **kwargs)
 
-    def _get_unit(self, variable_quantity: ur.Quantity | list[dict | Sensor]) -> str:
+    def _get_unit(self, variable_quantity: ur.Quantity | list[dict] | Sensor) -> str:
         """Obtain the unit from the variable quantity."""
         if isinstance(variable_quantity, ur.Quantity):
             unit = str(variable_quantity.units)

From 0284f3970ad783b6f2343ade65f6d9232582cb71 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 15:10:51 +0200
Subject: [PATCH 141/151] fix: compatibility with deprecated flex-context
 fields

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py |  7 +++-
 .../data/schemas/scheduling/__init__.py       |  4 ++
 flexmeasures/data/schemas/sensors.py          | 38 +++++++++++++++++++
 3 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index fb563f020c..1607be5387 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -14,6 +14,7 @@
 from flexmeasures.data.models.planning import Scheduler
 from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException
 from flexmeasures.data.models.planning.storage import (
+    MetaStorageScheduler,
     StorageScheduler,
     add_storage_constraints,
     validate_storage_constraints,
@@ -424,6 +425,7 @@ def test_charging_station_solver_day_2(
             "consumption_price": epex_da,
         },
     )
+    scheduler._load_time_series(charging_station)
     scheduler.config_deserialized = (
         True  # soc targets are already a DataFrame, names get underscore
     )
@@ -508,6 +510,7 @@ def test_fallback_to_unsolvable_problem(
         },
     }
     scheduler = StorageScheduler(**kwargs)
+    scheduler._load_time_series(charging_station)
     scheduler.config_deserialized = (
         True  # soc targets are already a DataFrame, names get underscore
     )
@@ -517,7 +520,8 @@ def test_fallback_to_unsolvable_problem(
         consumption_schedule = scheduler.compute(skip_validation=True)
 
     # check that the fallback scheduler provides a sensible fallback policy
-    fallback_scheduler = scheduler.fallback_scheduler_class(**kwargs)
+    fallback_scheduler: MetaStorageScheduler = scheduler.fallback_scheduler_class(**kwargs)
+    fallback_scheduler._load_time_series(charging_station)
     fallback_scheduler.config_deserialized = True
     consumption_schedule = fallback_scheduler.compute(skip_validation=True)
 
@@ -608,6 +612,7 @@ def test_building_solver_day_2(
             "consumption_price": consumption_price_sensor,
         },
     )
+    scheduler._load_time_series(battery)
     scheduler.config_deserialized = (
         True  # inflexible device sensors are already objects, names get underscore
     )
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 53f2588183..6d0a8f18b1 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -18,6 +18,7 @@
 from flexmeasures.data.schemas.sensors import (
     VariableQuantityField,
     SensorIdField,
+    TimeSeriesField,
 )
 from flexmeasures.data.schemas.utils import FMValidationError
 from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField
@@ -502,6 +503,9 @@ def __init__(self, *args, **kwargs):
                 "production_price",
             ):
                 field.load_time_series = True
+            # Compatibility with deprecated fields
+            elif field_var in ("consumption_price_sensor", "production_price_sensor"):
+                field = TimeSeriesField(to_unit="/MWh")
             else:
                 # Skip deserialization
                 field._deserialize = passthrough_deserializer()
diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py
index d29121c6c7..97adc14691 100644
--- a/flexmeasures/data/schemas/sensors.py
+++ b/flexmeasures/data/schemas/sensors.py
@@ -234,6 +234,44 @@ def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int:
         return sensor.id
 
 
+class TimeSeriesField(fields.Field):
+
+    def __init__(
+        self,
+        to_unit,
+        *args,
+        # timezone: str | None = None,
+        fill_sides: bool = False,
+        add_resolution: bool = False,
+        resolve_overlaps: str = "first",
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.to_unit = to_unit
+        self.fill_sides = fill_sides
+        self.add_resolution = add_resolution
+        self.resolve_overlaps = resolve_overlaps
+
+    def _deserialize(self, value: Sensor, **kwargs) -> pd.Series:
+        query_window = self.parent.query_window
+        resolution = self.parent.resolution
+        if self.add_resolution:
+            query_window = (
+                query_window[0] + resolution,
+                query_window[1] + resolution,
+            )
+        return get_continuous_series_sensor_or_quantity(
+            variable_quantity=value,
+            actuator=self.parent.asset,
+            unit=self._get_unit(value) if self.to_unit[0] == "/" else self.to_unit,
+            query_window=query_window,
+            resolution=resolution,
+            beliefs_before=self.parent.belief_time,
+            fill_sides=self.fill_sides,
+            resolve_overlaps=self.resolve_overlaps,
+        )
+
+
 class VariableQuantityField(MarshmallowClickMixin, fields.Field):
     def __init__(
         self,

From 15520c002d264017ab8d1349d6a6647791a0be04 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 09:14:38 +0200
Subject: [PATCH 142/151] fix: don't override field, just override field
 attribute to load the time series

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py       |  11 +-
 flexmeasures/data/schemas/sensors.py          | 106 +++++++++---------
 2 files changed, 58 insertions(+), 59 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 6d0a8f18b1..a2dfdee5ef 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -18,7 +18,6 @@
 from flexmeasures.data.schemas.sensors import (
     VariableQuantityField,
     SensorIdField,
-    TimeSeriesField,
 )
 from flexmeasures.data.schemas.utils import FMValidationError
 from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField
@@ -120,8 +119,12 @@ class FlexContextSchema(Schema):
         value_validator=series_range_validator(min=0),
     )
     # todo: deprecated since flexmeasures==0.23
-    consumption_price_sensor = SensorIdField(data_key="consumption-price-sensor")
-    production_price_sensor = SensorIdField(data_key="production-price-sensor")
+    consumption_price_sensor = SensorIdField(
+        data_key="consumption-price-sensor", fill_sides=True
+    )
+    production_price_sensor = SensorIdField(
+        data_key="production-price-sensor", fill_sides=True
+    )
     consumption_price = VariableQuantityField(
         "/MWh",
         required=False,
@@ -505,7 +508,7 @@ def __init__(self, *args, **kwargs):
                 field.load_time_series = True
             # Compatibility with deprecated fields
             elif field_var in ("consumption_price_sensor", "production_price_sensor"):
-                field = TimeSeriesField(to_unit="/MWh")
+                field.load_time_series = True
             else:
                 # Skip deserialization
                 field._deserialize = passthrough_deserializer()
diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py
index 97adc14691..c66b523560 100644
--- a/flexmeasures/data/schemas/sensors.py
+++ b/flexmeasures/data/schemas/sensors.py
@@ -2,6 +2,7 @@
 
 import numbers
 
+import timely_beliefs
 from flask import current_app
 from marshmallow import (
     Schema,
@@ -198,7 +199,15 @@ class Meta:
 class SensorIdField(MarshmallowClickMixin, fields.Int):
     """Field that deserializes to a Sensor and serializes back to an integer."""
 
-    def __init__(self, *args, unit: str | ur.Quantity | None = None, **kwargs):
+    def __init__(
+        self,
+        *args,
+        unit: str | ur.Quantity | None = None,
+        fill_sides: bool = False,
+        add_resolution: bool = False,
+        resolve_overlaps: str = "first",
+        **kwargs,
+    ):
         super().__init__(*args, **kwargs)
 
         if isinstance(unit, str):
@@ -207,71 +216,58 @@ def __init__(self, *args, unit: str | ur.Quantity | None = None, **kwargs):
             self.to_unit = unit
         else:
             self.to_unit = None
+        self.load_time_series = False
+        self.fill_sides = fill_sides
+        self.add_resolution = add_resolution
+        self.resolve_overlaps = resolve_overlaps
 
     @with_appcontext_if_needed()
-    def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor:
+    def _deserialize(
+        self, value: int, attr, obj, **kwargs
+    ) -> Sensor | timely_beliefs.BeliefsSeries:
         """Turn a sensor id into a Sensor."""
-        sensor = db.session.get(Sensor, value)
-        if sensor is None:
-            raise FMValidationError(f"No sensor found with id {value}.")
+        if not self.load_time_series:
+            sensor = db.session.get(Sensor, value)
+            if sensor is None:
+                raise FMValidationError(f"No sensor found with id {value}.")
 
-        # lazy loading now (sensor is somehow not in session after this)
-        sensor.generic_asset
-        sensor.generic_asset.generic_asset_type
+            # lazy loading now (sensor is somehow not in session after this)
+            sensor.generic_asset
+            sensor.generic_asset.generic_asset_type
 
-        # if the units are defined, check if the sensor data is convertible to the target units
-        if self.to_unit is not None and not units_are_convertible(
-            sensor.unit, str(self.to_unit.units)
-        ):
-            raise FMValidationError(
-                f"Cannot convert {sensor.unit} to {self.to_unit.units}"
-            )
+            # if the units are defined, check if the sensor data is convertible to the target units
+            if self.to_unit is not None and not units_are_convertible(
+                sensor.unit, str(self.to_unit.units)
+            ):
+                raise FMValidationError(
+                    f"Cannot convert {sensor.unit} to {self.to_unit.units}"
+                )
 
-        return sensor
+            return sensor
+        else:
+            query_window = self.parent.query_window
+            resolution = self.parent.resolution
+            if self.add_resolution:
+                query_window = (
+                    query_window[0] + resolution,
+                    query_window[1] + resolution,
+                )
+            return get_continuous_series_sensor_or_quantity(
+                variable_quantity=value,
+                actuator=self.parent.asset,
+                unit=value.unit,
+                query_window=query_window,
+                resolution=resolution,
+                beliefs_before=self.parent.belief_time,
+                fill_sides=self.fill_sides,
+                resolve_overlaps=self.resolve_overlaps,
+            )
 
     def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int:
         """Turn a Sensor into a sensor id."""
         return sensor.id
 
 
-class TimeSeriesField(fields.Field):
-
-    def __init__(
-        self,
-        to_unit,
-        *args,
-        # timezone: str | None = None,
-        fill_sides: bool = False,
-        add_resolution: bool = False,
-        resolve_overlaps: str = "first",
-        **kwargs,
-    ):
-        super().__init__(*args, **kwargs)
-        self.to_unit = to_unit
-        self.fill_sides = fill_sides
-        self.add_resolution = add_resolution
-        self.resolve_overlaps = resolve_overlaps
-
-    def _deserialize(self, value: Sensor, **kwargs) -> pd.Series:
-        query_window = self.parent.query_window
-        resolution = self.parent.resolution
-        if self.add_resolution:
-            query_window = (
-                query_window[0] + resolution,
-                query_window[1] + resolution,
-            )
-        return get_continuous_series_sensor_or_quantity(
-            variable_quantity=value,
-            actuator=self.parent.asset,
-            unit=self._get_unit(value) if self.to_unit[0] == "/" else self.to_unit,
-            query_window=query_window,
-            resolution=resolution,
-            beliefs_before=self.parent.belief_time,
-            fill_sides=self.fill_sides,
-            resolve_overlaps=self.resolve_overlaps,
-        )
-
-
 class VariableQuantityField(MarshmallowClickMixin, fields.Field):
     def __init__(
         self,
@@ -338,7 +334,7 @@ def __init__(
     @with_appcontext_if_needed()
     def _deserialize(
         self, value: dict[str, int] | list[dict] | str, attr, obj, **kwargs
-    ) -> Sensor | list[dict] | ur.Quantity:
+    ) -> Sensor | list[dict] | ur.Quantity | timely_beliefs.BeliefsSeries:
 
         if not self.load_time_series:
             if isinstance(value, dict):

From a3b9811fdfdc3c6140b345f747b5b17f5ffc5544 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 16:34:08 +0200
Subject: [PATCH 143/151] refactor: move ems_power_capacity_in_mw,
 ems_consumption_capacity_in_mw and ems_production_capacity_in_mw loading to
 schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 46 ++++++++-----------
 .../data/schemas/scheduling/__init__.py       |  6 +++
 2 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index caca130560..17dbd9c8af 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -181,35 +181,25 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
         # Create Series with EMS capacities
-        ems_power_capacity_in_mw = get_continuous_series_sensor_or_quantity(
-            variable_quantity=self.flex_context.get("ems_power_capacity_in_mw"),
-            actuator=asset,
-            unit="MW",
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            resolve_overlaps="min",
-        )
-        ems_consumption_capacity = get_continuous_series_sensor_or_quantity(
-            variable_quantity=self.flex_context.get("ems_consumption_capacity_in_mw"),
-            actuator=asset,
-            unit="MW",
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            max_value=ems_power_capacity_in_mw,
-            resolve_overlaps="min",
-        )
-        ems_production_capacity = -1 * get_continuous_series_sensor_or_quantity(
-            variable_quantity=self.flex_context.get("ems_production_capacity_in_mw"),
-            actuator=asset,
-            unit="MW",
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            max_value=ems_power_capacity_in_mw,
-            resolve_overlaps="min",
+        ems_power_capacity_in_mw = self.flex_context.get("ems_power_capacity_in_mw")
+        ems_consumption_capacity = self.flex_context.get(
+            "ems_consumption_capacity_in_mw"
         )
+        # if (match := pattern.search(data)) is not None:
+        #     # Do something with match
+        ems_production_capacity = self.flex_context.get("ems_production_capacity_in_mw")
+        if ems_consumption_capacity is not None:
+            ems_consumption_capacity = ems_consumption_capacity.clip(
+                upper=ems_power_capacity_in_mw
+            )
+        else:
+            ems_consumption_capacity = ems_power_capacity_in_mw
+        if ems_production_capacity is not None:
+            ems_production_capacity = -1 * (
+                ems_production_capacity.clip(upper=ems_power_capacity_in_mw)
+            )
+        else:
+            ems_production_capacity = -ems_power_capacity_in_mw
 
         # Set up commitments to optimise for
         commitments = []
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index a2dfdee5ef..1a1d6c4d7d 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -117,6 +117,7 @@ class FlexContextSchema(Schema):
         required=False,
         data_key="site-power-capacity",
         value_validator=series_range_validator(min=0),
+        resolve_overlaps="min",
     )
     # todo: deprecated since flexmeasures==0.23
     consumption_price_sensor = SensorIdField(
@@ -146,12 +147,14 @@ class FlexContextSchema(Schema):
         required=False,
         data_key="site-production-capacity",
         value_validator=series_range_validator(min=0),
+        resolve_overlaps="min",
     )
     ems_consumption_capacity_in_mw = VariableQuantityField(
         "MW",
         required=False,
         data_key="site-consumption-capacity",
         value_validator=series_range_validator(min=0),
+        resolve_overlaps="min",
     )
     ems_consumption_breach_price = VariableQuantityField(
         "/MW",
@@ -504,6 +507,9 @@ def __init__(self, *args, **kwargs):
                 "ems_peak_production_price",
                 "consumption_price",
                 "production_price",
+                "ems_power_capacity_in_mw",
+                "ems_consumption_capacity_in_mw",
+                "ems_production_capacity_in_mw",
             ):
                 field.load_time_series = True
             # Compatibility with deprecated fields

From a7a5e759d42afa1725a8ae03bc3365ae4087011d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 15:14:56 +0200
Subject: [PATCH 144/151] fix: determine resolution for loading time series
 data after deserializing flex-model

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/__init__.py |  2 ++
 flexmeasures/data/models/planning/storage.py  | 23 +++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index bdaec4b83c..e7a1980d92 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -44,6 +44,7 @@ class Scheduler:
     start: datetime
     end: datetime
     resolution: timedelta
+    load_resolution: timedelta
     belief_time: datetime
 
     round_to_decimals: int
@@ -114,6 +115,7 @@ def __init__(
         self.start = start
         self.end = end
         self.resolution = resolution
+        self.load_resolution = resolution
         self.belief_time = belief_time
         self.round_to_decimals = round_to_decimals
         if flex_model is None:
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 17dbd9c8af..ea23433bc4 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -814,13 +814,25 @@ def persist_flex_model(self):
             )
 
     def _load_time_series(self, asset):
+        if not self.load_resolution:
+            self._determine_load_resolution()
         self.flex_context = FlexContextTimeSeriesSchema(
             asset=asset,
             query_window=(self.start, self.end),
-            resolution=self.resolution,
+            resolution=self.load_resolution,
             belief_time=self.belief_time,
         ).load(self.flex_context)
 
+    def _determine_load_resolution(self):
+        """Determine resolution for loading time series data."""
+        if self.asset is not None:
+            sensors = [flex_model_d["sensor"] for flex_model_d in self.flex_model]
+            self.load_resolution = determine_minimum_resampling_resolution(
+                [s.event_resolution for s in sensors]
+            )
+        else:
+            self.load_resolution = self.resolution
+
     def deserialize_flex_config(self):
         """
         Deserialize storage flex model and the flex context against schemas.
@@ -845,10 +857,6 @@ def deserialize_flex_config(self):
             {**db_flex_context, **self.flex_context}
         )
 
-        # Load time series from flex-context
-        if self.load_time_series:
-            self._load_time_series(asset)
-
         if isinstance(self.flex_model, dict):
             # Check state of charge.
             # Preferably, a starting soc is given.
@@ -902,6 +910,11 @@ def deserialize_flex_config(self):
                 f"Unsupported type of flex-model: '{type(self.flex_model)}'"
             )
 
+        # Load time series from flex-context
+        self._determine_load_resolution()
+        if self.load_time_series:
+            self._load_time_series(asset)
+
         return self.flex_model
 
     def possibly_extend_end(self, soc_targets, sensor: Sensor = None):

From 28389cefd4c053958df1cfe37dcd891a871d22e5 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 17:28:23 +0200
Subject: [PATCH 145/151] Revert "refactor: move ems_power_capacity_in_mw,
 ems_consumption_capacity_in_mw and ems_production_capacity_in_mw loading to
 schema"

This reverts commit a3b9811fdfdc3c6140b345f747b5b17f5ffc5544.
---
 flexmeasures/data/models/planning/storage.py  | 46 +++++++++++--------
 .../data/schemas/scheduling/__init__.py       |  6 ---
 2 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index ea23433bc4..66f38309d0 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -181,25 +181,35 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
         # Create Series with EMS capacities
-        ems_power_capacity_in_mw = self.flex_context.get("ems_power_capacity_in_mw")
-        ems_consumption_capacity = self.flex_context.get(
-            "ems_consumption_capacity_in_mw"
+        ems_power_capacity_in_mw = get_continuous_series_sensor_or_quantity(
+            variable_quantity=self.flex_context.get("ems_power_capacity_in_mw"),
+            actuator=asset,
+            unit="MW",
+            query_window=(start, end),
+            resolution=resolution,
+            beliefs_before=belief_time,
+            resolve_overlaps="min",
+        )
+        ems_consumption_capacity = get_continuous_series_sensor_or_quantity(
+            variable_quantity=self.flex_context.get("ems_consumption_capacity_in_mw"),
+            actuator=asset,
+            unit="MW",
+            query_window=(start, end),
+            resolution=resolution,
+            beliefs_before=belief_time,
+            max_value=ems_power_capacity_in_mw,
+            resolve_overlaps="min",
+        )
+        ems_production_capacity = -1 * get_continuous_series_sensor_or_quantity(
+            variable_quantity=self.flex_context.get("ems_production_capacity_in_mw"),
+            actuator=asset,
+            unit="MW",
+            query_window=(start, end),
+            resolution=resolution,
+            beliefs_before=belief_time,
+            max_value=ems_power_capacity_in_mw,
+            resolve_overlaps="min",
         )
-        # if (match := pattern.search(data)) is not None:
-        #     # Do something with match
-        ems_production_capacity = self.flex_context.get("ems_production_capacity_in_mw")
-        if ems_consumption_capacity is not None:
-            ems_consumption_capacity = ems_consumption_capacity.clip(
-                upper=ems_power_capacity_in_mw
-            )
-        else:
-            ems_consumption_capacity = ems_power_capacity_in_mw
-        if ems_production_capacity is not None:
-            ems_production_capacity = -1 * (
-                ems_production_capacity.clip(upper=ems_power_capacity_in_mw)
-            )
-        else:
-            ems_production_capacity = -ems_power_capacity_in_mw
 
         # Set up commitments to optimise for
         commitments = []
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 1a1d6c4d7d..a2dfdee5ef 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -117,7 +117,6 @@ class FlexContextSchema(Schema):
         required=False,
         data_key="site-power-capacity",
         value_validator=series_range_validator(min=0),
-        resolve_overlaps="min",
     )
     # todo: deprecated since flexmeasures==0.23
     consumption_price_sensor = SensorIdField(
@@ -147,14 +146,12 @@ class FlexContextSchema(Schema):
         required=False,
         data_key="site-production-capacity",
         value_validator=series_range_validator(min=0),
-        resolve_overlaps="min",
     )
     ems_consumption_capacity_in_mw = VariableQuantityField(
         "MW",
         required=False,
         data_key="site-consumption-capacity",
         value_validator=series_range_validator(min=0),
-        resolve_overlaps="min",
     )
     ems_consumption_breach_price = VariableQuantityField(
         "/MW",
@@ -507,9 +504,6 @@ def __init__(self, *args, **kwargs):
                 "ems_peak_production_price",
                 "consumption_price",
                 "production_price",
-                "ems_power_capacity_in_mw",
-                "ems_consumption_capacity_in_mw",
-                "ems_production_capacity_in_mw",
             ):
                 field.load_time_series = True
             # Compatibility with deprecated fields

From 512e754589f58c0c83777b811e73d9ad1c438a6a Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 16:34:08 +0200
Subject: [PATCH 146/151] refactor: move ems_power_capacity_in_mw,
 ems_consumption_capacity_in_mw and ems_production_capacity_in_mw loading to
 schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 46 ++++++++-----------
 .../data/schemas/scheduling/__init__.py       |  6 +++
 2 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 66f38309d0..4a015eb98a 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -181,35 +181,25 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
 
         # Create Series with EMS capacities
-        ems_power_capacity_in_mw = get_continuous_series_sensor_or_quantity(
-            variable_quantity=self.flex_context.get("ems_power_capacity_in_mw"),
-            actuator=asset,
-            unit="MW",
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            resolve_overlaps="min",
-        )
-        ems_consumption_capacity = get_continuous_series_sensor_or_quantity(
-            variable_quantity=self.flex_context.get("ems_consumption_capacity_in_mw"),
-            actuator=asset,
-            unit="MW",
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            max_value=ems_power_capacity_in_mw,
-            resolve_overlaps="min",
-        )
-        ems_production_capacity = -1 * get_continuous_series_sensor_or_quantity(
-            variable_quantity=self.flex_context.get("ems_production_capacity_in_mw"),
-            actuator=asset,
-            unit="MW",
-            query_window=(start, end),
-            resolution=resolution,
-            beliefs_before=belief_time,
-            max_value=ems_power_capacity_in_mw,
-            resolve_overlaps="min",
+        ems_power_capacity_in_mw = self.flex_context.get("ems_power_capacity_in_mw")
+        ems_consumption_capacity = self.flex_context.get(
+            "ems_consumption_capacity_in_mw"
         )
+        ems_production_capacity = self.flex_context.get("ems_production_capacity_in_mw")
+        if ems_consumption_capacity is not None:
+            ems_consumption_capacity = ems_consumption_capacity.clip(
+                upper=ems_power_capacity_in_mw
+            ).fillna(ems_power_capacity_in_mw)
+        else:
+            ems_consumption_capacity = ems_power_capacity_in_mw
+        if ems_production_capacity is not None:
+            ems_production_capacity = -1 * (
+                ems_production_capacity.clip(upper=ems_power_capacity_in_mw).fillna(
+                    ems_power_capacity_in_mw
+                )
+            )
+        else:
+            ems_production_capacity = -ems_power_capacity_in_mw
 
         # Set up commitments to optimise for
         commitments = []
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index a2dfdee5ef..1a1d6c4d7d 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -117,6 +117,7 @@ class FlexContextSchema(Schema):
         required=False,
         data_key="site-power-capacity",
         value_validator=series_range_validator(min=0),
+        resolve_overlaps="min",
     )
     # todo: deprecated since flexmeasures==0.23
     consumption_price_sensor = SensorIdField(
@@ -146,12 +147,14 @@ class FlexContextSchema(Schema):
         required=False,
         data_key="site-production-capacity",
         value_validator=series_range_validator(min=0),
+        resolve_overlaps="min",
     )
     ems_consumption_capacity_in_mw = VariableQuantityField(
         "MW",
         required=False,
         data_key="site-consumption-capacity",
         value_validator=series_range_validator(min=0),
+        resolve_overlaps="min",
     )
     ems_consumption_breach_price = VariableQuantityField(
         "/MW",
@@ -504,6 +507,9 @@ def __init__(self, *args, **kwargs):
                 "ems_peak_production_price",
                 "consumption_price",
                 "production_price",
+                "ems_power_capacity_in_mw",
+                "ems_consumption_capacity_in_mw",
+                "ems_production_capacity_in_mw",
             ):
                 field.load_time_series = True
             # Compatibility with deprecated fields

From bb3341187fb057a308f8bf8fd4d78394dd288e2f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 17:50:08 +0200
Subject: [PATCH 147/151] refactor: use walrus operator

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 4a015eb98a..46ac034ba0 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -182,19 +182,15 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
 
         # Create Series with EMS capacities
         ems_power_capacity_in_mw = self.flex_context.get("ems_power_capacity_in_mw")
-        ems_consumption_capacity = self.flex_context.get(
-            "ems_consumption_capacity_in_mw"
-        )
-        ems_production_capacity = self.flex_context.get("ems_production_capacity_in_mw")
-        if ems_consumption_capacity is not None:
-            ems_consumption_capacity = ems_consumption_capacity.clip(
-                upper=ems_power_capacity_in_mw
-            ).fillna(ems_power_capacity_in_mw)
+        if (cap := self.flex_context.get("ems_consumption_capacity_in_mw")) is not None:
+            ems_consumption_capacity = cap.clip(upper=ems_power_capacity_in_mw).fillna(
+                ems_power_capacity_in_mw
+            )
         else:
             ems_consumption_capacity = ems_power_capacity_in_mw
-        if ems_production_capacity is not None:
+        if (cap := self.flex_context.get("ems_production_capacity_in_mw")) is not None:
             ems_production_capacity = -1 * (
-                ems_production_capacity.clip(upper=ems_power_capacity_in_mw).fillna(
+                cap.clip(upper=ems_power_capacity_in_mw).fillna(
                     ems_power_capacity_in_mw
                 )
             )

From 380623dcacd95f8c847e43c3ca5560feda39668e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 17:54:44 +0200
Subject: [PATCH 148/151] refactor: move ems_peak_consumption_in_mw and
 ems_peak_production_in_mw loading to schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py  | 30 +++++--------------
 .../data/schemas/scheduling/__init__.py       |  4 +++
 2 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 46ac034ba0..968c12b4d2 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -97,11 +97,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             resolution = determine_minimum_resampling_resolution(
                 [s.event_resolution for s in sensors]
             )
-            asset = self.asset
         else:
             # For backwards compatibility with the single asset scheduler
             sensors = [self.sensor]
-            asset = self.sensor.generic_asset
 
         # For backwards compatibility with the single asset scheduler
         flex_model = self.flex_model
@@ -227,16 +225,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
 
         # Set up peak commitments
         if self.flex_context.get("ems_peak_consumption_price") is not None:
-            ems_peak_consumption = get_continuous_series_sensor_or_quantity(
-                variable_quantity=self.flex_context.get("ems_peak_consumption_in_mw"),
-                actuator=asset,
-                unit="MW",
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                max_value=np.inf,  # np.nan -> np.inf to ignore commitment if no quantity is given
-                fill_sides=True,
-            )
+            ems_peak_consumption = self.flex_context.get("ems_peak_consumption_in_mw")
+            ems_peak_consumption.fillna(
+                np.inf
+            )  # np.nan -> np.inf to ignore commitment if no quantity is given
             ems_peak_consumption_price = self.flex_context.get(
                 "ems_peak_consumption_price"
             )
@@ -252,16 +244,10 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             )
             commitments.append(commitment)
         if self.flex_context.get("ems_peak_production_price") is not None:
-            ems_peak_production = get_continuous_series_sensor_or_quantity(
-                variable_quantity=self.flex_context.get("ems_peak_production_in_mw"),
-                actuator=asset,
-                unit="MW",
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                max_value=np.inf,  # np.nan -> np.inf to ignore commitment if no quantity is given
-                fill_sides=True,
-            )
+            ems_peak_production = self.flex_context.get("ems_peak_production_in_mw")
+            ems_peak_production.fillna(
+                np.inf
+            )  # np.nan -> np.inf to ignore commitment if no quantity is given
             ems_peak_production_price = self.flex_context.get(
                 "ems_peak_production_price"
             )
diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 1a1d6c4d7d..5aa9970d80 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -180,6 +180,7 @@ class FlexContextSchema(Schema):
         data_key="site-peak-consumption",
         value_validator=series_range_validator(min=0),
         default="0 kW",
+        fill_sides=True,
     )
     ems_peak_consumption_price = VariableQuantityField(
         "/MW",
@@ -197,6 +198,7 @@ class FlexContextSchema(Schema):
         data_key="site-peak-production",
         value_validator=series_range_validator(min=0),
         default="0 kW",
+        fill_sides=True,
     )
     ems_peak_production_price = VariableQuantityField(
         "/MW",
@@ -510,6 +512,8 @@ def __init__(self, *args, **kwargs):
                 "ems_power_capacity_in_mw",
                 "ems_consumption_capacity_in_mw",
                 "ems_production_capacity_in_mw",
+                "ems_peak_consumption_in_mw",
+                "ems_peak_production_in_mw",
             ):
                 field.load_time_series = True
             # Compatibility with deprecated fields

From c0acafe4d21f7a0db50cd63a6240c7835f1cf9ec Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 17:55:24 +0200
Subject: [PATCH 149/151] refactor: clean up after moving every
 VariableQuantityField in the flex-context schema

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/schemas/scheduling/__init__.py        | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py
index 5aa9970d80..d003e35365 100644
--- a/flexmeasures/data/schemas/scheduling/__init__.py
+++ b/flexmeasures/data/schemas/scheduling/__init__.py
@@ -498,23 +498,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.load_time_series = True
         for field_var, field in self.declared_fields.items():
-            if isinstance(field, VariableQuantityField) and field_var in (
-                "consumption_breach_price",
-                "production_breach_price",
-                "soc_minima_breach_price",
-                "soc_maxima_breach_price",
-                "ems_consumption_breach_price",
-                "ems_production_breach_price",
-                "ems_peak_consumption_price",
-                "ems_peak_production_price",
-                "consumption_price",
-                "production_price",
-                "ems_power_capacity_in_mw",
-                "ems_consumption_capacity_in_mw",
-                "ems_production_capacity_in_mw",
-                "ems_peak_consumption_in_mw",
-                "ems_peak_production_in_mw",
-            ):
+            if isinstance(field, VariableQuantityField):
                 field.load_time_series = True
             # Compatibility with deprecated fields
             elif field_var in ("consumption_price_sensor", "production_price_sensor"):

From 2093a0b87e7eda386b6757ddfb15f5c3a4646e63 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 18:01:17 +0200
Subject: [PATCH 150/151] fix: bug resolved in
 https://github.com/FlexMeasures/flexmeasures/pull/1433

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_storage.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py
index d970e2fc4f..67b98da607 100644
--- a/flexmeasures/data/models/planning/tests/test_storage.py
+++ b/flexmeasures/data/models/planning/tests/test_storage.py
@@ -140,8 +140,7 @@ def test_battery_relaxation(add_battery_assets, db):
             "soc-minima": [
                 {
                     "start": "2015-01-01T12:00:00+01:00",
-                    "end": "2015-01-01T18:00:00+01:00",
-                    # "duration": "PT6H",  # todo: fails validation, which points to a bug
+                    "duration": "PT6H",
                     "value": "0.8 MWh",
                 }
             ],

From fbd971c7dcad658580c661b6f0c6fcec1d0c0775 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 22 Apr 2025 20:38:44 +0200
Subject: [PATCH 151/151] style: black

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 1607be5387..a91b5b1569 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -520,7 +520,9 @@ def test_fallback_to_unsolvable_problem(
         consumption_schedule = scheduler.compute(skip_validation=True)
 
     # check that the fallback scheduler provides a sensible fallback policy
-    fallback_scheduler: MetaStorageScheduler = scheduler.fallback_scheduler_class(**kwargs)
+    fallback_scheduler: MetaStorageScheduler = scheduler.fallback_scheduler_class(
+        **kwargs
+    )
     fallback_scheduler._load_time_series(charging_station)
     fallback_scheduler.config_deserialized = True
     consumption_schedule = fallback_scheduler.compute(skip_validation=True)