From 00904f47c07f00004da423ba2b336dd9b49a2e97 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 8 Jan 2026 10:35:09 +0100 Subject: [PATCH 01/33] refactor: refactored SensorsToShowSchema to validate into new shape Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 103 +++++++++++++++----- 1 file changed, 78 insertions(+), 25 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 7e1e9b16dc..2c551a1d38 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -83,42 +83,95 @@ def _standardize_item(self, item) -> dict: Standardize different input formats to a consistent dictionary format. """ if isinstance(item, int): - return {"title": None, "sensors": [item]} + return {"title": None, "plots": {"sensor": item}} elif isinstance(item, list): if not all(isinstance(sensor_id, int) for sensor_id in item): raise ValidationError( "All elements in a list within 'sensors_to_show' must be integers." ) - return {"title": None, "sensors": item} + return {"title": None, "plots": {"sensors": item}} elif isinstance(item, dict): - if "title" not in item: - raise ValidationError("Dictionary must contain a 'title' key.") - else: - title = item["title"] - if not isinstance(title, str) and title is not None: - raise ValidationError("'title' value must be a string.") - - if "sensor" in item: - sensor = item["sensor"] - if not isinstance(sensor, int): - raise ValidationError("'sensor' value must be an integer.") - return {"title": title, "sensors": [sensor]} - elif "sensors" in item: - sensors = item["sensors"] - if not isinstance(sensors, list) or not all( - isinstance(sensor_id, int) for sensor_id in sensors - ): - raise ValidationError("'sensors' value must be a list of integers.") - return {"title": title, "sensors": sensors} - else: - raise ValidationError( - "Dictionary must contain either 'sensor' or 'sensors' key." - ) + return self._standardize_dict_item(item) else: raise ValidationError( "Invalid item type in 'sensors_to_show'. Expected int, list, or dict." ) + def _standardize_dict_item(self, item: dict) -> dict: + if "title" not in item: + raise ValidationError("Dictionary must contain a 'title' key.") + + title = item["title"] + if not isinstance(title, str) and title is not None: + raise ValidationError("'title' value must be a string.") + + if "sensor" in item: + sensor = item["sensor"] + if not isinstance(sensor, int): + raise ValidationError("'sensor' value must be an integer.") + return {"title": title, "plots": {"sensor": sensor}} + elif "sensors" in item: + sensors = item["sensors"] + if not isinstance(sensors, list) or not all( + isinstance(sensor_id, int) for sensor_id in sensors + ): + raise ValidationError("'sensors' value must be a list of integers.") + return {"title": title, "plots": {"sensors": sensors}} + elif "plots" in item: + plots = item["plots"] + if not isinstance(plots, list): + raise ValidationError("'plots' must be a list or dictionary.") + + for plot in plots: + self._validate_single_plot(plot) + + return {"title": title, "plots": plots} + else: + raise ValidationError( + "Dictionary must contain either 'sensor' or 'sensors' key." + ) + + def _validate_single_plot(self, plot): + if not isinstance(plot, dict): + raise ValidationError("Each plot in 'plots' must be a dictionary.") + + if "asset" in plot: + self._validate_asset_in_plot(plot) + + if "sensor" not in plot and "sensors" not in plot and "asset" not in plot: + raise ValidationError( + "Each plot must contain either 'sensor', 'sensors' or an 'asset' key." + ) + + def _validate_asset_in_plot(self, plot): + from flexmeasures.data.schemas.scheduling import ( + DBFlexContextSchema, + ) + from flexmeasures.data.schemas.scheduling.storage import ( + DBStorageFlexModelSchema, + ) + + if "flex-context" not in plot or "flex-model" not in plot: + raise ValidationError( + "When 'asset' is provided in a plot, 'flex-context' and 'flex-model' must also be provided." + ) + + self._validate_string_field_in_collection( + plot, "flex-context", DBFlexContextSchema.mapped_schema_keys.values() + ) + self._validate_string_field_in_collection( + plot, "flex-model", DBStorageFlexModelSchema.mapped_schema_keys.values() + ) + + def _validate_string_field_in_collection(self, data, field_name, valid_collection): + if field_name in data: + value = data[field_name] + if not isinstance(value, str): + raise ValidationError(f"'{field_name}' must be a string.") + + if value not in valid_collection: + raise ValidationError(f"'{field_name}' value '{value}' is not valid.") + @classmethod def flatten(cls, nested_list) -> list[int]: """ From b3afe9e4c63e82c83b4a5e45a1a7fb64ea0a500d Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 8 Jan 2026 20:36:15 +0100 Subject: [PATCH 02/33] chore: debugging - work in progress Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 5 +++ flexmeasures/data/schemas/generic_assets.py | 41 +++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index e9a3e460a1..4852b4fab2 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -286,6 +286,7 @@ def validate_sensors_to_show( return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show] sensor_ids_to_show = self.sensors_to_show + print("===========================: sensor_ids_to_show: ", sensor_ids_to_show) # Import the schema for validation from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema @@ -296,7 +297,11 @@ def validate_sensors_to_show( sensor_ids_to_show ) + print("===========================: ") + print("standardized_sensors_to_show: ", standardized_sensors_to_show) + sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show) + print("sensor_id_allowlist: ", sensor_id_allowlist) # Only allow showing sensors from assets owned by the user's organization, # except in play mode, where any sensor may be shown diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 2c551a1d38..2cba19339b 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -97,6 +97,47 @@ def _standardize_item(self, item) -> dict: "Invalid item type in 'sensors_to_show'. Expected int, list, or dict." ) + # def _standardize_item(self, item) -> dict: + # """ + # Standardize different input formats to a consistent dictionary format. + # """ + # if isinstance(item, int): + # return {"title": None, "sensors": [item]} + # elif isinstance(item, list): + # if not all(isinstance(sensor_id, int) for sensor_id in item): + # raise ValidationError( + # "All elements in a list within 'sensors_to_show' must be integers." + # ) + # return {"title": None, "sensors": item} + # elif isinstance(item, dict): + # if "title" not in item: + # raise ValidationError("Dictionary must contain a 'title' key.") + # else: + # title = item["title"] + # if not isinstance(title, str) and title is not None: + # raise ValidationError("'title' value must be a string.") + + # if "sensor" in item: + # sensor = item["sensor"] + # if not isinstance(sensor, int): + # raise ValidationError("'sensor' value must be an integer.") + # return {"title": title, "sensors": [sensor]} + # elif "sensors" in item: + # sensors = item["sensors"] + # if not isinstance(sensors, list) or not all( + # isinstance(sensor_id, int) for sensor_id in sensors + # ): + # raise ValidationError("'sensors' value must be a list of integers.") + # return {"title": title, "sensors": sensors} + # else: + # raise ValidationError( + # "Dictionary must contain either 'sensor' or 'sensors' key." + # ) + # else: + # raise ValidationError( + # "Invalid item type in 'sensors_to_show'. Expected int, list, or dict." + # ) + def _standardize_dict_item(self, item: dict) -> dict: if "title" not in item: raise ValidationError("Dictionary must contain a 'title' key.") From af66726ad3af412f1d2bd6390518c397011f473a Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 8 Jan 2026 21:21:21 +0100 Subject: [PATCH 03/33] chore: debug errors Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 13 ++++++------- flexmeasures/data/schemas/generic_assets.py | 9 +++++---- flexmeasures/utils/coding_utils.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 4852b4fab2..2dc2b494ca 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -286,7 +286,6 @@ def validate_sensors_to_show( return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show] sensor_ids_to_show = self.sensors_to_show - print("===========================: sensor_ids_to_show: ", sensor_ids_to_show) # Import the schema for validation from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema @@ -297,11 +296,7 @@ def validate_sensors_to_show( sensor_ids_to_show ) - print("===========================: ") - print("standardized_sensors_to_show: ", standardized_sensors_to_show) - sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show) - print("sensor_id_allowlist: ", sensor_id_allowlist) # Only allow showing sensors from assets owned by the user's organization, # except in play mode, where any sensor may be shown @@ -329,7 +324,7 @@ def validate_sensors_to_show( for entry in standardized_sensors_to_show: title = entry.get("title") - sensors = entry.get("sensors") + sensors = entry.get("plots", {}).get("sensors") accessible_sensors = [ accessible_sensor_map.get(sid) @@ -339,12 +334,16 @@ def validate_sensors_to_show( inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map] missed_sensor_ids.extend(inaccessible) if accessible_sensors: - sensors_to_show.append({"title": title, "sensors": accessible_sensors}) + sensors_to_show.append( + {"title": title, "plots": [{"sensors": accessible_sensors}]} + ) if missed_sensor_ids: current_app.logger.warning( f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {self}, as it is not accessible to user {current_user}." ) + print("=================== sensors_to_show: ", sensors_to_show) + return sensors_to_show @property diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 2cba19339b..29e2de31c4 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -238,10 +238,11 @@ def flatten(cls, nested_list) -> list[int]: if isinstance(s, list): all_objects.extend(s) elif isinstance(s, dict): - if "sensors" in s: - all_objects.extend(s["sensors"]) - if "sensor" in s: - all_objects.append(s["sensor"]) + if "plots" in s: + if "sensors" in s["plots"]: + all_objects.extend(s["plots"]["sensors"]) + if "sensor" in s["plots"]: + all_objects.append(s["plots"]["sensor"]) else: all_objects.append(s) return list(dict.fromkeys(all_objects).keys()) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 1270d4d516..dd3fce8504 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -1,4 +1,5 @@ -""" Various coding utils (e.g. around function decoration) """ +r"""Various coding utils (e.g. around function decorati +on)""" from __future__ import annotations @@ -93,7 +94,12 @@ def flatten_unique(nested_list_of_objects: list) -> list: if isinstance(s, list): all_objects.extend(s) elif isinstance(s, dict): - all_objects.extend(s["sensors"]) + if "plots" in s: + for entry in s["plots"]: + if "sensors" in entry: + all_objects.extend(entry["sensors"]) + if "sensor" in entry: + all_objects.append(entry["sensor"]) else: all_objects.append(s) return list(dict.fromkeys(all_objects).keys()) From 3219aa3407b3e1d3d03deaaf649780849b529663 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 8 Jan 2026 21:38:15 +0100 Subject: [PATCH 04/33] fix: fixed cahrts failing to render Signed-off-by: joshuaunity --- flexmeasures/data/models/charts/belief_charts.py | 9 ++++++++- flexmeasures/data/models/generic_assets.py | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 5fa2d2a1f9..70572c827b 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -514,7 +514,14 @@ def chart_for_multiple_sensors( title = entry.get("title") if title == "Charge Point sessions": continue - sensors = entry.get("sensors") + plots = entry.get("plots", []) + sensors = [] + for plot in plots: + if "sensors" in plot: + sensors.extend(plot.get("sensors")) + elif "sensor" in plot: + sensors.extend([plot.get("sensor")]) + # List the sensors that go into one row row_sensors: list["Sensor"] = sensors # noqa F821 diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 2dc2b494ca..ab78bb4d16 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -342,8 +342,6 @@ def validate_sensors_to_show( current_app.logger.warning( f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {self}, as it is not accessible to user {current_user}." ) - print("=================== sensors_to_show: ", sensors_to_show) - return sensors_to_show @property From 16111342ded757f31d84cf2a7b8b2cb9c3f08216 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 12 Jan 2026 13:29:17 +0100 Subject: [PATCH 05/33] chore: udpate test case with new schema changes Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 8 ++++---- flexmeasures/data/tests/test_SensorsToShowSchema.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 29e2de31c4..efe97ab130 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -83,13 +83,13 @@ def _standardize_item(self, item) -> dict: Standardize different input formats to a consistent dictionary format. """ if isinstance(item, int): - return {"title": None, "plots": {"sensor": item}} + return {"title": None, "plots": [{"sensor": item}]} elif isinstance(item, list): if not all(isinstance(sensor_id, int) for sensor_id in item): raise ValidationError( "All elements in a list within 'sensors_to_show' must be integers." ) - return {"title": None, "plots": {"sensors": item}} + return {"title": None, "plots": [{"sensors": item}]} elif isinstance(item, dict): return self._standardize_dict_item(item) else: @@ -150,14 +150,14 @@ def _standardize_dict_item(self, item: dict) -> dict: sensor = item["sensor"] if not isinstance(sensor, int): raise ValidationError("'sensor' value must be an integer.") - return {"title": title, "plots": {"sensor": sensor}} + return {"title": title, "plots": [{"sensor": sensor}]} elif "sensors" in item: sensors = item["sensors"] if not isinstance(sensors, list) or not all( isinstance(sensor_id, int) for sensor_id in sensors ): raise ValidationError("'sensors' value must be a list of integers.") - return {"title": title, "plots": {"sensors": sensors}} + return {"title": title, "plots": [{"sensors": sensors}]} elif "plots" in item: plots = item["plots"] if not isinstance(plots, list): diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index c5eb9919e3..ac0d274710 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -84,8 +84,8 @@ def test_string_json_input(): '[{"title": "Test", "sensors": [1, 2]}, {"title": "Test2", "sensors": [3]}]' ) expected_output = [ - {"title": "Test", "sensors": [1, 2]}, - {"title": "Test2", "sensors": [3]}, + {"title": "Test", "plots": [{"sensors": [1, 2]}]}, + {"title": "Test2", "plots": [{"sensors": [3]}]}, ] assert schema.deserialize(input_value) == expected_output From fbb9791aab0ac438c3aca4c80ada19caa0f3d611 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 12 Jan 2026 13:48:54 +0100 Subject: [PATCH 06/33] tests: adapting more testcases to new schema shape Signed-off-by: joshuaunity --- .../data/tests/test_SensorsToShowSchema.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index ac0d274710..8d17966cd7 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -7,7 +7,7 @@ def test_single_sensor_id(): schema = SensorsToShowSchema() input_value = [42] - expected_output = [{"title": None, "sensors": [42]}] + expected_output = [{"title": None, "plots": [{"sensor": 42}]}] assert schema.deserialize(input_value) == expected_output @@ -15,23 +15,23 @@ def test_list_of_sensor_ids(): schema = SensorsToShowSchema() input_value = [42, 43] expected_output = [ - {"title": None, "sensors": [42]}, - {"title": None, "sensors": [43]}, + {"title": None, "plots": [{"sensor": 42}]}, + {"title": None, "plots": [{"sensor": 43}]}, ] assert schema.deserialize(input_value) == expected_output def test_dict_with_title_and_single_sensor(): schema = SensorsToShowSchema() - input_value = [{"title": "Temperature", "sensor": 42}] - expected_output = [{"title": "Temperature", "sensors": [42]}] + input_value = [{"title": "Temperature", "plots": [{"sensor": 42}]}] + expected_output = [{"title": "Temperature", "plots": [{"sensor": 42}]}] assert schema.deserialize(input_value) == expected_output def test_dict_with_title_and_multiple_sensors(): schema = SensorsToShowSchema() - input_value = [{"title": "Pressure", "sensors": [42, 43]}] - expected_output = [{"title": "Pressure", "sensors": [42, 43]}] + input_value = [{"title": "Pressure", "plots": [{"sensors": [42, 43]}]}] + expected_output = [{"title": "Pressure", "plots": [{"sensors": [42, 43]}]}] assert schema.deserialize(input_value) == expected_output @@ -71,9 +71,9 @@ def test_mixed_valid_inputs(): 5, ] expected_output = [ - {"title": "Test", "sensors": [1, 2]}, - {"title": None, "sensors": [3, 4]}, - {"title": None, "sensors": [5]}, + {"title": "Test", "plots": [{"sensors": [1, 2]}]}, + {"title": None, "plots": [{"sensors": [3, 4]}]}, + {"title": None, "plots": [{"sensor": 5}]}, ] assert schema.deserialize(input_value) == expected_output From e6babb829a1e8203b4813b6021d1f55cbfcbfbf1 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 12 Jan 2026 16:56:12 +0100 Subject: [PATCH 07/33] fix: fix failing test Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index efe97ab130..2d66648d1d 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -239,10 +239,11 @@ def flatten(cls, nested_list) -> list[int]: all_objects.extend(s) elif isinstance(s, dict): if "plots" in s: - if "sensors" in s["plots"]: - all_objects.extend(s["plots"]["sensors"]) - if "sensor" in s["plots"]: - all_objects.append(s["plots"]["sensor"]) + for plot in s["plots"]: + if "sensors" in plot: + all_objects.extend(plot["sensors"]) + if "sensor" in plot: + all_objects.append(plot["sensor"]) else: all_objects.append(s) return list(dict.fromkeys(all_objects).keys()) From a1fd936518526a039486992877f831bbd101ce1c Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 13 Jan 2026 10:53:32 +0100 Subject: [PATCH 08/33] fix: fixed failing api due to logic oversight Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index ab78bb4d16..382f0716dd 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -324,7 +324,14 @@ def validate_sensors_to_show( for entry in standardized_sensors_to_show: title = entry.get("title") - sensors = entry.get("plots", {}).get("sensors") + sensors = [] + plots = entry.get("plots", []) + if len(plots) > 0: + for plot in plots: + if "sensor" in plot: + sensors.append(plot["sensor"]) + if "sensors" in plot: + sensors.extend(plot["sensors"]) accessible_sensors = [ accessible_sensor_map.get(sid) From db3a1fe74e508d2e32088b533dd2cab41caff693 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 14 Jan 2026 10:01:53 +0100 Subject: [PATCH 09/33] fix: handle asset plot entry Signed-off-by: joshuaunity --- flexmeasures/api/v3_0/assets.py | 1 + flexmeasures/data/models/generic_assets.py | 4 ++ flexmeasures/data/schemas/generic_assets.py | 71 ++++++--------------- flexmeasures/data/schemas/utils.py | 38 +++++++++++ 4 files changed, 64 insertions(+), 50 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 4182e7b4ff..cdd8d6212a 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -870,6 +870,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): - Assets """ sensors = flatten_unique(asset.validate_sensors_to_show()) + print("==========================", sensors) return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) @route("//auditlog") diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 382f0716dd..3128045892 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -287,6 +287,7 @@ def validate_sensors_to_show( sensor_ids_to_show = self.sensors_to_show # Import the schema for validation + from flexmeasures.data.schemas.utils import extract_sensors_from_flex_config from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema sensors_to_show_schema = SensorsToShowSchema() @@ -332,6 +333,9 @@ def validate_sensors_to_show( sensors.append(plot["sensor"]) if "sensors" in plot: sensors.extend(plot["sensors"]) + if "asset" in plot: + extracted_sensors = extract_sensors_from_flex_config(plot) + sensors.extend(extracted_sensors) accessible_sensors = [ accessible_sensor_map.get(sid) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 2d66648d1d..c7836a2139 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -19,6 +19,7 @@ from flexmeasures.data.schemas.utils import ( FMValidationError, MarshmallowClickMixin, + extract_sensors_from_flex_config, ) from flexmeasures.auth.policy import user_has_admin_access from flexmeasures.cli import is_running as running_as_cli @@ -97,47 +98,6 @@ def _standardize_item(self, item) -> dict: "Invalid item type in 'sensors_to_show'. Expected int, list, or dict." ) - # def _standardize_item(self, item) -> dict: - # """ - # Standardize different input formats to a consistent dictionary format. - # """ - # if isinstance(item, int): - # return {"title": None, "sensors": [item]} - # elif isinstance(item, list): - # if not all(isinstance(sensor_id, int) for sensor_id in item): - # raise ValidationError( - # "All elements in a list within 'sensors_to_show' must be integers." - # ) - # return {"title": None, "sensors": item} - # elif isinstance(item, dict): - # if "title" not in item: - # raise ValidationError("Dictionary must contain a 'title' key.") - # else: - # title = item["title"] - # if not isinstance(title, str) and title is not None: - # raise ValidationError("'title' value must be a string.") - - # if "sensor" in item: - # sensor = item["sensor"] - # if not isinstance(sensor, int): - # raise ValidationError("'sensor' value must be an integer.") - # return {"title": title, "sensors": [sensor]} - # elif "sensors" in item: - # sensors = item["sensors"] - # if not isinstance(sensors, list) or not all( - # isinstance(sensor_id, int) for sensor_id in sensors - # ): - # raise ValidationError("'sensors' value must be a list of integers.") - # return {"title": title, "sensors": sensors} - # else: - # raise ValidationError( - # "Dictionary must contain either 'sensor' or 'sensors' key." - # ) - # else: - # raise ValidationError( - # "Invalid item type in 'sensors_to_show'. Expected int, list, or dict." - # ) - def _standardize_dict_item(self, item: dict) -> dict: if "title" not in item: raise ValidationError("Dictionary must contain a 'title' key.") @@ -176,14 +136,24 @@ def _validate_single_plot(self, plot): if not isinstance(plot, dict): raise ValidationError("Each plot in 'plots' must be a dictionary.") - if "asset" in plot: - self._validate_asset_in_plot(plot) - if "sensor" not in plot and "sensors" not in plot and "asset" not in plot: raise ValidationError( "Each plot must contain either 'sensor', 'sensors' or an 'asset' key." ) + if "asset" in plot: + self._validate_asset_in_plot(plot) + if "sensor" in plot: + sensor = plot["sensor"] + if not isinstance(sensor, int): + raise ValidationError("'sensor' value must be an integer.") + if "sensors" in plot: + sensors = plot["sensors"] + if not isinstance(sensors, list) or not all( + isinstance(sensor_id, int) for sensor_id in sensors + ): + raise ValidationError("'sensors' value must be a list of integers.") + def _validate_asset_in_plot(self, plot): from flexmeasures.data.schemas.scheduling import ( DBFlexContextSchema, @@ -192,7 +162,7 @@ def _validate_asset_in_plot(self, plot): DBStorageFlexModelSchema, ) - if "flex-context" not in plot or "flex-model" not in plot: + if "flex-context" not in plot and "flex-model" not in plot: raise ValidationError( "When 'asset' is provided in a plot, 'flex-context' and 'flex-model' must also be provided." ) @@ -201,7 +171,7 @@ def _validate_asset_in_plot(self, plot): plot, "flex-context", DBFlexContextSchema.mapped_schema_keys.values() ) self._validate_string_field_in_collection( - plot, "flex-model", DBStorageFlexModelSchema.mapped_schema_keys.values() + plot, "flex-model", DBStorageFlexModelSchema().mapped_schema_keys.values() ) def _validate_string_field_in_collection(self, data, field_name, valid_collection): @@ -221,7 +191,7 @@ def flatten(cls, nested_list) -> list[int]: This method processes the following formats, for each of the entries of the nested list: - A list of sensor IDs: `[1, 2, 3]` - A list of dictionaries where each dictionary contains a `sensors` list or a `sensor` key: - `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}]` + `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, {"title": "Pressure", "plots": [{"sensor": 4}, {"sensors": [5,6]}]}]` - Mixed formats: `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, 4, 5, 1]` It extracts all sensor IDs, removes duplicates, and returns a flattened list of unique sensor IDs. @@ -232,7 +202,6 @@ def flatten(cls, nested_list) -> list[int]: Returns: list: A unique list of sensor IDs. """ - all_objects = [] for s in nested_list: if isinstance(s, list): @@ -244,8 +213,10 @@ def flatten(cls, nested_list) -> list[int]: all_objects.extend(plot["sensors"]) if "sensor" in plot: all_objects.append(plot["sensor"]) - else: - all_objects.append(s) + if "asset" in plot: + sensors = extract_sensors_from_flex_config(plot) + all_objects.extend(sensors) + return list(dict.fromkeys(all_objects).keys()) diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index b8ff581b75..26f35169c5 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -1,5 +1,6 @@ import click import marshmallow as ma +from typing import List from click import get_current_context from flask.cli import with_appcontext as with_cli_appcontext from pint import DefinitionSyntaxError, DimensionalityError, UndefinedUnitError @@ -83,3 +84,40 @@ def convert_to_quantity(value: str, to_unit: str) -> ur.Quantity: raise FMValidationError( f"Cannot convert value '{value}' to a valid quantity. {e}" ) + + +def extract_sensors_from_flex_config(plot: dict) -> List: + """ + Extracts a consolidated list of sensors from an asset based on + flex-context or flex-model definitions provided in a plot dictionary. + """ + all_sensors = [] + + from flexmeasures.data.schemas.generic_assets import ( + GenericAssetIdField, + ) # Import here to avoid circular imports + + asset = GenericAssetIdField().deserialize(plot.get("asset")) + + fields_to_check = { + "flex-context": asset.flex_context, + "flex-model": asset.flex_model, + } + + for plot_key, flex_config in fields_to_check.items(): + if plot_key in plot: + field_key = plot[plot_key] + data = flex_config or {} + field_value = data.get(field_key) + + if isinstance(field_value, dict): + # Add multiple sensors if they exist as a list + sensors = field_value.get("sensors", []) + all_sensors.extend(sensors) + + # Add a single sensor if it exists + sensor = field_value.get("sensor") + if sensor: + all_sensors.append(sensor) + + return all_sensors From a6dcc6e041a591199f44ea703a0ceb72479c5423 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 19 Jan 2026 16:56:47 +0100 Subject: [PATCH 10/33] chore: add changelog entry Signed-off-by: joshuaunity --- documentation/changelog.rst | 4 +++- flexmeasures/api/v3_0/assets.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 363ddd686a..b5807f6729 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -20,12 +20,14 @@ New features * Give ability to edit sensor timezone from the UI [see `PR #1900 `_] * Support creating schedules with only information known prior to some time, now also via the CLI (the API already supported it) [see `PR #1871 `_]. * Add ``fields`` param to the asset-listing endpoints, to save bandwidth in response data [see `PR #1884 `_] +* Support for flex-config in the ``SensorsToShowSchema`` [see `PR #1917 `_] + .. note:: For backwards-compatibility, the new ``fields`` parameter will only be fully active, i.e. also returning less fields per default, in v0.32. Set ``FLEXMEASURES_API_SUNSET_ACTIVE=True`` to test the full effect now. * Allow testing out the scheduling CLI without saving anything, using ``flexmeasures add schedule --dry-run`` [see `PR #1892 `_] * Allow unsupported ``flex-context`` or ``flex-model`` fields to be shown in the UI editors (they will be un-editable) [see `PR #1915 `_] -* Add back save buttons to both ``flex-context`` and ``flex-model`` UI editors [see `PR #1916 `_] +* Add back save buttons to both ``flex-context`` and ``flex-model`` UI editors [see `PR #1880 `_] Infrastructure / Support ---------------------- diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 853e48111b..d1cf46fa09 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -898,7 +898,6 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): - Assets """ sensors = flatten_unique(asset.validate_sensors_to_show()) - print("==========================", sensors) return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) @route("//auditlog") From 6fd81be374b68d95acc261475d16115d6ab48ad5 Mon Sep 17 00:00:00 2001 From: JDev <45713692+joshuaunity@users.noreply.github.com> Date: Tue, 20 Jan 2026 11:19:18 +0100 Subject: [PATCH 11/33] Update documentation/changelog.rst Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com> --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b5807f6729..0ba5ebc517 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -20,7 +20,7 @@ New features * Give ability to edit sensor timezone from the UI [see `PR #1900 `_] * Support creating schedules with only information known prior to some time, now also via the CLI (the API already supported it) [see `PR #1871 `_]. * Add ``fields`` param to the asset-listing endpoints, to save bandwidth in response data [see `PR #1884 `_] -* Support for flex-config in the ``SensorsToShowSchema`` [see `PR #1917 `_] +* Support for flex-config in the ``SensorsToShowSchema`` [see `PR #1904 `_] .. note:: For backwards-compatibility, the new ``fields`` parameter will only be fully active, i.e. also returning less fields per default, in v0.32. Set ``FLEXMEASURES_API_SUNSET_ACTIVE=True`` to test the full effect now. From d257c9485b9ce9417bfdebf89242003d59ded735 Mon Sep 17 00:00:00 2001 From: JDev <45713692+joshuaunity@users.noreply.github.com> Date: Mon, 26 Jan 2026 21:59:04 +0100 Subject: [PATCH 12/33] Update flexmeasures/data/schemas/utils.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolas Höning Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/data/schemas/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index 26f35169c5..52ffb65cc7 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -86,7 +86,7 @@ def convert_to_quantity(value: str, to_unit: str) -> ur.Quantity: ) -def extract_sensors_from_flex_config(plot: dict) -> List: +def extract_sensors_from_flex_config(plot: dict) -> list[Sensor]: """ Extracts a consolidated list of sensors from an asset based on flex-context or flex-model definitions provided in a plot dictionary. From 6e650de73e317f3bc56b9e9445066b15deb06f3c Mon Sep 17 00:00:00 2001 From: JDev <45713692+joshuaunity@users.noreply.github.com> Date: Mon, 26 Jan 2026 21:59:52 +0100 Subject: [PATCH 13/33] Update flexmeasures/data/schemas/generic_assets.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolas Höning Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/data/schemas/generic_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index c7836a2139..a2ba9d9e74 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -164,7 +164,7 @@ def _validate_asset_in_plot(self, plot): if "flex-context" not in plot and "flex-model" not in plot: raise ValidationError( - "When 'asset' is provided in a plot, 'flex-context' and 'flex-model' must also be provided." + "When 'asset' is provided in a plot, 'flex-context' or 'flex-model' must also be provided." ) self._validate_string_field_in_collection( From 24cc4a2dcb9ceeef5e3979a2b30ca060ef2911c2 Mon Sep 17 00:00:00 2001 From: JDev <45713692+joshuaunity@users.noreply.github.com> Date: Mon, 26 Jan 2026 22:00:18 +0100 Subject: [PATCH 14/33] Update flexmeasures/data/schemas/generic_assets.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolas Höning Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/data/schemas/generic_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index a2ba9d9e74..703f8114d5 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -178,7 +178,7 @@ def _validate_string_field_in_collection(self, data, field_name, valid_collectio if field_name in data: value = data[field_name] if not isinstance(value, str): - raise ValidationError(f"'{field_name}' must be a string.") + raise ValidationError(f"The value for '{field_name}' must be a string.") if value not in valid_collection: raise ValidationError(f"'{field_name}' value '{value}' is not valid.") From 328aba73059178619beb1fbb1b11d2d8011d1741 Mon Sep 17 00:00:00 2001 From: JDev <45713692+joshuaunity@users.noreply.github.com> Date: Mon, 26 Jan 2026 22:00:32 +0100 Subject: [PATCH 15/33] Update flexmeasures/data/schemas/generic_assets.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolas Höning Signed-off-by: JDev <45713692+joshuaunity@users.noreply.github.com> --- flexmeasures/data/schemas/generic_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 703f8114d5..659b1336dc 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -174,7 +174,7 @@ def _validate_asset_in_plot(self, plot): plot, "flex-model", DBStorageFlexModelSchema().mapped_schema_keys.values() ) - def _validate_string_field_in_collection(self, data, field_name, valid_collection): + def _validate_flex_config_field_is_valid_choice(self, plot_config, field_name, valid_collection): if field_name in data: value = data[field_name] if not isinstance(value, str): From 9d25020a398f159b9997cc4b342193f6db37c129 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 27 Jan 2026 10:34:28 +0100 Subject: [PATCH 16/33] fix: fixed schema bugs Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 13 +++++++------ flexmeasures/data/schemas/utils.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 659b1336dc..fa486ab99b 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -122,7 +122,6 @@ def _standardize_dict_item(self, item: dict) -> dict: plots = item["plots"] if not isinstance(plots, list): raise ValidationError("'plots' must be a list or dictionary.") - for plot in plots: self._validate_single_plot(plot) @@ -167,16 +166,18 @@ def _validate_asset_in_plot(self, plot): "When 'asset' is provided in a plot, 'flex-context' or 'flex-model' must also be provided." ) - self._validate_string_field_in_collection( + self._validate_flex_config_field_is_valid_choice( plot, "flex-context", DBFlexContextSchema.mapped_schema_keys.values() ) - self._validate_string_field_in_collection( + self._validate_flex_config_field_is_valid_choice( plot, "flex-model", DBStorageFlexModelSchema().mapped_schema_keys.values() ) - def _validate_flex_config_field_is_valid_choice(self, plot_config, field_name, valid_collection): - if field_name in data: - value = data[field_name] + def _validate_flex_config_field_is_valid_choice( + self, plot_config, field_name, valid_collection + ): + if field_name in plot_config: + value = plot_config[field_name] if not isinstance(value, str): raise ValidationError(f"The value for '{field_name}' must be a string.") diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index 52ffb65cc7..7b3f46db8c 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -1,11 +1,11 @@ import click import marshmallow as ma -from typing import List from click import get_current_context from flask.cli import with_appcontext as with_cli_appcontext from pint import DefinitionSyntaxError, DimensionalityError, UndefinedUnitError from flexmeasures.utils.unit_utils import to_preferred, ur +from flexmeasures.data.models.time_series import Sensor class MarshmallowClickMixin(click.ParamType): From a6596bc8013798a42448df488b2671f30770025f Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 27 Jan 2026 15:42:13 +0100 Subject: [PATCH 17/33] fix: fixed skipped validation step Signed-off-by: joshuaunity --- flexmeasures/data/services/generic_assets.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/services/generic_assets.py b/flexmeasures/data/services/generic_assets.py index ac8bd00e1a..3025b171d4 100644 --- a/flexmeasures/data/services/generic_assets.py +++ b/flexmeasures/data/services/generic_assets.py @@ -1,3 +1,4 @@ +from dictdiffer import diff from flask import current_app from sqlalchemy import delete @@ -6,7 +7,7 @@ from flexmeasures.data.models.audit_log import AssetAuditLog from flexmeasures.data.schemas.scheduling import DBFlexContextSchema from flexmeasures.data.schemas.scheduling.storage import DBStorageFlexModelSchema -from dictdiffer import diff +from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema """Services for managing assets""" @@ -120,6 +121,7 @@ def patch_asset(db_asset: GenericAsset, asset_data: dict) -> GenericAsset: schema_map = dict( flex_context=DBFlexContextSchema, flex_model=DBStorageFlexModelSchema, + sensors_to_show=SensorsToShowSchema, ) for k, v in asset_data.items(): @@ -135,7 +137,10 @@ def patch_asset(db_asset: GenericAsset, asset_data: dict) -> GenericAsset: continue if k in schema_map: # Validate the JSON field against the given schema - schema_map[k]().load(v) + if k != "sensors_to_show": + schema_map[k]().load(v) + else: + schema_map[k]().deserialize(v) if k.lower() in {"sensors_to_show", "flex_context", "flex_model"}: audit_log_data.append(format_json_field_change(k, getattr(db_asset, k), v)) From 8c01eaeb473a8f8f0cf1fd32cebf2913003f6eee Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 28 Jan 2026 07:47:27 +0100 Subject: [PATCH 18/33] chore: add docstring to schema functions Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 46 ++++++++++++++++--- .../data/tests/test_SensorsToShowSchema.py | 9 ++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index fa486ab99b..8233809184 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -81,7 +81,7 @@ def deserialize(self, value, **kwargs) -> list: def _standardize_item(self, item) -> dict: """ - Standardize different input formats to a consistent dictionary format. + Normalize various input formats (int, list, or dict) into a standard plot dictionary. """ if isinstance(item, int): return {"title": None, "plots": [{"sensor": item}]} @@ -99,12 +99,16 @@ def _standardize_item(self, item) -> dict: ) def _standardize_dict_item(self, item: dict) -> dict: - if "title" not in item: - raise ValidationError("Dictionary must contain a 'title' key.") + """ + Transform a dictionary-based sensor configuration into a standardized 'plots' structure. + Ensures 'title' is a string and processes 'sensor', 'sensors', or direct 'plots' keys. + """ + title = None - title = item["title"] - if not isinstance(title, str) and title is not None: - raise ValidationError("'title' value must be a string.") + if "title" in item: + title = item["title"] + if not isinstance(title, str) and title is not None: + title = None if "sensor" in item: sensor = item["sensor"] @@ -132,6 +136,10 @@ def _standardize_dict_item(self, item: dict) -> dict: ) def _validate_single_plot(self, plot): + """ + Perform structural validation on an individual plot dictionary. + Requires at least one of: 'sensor', 'sensors', or 'asset'. + """ if not isinstance(plot, dict): raise ValidationError("Each plot in 'plots' must be a dictionary.") @@ -154,6 +162,10 @@ def _validate_single_plot(self, plot): raise ValidationError("'sensors' value must be a list of integers.") def _validate_asset_in_plot(self, plot): + """ + Validate plots that reference a GenericAsset. + Ensures flex-config schemas are respected when an asset is provided. + """ from flexmeasures.data.schemas.scheduling import ( DBFlexContextSchema, ) @@ -176,14 +188,34 @@ def _validate_asset_in_plot(self, plot): def _validate_flex_config_field_is_valid_choice( self, plot_config, field_name, valid_collection ): + """ + Verify that the chosen flex-config field exists on the specific asset and matches + allowed schema keys. + """ if field_name in plot_config: value = plot_config[field_name] - if not isinstance(value, str): + asset_id = plot_config.get("asset") + asset = GenericAssetIdField().deserialize(asset_id) + + if asset is None: + raise ValidationError(f"Asset with ID {asset_id} does not exist.") + + if value and not isinstance(value, str): raise ValidationError(f"The value for '{field_name}' must be a string.") if value not in valid_collection: raise ValidationError(f"'{field_name}' value '{value}' is not valid.") + attr_to_check = ( + "flex_model" if field_name == "flex-model" else "flex_context" + ) + asset_flex_config = getattr(asset, attr_to_check, {}) + + if value not in asset_flex_config: + raise ValidationError( + f"The asset with ID '{asset_id}' does not have a '{value}' set in its '{attr_to_check}'." + ) + @classmethod def flatten(cls, nested_list) -> list[int]: """ diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index 8d17966cd7..d96a6b55a1 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -35,6 +35,15 @@ def test_dict_with_title_and_multiple_sensors(): assert schema.deserialize(input_value) == expected_output +def test_dict_with_asset_and_no_title_plot(): + schema = SensorsToShowSchema() + input_value = [{"plots": [{"asset": 44, "flex-model": "soc-min"}]}] + expected_output = [ + {"title": None, "plots": [{"asset": 44, "flex-model": "soc-min"}]} + ] + assert schema.deserialize(input_value) == expected_output + + def test_invalid_sensor_string_input(): schema = SensorsToShowSchema() with pytest.raises( From 39d03c424883f64e657857b27515abd7fa967370 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 28 Jan 2026 10:24:02 +0100 Subject: [PATCH 19/33] chore: little changes Signed-off-by: joshuaunity --- flexmeasures/data/models/generic_assets.py | 6 +++--- flexmeasures/data/schemas/generic_assets.py | 4 ++-- flexmeasures/data/services/generic_assets.py | 1 + flexmeasures/data/tests/test_SensorsToShowSchema.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 3128045892..46bb19383c 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -234,7 +234,7 @@ def validate_sensors_to_show( Steps: - The function deserializes the 'sensors_to_show' data from the database, ensuring that older formats are parsed correctly. - It checks if each sensor is accessible by the user and filters out any unauthorized sensors. - - The sensor structure is rebuilt according to the latest format, which allows for grouping sensors and adding optional titles. + - The sensor structure is rebuilt according to the latest format, which allows for grouping sensors and adding optional titles, all into a single 'plots' array. Details on format: - The 'sensors_to_show' attribute is defined as a list of sensor IDs or nested lists of sensor IDs (to indicate grouping). @@ -247,8 +247,8 @@ def validate_sensors_to_show( 2. List with titles and sensor groupings: sensors_to_show = [ - {"title": "Title 1", "sensor": 40}, - {"title": "Title 2", "sensors": [41, 42]}, + {"title": "Title 1", "plots": [{"sensor": 40}]}, + {"title": "Title 2", "plots": [{"sensors": [41, 42]}]}, [43, 44], 45, 46 ] diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 8233809184..4d28703e38 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -132,7 +132,7 @@ def _standardize_dict_item(self, item: dict) -> dict: return {"title": title, "plots": plots} else: raise ValidationError( - "Dictionary must contain either 'sensor' or 'sensors' key." + "Dictionary must contain either 'sensor', 'sensors' or 'plots' key." ) def _validate_single_plot(self, plot): @@ -223,7 +223,7 @@ def flatten(cls, nested_list) -> list[int]: This method processes the following formats, for each of the entries of the nested list: - A list of sensor IDs: `[1, 2, 3]` - - A list of dictionaries where each dictionary contains a `sensors` list or a `sensor` key: + - A list of dictionaries where each dictionary contains a `sensors` list, a `sensor` key or a `plots` key `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, {"title": "Pressure", "plots": [{"sensor": 4}, {"sensors": [5,6]}]}]` - Mixed formats: `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, 4, 5, 1]` diff --git a/flexmeasures/data/services/generic_assets.py b/flexmeasures/data/services/generic_assets.py index 3025b171d4..951206ddc5 100644 --- a/flexmeasures/data/services/generic_assets.py +++ b/flexmeasures/data/services/generic_assets.py @@ -140,6 +140,7 @@ def patch_asset(db_asset: GenericAsset, asset_data: dict) -> GenericAsset: if k != "sensors_to_show": schema_map[k]().load(v) else: + # we use `deserialize here because the `SensorsToShowSchema` is a "fields.Field" object rather than a "Schema" object schema_map[k]().deserialize(v) if k.lower() in {"sensors_to_show", "flex_context", "flex_model"}: diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index d96a6b55a1..275fe102a4 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -67,7 +67,7 @@ def test_invalid_sensor_dict_without_sensors_key(): input_value = [{"title": "Test", "something_else": 42}] with pytest.raises( ValidationError, - match="Dictionary must contain either 'sensor' or 'sensors' key.", + match="Dictionary must contain either 'sensor', 'sensors' or 'plots' key.", ): schema.deserialize(input_value) From cf0f18046d9a6521785800d4c195494a657123cf Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 28 Jan 2026 10:39:48 +0100 Subject: [PATCH 20/33] tests: expanding test case Signed-off-by: joshuaunity --- flexmeasures/data/tests/test_SensorsToShowSchema.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index 275fe102a4..9407e58cee 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -23,9 +23,11 @@ def test_list_of_sensor_ids(): def test_dict_with_title_and_single_sensor(): schema = SensorsToShowSchema() - input_value = [{"title": "Temperature", "plots": [{"sensor": 42}]}] + input_value_one = [{"title": "Temperature", "sensor": 42}] + input_value_two = [{"title": "Temperature", "plots": [{"sensor": 42}]}] expected_output = [{"title": "Temperature", "plots": [{"sensor": 42}]}] - assert schema.deserialize(input_value) == expected_output + assert schema.deserialize(input_value_one) == expected_output + assert schema.deserialize(input_value_two) == expected_output def test_dict_with_title_and_multiple_sensors(): From 41c9e8da43eff857de28a2004787c497e714c5a5 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 28 Jan 2026 11:21:58 +0100 Subject: [PATCH 21/33] refactor: more backward compatibility refactoring Signed-off-by: joshuaunity --- flexmeasures/data/schemas/utils.py | 4 ---- flexmeasures/utils/coding_utils.py | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index 7b3f46db8c..235b3dd103 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -111,10 +111,6 @@ def extract_sensors_from_flex_config(plot: dict) -> list[Sensor]: field_value = data.get(field_key) if isinstance(field_value, dict): - # Add multiple sensors if they exist as a list - sensors = field_value.get("sensors", []) - all_sensors.extend(sensors) - # Add a single sensor if it exists sensor = field_value.get("sensor") if sensor: diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index dd3fce8504..c31389b3ac 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -94,7 +94,11 @@ def flatten_unique(nested_list_of_objects: list) -> list: if isinstance(s, list): all_objects.extend(s) elif isinstance(s, dict): - if "plots" in s: + if "sensors" in s: + all_objects.extend(s["sensors"]) + elif "sensor" in s: + all_objects.append(s["sensor"]) + elif "plots" in s: for entry in s["plots"]: if "sensors" in entry: all_objects.extend(entry["sensors"]) From 577459ba71e2b7a30f2b5a4523bc193740bef65b Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Wed, 28 Jan 2026 11:39:46 +0100 Subject: [PATCH 22/33] chore: update docstring for SensorsToShowSchema Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 4d28703e38..2d24cca1d0 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -42,23 +42,24 @@ class SensorsToShowSchema(fields.Field): The `sensors_to_show` attribute defines which sensors should be displayed for a particular asset. It supports various input formats, which are standardized into a list of dictionaries, each containing - a `title` (optional) and a `sensors` list. The valid input formats include: + a `title` (optional) and a `plots` list, this list then consist of dictionaries with keys such as `sensor`, `asset` or `sensors`. - - A single sensor ID (int): `42` -> `{"title": None, "sensors": [42]}` - - A list of sensor IDs (list of ints): `[42, 43]` -> `{"title": None, "sensors": [42, 43]}` - - A dictionary with a title and sensor: `{"title": "Temperature", "sensor": 42}` -> `{"title": "Temperature", "sensors": [42]}` - - A dictionary with a title and sensors: `{"title": "Pressure", "sensors": [42, 43]}` + - A single sensor ID (int): `42` -> `{"title": None, "plots": [{"sensor": 42}]}` + - A list of sensor IDs (list of ints): `[42, 43]` -> `{"title": None, "plots": [{"sensors": [42, 43]}]}` + - A dictionary with a title and sensor: `{"title": "Temperature", "sensor": 42}` -> `{"title": "Temperature", "plots": [{"sensor": 42}]}` + - A dictionary with a title and sensors: `{"title": "Pressure", "sensors": [42, 43]}` -> `{"title": "Pressure", "plots": [{"sensors": [42, 43]}]}` Validation ensures that: - The input is either a list, integer, or dictionary. - - If the input is a dictionary, it must contain either `sensor` (int) or `sensors` (list of ints). + - If the input is a dictionary, it must contain either `sensor` (int), `sensors` (list of ints) or `plots` (list of dicts). - All sensor IDs must be valid integers. - Example Input: - - `[{"title": "Test", "sensors": [1, 2]}, {"title": None, "sensors": [3, 4]}, 5]` + Example Inputs: + - `[{"title": "Test", "plots": [{"sensor": 1}, {"sensor": 2}]}, {"title": "Another Test", "plots": [{"sensors": [3, 4]}]}, 5]` + - `[{"title": "Test", "sensors": [1, 2]}, {"title": None, "sensors": [3, 4]}, 5]` (Older format but still compatible) Example Output (Standardized): - - `[{"title": "Test", "sensors": [1, 2]}, {"title": None, "sensors": [3, 4]}, {"title": None, "sensors": [5]}]` + - `[{"title": "Test", "plots": [{"sensors": [1, 2]}]}, {"title": None, "plots": [{"sensors": [3, 4]}]}, {"title": None, "plots": [{"sensor": 5}]}]` """ def deserialize(self, value, **kwargs) -> list: From 68b2ac9142a4aed7c518310164dd2624193daed1 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 29 Jan 2026 14:38:47 +0100 Subject: [PATCH 23/33] refactor: support for old sensor to show format for flatenen functions Signed-off-by: joshuaunity --- documentation/views/asset-data.rst | 9 +++++---- flexmeasures/data/schemas/generic_assets.py | 6 ++++++ flexmeasures/utils/coding_utils.py | 10 ++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst index fe3d1a497d..61be23daa9 100644 --- a/documentation/views/asset-data.rst +++ b/documentation/views/asset-data.rst @@ -108,11 +108,11 @@ Editing the graphs dashboard Click the "Edit Graph" button to open the graph editor. -Use the "Add Graph" button to create graphs. For each graph, you can select one or more sensors, from all available sensors associated with the asset, including public sensors, and add them to your plot. +Use the "Add Graph" button to create graphs. For each graph, you can select one or more sensors, from all available sensors associated with the asset, including public sensors, and add them to your plot. -You can overlay data from multiple sensors on a single graph. To do this, click on an existing plot and add more sensors from the available options on the right. +In addition, you can add an asset's flex-config to the graph, as long as the value of that config is a sensor(e.g. `[{"title":"Power","plots":[{"sensor":2}]},{"title":"Costs","plots":[{"sensors":[5,6]}]}]`). -Finally, it is possible to set custom titles for any sensor graph by clicking on the "edit" button right next to the default or current title. +Finally, it is possible to set custom titles for any graph by clicking on the "edit" button right next to the default or current title. .. image:: https://github.com/FlexMeasures/screenshots/raw/main/screenshot-asset-editgraph.png :align: center @@ -120,7 +120,8 @@ Finally, it is possible to set custom titles for any sensor graph by clicking on | -Internally, the asset has a `sensors_to_show` field, which controls which sensor data appears in the plot. This can also be set by a script. The accepted format is a dictionary with a graph title and a lists of sensor IDs (e.g. `[{"title": "Power", "sensor": 2}, {"title": "Costs", "sensors": [5,6] }]`). +Internally, the asset has a `sensors_to_show` field, which controls which sensor data appears in the plot. This can also be set by a script or through the API. +The accepted format is a dictionary with a graph title and a lists of sensor IDs (e.g. `[{"title": "Power", "sensor": 2}, {"title": "Costs", "sensors": [5,6] }]`) or the new format with a plot key (e.g. `[{"title":"Power","plots":[{"sensor":2}]},{"title":"Costs","plots":[{"sensors":[5,6]}]}]`). Showing daily KPIs diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 2d24cca1d0..9c59aff554 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -240,6 +240,8 @@ def flatten(cls, nested_list) -> list[int]: for s in nested_list: if isinstance(s, list): all_objects.extend(s) + elif isinstance(s, int): + all_objects.append(s) elif isinstance(s, dict): if "plots" in s: for plot in s["plots"]: @@ -250,6 +252,10 @@ def flatten(cls, nested_list) -> list[int]: if "asset" in plot: sensors = extract_sensors_from_flex_config(plot) all_objects.extend(sensors) + elif "sensors" in s: + all_objects.extend(s["sensors"]) + elif "sensor" in s: + all_objects.append(s["sensor"]) return list(dict.fromkeys(all_objects).keys()) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index c31389b3ac..efad2af0cd 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -93,19 +93,29 @@ def flatten_unique(nested_list_of_objects: list) -> list: for s in nested_list_of_objects: if isinstance(s, list): all_objects.extend(s) + elif isinstance(s, int): + all_objects.append(s) elif isinstance(s, dict): if "sensors" in s: all_objects.extend(s["sensors"]) elif "sensor" in s: all_objects.append(s["sensor"]) elif "plots" in s: + from flexmeasures.data.schemas.utils import ( + extract_sensors_from_flex_config, + ) + for entry in s["plots"]: if "sensors" in entry: all_objects.extend(entry["sensors"]) if "sensor" in entry: all_objects.append(entry["sensor"]) + if "asset" in entry: + sensors = extract_sensors_from_flex_config(entry) + all_objects.extend(sensors) else: all_objects.append(s) + return list(dict.fromkeys(all_objects).keys()) From f65b1b670d2a953718d6a4fbb32490bbc719cae5 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 29 Jan 2026 18:15:28 +0100 Subject: [PATCH 24/33] tests: change test reference asset Signed-off-by: joshuaunity --- flexmeasures/data/tests/test_SensorsToShowSchema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index 9407e58cee..e2c9967258 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -39,9 +39,9 @@ def test_dict_with_title_and_multiple_sensors(): def test_dict_with_asset_and_no_title_plot(): schema = SensorsToShowSchema() - input_value = [{"plots": [{"asset": 44, "flex-model": "soc-min"}]}] + input_value = [{"plots": [{"asset": 4, "flex-model": "soc-min"}]}] expected_output = [ - {"title": None, "plots": [{"asset": 44, "flex-model": "soc-min"}]} + {"title": None, "plots": [{"asset": 4, "flex-model": "soc-min"}]} ] assert schema.deserialize(input_value) == expected_output From 2bbb7d25eb635905ef91cfd5209e5efae581109f Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Thu, 29 Jan 2026 18:24:43 +0100 Subject: [PATCH 25/33] test: apply fixture to test case due to asset resource not found Signed-off-by: joshuaunity --- flexmeasures/data/tests/test_SensorsToShowSchema.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index e2c9967258..56364bb64e 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -37,11 +37,12 @@ def test_dict_with_title_and_multiple_sensors(): assert schema.deserialize(input_value) == expected_output -def test_dict_with_asset_and_no_title_plot(): +def test_dict_with_asset_and_no_title_plot(setup_test_data): + asset_id = setup_test_data["wind-asset-1"].id schema = SensorsToShowSchema() - input_value = [{"plots": [{"asset": 4, "flex-model": "soc-min"}]}] + input_value = [{"plots": [{"asset": asset_id, "flex-model": "soc-min"}]}] expected_output = [ - {"title": None, "plots": [{"asset": 4, "flex-model": "soc-min"}]} + {"title": None, "plots": [{"asset": asset_id, "flex-model": "soc-min"}]} ] assert schema.deserialize(input_value) == expected_output From 2d2eb60ab4c87e2c0c82a19e37644fcab7a221fe Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 2 Feb 2026 05:56:49 +0100 Subject: [PATCH 26/33] chore: multiple followups across docs and schema based on PR request changes Signed-off-by: joshuaunity --- documentation/views/asset-data.rst | 2 +- flexmeasures/data/schemas/generic_assets.py | 8 +++++--- .../data/tests/test_SensorsToShowSchema.py | 17 ----------------- flexmeasures/utils/coding_utils.py | 6 +++--- 4 files changed, 9 insertions(+), 24 deletions(-) diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst index 61be23daa9..b2e5e81e69 100644 --- a/documentation/views/asset-data.rst +++ b/documentation/views/asset-data.rst @@ -121,7 +121,7 @@ Finally, it is possible to set custom titles for any graph by clicking on the "e | Internally, the asset has a `sensors_to_show` field, which controls which sensor data appears in the plot. This can also be set by a script or through the API. -The accepted format is a dictionary with a graph title and a lists of sensor IDs (e.g. `[{"title": "Power", "sensor": 2}, {"title": "Costs", "sensors": [5,6] }]`) or the new format with a plot key (e.g. `[{"title":"Power","plots":[{"sensor":2}]},{"title":"Costs","plots":[{"sensors":[5,6]}]}]`). +The accepted format is a dictionary with a graph title followed by a plot containing senors or asset fex-config reference(e.g. `[{"title":"Power","plots":[{"sensor":2}]},{"title":"Costs","plots":[{"sensor":5},{"asset":10,"flex-model":"soc-min"},]}]`). Showing daily KPIs diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 9c59aff554..0231f55241 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -104,12 +104,14 @@ def _standardize_dict_item(self, item: dict) -> dict: Transform a dictionary-based sensor configuration into a standardized 'plots' structure. Ensures 'title' is a string and processes 'sensor', 'sensors', or direct 'plots' keys. """ - title = None + title = "No Title" if "title" in item: title = item["title"] - if not isinstance(title, str) and title is not None: - title = None + if not isinstance(title, str): + raise ValidationError("'title' value must be a string.") + else: + item["title"] = title if "sensor" in item: sensor = item["sensor"] diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index 56364bb64e..95ca1ecbb6 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -102,23 +102,6 @@ def test_string_json_input(): assert schema.deserialize(input_value) == expected_output -# New test cases for misspelled or missing title and mixed sensor/sensors formats - - -def test_dict_missing_title_key(): - schema = SensorsToShowSchema() - input_value = [{"sensor": 42}] - with pytest.raises(ValidationError, match="Dictionary must contain a 'title' key."): - schema.deserialize(input_value) - - -def test_dict_misspelled_title_key(): - schema = SensorsToShowSchema() - input_value = [{"titel": "Temperature", "sensor": 42}] # Misspelled 'title' - with pytest.raises(ValidationError, match="Dictionary must contain a 'title' key."): - schema.deserialize(input_value) - - def test_dict_with_sensor_as_list(): schema = SensorsToShowSchema() input_value = [{"title": "Temperature", "sensor": [42]}] diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index efad2af0cd..0c1607efc6 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -1,5 +1,4 @@ -r"""Various coding utils (e.g. around function decorati -on)""" +r"""Various coding utils (e.g. around function decoration)""" from __future__ import annotations @@ -81,10 +80,11 @@ def flatten_unique(nested_list_of_objects: list) -> list: - Lists of sensor IDs - Dictionaries with a `sensors` key - Nested lists (one level) + - Dictionaries with `plots` key containing lists of sensors or asset's flex-config reference Example: Input: - [1, [2, 20, 6], 10, [6, 2], {"title":None,"sensors": [10, 15]}, 15] + [1, [2, 20, 6], 10, [6, 2], {"title":None,"sensors": [10, 15]}, 15, {"plots": [{"sensor": 1}, {"sensors": [20, 6]}]}] Output: [1, 2, 20, 6, 10, 15] From 5e2d55d5742ad691ddd7110732e19a4d7c59c28c Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Mon, 2 Feb 2026 20:17:59 +0100 Subject: [PATCH 27/33] fix: fixed failing test Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index f7ee46b822..7d09b13346 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -99,7 +99,7 @@ def _standardize_dict_item(self, item: dict) -> dict: if "title" in item: title = item["title"] - if not isinstance(title, str): + if not isinstance(title, str) and title is not None: raise ValidationError("'title' value must be a string.") else: item["title"] = title From f2d905fe85da02113c61076ffe8a7fe92e652b83 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 3 Feb 2026 11:24:29 +0100 Subject: [PATCH 28/33] chore: update tpy accoutn data relating to sensors_to_show Signed-off-by: joshuaunity --- flexmeasures/cli/data_add.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 6e88a6aa62..6051b2d644 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1923,19 +1923,23 @@ def create_asset_with_one_sensor( db.session.flush() battery = discharging_sensor.generic_asset battery.sensors_to_show = [ - {"title": "Prices", "sensor": day_ahead_sensor.id}, + {"title": "Prices", "plots": [{"sensor": day_ahead_sensor.id}]}, { "title": "Power flows", - "sensors": [production_sensor.id, discharging_sensor.id], + "plots": [ + {"sensors": [production_sensor.id, discharging_sensor.id]}, + ], }, ] # the site gets a similar dashboard (TODO: after #1801, add also capacity constraint) building_asset.sensors_to_show = [ - {"title": "Prices", "sensor": day_ahead_sensor.id}, + {"title": "Prices", "plots": [{"sensor": day_ahead_sensor.id}]}, { "title": "Power flows", - "sensors": [production_sensor.id, discharging_sensor.id], + "plots": [ + {"sensors": [production_sensor.id, discharging_sensor.id]}, + ], }, ] @@ -1975,10 +1979,10 @@ def create_asset_with_one_sensor( process = shiftable_power.generic_asset process.sensors_to_show = [ - {"title": "Prices", "sensor": day_ahead_sensor.id}, - {"title": "Inflexible", "sensor": inflexible_power.id}, - {"title": "Breakable", "sensor": breakable_power.id}, - {"title": "Shiftable", "sensor": shiftable_power.id}, + {"title": "Prices", "plots": [{"sensor": day_ahead_sensor.id}]}, + {"title": "Inflexible", "plots": [{"sensor": inflexible_power.id}]}, + {"title": "Breakable", "plots": [{"sensor": breakable_power.id}]}, + {"title": "Shiftable", "plots": [{"sensor": shiftable_power.id}]}, ] db.session.commit() From c1fd4bfb677d4070ae5fde36af7f48341340a7f9 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 3 Feb 2026 12:45:35 +0100 Subject: [PATCH 29/33] tests: update test cases Signed-off-by: joshuaunity --- .../data/tests/test_SensorsToShowSchema.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index 95ca1ecbb6..8c3c0541eb 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -7,7 +7,7 @@ def test_single_sensor_id(): schema = SensorsToShowSchema() input_value = [42] - expected_output = [{"title": None, "plots": [{"sensor": 42}]}] + expected_output = [{"title": "No Title", "plots": [{"sensor": 42}]}] assert schema.deserialize(input_value) == expected_output @@ -15,8 +15,8 @@ def test_list_of_sensor_ids(): schema = SensorsToShowSchema() input_value = [42, 43] expected_output = [ - {"title": None, "plots": [{"sensor": 42}]}, - {"title": None, "plots": [{"sensor": 43}]}, + {"title": "No Title", "plots": [{"sensor": 42}]}, + {"title": "No Title", "plots": [{"sensor": 43}]}, ] assert schema.deserialize(input_value) == expected_output @@ -42,7 +42,7 @@ def test_dict_with_asset_and_no_title_plot(setup_test_data): schema = SensorsToShowSchema() input_value = [{"plots": [{"asset": asset_id, "flex-model": "soc-min"}]}] expected_output = [ - {"title": None, "plots": [{"asset": asset_id, "flex-model": "soc-min"}]} + {"title": "No Title", "plots": [{"asset": asset_id, "flex-model": "soc-min"}]} ] assert schema.deserialize(input_value) == expected_output @@ -79,13 +79,13 @@ def test_mixed_valid_inputs(): schema = SensorsToShowSchema() input_value = [ {"title": "Test", "sensors": [1, 2]}, - {"title": None, "sensors": [3, 4]}, + {"title": "No Title", "sensors": [3, 4]}, 5, ] expected_output = [ {"title": "Test", "plots": [{"sensors": [1, 2]}]}, - {"title": None, "plots": [{"sensors": [3, 4]}]}, - {"title": None, "plots": [{"sensor": 5}]}, + {"title": "No Title", "plots": [{"sensors": [3, 4]}]}, + {"title": "No Title", "plots": [{"sensor": 5}]}, ] assert schema.deserialize(input_value) == expected_output From 58af007327e2f81c7e53751516bf7d561b0628cd Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 3 Feb 2026 13:03:20 +0100 Subject: [PATCH 30/33] refactor: refactored util function for backward compatibility Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 14 +++++++------- .../data/tests/test_SensorsToShowSchema.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 7d09b13346..30882305b3 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -95,14 +95,14 @@ def _standardize_dict_item(self, item: dict) -> dict: Transform a dictionary-based sensor configuration into a standardized 'plots' structure. Ensures 'title' is a string and processes 'sensor', 'sensors', or direct 'plots' keys. """ - title = "No Title" - if "title" in item: - title = item["title"] - if not isinstance(title, str) and title is not None: - raise ValidationError("'title' value must be a string.") - else: - item["title"] = title + # Get the value, default to "No Title" if the key doesn't exist + title = item.get("title", "No Title") + + if title is not None and not isinstance(title, str): + raise ValidationError("'title' value must be a string.") + + item["title"] = title if "sensor" in item: sensor = item["sensor"] diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index 8c3c0541eb..95ca1ecbb6 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -7,7 +7,7 @@ def test_single_sensor_id(): schema = SensorsToShowSchema() input_value = [42] - expected_output = [{"title": "No Title", "plots": [{"sensor": 42}]}] + expected_output = [{"title": None, "plots": [{"sensor": 42}]}] assert schema.deserialize(input_value) == expected_output @@ -15,8 +15,8 @@ def test_list_of_sensor_ids(): schema = SensorsToShowSchema() input_value = [42, 43] expected_output = [ - {"title": "No Title", "plots": [{"sensor": 42}]}, - {"title": "No Title", "plots": [{"sensor": 43}]}, + {"title": None, "plots": [{"sensor": 42}]}, + {"title": None, "plots": [{"sensor": 43}]}, ] assert schema.deserialize(input_value) == expected_output @@ -42,7 +42,7 @@ def test_dict_with_asset_and_no_title_plot(setup_test_data): schema = SensorsToShowSchema() input_value = [{"plots": [{"asset": asset_id, "flex-model": "soc-min"}]}] expected_output = [ - {"title": "No Title", "plots": [{"asset": asset_id, "flex-model": "soc-min"}]} + {"title": None, "plots": [{"asset": asset_id, "flex-model": "soc-min"}]} ] assert schema.deserialize(input_value) == expected_output @@ -79,13 +79,13 @@ def test_mixed_valid_inputs(): schema = SensorsToShowSchema() input_value = [ {"title": "Test", "sensors": [1, 2]}, - {"title": "No Title", "sensors": [3, 4]}, + {"title": None, "sensors": [3, 4]}, 5, ] expected_output = [ {"title": "Test", "plots": [{"sensors": [1, 2]}]}, - {"title": "No Title", "plots": [{"sensors": [3, 4]}]}, - {"title": "No Title", "plots": [{"sensor": 5}]}, + {"title": None, "plots": [{"sensors": [3, 4]}]}, + {"title": None, "plots": [{"sensor": 5}]}, ] assert schema.deserialize(input_value) == expected_output From a29eda2f93c71484b268456b797e749e37cc8aff Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 3 Feb 2026 13:33:33 +0100 Subject: [PATCH 31/33] tests: fixed failing tests - phase 2 Signed-off-by: joshuaunity --- flexmeasures/data/schemas/generic_assets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 30882305b3..a6e2e0b8df 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -97,12 +97,12 @@ def _standardize_dict_item(self, item: dict) -> dict: """ # Get the value, default to "No Title" if the key doesn't exist - title = item.get("title", "No Title") + title = item.get("title", None) if title is not None and not isinstance(title, str): raise ValidationError("'title' value must be a string.") - item["title"] = title + item["title"] = title or "No Title" if "sensor" in item: sensor = item["sensor"] From af3f3cac55ec7ee45575d1aa2e44e94e477e3811 Mon Sep 17 00:00:00 2001 From: joshuaunity Date: Tue, 24 Feb 2026 11:21:47 +0100 Subject: [PATCH 32/33] fix: Fix failing pipeline due to code indentation Signed-off-by: joshuaunity --- flexmeasures/data/schemas/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index ef1da2046c..a628add7a8 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -85,7 +85,7 @@ def convert_to_quantity(value: str, to_unit: str) -> ur.Quantity: f"Cannot convert value '{value}' to a valid quantity. {e}" ) - + def extract_sensors_from_flex_config(plot: dict) -> list[Sensor]: """ Extracts a consolidated list of sensors from an asset based on @@ -118,8 +118,8 @@ def extract_sensors_from_flex_config(plot: dict) -> list[Sensor]: return all_sensors - - def snake_to_kebab(key: str) -> str: + +def snake_to_kebab(key: str) -> str: """Convert snake_case to kebab-case.""" return key.replace("_", "-") From e0683d4cf669865a200836a7bc7f337134d06404 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Wed, 25 Feb 2026 10:04:56 +0100 Subject: [PATCH 33/33] Feat/clean up duplicate flattening implementation (#1984) * refactor: move extract_sensors_from_flex_config to schemas/generic_assets.py Signed-off-by: F.N. Claessen * delete: internal import no longer needed Signed-off-by: F.N. Claessen * refactor: rename util method to flatten_sensors_to_show Signed-off-by: F.N. Claessen * refactor: get rid of duplicate implementation for flattening sensors to show Signed-off-by: F.N. Claessen * feat: make doctest out of example Signed-off-by: F.N. Claessen * style: rst-style docstrings Signed-off-by: F.N. Claessen * feat: support multiple flex-config field names Signed-off-by: F.N. Claessen * feat: test flatten function Signed-off-by: F.N. Claessen --------- Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 6 +- flexmeasures/conftest.py | 25 ++++++ .../data/models/charts/belief_charts.py | 5 +- flexmeasures/data/models/generic_assets.py | 13 ++- flexmeasures/data/schemas/generic_assets.py | 88 ++++++++++++++----- flexmeasures/data/schemas/utils.py | 34 ------- .../data/tests/test_SensorsToShowSchema.py | 33 +++++++ flexmeasures/utils/coding_utils.py | 48 ---------- 8 files changed, 140 insertions(+), 112 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index d63bbc2dda..b2f14db488 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -47,6 +47,7 @@ GenericAssetSchema as AssetSchema, GenericAssetIdField as AssetIdField, GenericAssetTypeSchema as AssetTypeSchema, + SensorsToShowSchema, ) from flexmeasures.data.schemas.scheduling.storage import StorageFlexModelSchema from flexmeasures.data.schemas.scheduling import AssetTriggerSchema, FlexContextSchema @@ -61,9 +62,6 @@ ) from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.schemas.assets import default_response_fields -from flexmeasures.utils.coding_utils import ( - flatten_unique, -) from flexmeasures.ui.utils.view_utils import clear_session, set_session_variables from flexmeasures.auth.policy import check_access from flexmeasures.data.schemas.sensors import ( @@ -896,7 +894,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): tags: - Assets """ - sensors = flatten_unique(asset.validate_sensors_to_show()) + sensors = SensorsToShowSchema.flatten(asset.validate_sensors_to_show()) return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) @route("//auditlog") diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index d3043286e4..f737bb4c82 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -548,6 +548,31 @@ def create_assets( ), ) db.session.add(sensor) + scc_sensor = Sensor( + name="site-consumption-capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + db.session.add(scc_sensor) + cc_sensor = Sensor( + name="consumption-capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + db.session.add(cc_sensor) + pc_sensor = Sensor( + name="production-capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + db.session.add(pc_sensor) + db.session.flush() # assign sensor IDs + asset.flex_model["consumption-capacity"] = {"sensor": cc_sensor.id} + asset.flex_model["production-capacity"] = {"sensor": pc_sensor.id} + asset.flex_context["site-consumption-capacity"] = {"sensor": scc_sensor.id} assets.append(asset) # one day of test data (one complete sine curve) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index e155438520..4cf882d9ee 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -11,7 +11,6 @@ from flexmeasures.utils.flexmeasures_inflection import ( capitalize, ) -from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.utils.unit_utils import find_smallest_common_unit, get_unit_dimension @@ -499,8 +498,10 @@ def chart_for_multiple_sensors( combine_legend: bool = True, **override_chart_specs: dict, ): + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + # Determine the shared data resolution - all_shown_sensors = flatten_unique(sensors_to_show) + all_shown_sensors = SensorsToShowSchema.flatten(sensors_to_show) condition = list( sensor.event_resolution for sensor in all_shown_sensors diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 46bb19383c..48072f0556 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -30,7 +30,6 @@ CONSULTANT_ROLE, ) from flexmeasures.utils import geo_utils -from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution from flexmeasures.utils.unit_utils import find_smallest_common_unit @@ -287,7 +286,9 @@ def validate_sensors_to_show( sensor_ids_to_show = self.sensors_to_show # Import the schema for validation - from flexmeasures.data.schemas.utils import extract_sensors_from_flex_config + from flexmeasures.data.schemas.generic_assets import ( + extract_sensors_from_flex_config, + ) from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema sensors_to_show_schema = SensorsToShowSchema() @@ -662,8 +663,10 @@ def chart( :param resolution: optionally set the resolution of data being displayed :returns: JSON string defining vega-lite chart specs """ + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + processed_sensors_to_show = self.validate_sensors_to_show() - sensors = flatten_unique(processed_sensors_to_show) + sensors = SensorsToShowSchema.flatten(processed_sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -1023,7 +1026,9 @@ def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) } """ - sensor_ids = [s.id for s in flatten_unique(sensors)] + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + + sensor_ids = [s.id for s in SensorsToShowSchema.flatten(sensors)] start, end = get_timerange(sensor_ids) return dict(start=start, end=end) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index a6e2e0b8df..472b63aafb 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -3,7 +3,7 @@ from datetime import timedelta import json from http import HTTPStatus -from typing import Any +from typing import Any, TYPE_CHECKING from flask import abort from marshmallow import validates, ValidationError, fields, validates_schema @@ -11,7 +11,8 @@ from flask_security import current_user from sqlalchemy import select - +if TYPE_CHECKING: + from flexmeasures import Sensor from flexmeasures.data import ma, db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType @@ -21,7 +22,6 @@ from flexmeasures.data.schemas.utils import ( FMValidationError, MarshmallowClickMixin, - extract_sensors_from_flex_config, ) from flexmeasures.auth.policy import user_has_admin_access from flexmeasures.cli import is_running as running_as_cli @@ -211,23 +211,34 @@ def _validate_flex_config_field_is_valid_choice( ) @classmethod - def flatten(cls, nested_list) -> list[int]: + def flatten(cls, nested_list: list) -> list[int] | list[Sensor]: """ - Flatten a nested list of sensors or sensor dictionaries into a unique list of sensor IDs. - - This method processes the following formats, for each of the entries of the nested list: - - A list of sensor IDs: `[1, 2, 3]` - - A list of dictionaries where each dictionary contains a `sensors` list, a `sensor` key or a `plots` key - `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, {"title": "Pressure", "plots": [{"sensor": 4}, {"sensors": [5,6]}]}]` - - Mixed formats: `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, 4, 5, 1]` - - It extracts all sensor IDs, removes duplicates, and returns a flattened list of unique sensor IDs. - - Args: - nested_list (list): A list containing sensor IDs, or dictionaries with `sensors` or `sensor` keys. - - Returns: - list: A unique list of sensor IDs. + Flatten a nested list of sensor IDs into a unique list. Also works for Sensor objects. + + This method processes the following formats for each entry in the list: + 1. A single sensor ID: + `3` + 2. A list of sensor IDs: + `[1, 2]` + 3. A dictionary with a `sensor` key: + `{"sensor": 3}` + 4. A dictionary with a `sensors` key: + `{"sensors": [1, 2]}` + 5. A dictionary with a `plots` key, containing a list of dictionaries, + each with a `sensor` or `sensors` key: + `{"plots": [{"sensor": 4}, {"sensors": [5, 6]}]}` + 6. A dictionary under the `plots` key containing the `asset` key together with a `flex-model` or `flex-context` key, + containing a field name or a list of field names: + `{"plots": [{"asset": 100, "flex-model": ["consumption-capacity", "production-capacity"], "flex-context": "site-power-capacity"}}` + 7. Mixed formats: + `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, {"title": "Pressure", "plots": [{"sensor": 4}, {"sensors": [5, 6]}]}]` + + Example: + >>> SensorsToShowSchema.flatten([1, [2, 20, 6], 10, [6, 2], {"title": None,"sensors": [10, 15]}, 15, {"plots": [{"sensor": 1}, {"sensors": [20, 8]}]}]) + [1, 2, 20, 6, 10, 15, 8] + + :param nested_list: A list containing sensor IDs, or dictionaries with `sensors` or `sensor` keys. + :returns: A unique list of sensor IDs, or a unique list of Sensors """ all_objects = [] for s in nested_list: @@ -249,7 +260,6 @@ def flatten(cls, nested_list) -> list[int]: all_objects.extend(s["sensors"]) elif "sensor" in s: all_objects.append(s["sensor"]) - return list(dict.fromkeys(all_objects).keys()) @@ -427,3 +437,41 @@ def _deserialize(self, value: Any, attr, data, **kwargs) -> GenericAsset: def _serialize(self, value: GenericAsset, attr, obj, **kwargs) -> int: """Turn a GenericAsset into a generic asset id.""" return value.id + + +def extract_sensors_from_flex_config(plot: dict) -> list[Sensor]: + """ + Extracts a consolidated list of sensors from an asset based on + flex-context or flex-model definitions provided in a plot dictionary. + """ + all_sensors = [] + + asset = GenericAssetIdField().deserialize(plot.get("asset")) + + fields_to_check = { + "flex-context": asset.flex_context, + "flex-model": asset.flex_model, + } + + for plot_key, flex_config in fields_to_check.items(): + if plot_key not in plot: + continue + + field_keys = plot[plot_key] + data = flex_config or {} + + if isinstance(field_keys, str): + field_keys = [field_keys] + elif not isinstance(field_keys, list): + continue + + for field_key in field_keys: + field_value = data.get(field_key) + + if isinstance(field_value, dict): + # Add a single sensor if it exists + sensor = field_value.get("sensor") + if sensor: + all_sensors.append(sensor) + + return all_sensors diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index a628add7a8..149d891f39 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -5,7 +5,6 @@ from pint import DefinitionSyntaxError, DimensionalityError, UndefinedUnitError from flexmeasures.utils.unit_utils import to_preferred, ur -from flexmeasures.data.models.time_series import Sensor class MarshmallowClickMixin(click.ParamType): @@ -86,39 +85,6 @@ def convert_to_quantity(value: str, to_unit: str) -> ur.Quantity: ) -def extract_sensors_from_flex_config(plot: dict) -> list[Sensor]: - """ - Extracts a consolidated list of sensors from an asset based on - flex-context or flex-model definitions provided in a plot dictionary. - """ - all_sensors = [] - - from flexmeasures.data.schemas.generic_assets import ( - GenericAssetIdField, - ) # Import here to avoid circular imports - - asset = GenericAssetIdField().deserialize(plot.get("asset")) - - fields_to_check = { - "flex-context": asset.flex_context, - "flex-model": asset.flex_model, - } - - for plot_key, flex_config in fields_to_check.items(): - if plot_key in plot: - field_key = plot[plot_key] - data = flex_config or {} - field_value = data.get(field_key) - - if isinstance(field_value, dict): - # Add a single sensor if it exists - sensor = field_value.get("sensor") - if sensor: - all_sensors.append(sensor) - - return all_sensors - - def snake_to_kebab(key: str) -> str: """Convert snake_case to kebab-case.""" return key.replace("_", "-") diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index 95ca1ecbb6..2780baf5b2 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -1,6 +1,7 @@ import pytest from marshmallow import ValidationError +from flexmeasures import Sensor from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema @@ -47,6 +48,38 @@ def test_dict_with_asset_and_no_title_plot(setup_test_data): assert schema.deserialize(input_value) == expected_output +def _get_sensor_by_name(sensors: list[Sensor], name: str) -> Sensor: + for sensor in sensors: + if sensor.name == name: + return sensor + raise ValueError(f"Sensor {name} not found") + + +def test_flatten_with_multiple_flex_config_fields(setup_test_data): + asset = setup_test_data["wind-asset-1"] + schema = SensorsToShowSchema() + input_value = [ + { + "plots": [ + { + "asset": asset.id, + "flex-model": ["consumption-capacity", "production-capacity"], + "flex-context": "site-consumption-capacity", + } + ] + } + ] + expected_output = [ + _get_sensor_by_name(asset.sensors, name).id + for name in ( + "site-consumption-capacity", + "consumption-capacity", + "production-capacity", + ) + ] + assert schema.flatten(input_value) == expected_output + + def test_invalid_sensor_string_input(): schema = SensorsToShowSchema() with pytest.raises( diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 0c1607efc6..0a951bc053 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -71,54 +71,6 @@ def sort_dict(unsorted_dict: dict) -> dict: return sorted_dict -# This function is used for sensors_to_show in follow-up PR it will be moved and renamed to flatten_sensors_to_show -def flatten_unique(nested_list_of_objects: list) -> list: - """ - Get unique sensor IDs from a list of `sensors_to_show`. - - Handles: - - Lists of sensor IDs - - Dictionaries with a `sensors` key - - Nested lists (one level) - - Dictionaries with `plots` key containing lists of sensors or asset's flex-config reference - - Example: - Input: - [1, [2, 20, 6], 10, [6, 2], {"title":None,"sensors": [10, 15]}, 15, {"plots": [{"sensor": 1}, {"sensors": [20, 6]}]}] - - Output: - [1, 2, 20, 6, 10, 15] - """ - all_objects = [] - for s in nested_list_of_objects: - if isinstance(s, list): - all_objects.extend(s) - elif isinstance(s, int): - all_objects.append(s) - elif isinstance(s, dict): - if "sensors" in s: - all_objects.extend(s["sensors"]) - elif "sensor" in s: - all_objects.append(s["sensor"]) - elif "plots" in s: - from flexmeasures.data.schemas.utils import ( - extract_sensors_from_flex_config, - ) - - for entry in s["plots"]: - if "sensors" in entry: - all_objects.extend(entry["sensors"]) - if "sensor" in entry: - all_objects.append(entry["sensor"]) - if "asset" in entry: - sensors = extract_sensors_from_flex_config(entry) - all_objects.extend(sensors) - else: - all_objects.append(s) - - return list(dict.fromkeys(all_objects).keys()) - - def timeit(func): """Decorator for printing the time it took to execute the decorated function."""