diff --git a/documentation/changelog.rst b/documentation/changelog.rst index f9c1a71d0a..d5023925e0 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -25,6 +25,8 @@ New features * Give ability to edit sensor timezone from the UI [see `PR #1900 `_] * Support creating schedules with only information known prior to some time, now also via the CLI (the API already supported it) [see `PR #1871 `_]. * Add ``fields`` param to the asset-listing endpoints, to save bandwidth in response data [see `PR #1884 `_] +* Support for flex-config in the ``SensorsToShowSchema`` [see `PR #1904 `_] + .. note:: For backwards-compatibility, the new ``fields`` parameter will only be fully active, i.e. also returning less fields per default, in v0.32. Set ``FLEXMEASURES_API_SUNSET_ACTIVE=True`` to test the full effect now. diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst index fe3d1a497d..b2e5e81e69 100644 --- a/documentation/views/asset-data.rst +++ b/documentation/views/asset-data.rst @@ -108,11 +108,11 @@ Editing the graphs dashboard Click the "Edit Graph" button to open the graph editor. -Use the "Add Graph" button to create graphs. For each graph, you can select one or more sensors, from all available sensors associated with the asset, including public sensors, and add them to your plot. +Use the "Add Graph" button to create graphs. For each graph, you can select one or more sensors, from all available sensors associated with the asset, including public sensors, and add them to your plot. -You can overlay data from multiple sensors on a single graph. To do this, click on an existing plot and add more sensors from the available options on the right. +In addition, you can add an asset's flex-config to the graph, as long as the value of that config is a sensor(e.g. `[{"title":"Power","plots":[{"sensor":2}]},{"title":"Costs","plots":[{"sensors":[5,6]}]}]`). -Finally, it is possible to set custom titles for any sensor graph by clicking on the "edit" button right next to the default or current title. +Finally, it is possible to set custom titles for any graph by clicking on the "edit" button right next to the default or current title. .. image:: https://github.com/FlexMeasures/screenshots/raw/main/screenshot-asset-editgraph.png :align: center @@ -120,7 +120,8 @@ Finally, it is possible to set custom titles for any sensor graph by clicking on | -Internally, the asset has a `sensors_to_show` field, which controls which sensor data appears in the plot. This can also be set by a script. The accepted format is a dictionary with a graph title and a lists of sensor IDs (e.g. `[{"title": "Power", "sensor": 2}, {"title": "Costs", "sensors": [5,6] }]`). +Internally, the asset has a `sensors_to_show` field, which controls which sensor data appears in the plot. This can also be set by a script or through the API. +The accepted format is a dictionary with a graph title followed by a plot containing senors or asset fex-config reference(e.g. `[{"title":"Power","plots":[{"sensor":2}]},{"title":"Costs","plots":[{"sensor":5},{"asset":10,"flex-model":"soc-min"},]}]`). Showing daily KPIs diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index d63bbc2dda..b2f14db488 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -47,6 +47,7 @@ GenericAssetSchema as AssetSchema, GenericAssetIdField as AssetIdField, GenericAssetTypeSchema as AssetTypeSchema, + SensorsToShowSchema, ) from flexmeasures.data.schemas.scheduling.storage import StorageFlexModelSchema from flexmeasures.data.schemas.scheduling import AssetTriggerSchema, FlexContextSchema @@ -61,9 +62,6 @@ ) from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.schemas.assets import default_response_fields -from flexmeasures.utils.coding_utils import ( - flatten_unique, -) from flexmeasures.ui.utils.view_utils import clear_session, set_session_variables from flexmeasures.auth.policy import check_access from flexmeasures.data.schemas.sensors import ( @@ -896,7 +894,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): tags: - Assets """ - sensors = flatten_unique(asset.validate_sensors_to_show()) + sensors = SensorsToShowSchema.flatten(asset.validate_sensors_to_show()) return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) @route("//auditlog") diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 2c8ab23e3e..e04503f876 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1824,19 +1824,23 @@ def create_asset_with_one_sensor( db.session.flush() battery = discharging_sensor.generic_asset battery.sensors_to_show = [ - {"title": "Prices", "sensor": day_ahead_sensor.id}, + {"title": "Prices", "plots": [{"sensor": day_ahead_sensor.id}]}, { "title": "Power flows", - "sensors": [production_sensor.id, discharging_sensor.id], + "plots": [ + {"sensors": [production_sensor.id, discharging_sensor.id]}, + ], }, ] # the site gets a similar dashboard (TODO: after #1801, add also capacity constraint) building_asset.sensors_to_show = [ - {"title": "Prices", "sensor": day_ahead_sensor.id}, + {"title": "Prices", "plots": [{"sensor": day_ahead_sensor.id}]}, { "title": "Power flows", - "sensors": [production_sensor.id, discharging_sensor.id], + "plots": [ + {"sensors": [production_sensor.id, discharging_sensor.id]}, + ], }, ] @@ -1876,10 +1880,10 @@ def create_asset_with_one_sensor( process = shiftable_power.generic_asset process.sensors_to_show = [ - {"title": "Prices", "sensor": day_ahead_sensor.id}, - {"title": "Inflexible", "sensor": inflexible_power.id}, - {"title": "Breakable", "sensor": breakable_power.id}, - {"title": "Shiftable", "sensor": shiftable_power.id}, + {"title": "Prices", "plots": [{"sensor": day_ahead_sensor.id}]}, + {"title": "Inflexible", "plots": [{"sensor": inflexible_power.id}]}, + {"title": "Breakable", "plots": [{"sensor": breakable_power.id}]}, + {"title": "Shiftable", "plots": [{"sensor": shiftable_power.id}]}, ] db.session.commit() diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index d3043286e4..f737bb4c82 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -548,6 +548,31 @@ def create_assets( ), ) db.session.add(sensor) + scc_sensor = Sensor( + name="site-consumption-capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + db.session.add(scc_sensor) + cc_sensor = Sensor( + name="consumption-capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + db.session.add(cc_sensor) + pc_sensor = Sensor( + name="production-capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + db.session.add(pc_sensor) + db.session.flush() # assign sensor IDs + asset.flex_model["consumption-capacity"] = {"sensor": cc_sensor.id} + asset.flex_model["production-capacity"] = {"sensor": pc_sensor.id} + asset.flex_context["site-consumption-capacity"] = {"sensor": scc_sensor.id} assets.append(asset) # one day of test data (one complete sine curve) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index e7c5213b60..4cf882d9ee 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -11,7 +11,6 @@ from flexmeasures.utils.flexmeasures_inflection import ( capitalize, ) -from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.utils.unit_utils import find_smallest_common_unit, get_unit_dimension @@ -499,8 +498,10 @@ def chart_for_multiple_sensors( combine_legend: bool = True, **override_chart_specs: dict, ): + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + # Determine the shared data resolution - all_shown_sensors = flatten_unique(sensors_to_show) + all_shown_sensors = SensorsToShowSchema.flatten(sensors_to_show) condition = list( sensor.event_resolution for sensor in all_shown_sensors @@ -528,7 +529,14 @@ def chart_for_multiple_sensors( title = entry.get("title") if title == "Charge Point sessions": continue - sensors = entry.get("sensors") + plots = entry.get("plots", []) + sensors = [] + for plot in plots: + if "sensors" in plot: + sensors.extend(plot.get("sensors")) + elif "sensor" in plot: + sensors.extend([plot.get("sensor")]) + # List the sensors that go into one row row_sensors: list["Sensor"] = sensors # noqa F821 diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index e9a3e460a1..48072f0556 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -30,7 +30,6 @@ CONSULTANT_ROLE, ) from flexmeasures.utils import geo_utils -from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution from flexmeasures.utils.unit_utils import find_smallest_common_unit @@ -234,7 +233,7 @@ def validate_sensors_to_show( Steps: - The function deserializes the 'sensors_to_show' data from the database, ensuring that older formats are parsed correctly. - It checks if each sensor is accessible by the user and filters out any unauthorized sensors. - - The sensor structure is rebuilt according to the latest format, which allows for grouping sensors and adding optional titles. + - The sensor structure is rebuilt according to the latest format, which allows for grouping sensors and adding optional titles, all into a single 'plots' array. Details on format: - The 'sensors_to_show' attribute is defined as a list of sensor IDs or nested lists of sensor IDs (to indicate grouping). @@ -247,8 +246,8 @@ def validate_sensors_to_show( 2. List with titles and sensor groupings: sensors_to_show = [ - {"title": "Title 1", "sensor": 40}, - {"title": "Title 2", "sensors": [41, 42]}, + {"title": "Title 1", "plots": [{"sensor": 40}]}, + {"title": "Title 2", "plots": [{"sensors": [41, 42]}]}, [43, 44], 45, 46 ] @@ -287,6 +286,9 @@ def validate_sensors_to_show( sensor_ids_to_show = self.sensors_to_show # Import the schema for validation + from flexmeasures.data.schemas.generic_assets import ( + extract_sensors_from_flex_config, + ) from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema sensors_to_show_schema = SensorsToShowSchema() @@ -324,7 +326,17 @@ def validate_sensors_to_show( for entry in standardized_sensors_to_show: title = entry.get("title") - sensors = entry.get("sensors") + sensors = [] + plots = entry.get("plots", []) + if len(plots) > 0: + for plot in plots: + if "sensor" in plot: + sensors.append(plot["sensor"]) + if "sensors" in plot: + sensors.extend(plot["sensors"]) + if "asset" in plot: + extracted_sensors = extract_sensors_from_flex_config(plot) + sensors.extend(extracted_sensors) accessible_sensors = [ accessible_sensor_map.get(sid) @@ -334,7 +346,9 @@ def validate_sensors_to_show( inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map] missed_sensor_ids.extend(inaccessible) if accessible_sensors: - sensors_to_show.append({"title": title, "sensors": accessible_sensors}) + sensors_to_show.append( + {"title": title, "plots": [{"sensors": accessible_sensors}]} + ) if missed_sensor_ids: current_app.logger.warning( @@ -649,8 +663,10 @@ def chart( :param resolution: optionally set the resolution of data being displayed :returns: JSON string defining vega-lite chart specs """ + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + processed_sensors_to_show = self.validate_sensors_to_show() - sensors = flatten_unique(processed_sensors_to_show) + sensors = SensorsToShowSchema.flatten(processed_sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -1010,7 +1026,9 @@ def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) } """ - sensor_ids = [s.id for s in flatten_unique(sensors)] + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + + sensor_ids = [s.id for s in SensorsToShowSchema.flatten(sensors)] start, end = get_timerange(sensor_ids) return dict(start=start, end=end) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 16ca0a1ba6..472b63aafb 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -3,7 +3,7 @@ from datetime import timedelta import json from http import HTTPStatus -from typing import Any +from typing import Any, TYPE_CHECKING from flask import abort from marshmallow import validates, ValidationError, fields, validates_schema @@ -11,7 +11,8 @@ from flask_security import current_user from sqlalchemy import select - +if TYPE_CHECKING: + from flexmeasures import Sensor from flexmeasures.data import ma, db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType @@ -32,23 +33,24 @@ class SensorsToShowSchema(fields.Field): The `sensors_to_show` attribute defines which sensors should be displayed for a particular asset. It supports various input formats, which are standardized into a list of dictionaries, each containing - a `title` (optional) and a `sensors` list. The valid input formats include: + a `title` (optional) and a `plots` list, this list then consist of dictionaries with keys such as `sensor`, `asset` or `sensors`. - - A single sensor ID (int): `42` -> `{"title": None, "sensors": [42]}` - - A list of sensor IDs (list of ints): `[42, 43]` -> `{"title": None, "sensors": [42, 43]}` - - A dictionary with a title and sensor: `{"title": "Temperature", "sensor": 42}` -> `{"title": "Temperature", "sensors": [42]}` - - A dictionary with a title and sensors: `{"title": "Pressure", "sensors": [42, 43]}` + - A single sensor ID (int): `42` -> `{"title": None, "plots": [{"sensor": 42}]}` + - A list of sensor IDs (list of ints): `[42, 43]` -> `{"title": None, "plots": [{"sensors": [42, 43]}]}` + - A dictionary with a title and sensor: `{"title": "Temperature", "sensor": 42}` -> `{"title": "Temperature", "plots": [{"sensor": 42}]}` + - A dictionary with a title and sensors: `{"title": "Pressure", "sensors": [42, 43]}` -> `{"title": "Pressure", "plots": [{"sensors": [42, 43]}]}` Validation ensures that: - The input is either a list, integer, or dictionary. - - If the input is a dictionary, it must contain either `sensor` (int) or `sensors` (list of ints). + - If the input is a dictionary, it must contain either `sensor` (int), `sensors` (list of ints) or `plots` (list of dicts). - All sensor IDs must be valid integers. - Example Input: - - `[{"title": "Test", "sensors": [1, 2]}, {"title": None, "sensors": [3, 4]}, 5]` + Example Inputs: + - `[{"title": "Test", "plots": [{"sensor": 1}, {"sensor": 2}]}, {"title": "Another Test", "plots": [{"sensors": [3, 4]}]}, 5]` + - `[{"title": "Test", "sensors": [1, 2]}, {"title": None, "sensors": [3, 4]}, 5]` (Older format but still compatible) Example Output (Standardized): - - `[{"title": "Test", "sensors": [1, 2]}, {"title": None, "sensors": [3, 4]}, {"title": None, "sensors": [5]}]` + - `[{"title": "Test", "plots": [{"sensors": [1, 2]}]}, {"title": None, "plots": [{"sensors": [3, 4]}]}, {"title": None, "plots": [{"sensor": 5}]}]` """ def deserialize(self, value, **kwargs) -> list: @@ -71,76 +73,193 @@ def deserialize(self, value, **kwargs) -> list: def _standardize_item(self, item) -> dict: """ - Standardize different input formats to a consistent dictionary format. + Normalize various input formats (int, list, or dict) into a standard plot dictionary. """ if isinstance(item, int): - return {"title": None, "sensors": [item]} + return {"title": None, "plots": [{"sensor": item}]} elif isinstance(item, list): if not all(isinstance(sensor_id, int) for sensor_id in item): raise ValidationError( "All elements in a list within 'sensors_to_show' must be integers." ) - return {"title": None, "sensors": item} + return {"title": None, "plots": [{"sensors": item}]} elif isinstance(item, dict): - if "title" not in item: - raise ValidationError("Dictionary must contain a 'title' key.") - else: - title = item["title"] - if not isinstance(title, str) and title is not None: - raise ValidationError("'title' value must be a string.") - - if "sensor" in item: - sensor = item["sensor"] - if not isinstance(sensor, int): - raise ValidationError("'sensor' value must be an integer.") - return {"title": title, "sensors": [sensor]} - elif "sensors" in item: - sensors = item["sensors"] - if not isinstance(sensors, list) or not all( - isinstance(sensor_id, int) for sensor_id in sensors - ): - raise ValidationError("'sensors' value must be a list of integers.") - return {"title": title, "sensors": sensors} - else: - raise ValidationError( - "Dictionary must contain either 'sensor' or 'sensors' key." - ) + return self._standardize_dict_item(item) else: raise ValidationError( "Invalid item type in 'sensors_to_show'. Expected int, list, or dict." ) - @classmethod - def flatten(cls, nested_list) -> list[int]: + def _standardize_dict_item(self, item: dict) -> dict: + """ + Transform a dictionary-based sensor configuration into a standardized 'plots' structure. + Ensures 'title' is a string and processes 'sensor', 'sensors', or direct 'plots' keys. + """ + + # Get the value, default to "No Title" if the key doesn't exist + title = item.get("title", None) + + if title is not None and not isinstance(title, str): + raise ValidationError("'title' value must be a string.") + + item["title"] = title or "No Title" + + if "sensor" in item: + sensor = item["sensor"] + if not isinstance(sensor, int): + raise ValidationError("'sensor' value must be an integer.") + return {"title": title, "plots": [{"sensor": sensor}]} + elif "sensors" in item: + sensors = item["sensors"] + if not isinstance(sensors, list) or not all( + isinstance(sensor_id, int) for sensor_id in sensors + ): + raise ValidationError("'sensors' value must be a list of integers.") + return {"title": title, "plots": [{"sensors": sensors}]} + elif "plots" in item: + plots = item["plots"] + if not isinstance(plots, list): + raise ValidationError("'plots' must be a list or dictionary.") + for plot in plots: + self._validate_single_plot(plot) + + return {"title": title, "plots": plots} + else: + raise ValidationError( + "Dictionary must contain either 'sensor', 'sensors' or 'plots' key." + ) + + def _validate_single_plot(self, plot): + """ + Perform structural validation on an individual plot dictionary. + Requires at least one of: 'sensor', 'sensors', or 'asset'. """ - Flatten a nested list of sensors or sensor dictionaries into a unique list of sensor IDs. + if not isinstance(plot, dict): + raise ValidationError("Each plot in 'plots' must be a dictionary.") - This method processes the following formats, for each of the entries of the nested list: - - A list of sensor IDs: `[1, 2, 3]` - - A list of dictionaries where each dictionary contains a `sensors` list or a `sensor` key: - `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}]` - - Mixed formats: `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, 4, 5, 1]` + if "sensor" not in plot and "sensors" not in plot and "asset" not in plot: + raise ValidationError( + "Each plot must contain either 'sensor', 'sensors' or an 'asset' key." + ) - It extracts all sensor IDs, removes duplicates, and returns a flattened list of unique sensor IDs. + if "asset" in plot: + self._validate_asset_in_plot(plot) + if "sensor" in plot: + sensor = plot["sensor"] + if not isinstance(sensor, int): + raise ValidationError("'sensor' value must be an integer.") + if "sensors" in plot: + sensors = plot["sensors"] + if not isinstance(sensors, list) or not all( + isinstance(sensor_id, int) for sensor_id in sensors + ): + raise ValidationError("'sensors' value must be a list of integers.") + + def _validate_asset_in_plot(self, plot): + """ + Validate plots that reference a GenericAsset. + Ensures flex-config schemas are respected when an asset is provided. + """ + from flexmeasures.data.schemas.scheduling import ( + DBFlexContextSchema, + ) + from flexmeasures.data.schemas.scheduling.storage import ( + DBStorageFlexModelSchema, + ) + + if "flex-context" not in plot and "flex-model" not in plot: + raise ValidationError( + "When 'asset' is provided in a plot, 'flex-context' or 'flex-model' must also be provided." + ) - Args: - nested_list (list): A list containing sensor IDs, or dictionaries with `sensors` or `sensor` keys. + self._validate_flex_config_field_is_valid_choice( + plot, "flex-context", DBFlexContextSchema.mapped_schema_keys.values() + ) + self._validate_flex_config_field_is_valid_choice( + plot, "flex-model", DBStorageFlexModelSchema().mapped_schema_keys.values() + ) - Returns: - list: A unique list of sensor IDs. + def _validate_flex_config_field_is_valid_choice( + self, plot_config, field_name, valid_collection + ): + """ + Verify that the chosen flex-config field exists on the specific asset and matches + allowed schema keys. """ + if field_name in plot_config: + value = plot_config[field_name] + asset_id = plot_config.get("asset") + asset = GenericAssetIdField().deserialize(asset_id) + + if asset is None: + raise ValidationError(f"Asset with ID {asset_id} does not exist.") + + if value and not isinstance(value, str): + raise ValidationError(f"The value for '{field_name}' must be a string.") + + if value not in valid_collection: + raise ValidationError(f"'{field_name}' value '{value}' is not valid.") + attr_to_check = ( + "flex_model" if field_name == "flex-model" else "flex_context" + ) + asset_flex_config = getattr(asset, attr_to_check, {}) + + if value not in asset_flex_config: + raise ValidationError( + f"The asset with ID '{asset_id}' does not have a '{value}' set in its '{attr_to_check}'." + ) + + @classmethod + def flatten(cls, nested_list: list) -> list[int] | list[Sensor]: + """ + Flatten a nested list of sensor IDs into a unique list. Also works for Sensor objects. + + This method processes the following formats for each entry in the list: + 1. A single sensor ID: + `3` + 2. A list of sensor IDs: + `[1, 2]` + 3. A dictionary with a `sensor` key: + `{"sensor": 3}` + 4. A dictionary with a `sensors` key: + `{"sensors": [1, 2]}` + 5. A dictionary with a `plots` key, containing a list of dictionaries, + each with a `sensor` or `sensors` key: + `{"plots": [{"sensor": 4}, {"sensors": [5, 6]}]}` + 6. A dictionary under the `plots` key containing the `asset` key together with a `flex-model` or `flex-context` key, + containing a field name or a list of field names: + `{"plots": [{"asset": 100, "flex-model": ["consumption-capacity", "production-capacity"], "flex-context": "site-power-capacity"}}` + 7. Mixed formats: + `[{"title": "Temperature", "sensors": [1, 2]}, {"title": "Pressure", "sensor": 3}, {"title": "Pressure", "plots": [{"sensor": 4}, {"sensors": [5, 6]}]}]` + + Example: + >>> SensorsToShowSchema.flatten([1, [2, 20, 6], 10, [6, 2], {"title": None,"sensors": [10, 15]}, 15, {"plots": [{"sensor": 1}, {"sensors": [20, 8]}]}]) + [1, 2, 20, 6, 10, 15, 8] + + :param nested_list: A list containing sensor IDs, or dictionaries with `sensors` or `sensor` keys. + :returns: A unique list of sensor IDs, or a unique list of Sensors + """ all_objects = [] for s in nested_list: if isinstance(s, list): all_objects.extend(s) + elif isinstance(s, int): + all_objects.append(s) elif isinstance(s, dict): - if "sensors" in s: + if "plots" in s: + for plot in s["plots"]: + if "sensors" in plot: + all_objects.extend(plot["sensors"]) + if "sensor" in plot: + all_objects.append(plot["sensor"]) + if "asset" in plot: + sensors = extract_sensors_from_flex_config(plot) + all_objects.extend(sensors) + elif "sensors" in s: all_objects.extend(s["sensors"]) - if "sensor" in s: + elif "sensor" in s: all_objects.append(s["sensor"]) - else: - all_objects.append(s) return list(dict.fromkeys(all_objects).keys()) @@ -318,3 +437,41 @@ def _deserialize(self, value: Any, attr, data, **kwargs) -> GenericAsset: def _serialize(self, value: GenericAsset, attr, obj, **kwargs) -> int: """Turn a GenericAsset into a generic asset id.""" return value.id + + +def extract_sensors_from_flex_config(plot: dict) -> list[Sensor]: + """ + Extracts a consolidated list of sensors from an asset based on + flex-context or flex-model definitions provided in a plot dictionary. + """ + all_sensors = [] + + asset = GenericAssetIdField().deserialize(plot.get("asset")) + + fields_to_check = { + "flex-context": asset.flex_context, + "flex-model": asset.flex_model, + } + + for plot_key, flex_config in fields_to_check.items(): + if plot_key not in plot: + continue + + field_keys = plot[plot_key] + data = flex_config or {} + + if isinstance(field_keys, str): + field_keys = [field_keys] + elif not isinstance(field_keys, list): + continue + + for field_key in field_keys: + field_value = data.get(field_key) + + if isinstance(field_value, dict): + # Add a single sensor if it exists + sensor = field_value.get("sensor") + if sensor: + all_sensors.append(sensor) + + return all_sensors diff --git a/flexmeasures/data/services/generic_assets.py b/flexmeasures/data/services/generic_assets.py index ac8bd00e1a..951206ddc5 100644 --- a/flexmeasures/data/services/generic_assets.py +++ b/flexmeasures/data/services/generic_assets.py @@ -1,3 +1,4 @@ +from dictdiffer import diff from flask import current_app from sqlalchemy import delete @@ -6,7 +7,7 @@ from flexmeasures.data.models.audit_log import AssetAuditLog from flexmeasures.data.schemas.scheduling import DBFlexContextSchema from flexmeasures.data.schemas.scheduling.storage import DBStorageFlexModelSchema -from dictdiffer import diff +from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema """Services for managing assets""" @@ -120,6 +121,7 @@ def patch_asset(db_asset: GenericAsset, asset_data: dict) -> GenericAsset: schema_map = dict( flex_context=DBFlexContextSchema, flex_model=DBStorageFlexModelSchema, + sensors_to_show=SensorsToShowSchema, ) for k, v in asset_data.items(): @@ -135,7 +137,11 @@ def patch_asset(db_asset: GenericAsset, asset_data: dict) -> GenericAsset: continue if k in schema_map: # Validate the JSON field against the given schema - schema_map[k]().load(v) + if k != "sensors_to_show": + schema_map[k]().load(v) + else: + # we use `deserialize here because the `SensorsToShowSchema` is a "fields.Field" object rather than a "Schema" object + schema_map[k]().deserialize(v) if k.lower() in {"sensors_to_show", "flex_context", "flex_model"}: audit_log_data.append(format_json_field_change(k, getattr(db_asset, k), v)) diff --git a/flexmeasures/data/tests/test_SensorsToShowSchema.py b/flexmeasures/data/tests/test_SensorsToShowSchema.py index c5eb9919e3..2780baf5b2 100644 --- a/flexmeasures/data/tests/test_SensorsToShowSchema.py +++ b/flexmeasures/data/tests/test_SensorsToShowSchema.py @@ -1,13 +1,14 @@ import pytest from marshmallow import ValidationError +from flexmeasures import Sensor from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema def test_single_sensor_id(): schema = SensorsToShowSchema() input_value = [42] - expected_output = [{"title": None, "sensors": [42]}] + expected_output = [{"title": None, "plots": [{"sensor": 42}]}] assert schema.deserialize(input_value) == expected_output @@ -15,26 +16,70 @@ def test_list_of_sensor_ids(): schema = SensorsToShowSchema() input_value = [42, 43] expected_output = [ - {"title": None, "sensors": [42]}, - {"title": None, "sensors": [43]}, + {"title": None, "plots": [{"sensor": 42}]}, + {"title": None, "plots": [{"sensor": 43}]}, ] assert schema.deserialize(input_value) == expected_output def test_dict_with_title_and_single_sensor(): schema = SensorsToShowSchema() - input_value = [{"title": "Temperature", "sensor": 42}] - expected_output = [{"title": "Temperature", "sensors": [42]}] - assert schema.deserialize(input_value) == expected_output + input_value_one = [{"title": "Temperature", "sensor": 42}] + input_value_two = [{"title": "Temperature", "plots": [{"sensor": 42}]}] + expected_output = [{"title": "Temperature", "plots": [{"sensor": 42}]}] + assert schema.deserialize(input_value_one) == expected_output + assert schema.deserialize(input_value_two) == expected_output def test_dict_with_title_and_multiple_sensors(): schema = SensorsToShowSchema() - input_value = [{"title": "Pressure", "sensors": [42, 43]}] - expected_output = [{"title": "Pressure", "sensors": [42, 43]}] + input_value = [{"title": "Pressure", "plots": [{"sensors": [42, 43]}]}] + expected_output = [{"title": "Pressure", "plots": [{"sensors": [42, 43]}]}] + assert schema.deserialize(input_value) == expected_output + + +def test_dict_with_asset_and_no_title_plot(setup_test_data): + asset_id = setup_test_data["wind-asset-1"].id + schema = SensorsToShowSchema() + input_value = [{"plots": [{"asset": asset_id, "flex-model": "soc-min"}]}] + expected_output = [ + {"title": None, "plots": [{"asset": asset_id, "flex-model": "soc-min"}]} + ] assert schema.deserialize(input_value) == expected_output +def _get_sensor_by_name(sensors: list[Sensor], name: str) -> Sensor: + for sensor in sensors: + if sensor.name == name: + return sensor + raise ValueError(f"Sensor {name} not found") + + +def test_flatten_with_multiple_flex_config_fields(setup_test_data): + asset = setup_test_data["wind-asset-1"] + schema = SensorsToShowSchema() + input_value = [ + { + "plots": [ + { + "asset": asset.id, + "flex-model": ["consumption-capacity", "production-capacity"], + "flex-context": "site-consumption-capacity", + } + ] + } + ] + expected_output = [ + _get_sensor_by_name(asset.sensors, name).id + for name in ( + "site-consumption-capacity", + "consumption-capacity", + "production-capacity", + ) + ] + assert schema.flatten(input_value) == expected_output + + def test_invalid_sensor_string_input(): schema = SensorsToShowSchema() with pytest.raises( @@ -58,7 +103,7 @@ def test_invalid_sensor_dict_without_sensors_key(): input_value = [{"title": "Test", "something_else": 42}] with pytest.raises( ValidationError, - match="Dictionary must contain either 'sensor' or 'sensors' key.", + match="Dictionary must contain either 'sensor', 'sensors' or 'plots' key.", ): schema.deserialize(input_value) @@ -71,9 +116,9 @@ def test_mixed_valid_inputs(): 5, ] expected_output = [ - {"title": "Test", "sensors": [1, 2]}, - {"title": None, "sensors": [3, 4]}, - {"title": None, "sensors": [5]}, + {"title": "Test", "plots": [{"sensors": [1, 2]}]}, + {"title": None, "plots": [{"sensors": [3, 4]}]}, + {"title": None, "plots": [{"sensor": 5}]}, ] assert schema.deserialize(input_value) == expected_output @@ -84,29 +129,12 @@ def test_string_json_input(): '[{"title": "Test", "sensors": [1, 2]}, {"title": "Test2", "sensors": [3]}]' ) expected_output = [ - {"title": "Test", "sensors": [1, 2]}, - {"title": "Test2", "sensors": [3]}, + {"title": "Test", "plots": [{"sensors": [1, 2]}]}, + {"title": "Test2", "plots": [{"sensors": [3]}]}, ] assert schema.deserialize(input_value) == expected_output -# New test cases for misspelled or missing title and mixed sensor/sensors formats - - -def test_dict_missing_title_key(): - schema = SensorsToShowSchema() - input_value = [{"sensor": 42}] - with pytest.raises(ValidationError, match="Dictionary must contain a 'title' key."): - schema.deserialize(input_value) - - -def test_dict_misspelled_title_key(): - schema = SensorsToShowSchema() - input_value = [{"titel": "Temperature", "sensor": 42}] # Misspelled 'title' - with pytest.raises(ValidationError, match="Dictionary must contain a 'title' key."): - schema.deserialize(input_value) - - def test_dict_with_sensor_as_list(): schema = SensorsToShowSchema() input_value = [{"title": "Temperature", "sensor": [42]}] diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 1270d4d516..0a951bc053 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -1,4 +1,4 @@ -""" Various coding utils (e.g. around function decoration) """ +r"""Various coding utils (e.g. around function decoration)""" from __future__ import annotations @@ -71,34 +71,6 @@ def sort_dict(unsorted_dict: dict) -> dict: return sorted_dict -# This function is used for sensors_to_show in follow-up PR it will be moved and renamed to flatten_sensors_to_show -def flatten_unique(nested_list_of_objects: list) -> list: - """ - Get unique sensor IDs from a list of `sensors_to_show`. - - Handles: - - Lists of sensor IDs - - Dictionaries with a `sensors` key - - Nested lists (one level) - - Example: - Input: - [1, [2, 20, 6], 10, [6, 2], {"title":None,"sensors": [10, 15]}, 15] - - Output: - [1, 2, 20, 6, 10, 15] - """ - all_objects = [] - for s in nested_list_of_objects: - if isinstance(s, list): - all_objects.extend(s) - elif isinstance(s, dict): - all_objects.extend(s["sensors"]) - else: - all_objects.append(s) - return list(dict.fromkeys(all_objects).keys()) - - def timeit(func): """Decorator for printing the time it took to execute the decorated function."""