From 6ecb17f6d5b547a6b7bff28833bbeda7d81864c8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:46:24 +0200 Subject: [PATCH 01/28] feature: allow checking permissions on optional fields Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 2655bb42a2..8c09260713 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -178,7 +178,10 @@ def decorated_view(*args, **kwargs): elif ctx_arg_pos is not None: context_from_args = args[ctx_arg_pos] elif ctx_arg_name is not None: - context_from_args = kwargs[ctx_arg_name] + context_from_args = kwargs.get(ctx_arg_name) + # skip check in case (optional) argument was not passed + if context_from_args is None: + return fn(*args, **kwargs) elif len(args) > 0: context_from_args = args[0] From 2546546a26443d696ff265b608884ea35e425eb2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:52:03 +0200 Subject: [PATCH 02/28] feature: decorator supports custom error handler Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 8c09260713..9dcf94ba3e 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -111,6 +111,7 @@ def permission_required_for_context( ctx_arg_name: str | None = None, ctx_loader: Callable | None = None, pass_ctx_to_loader: bool = False, + error_handler: Callable | None = None, ): """ This decorator can be used to make sure that the current user has the necessary permission to access the context. @@ -119,6 +120,7 @@ def permission_required_for_context( A 403 response is raised if there is no principal for the required permission. A 401 response is raised if the user is not authenticated at all. + A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and ctx_arg_name. We will now explain how to load a context, and give an example: @@ -199,7 +201,12 @@ def decorated_view(*args, **kwargs): else: context = context_from_args - check_access(context, permission) + try: + check_access(context, permission) + except Exception as e: + if error_handler: + return error_handler(context, permission, ctx_arg_name) + raise e return fn(*args, **kwargs) From c3a592053293f45b5765138c8c49e4698586a958 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:53:22 +0200 Subject: [PATCH 03/28] docs: add inline note explaining status code Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index be7b67760f..e7fd1f40db 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -84,7 +84,7 @@ def test_trigger_schedule_with_invalid_flexmodel( ) print("Server responded with:\n%s" % trigger_schedule_response.json) check_deprecation(trigger_schedule_response, deprecation=None, sunset=None) - assert trigger_schedule_response.status_code == 422 + assert trigger_schedule_response.status_code == 422 # Unprocessable entity assert field in trigger_schedule_response.json["message"]["json"] if isinstance(trigger_schedule_response.json["message"]["json"], str): # ValueError From 482dd0a9a05da20974d93a528a23e989e7e30793 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:56:23 +0200 Subject: [PATCH 04/28] feature: flex_context_loader lists all sensors contained in a flex-context Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/utils.py | 69 ++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 908cf65e6f..b1ba739fa0 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -441,3 +441,72 @@ def nanmin_of_series_and_value(s: pd.Series, value: float | pd.Series) -> pd.Ser # [right]: datetime64[ns, UTC] value = value.tz_convert("UTC") return s.fillna(value).clip(upper=value) + + +def flex_context_loader(context) -> list[tuple[Sensor, str]]: + sensor_ids = find_sensor_ids(context, "flex-context") + sensors = [ + (db.session.get(Sensor, sensor_id), field_name) + for sensor_id, field_name in sensor_ids + ] + return sensors + + +def find_sensor_ids(data, parent_key="") -> list[tuple[int, str]]: + """ + Recursively find all sensor IDs in a nested dictionary or list along with the fields referring to them. + + Args: + data (dict or list): The input data which can be a dictionary or a list containing nested dictionaries and lists. + parent_key (str): The key of the parent element in the recursion, used to track the referring fields. + + Returns: + list: A list of tuples, each containing a sensor ID and the referring field. + + Example: + nested_dict = { + "flex-model": [ + { + "sensor": 931, + "soc-at-start": 12.1, + "soc-unit": "kWh", + "soc-targets": [ + { + "value": 25, + "datetime": "2015-06-02T16:00:00+00:00" + }, + ], + "soc-minima": {"sensor": 300}, + "soc-min": 10, + "soc-max": 25, + "charging-efficiency": "120%", + "discharging-efficiency": {"sensor": 98}, + "storage-efficiency": 0.9999, + "power-capacity": "25kW", + "consumption-capacity": {"sensor": 42}, + "production-capacity": "30 kW" + }, + ], + } + + sensor_ids = find_sensor_ids(nested_dict) + print(sensor_ids) # Output: [(931, 'sensor'), (300, 'soc-minima.sensor'), (98, 'discharging-efficiency.sensor'), (42, 'consumption-capacity.sensor')] + """ + sensor_ids = [] + + if isinstance(data, dict): + for key, value in data.items(): + new_parent_key = f"{parent_key}.{key}" if parent_key else key + if key[-6:] == "sensor": + sensor_ids.append((value, new_parent_key)) + elif key[-7:] == "sensors": + for v in value: + sensor_ids.append((v, new_parent_key)) + else: + sensor_ids.extend(find_sensor_ids(value, new_parent_key)) + elif isinstance(data, list): + for index, item in enumerate(data): + new_parent_key = f"{parent_key}[{index}]" + sensor_ids.extend(find_sensor_ids(item, new_parent_key)) + + return sensor_ids From cc7248b42b98ab348abadf5e0a97a0a8de7b199e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:01:45 +0200 Subject: [PATCH 05/28] feature: support context loader that returns multiple contexts Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 9dcf94ba3e..70d710c07d 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -120,7 +120,7 @@ def permission_required_for_context( A 403 response is raised if there is no principal for the required permission. A 401 response is raised if the user is not authenticated at all. - A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and ctx_arg_name. + A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and a context origin. We will now explain how to load a context, and give an example: @@ -148,6 +148,7 @@ def view(resource_id: int, the_resource: Resource): The ctx_loader: The ctx_loader can be a function without arguments or it takes the context loaded from the arguments as input (using pass_ctx_to_loader=True). + It should return the context or a list of contexts. A special case is useful when the arguments contain the context ID (not the instance). Then, the loader can be a subclass of AuthModelMixin, and this decorator will look up the instance. @@ -201,12 +202,22 @@ def decorated_view(*args, **kwargs): else: context = context_from_args - try: - check_access(context, permission) - except Exception as e: - if error_handler: - return error_handler(context, permission, ctx_arg_name) - raise e + # Check access for possibly multiple contexts + if not isinstance(context, list): + context = [context] + for ctx in context: + if isinstance(ctx, tuple): + c = ctx[0] # the context + origin = ctx[1] # the context loader may narrow down the origin of the context (e.g. a nested field rather than a function argument) + else: + c = ctx + origin = ctx_arg_name + try: + check_access(c, permission) + except Exception as e: + if error_handler: + return error_handler(c, permission, origin) + raise e return fn(*args, **kwargs) From 0b41e772f2571fad8504898891bd4aeca341f2ec Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:04:14 +0200 Subject: [PATCH 06/28] feature: check permissions on sensors referenced in flex-context Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index edb31eace8..0d6a953db8 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -33,6 +33,7 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.models.planning.utils import flex_context_loader from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField @@ -225,6 +226,16 @@ def get_data(self, sensor_data_description: dict): location="json", ) @permission_required_for_context("create-children", ctx_arg_name="sensor") + @permission_required_for_context( + "read", + ctx_arg_name="flex_context", + ctx_loader=flex_context_loader, + pass_ctx_to_loader=True, + error_handler=lambda context, permission, origin: invalid_flex_config( + f"User has no {permission} authorization on sensor {context.id}", + origin, + ), + ) def trigger_schedule( self, sensor: Sensor, From e616d1cfc71b7cbb2a9456a28bfedf5d474d30b4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:04:38 +0200 Subject: [PATCH 07/28] feature: add test checking permissions Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 13 +++++ .../api/v3_0/tests/test_sensor_schedules.py | 48 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 74a70383f0..8d2277892f 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -201,3 +201,16 @@ def add_temperature_measurements(db, source: Source, sensor: Sensor): for event_start, event_value in zip(event_starts, event_values) ] db.session.add_all(beliefs) + + +@pytest.fixture(scope="module") +def setup_capacity_sensor_on_asset_in_supplier_account(db, setup_generic_assets): + asset = setup_generic_assets["test_wind_turbine"] + sensor = Sensor( + name="capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MVA", + ) + db.session.add(sensor) + return sensor diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index e7fd1f40db..4428be296c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -422,3 +422,51 @@ def test_get_schedule_fallback_not_redirect( assert schedule["scheduler_info"]["scheduler"] == "StorageFallbackScheduler" app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False + + +@pytest.mark.parametrize( + "message, flex_config, field, err_msg", + [ + (message_for_trigger_schedule(), "flex-context", "site-consumption-capacity", "no read authorization"), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_with_unauthorized_sensor( + app, + add_battery_assets, + setup_capacity_sensor_on_asset_in_supplier_account, + keep_scheduling_queue_empty, + message, + flex_config, + field, + err_msg, + requesting_user, +): + """Test triggering a schedule using a flex config that refers to a capacity sensor from a different account. + + The user is not authorized to read sensors from the other account, so we expect a 403 (Forbidden) response. + """ + sensor = add_battery_assets["Test battery"].sensors[0] + with app.test_client() as client: + if flex_config not in message: + message[flex_config] = {} + sensor_id = setup_capacity_sensor_on_asset_in_supplier_account.id + message[flex_config][field] = {"sensor": sensor_id} + + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + print("Server responded with:\n%s" % trigger_schedule_response.json) + assert trigger_schedule_response.status_code == 422 # Unprocessable entity + assert f"{flex_config}.{field}.sensor" in trigger_schedule_response.json["message"] + if isinstance(trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"], str): + # ValueError + assert err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"] + else: + # ValidationError (marshmallow) + assert ( + err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"][field][0] + ) From 019d77242e7711551e1517fd1956cbc2df5a1866 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:08:44 +0200 Subject: [PATCH 08/28] fix: response with field names Signed-off-by: F.N. Claessen --- flexmeasures/api/common/responses.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index c0e959a62f..1f0d3bc763 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -274,10 +274,12 @@ def fallback_schedule_redirect(message: str, location: str) -> ResponseTuple: ) -def invalid_flex_config(message: str) -> ResponseTuple: +def invalid_flex_config(message: str, field_name: str | None = None) -> ResponseTuple: return ( dict( - result="Rejected", status="UNPROCESSABLE_ENTITY", message=dict(json=message) + result="Rejected", + status="UNPROCESSABLE_ENTITY", + message={field_name if field_name else "json": message}, ), 422, ) From aa27d79d5c659d00a13ed44d448e924ea8739e3e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 09:54:46 +0200 Subject: [PATCH 09/28] feature: check permissions on sensors referenced in flex-model Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 12 +++++++++++- flexmeasures/data/models/planning/utils.py | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 0d6a953db8..0a0a7b3cc0 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -33,7 +33,7 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.models.planning.utils import flex_context_loader +from flexmeasures.data.models.planning.utils import flex_context_loader, flex_model_loader from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField @@ -226,6 +226,16 @@ def get_data(self, sensor_data_description: dict): location="json", ) @permission_required_for_context("create-children", ctx_arg_name="sensor") + @permission_required_for_context( + "read", + ctx_arg_name="flex_model", + ctx_loader=flex_model_loader, + pass_ctx_to_loader=True, + error_handler=lambda context, permission, origin: invalid_flex_config( + f"User has no {permission} authorization on sensor {context.id}", + origin, + ), + ) @permission_required_for_context( "read", ctx_arg_name="flex_context", diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index b1ba739fa0..ce0650cb99 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -443,6 +443,15 @@ def nanmin_of_series_and_value(s: pd.Series, value: float | pd.Series) -> pd.Ser return s.fillna(value).clip(upper=value) +def flex_model_loader(context) -> list[tuple[Sensor, str]]: + sensor_ids = find_sensor_ids(context, "flex-model") + sensors = [ + (db.session.get(Sensor, sensor_id), field_name) + for sensor_id, field_name in sensor_ids + ] + return sensors + + def flex_context_loader(context) -> list[tuple[Sensor, str]]: sensor_ids = find_sensor_ids(context, "flex-context") sensors = [ From 6ea9202c6374075ed1e3d14a379e59c05bf8a468 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 10:15:11 +0200 Subject: [PATCH 10/28] refactor: sensor_loader Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/utils.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index ce0650cb99..c4956ced7f 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +from functools import partial from packaging import version from datetime import date, datetime, timedelta @@ -443,8 +444,14 @@ def nanmin_of_series_and_value(s: pd.Series, value: float | pd.Series) -> pd.Ser return s.fillna(value).clip(upper=value) -def flex_model_loader(context) -> list[tuple[Sensor, str]]: - sensor_ids = find_sensor_ids(context, "flex-model") +def sensor_loader(data, parent_key: str) -> list[tuple[Sensor, str]]: + """Load all sensors referenced by their ID in a nested dict or list, along with the fields referring to them. + + :param data: nested dict or list + :param parent_key: 'flex-model' or 'flex-context' + :returns: list of sensor-field tuples + """ + sensor_ids = find_sensor_ids(data, parent_key) sensors = [ (db.session.get(Sensor, sensor_id), field_name) for sensor_id, field_name in sensor_ids @@ -452,13 +459,8 @@ def flex_model_loader(context) -> list[tuple[Sensor, str]]: return sensors -def flex_context_loader(context) -> list[tuple[Sensor, str]]: - sensor_ids = find_sensor_ids(context, "flex-context") - sensors = [ - (db.session.get(Sensor, sensor_id), field_name) - for sensor_id, field_name in sensor_ids - ] - return sensors +flex_model_loader = partial(sensor_loader, parent_key="flex-model") +flex_context_loader = partial(sensor_loader, parent_key="flex-context") def find_sensor_ids(data, parent_key="") -> list[tuple[int, str]]: From f72ab1489a65416536b64bac6508ae1c4c0c4a38 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 10:18:25 +0200 Subject: [PATCH 11/28] feature: add test case for unauthorized sensor in flex-model Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 4428be296c..ae30d08288 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -428,6 +428,7 @@ def test_get_schedule_fallback_not_redirect( "message, flex_config, field, err_msg", [ (message_for_trigger_schedule(), "flex-context", "site-consumption-capacity", "no read authorization"), + (message_for_trigger_schedule(), "flex-model", "site-consumption-capacity", "no read authorization"), ], ) @pytest.mark.parametrize( From edfa646863dfc4daf642b7a54843106542ac3322 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 10:33:24 +0200 Subject: [PATCH 12/28] style: black Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensor_schedules.py | 35 +++++++++++++++---- flexmeasures/auth/decorators.py | 3 +- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index ae30d08288..2d6c585484 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -427,8 +427,18 @@ def test_get_schedule_fallback_not_redirect( @pytest.mark.parametrize( "message, flex_config, field, err_msg", [ - (message_for_trigger_schedule(), "flex-context", "site-consumption-capacity", "no read authorization"), - (message_for_trigger_schedule(), "flex-model", "site-consumption-capacity", "no read authorization"), + ( + message_for_trigger_schedule(), + "flex-context", + "site-consumption-capacity", + "no read authorization", + ), + ( + message_for_trigger_schedule(), + "flex-model", + "site-consumption-capacity", + "no read authorization", + ), ], ) @pytest.mark.parametrize( @@ -462,12 +472,25 @@ def test_trigger_schedule_with_unauthorized_sensor( ) print("Server responded with:\n%s" % trigger_schedule_response.json) assert trigger_schedule_response.status_code == 422 # Unprocessable entity - assert f"{flex_config}.{field}.sensor" in trigger_schedule_response.json["message"] - if isinstance(trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"], str): + assert ( + f"{flex_config}.{field}.sensor" in trigger_schedule_response.json["message"] + ) + if isinstance( + trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"], + str, + ): # ValueError - assert err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"] + assert ( + err_msg + in trigger_schedule_response.json["message"][ + f"{flex_config}.{field}.sensor" + ] + ) else: # ValidationError (marshmallow) assert ( - err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"][field][0] + err_msg + in trigger_schedule_response.json["message"][ + f"{flex_config}.{field}.sensor" + ][field][0] ) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 70d710c07d..01c3910278 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -208,7 +208,8 @@ def decorated_view(*args, **kwargs): for ctx in context: if isinstance(ctx, tuple): c = ctx[0] # the context - origin = ctx[1] # the context loader may narrow down the origin of the context (e.g. a nested field rather than a function argument) + # the context loader may narrow down the origin of the context (e.g. a nested field rather than a function argument) + origin = ctx[1] else: c = ctx origin = ctx_arg_name From 16684e68600bca817ff11953baae862e23cffb39 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 10:35:44 +0200 Subject: [PATCH 13/28] style: black once more Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 0a0a7b3cc0..9a1e487bc1 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -33,7 +33,10 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.models.planning.utils import flex_context_loader, flex_model_loader +from flexmeasures.data.models.planning.utils import ( + flex_context_loader, + flex_model_loader, +) from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField From 7a12c4974f06c6f6bb6de582b2602b1767611a89 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:01:38 +0200 Subject: [PATCH 14/28] style: flake8 Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 75 ++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 01c3910278..0907e1c95e 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -147,7 +147,7 @@ def view(resource_id: int, the_resource: Resource): The ctx_loader: - The ctx_loader can be a function without arguments or it takes the context loaded from the arguments as input (using pass_ctx_to_loader=True). + The ctx_loader can be a function without arguments, or it takes the context loaded from the arguments as input (using pass_ctx_to_loader=True). It should return the context or a list of contexts. A special case is useful when the arguments contain the context ID (not the instance). Then, the loader can be a subclass of AuthModelMixin, and this decorator will look up the instance. @@ -171,43 +171,20 @@ def post(self, resource_data: dict): def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): - # load & check context - context: AuthModelMixin = None - - # first set context_from_args, if possible - context_from_args: AuthModelMixin = None - if ctx_arg_pos is not None and ctx_arg_name is not None: - context_from_args = args[ctx_arg_pos][ctx_arg_name] - elif ctx_arg_pos is not None: - context_from_args = args[ctx_arg_pos] - elif ctx_arg_name is not None: - context_from_args = kwargs.get(ctx_arg_name) - # skip check in case (optional) argument was not passed - if context_from_args is None: - return fn(*args, **kwargs) - elif len(args) > 0: - context_from_args = args[0] - - # if a loader is given, use that, otherwise fall back to context_from_args - if ctx_loader is not None: - if pass_ctx_to_loader: - if inspect.isclass(ctx_loader) and issubclass( - ctx_loader, AuthModelMixin - ): - context = db.session.get(ctx_loader, context_from_args) - else: - context = ctx_loader(context_from_args) - else: - context = ctx_loader() - else: - context = context_from_args + context = load_context( + ctx_arg_pos, ctx_arg_name, ctx_loader, pass_ctx_to_loader, args, kwargs + ) + + # skip check in case (optional) argument was not passed + if context is None: + return fn(*args, **kwargs) # Check access for possibly multiple contexts if not isinstance(context, list): context = [context] for ctx in context: if isinstance(ctx, tuple): - c = ctx[0] # the context + c = ctx[0] # c[0] is the context, c[1] is its origin # the context loader may narrow down the origin of the context (e.g. a nested field rather than a function argument) origin = ctx[1] else: @@ -215,7 +192,7 @@ def decorated_view(*args, **kwargs): origin = ctx_arg_name try: check_access(c, permission) - except Exception as e: + except Exception as e: # noqa: B902 if error_handler: return error_handler(c, permission, origin) raise e @@ -225,3 +202,35 @@ def decorated_view(*args, **kwargs): return decorated_view return wrapper + + +def load_context( + ctx_arg_pos, ctx_arg_name, ctx_loader, pass_ctx_to_loader, args, kwargs +) -> AuthModelMixin | None: + + # first set context_from_args, if possible + context_from_args: AuthModelMixin | None = None + if ctx_arg_pos is not None and ctx_arg_name is not None: + context_from_args = args[ctx_arg_pos][ctx_arg_name] + elif ctx_arg_pos is not None: + context_from_args = args[ctx_arg_pos] + elif ctx_arg_name is not None: + context_from_args = kwargs.get(ctx_arg_name) + # skip check in case (optional) argument was not passed + if context_from_args is None: + return None + elif len(args) > 0: + context_from_args = args[0] + + # if a loader is given, use that, otherwise fall back to context_from_args + if ctx_loader is not None: + if pass_ctx_to_loader: + if inspect.isclass(ctx_loader) and issubclass(ctx_loader, AuthModelMixin): + context = db.session.get(ctx_loader, context_from_args) + else: + context = ctx_loader(context_from_args) + else: + context = ctx_loader() + else: + context = context_from_args + return context From c7938fa874f33836c7961d47f9de1db1121e2994 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:15:49 +0200 Subject: [PATCH 15/28] docs: fix test docstring Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 2d6c585484..ff3a8a2acc 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -457,7 +457,8 @@ def test_trigger_schedule_with_unauthorized_sensor( ): """Test triggering a schedule using a flex config that refers to a capacity sensor from a different account. - The user is not authorized to read sensors from the other account, so we expect a 403 (Forbidden) response. + The user is not authorized to read sensors from the other account, + so we expect a 422 (Unprocessable entity) response referring to the relevant flex-config field. """ sensor = add_battery_assets["Test battery"].sensors[0] with app.test_client() as client: From 8733408b883e09d9addbc8c956b7210c005a03be Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:17:16 +0200 Subject: [PATCH 16/28] docs: API changelog entry Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 3f32d6313a..460c20ca96 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,6 +6,11 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace. +v3.0-19 | 2024-05-24 +"""""""""""""""""""" +- Add authorization check on sensors referred to in flex-model and flex-context fields for `/sensors//schedules/trigger` (POST). + + v3.0-18 | 2024-03-07 """""""""""""""""""" - Add support for providing a sensor definition to the ``soc-minima``, ``soc-maxima`` and ``soc-targets`` flex-model fields for `/sensors//schedules/trigger` (POST). From 38d82412e995709e2ee72faaf8325a0a93fc9a76 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:19:22 +0200 Subject: [PATCH 17/28] docs: main changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b3f57f390e..81e10a87ae 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,6 +7,11 @@ FlexMeasures Changelog v0.22.0 | June XX, 2024 ============================ +Bugfixes +----------- + +* Add authorization check on sensors referred to in ``flex-model`` and ``flex-context`` fields [see `PR #1071 `_] + New features ------------- From 97aa1d742ff105c5a812fe782ecee57e2a846860 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 8 Jun 2024 22:10:42 +0200 Subject: [PATCH 18/28] docs: update docstring for supported ctx_loader return values Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 0907e1c95e..96e5a4a379 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -148,7 +148,8 @@ def view(resource_id: int, the_resource: Resource): The ctx_loader: The ctx_loader can be a function without arguments, or it takes the context loaded from the arguments as input (using pass_ctx_to_loader=True). - It should return the context or a list of contexts. + It should return the context, a list of contexts, or a list of (context, origin) tuples, + where an origin defines where (e.g. what API field) the context came from, to help with responding with more informative errors. A special case is useful when the arguments contain the context ID (not the instance). Then, the loader can be a subclass of AuthModelMixin, and this decorator will look up the instance. From b2ec43ae00113ee3a7ce7d7e85f465d1e6c32a13 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 8 Jun 2024 22:13:35 +0200 Subject: [PATCH 19/28] fix: narrow down skipping auth check for optional keyword arguments only. Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 96e5a4a379..52dd7c8fd3 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -172,14 +172,15 @@ def post(self, resource_data: dict): def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): + + # skip check in case (optional) keyword argument was not passed + if ctx_arg_name is not None and kwargs.get(ctx_arg_name) is None: + return fn(*args, **kwargs) + context = load_context( ctx_arg_pos, ctx_arg_name, ctx_loader, pass_ctx_to_loader, args, kwargs ) - # skip check in case (optional) argument was not passed - if context is None: - return fn(*args, **kwargs) - # Check access for possibly multiple contexts if not isinstance(context, list): context = [context] From d7862fcc12df64f8c87315f2d8401cf5d711b071 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 8 Jun 2024 23:53:14 +0200 Subject: [PATCH 20/28] fix: permission_required_for_context returns 403 rather than 422 Signed-off-by: F.N. Claessen --- flexmeasures/api/common/responses.py | 12 ++++++++++-- flexmeasures/api/v3_0/sensors.py | 13 +++++++------ .../api/v3_0/tests/test_sensor_schedules.py | 19 +++++++++++-------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index 1f0d3bc763..57c3b73572 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -177,6 +177,7 @@ def invalid_role(requested_access_role: str) -> ResponseTuple: def invalid_sender( required_permissions: list[str] | None = None, + field_name: str | None = None, ) -> ResponseTuple: """ Signify that the sender is invalid to perform the request. Fits well with 403 errors. @@ -186,7 +187,11 @@ def invalid_sender( if required_permissions: message += f" It requires {p.join(required_permissions)} permission(s)." return ( - dict(result="Rejected", status="INVALID_SENDER", message=message), + dict( + result="Rejected", + status="INVALID_SENDER", + message={"json": {field_name: message}} if field_name else message, + ), FORBIDDEN_STATUS_CODE, ) @@ -275,11 +280,14 @@ def fallback_schedule_redirect(message: str, location: str) -> ResponseTuple: def invalid_flex_config(message: str, field_name: str | None = None) -> ResponseTuple: + """The flex-config is always part of the JSON payload.""" return ( dict( result="Rejected", status="UNPROCESSABLE_ENTITY", - message={field_name if field_name else "json": message}, + message={"json": {field_name: message}} + if field_name + else {"json": message}, ), 422, ) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 4abddb8ceb..210da85b74 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -18,6 +18,7 @@ unrecognized_event, unknown_schedule, invalid_flex_config, + invalid_sender, fallback_schedule_redirect, ) from flexmeasures.api.common.utils.validators import ( @@ -236,9 +237,9 @@ def get_data(self, sensor_data_description: dict): ctx_arg_name="flex_model", ctx_loader=flex_model_loader, pass_ctx_to_loader=True, - error_handler=lambda context, permission, origin: invalid_flex_config( - f"User has no {permission} authorization on sensor {context.id}", - origin, + error_handler=lambda context, permission, origin: invalid_sender( + required_permissions=[f"{permission} sensor {context.id}"], + field_name=origin, ), ) @permission_required_for_context( @@ -246,9 +247,9 @@ def get_data(self, sensor_data_description: dict): ctx_arg_name="flex_context", ctx_loader=flex_context_loader, pass_ctx_to_loader=True, - error_handler=lambda context, permission, origin: invalid_flex_config( - f"User has no {permission} authorization on sensor {context.id}", - origin, + error_handler=lambda context, permission, origin: invalid_sender( + required_permissions=[f"{permission} sensor {context.id}"], + field_name=origin, ), ) def trigger_schedule( diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index ff3a8a2acc..3270c0c5f8 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -431,13 +431,13 @@ def test_get_schedule_fallback_not_redirect( message_for_trigger_schedule(), "flex-context", "site-consumption-capacity", - "no read authorization", + "requires read sensor", ), ( message_for_trigger_schedule(), "flex-model", "site-consumption-capacity", - "no read authorization", + "requires read sensor", ), ], ) @@ -458,7 +458,7 @@ def test_trigger_schedule_with_unauthorized_sensor( """Test triggering a schedule using a flex config that refers to a capacity sensor from a different account. The user is not authorized to read sensors from the other account, - so we expect a 422 (Unprocessable entity) response referring to the relevant flex-config field. + so we expect a 403 (Forbidden) response referring to the relevant flex-config field. """ sensor = add_battery_assets["Test battery"].sensors[0] with app.test_client() as client: @@ -472,18 +472,21 @@ def test_trigger_schedule_with_unauthorized_sensor( json=message, ) print("Server responded with:\n%s" % trigger_schedule_response.json) - assert trigger_schedule_response.status_code == 422 # Unprocessable entity + assert trigger_schedule_response.status_code == 403 # Forbidden assert ( - f"{flex_config}.{field}.sensor" in trigger_schedule_response.json["message"] + f"{flex_config}.{field}.sensor" + in trigger_schedule_response.json["message"]["json"] ) if isinstance( - trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"], + trigger_schedule_response.json["message"]["json"][ + f"{flex_config}.{field}.sensor" + ], str, ): # ValueError assert ( err_msg - in trigger_schedule_response.json["message"][ + in trigger_schedule_response.json["message"]["json"][ f"{flex_config}.{field}.sensor" ] ) @@ -491,7 +494,7 @@ def test_trigger_schedule_with_unauthorized_sensor( # ValidationError (marshmallow) assert ( err_msg - in trigger_schedule_response.json["message"][ + in trigger_schedule_response.json["message"]["json"][ f"{flex_config}.{field}.sensor" ][field][0] ) From 07b864c8ffb3ca22fe724ad0662dcd2db3e9c6f9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 10 Jun 2024 14:40:25 +0200 Subject: [PATCH 21/28] fix: typo Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/tests/test_times.py | 2 +- flexmeasures/data/schemas/times.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/tests/test_times.py b/flexmeasures/data/schemas/tests/test_times.py index 8f3415a525..f113aae3a1 100644 --- a/flexmeasures/data/schemas/tests/test_times.py +++ b/flexmeasures/data/schemas/tests/test_times.py @@ -65,7 +65,7 @@ def test_duration_field_nominal_grounded( ("1H", "Unable to parse duration string"), ("PP1M", "time designator 'T' missing"), ("PT2D", "Unrecognised ISO 8601 date format"), - ("PT40S", "FlexMeasures only support multiples of 1 minute."), + ("PT40S", "FlexMeasures only supports multiples of 1 minute."), ], ) def test_duration_field_invalid(duration_input, error_msg): diff --git a/flexmeasures/data/schemas/times.py b/flexmeasures/data/schemas/times.py index 625099b531..9367e72704 100644 --- a/flexmeasures/data/schemas/times.py +++ b/flexmeasures/data/schemas/times.py @@ -36,7 +36,7 @@ def _deserialize(self, value, attr, obj, **kwargs) -> timedelta | isodate.Durati ) if duration_value.seconds % 60 != 0 or duration_value.microseconds != 0: raise DurationValidationError( - "FlexMeasures only support multiples of 1 minute." + "FlexMeasures only supports multiples of 1 minute." ) return duration_value From b07d76f58a07ad2cd1fde8dac4eb4c7d29e56d1d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 10 Jun 2024 23:41:39 +0200 Subject: [PATCH 22/28] Revert "fix: narrow down skipping auth check for optional keyword arguments only." This reverts commit b2ec43ae00113ee3a7ce7d7e85f465d1e6c32a13. --- flexmeasures/auth/decorators.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 52dd7c8fd3..96e5a4a379 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -172,15 +172,14 @@ def post(self, resource_data: dict): def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): - - # skip check in case (optional) keyword argument was not passed - if ctx_arg_name is not None and kwargs.get(ctx_arg_name) is None: - return fn(*args, **kwargs) - context = load_context( ctx_arg_pos, ctx_arg_name, ctx_loader, pass_ctx_to_loader, args, kwargs ) + # skip check in case (optional) argument was not passed + if context is None: + return fn(*args, **kwargs) + # Check access for possibly multiple contexts if not isinstance(context, list): context = [context] From ff73a4c2159794438e28426acc95124ffbccb0af Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 10 Jun 2024 23:43:39 +0200 Subject: [PATCH 23/28] fix: only return None in case of no context from args, but not when context loaders loads a None Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 96e5a4a379..ae42128cd7 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -234,4 +234,6 @@ def load_context( context = ctx_loader() else: context = context_from_args + if context is None: + raise LookupError(f"No context could be loaded from {context_from_args}.") return context From 110d027e9656daa03a508d4aa90dbac4ba21e703 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 10 Jun 2024 23:51:35 +0200 Subject: [PATCH 24/28] fix: mypy Signed-off-by: F.N. Claessen --- flexmeasures/api/common/responses.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index 57c3b73572..1e1df3aad0 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -279,15 +279,10 @@ def fallback_schedule_redirect(message: str, location: str) -> ResponseTuple: ) -def invalid_flex_config(message: str, field_name: str | None = None) -> ResponseTuple: - """The flex-config is always part of the JSON payload.""" +def invalid_flex_config(message: str) -> ResponseTuple: return ( dict( - result="Rejected", - status="UNPROCESSABLE_ENTITY", - message={"json": {field_name: message}} - if field_name - else {"json": message}, + result="Rejected", status="UNPROCESSABLE_ENTITY", message=dict(json=message) ), 422, ) From 934ac5f0a398f4d1140795e8f1a416596c602392 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 29 Nov 2025 10:51:18 +0100 Subject: [PATCH 25/28] fix: pass_if_no_context_found Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 2 ++ flexmeasures/auth/decorators.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 6406314945..eb2ee180ab 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -604,6 +604,7 @@ def get_data(self, id: int, **sensor_data_description: dict): ctx_arg_name="flex_model", ctx_loader=flex_model_loader, pass_ctx_to_loader=True, + pass_if_no_context_found=True, error_handler=lambda context, permission, origin: invalid_sender( required_permissions=[f"{permission} sensor {context.id}"], field_name=origin, @@ -614,6 +615,7 @@ def get_data(self, id: int, **sensor_data_description: dict): ctx_arg_name="flex_context", ctx_loader=flex_context_loader, pass_ctx_to_loader=True, + pass_if_no_context_found=True, error_handler=lambda context, permission, origin: invalid_sender( required_permissions=[f"{permission} sensor {context.id}"], field_name=origin, diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index e2025ad3d2..76a6da1c96 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -110,6 +110,7 @@ def permission_required_for_context( ctx_arg_name: str | None = None, ctx_loader: Callable | None = None, pass_ctx_to_loader: bool = False, + pass_if_no_context_found: bool = False, error_handler: Callable | None = None, ): """ @@ -176,7 +177,7 @@ def decorated_view(*args, **kwargs): ) # skip check in case (optional) argument was not passed - if context is None: + if context is None and pass_if_no_context_found: return fn(*args, **kwargs) # Check access for possibly multiple contexts From a53db55279610326b7156046df2ed8fc3c7bac74 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 29 Nov 2025 10:56:36 +0100 Subject: [PATCH 26/28] refactor: set load_defaults for flex_model and flex_context instead; so now permission_required_for_context still doesn't work with optional contexts Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 5 ++--- flexmeasures/auth/decorators.py | 5 ----- flexmeasures/ui/static/openapi-specs.json | 2 ++ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index eb2ee180ab..b9c3f5278a 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -119,13 +119,14 @@ class TriggerScheduleKwargsSchema(Schema): metadata=dict( description="The flex-model is validated according to the scheduler's `FlexModelSchema`.", ), + load_default={}, ) flex_context = fields.Dict( - required=False, data_key="flex-context", metadata=dict( description="The flex-context is validated according to the scheduler's `FlexContextSchema`.", ), + load_default={}, ) force_new_job_creation = fields.Boolean( required=False, @@ -604,7 +605,6 @@ def get_data(self, id: int, **sensor_data_description: dict): ctx_arg_name="flex_model", ctx_loader=flex_model_loader, pass_ctx_to_loader=True, - pass_if_no_context_found=True, error_handler=lambda context, permission, origin: invalid_sender( required_permissions=[f"{permission} sensor {context.id}"], field_name=origin, @@ -615,7 +615,6 @@ def get_data(self, id: int, **sensor_data_description: dict): ctx_arg_name="flex_context", ctx_loader=flex_context_loader, pass_ctx_to_loader=True, - pass_if_no_context_found=True, error_handler=lambda context, permission, origin: invalid_sender( required_permissions=[f"{permission} sensor {context.id}"], field_name=origin, diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 76a6da1c96..85a5fb6900 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -110,7 +110,6 @@ def permission_required_for_context( ctx_arg_name: str | None = None, ctx_loader: Callable | None = None, pass_ctx_to_loader: bool = False, - pass_if_no_context_found: bool = False, error_handler: Callable | None = None, ): """ @@ -176,10 +175,6 @@ def decorated_view(*args, **kwargs): ctx_arg_pos, ctx_arg_name, ctx_loader, pass_ctx_to_loader, args, kwargs ) - # skip check in case (optional) argument was not passed - if context is None and pass_if_no_context_found: - return fn(*args, **kwargs) - # Check access for possibly multiple contexts if not isinstance(context, list): context = [context] diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index e1ce0f9019..97c41995da 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4339,11 +4339,13 @@ }, "flex-model": { "type": "object", + "default": {}, "description": "The flex-model is validated according to the scheduler's `FlexModelSchema`.", "additionalProperties": {} }, "flex-context": { "type": "object", + "default": {}, "description": "The flex-context is validated according to the scheduler's `FlexContextSchema`.", "additionalProperties": {} }, From 3e78412798a01b106fe214364aacd66b398f1443 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 29 Nov 2025 15:36:57 +0100 Subject: [PATCH 27/28] feat: also apply auth decorators to asset trigger endpoint Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 25 +++++++++++++ flexmeasures/data/models/planning/utils.py | 41 ++++++++++++++-------- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 88e93b0300..a7a32710c6 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -53,6 +53,7 @@ from flexmeasures.api.common.utils.api_utils import get_accessible_accounts from flexmeasures.api.common.responses import ( invalid_flex_config, + invalid_sender, request_processed, ) from flexmeasures.api.common.schemas.users import AccountIdField @@ -64,6 +65,10 @@ from flexmeasures.data.schemas.sensors import ( SensorSchema, ) +from flexmeasures.data.models.planning.utils import ( + flex_context_loader, + flex_model_loader, +) from flexmeasures.data.models.time_series import Sensor from flexmeasures.utils.time_utils import naturalized_datetime_str from flexmeasures.data.utils import get_downsample_function_and_value @@ -1189,6 +1194,26 @@ def update_keep_legends_below_graphs(self, **kwargs): # Simplification of checking for create-children access on each of the flexible sensors, # which assumes each of the flexible sensors belongs to the given asset. @permission_required_for_context("create-children", ctx_arg_name="asset") + @permission_required_for_context( + "read", + ctx_arg_name="flex_model", + ctx_loader=flex_model_loader, + pass_ctx_to_loader=True, + error_handler=lambda context, permission, origin: invalid_sender( + required_permissions=[f"{permission} sensor {context.id}"], + field_name=origin, + ), + ) + @permission_required_for_context( + "read", + ctx_arg_name="flex_context", + ctx_loader=flex_context_loader, + pass_ctx_to_loader=True, + error_handler=lambda context, permission, origin: invalid_sender( + required_permissions=[f"{permission} sensor {context.id}"], + field_name=origin, + ), + ) def trigger_schedule( self, asset: GenericAsset, diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index bb8174d612..d2df0ad871 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -10,11 +10,11 @@ import numpy as np import timely_beliefs as tb -from flexmeasures.data import db from flexmeasures.data.models.planning.exceptions import UnknownPricesException from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.planning import StockCommitment from flexmeasures.data.queries.utils import simplify_index +from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.utils.flexmeasures_inflection import capitalize, pluralize from flexmeasures.utils.unit_utils import ur, convert_units @@ -565,11 +565,7 @@ def sensor_loader(data, parent_key: str) -> list[tuple[Sensor, str]]: :param parent_key: 'flex-model' or 'flex-context' :returns: list of sensor-field tuples """ - sensor_ids = find_sensor_ids(data, parent_key) - sensors = [ - (db.session.get(Sensor, sensor_id), field_name) - for sensor_id, field_name in sensor_ids - ] + sensors = find_sensors(data, parent_key) return sensors @@ -577,16 +573,16 @@ def sensor_loader(data, parent_key: str) -> list[tuple[Sensor, str]]: flex_context_loader = partial(sensor_loader, parent_key="flex-context") -def find_sensor_ids(data, parent_key="") -> list[tuple[int, str]]: +def find_sensors(data, parent_key="") -> list[tuple[Sensor, str]]: """ - Recursively find all sensor IDs in a nested dictionary or list along with the fields referring to them. + Recursively find all sensors in a nested dictionary or list along with the fields referring to them. Args: data (dict or list): The input data which can be a dictionary or a list containing nested dictionaries and lists. parent_key (str): The key of the parent element in the recursion, used to track the referring fields. Returns: - list: A list of tuples, each containing a sensor ID and the referring field. + list: A list of tuples, each containing a sensor and the field that referred to it. Example: nested_dict = { @@ -614,8 +610,8 @@ def find_sensor_ids(data, parent_key="") -> list[tuple[int, str]]: ], } - sensor_ids = find_sensor_ids(nested_dict) - print(sensor_ids) # Output: [(931, 'sensor'), (300, 'soc-minima.sensor'), (98, 'discharging-efficiency.sensor'), (42, 'consumption-capacity.sensor')] + sensors = find_sensors(nested_dict) + print(sensors) # Output: [(, 'sensor'), (, 'soc-minima.sensor'), (, 'discharging-efficiency.sensor'), (, 'consumption-capacity.sensor')] """ sensor_ids = [] @@ -623,15 +619,30 @@ def find_sensor_ids(data, parent_key="") -> list[tuple[int, str]]: for key, value in data.items(): new_parent_key = f"{parent_key}.{key}" if parent_key else key if key[-6:] == "sensor": - sensor_ids.append((value, new_parent_key)) + sensor = deserialize_to_sensor_if_needed(value) + sensor_ids.append((sensor, new_parent_key)) elif key[-7:] == "sensors": for v in value: - sensor_ids.append((v, new_parent_key)) + sensor = deserialize_to_sensor_if_needed(v) + sensor_ids.append((sensor, new_parent_key)) else: - sensor_ids.extend(find_sensor_ids(value, new_parent_key)) + sensor_ids.extend(find_sensors(value, new_parent_key)) elif isinstance(data, list): for index, item in enumerate(data): new_parent_key = f"{parent_key}[{index}]" - sensor_ids.extend(find_sensor_ids(item, new_parent_key)) + sensor_ids.extend(find_sensors(item, new_parent_key)) return sensor_ids + + +def deserialize_to_sensor_if_needed(value: int | Sensor) -> Sensor: + """Ensure all sensor values are deserialized. + + The multi-asset flex-model already deserialized the power sensors of each device. + Other fields still need to be deserialized. + """ + if isinstance(value, Sensor): + sensor = value + else: + sensor = SensorIdField().deserialize(value) + return sensor From 70e36159b398d700f6827a785c0b33516fc2d843 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 29 Nov 2025 15:42:49 +0100 Subject: [PATCH 28/28] refactor: one less method Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/utils.py | 42 ++++++++-------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index d2df0ad871..a3ac8cfcb2 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -558,31 +558,15 @@ def initialize_device_commitment( return stock_commitment -def sensor_loader(data, parent_key: str) -> list[tuple[Sensor, str]]: - """Load all sensors referenced by their ID in a nested dict or list, along with the fields referring to them. +def sensor_loader( + data: dict | list[dict], parent_key: str = "" +) -> list[tuple[Sensor, str]]: + """Recursively load all sensors referenced by ID in a nested dict or list, along with the fields referring to them. - :param data: nested dict or list - :param parent_key: 'flex-model' or 'flex-context' - :returns: list of sensor-field tuples - """ - sensors = find_sensors(data, parent_key) - return sensors - - -flex_model_loader = partial(sensor_loader, parent_key="flex-model") -flex_context_loader = partial(sensor_loader, parent_key="flex-context") - - -def find_sensors(data, parent_key="") -> list[tuple[Sensor, str]]: - """ - Recursively find all sensors in a nested dictionary or list along with the fields referring to them. - - Args: - data (dict or list): The input data which can be a dictionary or a list containing nested dictionaries and lists. - parent_key (str): The key of the parent element in the recursion, used to track the referring fields. - - Returns: - list: A list of tuples, each containing a sensor and the field that referred to it. + :param data: Nested dict or list thereof. + :param parent_key: The key of the parent element in the recursion, used to track the referring fields. + For example, 'flex-model' or 'flex-context'. + :returns: A list of tuples, each containing a sensor and the field that referred to it. Example: nested_dict = { @@ -610,7 +594,7 @@ def find_sensors(data, parent_key="") -> list[tuple[Sensor, str]]: ], } - sensors = find_sensors(nested_dict) + sensors = sensor_loader(nested_dict) print(sensors) # Output: [(, 'sensor'), (, 'soc-minima.sensor'), (, 'discharging-efficiency.sensor'), (, 'consumption-capacity.sensor')] """ sensor_ids = [] @@ -626,11 +610,11 @@ def find_sensors(data, parent_key="") -> list[tuple[Sensor, str]]: sensor = deserialize_to_sensor_if_needed(v) sensor_ids.append((sensor, new_parent_key)) else: - sensor_ids.extend(find_sensors(value, new_parent_key)) + sensor_ids.extend(sensor_loader(value, new_parent_key)) elif isinstance(data, list): for index, item in enumerate(data): new_parent_key = f"{parent_key}[{index}]" - sensor_ids.extend(find_sensors(item, new_parent_key)) + sensor_ids.extend(sensor_loader(item, new_parent_key)) return sensor_ids @@ -646,3 +630,7 @@ def deserialize_to_sensor_if_needed(value: int | Sensor) -> Sensor: else: sensor = SensorIdField().deserialize(value) return sensor + + +flex_model_loader = partial(sensor_loader, parent_key="flex-model") +flex_context_loader = partial(sensor_loader, parent_key="flex-context")