From 614a12e3e10215a94bf305d4cf06060c1f7bd46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 11 Dec 2024 09:43:33 +0100 Subject: [PATCH 01/13] LayerCollection get or create unique collection --- geoengine/layers.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/geoengine/layers.py b/geoengine/layers.py index fb3f6457..26a92890 100644 --- a/geoengine/layers.py +++ b/geoengine/layers.py @@ -16,6 +16,7 @@ from geoengine.error import ModificationNotOnLayerDbException, InputException from geoengine.tasks import Task, TaskId from geoengine.types import Symbology +from geoengine.permissions import Resource, add_permission, RoleId, Permission from geoengine.workflow import Workflow, WorkflowId from geoengine.workflow_builder.operators import Operator as WorkflowBuilderOperator @@ -456,6 +457,48 @@ def search(self, search_string: str, *, return listings + def get_or_create_unique_collection( + self, + collection_name: str, + create_collection_description: Optional[str] = None, + delete_existing_with_same_name: bool = False, + create_permissions_tuples: Optional[List[(RoleId, Permission)]] = None + ) -> LayerCollection: + ''' + Get a unique child by name OR if it does not exist create it. + Removes existing collections with same name if forced! + Sets permissions if the collection is created from a list of tuples + ''' + parent_collection = self.reload() # reload just to be safe since self's state change on the server + existing_collections = parent_collection.get_items_by_name(collection_name) + + if delete_existing_with_same_name and len(existing_collections) > 0: + for c in existing_collections: + c.load().remove() + parent_collection = parent_collection.reload() + existing_collections = parent_collection.get_items_by_name(collection_name) + + if len(existing_collections) == 0: + new_desc = create_collection_description if create_collection_description is not None else collection_name + new_collection = parent_collection.add_collection(collection_name, new_desc) + new_ressource = Resource.from_layer_collection_id(new_collection) + + if create_permissions_tuples is not None: + for (role, perm) in create_permissions_tuples: + add_permission(role, new_ressource, perm) + parent_collection = parent_collection.reload() + existing_collections = parent_collection.get_items_by_name(collection_name) + + if len(existing_collections) == 0: + raise KeyError( + f"No collection with name {collection_name} exists in {parent_collection.name} and none was created!" + ) + + if len(existing_collections) > 1: + raise KeyError(f"Multiple collections with name {collection_name} exist in {parent_collection.name}") + + return existing_collections[0].load() + @dataclass(repr=False) class Layer: From 27d1cfbf3bf215a8ea731030e83139b10f7275cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 11 Dec 2024 10:15:58 +0100 Subject: [PATCH 02/13] cyclic imports and instances --- geoengine/layers.py | 20 +++++++++++++------- geoengine/permissions.py | 6 +++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/geoengine/layers.py b/geoengine/layers.py index 26a92890..bc8854b2 100644 --- a/geoengine/layers.py +++ b/geoengine/layers.py @@ -7,7 +7,7 @@ from enum import auto from io import StringIO import os -from typing import Any, Dict, Generic, List, Literal, NewType, Optional, TypeVar, Union, cast +from typing import Any, Dict, Generic, List, Literal, NewType, Optional, TypeVar, Union, cast, Tuple from uuid import UUID import json from strenum import LowercaseStrEnum @@ -16,7 +16,7 @@ from geoengine.error import ModificationNotOnLayerDbException, InputException from geoengine.tasks import Task, TaskId from geoengine.types import Symbology -from geoengine.permissions import Resource, add_permission, RoleId, Permission +import geoengine.permissions as ge_permissions from geoengine.workflow import Workflow, WorkflowId from geoengine.workflow_builder.operators import Operator as WorkflowBuilderOperator @@ -462,7 +462,7 @@ def get_or_create_unique_collection( collection_name: str, create_collection_description: Optional[str] = None, delete_existing_with_same_name: bool = False, - create_permissions_tuples: Optional[List[(RoleId, Permission)]] = None + create_permissions_tuples: Optional[List[Tuple[ge_permissions.RoleId, ge_permissions.Permission]]] = None ) -> LayerCollection: ''' Get a unique child by name OR if it does not exist create it. @@ -474,18 +474,20 @@ def get_or_create_unique_collection( if delete_existing_with_same_name and len(existing_collections) > 0: for c in existing_collections: - c.load().remove() + actual = c.load() + if isinstance(actual, LayerCollection): + actual.remove() parent_collection = parent_collection.reload() existing_collections = parent_collection.get_items_by_name(collection_name) if len(existing_collections) == 0: new_desc = create_collection_description if create_collection_description is not None else collection_name new_collection = parent_collection.add_collection(collection_name, new_desc) - new_ressource = Resource.from_layer_collection_id(new_collection) + new_ressource = ge_permissions.Resource.from_layer_collection_id(new_collection) if create_permissions_tuples is not None: for (role, perm) in create_permissions_tuples: - add_permission(role, new_ressource, perm) + ge_permissions.add_permission(role, new_ressource, perm) parent_collection = parent_collection.reload() existing_collections = parent_collection.get_items_by_name(collection_name) @@ -497,7 +499,11 @@ def get_or_create_unique_collection( if len(existing_collections) > 1: raise KeyError(f"Multiple collections with name {collection_name} exist in {parent_collection.name}") - return existing_collections[0].load() + res = existing_collections[0].load() + if isinstance(res, Layer): + raise TypeError(f"Found a Layer not a Layer collection for {collection_name}") + + return cast(LayerCollection, existing_collections[0].load()) # we know that it is a collection since check that @dataclass(repr=False) diff --git a/geoengine/permissions.py b/geoengine/permissions.py index 2e5697ad..3d16665d 100644 --- a/geoengine/permissions.py +++ b/geoengine/permissions.py @@ -14,7 +14,7 @@ from geoengine.auth import get_session from geoengine.datasets import DatasetName from geoengine.error import GeoEngineException -from geoengine.layers import LayerCollectionId, LayerId +import geoengine.layers as ge_layers class RoleId: @@ -89,12 +89,12 @@ def __init__(self, resource_type: Literal['dataset', 'layer', 'layerCollection'] self.__id = resource_id @classmethod - def from_layer_id(cls, layer_id: LayerId) -> Resource: + def from_layer_id(cls, layer_id: ge_layers.LayerId) -> Resource: '''Create a resource id from a layer id''' return Resource('layer', str(layer_id)) @classmethod - def from_layer_collection_id(cls, layer_collection_id: LayerCollectionId) -> Resource: + def from_layer_collection_id(cls, layer_collection_id: ge_layers.LayerCollectionId) -> Resource: '''Create a resource id from a layer collection id''' return Resource('layerCollection', str(layer_collection_id)) From c70d3254ab081b3bc746c2266a19d134df722398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 11 Dec 2024 23:47:34 +0100 Subject: [PATCH 03/13] resolve dependency circle, add method to get dataset info from server --- geoengine/__init__.py | 5 +- geoengine/datasets.py | 77 ++++---------------- geoengine/layers.py | 19 ++--- geoengine/permissions.py | 45 +----------- geoengine/resource_identifier.py | 118 +++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 122 deletions(-) create mode 100644 geoengine/resource_identifier.py diff --git a/geoengine/__init__.py b/geoengine/__init__.py index fca41ddb..f5985ddd 100644 --- a/geoengine/__init__.py +++ b/geoengine/__init__.py @@ -16,11 +16,10 @@ SpatialReferenceMismatchException, check_response_for_error, ModificationNotOnLayerDbException, \ InvalidUrlException, MissingFieldInResponseException from .layers import Layer, LayerCollection, LayerListing, LayerCollectionListing, \ - LayerId, LayerCollectionId, LayerProviderId, \ layer_collection, layer from .ml import register_ml_model, MlModelConfig from .permissions import add_permission, remove_permission, add_role, remove_role, assign_role, revoke_role, \ - ADMIN_ROLE_ID, REGISTERED_USER_ROLE_ID, ANONYMOUS_USER_ROLE_ID, Permission, Resource, UserId, RoleId + ADMIN_ROLE_ID, REGISTERED_USER_ROLE_ID, ANONYMOUS_USER_ROLE_ID, Permission, UserId, RoleId from .tasks import Task, TaskId from .types import QueryRectangle, GeoTransform, \ RasterResultDescriptor, Provenance, UnitlessMeasurement, ContinuousMeasurement, \ @@ -28,6 +27,8 @@ RasterSymbology, VectorSymbology, VectorDataType, VectorResultDescriptor, VectorColumnInfo, \ FeatureDataType, RasterBandDescriptor, DEFAULT_ISO_TIME_FORMAT, RasterColorizer, SingleBandRasterColorizer, \ MultiBandRasterColorizer +from .resource_identifier import LAYER_DB_PROVIDER_ID, LAYER_DB_ROOT_COLLECTION_ID, DatasetName, UploadId, \ + LayerId, LayerCollectionId, LayerProviderId, Resource from .util import clamp_datetime_ms_ns from .workflow import WorkflowId, Workflow, workflow_by_id, register_workflow, get_quota, update_quota diff --git a/geoengine/datasets.py b/geoengine/datasets.py index 3a9d953a..1d41893e 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -10,6 +10,7 @@ from uuid import UUID import tempfile from attr import dataclass +import geoengine_openapi_client.models import numpy as np import geopandas as gpd import geoengine_openapi_client @@ -19,6 +20,7 @@ from geoengine.types import Provenance, RasterSymbology, TimeStep, \ TimeStepGranularity, VectorDataType, VectorResultDescriptor, VectorColumnInfo, \ UnitlessMeasurement, FeatureDataType +from geoengine.resource_identifier import Resource, UploadId, DatasetName class UnixTimeStampType(Enum): @@ -256,71 +258,6 @@ def to_api_enum(self) -> geoengine_openapi_client.OgrSourceErrorSpec: return geoengine_openapi_client.OgrSourceErrorSpec(self.value) -class DatasetName: - '''A wrapper for a dataset id''' - - __dataset_name: str - - def __init__(self, dataset_name: str) -> None: - self.__dataset_name = dataset_name - - @classmethod - def from_response(cls, response: geoengine_openapi_client.CreateDatasetHandler200Response) -> DatasetName: - '''Parse a http response to an `DatasetId`''' - return DatasetName(response.dataset_name) - - def __str__(self) -> str: - return self.__dataset_name - - def __repr__(self) -> str: - return str(self) - - def __eq__(self, other) -> bool: - '''Checks if two dataset ids are equal''' - if not isinstance(other, self.__class__): - return False - - return self.__dataset_name == other.__dataset_name # pylint: disable=protected-access - - def to_api_dict(self) -> geoengine_openapi_client.CreateDatasetHandler200Response: - return geoengine_openapi_client.CreateDatasetHandler200Response( - dataset_name=str(self.__dataset_name) - ) - - -class UploadId: - '''A wrapper for an upload id''' - - __upload_id: UUID - - def __init__(self, upload_id: UUID) -> None: - self.__upload_id = upload_id - - @classmethod - def from_response(cls, response: geoengine_openapi_client.AddCollection200Response) -> UploadId: - '''Parse a http response to an `UploadId`''' - return UploadId(UUID(response.id)) - - def __str__(self) -> str: - return str(self.__upload_id) - - def __repr__(self) -> str: - return str(self) - - def __eq__(self, other) -> bool: - '''Checks if two upload ids are equal''' - if not isinstance(other, self.__class__): - return False - - return self.__upload_id == other.__upload_id # pylint: disable=protected-access - - def to_api_dict(self) -> geoengine_openapi_client.AddCollection200Response: - '''Converts the upload id to a dict for the api''' - return geoengine_openapi_client.AddCollection200Response( - id=str(self.__upload_id) - ) - - class AddDatasetProperties(): '''The properties for adding a dataset''' name: Optional[str] @@ -630,3 +567,13 @@ def list_datasets(offset: int = 0, ) return response + + +def dataset_info_by_name(dataset_name: DatasetName, timeout: int = 60) -> geoengine_openapi_client.models.Dataset: + '''Delete a dataset. The dataset must be owned by the caller.''' + + session = get_session() + + with geoengine_openapi_client.ApiClient(session.configuration) as api_client: + datasets_api = geoengine_openapi_client.DatasetsApi(api_client) + return datasets_api.get_dataset_handler(str(dataset_name), _request_timeout=timeout) diff --git a/geoengine/layers.py b/geoengine/layers.py index bc8854b2..b2606c11 100644 --- a/geoengine/layers.py +++ b/geoengine/layers.py @@ -7,7 +7,7 @@ from enum import auto from io import StringIO import os -from typing import Any, Dict, Generic, List, Literal, NewType, Optional, TypeVar, Union, cast, Tuple +from typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union, cast, Tuple from uuid import UUID import json from strenum import LowercaseStrEnum @@ -16,16 +16,11 @@ from geoengine.error import ModificationNotOnLayerDbException, InputException from geoengine.tasks import Task, TaskId from geoengine.types import Symbology -import geoengine.permissions as ge_permissions +from geoengine.permissions import RoleId, Permission, add_permission from geoengine.workflow import Workflow, WorkflowId from geoengine.workflow_builder.operators import Operator as WorkflowBuilderOperator - -LayerId = NewType('LayerId', str) -LayerCollectionId = NewType('LayerCollectionId', str) -LayerProviderId = NewType('LayerProviderId', UUID) - -LAYER_DB_PROVIDER_ID = LayerProviderId(UUID('ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74')) -LAYER_DB_ROOT_COLLECTION_ID = LayerCollectionId('05102bb3-a855-4a37-8a8a-30026a91fef1') +from geoengine.resource_identifier import LayerCollectionId, LayerId, LayerProviderId, \ + LAYER_DB_PROVIDER_ID, Resource class LayerCollectionListingType(LowercaseStrEnum): @@ -462,7 +457,7 @@ def get_or_create_unique_collection( collection_name: str, create_collection_description: Optional[str] = None, delete_existing_with_same_name: bool = False, - create_permissions_tuples: Optional[List[Tuple[ge_permissions.RoleId, ge_permissions.Permission]]] = None + create_permissions_tuples: Optional[List[Tuple[RoleId, Permission]]] = None ) -> LayerCollection: ''' Get a unique child by name OR if it does not exist create it. @@ -483,11 +478,11 @@ def get_or_create_unique_collection( if len(existing_collections) == 0: new_desc = create_collection_description if create_collection_description is not None else collection_name new_collection = parent_collection.add_collection(collection_name, new_desc) - new_ressource = ge_permissions.Resource.from_layer_collection_id(new_collection) + new_ressource = Resource.from_layer_collection_id(new_collection) if create_permissions_tuples is not None: for (role, perm) in create_permissions_tuples: - ge_permissions.add_permission(role, new_ressource, perm) + add_permission(role, new_ressource, perm) parent_collection = parent_collection.reload() existing_collections = parent_collection.get_items_by_name(collection_name) diff --git a/geoengine/permissions.py b/geoengine/permissions.py index 3d16665d..968c8d99 100644 --- a/geoengine/permissions.py +++ b/geoengine/permissions.py @@ -6,15 +6,14 @@ from enum import Enum import ast -from typing import Dict, Literal, Any +from typing import Dict from uuid import UUID import geoengine_openapi_client from geoengine.auth import get_session -from geoengine.datasets import DatasetName +from geoengine.resource_identifier import Resource from geoengine.error import GeoEngineException -import geoengine.layers as ge_layers class RoleId: @@ -79,46 +78,6 @@ def __repr__(self) -> str: return repr(self.__user_id) -class Resource: - '''A wrapper for a resource id''' - - def __init__(self, resource_type: Literal['dataset', 'layer', 'layerCollection'], - resource_id: str) -> None: - '''Create a resource id''' - self.__type = resource_type - self.__id = resource_id - - @classmethod - def from_layer_id(cls, layer_id: ge_layers.LayerId) -> Resource: - '''Create a resource id from a layer id''' - return Resource('layer', str(layer_id)) - - @classmethod - def from_layer_collection_id(cls, layer_collection_id: ge_layers.LayerCollectionId) -> Resource: - '''Create a resource id from a layer collection id''' - return Resource('layerCollection', str(layer_collection_id)) - - @classmethod - def from_dataset_name(cls, dataset_name: DatasetName) -> Resource: - '''Create a resource id from a dataset id''' - return Resource('dataset', str(dataset_name)) - - def to_api_dict(self) -> geoengine_openapi_client.Resource: - '''Convert to a dict for the API''' - inner: Any = None - - if self.__type == "layer": - inner = geoengine_openapi_client.LayerResource(type="layer", id=self.__id) - elif self.__type == "layerCollection": - inner = geoengine_openapi_client.LayerCollectionResource(type="layerCollection", id=self.__id) - elif self.__type == "project": - inner = geoengine_openapi_client.ProjectResource(type="project", id=self.__id) - elif self.__type == "dataset": - inner = geoengine_openapi_client.DatasetResource(type="dataset", id=self.__id) - - return geoengine_openapi_client.Resource(inner) - - class Permission(str, Enum): '''A permission''' READ = 'Read' diff --git a/geoengine/resource_identifier.py b/geoengine/resource_identifier.py new file mode 100644 index 00000000..4b9b244e --- /dev/null +++ b/geoengine/resource_identifier.py @@ -0,0 +1,118 @@ +''' Types that identify a ressource in the Geo Engine''' + +from __future__ import annotations +from typing import Any, Literal, NewType +from uuid import UUID +import geoengine_openapi_client + +LayerId = NewType('LayerId', str) +LayerCollectionId = NewType('LayerCollectionId', str) +LayerProviderId = NewType('LayerProviderId', UUID) + +LAYER_DB_PROVIDER_ID = LayerProviderId(UUID('ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74')) +LAYER_DB_ROOT_COLLECTION_ID = LayerCollectionId('05102bb3-a855-4a37-8a8a-30026a91fef1') + + +class DatasetName: + '''A wrapper for a dataset id''' + + __dataset_name: str + + def __init__(self, dataset_name: str) -> None: + self.__dataset_name = dataset_name + + @classmethod + def from_response(cls, response: geoengine_openapi_client.CreateDatasetHandler200Response) -> DatasetName: + '''Parse a http response to an `DatasetId`''' + return DatasetName(response.dataset_name) + + def __str__(self) -> str: + return self.__dataset_name + + def __repr__(self) -> str: + return str(self) + + def __eq__(self, other) -> bool: + '''Checks if two dataset ids are equal''' + if not isinstance(other, self.__class__): + return False + + return self.__dataset_name == other.__dataset_name # pylint: disable=protected-access + + def to_api_dict(self) -> geoengine_openapi_client.CreateDatasetHandler200Response: + return geoengine_openapi_client.CreateDatasetHandler200Response( + dataset_name=str(self.__dataset_name) + ) + + +class UploadId: + '''A wrapper for an upload id''' + + __upload_id: UUID + + def __init__(self, upload_id: UUID) -> None: + self.__upload_id = upload_id + + @classmethod + def from_response(cls, response: geoengine_openapi_client.AddCollection200Response) -> UploadId: + '''Parse a http response to an `UploadId`''' + return UploadId(UUID(response.id)) + + def __str__(self) -> str: + return str(self.__upload_id) + + def __repr__(self) -> str: + return str(self) + + def __eq__(self, other) -> bool: + '''Checks if two upload ids are equal''' + if not isinstance(other, self.__class__): + return False + + return self.__upload_id == other.__upload_id # pylint: disable=protected-access + + def to_api_dict(self) -> geoengine_openapi_client.AddCollection200Response: + '''Converts the upload id to a dict for the api''' + return geoengine_openapi_client.AddCollection200Response( + id=str(self.__upload_id) + ) + + +class Resource: + '''A wrapper for a resource id''' + + def __init__(self, resource_type: Literal['dataset', 'layer', 'layerCollection'], + resource_id: str) -> None: + '''Create a resource id''' + self.__type = resource_type + self.__id = resource_id + + @classmethod + def from_layer_id(cls, layer_id: LayerId) -> Resource: + '''Create a resource id from a layer id''' + return Resource('layer', str(layer_id)) + + @classmethod + def from_layer_collection_id(cls, layer_collection_id: LayerCollectionId) -> Resource: + '''Create a resource id from a layer collection id''' + return Resource('layerCollection', str(layer_collection_id)) + + @classmethod + def from_dataset_name(cls, dataset_name: DatasetName) -> Resource: + '''Create a resource id from a dataset id''' + return Resource('dataset', str(dataset_name)) + + def to_api_dict(self) -> geoengine_openapi_client.Resource: + '''Convert to a dict for the API''' + inner: Any = None + + if self.__type == "layer": + inner = geoengine_openapi_client.LayerResource(type="layer", id=self.__id) + elif self.__type == "layerCollection": + inner = geoengine_openapi_client.LayerCollectionResource(type="layerCollection", id=self.__id) + elif self.__type == "project": + inner = geoengine_openapi_client.ProjectResource(type="project", id=self.__id) + elif self.__type == "dataset": + inner = geoengine_openapi_client.DatasetResource(type="dataset", id=self.__id) + + return geoengine_openapi_client.Resource(inner) From ac66bd57fa4718ee4810c92fc1ba6c9ce1bb0629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Thu, 12 Dec 2024 09:42:52 +0100 Subject: [PATCH 04/13] reorder imports --- geoengine/datasets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geoengine/datasets.py b/geoengine/datasets.py index 1d41893e..fd9de7a8 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -10,17 +10,17 @@ from uuid import UUID import tempfile from attr import dataclass +import geoengine_openapi_client import geoengine_openapi_client.models import numpy as np import geopandas as gpd -import geoengine_openapi_client from geoengine import api from geoengine.error import InputException, MissingFieldInResponseException from geoengine.auth import get_session from geoengine.types import Provenance, RasterSymbology, TimeStep, \ TimeStepGranularity, VectorDataType, VectorResultDescriptor, VectorColumnInfo, \ UnitlessMeasurement, FeatureDataType -from geoengine.resource_identifier import Resource, UploadId, DatasetName +from geoengine.resource_identifier import UploadId, DatasetName class UnixTimeStampType(Enum): From f9fe7e5fcd809ed40f44dc1ddc4763cfbf51a054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Mon, 16 Dec 2024 12:00:14 +0100 Subject: [PATCH 05/13] add dataset / layers --- geoengine/datasets.py | 49 +++++++++++++++++++++++++++++++++++++++---- geoengine/layers.py | 20 ++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/geoengine/datasets.py b/geoengine/datasets.py index fd9de7a8..6cd157d4 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -5,12 +5,13 @@ from __future__ import annotations from abc import abstractmethod from pathlib import Path -from typing import List, NamedTuple, Optional, Union, Literal +from typing import List, NamedTuple, Optional, Union, Literal, Tuple from enum import Enum from uuid import UUID import tempfile from attr import dataclass import geoengine_openapi_client +import geoengine_openapi_client.exceptions import geoengine_openapi_client.models import numpy as np import geopandas as gpd @@ -20,7 +21,8 @@ from geoengine.types import Provenance, RasterSymbology, TimeStep, \ TimeStepGranularity, VectorDataType, VectorResultDescriptor, VectorColumnInfo, \ UnitlessMeasurement, FeatureDataType -from geoengine.resource_identifier import UploadId, DatasetName +from geoengine.resource_identifier import Resource, UploadId, DatasetName +from geoengine.permissions import RoleId, Permission, add_permission class UnixTimeStampType(Enum): @@ -532,6 +534,37 @@ def add_dataset(data_store: Union[Volume, UploadId], return DatasetName.from_response(response) +def add_or_replace_dataset_with_permissions(data_store: Union[Volume, UploadId], + properties: AddDatasetProperties, + meta_data: geoengine_openapi_client.MetaDataDefinition, + permission_tuples: Optional[List[Tuple[RoleId, Permission]]] = None, + replace_existing=False, + timeout: int = 60) -> DatasetName: + ''' + Add a dataset to the Geo Engine and set permissions. + Replaces existing datasets if forced! + ''' + # pylint: disable=too-many-arguments,too-many-positional-arguments + + if replace_existing and properties.name is not None: + dataset_name = DatasetName(properties.name) + if dataset_info_by_name(dataset_name, timeout=timeout) is not None: + delete_dataset(dataset_name) + + dataset_name = add_dataset(data_store=data_store, properties=properties, meta_data=meta_data, timeout=timeout) + + # handle pemission setting + dataset_from_server = dataset_info_by_name(dataset_name, timeout=timeout) + assert dataset_from_server is not None, "Dataset added to the server must be resolvable!" + + if permission_tuples is not None: + dataset_res = Resource.from_dataset_name(DatasetName(dataset_from_server.id)) + for (role, perm) in permission_tuples: + add_permission(role, dataset_res, perm, timeout=timeout) + + return dataset_name + + def delete_dataset(dataset_name: DatasetName, timeout: int = 60) -> None: '''Delete a dataset. The dataset must be owned by the caller.''' @@ -569,11 +602,19 @@ def list_datasets(offset: int = 0, return response -def dataset_info_by_name(dataset_name: DatasetName, timeout: int = 60) -> geoengine_openapi_client.models.Dataset: +def dataset_info_by_name( + dataset_name: DatasetName, timeout: int = 60 +) -> geoengine_openapi_client.models.Dataset | None: '''Delete a dataset. The dataset must be owned by the caller.''' session = get_session() with geoengine_openapi_client.ApiClient(session.configuration) as api_client: datasets_api = geoengine_openapi_client.DatasetsApi(api_client) - return datasets_api.get_dataset_handler(str(dataset_name), _request_timeout=timeout) + res = None + try: + res = datasets_api.get_dataset_handler(str(dataset_name), _request_timeout=timeout) + except geoengine_openapi_client.exceptions.BadRequestException as e: + if 'CannotLoadDataset' not in e.body: + raise e + return res diff --git a/geoengine/layers.py b/geoengine/layers.py index b2606c11..ce367ae7 100644 --- a/geoengine/layers.py +++ b/geoengine/layers.py @@ -315,6 +315,26 @@ def add_layer(self, return layer_id + def add_layer_with_permissions(self, + name: str, + description: str, + workflow: Union[Dict[str, Any], WorkflowBuilderOperator], # TODO: improve type + symbology: Optional[Symbology], + permission_tuples: Optional[List[Tuple[RoleId, Permission]]] = None, + timeout: int = 60) -> LayerId: + ''' + Add a layer to this collection and set permissions. + ''' + + layer_id = self.add_layer(name, description, workflow, symbology, timeout) + + if permission_tuples is not None: + res = Resource.from_layer_id(layer_id) + for (role, perm) in permission_tuples: + add_permission(role, res, perm) + + return layer_id + def add_existing_layer(self, existing_layer: Union[LayerListing, Layer, LayerId], timeout: int = 60): From 3ce013ac3e04f0f382ef19a8aae651737f5a95a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Mon, 16 Dec 2024 14:47:32 +0100 Subject: [PATCH 06/13] raster downloader: add bands --- geoengine/raster_workflow_rio_writer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index 7e70a566..e211e462 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -175,10 +175,12 @@ async def query_and_write(self, query: QueryRectangle): self.create_tiling_geo_transform_width_height(query) - assert self.workflow is not None, "The workflow must be set" + assert self.bands is not None, "The bands must be set" + bands = list(range(0, len(self.bands))) + assert self.workflow is not None, "The workflow must be set" try: - async for tile in self.workflow.raster_stream(query): + async for tile in self.workflow.raster_stream(query, bands=bands): if self.current_time != tile.time: self.close_current_dataset() self.current_time = tile.time From afbb3266deefc7442a8060e8b753749384d2a1be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Tue, 17 Dec 2024 15:00:27 +0100 Subject: [PATCH 07/13] raster writer: skip empty timestaps --- geoengine/raster.py | 6 ++++++ geoengine/raster_workflow_rio_writer.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/geoengine/raster.py b/geoengine/raster.py index 3d5199f2..b7793729 100644 --- a/geoengine/raster.py +++ b/geoengine/raster.py @@ -213,6 +213,12 @@ def spatial_partition(self) -> gety.SpatialPartition2D: def spatial_resolution(self) -> gety.SpatialResolution: return self.geo_transform.spatial_resolution() + def is_empty(self) -> bool: + ''' Returns true if the tile is empty''' + num_pixels = self.size_x * self.size_y + num_nulls = self.data.null_count + return num_pixels == num_nulls + @staticmethod def from_ge_record_batch(record_batch: pa.RecordBatch) -> RasterTile2D: '''Create a RasterTile2D from an Arrow record batch recieved from the Geo Engine''' diff --git a/geoengine/raster_workflow_rio_writer.py b/geoengine/raster_workflow_rio_writer.py index e211e462..1ca0b50e 100644 --- a/geoengine/raster_workflow_rio_writer.py +++ b/geoengine/raster_workflow_rio_writer.py @@ -170,7 +170,7 @@ def __create_new_dataset(self, query: QueryRectangle): self.current_dataset = rio_dataset - async def query_and_write(self, query: QueryRectangle): + async def query_and_write(self, query: QueryRectangle, skip_empty_times=True): ''' Query the raster workflow and write the tiles to the dataset.''' self.create_tiling_geo_transform_width_height(query) @@ -184,6 +184,11 @@ async def query_and_write(self, query: QueryRectangle): if self.current_time != tile.time: self.close_current_dataset() self.current_time = tile.time + + if tile.is_empty() and skip_empty_times: + continue + + if self.current_dataset is None: self.__create_new_dataset(query) assert self.current_time == tile.time, "The time of the current dataset does not match the tile" From 041cd99b5aad4074c7717449938d2a629a3e9331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 5 Feb 2025 15:03:18 +0100 Subject: [PATCH 08/13] test_layers using TestInstance --- geoengine/__init__.py | 4 +- geoengine/layers.py | 11 +- geoengine/ml.py | 34 +- geoengine/permissions.py | 5 +- geoengine/resource_identifier.py | 34 +- geoengine/types.py | 24 +- tests/test_layers.py | 556 ++++++------------------------- tests/test_workflow_storage.py | 3 +- 8 files changed, 174 insertions(+), 497 deletions(-) diff --git a/geoengine/__init__.py b/geoengine/__init__.py index 5ca7f282..2582d2e0 100644 --- a/geoengine/__init__.py +++ b/geoengine/__init__.py @@ -18,7 +18,7 @@ InvalidUrlException, MissingFieldInResponseException, OGCXMLError from .layers import Layer, LayerCollection, LayerListing, LayerCollectionListing, \ layer_collection, layer -from .ml import register_ml_model, MlModelConfig, MlModelName +from .ml import register_ml_model, MlModelConfig from .permissions import add_permission, remove_permission, add_role, remove_role, assign_role, revoke_role, \ ADMIN_ROLE_ID, REGISTERED_USER_ROLE_ID, ANONYMOUS_USER_ROLE_ID, Permission, UserId, RoleId from .tasks import Task, TaskId @@ -29,7 +29,7 @@ FeatureDataType, RasterBandDescriptor, DEFAULT_ISO_TIME_FORMAT, RasterColorizer, SingleBandRasterColorizer, \ MultiBandRasterColorizer from .resource_identifier import LAYER_DB_PROVIDER_ID, LAYER_DB_ROOT_COLLECTION_ID, DatasetName, UploadId, \ - LayerId, LayerCollectionId, LayerProviderId, Resource + LayerId, LayerCollectionId, LayerProviderId, Resource, MlModelName from .util import clamp_datetime_ms_ns from .workflow import WorkflowId, Workflow, workflow_by_id, register_workflow, get_quota, update_quota, data_usage, \ diff --git a/geoengine/layers.py b/geoengine/layers.py index ce367ae7..2f730eeb 100644 --- a/geoengine/layers.py +++ b/geoengine/layers.py @@ -520,6 +520,15 @@ def get_or_create_unique_collection( return cast(LayerCollection, existing_collections[0].load()) # we know that it is a collection since check that + def __eq__(self, other): + ''' Tests if two layer listings are identical ''' + if not isinstance(other, self.__class__): + return False + + return self.name == other.name and self.description == other.description \ + and self.provider_id == other.provider_id \ + and self.collection_id == other.collection_id and self.items == other.items + @dataclass(repr=False) class Layer: @@ -727,7 +736,7 @@ def layer(layer_id: LayerId, with geoengine_openapi_client.ApiClient(session.configuration) as api_client: layers_api = geoengine_openapi_client.LayersApi(api_client) - response = layers_api.layer_handler(str(layer_provider_id), layer_id, _request_timeout=timeout) + response = layers_api.layer_handler(str(layer_provider_id), str(layer_id), _request_timeout=timeout) return Layer.from_response(response) diff --git a/geoengine/ml.py b/geoengine/ml.py index adcdeda2..bb62fdab 100644 --- a/geoengine/ml.py +++ b/geoengine/ml.py @@ -12,7 +12,7 @@ from geoengine_openapi_client.models import MlModelMetadata, MlModel, RasterDataType import geoengine_openapi_client from geoengine.auth import get_session -from geoengine.datasets import UploadId +from geoengine.resource_identifier import UploadId, MlModelName from geoengine.error import InputException @@ -25,38 +25,6 @@ class MlModelConfig: description: str = "My Ml Model Description" -class MlModelName: - '''A wrapper for an MlModel name''' - - __ml_model_name: str - - def __init__(self, ml_model_name: str) -> None: - self.__ml_model_name = ml_model_name - - @classmethod - def from_response(cls, response: geoengine_openapi_client.models.MlModelNameResponse) -> MlModelName: - '''Parse a http response to an `DatasetName`''' - return MlModelName(response.ml_model_name) - - def __str__(self) -> str: - return self.__ml_model_name - - def __repr__(self) -> str: - return str(self) - - def __eq__(self, other) -> bool: - '''Checks if two dataset names are equal''' - if not isinstance(other, self.__class__): - return False - - return self.__ml_model_name == other.__ml_model_name # pylint: disable=protected-access - - def to_api_dict(self) -> geoengine_openapi_client.models.MlModelNameResponse: - return geoengine_openapi_client.models.MlModelNameResponse( - ml_model_name=str(self.__ml_model_name) - ) - - def register_ml_model(onnx_model: ModelProto, model_config: MlModelConfig, upload_timeout: int = 3600, diff --git a/geoengine/permissions.py b/geoengine/permissions.py index d751c7fc..2876ec05 100644 --- a/geoengine/permissions.py +++ b/geoengine/permissions.py @@ -5,8 +5,7 @@ from __future__ import annotations import ast from uuid import UUID -from typing import Dict, List, Literal, Any, Union -from typing import Dict +from typing import Dict, List, Union from enum import Enum import geoengine_openapi_client.models.role @@ -14,8 +13,6 @@ import geoengine_openapi_client.api import geoengine_openapi_client -from geoengine.ml import MlModelName -from geoengine.layers import LayerCollectionId, LayerId from geoengine.error import GeoEngineException from geoengine.resource_identifier import Resource from geoengine.auth import get_session diff --git a/geoengine/resource_identifier.py b/geoengine/resource_identifier.py index 72030386..e304a68c 100644 --- a/geoengine/resource_identifier.py +++ b/geoengine/resource_identifier.py @@ -5,8 +5,6 @@ from uuid import UUID import geoengine_openapi_client -from geoengine.ml import MlModelName - LayerId = NewType('LayerId', str) LayerCollectionId = NewType('LayerCollectionId', str) LayerProviderId = NewType('LayerProviderId', UUID) @@ -15,6 +13,38 @@ LAYER_DB_ROOT_COLLECTION_ID = LayerCollectionId('05102bb3-a855-4a37-8a8a-30026a91fef1') +class MlModelName: + '''A wrapper for an MlModel name''' + + __ml_model_name: str + + def __init__(self, ml_model_name: str) -> None: + self.__ml_model_name = ml_model_name + + @classmethod + def from_response(cls, response: geoengine_openapi_client.models.MlModelNameResponse) -> MlModelName: + '''Parse a http response to an `DatasetName`''' + return MlModelName(response.ml_model_name) + + def __str__(self) -> str: + return self.__ml_model_name + + def __repr__(self) -> str: + return str(self) + + def __eq__(self, other) -> bool: + '''Checks if two dataset names are equal''' + if not isinstance(other, self.__class__): + return False + + return self.__ml_model_name == other.__ml_model_name # pylint: disable=protected-access + + def to_api_dict(self) -> geoengine_openapi_client.models.MlModelNameResponse: + return geoengine_openapi_client.models.MlModelNameResponse( + ml_model_name=str(self.__ml_model_name) + ) + + class DatasetName: '''A wrapper for a dataset name''' diff --git a/geoengine/types.py b/geoengine/types.py index fbdc8aab..1e538abf 100644 --- a/geoengine/types.py +++ b/geoengine/types.py @@ -959,6 +959,9 @@ def from_response(response: geoengine_openapi_client.Symbology) -> Symbology: raise InputException("Invalid symbology type") + def __repr__(self): + "Symbology" + class VectorSymbology(Symbology): '''A vector symbology''' @@ -1065,22 +1068,22 @@ def to_api_dict(self) -> geoengine_openapi_client.RasterColorizer: class RasterSymbology(Symbology): '''A raster symbology''' - __opacity: float - __raster_colorizer: RasterColorizer + opacity: float + raster_colorizer: RasterColorizer def __init__(self, raster_colorizer: RasterColorizer, opacity: float = 1.0) -> None: '''Initialize a new `RasterSymbology`''' - self.__raster_colorizer = raster_colorizer - self.__opacity = opacity + self.raster_colorizer = raster_colorizer + self.opacity = opacity def to_api_dict(self) -> geoengine_openapi_client.Symbology: '''Convert the raster symbology to a dictionary''' return geoengine_openapi_client.Symbology(geoengine_openapi_client.RasterSymbology( type='raster', - raster_colorizer=self.__raster_colorizer.to_api_dict(), - opacity=self.__opacity, + raster_colorizer=self.raster_colorizer.to_api_dict(), + opacity=self.opacity, )) @staticmethod @@ -1092,7 +1095,14 @@ def from_response_raster(response: geoengine_openapi_client.RasterSymbology) -> return RasterSymbology(raster_colorizer, response.opacity) def __repr__(self) -> str: - return super().__repr__() + f"({self.__raster_colorizer}, {self.__opacity})" + return str(self.__class__) + f"({self.raster_colorizer}, {self.opacity})" + + def __eq__(self, value): + '''Check if two RasterSymbologies are equal''' + + if not isinstance(value, self.__class__): + return False + return self.opacity == value.opacity and self.raster_colorizer == value.raster_colorizer class DataId: # pylint: disable=too-few-public-methods diff --git a/tests/test_layers.py b/tests/test_layers.py index ea6fa98c..d57b8863 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -5,10 +5,11 @@ import geoengine as ge from geoengine import StoredDataset, BadRequestException from geoengine import api -from geoengine.datasets import DatasetName, UploadId +from geoengine.resource_identifier import LAYER_DB_PROVIDER_ID from geoengine.layers import Layer, LayerId, LayerProviderId +from geoengine.tasks import TaskStatus from geoengine.types import RasterSymbology -from . import UrllibMocker +from tests.ge_test import GeoEngineTestInstance class LayerTests(unittest.TestCase): @@ -21,423 +22,130 @@ def setUp(self) -> None: def test_layer(self): """Test `add_layer`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) - - # pylint: disable=line-too-long - m.get( - 'http://mock-instance/layers/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/9ee3619e-d0f9-4ced-9c44-3d407c3aed69', - json={ - "description": "Land Cover derived from MODIS/Terra+Aqua Land Cover", - "id": { - "layerId": "9ee3619e-d0f9-4ced-9c44-3d407c3aed69", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "metadata": {}, - "name": "Land Cover", - "properties": [], - "symbology": { - "rasterColorizer": { - "type": "singleBand", - "band": 0, - "bandColorizer": { - "colors": { - "0": [134, 201, 227, 255], - "1": [30, 129, 62, 255], - "2": [59, 194, 212, 255], - "3": [157, 194, 63, 255], - "4": [159, 225, 127, 255], - "5": [125, 194, 127, 255], - "6": [195, 127, 126, 255], - "7": [188, 221, 190, 255], - "8": [224, 223, 133, 255], - "9": [226, 221, 7, 255], - "10": [223, 192, 125, 255], - "11": [66, 128, 189, 255], - "12": [225, 222, 127, 255], - "13": [253, 2, 0, 255], - "14": [162, 159, 66, 255], - "15": [255, 255, 255, 255], - "16": [192, 192, 192, 255] - }, - "defaultColor": [0, 0, 0, 0], - "noDataColor": [0, 0, 0, 0], - "type": "palette" - } - }, - "opacity": 1, - "type": "raster" + expected = ge.Layer( + name='Natural Earth II – RGB', + description='A raster with three bands for RGB visualization', + layer_id=LayerId('83866f7b-dcee-47b8-9242-e5636ceaf402'), + provider_id=LAYER_DB_PROVIDER_ID, + metadata={}, + properties=[], + symbology=ge.RasterSymbology( + raster_colorizer=ge.MultiBandRasterColorizer( + red_band=2, + green_band=1, + blue_band=0, + red_min=0.0, + red_max=255.0, + red_scale=1.0, + green_min=0.0, + green_max=255.0, + green_scale=1.0, + blue_min=0.0, + blue_max=255.0, + blue_scale=1.0 + ), + opacity=1.0 + ), + workflow={ + 'operator': { + 'params': {'renameBands': { + 'type': 'rename', 'values': ['blue', 'green', 'red'] + } }, - "workflow": { - "operator": { - "params": { - "data": "ndvi" - }, - "type": "GdalSource" + 'sources': + {'rasters': [ + {'type': 'GdalSource', 'params': { + 'data': 'ne2_raster_blue' + } }, - "type": "Raster" - } - }) + {'type': 'GdalSource', 'params': { + 'data': 'ne2_raster_green' + } + }, {'type': 'GdalSource', 'params': { + 'data': 'ne2_raster_red' + } + } + ] + }, 'type': 'RasterStacker' + }, + 'type': 'Raster' + } + + ) + + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address()) - layer = ge.layer('9ee3619e-d0f9-4ced-9c44-3d407c3aed69', 'ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b') + layer = ge.layer(LayerId('83866f7b-dcee-47b8-9242-e5636ceaf402'), LAYER_DB_PROVIDER_ID) self.assertEqual( - layer.to_api_dict(), - api.Layer( - name='Land Cover', - description="Land Cover derived from MODIS/Terra+Aqua Land Cover", - id=api.ProviderLayerId( - layer_id='9ee3619e-d0f9-4ced-9c44-3d407c3aed69', - provider_id='ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b' - ), - metadata={}, - properties=[], - symbology=api.Symbology(api.RasterSymbology( - opacity=1, - raster_colorizer=api.RasterColorizer(api.SingleBandRasterColorizer( - type='singleBand', - band=0, - band_colorizer=api.Colorizer(api.PaletteColorizer( - colors={ - "0.0": [134, 201, 227, 255], - "1.0": [30, 129, 62, 255], - "2.0": [59, 194, 212, 255], - "3.0": [157, 194, 63, 255], - "4.0": [159, 225, 127, 255], - "5.0": [125, 194, 127, 255], - "6.0": [195, 127, 126, 255], - "7.0": [188, 221, 190, 255], - "8.0": [224, 223, 133, 255], - "9.0": [226, 221, 7, 255], - "10.0": [223, 192, 125, 255], - "11.0": [66, 128, 189, 255], - "12.0": [225, 222, 127, 255], - "13.0": [253, 2, 0, 255], - "14.0": [162, 159, 66, 255], - "15.0": [255, 255, 255, 255], - "16.0": [192, 192, 192, 255] - }, - default_color=[0, 0, 0, 0], - no_data_color=[0, 0, 0, 0], - type="palette" - )))), - type="raster" - )), - workflow={ - "operator": { - "params": { - "data": "ndvi" - }, - "type": "GdalSource" - }, - "type": "Raster" - } - ) + layer, + expected ) def test_layer_collection(self): """Test `add_layer`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/546073b6-d535-4205-b601-99675c9f6dd7?offset=0&limit=20', - json={ - "description": "Basic Layers for all Datasets", - "entryLabel": None, - "id": { - "collectionId": "546073b6-d535-4205-b601-99675c9f6dd7", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "items": [ - { - "description": "Land Cover derived from MODIS/Terra+Aqua Land Cover", - "id": { - "layerId": "9ee3619e-d0f9-4ced-9c44-3d407c3aed69", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "name": "Land Cover", - "type": "layer" - }, - { - "description": "NDVI data from MODIS", - "id": { - "layerId": "36574dc3-560a-4b09-9d22-d5945f2b8093", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "name": "NDVI", - "type": "layer" - } - ], - "name": "Datasets", - "properties": [] - } - ) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address()) layer_collection = ge.layer_collection( - '546073b6-d535-4205-b601-99675c9f6dd7', - 'ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b' + LayerId('272bf675-2e27-4412-824c-287c1e6841ac'), + LAYER_DB_PROVIDER_ID ) + print(layer_collection) + self.assertEqual( - layer_collection.__dict__, + layer_collection, ge.LayerCollection( - name='Datasets', - description='Basic Layers for all Datasets', - collection_id=ge.LayerCollectionId('546073b6-d535-4205-b601-99675c9f6dd7'), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), + name='A test collection', + description='Some layers for testing and an empty subcollection', + collection_id=ge.LayerCollectionId('272bf675-2e27-4412-824c-287c1e6841ac'), + provider_id=LAYER_DB_PROVIDER_ID, items=[ + ge.LayerCollectionListing( + listing_id=ge.LayerCollectionId('a29f77cc-51ce-466b-86ef-d0ab2170bc0a'), + provider_id=LAYER_DB_PROVIDER_ID, + name='An empty collection', + description='There is nothing here' + ), + ge.LayerListing( + listing_id=ge.LayerCollectionId('83866f7b-dcee-47b8-9242-e5636ceaf402'), + provider_id=LAYER_DB_PROVIDER_ID, + name='Natural Earth II – RGB', + description='A raster with three bands for RGB visualization' + ), ge.LayerListing( - listing_id=ge.LayerId('9ee3619e-d0f9-4ced-9c44-3d407c3aed69'), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), - name='Land Cover', - description='Land Cover derived from MODIS/Terra+Aqua Land Cover', + listing_id=ge.LayerId('b75db46e-2b9a-4a86-b33f-bc06a73cd711'), + provider_id=LAYER_DB_PROVIDER_ID, + name='Ports in Germany', + description='Natural Earth Ports point filtered with Germany polygon', ), ge.LayerListing( - listing_id=ge.LayerId('36574dc3-560a-4b09-9d22-d5945f2b8093'), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), - name='NDVI', - description='NDVI data from MODIS', + listing_id=ge.LayerId('c078db52-2dc6-4838-ad75-340cefeab476'), + provider_id=LAYER_DB_PROVIDER_ID, + name='Stacked Raster', + description='A raster with two bands for testing', ), ], - ).__dict__ + ) ) def test_layer_collection_modification(self): """Test addition and removal to a data collection.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/05102bb3-a855-4a37-8a8a-30026a91fef1?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "Root collection for LayerDB", - "entryLabel": None, - "id": { - "collectionId": "05102bb3-a855-4a37-8a8a-30026a91fef1", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "LayerDB", - "properties": [] - }, - ) - - m.post( - 'http://mock-instance/layerDb/collections/05102bb3-a855-4a37-8a8a-30026a91fef1/collections', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "id": "490ef009-aa7a-44b0-bbef-73cfb5916b55" - } - ) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/490ef009-aa7a-44b0-bbef-73cfb5916b55?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "test description", - "entryLabel": None, - "id": { - "collectionId": "490ef009-aa7a-44b0-bbef-73cfb5916b55", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "my test collection", - "properties": [] - }, - ) - - m.post( - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "id": "64221c85-22df-4d30-9c97-605e5c498629" - } - ) - - m.post( - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/layers', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - expected_request_body={ - 'name': 'ports clone', - 'description': 'test description', - 'workflow': { - 'type': 'Vector', - 'operator': { - 'type': 'PointInPolygonFilter', - 'params': {}, - 'sources': { - 'points': { - 'type': 'OgrSource', - 'params': { - 'data': "ne_10m_ports", - 'attributeProjection': None, - 'attributeFilters': None - } - }, - 'polygons': { - 'type': 'OgrSource', - 'params': { - 'data': "germany_outline", - 'attributeProjection': None, - 'attributeFilters': None - } - }, - } - } - }, - 'symbology': None - }, - json={ - "id": "fbffb07e-d8b7-4688-98a5-4665988e6ae3" - } - ) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/64221c85-22df-4d30-9c97-605e5c498629?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "another description", - "entryLabel": None, - "id": { - "collectionId": "64221c85-22df-4d30-9c97-605e5c498629", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "sub collection", - "properties": [] - }, - ) - - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/64221c85-22df-4d30-9c97-605e5c498629/layers/fbffb07e-d8b7-4688-98a5-4665988e6ae3', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.get( - 'http://mock-instance/layers/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/fbffb07e-d8b7-4688-98a5-4665988e6ae3', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - 'id': { - "layerId": "fbffb07e-d8b7-4688-98a5-4665988e6ae3", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - 'name': 'ports clone', - 'description': 'test description', - 'workflow': { - 'type': 'Vector', - 'operator': { - 'type': 'PointInPolygonFilter', - 'params': {}, - 'sources': { - 'points': { - 'type': 'OgrSource', - 'params': { - 'data': "ne_10m_ports", - 'attributeProjection': None, - 'attributeFilters': None - } - }, - 'polygons': { - 'type': 'OgrSource', - 'params': { - 'data': "germany_outline", - 'attributeProjection': None, - 'attributeFilters': None - } - } - } - } - }, - 'symbology': None, - "metadata": {}, - "properties": [], - }, - ) - - m.post( - 'http://mock-instance/layerDb/collections/64221c85-22df-4d30-9c97-605e5c498629/collections', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - expected_request_body={ - 'name': 'sub sub collection', - 'description': 'yet another description', - }, - json={ - "id": "cd5c3f0f-c682-4f49-820d-8d704f25e803" - }, - ) - - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections/cd5c3f0f-c682-4f49-820d-8d704f25e803', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/cd5c3f0f-c682-4f49-820d-8d704f25e803?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "yet another description", - "entryLabel": None, - "id": { - "collectionId": "cd5c3f0f-c682-4f49-820d-8d704f25e803", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "sub sub collection", - "properties": [] - }, - ) - - m.delete( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections/64221c85-22df-4d30-9c97-605e5c498629', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.delete( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/layers/fbffb07e-d8b7-4688-98a5-4665988e6ae3', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.delete( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections/cd5c3f0f-c682-4f49-820d-8d704f25e803', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - m.delete( - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) root_of_layerdb = ge.layer_collection('05102bb3-a855-4a37-8a8a-30026a91fef1') @@ -497,67 +205,21 @@ def test_layer_collection_modification(self): def test_save_as_dataset(self): """Test `layer.save_as_dataset`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - # Success case - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layers/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/9ee3619e-d0f9-4ced-9c44-3d407c3aed69/dataset', - json={'taskId': '7f210984-8f2d-44f6-b211-ededada17598'}, - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}) - - m.get('http://mock-instance/tasks/7f210984-8f2d-44f6-b211-ededada17598/status', - # pylint: disable=line-too-long - json={'status': 'completed', - 'info': {'dataset': '94230f0b-4e8a-4cba-9adc-3ace837fe5d4', - 'upload': '3086f494-d5a4-4b51-a14b-3b29f8bf7bb0'}, - 'timeTotal': '00:00:00', - 'taskType': 'create-dataset', - 'description': 'Creating dataset Test Raster Layer from layer 86c81654-e572-42ed-96ee-8b38ebcd84ab', - 'timeStarted': '2023-02-16T15:25:45.390Z'}, ) - - # Some processing error occurred - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layers/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/86c81654-e572-42ed-96ee-8b38ebcd84ab/dataset', - status_code=400, - json={'error': 'Some Processing Error', - 'message': 'Some Processing Message'}, - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}) - - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address()) # Success case - layer = ge.Layer( - name='Test Raster Layer', - description='Test Raster Layer Description', - layer_id=ge.LayerId(UUID('9ee3619e-d0f9-4ced-9c44-3d407c3aed69')), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), - workflow={ - "operator": { - "params": { - "data": "ndvi" - }, - "type": "GdalSource" - }, - "type": "Raster" - }, - symbology=None, - properties=[], - metadata={}, - ) + layer = ge.layer(LayerId('83866f7b-dcee-47b8-9242-e5636ceaf402'), LAYER_DB_PROVIDER_ID) task = layer.save_as_dataset() - task_status = task.get_status() + task_status = task.wait_for_finish() + self.assertEqual(task_status.status, TaskStatus.COMPLETED) stored_dataset = StoredDataset.from_response(task_status.info) - - self.assertEqual(stored_dataset.dataset_name, DatasetName("94230f0b-4e8a-4cba-9adc-3ace837fe5d4")) - self.assertEqual(stored_dataset.upload_id, UploadId(UUID("3086f494-d5a4-4b51-a14b-3b29f8bf7bb0"))) + self.assertNotEqual(stored_dataset.dataset_name, '') + self.assertNotEqual(stored_dataset.upload_id, '') # Some processing error occurred (e.g., layer does not exist) layer = ge.Layer( @@ -592,9 +254,9 @@ def test_layer_repr_html_does_not_crash(self): provider_id=LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), workflow={ "operator": { - "params": { - "data": "ndvi" - }, + "params": { + "data": "ndvi" + }, "type": "GdalSource" }, "type": "Raster" diff --git a/tests/test_workflow_storage.py b/tests/test_workflow_storage.py index 170b39a1..a953d5e4 100644 --- a/tests/test_workflow_storage.py +++ b/tests/test_workflow_storage.py @@ -3,7 +3,8 @@ import unittest from uuid import UUID import geoengine_openapi_client -from geoengine.datasets import DatasetName, UploadId, StoredDataset +from geoengine.datasets import DatasetName, StoredDataset +from geoengine.resource_identifier import UploadId import geoengine as ge from . import UrllibMocker From e79235bb876cc20e79295a94fdc780bf28827175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 5 Feb 2025 16:49:46 +0100 Subject: [PATCH 09/13] add test for get_or_create_unique_collection --- tests/test_layers.py | 67 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/tests/test_layers.py b/tests/test_layers.py index d57b8863..f0ebb936 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -5,7 +5,8 @@ import geoengine as ge from geoengine import StoredDataset, BadRequestException from geoengine import api -from geoengine.resource_identifier import LAYER_DB_PROVIDER_ID +from geoengine.permissions import REGISTERED_USER_ROLE_ID, Permission, PermissionListing, Role +from geoengine.resource_identifier import LAYER_DB_PROVIDER_ID, Resource from geoengine.layers import Layer, LayerId, LayerProviderId from geoengine.tasks import TaskStatus from geoengine.types import RasterSymbology @@ -304,6 +305,70 @@ def test_layer_repr_html_does_not_crash(self): _html = layer._repr_html_() # pylint: disable=protected-access + def test_layer_collection_advanced_modification(self): + """Test addition and overwrite to a data collection.""" + + permisions = [(REGISTERED_USER_ROLE_ID, Permission.READ)] + + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() + + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) + + root_of_layerdb = ge.layer_collection('05102bb3-a855-4a37-8a8a-30026a91fef1') + + # create a new collection with permissions + created_collection = root_of_layerdb.get_or_create_unique_collection( + "my test collection", + create_collection_description="the first collection description", + create_permissions_tuples=permisions) + + root_of_layerdb = root_of_layerdb.reload() + + collection_in = root_of_layerdb.get_items_by_name("my test collection")[0].load() + + self.assertEqual(created_collection.description, "the first collection description") + self.assertEqual(created_collection, collection_in) + + expected_permission = PermissionListing( + role=Role(role_name="user", role_id=REGISTERED_USER_ROLE_ID), + resource=Resource.from_layer_collection_id(created_collection.collection_id), + permission=Permission.READ + ) + + self.assertIn(expected_permission, + ge.permissions.list_permissions( + Resource.from_layer_collection_id(created_collection.collection_id) + ) + ) + + # get the existing collection (no overwrite) + existing_collection = root_of_layerdb.get_or_create_unique_collection( + "my test collection", + create_collection_description="the second collection description", + create_permissions_tuples=permisions) + + root_of_layerdb = root_of_layerdb.reload() + + collection_in = root_of_layerdb.get_items_by_name("my test collection")[0].load() + self.assertEqual(existing_collection.description, "the first collection description") + self.assertEqual(existing_collection, collection_in) + + # now overwrite existing collection + overwrite_collection = root_of_layerdb.get_or_create_unique_collection( + "my test collection", + create_collection_description="the third collection description", + create_permissions_tuples=permisions, + delete_existing_with_same_name=True + ) + + root_of_layerdb = root_of_layerdb.reload() + + collection_in = root_of_layerdb.get_items_by_name("my test collection")[0].load() + self.assertEqual(overwrite_collection.description, "the third collection description") + self.assertEqual(overwrite_collection, collection_in) + if __name__ == '__main__': unittest.main() From f14b968b38b6c344d49183b25b87354a9113de69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 5 Feb 2025 17:05:37 +0100 Subject: [PATCH 10/13] also test add_layer_with_permissions --- tests/test_layers.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_layers.py b/tests/test_layers.py index f0ebb936..ce5927fd 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -369,6 +369,44 @@ def test_layer_collection_advanced_modification(self): self.assertEqual(overwrite_collection.description, "the third collection description") self.assertEqual(overwrite_collection, collection_in) + new_layer = overwrite_collection.add_layer_with_permissions( + name="ports clone", + description="test description", + workflow={ + "type": "Vector", + "operator": { + "type": "PointInPolygonFilter", + "params": {}, + "sources": { + "points": { + "type": "OgrSource", + "params": { + "data": "ne_10m_ports", + "attributeProjection": None, + "attributeFilters": None + } + }, + "polygons": { + "type": "OgrSource", + "params": { + "data": "germany_outline", + "attributeProjection": None, + "attributeFilters": None + } + } + } + } + }, + symbology=None, + permission_tuples=permisions) + + expected_permission = PermissionListing( + role=Role(role_name="user", role_id=REGISTERED_USER_ROLE_ID), + resource=Resource.from_layer_id(new_layer), + permission=Permission.READ + ) + self.assertIn(expected_permission, ge.permissions.list_permissions(Resource.from_layer_id(new_layer))) + if __name__ == '__main__': unittest.main() From f2a9bd86c0b2a242b91bbaf3b9cda51937c3aace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 5 Feb 2025 17:13:40 +0100 Subject: [PATCH 11/13] use test instance for test_datasets --- tests/test_datasets.py | 78 ++++++------------------------------------ 1 file changed, 11 insertions(+), 67 deletions(-) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index c4027513..6bf200f3 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -2,7 +2,7 @@ import unittest import geoengine as ge -from . import UrllibMocker +from tests.ge_test import GeoEngineTestInstance class DatasetsTests(unittest.TestCase): @@ -15,81 +15,25 @@ def setUp(self) -> None: def test_list_datasets(self): """Test `GET /datasets`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - m.get( - 'http://mock-instance/datasets?filter=foo&order=NameAsc&offset=1&limit=2', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json=[{ - 'id': '35383178-4b70-421f-af82-c345c81d2e13', - 'name': 'foobar', - 'displayName': 'Land Cover', - 'description': 'Land Cover', - 'tags': [], - 'sourceOperator': 'GdalSource', - 'resultDescriptor': {'type': 'raster', - 'dataType': 'U8', - 'spatialReference': 'EPSG:4326', - 'bands': [{'name': 'band', - 'measurement': - {'type': 'classification', - 'measurement': 'Land Cover', - 'classes': {'0': 'Water Bodies', - '1': 'Evergreen Needleleaf Forests', - '10': 'Grasslands', - '11': 'Permanent Wtlands', - '12': 'Croplands', - '13': 'Urban and Built-Up', - '14': 'Cropland-Natural Vegetation Mosaics', - '15': 'Snow and Ice', - '16': 'Barren or Sparsely Vegetated', - '2': 'Evergreen Broadleaf Forests', - '3': 'Deciduous Needleleaf Forests', - '4': 'Deciduous Broadleleaf Forests', - '5': 'Mixed Forests', - '6': 'Closed Shrublands', - '7': 'Open Shrublands', - '8': 'Woody Savannas', - '9': 'Savannas'}}}], - 'time': None, - 'bbox': {'upperLeftCoordinate': {'x': -180.0, 'y': 90.0}, - 'lowerRightCoordinate': {'x': 180.0, 'y': -90.0}}, - 'resolution': {'x': 0.1, 'y': 0.1}}, - 'symbology': {'type': 'raster', - 'opacity': 1.0, - 'rasterColorizer': { - 'type': 'singleBand', - 'band': 0, - 'bandColorizer': { - 'type': 'linearGradient', - 'breakpoints': [{'value': 0.0, 'color': [0, 0, 255, 255]}, - {'value': 8.0, 'color': [0, 255, 0, 255]}, - {'value': 16.0, 'color': [255, 0, 0, 255]}], - 'noDataColor': [0, 0, 0, 0], - 'overColor': [0, 0, 0, 0], - 'underColor': [0, 0, 0, 0]}}}} - ]) - - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address()) datasets = ge.list_datasets( - offset=1, - limit=2, + offset=0, + limit=10, order=ge.DatasetListOrder.NAME_ASC, - name_filter='foo' + name_filter="Natural Earth II" ) - self.assertEqual(len(datasets), 1) + self.assertEqual(len(datasets), 3) dataset = datasets[0] - self.assertEqual(dataset.name, 'foobar') - self.assertEqual(dataset.display_name, 'Land Cover') + self.assertEqual(dataset.name, 'ne2_raster_blue') + self.assertEqual(dataset.display_name, 'Natural Earth II – Blue') self.assertEqual(dataset.result_descriptor.actual_instance.type, 'raster') From 9f0ec208397624fbb2d1682ccae6d14f6da07211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Thu, 6 Feb 2025 13:07:12 +0100 Subject: [PATCH 12/13] add test for add_dataset, add_or_replace_dataset_with_permissions, and dataset_info_by_name --- geoengine/__init__.py | 51 +++---- geoengine/datasets.py | 38 +++-- tests/test_datasets.py | 335 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 383 insertions(+), 41 deletions(-) diff --git a/geoengine/__init__.py b/geoengine/__init__.py index 2582d2e0..af2ff8b9 100644 --- a/geoengine/__init__.py +++ b/geoengine/__init__.py @@ -3,41 +3,40 @@ from pkg_resources import get_distribution from requests import utils from pydantic import ValidationError - from geoengine_openapi_client.exceptions import BadRequestException, OpenApiException, ApiTypeError, ApiValueError, \ ApiKeyError, ApiAttributeError, ApiException, NotFoundException from geoengine_openapi_client import UsageSummaryGranularity -from .auth import Session, get_session, initialize, reset -from .colorizer import Colorizer, ColorBreakpoint, LinearGradientColorizer, PaletteColorizer, \ - LogarithmicGradientColorizer -from .datasets import upload_dataframe, StoredDataset, add_dataset, volumes, AddDatasetProperties, \ - delete_dataset, list_datasets, DatasetListOrder, OgrSourceDatasetTimeType, OgrOnError -from .error import GeoEngineException, InputException, UninitializedException, TypeException, \ - MethodNotCalledOnPlotException, MethodNotCalledOnRasterException, MethodNotCalledOnVectorException, \ - SpatialReferenceMismatchException, check_response_for_error, ModificationNotOnLayerDbException, \ - InvalidUrlException, MissingFieldInResponseException, OGCXMLError -from .layers import Layer, LayerCollection, LayerListing, LayerCollectionListing, \ - layer_collection, layer -from .ml import register_ml_model, MlModelConfig -from .permissions import add_permission, remove_permission, add_role, remove_role, assign_role, revoke_role, \ - ADMIN_ROLE_ID, REGISTERED_USER_ROLE_ID, ANONYMOUS_USER_ROLE_ID, Permission, UserId, RoleId -from .tasks import Task, TaskId + +from . import workflow_builder +from .raster_workflow_rio_writer import RasterWorkflowRioWriter +from .raster import RasterTile2D +from .workflow import WorkflowId, Workflow, workflow_by_id, register_workflow, get_quota, update_quota, data_usage, \ + data_usage_summary +from .util import clamp_datetime_ms_ns +from .resource_identifier import LAYER_DB_PROVIDER_ID, LAYER_DB_ROOT_COLLECTION_ID, DatasetName, UploadId, \ + LayerId, LayerCollectionId, LayerProviderId, Resource, MlModelName from .types import QueryRectangle, GeoTransform, \ RasterResultDescriptor, Provenance, UnitlessMeasurement, ContinuousMeasurement, \ ClassificationMeasurement, BoundingBox2D, TimeInterval, SpatialResolution, SpatialPartition2D, \ RasterSymbology, VectorSymbology, VectorDataType, VectorResultDescriptor, VectorColumnInfo, \ FeatureDataType, RasterBandDescriptor, DEFAULT_ISO_TIME_FORMAT, RasterColorizer, SingleBandRasterColorizer, \ MultiBandRasterColorizer -from .resource_identifier import LAYER_DB_PROVIDER_ID, LAYER_DB_ROOT_COLLECTION_ID, DatasetName, UploadId, \ - LayerId, LayerCollectionId, LayerProviderId, Resource, MlModelName - -from .util import clamp_datetime_ms_ns -from .workflow import WorkflowId, Workflow, workflow_by_id, register_workflow, get_quota, update_quota, data_usage, \ - data_usage_summary -from .raster import RasterTile2D -from .raster_workflow_rio_writer import RasterWorkflowRioWriter - -from . import workflow_builder +from .tasks import Task, TaskId +from .permissions import add_permission, remove_permission, add_role, remove_role, assign_role, revoke_role, \ + ADMIN_ROLE_ID, REGISTERED_USER_ROLE_ID, ANONYMOUS_USER_ROLE_ID, Permission, UserId, RoleId +from .ml import register_ml_model, MlModelConfig +from .layers import Layer, LayerCollection, LayerListing, LayerCollectionListing, \ + layer_collection, layer +from .error import GeoEngineException, InputException, UninitializedException, TypeException, \ + MethodNotCalledOnPlotException, MethodNotCalledOnRasterException, MethodNotCalledOnVectorException, \ + SpatialReferenceMismatchException, check_response_for_error, ModificationNotOnLayerDbException, \ + InvalidUrlException, MissingFieldInResponseException, OGCXMLError +from .auth import Session, get_session, initialize, reset +from .colorizer import Colorizer, ColorBreakpoint, LinearGradientColorizer, PaletteColorizer, \ + LogarithmicGradientColorizer +from .datasets import upload_dataframe, StoredDataset, add_dataset, volumes, AddDatasetProperties, \ + delete_dataset, list_datasets, DatasetListOrder, OgrSourceDatasetTimeType, OgrOnError, \ + add_or_replace_dataset_with_permissions, dataset_info_by_name DEFAULT_USER_AGENT = f'geoengine-python/{get_distribution("geoengine").version}' diff --git a/geoengine/datasets.py b/geoengine/datasets.py index 6cd157d4..6fd6c289 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -546,21 +546,26 @@ def add_or_replace_dataset_with_permissions(data_store: Union[Volume, UploadId], ''' # pylint: disable=too-many-arguments,too-many-positional-arguments - if replace_existing and properties.name is not None: - dataset_name = DatasetName(properties.name) - if dataset_info_by_name(dataset_name, timeout=timeout) is not None: - delete_dataset(dataset_name) - - dataset_name = add_dataset(data_store=data_store, properties=properties, meta_data=meta_data, timeout=timeout) + def add_dataset_and_permissions() -> DatasetName: + dataset_name = add_dataset(data_store=data_store, properties=properties, meta_data=meta_data, timeout=timeout) + if permission_tuples is not None: + dataset_res = Resource.from_dataset_name(dataset_name) + for (role, perm) in permission_tuples: + add_permission(role, dataset_res, perm, timeout=timeout) + return dataset_name - # handle pemission setting - dataset_from_server = dataset_info_by_name(dataset_name, timeout=timeout) - assert dataset_from_server is not None, "Dataset added to the server must be resolvable!" + if properties.name is None: + dataset_name = add_dataset_and_permissions() - if permission_tuples is not None: - dataset_res = Resource.from_dataset_name(DatasetName(dataset_from_server.id)) - for (role, perm) in permission_tuples: - add_permission(role, dataset_res, perm, timeout=timeout) + else: + dataset_name = DatasetName(properties.name) + dataset_info = dataset_info_by_name(dataset_name) + if dataset_info is None: # dataset is not existing + dataset_name = add_dataset_and_permissions() + else: + if replace_existing: # dataset exists and we overwrite it + delete_dataset(dataset_name) + dataset_name = add_dataset_and_permissions() return dataset_name @@ -603,9 +608,12 @@ def list_datasets(offset: int = 0, def dataset_info_by_name( - dataset_name: DatasetName, timeout: int = 60 + dataset_name: Union[DatasetName, str], timeout: int = 60 ) -> geoengine_openapi_client.models.Dataset | None: - '''Delete a dataset. The dataset must be owned by the caller.''' + '''Get dataset information.''' + + if not isinstance(dataset_name, DatasetName): + dataset_name = DatasetName(dataset_name) session = get_session() diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 6bf200f3..5bf19765 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -1,7 +1,12 @@ """Tests for the datasets module.""" import unittest + +import geoengine_openapi_client import geoengine as ge +from geoengine.permissions import REGISTERED_USER_ROLE_ID, Permission, PermissionListing, Role +from geoengine.resource_identifier import Resource +from geoengine.types import RasterBandDescriptor from tests.ge_test import GeoEngineTestInstance @@ -36,6 +41,336 @@ def test_list_datasets(self): self.assertEqual(dataset.display_name, 'Natural Earth II – Blue') self.assertEqual(dataset.result_descriptor.actual_instance.type, 'raster') + def test_add_dataset(self): + """Test `add_datset`.""" + + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() + + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) + + volumes = ge.volumes() + + geo_transform = ge.GeoTransform( + x_min=180.0, + y_max=90.0, + x_pixel_size=0.1, + y_pixel_size=-0.1 + ) + + gdal_params = geoengine_openapi_client.GdalDatasetParameters.from_dict({ + "filePath": "raster/landcover/landcover.tif", + "rasterbandChannel": 1, + "geoTransform": geo_transform.to_api_dict(), + "width": 3600, + "height": 1800, + "fileNotFoundHandling": geoengine_openapi_client.FileNotFoundHandling.NODATA, + "noDataValue": None, + "propertiesMapping": None, + "gdalOpenOptions": None, + "gdalConfigOptions": None, + "allowAlphabandAsMask": True + }) + + result_descriptor_measurement = ge.ClassificationMeasurement( + measurement="Land Cover", + classes={ + "0": "Water Bodies", + "1": "Evergreen Needleleaf Forests", + "2": "Evergreen Broadleaf Forests", + "3": "Deciduous Needleleaf Forests", + "4": "Deciduous Broadleleaf Forests", + "5": "Mixed Forests", + "6": "Closed Shrublands", + "7": "Open Shrublands", + "8": "Woody Savannas", + "9": "Savannas", + "10": "Grasslands", + "11": "Permanent Wtlands", + "12": "Croplands", + "13": "Urban and Built-Up", + "14": "Cropland-Natural Vegetation Mosaics", + "15": "Snow and Ice", + "16": "Barren or Sparsely Vegetated" + } + + ) + + result_descriptor = ge.RasterResultDescriptor( + "U8", + [RasterBandDescriptor("band", result_descriptor_measurement)], + "EPSG:4326", + spatial_bounds=ge.SpatialPartition2D(-180.0, -90.0, 180.0, 90.0), + spatial_resolution=ge.SpatialResolution(0.1, 0.1) + ) + + meta_data = geoengine_openapi_client.GdalMetaDataStatic.from_dict({ + "type": "GdalStatic", + "time": None, + "params": gdal_params, + "resultDescriptor": result_descriptor.to_api_dict().to_dict(), + }) + + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=8, color=(0, 255, 0, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[ge.Provenance( + citation="The data was obtained from .", + uri="https://lpdaac.usgs.gov/products/mcd12c1v006/", + license="All data distributed by the LP DAAC contain no restrictions on the data reuse." + )], + + + ) + + dataset_name = ge.add_dataset( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + + def test_add_dataset_with_permissions(self): + """Test `add_datset`.""" + + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() + + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) + + volumes = ge.volumes() + + geo_transform = ge.GeoTransform( + x_min=180.0, + y_max=90.0, + x_pixel_size=0.1, + y_pixel_size=-0.1 + ) + + gdal_params = geoengine_openapi_client.GdalDatasetParameters.from_dict({ + "filePath": "raster/landcover/landcover.tif", + "rasterbandChannel": 1, + "geoTransform": geo_transform.to_api_dict(), + "width": 3600, + "height": 1800, + "fileNotFoundHandling": geoengine_openapi_client.FileNotFoundHandling.NODATA, + "noDataValue": None, + "propertiesMapping": None, + "gdalOpenOptions": None, + "gdalConfigOptions": None, + "allowAlphabandAsMask": True + }) + + result_descriptor_measurement = ge.ClassificationMeasurement( + measurement="Land Cover", + classes={ + "0": "Water Bodies", + "1": "Evergreen Needleleaf Forests", + "2": "Evergreen Broadleaf Forests", + "3": "Deciduous Needleleaf Forests", + "4": "Deciduous Broadleleaf Forests", + "5": "Mixed Forests", + "6": "Closed Shrublands", + "7": "Open Shrublands", + "8": "Woody Savannas", + "9": "Savannas", + "10": "Grasslands", + "11": "Permanent Wtlands", + "12": "Croplands", + "13": "Urban and Built-Up", + "14": "Cropland-Natural Vegetation Mosaics", + "15": "Snow and Ice", + "16": "Barren or Sparsely Vegetated" + } + + ) + + result_descriptor = ge.RasterResultDescriptor( + "U8", + [RasterBandDescriptor("band", result_descriptor_measurement)], + "EPSG:4326", + spatial_bounds=ge.SpatialPartition2D(-180.0, -90.0, 180.0, 90.0), + spatial_resolution=ge.SpatialResolution(0.1, 0.1) + ) + + meta_data = geoengine_openapi_client.GdalMetaDataStatic.from_dict({ + "type": "GdalStatic", + "time": None, + "params": gdal_params, + "resultDescriptor": result_descriptor.to_api_dict().to_dict(), + }) + + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[], + ) + + permisions = [(REGISTERED_USER_ROLE_ID, Permission.READ)] + + dataset_name = ge.add_or_replace_dataset_with_permissions( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + permission_tuples=permisions + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + dataset_info = ge.dataset_info_by_name(ge.DatasetName('MCD12C1_test')) + self.assertEqual( + dataset_info.name, + 'MCD12C1_test' + ) + self.assertEqual( + dataset_info.description, + 'Land Cover' + ) + + expected_permission = PermissionListing( + role=Role(role_name="user", role_id=REGISTERED_USER_ROLE_ID), + resource=Resource.from_dataset_name(dataset_name), + permission=Permission.READ + ) + self.assertIn( + expected_permission, + ge.permissions.list_permissions(Resource.from_dataset_name(dataset_name)) + ) + + # now get without overwrite + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover 2", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[], + ) + + dataset_name = ge.add_or_replace_dataset_with_permissions( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + permission_tuples=permisions + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + dataset_info = ge.dataset_info_by_name(ge.DatasetName('MCD12C1_test')) + self.assertEqual( + dataset_info.name, + 'MCD12C1_test' + ) + self.assertEqual( + dataset_info.description, + 'Land Cover' # Still the first value, since no overwrite + ) + + # now overwrite + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover 3", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[], + ) + + dataset_name = ge.add_or_replace_dataset_with_permissions( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + permission_tuples=permisions, + replace_existing=True + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + dataset_info = ge.dataset_info_by_name(ge.DatasetName('MCD12C1_test')) + self.assertEqual( + dataset_info.name, + 'MCD12C1_test' + ) + self.assertEqual( + dataset_info.description, + 'Land Cover 3' # Now the third value, replaced with new dataset + ) + if __name__ == '__main__': unittest.main() From 0dddc43e2f6de46e3db6906eeebe213d2f19a617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Wed, 19 Feb 2025 10:41:58 +0100 Subject: [PATCH 13/13] handle typing --- geoengine/datasets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geoengine/datasets.py b/geoengine/datasets.py index 53f20e26..05b4ae5b 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -629,6 +629,7 @@ def dataset_info_by_name( try: res = datasets_api.get_dataset_handler(str(dataset_name), _request_timeout=timeout) except geoengine_openapi_client.exceptions.BadRequestException as e: - if 'CannotLoadDataset' not in e.body: + e_body = e.body + if isinstance(e_body, str) and 'CannotLoadDataset' not in e_body: raise e return res