diff --git a/geoengine/__init__.py b/geoengine/__init__.py index 296cc077..af2ff8b9 100644 --- a/geoengine/__init__.py +++ b/geoengine/__init__.py @@ -3,40 +3,40 @@ from pkg_resources import get_distribution from requests import utils from pydantic import ValidationError - from geoengine_openapi_client.exceptions import BadRequestException, OpenApiException, ApiTypeError, ApiValueError, \ ApiKeyError, ApiAttributeError, ApiException, NotFoundException from geoengine_openapi_client import UsageSummaryGranularity -from .auth import Session, get_session, initialize, reset -from .colorizer import Colorizer, ColorBreakpoint, LinearGradientColorizer, PaletteColorizer, \ - LogarithmicGradientColorizer -from .datasets import upload_dataframe, StoredDataset, add_dataset, volumes, AddDatasetProperties, \ - delete_dataset, list_datasets, DatasetListOrder, OgrSourceDatasetTimeType, OgrOnError -from .error import GeoEngineException, InputException, UninitializedException, TypeException, \ - MethodNotCalledOnPlotException, MethodNotCalledOnRasterException, MethodNotCalledOnVectorException, \ - SpatialReferenceMismatchException, check_response_for_error, ModificationNotOnLayerDbException, \ - InvalidUrlException, MissingFieldInResponseException, OGCXMLError -from .layers import Layer, LayerCollection, LayerListing, LayerCollectionListing, \ - LayerId, LayerCollectionId, LayerProviderId, \ - layer_collection, layer -from .ml import register_ml_model, MlModelConfig, MlModelName -from .permissions import add_permission, remove_permission, add_role, remove_role, assign_role, revoke_role, \ - ADMIN_ROLE_ID, REGISTERED_USER_ROLE_ID, ANONYMOUS_USER_ROLE_ID, Permission, Resource, UserId, RoleId -from .tasks import Task, TaskId + +from . import workflow_builder +from .raster_workflow_rio_writer import RasterWorkflowRioWriter +from .raster import RasterTile2D +from .workflow import WorkflowId, Workflow, workflow_by_id, register_workflow, get_quota, update_quota, data_usage, \ + data_usage_summary +from .util import clamp_datetime_ms_ns +from .resource_identifier import LAYER_DB_PROVIDER_ID, LAYER_DB_ROOT_COLLECTION_ID, DatasetName, UploadId, \ + LayerId, LayerCollectionId, LayerProviderId, Resource, MlModelName from .types import QueryRectangle, GeoTransform, \ RasterResultDescriptor, Provenance, UnitlessMeasurement, ContinuousMeasurement, \ ClassificationMeasurement, BoundingBox2D, TimeInterval, SpatialResolution, SpatialPartition2D, \ RasterSymbology, VectorSymbology, VectorDataType, VectorResultDescriptor, VectorColumnInfo, \ FeatureDataType, RasterBandDescriptor, DEFAULT_ISO_TIME_FORMAT, RasterColorizer, SingleBandRasterColorizer, \ MultiBandRasterColorizer - -from .util import clamp_datetime_ms_ns -from .workflow import WorkflowId, Workflow, workflow_by_id, register_workflow, get_quota, update_quota, data_usage, \ - data_usage_summary -from .raster import RasterTile2D -from .raster_workflow_rio_writer import RasterWorkflowRioWriter - -from . import workflow_builder +from .tasks import Task, TaskId +from .permissions import add_permission, remove_permission, add_role, remove_role, assign_role, revoke_role, \ + ADMIN_ROLE_ID, REGISTERED_USER_ROLE_ID, ANONYMOUS_USER_ROLE_ID, Permission, UserId, RoleId +from .ml import register_ml_model, MlModelConfig +from .layers import Layer, LayerCollection, LayerListing, LayerCollectionListing, \ + layer_collection, layer +from .error import GeoEngineException, InputException, UninitializedException, TypeException, \ + MethodNotCalledOnPlotException, MethodNotCalledOnRasterException, MethodNotCalledOnVectorException, \ + SpatialReferenceMismatchException, check_response_for_error, ModificationNotOnLayerDbException, \ + InvalidUrlException, MissingFieldInResponseException, OGCXMLError +from .auth import Session, get_session, initialize, reset +from .colorizer import Colorizer, ColorBreakpoint, LinearGradientColorizer, PaletteColorizer, \ + LogarithmicGradientColorizer +from .datasets import upload_dataframe, StoredDataset, add_dataset, volumes, AddDatasetProperties, \ + delete_dataset, list_datasets, DatasetListOrder, OgrSourceDatasetTimeType, OgrOnError, \ + add_or_replace_dataset_with_permissions, dataset_info_by_name DEFAULT_USER_AGENT = f'geoengine-python/{get_distribution("geoengine").version}' diff --git a/geoengine/datasets.py b/geoengine/datasets.py index 1ad9fbc4..05b4ae5b 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -5,20 +5,24 @@ from __future__ import annotations from abc import abstractmethod from pathlib import Path -from typing import List, NamedTuple, Optional, Union, Literal +from typing import List, NamedTuple, Optional, Union, Literal, Tuple from enum import Enum from uuid import UUID import tempfile from attr import dataclass +import geoengine_openapi_client +import geoengine_openapi_client.exceptions +import geoengine_openapi_client.models import numpy as np import geopandas as gpd -import geoengine_openapi_client from geoengine import api from geoengine.error import InputException, MissingFieldInResponseException from geoengine.auth import get_session from geoengine.types import Provenance, RasterSymbology, TimeStep, \ TimeStepGranularity, VectorDataType, VectorResultDescriptor, VectorColumnInfo, \ UnitlessMeasurement, FeatureDataType +from geoengine.resource_identifier import Resource, UploadId, DatasetName +from geoengine.permissions import RoleId, Permission, add_permission class UnixTimeStampType(Enum): @@ -256,71 +260,6 @@ def to_api_enum(self) -> geoengine_openapi_client.OgrSourceErrorSpec: return geoengine_openapi_client.OgrSourceErrorSpec(self.value) -class DatasetName: - '''A wrapper for a dataset name''' - - __dataset_name: str - - def __init__(self, dataset_name: str) -> None: - self.__dataset_name = dataset_name - - @classmethod - def from_response(cls, response: geoengine_openapi_client.CreateDatasetHandler200Response) -> DatasetName: - '''Parse a http response to an `DatasetName`''' - return DatasetName(response.dataset_name) - - def __str__(self) -> str: - return self.__dataset_name - - def __repr__(self) -> str: - return str(self) - - def __eq__(self, other) -> bool: - '''Checks if two dataset names are equal''' - if not isinstance(other, self.__class__): - return False - - return self.__dataset_name == other.__dataset_name # pylint: disable=protected-access - - def to_api_dict(self) -> geoengine_openapi_client.CreateDatasetHandler200Response: - return geoengine_openapi_client.CreateDatasetHandler200Response( - dataset_name=str(self.__dataset_name) - ) - - -class UploadId: - '''A wrapper for an upload id''' - - __upload_id: UUID - - def __init__(self, upload_id: UUID) -> None: - self.__upload_id = upload_id - - @classmethod - def from_response(cls, response: geoengine_openapi_client.AddCollection200Response) -> UploadId: - '''Parse a http response to an `UploadId`''' - return UploadId(UUID(response.id)) - - def __str__(self) -> str: - return str(self.__upload_id) - - def __repr__(self) -> str: - return str(self) - - def __eq__(self, other) -> bool: - '''Checks if two upload ids are equal''' - if not isinstance(other, self.__class__): - return False - - return self.__upload_id == other.__upload_id # pylint: disable=protected-access - - def to_api_dict(self) -> geoengine_openapi_client.AddCollection200Response: - '''Converts the upload id to a dict for the api''' - return geoengine_openapi_client.AddCollection200Response( - id=str(self.__upload_id) - ) - - class AddDatasetProperties(): '''The properties for adding a dataset''' name: Optional[str] @@ -601,6 +540,42 @@ def add_dataset(data_store: Union[Volume, UploadId], return DatasetName.from_response(response) +def add_or_replace_dataset_with_permissions(data_store: Union[Volume, UploadId], + properties: AddDatasetProperties, + meta_data: geoengine_openapi_client.MetaDataDefinition, + permission_tuples: Optional[List[Tuple[RoleId, Permission]]] = None, + replace_existing=False, + timeout: int = 60) -> DatasetName: + ''' + Add a dataset to the Geo Engine and set permissions. + Replaces existing datasets if forced! + ''' + # pylint: disable=too-many-arguments,too-many-positional-arguments + + def add_dataset_and_permissions() -> DatasetName: + dataset_name = add_dataset(data_store=data_store, properties=properties, meta_data=meta_data, timeout=timeout) + if permission_tuples is not None: + dataset_res = Resource.from_dataset_name(dataset_name) + for (role, perm) in permission_tuples: + add_permission(role, dataset_res, perm, timeout=timeout) + return dataset_name + + if properties.name is None: + dataset_name = add_dataset_and_permissions() + + else: + dataset_name = DatasetName(properties.name) + dataset_info = dataset_info_by_name(dataset_name) + if dataset_info is None: # dataset is not existing + dataset_name = add_dataset_and_permissions() + else: + if replace_existing: # dataset exists and we overwrite it + delete_dataset(dataset_name) + dataset_name = add_dataset_and_permissions() + + return dataset_name + + def delete_dataset(dataset_name: DatasetName, timeout: int = 60) -> None: '''Delete a dataset. The dataset must be owned by the caller.''' @@ -636,3 +611,25 @@ def list_datasets(offset: int = 0, ) return response + + +def dataset_info_by_name( + dataset_name: Union[DatasetName, str], timeout: int = 60 +) -> geoengine_openapi_client.models.Dataset | None: + '''Get dataset information.''' + + if not isinstance(dataset_name, DatasetName): + dataset_name = DatasetName(dataset_name) + + session = get_session() + + with geoengine_openapi_client.ApiClient(session.configuration) as api_client: + datasets_api = geoengine_openapi_client.DatasetsApi(api_client) + res = None + try: + res = datasets_api.get_dataset_handler(str(dataset_name), _request_timeout=timeout) + except geoengine_openapi_client.exceptions.BadRequestException as e: + e_body = e.body + if isinstance(e_body, str) and 'CannotLoadDataset' not in e_body: + raise e + return res diff --git a/geoengine/layers.py b/geoengine/layers.py index fb3f6457..2f730eeb 100644 --- a/geoengine/layers.py +++ b/geoengine/layers.py @@ -7,7 +7,7 @@ from enum import auto from io import StringIO import os -from typing import Any, Dict, Generic, List, Literal, NewType, Optional, TypeVar, Union, cast +from typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union, cast, Tuple from uuid import UUID import json from strenum import LowercaseStrEnum @@ -16,15 +16,11 @@ from geoengine.error import ModificationNotOnLayerDbException, InputException from geoengine.tasks import Task, TaskId from geoengine.types import Symbology +from geoengine.permissions import RoleId, Permission, add_permission from geoengine.workflow import Workflow, WorkflowId from geoengine.workflow_builder.operators import Operator as WorkflowBuilderOperator - -LayerId = NewType('LayerId', str) -LayerCollectionId = NewType('LayerCollectionId', str) -LayerProviderId = NewType('LayerProviderId', UUID) - -LAYER_DB_PROVIDER_ID = LayerProviderId(UUID('ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74')) -LAYER_DB_ROOT_COLLECTION_ID = LayerCollectionId('05102bb3-a855-4a37-8a8a-30026a91fef1') +from geoengine.resource_identifier import LayerCollectionId, LayerId, LayerProviderId, \ + LAYER_DB_PROVIDER_ID, Resource class LayerCollectionListingType(LowercaseStrEnum): @@ -319,6 +315,26 @@ def add_layer(self, return layer_id + def add_layer_with_permissions(self, + name: str, + description: str, + workflow: Union[Dict[str, Any], WorkflowBuilderOperator], # TODO: improve type + symbology: Optional[Symbology], + permission_tuples: Optional[List[Tuple[RoleId, Permission]]] = None, + timeout: int = 60) -> LayerId: + ''' + Add a layer to this collection and set permissions. + ''' + + layer_id = self.add_layer(name, description, workflow, symbology, timeout) + + if permission_tuples is not None: + res = Resource.from_layer_id(layer_id) + for (role, perm) in permission_tuples: + add_permission(role, res, perm) + + return layer_id + def add_existing_layer(self, existing_layer: Union[LayerListing, Layer, LayerId], timeout: int = 60): @@ -456,6 +472,63 @@ def search(self, search_string: str, *, return listings + def get_or_create_unique_collection( + self, + collection_name: str, + create_collection_description: Optional[str] = None, + delete_existing_with_same_name: bool = False, + create_permissions_tuples: Optional[List[Tuple[RoleId, Permission]]] = None + ) -> LayerCollection: + ''' + Get a unique child by name OR if it does not exist create it. + Removes existing collections with same name if forced! + Sets permissions if the collection is created from a list of tuples + ''' + parent_collection = self.reload() # reload just to be safe since self's state change on the server + existing_collections = parent_collection.get_items_by_name(collection_name) + + if delete_existing_with_same_name and len(existing_collections) > 0: + for c in existing_collections: + actual = c.load() + if isinstance(actual, LayerCollection): + actual.remove() + parent_collection = parent_collection.reload() + existing_collections = parent_collection.get_items_by_name(collection_name) + + if len(existing_collections) == 0: + new_desc = create_collection_description if create_collection_description is not None else collection_name + new_collection = parent_collection.add_collection(collection_name, new_desc) + new_ressource = Resource.from_layer_collection_id(new_collection) + + if create_permissions_tuples is not None: + for (role, perm) in create_permissions_tuples: + add_permission(role, new_ressource, perm) + parent_collection = parent_collection.reload() + existing_collections = parent_collection.get_items_by_name(collection_name) + + if len(existing_collections) == 0: + raise KeyError( + f"No collection with name {collection_name} exists in {parent_collection.name} and none was created!" + ) + + if len(existing_collections) > 1: + raise KeyError(f"Multiple collections with name {collection_name} exist in {parent_collection.name}") + + res = existing_collections[0].load() + if isinstance(res, Layer): + raise TypeError(f"Found a Layer not a Layer collection for {collection_name}") + + return cast(LayerCollection, existing_collections[0].load()) # we know that it is a collection since check that + + def __eq__(self, other): + ''' Tests if two layer listings are identical ''' + if not isinstance(other, self.__class__): + return False + + return self.name == other.name and self.description == other.description \ + and self.provider_id == other.provider_id \ + and self.collection_id == other.collection_id and self.items == other.items + @dataclass(repr=False) class Layer: @@ -663,7 +736,7 @@ def layer(layer_id: LayerId, with geoengine_openapi_client.ApiClient(session.configuration) as api_client: layers_api = geoengine_openapi_client.LayersApi(api_client) - response = layers_api.layer_handler(str(layer_provider_id), layer_id, _request_timeout=timeout) + response = layers_api.layer_handler(str(layer_provider_id), str(layer_id), _request_timeout=timeout) return Layer.from_response(response) diff --git a/geoengine/ml.py b/geoengine/ml.py index dd123b46..468b0a15 100644 --- a/geoengine/ml.py +++ b/geoengine/ml.py @@ -12,7 +12,7 @@ from geoengine_openapi_client.models import MlModelMetadata, MlModel, RasterDataType import geoengine_openapi_client from geoengine.auth import get_session -from geoengine.datasets import UploadId +from geoengine.resource_identifier import UploadId, MlModelName from geoengine.error import InputException @@ -25,38 +25,6 @@ class MlModelConfig: description: str = "My Ml Model Description" -class MlModelName: - '''A wrapper for an MlModel name''' - - __ml_model_name: str - - def __init__(self, ml_model_name: str) -> None: - self.__ml_model_name = ml_model_name - - @classmethod - def from_response(cls, response: geoengine_openapi_client.models.MlModelNameResponse) -> MlModelName: - '''Parse a http response to an `DatasetName`''' - return MlModelName(response.ml_model_name) - - def __str__(self) -> str: - return self.__ml_model_name - - def __repr__(self) -> str: - return str(self) - - def __eq__(self, other) -> bool: - '''Checks if two dataset names are equal''' - if not isinstance(other, self.__class__): - return False - - return self.__ml_model_name == other.__ml_model_name # pylint: disable=protected-access - - def to_api_dict(self) -> geoengine_openapi_client.models.MlModelNameResponse: - return geoengine_openapi_client.models.MlModelNameResponse( - ml_model_name=str(self.__ml_model_name) - ) - - def register_ml_model(onnx_model: ModelProto, model_config: MlModelConfig, upload_timeout: int = 3600, diff --git a/geoengine/permissions.py b/geoengine/permissions.py index 99f308ab..2876ec05 100644 --- a/geoengine/permissions.py +++ b/geoengine/permissions.py @@ -3,22 +3,19 @@ ''' from __future__ import annotations -from enum import Enum - import ast -from typing import Dict, List, Literal, Any, Union from uuid import UUID +from typing import Dict, List, Union +from enum import Enum -import geoengine_openapi_client -import geoengine_openapi_client.api -import geoengine_openapi_client.models import geoengine_openapi_client.models.role +import geoengine_openapi_client.models +import geoengine_openapi_client.api +import geoengine_openapi_client -from geoengine.auth import get_session -from geoengine.datasets import DatasetName from geoengine.error import GeoEngineException -from geoengine.layers import LayerCollectionId, LayerId -from geoengine.ml import MlModelName +from geoengine.resource_identifier import Resource +from geoengine.auth import get_session class RoleId: @@ -126,89 +123,6 @@ def __repr__(self) -> str: return repr(self.__user_id) -class Resource: - '''A wrapper for a resource id''' - - id: str - type: Literal['dataset', 'layer', 'layerCollection', 'mlModel', 'project'] - - def __init__(self, resource_type: Literal['dataset', 'layer', 'layerCollection', 'mlModel', 'project'], - resource_id: str) -> None: - '''Create a resource id''' - self.type = resource_type - self.id = resource_id - - @classmethod - def from_layer_id(cls, layer_id: LayerId) -> Resource: - '''Create a resource id from a layer id''' - return Resource('layer', str(layer_id)) - - @classmethod - def from_layer_collection_id(cls, layer_collection_id: LayerCollectionId) -> Resource: - '''Create a resource id from a layer collection id''' - return Resource('layerCollection', str(layer_collection_id)) - - @classmethod - def from_dataset_name(cls, dataset_name: Union[DatasetName, str]) -> Resource: - '''Create a resource id from a dataset name''' - if isinstance(dataset_name, DatasetName): - dataset_name = str(dataset_name) - return Resource('dataset', dataset_name) - - @classmethod - def from_ml_model_name(cls, ml_model_name: Union[MlModelName, str]) -> Resource: - '''Create a resource from an ml model name''' - if isinstance(ml_model_name, MlModelName): - ml_model_name = str(ml_model_name) - return Resource('mlModel', ml_model_name) - - def to_api_dict(self) -> geoengine_openapi_client.Resource: - '''Convert to a dict for the API''' - inner: Any = None - - if self.type == "layer": - inner = geoengine_openapi_client.LayerResource(type="layer", id=self.id) - elif self.type == "layerCollection": - inner = geoengine_openapi_client.LayerCollectionResource(type="layerCollection", id=self.id) - elif self.type == "project": - inner = geoengine_openapi_client.ProjectResource(type="project", id=self.id) - elif self.type == "dataset": - inner = geoengine_openapi_client.DatasetResource(type="dataset", id=self.id) - elif self.type == "mlModel": - inner = geoengine_openapi_client.MlModelResource(type="mlModel", id=self.id) - else: - raise KeyError(f"Unknown resource type: {self.type}") - - return geoengine_openapi_client.Resource(inner) - - @classmethod - def from_response(cls, response: geoengine_openapi_client.Resource) -> Resource: - '''Convert to a dict for the API''' - inner: Resource - if isinstance(response.actual_instance, geoengine_openapi_client.LayerResource): - inner = Resource('layer', response.actual_instance.id) - elif isinstance(response.actual_instance, geoengine_openapi_client.LayerCollectionResource): - inner = Resource('layerCollection', response.actual_instance.id) - elif isinstance(response.actual_instance, geoengine_openapi_client.ProjectResource): - inner = Resource('project', response.actual_instance.id) - elif isinstance(response.actual_instance, geoengine_openapi_client.DatasetResource): - inner = Resource('dataset', response.actual_instance.id) - elif isinstance(response.actual_instance, geoengine_openapi_client.MlModelResource): - inner = Resource('mlModel', response.actual_instance.id) - else: - raise KeyError(f"Unknown resource type from API: {response.actual_instance}") - return inner - - def __repr__(self): - return 'id: ' + repr(self.id) + ', type: ' + repr(self.type) - - def __eq__(self, value): - '''Checks if two listings are equal''' - if not isinstance(value, self.__class__): - return False - return self.id == value.id and self.type == value.type - - class PermissionListing: """ PermissionListing diff --git a/geoengine/resource_identifier.py b/geoengine/resource_identifier.py new file mode 100644 index 00000000..e304a68c --- /dev/null +++ b/geoengine/resource_identifier.py @@ -0,0 +1,193 @@ +''' Types that identify a ressource in the Geo Engine''' + +from __future__ import annotations +from typing import Any, Literal, NewType, Union +from uuid import UUID +import geoengine_openapi_client + +LayerId = NewType('LayerId', str) +LayerCollectionId = NewType('LayerCollectionId', str) +LayerProviderId = NewType('LayerProviderId', UUID) + +LAYER_DB_PROVIDER_ID = LayerProviderId(UUID('ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74')) +LAYER_DB_ROOT_COLLECTION_ID = LayerCollectionId('05102bb3-a855-4a37-8a8a-30026a91fef1') + + +class MlModelName: + '''A wrapper for an MlModel name''' + + __ml_model_name: str + + def __init__(self, ml_model_name: str) -> None: + self.__ml_model_name = ml_model_name + + @classmethod + def from_response(cls, response: geoengine_openapi_client.models.MlModelNameResponse) -> MlModelName: + '''Parse a http response to an `DatasetName`''' + return MlModelName(response.ml_model_name) + + def __str__(self) -> str: + return self.__ml_model_name + + def __repr__(self) -> str: + return str(self) + + def __eq__(self, other) -> bool: + '''Checks if two dataset names are equal''' + if not isinstance(other, self.__class__): + return False + + return self.__ml_model_name == other.__ml_model_name # pylint: disable=protected-access + + def to_api_dict(self) -> geoengine_openapi_client.models.MlModelNameResponse: + return geoengine_openapi_client.models.MlModelNameResponse( + ml_model_name=str(self.__ml_model_name) + ) + + +class DatasetName: + '''A wrapper for a dataset name''' + + __dataset_name: str + + def __init__(self, dataset_name: str) -> None: + self.__dataset_name = dataset_name + + @classmethod + def from_response(cls, response: geoengine_openapi_client.CreateDatasetHandler200Response) -> DatasetName: + '''Parse a http response to an `DatasetName`''' + return DatasetName(response.dataset_name) + + def __str__(self) -> str: + return self.__dataset_name + + def __repr__(self) -> str: + return str(self) + + def __eq__(self, other) -> bool: + '''Checks if two dataset names are equal''' + if not isinstance(other, self.__class__): + return False + + return self.__dataset_name == other.__dataset_name # pylint: disable=protected-access + + def to_api_dict(self) -> geoengine_openapi_client.CreateDatasetHandler200Response: + return geoengine_openapi_client.CreateDatasetHandler200Response( + dataset_name=str(self.__dataset_name) + ) + + +class UploadId: + '''A wrapper for an upload id''' + + __upload_id: UUID + + def __init__(self, upload_id: UUID) -> None: + self.__upload_id = upload_id + + @classmethod + def from_response(cls, response: geoengine_openapi_client.AddCollection200Response) -> UploadId: + '''Parse a http response to an `UploadId`''' + return UploadId(UUID(response.id)) + + def __str__(self) -> str: + return str(self.__upload_id) + + def __repr__(self) -> str: + return str(self) + + def __eq__(self, other) -> bool: + '''Checks if two upload ids are equal''' + if not isinstance(other, self.__class__): + return False + + return self.__upload_id == other.__upload_id # pylint: disable=protected-access + + def to_api_dict(self) -> geoengine_openapi_client.AddCollection200Response: + '''Converts the upload id to a dict for the api''' + return geoengine_openapi_client.AddCollection200Response( + id=str(self.__upload_id) + ) + + +class Resource: + '''A wrapper for a resource id''' + + id: str + type: Literal['dataset', 'layer', 'layerCollection', 'mlModel', 'project'] + + def __init__(self, resource_type: Literal['dataset', 'layer', 'layerCollection', 'mlModel', 'project'], + resource_id: str) -> None: + '''Create a resource id''' + self.type = resource_type + self.id = resource_id + + @classmethod + def from_layer_id(cls, layer_id: LayerId) -> Resource: + '''Create a resource id from a layer id''' + return Resource('layer', str(layer_id)) + + @classmethod + def from_layer_collection_id(cls, layer_collection_id: LayerCollectionId) -> Resource: + '''Create a resource id from a layer collection id''' + return Resource('layerCollection', str(layer_collection_id)) + + @classmethod + def from_dataset_name(cls, dataset_name: Union[DatasetName, str]) -> Resource: + '''Create a resource id from a dataset name''' + if isinstance(dataset_name, DatasetName): + dataset_name = str(dataset_name) + return Resource('dataset', dataset_name) + + @classmethod + def from_ml_model_name(cls, ml_model_name: Union[MlModelName, str]) -> Resource: + '''Create a resource from an ml model name''' + if isinstance(ml_model_name, MlModelName): + ml_model_name = str(ml_model_name) + return Resource('mlModel', ml_model_name) + + def to_api_dict(self) -> geoengine_openapi_client.Resource: + '''Convert to a dict for the API''' + inner: Any = None + + if self.type == "layer": + inner = geoengine_openapi_client.LayerResource(type="layer", id=self.id) + elif self.type == "layerCollection": + inner = geoengine_openapi_client.LayerCollectionResource(type="layerCollection", id=self.id) + elif self.type == "project": + inner = geoengine_openapi_client.ProjectResource(type="project", id=self.id) + elif self.type == "dataset": + inner = geoengine_openapi_client.DatasetResource(type="dataset", id=self.id) + elif self.type == "mlModel": + inner = geoengine_openapi_client.MlModelResource(type="mlModel", id=self.id) + else: + raise KeyError(f"Unknown resource type: {self.type}") + + return geoengine_openapi_client.Resource(inner) + + @classmethod + def from_response(cls, response: geoengine_openapi_client.Resource) -> Resource: + '''Convert to a dict for the API''' + inner: Resource + if isinstance(response.actual_instance, geoengine_openapi_client.LayerResource): + inner = Resource('layer', response.actual_instance.id) + elif isinstance(response.actual_instance, geoengine_openapi_client.LayerCollectionResource): + inner = Resource('layerCollection', response.actual_instance.id) + elif isinstance(response.actual_instance, geoengine_openapi_client.ProjectResource): + inner = Resource('project', response.actual_instance.id) + elif isinstance(response.actual_instance, geoengine_openapi_client.DatasetResource): + inner = Resource('dataset', response.actual_instance.id) + elif isinstance(response.actual_instance, geoengine_openapi_client.MlModelResource): + inner = Resource('mlModel', response.actual_instance.id) + else: + raise KeyError(f"Unknown resource type from API: {response.actual_instance}") + return inner + + def __repr__(self): + return 'id: ' + repr(self.id) + ', type: ' + repr(self.type) + + def __eq__(self, value): + '''Checks if two listings are equal''' + if not isinstance(value, self.__class__): + return False + return self.id == value.id and self.type == value.type diff --git a/geoengine/types.py b/geoengine/types.py index afff0d40..e8189003 100644 --- a/geoengine/types.py +++ b/geoengine/types.py @@ -982,6 +982,9 @@ def from_response(response: geoengine_openapi_client.Symbology) -> Symbology: raise InputException("Invalid symbology type") + def __repr__(self): + "Symbology" + class VectorSymbology(Symbology): '''A vector symbology''' @@ -1088,22 +1091,22 @@ def to_api_dict(self) -> geoengine_openapi_client.RasterColorizer: class RasterSymbology(Symbology): '''A raster symbology''' - __opacity: float - __raster_colorizer: RasterColorizer + opacity: float + raster_colorizer: RasterColorizer def __init__(self, raster_colorizer: RasterColorizer, opacity: float = 1.0) -> None: '''Initialize a new `RasterSymbology`''' - self.__raster_colorizer = raster_colorizer - self.__opacity = opacity + self.raster_colorizer = raster_colorizer + self.opacity = opacity def to_api_dict(self) -> geoengine_openapi_client.Symbology: '''Convert the raster symbology to a dictionary''' return geoengine_openapi_client.Symbology(geoengine_openapi_client.RasterSymbology( type='raster', - raster_colorizer=self.__raster_colorizer.to_api_dict(), - opacity=self.__opacity, + raster_colorizer=self.raster_colorizer.to_api_dict(), + opacity=self.opacity, )) @staticmethod @@ -1115,7 +1118,14 @@ def from_response_raster(response: geoengine_openapi_client.RasterSymbology) -> return RasterSymbology(raster_colorizer, response.opacity) def __repr__(self) -> str: - return super().__repr__() + f"({self.__raster_colorizer}, {self.__opacity})" + return str(self.__class__) + f"({self.raster_colorizer}, {self.opacity})" + + def __eq__(self, value): + '''Check if two RasterSymbologies are equal''' + + if not isinstance(value, self.__class__): + return False + return self.opacity == value.opacity and self.raster_colorizer == value.raster_colorizer class DataId: # pylint: disable=too-few-public-methods diff --git a/tests/test_datasets.py b/tests/test_datasets.py index c4027513..5bf19765 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -1,8 +1,13 @@ """Tests for the datasets module.""" import unittest + +import geoengine_openapi_client import geoengine as ge -from . import UrllibMocker +from geoengine.permissions import REGISTERED_USER_ROLE_ID, Permission, PermissionListing, Role +from geoengine.resource_identifier import Resource +from geoengine.types import RasterBandDescriptor +from tests.ge_test import GeoEngineTestInstance class DatasetsTests(unittest.TestCase): @@ -15,83 +20,357 @@ def setUp(self) -> None: def test_list_datasets(self): """Test `GET /datasets`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - m.get( - 'http://mock-instance/datasets?filter=foo&order=NameAsc&offset=1&limit=2', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json=[{ - 'id': '35383178-4b70-421f-af82-c345c81d2e13', - 'name': 'foobar', - 'displayName': 'Land Cover', - 'description': 'Land Cover', - 'tags': [], - 'sourceOperator': 'GdalSource', - 'resultDescriptor': {'type': 'raster', - 'dataType': 'U8', - 'spatialReference': 'EPSG:4326', - 'bands': [{'name': 'band', - 'measurement': - {'type': 'classification', - 'measurement': 'Land Cover', - 'classes': {'0': 'Water Bodies', - '1': 'Evergreen Needleleaf Forests', - '10': 'Grasslands', - '11': 'Permanent Wtlands', - '12': 'Croplands', - '13': 'Urban and Built-Up', - '14': 'Cropland-Natural Vegetation Mosaics', - '15': 'Snow and Ice', - '16': 'Barren or Sparsely Vegetated', - '2': 'Evergreen Broadleaf Forests', - '3': 'Deciduous Needleleaf Forests', - '4': 'Deciduous Broadleleaf Forests', - '5': 'Mixed Forests', - '6': 'Closed Shrublands', - '7': 'Open Shrublands', - '8': 'Woody Savannas', - '9': 'Savannas'}}}], - 'time': None, - 'bbox': {'upperLeftCoordinate': {'x': -180.0, 'y': 90.0}, - 'lowerRightCoordinate': {'x': 180.0, 'y': -90.0}}, - 'resolution': {'x': 0.1, 'y': 0.1}}, - 'symbology': {'type': 'raster', - 'opacity': 1.0, - 'rasterColorizer': { - 'type': 'singleBand', - 'band': 0, - 'bandColorizer': { - 'type': 'linearGradient', - 'breakpoints': [{'value': 0.0, 'color': [0, 0, 255, 255]}, - {'value': 8.0, 'color': [0, 255, 0, 255]}, - {'value': 16.0, 'color': [255, 0, 0, 255]}], - 'noDataColor': [0, 0, 0, 0], - 'overColor': [0, 0, 0, 0], - 'underColor': [0, 0, 0, 0]}}}} - ]) - - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address()) datasets = ge.list_datasets( - offset=1, - limit=2, + offset=0, + limit=10, order=ge.DatasetListOrder.NAME_ASC, - name_filter='foo' + name_filter="Natural Earth II" ) - self.assertEqual(len(datasets), 1) + self.assertEqual(len(datasets), 3) dataset = datasets[0] - self.assertEqual(dataset.name, 'foobar') - self.assertEqual(dataset.display_name, 'Land Cover') + self.assertEqual(dataset.name, 'ne2_raster_blue') + self.assertEqual(dataset.display_name, 'Natural Earth II – Blue') self.assertEqual(dataset.result_descriptor.actual_instance.type, 'raster') + def test_add_dataset(self): + """Test `add_datset`.""" + + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() + + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) + + volumes = ge.volumes() + + geo_transform = ge.GeoTransform( + x_min=180.0, + y_max=90.0, + x_pixel_size=0.1, + y_pixel_size=-0.1 + ) + + gdal_params = geoengine_openapi_client.GdalDatasetParameters.from_dict({ + "filePath": "raster/landcover/landcover.tif", + "rasterbandChannel": 1, + "geoTransform": geo_transform.to_api_dict(), + "width": 3600, + "height": 1800, + "fileNotFoundHandling": geoengine_openapi_client.FileNotFoundHandling.NODATA, + "noDataValue": None, + "propertiesMapping": None, + "gdalOpenOptions": None, + "gdalConfigOptions": None, + "allowAlphabandAsMask": True + }) + + result_descriptor_measurement = ge.ClassificationMeasurement( + measurement="Land Cover", + classes={ + "0": "Water Bodies", + "1": "Evergreen Needleleaf Forests", + "2": "Evergreen Broadleaf Forests", + "3": "Deciduous Needleleaf Forests", + "4": "Deciduous Broadleleaf Forests", + "5": "Mixed Forests", + "6": "Closed Shrublands", + "7": "Open Shrublands", + "8": "Woody Savannas", + "9": "Savannas", + "10": "Grasslands", + "11": "Permanent Wtlands", + "12": "Croplands", + "13": "Urban and Built-Up", + "14": "Cropland-Natural Vegetation Mosaics", + "15": "Snow and Ice", + "16": "Barren or Sparsely Vegetated" + } + + ) + + result_descriptor = ge.RasterResultDescriptor( + "U8", + [RasterBandDescriptor("band", result_descriptor_measurement)], + "EPSG:4326", + spatial_bounds=ge.SpatialPartition2D(-180.0, -90.0, 180.0, 90.0), + spatial_resolution=ge.SpatialResolution(0.1, 0.1) + ) + + meta_data = geoengine_openapi_client.GdalMetaDataStatic.from_dict({ + "type": "GdalStatic", + "time": None, + "params": gdal_params, + "resultDescriptor": result_descriptor.to_api_dict().to_dict(), + }) + + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=8, color=(0, 255, 0, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[ge.Provenance( + citation="The data was obtained from .", + uri="https://lpdaac.usgs.gov/products/mcd12c1v006/", + license="All data distributed by the LP DAAC contain no restrictions on the data reuse." + )], + + + ) + + dataset_name = ge.add_dataset( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + + def test_add_dataset_with_permissions(self): + """Test `add_datset`.""" + + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() + + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) + + volumes = ge.volumes() + + geo_transform = ge.GeoTransform( + x_min=180.0, + y_max=90.0, + x_pixel_size=0.1, + y_pixel_size=-0.1 + ) + + gdal_params = geoengine_openapi_client.GdalDatasetParameters.from_dict({ + "filePath": "raster/landcover/landcover.tif", + "rasterbandChannel": 1, + "geoTransform": geo_transform.to_api_dict(), + "width": 3600, + "height": 1800, + "fileNotFoundHandling": geoengine_openapi_client.FileNotFoundHandling.NODATA, + "noDataValue": None, + "propertiesMapping": None, + "gdalOpenOptions": None, + "gdalConfigOptions": None, + "allowAlphabandAsMask": True + }) + + result_descriptor_measurement = ge.ClassificationMeasurement( + measurement="Land Cover", + classes={ + "0": "Water Bodies", + "1": "Evergreen Needleleaf Forests", + "2": "Evergreen Broadleaf Forests", + "3": "Deciduous Needleleaf Forests", + "4": "Deciduous Broadleleaf Forests", + "5": "Mixed Forests", + "6": "Closed Shrublands", + "7": "Open Shrublands", + "8": "Woody Savannas", + "9": "Savannas", + "10": "Grasslands", + "11": "Permanent Wtlands", + "12": "Croplands", + "13": "Urban and Built-Up", + "14": "Cropland-Natural Vegetation Mosaics", + "15": "Snow and Ice", + "16": "Barren or Sparsely Vegetated" + } + + ) + + result_descriptor = ge.RasterResultDescriptor( + "U8", + [RasterBandDescriptor("band", result_descriptor_measurement)], + "EPSG:4326", + spatial_bounds=ge.SpatialPartition2D(-180.0, -90.0, 180.0, 90.0), + spatial_resolution=ge.SpatialResolution(0.1, 0.1) + ) + + meta_data = geoengine_openapi_client.GdalMetaDataStatic.from_dict({ + "type": "GdalStatic", + "time": None, + "params": gdal_params, + "resultDescriptor": result_descriptor.to_api_dict().to_dict(), + }) + + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[], + ) + + permisions = [(REGISTERED_USER_ROLE_ID, Permission.READ)] + + dataset_name = ge.add_or_replace_dataset_with_permissions( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + permission_tuples=permisions + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + dataset_info = ge.dataset_info_by_name(ge.DatasetName('MCD12C1_test')) + self.assertEqual( + dataset_info.name, + 'MCD12C1_test' + ) + self.assertEqual( + dataset_info.description, + 'Land Cover' + ) + + expected_permission = PermissionListing( + role=Role(role_name="user", role_id=REGISTERED_USER_ROLE_ID), + resource=Resource.from_dataset_name(dataset_name), + permission=Permission.READ + ) + self.assertIn( + expected_permission, + ge.permissions.list_permissions(Resource.from_dataset_name(dataset_name)) + ) + + # now get without overwrite + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover 2", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[], + ) + + dataset_name = ge.add_or_replace_dataset_with_permissions( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + permission_tuples=permisions + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + dataset_info = ge.dataset_info_by_name(ge.DatasetName('MCD12C1_test')) + self.assertEqual( + dataset_info.name, + 'MCD12C1_test' + ) + self.assertEqual( + dataset_info.description, + 'Land Cover' # Still the first value, since no overwrite + ) + + # now overwrite + add_dataset_properties = ge.AddDatasetProperties( + name="MCD12C1_test", + display_name="Land Cover TEST", + source_operator="GdalSource", + description="Land Cover 3", + symbology=ge.RasterSymbology( + opacity=1.0, + raster_colorizer=ge.SingleBandRasterColorizer( + band=0, + band_colorizer=ge.LinearGradientColorizer( + breakpoints=[ + ge.ColorBreakpoint(value=0, color=(0, 0, 255, 255)), + ge.ColorBreakpoint(value=16, color=(255, 0, 0, 255)) + ], + no_data_color=(0, 0, 0, 0), + over_color=(0, 0, 0, 0), + under_color=(0, 0, 0, 0) + ) + ) + ), + provenance=[], + ) + + dataset_name = ge.add_or_replace_dataset_with_permissions( + volumes[0], + add_dataset_properties, + geoengine_openapi_client.MetaDataDefinition( + meta_data, + ), + permission_tuples=permisions, + replace_existing=True + ) + + self.assertEqual(dataset_name, ge.DatasetName('MCD12C1_test')) + self.assertEqual(len(ge.list_datasets(name_filter='Land Cover TEST')), 1) + dataset_info = ge.dataset_info_by_name(ge.DatasetName('MCD12C1_test')) + self.assertEqual( + dataset_info.name, + 'MCD12C1_test' + ) + self.assertEqual( + dataset_info.description, + 'Land Cover 3' # Now the third value, replaced with new dataset + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_layers.py b/tests/test_layers.py index ea6fa98c..ce5927fd 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -5,10 +5,12 @@ import geoengine as ge from geoengine import StoredDataset, BadRequestException from geoengine import api -from geoengine.datasets import DatasetName, UploadId +from geoengine.permissions import REGISTERED_USER_ROLE_ID, Permission, PermissionListing, Role +from geoengine.resource_identifier import LAYER_DB_PROVIDER_ID, Resource from geoengine.layers import Layer, LayerId, LayerProviderId +from geoengine.tasks import TaskStatus from geoengine.types import RasterSymbology -from . import UrllibMocker +from tests.ge_test import GeoEngineTestInstance class LayerTests(unittest.TestCase): @@ -21,423 +23,130 @@ def setUp(self) -> None: def test_layer(self): """Test `add_layer`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) - - # pylint: disable=line-too-long - m.get( - 'http://mock-instance/layers/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/9ee3619e-d0f9-4ced-9c44-3d407c3aed69', - json={ - "description": "Land Cover derived from MODIS/Terra+Aqua Land Cover", - "id": { - "layerId": "9ee3619e-d0f9-4ced-9c44-3d407c3aed69", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "metadata": {}, - "name": "Land Cover", - "properties": [], - "symbology": { - "rasterColorizer": { - "type": "singleBand", - "band": 0, - "bandColorizer": { - "colors": { - "0": [134, 201, 227, 255], - "1": [30, 129, 62, 255], - "2": [59, 194, 212, 255], - "3": [157, 194, 63, 255], - "4": [159, 225, 127, 255], - "5": [125, 194, 127, 255], - "6": [195, 127, 126, 255], - "7": [188, 221, 190, 255], - "8": [224, 223, 133, 255], - "9": [226, 221, 7, 255], - "10": [223, 192, 125, 255], - "11": [66, 128, 189, 255], - "12": [225, 222, 127, 255], - "13": [253, 2, 0, 255], - "14": [162, 159, 66, 255], - "15": [255, 255, 255, 255], - "16": [192, 192, 192, 255] - }, - "defaultColor": [0, 0, 0, 0], - "noDataColor": [0, 0, 0, 0], - "type": "palette" - } - }, - "opacity": 1, - "type": "raster" + expected = ge.Layer( + name='Natural Earth II – RGB', + description='A raster with three bands for RGB visualization', + layer_id=LayerId('83866f7b-dcee-47b8-9242-e5636ceaf402'), + provider_id=LAYER_DB_PROVIDER_ID, + metadata={}, + properties=[], + symbology=ge.RasterSymbology( + raster_colorizer=ge.MultiBandRasterColorizer( + red_band=2, + green_band=1, + blue_band=0, + red_min=0.0, + red_max=255.0, + red_scale=1.0, + green_min=0.0, + green_max=255.0, + green_scale=1.0, + blue_min=0.0, + blue_max=255.0, + blue_scale=1.0 + ), + opacity=1.0 + ), + workflow={ + 'operator': { + 'params': {'renameBands': { + 'type': 'rename', 'values': ['blue', 'green', 'red'] + } }, - "workflow": { - "operator": { - "params": { - "data": "ndvi" - }, - "type": "GdalSource" + 'sources': + {'rasters': [ + {'type': 'GdalSource', 'params': { + 'data': 'ne2_raster_blue' + } }, - "type": "Raster" - } - }) + {'type': 'GdalSource', 'params': { + 'data': 'ne2_raster_green' + } + }, {'type': 'GdalSource', 'params': { + 'data': 'ne2_raster_red' + } + } + ] + }, 'type': 'RasterStacker' + }, + 'type': 'Raster' + } - ge.initialize("http://mock-instance") + ) - layer = ge.layer('9ee3619e-d0f9-4ced-9c44-3d407c3aed69', 'ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b') + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() + + ge.initialize(ge_instance.address()) + + layer = ge.layer(LayerId('83866f7b-dcee-47b8-9242-e5636ceaf402'), LAYER_DB_PROVIDER_ID) self.assertEqual( - layer.to_api_dict(), - api.Layer( - name='Land Cover', - description="Land Cover derived from MODIS/Terra+Aqua Land Cover", - id=api.ProviderLayerId( - layer_id='9ee3619e-d0f9-4ced-9c44-3d407c3aed69', - provider_id='ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b' - ), - metadata={}, - properties=[], - symbology=api.Symbology(api.RasterSymbology( - opacity=1, - raster_colorizer=api.RasterColorizer(api.SingleBandRasterColorizer( - type='singleBand', - band=0, - band_colorizer=api.Colorizer(api.PaletteColorizer( - colors={ - "0.0": [134, 201, 227, 255], - "1.0": [30, 129, 62, 255], - "2.0": [59, 194, 212, 255], - "3.0": [157, 194, 63, 255], - "4.0": [159, 225, 127, 255], - "5.0": [125, 194, 127, 255], - "6.0": [195, 127, 126, 255], - "7.0": [188, 221, 190, 255], - "8.0": [224, 223, 133, 255], - "9.0": [226, 221, 7, 255], - "10.0": [223, 192, 125, 255], - "11.0": [66, 128, 189, 255], - "12.0": [225, 222, 127, 255], - "13.0": [253, 2, 0, 255], - "14.0": [162, 159, 66, 255], - "15.0": [255, 255, 255, 255], - "16.0": [192, 192, 192, 255] - }, - default_color=[0, 0, 0, 0], - no_data_color=[0, 0, 0, 0], - type="palette" - )))), - type="raster" - )), - workflow={ - "operator": { - "params": { - "data": "ndvi" - }, - "type": "GdalSource" - }, - "type": "Raster" - } - ) + layer, + expected ) def test_layer_collection(self): """Test `add_layer`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/546073b6-d535-4205-b601-99675c9f6dd7?offset=0&limit=20', - json={ - "description": "Basic Layers for all Datasets", - "entryLabel": None, - "id": { - "collectionId": "546073b6-d535-4205-b601-99675c9f6dd7", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "items": [ - { - "description": "Land Cover derived from MODIS/Terra+Aqua Land Cover", - "id": { - "layerId": "9ee3619e-d0f9-4ced-9c44-3d407c3aed69", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "name": "Land Cover", - "type": "layer" - }, - { - "description": "NDVI data from MODIS", - "id": { - "layerId": "36574dc3-560a-4b09-9d22-d5945f2b8093", - "providerId": "ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b" - }, - "name": "NDVI", - "type": "layer" - } - ], - "name": "Datasets", - "properties": [] - } - ) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address()) layer_collection = ge.layer_collection( - '546073b6-d535-4205-b601-99675c9f6dd7', - 'ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b' + LayerId('272bf675-2e27-4412-824c-287c1e6841ac'), + LAYER_DB_PROVIDER_ID ) + print(layer_collection) + self.assertEqual( - layer_collection.__dict__, + layer_collection, ge.LayerCollection( - name='Datasets', - description='Basic Layers for all Datasets', - collection_id=ge.LayerCollectionId('546073b6-d535-4205-b601-99675c9f6dd7'), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), + name='A test collection', + description='Some layers for testing and an empty subcollection', + collection_id=ge.LayerCollectionId('272bf675-2e27-4412-824c-287c1e6841ac'), + provider_id=LAYER_DB_PROVIDER_ID, items=[ + ge.LayerCollectionListing( + listing_id=ge.LayerCollectionId('a29f77cc-51ce-466b-86ef-d0ab2170bc0a'), + provider_id=LAYER_DB_PROVIDER_ID, + name='An empty collection', + description='There is nothing here' + ), ge.LayerListing( - listing_id=ge.LayerId('9ee3619e-d0f9-4ced-9c44-3d407c3aed69'), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), - name='Land Cover', - description='Land Cover derived from MODIS/Terra+Aqua Land Cover', + listing_id=ge.LayerCollectionId('83866f7b-dcee-47b8-9242-e5636ceaf402'), + provider_id=LAYER_DB_PROVIDER_ID, + name='Natural Earth II – RGB', + description='A raster with three bands for RGB visualization' ), ge.LayerListing( - listing_id=ge.LayerId('36574dc3-560a-4b09-9d22-d5945f2b8093'), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), - name='NDVI', - description='NDVI data from MODIS', + listing_id=ge.LayerId('b75db46e-2b9a-4a86-b33f-bc06a73cd711'), + provider_id=LAYER_DB_PROVIDER_ID, + name='Ports in Germany', + description='Natural Earth Ports point filtered with Germany polygon', + ), + ge.LayerListing( + listing_id=ge.LayerId('c078db52-2dc6-4838-ad75-340cefeab476'), + provider_id=LAYER_DB_PROVIDER_ID, + name='Stacked Raster', + description='A raster with two bands for testing', ), ], - ).__dict__ + ) ) def test_layer_collection_modification(self): """Test addition and removal to a data collection.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/05102bb3-a855-4a37-8a8a-30026a91fef1?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "Root collection for LayerDB", - "entryLabel": None, - "id": { - "collectionId": "05102bb3-a855-4a37-8a8a-30026a91fef1", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "LayerDB", - "properties": [] - }, - ) - - m.post( - 'http://mock-instance/layerDb/collections/05102bb3-a855-4a37-8a8a-30026a91fef1/collections', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "id": "490ef009-aa7a-44b0-bbef-73cfb5916b55" - } - ) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/490ef009-aa7a-44b0-bbef-73cfb5916b55?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "test description", - "entryLabel": None, - "id": { - "collectionId": "490ef009-aa7a-44b0-bbef-73cfb5916b55", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "my test collection", - "properties": [] - }, - ) - - m.post( - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "id": "64221c85-22df-4d30-9c97-605e5c498629" - } - ) - - m.post( - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/layers', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - expected_request_body={ - 'name': 'ports clone', - 'description': 'test description', - 'workflow': { - 'type': 'Vector', - 'operator': { - 'type': 'PointInPolygonFilter', - 'params': {}, - 'sources': { - 'points': { - 'type': 'OgrSource', - 'params': { - 'data': "ne_10m_ports", - 'attributeProjection': None, - 'attributeFilters': None - } - }, - 'polygons': { - 'type': 'OgrSource', - 'params': { - 'data': "germany_outline", - 'attributeProjection': None, - 'attributeFilters': None - } - }, - } - } - }, - 'symbology': None - }, - json={ - "id": "fbffb07e-d8b7-4688-98a5-4665988e6ae3" - } - ) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/64221c85-22df-4d30-9c97-605e5c498629?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "another description", - "entryLabel": None, - "id": { - "collectionId": "64221c85-22df-4d30-9c97-605e5c498629", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "sub collection", - "properties": [] - }, - ) - - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/64221c85-22df-4d30-9c97-605e5c498629/layers/fbffb07e-d8b7-4688-98a5-4665988e6ae3', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - m.get( - 'http://mock-instance/layers/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/fbffb07e-d8b7-4688-98a5-4665988e6ae3', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - 'id': { - "layerId": "fbffb07e-d8b7-4688-98a5-4665988e6ae3", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - 'name': 'ports clone', - 'description': 'test description', - 'workflow': { - 'type': 'Vector', - 'operator': { - 'type': 'PointInPolygonFilter', - 'params': {}, - 'sources': { - 'points': { - 'type': 'OgrSource', - 'params': { - 'data': "ne_10m_ports", - 'attributeProjection': None, - 'attributeFilters': None - } - }, - 'polygons': { - 'type': 'OgrSource', - 'params': { - 'data': "germany_outline", - 'attributeProjection': None, - 'attributeFilters': None - } - } - } - } - }, - 'symbology': None, - "metadata": {}, - "properties": [], - }, - ) - - m.post( - 'http://mock-instance/layerDb/collections/64221c85-22df-4d30-9c97-605e5c498629/collections', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - expected_request_body={ - 'name': 'sub sub collection', - 'description': 'yet another description', - }, - json={ - "id": "cd5c3f0f-c682-4f49-820d-8d704f25e803" - }, - ) - - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections/cd5c3f0f-c682-4f49-820d-8d704f25e803', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.get( - # pylint: disable=line-too-long - 'http://mock-instance/layers/collections/ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74/cd5c3f0f-c682-4f49-820d-8d704f25e803?offset=0&limit=20', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - json={ - "description": "yet another description", - "entryLabel": None, - "id": { - "collectionId": "cd5c3f0f-c682-4f49-820d-8d704f25e803", - "providerId": "ce5e84db-cbf9-48a2-9a32-d4b7cc56ea74" - }, - "items": [], - "name": "sub sub collection", - "properties": [] - }, - ) - - m.delete( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections/64221c85-22df-4d30-9c97-605e5c498629', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.delete( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/layers/fbffb07e-d8b7-4688-98a5-4665988e6ae3', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.delete( - # pylint: disable=line-too-long - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55/collections/cd5c3f0f-c682-4f49-820d-8d704f25e803', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - m.delete( - 'http://mock-instance/layerDb/collections/490ef009-aa7a-44b0-bbef-73cfb5916b55', - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}, - ) - - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) root_of_layerdb = ge.layer_collection('05102bb3-a855-4a37-8a8a-30026a91fef1') @@ -497,67 +206,21 @@ def test_layer_collection_modification(self): def test_save_as_dataset(self): """Test `layer.save_as_dataset`.""" - with UrllibMocker() as m: - m.post('http://mock-instance/anonymous', json={ - "id": "c4983c3e-9b53-47ae-bda9-382223bd5081", - "project": None, - "view": None - }) + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() - # Success case - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layers/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/9ee3619e-d0f9-4ced-9c44-3d407c3aed69/dataset', - json={'taskId': '7f210984-8f2d-44f6-b211-ededada17598'}, - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}) - - m.get('http://mock-instance/tasks/7f210984-8f2d-44f6-b211-ededada17598/status', - # pylint: disable=line-too-long - json={'status': 'completed', - 'info': {'dataset': '94230f0b-4e8a-4cba-9adc-3ace837fe5d4', - 'upload': '3086f494-d5a4-4b51-a14b-3b29f8bf7bb0'}, - 'timeTotal': '00:00:00', - 'taskType': 'create-dataset', - 'description': 'Creating dataset Test Raster Layer from layer 86c81654-e572-42ed-96ee-8b38ebcd84ab', - 'timeStarted': '2023-02-16T15:25:45.390Z'}, ) - - # Some processing error occurred - m.post( - # pylint: disable=line-too-long - 'http://mock-instance/layers/ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b/86c81654-e572-42ed-96ee-8b38ebcd84ab/dataset', - status_code=400, - json={'error': 'Some Processing Error', - 'message': 'Some Processing Message'}, - request_headers={'Authorization': 'Bearer c4983c3e-9b53-47ae-bda9-382223bd5081'}) - - ge.initialize("http://mock-instance") + ge.initialize(ge_instance.address()) # Success case - layer = ge.Layer( - name='Test Raster Layer', - description='Test Raster Layer Description', - layer_id=ge.LayerId(UUID('9ee3619e-d0f9-4ced-9c44-3d407c3aed69')), - provider_id=ge.LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), - workflow={ - "operator": { - "params": { - "data": "ndvi" - }, - "type": "GdalSource" - }, - "type": "Raster" - }, - symbology=None, - properties=[], - metadata={}, - ) + layer = ge.layer(LayerId('83866f7b-dcee-47b8-9242-e5636ceaf402'), LAYER_DB_PROVIDER_ID) task = layer.save_as_dataset() - task_status = task.get_status() + task_status = task.wait_for_finish() + self.assertEqual(task_status.status, TaskStatus.COMPLETED) stored_dataset = StoredDataset.from_response(task_status.info) - - self.assertEqual(stored_dataset.dataset_name, DatasetName("94230f0b-4e8a-4cba-9adc-3ace837fe5d4")) - self.assertEqual(stored_dataset.upload_id, UploadId(UUID("3086f494-d5a4-4b51-a14b-3b29f8bf7bb0"))) + self.assertNotEqual(stored_dataset.dataset_name, '') + self.assertNotEqual(stored_dataset.upload_id, '') # Some processing error occurred (e.g., layer does not exist) layer = ge.Layer( @@ -592,9 +255,9 @@ def test_layer_repr_html_does_not_crash(self): provider_id=LayerProviderId(UUID('ac50ed0d-c9a0-41f8-9ce8-35fc9e38299b')), workflow={ "operator": { - "params": { - "data": "ndvi" - }, + "params": { + "data": "ndvi" + }, "type": "GdalSource" }, "type": "Raster" @@ -642,6 +305,108 @@ def test_layer_repr_html_does_not_crash(self): _html = layer._repr_html_() # pylint: disable=protected-access + def test_layer_collection_advanced_modification(self): + """Test addition and overwrite to a data collection.""" + + permisions = [(REGISTERED_USER_ROLE_ID, Permission.READ)] + + # TODO: use `enterContext(cm)` instead of `with cm:` in Python 3.11 + with GeoEngineTestInstance() as ge_instance: + ge_instance.wait_for_ready() + + ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin")) + + root_of_layerdb = ge.layer_collection('05102bb3-a855-4a37-8a8a-30026a91fef1') + + # create a new collection with permissions + created_collection = root_of_layerdb.get_or_create_unique_collection( + "my test collection", + create_collection_description="the first collection description", + create_permissions_tuples=permisions) + + root_of_layerdb = root_of_layerdb.reload() + + collection_in = root_of_layerdb.get_items_by_name("my test collection")[0].load() + + self.assertEqual(created_collection.description, "the first collection description") + self.assertEqual(created_collection, collection_in) + + expected_permission = PermissionListing( + role=Role(role_name="user", role_id=REGISTERED_USER_ROLE_ID), + resource=Resource.from_layer_collection_id(created_collection.collection_id), + permission=Permission.READ + ) + + self.assertIn(expected_permission, + ge.permissions.list_permissions( + Resource.from_layer_collection_id(created_collection.collection_id) + ) + ) + + # get the existing collection (no overwrite) + existing_collection = root_of_layerdb.get_or_create_unique_collection( + "my test collection", + create_collection_description="the second collection description", + create_permissions_tuples=permisions) + + root_of_layerdb = root_of_layerdb.reload() + + collection_in = root_of_layerdb.get_items_by_name("my test collection")[0].load() + self.assertEqual(existing_collection.description, "the first collection description") + self.assertEqual(existing_collection, collection_in) + + # now overwrite existing collection + overwrite_collection = root_of_layerdb.get_or_create_unique_collection( + "my test collection", + create_collection_description="the third collection description", + create_permissions_tuples=permisions, + delete_existing_with_same_name=True + ) + + root_of_layerdb = root_of_layerdb.reload() + + collection_in = root_of_layerdb.get_items_by_name("my test collection")[0].load() + self.assertEqual(overwrite_collection.description, "the third collection description") + self.assertEqual(overwrite_collection, collection_in) + + new_layer = overwrite_collection.add_layer_with_permissions( + name="ports clone", + description="test description", + workflow={ + "type": "Vector", + "operator": { + "type": "PointInPolygonFilter", + "params": {}, + "sources": { + "points": { + "type": "OgrSource", + "params": { + "data": "ne_10m_ports", + "attributeProjection": None, + "attributeFilters": None + } + }, + "polygons": { + "type": "OgrSource", + "params": { + "data": "germany_outline", + "attributeProjection": None, + "attributeFilters": None + } + } + } + } + }, + symbology=None, + permission_tuples=permisions) + + expected_permission = PermissionListing( + role=Role(role_name="user", role_id=REGISTERED_USER_ROLE_ID), + resource=Resource.from_layer_id(new_layer), + permission=Permission.READ + ) + self.assertIn(expected_permission, ge.permissions.list_permissions(Resource.from_layer_id(new_layer))) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_workflow_storage.py b/tests/test_workflow_storage.py index 5bd46103..f15c27a4 100644 --- a/tests/test_workflow_storage.py +++ b/tests/test_workflow_storage.py @@ -3,7 +3,8 @@ import unittest from uuid import UUID import geoengine_openapi_client -from geoengine.datasets import DatasetName, UploadId, StoredDataset +from geoengine.datasets import DatasetName, StoredDataset +from geoengine.resource_identifier import UploadId import geoengine as ge from . import UrllibMocker