From 0f048dbbc79d5bae9f5c7c6821f33e25c9fa699f Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 26 Jan 2022 18:32:55 +0100 Subject: [PATCH 01/24] [Fixes #8689] Extend the ResourceBase metadata model with an opaque JSONField --- geonode/base/forms.py | 32 +++++++++++ .../0062_resourcebase_extra_metadata.py | 19 +++++++ geonode/base/models.py | 3 ++ geonode/layers/templates/layouts/panels.html | 6 +++ geonode/layers/tests.py | 53 ++++++++++++++++++- geonode/layers/views.py | 3 +- geonode/settings.py | 31 +++++++++++ requirements.txt | 1 + setup.cfg | 1 + 9 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 geonode/base/migrations/0062_resourcebase_extra_metadata.py diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 939e61c9998..b104107b073 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -16,6 +16,8 @@ # along with this program. If not, see . # ######################################################################### +import json +from schema import Schema import re import html import logging @@ -477,6 +479,8 @@ class ResourceBaseForm(TranslationModelForm): regions.widget.attrs = {"size": 20} + extra_metadata = forms.CharField(required=False, widget=forms.Textarea) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) for field in self.fields: @@ -524,6 +528,34 @@ def clean_title(self): title = title.replace(",", "_") return title + def clean_extra_metadata(self): + cleaned_data = self.cleaned_data + if not cleaned_data.get('extra_metadata', []): + return cleaned_data + + # starting validation of extra metadata passed via JSON + # if schema for metadata validation is not defined, an error is raised + resource_type = self.instance.polymorphic_ctype.model + extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) + if not extra_metadata_validation_schema: + raise forms.ValidationError( + f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {self.instance.polymorphic_ctype.model}" + ) + # starting json structure validation. The Field can contain multiple metadata + + try: + _extra_as_json = json.loads(cleaned_data.get('extra_metadata')) + except Exception: + raise forms.ValidationError("The value provided for the Extra metadata field is not a valid JSON") + + # looping on all the single metadata provided. If it doen't match the schema an error is raised + for _index, _metadata in enumerate(_extra_as_json): + try: + Schema(extra_metadata_validation_schema).validate(_metadata) + except Exception as e: + raise forms.ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") + return cleaned_data.get('extra_metadata') + class Meta: exclude = ( 'contacts', diff --git a/geonode/base/migrations/0062_resourcebase_extra_metadata.py b/geonode/base/migrations/0062_resourcebase_extra_metadata.py new file mode 100644 index 00000000000..0f48abef8c3 --- /dev/null +++ b/geonode/base/migrations/0062_resourcebase_extra_metadata.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.24 on 2022-01-26 14:04 + +from django.db import migrations +import django_jsonfield_backport.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0061_auto_20211117_1238'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='extra_metadata', + field=django_jsonfield_backport.models.JSONField(blank=True, default=list, null=True), + ), + ] diff --git a/geonode/base/models.py b/geonode/base/models.py index cf0d92b1cb8..21f80e6bafe 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -43,6 +43,7 @@ from django.core.files.storage import default_storage as storage from django.utils.html import strip_tags from mptt.models import MPTTModel, TreeForeignKey +from django_jsonfield_backport.models import JSONField from PIL import Image, ImageOps @@ -961,6 +962,8 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): default=False, help_text=_('If true, will be excluded from search')) + extra_metadata = JSONField(null=True, default=list, blank=True) + objects = ResourceBaseManager() class Meta: diff --git a/geonode/layers/templates/layouts/panels.html b/geonode/layers/templates/layouts/panels.html index df5073d8503..4d2d6b6cbe3 100644 --- a/geonode/layers/templates/layouts/panels.html +++ b/geonode/layers/templates/layouts/panels.html @@ -544,6 +544,12 @@ {{ layer_form.spatial_representation_type }} {% endblock layer_spatial_representation_type %} + {% block layer_extra_metadata %} +
+ + {{ layer_form.extra_metadata }} +
+ {% endblock layer_extra_metadata %}
diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index ac690b14deb..8218196fe6d 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -65,7 +65,7 @@ from geonode.people.utils import get_valid_user from geonode.base.populate_test_data import all_public, create_single_layer from geonode.base.models import TopicCategory, License, Region, Link -from geonode.layers.forms import JSONField, LayerUploadForm +from geonode.layers.forms import JSONField, LayerForm, LayerUploadForm from geonode.utils import check_ogc_backend, set_resource_default_links from geonode.layers import LayersAppConfig from geonode.tests.utils import NotificationsTestsHelper @@ -1945,3 +1945,54 @@ def test_give_single_file_should_return_False(self): request.FILES['base_file'] = f actual = is_sld_upload_only(request) self.assertFalse(actual) + + +class TestLayerForm(GeoNodeBaseTestSupport): + def setUp(self) -> None: + self.user = get_user_model().objects.get(username='admin') + self.layer = create_single_layer("my_single_layer", owner=self.user) + self.sut = LayerForm + + def test_resource_form_is_invalid_extra_metadata_not_json_format(self): + self.client.login(username="admin", password="admin") + url = reverse("layer_metadata", args=(self.layer.alternate,)) + response = self.client.post(url, data={ + "resource-owner": self.layer.owner.id, + "resource-title": "layer_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "not-a-json" + }) + expected = {"success": False, "errors": ["extra_metadata: The value insered for the Extra metadata field is not a valid JSON"]} + self.assertDictEqual(expected, response.json()) + + @override_settings(EXTRA_METADATA_SCHEMA={"key": "value"}) + def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): + self.client.login(username="admin", password="admin") + url = reverse("layer_metadata", args=(self.layer.alternate,)) + response = self.client.post(url, data={ + "resource-owner": self.layer.owner.id, + "resource-title": "layer_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "[{'key': 'value'}]" + }) + expected = {"success": False, "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource layer"]} + self.assertDictEqual(expected, response.json()) + + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): + self.client.login(username="admin", password="admin") + url = reverse("layer_metadata", args=(self.layer.alternate,)) + response = self.client.post(url, data={ + "resource-owner": self.layer.owner.id, + "resource-title": "layer_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": '[{"key": "value"},{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + }) + expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" + self.assertIn(expected, response.json()['errors'][0]) + \ No newline at end of file diff --git a/geonode/layers/views.py b/geonode/layers/views.py index fba87c58cf0..ffa06751d8b 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -853,8 +853,7 @@ def layer_metadata( logger.error(f"Layer Metadata form is not valid: {layer_form.errors}") out = { 'success': False, - 'errors': [ - re.sub(re.compile('<.*?>'), '', str(err)) for err in layer_form.errors] + "errors": [f"{x}: {y[0].messages[0]}" for x, y in layer_form.errors.as_data().items()] } return HttpResponse( json.dumps(out), diff --git a/geonode/settings.py b/geonode/settings.py index 8853e062db6..0c7e60e2fd0 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2163,3 +2163,34 @@ def get_geonode_catalogue_service(): DEFAULT_MAX_UPLOAD_SIZE = int(os.getenv('DEFAULT_MAX_UPLOAD_SIZE', 104857600)) # 100 MB DEFAULT_MAX_BEFORE_UPLOAD_SIZE = int(os.getenv('DEFAULT_MAX_BEFORE_UPLOAD_SIZE', 524288000)) # 500 MB + +''' +Default schema used to store extra and dynamic metadata for the resource +''' +DEFAULT_EXTRA_METADATA_SCHEMA = { + "name": str, + "slug": str, + "help_text": str, + "field_type": object, + "value": object, + "category": str +} + +''' +If present, will extend the available metadata schema used for store +new value for each resource. By default overrided the existing one. +The expected schema is the same as the default +''' +CUSTOM_METADATA_SCHEMA = os.getenv('CUSTOM_METADATA_SCHEMA ', {}) + +''' +Variable used to actually get the expected metadata schema for each resource_type. +In this way, each resource type can have a different metadata schema +''' + +EXTRA_METADATA_SCHEMA = {**{ + "map": os.getenv('MAP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), + "layer": os.getenv('DATASET_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), + "document": os.getenv('DOCUMENT_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), + "geoapp": os.getenv('GEOAPP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA) +}, **CUSTOM_METADATA_SCHEMA} diff --git a/requirements.txt b/requirements.txt index 6b20bce60d5..f2b70a36d19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ jsonfield==3.1.0 jsonschema==3.2.0 pyrsistent==0.17.3 zipstream-new==1.1.8 +schema==0.7.5 # Django Apps django-allauth==0.44.0 diff --git a/setup.cfg b/setup.cfg index 9ee168d3c23..1c41251e132 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,7 @@ install_requires = jsonschema==3.2.0 pyrsistent==0.17.3 zipstream-new==1.1.8 + schema==0.7.5 # Django Apps django-allauth==0.44.0 From 20a1c1439a1de0ad2a79ae573cc75b9bd45d48d2 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 27 Jan 2022 09:59:36 +0100 Subject: [PATCH 02/24] [Fixes #8689] Fix missing resource_type for new form instances --- geonode/base/forms.py | 6 +++++- geonode/layers/tests.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index b104107b073..92ed4f2d156 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -535,7 +535,11 @@ def clean_extra_metadata(self): # starting validation of extra metadata passed via JSON # if schema for metadata validation is not defined, an error is raised - resource_type = self.instance.polymorphic_ctype.model + resource_type = ( + self.instance.polymorphic_ctype.model + if self.instance.polymorphic_ctype + else self.instance.class_name.lower() + ) extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) if not extra_metadata_validation_schema: raise forms.ValidationError( diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index 8218196fe6d..de028589d62 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -1995,4 +1995,16 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): }) expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" self.assertIn(expected, response.json()['errors'][0]) + + + def test_resource_form_is_valid_extra_metadata(self): + form = self.sut(data={ + "owner": self.layer.owner.id, + "title": "layer_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + }) + self.assertTrue(form.is_valid()) \ No newline at end of file From 95aa9c78b6be1e3cf214723e604a4fed71f42bf2 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 27 Jan 2022 11:08:17 +0100 Subject: [PATCH 03/24] [Fixes #8689] Add test and UI fix for doc, maps and geoapps --- geonode/base/forms.py | 2 +- geonode/base/populate_test_data.py | 8 +- .../templates/layouts/doc_panels.html | 8 +- geonode/documents/tests.py | 5 +- .../geoapps/templates/layouts/app_panels.html | 6 ++ geonode/geoapps/tests.py | 77 +++++++++++++++++++ geonode/geoapps/views.py | 12 ++- .../maps/templates/layouts/map_panels.html | 6 ++ geonode/maps/tests.py | 65 ++++++++++++++++ geonode/maps/views.py | 12 ++- 10 files changed, 191 insertions(+), 10 deletions(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 92ed4f2d156..897e9ea426e 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -543,7 +543,7 @@ def clean_extra_metadata(self): extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) if not extra_metadata_validation_schema: raise forms.ValidationError( - f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {self.instance.polymorphic_ctype.model}" + f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}" ) # starting json structure validation. The Field can contain multiple metadata diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py index aa1862cee93..cca8d8a6548 100644 --- a/geonode/base/populate_test_data.py +++ b/geonode/base/populate_test_data.py @@ -355,7 +355,7 @@ def create_single_layer(name, keywords=None, owner=None, group=None): return layer -def create_single_map(name, keywords=None): +def create_single_map(name, keywords=None, owner=None): admin, created = get_user_model().objects.get_or_create(username='admin') if created: admin.is_superuser = True @@ -374,7 +374,7 @@ def create_single_map(name, keywords=None): projection='EPSG:4326', center_x=42, center_y=-73, - owner=user, + owner=owner or user, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid='EPSG:4326', @@ -390,7 +390,7 @@ def create_single_map(name, keywords=None): return m -def create_single_doc(name, keywords=None): +def create_single_doc(name, keywords=None, owner=None): admin, created = get_user_model().objects.get_or_create(username='admin') if created: admin.is_superuser = True @@ -406,7 +406,7 @@ def create_single_doc(name, keywords=None): m = Document( title=title, abstract=abstract, - owner=user, + owner=owner or user, bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid='EPSG:4326', diff --git a/geonode/documents/templates/layouts/doc_panels.html b/geonode/documents/templates/layouts/doc_panels.html index f3dcc8cdf6c..7a9f9d7c0ab 100644 --- a/geonode/documents/templates/layouts/doc_panels.html +++ b/geonode/documents/templates/layouts/doc_panels.html @@ -539,7 +539,13 @@ {{ document_form.spatial_representation_type }}
- {% endblock doc_spatial_representation_type %} + {% endblock doc_spatial_representation_type %} + {% block doc_extra_metadata %} +
+ + {{ document_form.extra_metadata }} +
+ {% endblock doc_extra_metadata %}
diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 1ad4b404797..128e313fdd0 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -22,6 +22,7 @@ when you run "manage.py test". """ +from django.test import override_settings from geonode.tests.base import GeoNodeBaseTestSupport import os @@ -40,7 +41,7 @@ from guardian.shortcuts import get_perms, get_anonymous_user -from .forms import DocumentCreateForm +from .forms import DocumentCreateForm, DocumentForm from geonode.groups.models import ( GroupProfile, @@ -53,7 +54,7 @@ from geonode.documents import DocumentsAppConfig from geonode.documents.forms import DocumentFormMixin from geonode.tests.utils import NotificationsTestsHelper -from geonode.base.populate_test_data import create_models +from geonode.base.populate_test_data import create_models, create_single_doc from geonode.documents.enumerations import DOCUMENT_TYPE_MAP from geonode.documents.models import Document, DocumentResourceLink diff --git a/geonode/geoapps/templates/layouts/app_panels.html b/geonode/geoapps/templates/layouts/app_panels.html index 5decca27fde..40800356fbc 100644 --- a/geonode/geoapps/templates/layouts/app_panels.html +++ b/geonode/geoapps/templates/layouts/app_panels.html @@ -470,6 +470,12 @@ {{ geoapp_form.spatial_representation_type }}
+ {% block geoapp_extra_metadata %} +
+ + {{ geoapp_form.extra_metadata }} +
+ {% endblock geoapp_extra_metadata %}
diff --git a/geonode/geoapps/tests.py b/geonode/geoapps/tests.py index 1dc371bddd4..adc3d2181b1 100644 --- a/geonode/geoapps/tests.py +++ b/geonode/geoapps/tests.py @@ -16,8 +16,10 @@ # along with this program. If not, see . # ######################################################################### +from django.test import override_settings from django.urls import reverse from django.contrib.auth import get_user_model +from geonode.geoapps.forms import GeoAppForm from geonode.geoapps.models import GeoApp from geonode.tests.base import GeoNodeBaseTestSupport @@ -25,6 +27,16 @@ class TestGeoAppViews(GeoNodeBaseTestSupport): + def setUp(self) -> None: + self.user = get_user_model().objects.get(username='admin') + self.geoapp = GeoApp.objects.create( + name="name", + title="geoapp_titlte", + thumbnail_url='initial', + owner=self.user + ) + self.sut = GeoAppForm + def test_update_geoapp_metadata(self): bobby = get_user_model().objects.get(username='bobby') gep_app = GeoApp.objects.create( @@ -48,3 +60,68 @@ def test_update_geoapp_metadata(self): self.assertEqual(GeoApp.objects.get(id=gep_app.id).title, 'New title') # Check uuid is populate self.assertTrue(GeoApp.objects.get(id=gep_app.id).uuid) + + def test_resource_form_is_invalid_extra_metadata_not_json_format(self): + self.client.login(username="admin", password="admin") + url = reverse("geoapp_metadata", args=(self.geoapp.id,)) + response = self.client.post(url, data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "not-a-json" + }) + expected = {"success": False, "errors": ["extra_metadata: The value provided for the Extra metadata field is not a valid JSON"]} + self.assertDictEqual(expected, response.json()) + + @override_settings(EXTRA_METADATA_SCHEMA={"key": "value"}) + def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): + self.client.login(username="admin", password="admin") + url = reverse("geoapp_metadata", args=(self.geoapp.id,)) + response = self.client.post(url, data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "[{'key': 'value'}]" + }) + expected = {"success": False, "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource geoapp"]} + self.assertDictEqual(expected, response.json()) + + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): + self.client.login(username="admin", password="admin") + url = reverse("geoapp_metadata", args=(self.geoapp.id,)) + response = self.client.post(url, data={ + "resource-owner": self.geoapp.owner.id, + "resource-title": "geoapp_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": '[{"key": "value"},{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + }) + expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" + self.assertIn(expected, response.json()['errors'][0]) + + @override_settings(EXTRA_METADATA_SCHEMA={ + "geoapp": { + "name": str, + "slug": str, + "help_text": str, + "field_type": object, + "value": object, + "category": str + } + }) + def test_resource_form_is_valid_extra_metadata(self): + form = self.sut(data={ + "owner": self.geoapp.owner.id, + "title": "geoapp_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + }) + self.assertTrue(form.is_valid()) + \ No newline at end of file diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py index 23b07709bc7..eb576e68fd5 100644 --- a/geonode/geoapps/views.py +++ b/geonode/geoapps/views.py @@ -486,7 +486,17 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T logger.error(tb) return HttpResponse(json.dumps({'message': message})) - + else: + errors_list = {**geoapp_form.errors.as_data(), **category_form.errors.as_data(), **tkeywords_form.errors.as_data()} + logger.error(f"GeoApp Metadata form is not valid: {errors_list}") + out = { + 'success': False, + "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()] + } + return HttpResponse( + json.dumps(out), + content_type='application/json', + status=400) # - POST Request Ends here - # Request.GET diff --git a/geonode/maps/templates/layouts/map_panels.html b/geonode/maps/templates/layouts/map_panels.html index 122ed38fd93..4cf5516befd 100644 --- a/geonode/maps/templates/layouts/map_panels.html +++ b/geonode/maps/templates/layouts/map_panels.html @@ -528,6 +528,12 @@ {{ map_form.spatial_representation_type }}
+ {% block map_extra_metadata %} +
+ + {{ map_form.extra_metadata }} +
+ {% endblock map_extra_metadata %}
diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index b987e7f362b..c5270eae906 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -30,6 +30,8 @@ from django.contrib.auth.models import Group from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType +from geonode.base.populate_test_data import create_single_map +from geonode.maps.forms import MapForm from geonode.maps.models import Map, MapLayer from geonode.settings import on_travis @@ -1098,3 +1100,66 @@ def testMapsNotifications(self): rating=5) rating.save() self.assertTrue(self.check_notification_out('map_rated', self.u)) + + +class TestMapForm(GeoNodeBaseTestSupport): + def setUp(self) -> None: + self.user = get_user_model().objects.get(username='admin') + self.map = create_single_map("single_map", owner=self.user) + self.sut = MapForm + + def test_resource_form_is_invalid_extra_metadata_not_json_format(self): + self.client.login(username="admin", password="admin") + url = reverse("map_metadata", args=(self.map.id,)) + response = self.client.post(url, data={ + "resource-owner": self.map.owner.id, + "resource-title": "map_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "not-a-json" + }) + expected = {"success": False, "errors": ["extra_metadata: The value provided for the Extra metadata field is not a valid JSON"]} + self.assertDictEqual(expected, response.json()) + + @override_settings(EXTRA_METADATA_SCHEMA={"key": "value"}) + def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): + self.client.login(username="admin", password="admin") + url = reverse("map_metadata", args=(self.map.id,)) + response = self.client.post(url, data={ + "resource-owner": self.map.owner.id, + "resource-title": "map_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": "[{'key': 'value'}]" + }) + expected = {"success": False, "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource map"]} + self.assertDictEqual(expected, response.json()) + + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): + self.client.login(username="admin", password="admin") + url = reverse("map_metadata", args=(self.map.id,)) + response = self.client.post(url, data={ + "resource-owner": self.map.owner.id, + "resource-title": "map_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + "resource-extra_metadata": '[{"key": "value"},{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + }) + expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" + self.assertIn(expected, response.json()['errors'][0]) + + + def test_resource_form_is_valid_extra_metadata(self): + form = self.sut(data={ + "owner": self.map.owner.id, + "title": "map_title", + "date": "2022-01-24 16:38 pm", + "date_type": "creation", + "language": "eng", + "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + }) + self.assertTrue(form.is_valid()) + \ No newline at end of file diff --git a/geonode/maps/views.py b/geonode/maps/views.py index 70c5650c2eb..4965d931abe 100644 --- a/geonode/maps/views.py +++ b/geonode/maps/views.py @@ -369,7 +369,17 @@ def map_metadata( map_obj.save(notify=True) return HttpResponse(json.dumps({'message': message})) - + else: + errors_list = {**map_form.errors.as_data(), **category_form.errors.as_data(), **tkeywords_form.errors.as_data()} + logger.error(f"GeoApp Metadata form is not valid: {errors_list}") + out = { + 'success': False, + "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()] + } + return HttpResponse( + json.dumps(out), + content_type='application/json', + status=400) # - POST Request Ends here - # Request.GET From 38de0f0a9d9fd32ddf8fbf57ae778ab2b27917f1 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 27 Jan 2022 11:11:56 +0100 Subject: [PATCH 04/24] [Fixes #8689] Fix flakee8 formatting --- geonode/documents/tests.py | 5 ++--- geonode/geoapps/tests.py | 7 +++---- geonode/layers/tests.py | 6 ++---- geonode/maps/tests.py | 6 ++---- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 128e313fdd0..1ad4b404797 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -22,7 +22,6 @@ when you run "manage.py test". """ -from django.test import override_settings from geonode.tests.base import GeoNodeBaseTestSupport import os @@ -41,7 +40,7 @@ from guardian.shortcuts import get_perms, get_anonymous_user -from .forms import DocumentCreateForm, DocumentForm +from .forms import DocumentCreateForm from geonode.groups.models import ( GroupProfile, @@ -54,7 +53,7 @@ from geonode.documents import DocumentsAppConfig from geonode.documents.forms import DocumentFormMixin from geonode.tests.utils import NotificationsTestsHelper -from geonode.base.populate_test_data import create_models, create_single_doc +from geonode.base.populate_test_data import create_models from geonode.documents.enumerations import DOCUMENT_TYPE_MAP from geonode.documents.models import Document, DocumentResourceLink diff --git a/geonode/geoapps/tests.py b/geonode/geoapps/tests.py index adc3d2181b1..2b9b1bec8b0 100644 --- a/geonode/geoapps/tests.py +++ b/geonode/geoapps/tests.py @@ -36,7 +36,7 @@ def setUp(self) -> None: owner=self.user ) self.sut = GeoAppForm - + def test_update_geoapp_metadata(self): bobby = get_user_model().objects.get(username='bobby') gep_app = GeoApp.objects.create( @@ -89,7 +89,7 @@ def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): }) expected = {"success": False, "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource geoapp"]} self.assertDictEqual(expected, response.json()) - + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): self.client.login(username="admin", password="admin") url = reverse("geoapp_metadata", args=(self.geoapp.id,)) @@ -103,7 +103,7 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): }) expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" self.assertIn(expected, response.json()['errors'][0]) - + @override_settings(EXTRA_METADATA_SCHEMA={ "geoapp": { "name": str, @@ -124,4 +124,3 @@ def test_resource_form_is_valid_extra_metadata(self): "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' }) self.assertTrue(form.is_valid()) - \ No newline at end of file diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index de028589d62..c7c97a5c64b 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -1981,7 +1981,7 @@ def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): }) expected = {"success": False, "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource layer"]} self.assertDictEqual(expected, response.json()) - + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): self.client.login(username="admin", password="admin") url = reverse("layer_metadata", args=(self.layer.alternate,)) @@ -1995,8 +1995,7 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): }) expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" self.assertIn(expected, response.json()['errors'][0]) - - + def test_resource_form_is_valid_extra_metadata(self): form = self.sut(data={ "owner": self.layer.owner.id, @@ -2007,4 +2006,3 @@ def test_resource_form_is_valid_extra_metadata(self): "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' }) self.assertTrue(form.is_valid()) - \ No newline at end of file diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index c5270eae906..8e26ab11eff 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -1136,7 +1136,7 @@ def test_resource_form_is_invalid_extra_metadata_not_schema_in_settings(self): }) expected = {"success": False, "errors": ["extra_metadata: EXTRA_METADATA_SCHEMA validation schema is not available for resource map"]} self.assertDictEqual(expected, response.json()) - + def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): self.client.login(username="admin", password="admin") url = reverse("map_metadata", args=(self.map.id,)) @@ -1150,8 +1150,7 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): }) expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" self.assertIn(expected, response.json()['errors'][0]) - - + def test_resource_form_is_valid_extra_metadata(self): form = self.sut(data={ "owner": self.map.owner.id, @@ -1162,4 +1161,3 @@ def test_resource_form_is_valid_extra_metadata(self): "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' }) self.assertTrue(form.is_valid()) - \ No newline at end of file From b2ba2c68b173acccc8ce2883b3dc1dcbead10bc0 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 27 Jan 2022 11:43:05 +0100 Subject: [PATCH 05/24] [Fixes #8689] Extra metadata json saved with format --- geonode/base/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 897e9ea426e..f0567c2b1af 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -558,7 +558,8 @@ def clean_extra_metadata(self): Schema(extra_metadata_validation_schema).validate(_metadata) except Exception as e: raise forms.ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") - return cleaned_data.get('extra_metadata') + # conerted because in this case, we can store a well formated json instead of the user input + return json.dumps(json.loads(cleaned_data.get('extra_metadata')), indent=4) class Meta: exclude = ( From 318a2f0c447f862cf672f514a390fd6ee5e32434 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 27 Jan 2022 15:17:35 +0100 Subject: [PATCH 06/24] [Fixes #8689] Refactor validation def, start defining endpoint for API --- geonode/base/api/serializers.py | 11 ++++++++ geonode/base/forms.py | 35 ++--------------------- geonode/base/utils.py | 37 +++++++++++++++++++++++- geonode/documents/views.py | 2 ++ geonode/geoapps/views.py | 6 +++- geonode/layers/views.py | 3 ++ geonode/maps/api/views.py | 50 ++++++++++++++++++++++++++++++++- geonode/maps/views.py | 4 ++- 8 files changed, 112 insertions(+), 36 deletions(-) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 581ff8c66f1..ec4dc2209ac 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -16,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +import json from slugify import slugify from urllib.parse import urljoin @@ -262,7 +263,17 @@ def __init__(self, **kwargs): def get_attribute(self, instance): return build_absolute_uri(instance.detail_url) +class ExtraMetadataSerializer(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + return json.loads(instance.extra_metadata) + + def to_representation(self, value): + return json.loads(value) + class ThumbnailUrlField(DynamicComputedField): def __init__(self, **kwargs): diff --git a/geonode/base/forms.py b/geonode/base/forms.py index f0567c2b1af..226aa15be07 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -17,7 +17,6 @@ # ######################################################################### import json -from schema import Schema import re import html import logging @@ -46,6 +45,7 @@ License, Region, ResourceBase, Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel, ThesaurusLabel, TopicCategory) +from geonode.base.utils import validate_extra_metadata from geonode.base.widgets import TaggitSelect2Custom from geonode.documents.models import Document from geonode.layers.models import Layer @@ -529,37 +529,8 @@ def clean_title(self): return title def clean_extra_metadata(self): - cleaned_data = self.cleaned_data - if not cleaned_data.get('extra_metadata', []): - return cleaned_data - - # starting validation of extra metadata passed via JSON - # if schema for metadata validation is not defined, an error is raised - resource_type = ( - self.instance.polymorphic_ctype.model - if self.instance.polymorphic_ctype - else self.instance.class_name.lower() - ) - extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) - if not extra_metadata_validation_schema: - raise forms.ValidationError( - f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}" - ) - # starting json structure validation. The Field can contain multiple metadata - - try: - _extra_as_json = json.loads(cleaned_data.get('extra_metadata')) - except Exception: - raise forms.ValidationError("The value provided for the Extra metadata field is not a valid JSON") - - # looping on all the single metadata provided. If it doen't match the schema an error is raised - for _index, _metadata in enumerate(_extra_as_json): - try: - Schema(extra_metadata_validation_schema).validate(_metadata) - except Exception as e: - raise forms.ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") - # conerted because in this case, we can store a well formated json instead of the user input - return json.dumps(json.loads(cleaned_data.get('extra_metadata')), indent=4) + cleaned_data = self.cleaned_data.get('extra_metadata', []) + return json.dumps(validate_extra_metadata(cleaned_data, self.instance), indent=4) class Meta: exclude = ( diff --git a/geonode/base/utils.py b/geonode/base/utils.py index 11d7f877588..829b335a66d 100644 --- a/geonode/base/utils.py +++ b/geonode/base/utils.py @@ -21,6 +21,7 @@ """ # Standard Modules +import json import re import logging from urllib.parse import urljoin @@ -31,7 +32,7 @@ # Django functionality from django.conf import settings from django.contrib.auth import get_user_model - +from django.core.exceptions import ValidationError # Geonode functionality from guardian.shortcuts import get_perms, remove_perm, assign_perm @@ -41,6 +42,7 @@ get_thumbs, remove_thumb) from geonode.utils import get_legend_url +from schema import Schema logger = logging.getLogger('geonode.base.utils') @@ -207,3 +209,36 @@ def _restore_owner_permissions(self): assign_perm(perm, self.resource.owner, self.resource) elif perm not in {'change_resourcebase_permissions', 'publish_resourcebase'}: assign_perm(perm, self.resource.owner, self.resource) + + +def validate_extra_metadata(data, instance): + if not data: + return data + + # starting validation of extra metadata passed via JSON + # if schema for metadata validation is not defined, an error is raised + resource_type = ( + instance.polymorphic_ctype.model + if instance.polymorphic_ctype + else instance.class_name.lower() + ) + extra_metadata_validation_schema = settings.EXTRA_METADATA_SCHEMA.get(resource_type, None) + if not extra_metadata_validation_schema: + raise ValidationError( + f"EXTRA_METADATA_SCHEMA validation schema is not available for resource {resource_type}" + ) + # starting json structure validation. The Field can contain multiple metadata + try: + if isinstance(data, str): + data = json.loads(data) + except Exception: + raise ValidationError("The value provided for the Extra metadata field is not a valid JSON") + + # looping on all the single metadata provided. If it doen't match the schema an error is raised + for _index, _metadata in enumerate(data): + try: + Schema(extra_metadata_validation_schema).validate(_metadata) + except Exception as e: + raise ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") + # conerted because in this case, we can store a well formated json instead of the user input + return data diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 6b74f7df48a..6b7cbe323bb 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -229,6 +229,8 @@ def form_valid(self, form): if settings.RESOURCE_PUBLISHING: self.object.is_published = False self.object.was_published = False + + self.object.extra_metadata = json.loads(self.object.extra_metadata) self.object.save() form.save_many2many() diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py index eb576e68fd5..6a41c2fe42f 100644 --- a/geonode/geoapps/views.py +++ b/geonode/geoapps/views.py @@ -451,6 +451,9 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T geoapp_obj.regions.clear() geoapp_obj.regions.add(*new_regions) geoapp_obj.category = new_category + + geoapp_obj.extra_metadata = json.loads(geoapp_form.cleaned_data['extra_metadata']) + geoapp_obj.save(notify=True) register_event(request, EventType.EVENT_CHANGE_METADATA, geoapp_obj) @@ -486,7 +489,8 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T logger.error(tb) return HttpResponse(json.dumps({'message': message})) - else: + elif request.method == "POST" and (not geoapp_form.is_valid( + ) or not category_form.is_valid() or not tkeywords_form.is_valid()): errors_list = {**geoapp_form.errors.as_data(), **category_form.errors.as_data(), **tkeywords_form.errors.as_data()} logger.error(f"GeoApp Metadata form is not valid: {errors_list}") out = { diff --git a/geonode/layers/views.py b/geonode/layers/views.py index ffa06751d8b..c780a21bd94 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -1003,6 +1003,7 @@ def layer_metadata( la.visible = form["visible"] la.display_order = form["display_order"] la.featureinfo_type = form["featureinfo_type"] + la.save() if new_poc is not None or new_author is not None: @@ -1022,6 +1023,8 @@ def layer_metadata( layer.regions.add(*new_regions) layer.category = new_category + layer.extra_metadata = json.loads(layer_form.cleaned_data['extra_metadata']) + up_sessions = UploadSession.objects.filter(layer=layer) if up_sessions.count() > 0 and up_sessions[0].user != layer.owner: up_sessions.update(user=layer.owner) diff --git a/geonode/maps/api/views.py b/geonode/maps/api/views.py index dbac2140d43..571ce68bc53 100644 --- a/geonode/maps/api/views.py +++ b/geonode/maps/api/views.py @@ -16,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +from importlib.metadata import metadata from drf_spectacular.utils import extend_schema from dynamic_rest.viewsets import DynamicModelViewSet @@ -28,8 +29,10 @@ from oauth2_provider.contrib.rest_framework import OAuth2Authentication from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.base.api.serializers import ExtraMetadataSerializer +from geonode.base.utils import validate_extra_metadata from geonode.layers.api.serializers import LayerSerializer from geonode.maps.models import Map @@ -70,3 +73,48 @@ def local_layers(self, request, pk=None): map = self.get_object() resources = map.local_layers return Response(LayerSerializer(embed=True, many=True).to_representation(resources)) + + + @extend_schema( + methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + permission_classes=[ + IsOwnerOrAdmin, + ], + url_path=r"extra_metadata", # noqa + url_name="map-extra-metadata", + ) + def extra_metadata(self, request, pk=None): + _obj = self.get_object() + if request.method == "GET": + # get list of available metadata + return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) + + try: + extra_metadata = validate_extra_metadata(request.data, _obj) + except Exception as e: + return Response(status=500, data=e.args[0]) + + if request.method == "PUT": + # update all metadata + _obj.extra_metadata = extra_metadata + _obj.save() + logger.info("metadata updated for the selected resource") + return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) + elif request.method == "DELETE": + # delete single metadata + metadata_to_keep = _obj.extra_metadata.copy() + for _metadata in extra_metadata: + if _metadata in _obj.extra_metadata: + metadata_to_keep.remove(_obj.extra_metadata.index(_metadata)) + _obj.extra_metadata = metadata_to_keep + _obj.save() + return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) + elif request.method == "POST": + # add new metadata + return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata), status=201) + else: + return Response(status="404") diff --git a/geonode/maps/views.py b/geonode/maps/views.py index 4965d931abe..2c069b9d684 100644 --- a/geonode/maps/views.py +++ b/geonode/maps/views.py @@ -333,6 +333,7 @@ def map_metadata( map_obj.regions.clear() map_obj.regions.add(*new_regions) map_obj.category = new_category + map_obj.extra_metadata = json.loads(map_form.cleaned_data['extra_metadata']) register_event(request, EventType.EVENT_CHANGE_METADATA, map_obj) if not ajax: @@ -369,7 +370,8 @@ def map_metadata( map_obj.save(notify=True) return HttpResponse(json.dumps({'message': message})) - else: + elif request.method == "POST" and (not map_form.is_valid( + ) or not category_form.is_valid() or not tkeywords_form.is_valid()): errors_list = {**map_form.errors.as_data(), **category_form.errors.as_data(), **tkeywords_form.errors.as_data()} logger.error(f"GeoApp Metadata form is not valid: {errors_list}") out = { From f1083448edee9d2ea3f8507c8b1c4d8ec1359922 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 27 Jan 2022 17:00:43 +0100 Subject: [PATCH 07/24] [Fixes #8689] Definition of extra-metadata endpoints for resources --- geonode/base/api/serializers.py | 2 -- geonode/base/api/views.py | 33 +++++++++++++++++ geonode/documents/api/tests.py | 62 +++++++++++++++++++++++++++++++- geonode/documents/api/views.py | 19 +++++++++- geonode/geoapps/api/tests.py | 63 +++++++++++++++++++++++++++++++++ geonode/geoapps/api/views.py | 21 ++++++++++- geonode/layers/api/tests.py | 62 +++++++++++++++++++++++++++++++- geonode/layers/api/views.py | 22 ++++++++++-- geonode/maps/api/tests.py | 61 ++++++++++++++++++++++++++++++- geonode/maps/api/views.py | 39 +++----------------- 10 files changed, 340 insertions(+), 44 deletions(-) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index ec4dc2209ac..137945bf317 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -271,8 +271,6 @@ def __init__(self, **kwargs): def get_attribute(self, instance): return json.loads(instance.extra_metadata) - def to_representation(self, value): - return json.loads(value) class ThumbnailUrlField(DynamicComputedField): diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 8b4c3d7778b..935691c0686 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -70,6 +70,8 @@ ThesaurusKeywordSerializer, ) from .pagination import GeoNodeApiPagination +from geonode.base.api.serializers import ExtraMetadataSerializer +from geonode.base.utils import validate_extra_metadata import logging @@ -466,3 +468,34 @@ def set_thumbnail_from_bbox(self, request, resource_id): traceback.print_exc() logger.error(e) return Response(data={"message": e.args[0], "success": False}, status=500, exception=True) + + +def common_extra_metadata_handler(request, _obj): + if request.method == "GET": + # get list of available metadata + return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) + + try: + extra_metadata = validate_extra_metadata(request.data, _obj) + except Exception as e: + return Response(status=500, data=e.args[0]) + + if request.method == "PUT": + # update all metadata + ResourceBase.objects.filter(id=_obj.id).update(extra_metadata=extra_metadata) + logger.info("metadata updated for the selected resource") + return Response(ExtraMetadataSerializer().to_representation(extra_metadata)) + elif request.method == "DELETE": + # delete single metadata + metadata_to_keep = [dict(sorted(x.items())) for x in _obj.extra_metadata] + old_metadata = metadata_to_keep.copy() + for _metadata in extra_metadata: + _m = dict(sorted(_metadata.items())) + if _m in old_metadata: + metadata_to_keep.remove(_m) + ResourceBase.objects.filter(id=_obj.id).update(extra_metadata=metadata_to_keep) + return Response(ExtraMetadataSerializer().to_representation(metadata_to_keep)) + elif request.method == "POST": + # add new metadata + ResourceBase.objects.filter(id=_obj.id).update(extra_metadata=_obj.extra_metadata + extra_metadata) + return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata + extra_metadata), status=201) diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index 7c5e0c7dd33..47d1850637d 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -27,8 +27,9 @@ from geonode.documents.models import Document from geonode import geoserver +from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend -from geonode.base.populate_test_data import create_models +from geonode.base.populate_test_data import create_models, create_single_doc logger = logging.getLogger(__name__) @@ -89,3 +90,62 @@ def test_documents(self): # import json # logger.error(f"{json.dumps(layers_data)}") + + +class TestExtraMetadataUploadApi(GeoNodeBaseTestSupport): + def setUp(self): + self.doc = create_single_doc('document') + self.metadata = { + "name": "metadata-name", + "slug": "metadata-slug", + "help_text": "this is the help text", + "field_type": "str", + "value": "my value", + "category": "cat1" + } + self.doc.extra_metadata = [self.metadata] + self.doc.save() + + def test_get_will_return_the_list_of_extra_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('documents-extra_metadata', args=[self.doc.id]) + response = self.client.get(url, content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([self.metadata], response.json()) + + def test_put_will_update_the_whole_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('documents-extra_metadata', args=[self.doc.id]) + input_metadata = { + "name": "metadata-updated", + "slug": "metadata-slug-updated", + "help_text": "this is the help text-updated", + "field_type": "str-updated", + "value": "my value-updated", + "category": "cat1-updated" + } + response = self.client.put(url, data=[input_metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([input_metadata], response.json()) + + def test_post_will_add_new_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('documents-extra_metadata', args=[self.doc.id]) + input_metadata = { + "name": "metadata-new", + "slug": "metadata-slug-new", + "help_text": "this is the help text-new", + "field_type": "str-new", + "value": "my value-new", + "category": "cat1-new" + } + response = self.client.post(url, data=[input_metadata], content_type='application/json') + self.assertTrue(201, response.status_code) + self.assertEqual(2, len(response.json())) + + def test_delete_will_delete_single_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('documents-extra_metadata', args=[self.doc.id]) + response = self.client.delete(url, data=[self.metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([], response.json()) diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index cd80505faff..3fa53498e4e 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -27,8 +27,9 @@ from oauth2_provider.contrib.rest_framework import OAuth2Authentication from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.base.api.views import common_extra_metadata_handler from geonode.documents.models import Document from geonode.base.models import ResourceBase @@ -74,3 +75,19 @@ def linked_resources(self, request, pk=None): result_page = paginator.paginate_queryset(resources, request) serializer = ResourceBaseSerializer(result_page, embed=True, many=True) return paginator.get_paginated_response({"resources": serializer.data}) + + + @extend_schema( + methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + permission_classes=[ + IsOwnerOrAdmin, + ], + url_path=r"extra_metadata", # noqa + url_name="extra_metadata", + ) + def extra_metadata(self, request, pk=None): + return common_extra_metadata_handler(request, self.get_object()) diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index c86219aa3f2..3db4e3f64ed 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -27,6 +27,7 @@ from geonode.geoapps.models import GeoApp, GeoAppData from geonode import geoserver +from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend from geonode.base.populate_test_data import create_models @@ -187,3 +188,65 @@ def test_geoapps_crud(self): response = self.client.get( f"{url}?include[]=data", format='json') self.assertEqual(response.status_code, 404) # 404 - Not Found + + +class TestExtraMetadataGeoAppApi(GeoNodeBaseTestSupport): + def setUp(self): + self.geo_app = GeoApp.objects.create( + title="Test GeoApp", + owner=get_user_model().objects.get(username='admin') + ) + self.metadata = { + "name": "metadata-name", + "slug": "metadata-slug", + "help_text": "this is the help text", + "field_type": "str", + "value": "my value", + "category": "cat1" + } + self.geo_app.extra_metadata = [self.metadata] + self.geo_app.save() + + def test_get_will_return_the_list_of_extra_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) + response = self.client.get(url, content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([self.metadata], response.json()) + + def test_put_will_update_the_whole_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) + input_metadata = { + "name": "metadata-updated", + "slug": "metadata-slug-updated", + "help_text": "this is the help text-updated", + "field_type": "str-updated", + "value": "my value-updated", + "category": "cat1-updated" + } + response = self.client.put(url, data=[input_metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([input_metadata], response.json()) + + def test_post_will_add_new_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) + input_metadata = { + "name": "metadata-new", + "slug": "metadata-slug-new", + "help_text": "this is the help text-new", + "field_type": "str-new", + "value": "my value-new", + "category": "cat1-new" + } + response = self.client.post(url, data=[input_metadata], content_type='application/json') + self.assertTrue(201, response.status_code) + self.assertEqual(2, len(response.json())) + + def test_delete_will_delete_single_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) + response = self.client.delete(url, data=[self.metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([], response.json()) diff --git a/geonode/geoapps/api/views.py b/geonode/geoapps/api/views.py index aa444b450cf..5d4a350aa47 100644 --- a/geonode/geoapps/api/views.py +++ b/geonode/geoapps/api/views.py @@ -22,9 +22,12 @@ from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.authentication import SessionAuthentication, BasicAuthentication from oauth2_provider.contrib.rest_framework import OAuth2Authentication +from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action +from geonode.base.api.views import common_extra_metadata_handler from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination from geonode.geoapps.models import GeoApp @@ -49,3 +52,19 @@ class GeoAppViewSet(DynamicModelViewSet): queryset = GeoApp.objects.all() serializer_class = GeoAppSerializer pagination_class = GeoNodeApiPagination + + + @extend_schema( + methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + permission_classes=[ + IsOwnerOrAdmin, + ], + url_path=r"extra_metadata", # noqa + url_name="extra-metadata", + ) + def extra_metadata(self, request, pk=None): + return common_extra_metadata_handler(request, self.get_object()) diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py index 08451f3f882..06496581255 100644 --- a/geonode/layers/api/tests.py +++ b/geonode/layers/api/tests.py @@ -24,8 +24,10 @@ from geonode import geoserver from geonode.layers.models import Layer +from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend -from geonode.base.populate_test_data import create_models +from geonode.base.populate_test_data import create_models, create_single_layer +from geonode.geoserver.createlayer.utils import create_layer logger = logging.getLogger(__name__) @@ -96,3 +98,61 @@ def test_raw_HTML_stripped_properties(self): self.assertEqual(response.data['layer']['raw_constraints_other'], "None") self.assertEqual(response.data['layer']['raw_supplemental_information'], "No information provided í £682m") self.assertEqual(response.data['layer']['raw_data_quality_statement'], "OK 1 2 a b") + + +class TestExtraMetadataLayersApi(GeoNodeBaseTestSupport): + def setUp(self): + self.layer = create_single_layer('single_layer') + self.metadata = { + "name": "metadata-name", + "slug": "metadata-slug", + "help_text": "this is the help text", + "field_type": "str", + "value": "my value", + "category": "cat1" + } + Layer.objects.filter(id=self.layer.id).update(extra_metadata=[self.metadata]) + + def test_get_will_return_the_list_of_extra_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('layers-extra-metadata', args=[self.layer.id]) + response = self.client.get(url, content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([self.metadata], response.json()) + + def test_put_will_update_the_whole_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('layers-extra-metadata', args=[self.layer.id]) + input_metadata = { + "name": "metadata-updated", + "slug": "metadata-slug-updated", + "help_text": "this is the help text-updated", + "field_type": "str-updated", + "value": "my value-updated", + "category": "cat1-updated" + } + response = self.client.put(url, data=[input_metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([input_metadata], response.json()) + + def test_post_will_add_new_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('layers-extra-metadata', args=[self.layer.id]) + input_metadata = { + "name": "metadata-new", + "slug": "metadata-slug-new", + "help_text": "this is the help text-new", + "field_type": "str-new", + "value": "my value-new", + "category": "cat1-new" + } + response = self.client.post(url, data=[input_metadata], content_type='application/json') + self.assertTrue(201, response.status_code) + self.assertEqual(2, len(response.json())) + + def test_delete_will_delete_single_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('layers-extra-metadata', args=[self.layer.id]) + response = self.client.delete(url, data=[self.metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([], response.json()) diff --git a/geonode/layers/api/views.py b/geonode/layers/api/views.py index 9cf70249387..23406ab052e 100644 --- a/geonode/layers/api/views.py +++ b/geonode/layers/api/views.py @@ -22,9 +22,11 @@ from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.authentication import SessionAuthentication, BasicAuthentication from oauth2_provider.contrib.rest_framework import OAuth2Authentication - +from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action +from geonode.base.api.views import common_extra_metadata_handler from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination from geonode.layers.models import Layer @@ -49,3 +51,19 @@ class LayerViewSet(DynamicModelViewSet): queryset = Layer.objects.all() serializer_class = LayerSerializer pagination_class = GeoNodeApiPagination + + + @extend_schema( + methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + permission_classes=[ + IsOwnerOrAdmin, + ], + url_path=r"extra_metadata", # noqa + url_name="extra-metadata", + ) + def extra_metadata(self, request, pk=None): + return common_extra_metadata_handler(request, self.get_object()) diff --git a/geonode/maps/api/tests.py b/geonode/maps/api/tests.py index fe655a8c3f6..668528ddbb2 100644 --- a/geonode/maps/api/tests.py +++ b/geonode/maps/api/tests.py @@ -28,8 +28,9 @@ from geonode.maps.models import Map from geonode import geoserver +from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend -from geonode.base.populate_test_data import create_models +from geonode.base.populate_test_data import create_models, create_single_map logger = logging.getLogger(__name__) @@ -117,3 +118,61 @@ def test_maps(self): self.assertTrue(len(response.data) > 0) self.assertTrue('data' in response.data) self.assertTrue('attributes' in response.data) + + +class TestExtraMetadataMapsApi(GeoNodeBaseTestSupport): + def setUp(self): + self.map = create_single_map('single_map') + self.metadata = { + "name": "metadata-name", + "slug": "metadata-slug", + "help_text": "this is the help text", + "field_type": "str", + "value": "my value", + "category": "cat1" + } + Map.objects.filter(id=self.map.id).update(extra_metadata=[self.metadata]) + + def test_get_will_return_the_list_of_extra_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('maps-extra-metadata', args=[self.map.id]) + response = self.client.get(url, content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([self.metadata], response.json()) + + def test_put_will_update_the_whole_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('maps-extra-metadata', args=[self.map.id]) + input_metadata = { + "name": "metadata-updated", + "slug": "metadata-slug-updated", + "help_text": "this is the help text-updated", + "field_type": "str-updated", + "value": "my value-updated", + "category": "cat1-updated" + } + response = self.client.put(url, data=[input_metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([input_metadata], response.json()) + + def test_post_will_add_new_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('maps-extra-metadata', args=[self.map.id]) + input_metadata = { + "name": "metadata-new", + "slug": "metadata-slug-new", + "help_text": "this is the help text-new", + "field_type": "str-new", + "value": "my value-new", + "category": "cat1-new" + } + response = self.client.post(url, data=[input_metadata], content_type='application/json') + self.assertTrue(201, response.status_code) + self.assertEqual(2, len(response.json())) + + def test_delete_will_delete_single_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('maps-extra-metadata', args=[self.map.id]) + response = self.client.delete(url, data=[self.metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([], response.json()) diff --git a/geonode/maps/api/views.py b/geonode/maps/api/views.py index 571ce68bc53..3f794c1567b 100644 --- a/geonode/maps/api/views.py +++ b/geonode/maps/api/views.py @@ -16,7 +16,6 @@ # along with this program. If not, see . # ######################################################################### -from importlib.metadata import metadata from drf_spectacular.utils import extend_schema from dynamic_rest.viewsets import DynamicModelViewSet @@ -31,8 +30,7 @@ from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination -from geonode.base.api.serializers import ExtraMetadataSerializer -from geonode.base.utils import validate_extra_metadata +from geonode.base.api.views import common_extra_metadata_handler from geonode.layers.api.serializers import LayerSerializer from geonode.maps.models import Map @@ -74,7 +72,7 @@ def local_layers(self, request, pk=None): resources = map.local_layers return Response(LayerSerializer(embed=True, many=True).to_representation(resources)) - + @extend_schema( methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" ) @@ -85,36 +83,7 @@ def local_layers(self, request, pk=None): IsOwnerOrAdmin, ], url_path=r"extra_metadata", # noqa - url_name="map-extra-metadata", + url_name="extra-metadata", ) def extra_metadata(self, request, pk=None): - _obj = self.get_object() - if request.method == "GET": - # get list of available metadata - return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) - - try: - extra_metadata = validate_extra_metadata(request.data, _obj) - except Exception as e: - return Response(status=500, data=e.args[0]) - - if request.method == "PUT": - # update all metadata - _obj.extra_metadata = extra_metadata - _obj.save() - logger.info("metadata updated for the selected resource") - return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) - elif request.method == "DELETE": - # delete single metadata - metadata_to_keep = _obj.extra_metadata.copy() - for _metadata in extra_metadata: - if _metadata in _obj.extra_metadata: - metadata_to_keep.remove(_obj.extra_metadata.index(_metadata)) - _obj.extra_metadata = metadata_to_keep - _obj.save() - return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) - elif request.method == "POST": - # add new metadata - return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata), status=201) - else: - return Response(status="404") + return common_extra_metadata_handler(request, self.get_object()) From e20c65613dc4f398e101bc96ac3e10237b6568f7 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Fri, 28 Jan 2022 12:36:32 +0100 Subject: [PATCH 08/24] [Fixes #8689] Converting metadata from jsonfield to manytomany relation --- .../migrations/0063_auto_20220128_1042.py | 27 +++++++++++++++++++ .../migrations/0064_resourcebase_metadata.py | 18 +++++++++++++ geonode/base/models.py | 16 ++++++++++- geonode/geoapps/views.py | 21 ++++++++++----- geonode/layers/api/views.py | 2 +- geonode/layers/views.py | 3 ++- geonode/settings.py | 3 ++- 7 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 geonode/base/migrations/0063_auto_20220128_1042.py create mode 100644 geonode/base/migrations/0064_resourcebase_metadata.py diff --git a/geonode/base/migrations/0063_auto_20220128_1042.py b/geonode/base/migrations/0063_auto_20220128_1042.py new file mode 100644 index 00000000000..ee53e668f85 --- /dev/null +++ b/geonode/base/migrations/0063_auto_20220128_1042.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.24 on 2022-01-28 10:42 + +from django.db import migrations, models +import django.db.models.deletion +import django_jsonfield_backport.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0062_resourcebase_extra_metadata'), + ] + + operations = [ + migrations.RemoveField( + model_name='resourcebase', + name='extra_metadata', + ), + migrations.CreateModel( + name='ExtraMetadata', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('metadata', django_jsonfield_backport.models.JSONField(blank=True, default=dict, null=True)), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase')), + ], + ), + ] diff --git a/geonode/base/migrations/0064_resourcebase_metadata.py b/geonode/base/migrations/0064_resourcebase_metadata.py new file mode 100644 index 00000000000..5e12788dd25 --- /dev/null +++ b/geonode/base/migrations/0064_resourcebase_metadata.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2022-01-28 10:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0063_auto_20220128_1042'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='metadata', + field=models.ManyToManyField(blank=True, null=True, to='base.ExtraMetadata', verbose_name='Extra Metadata'), + ), + ] diff --git a/geonode/base/models.py b/geonode/base/models.py index 21f80e6bafe..23dad309db5 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -53,6 +53,7 @@ from polymorphic.models import PolymorphicModel from polymorphic.managers import PolymorphicManager from pinax.ratings.models import OverallRating +from sqlalchemy import JSON from taggit.models import TagBase, ItemBase from taggit.managers import TaggableManager, _TaggableManager @@ -962,7 +963,11 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): default=False, help_text=_('If true, will be excluded from search')) - extra_metadata = JSONField(null=True, default=list, blank=True) + metadata = models.ManyToManyField( + "ExtraMetadata", + verbose_name=_('Extra Metadata'), + null=True, + blank=True) objects = ResourceBaseManager() @@ -2088,3 +2093,12 @@ def rating_post_save(instance, *args, **kwargs): signals.post_save.connect(rating_post_save, sender=OverallRating) + + +class ExtraMetadata(models.Model): + resource = models.ForeignKey( + ResourceBase, + null=False, + blank=False, + on_delete=models.CASCADE) + metadata = JSONField(null=True, default=dict, blank=True) diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py index 6a41c2fe42f..d268fc7fdd8 100644 --- a/geonode/geoapps/views.py +++ b/geonode/geoapps/views.py @@ -44,10 +44,7 @@ from geonode.people.forms import ProfileForm from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm -from geonode.base.models import ( - Thesaurus, - TopicCategory -) +from geonode.base.models import ExtraMetadata, Thesaurus, TopicCategory from geonode.utils import ( resolve_object, @@ -350,6 +347,9 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T else: geoapp_form = GeoAppForm(instance=geoapp_obj, prefix="resource") + if geoapp_obj.metadata.exists(): + geoapp_form.fields['extra_metadata'].initial = [json.dumps(x.metadata, indent=4) for x in geoapp_obj.metadata.all()] + geoapp_form.disable_keywords_widget_for_non_superuser(request.user) category_form = CategoryForm( prefix="category_choice_field", @@ -452,10 +452,17 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T geoapp_obj.regions.add(*new_regions) geoapp_obj.category = new_category - geoapp_obj.extra_metadata = json.loads(geoapp_form.cleaned_data['extra_metadata']) - geoapp_obj.save(notify=True) - + # clearing old metadata from the resource + geoapp_obj.metadata.clear() + # creating new metadata for the resource + for _m in json.loads(geoapp_form.cleaned_data['extra_metadata']): + new_m = ExtraMetadata.objects.create( + resource=geoapp_obj, + metadata=_m + ) + geoapp_obj.metadata.add(new_m) + register_event(request, EventType.EVENT_CHANGE_METADATA, geoapp_obj) if not ajax: return HttpResponseRedirect( diff --git a/geonode/layers/api/views.py b/geonode/layers/api/views.py index 23406ab052e..f0555160f08 100644 --- a/geonode/layers/api/views.py +++ b/geonode/layers/api/views.py @@ -66,4 +66,4 @@ class LayerViewSet(DynamicModelViewSet): url_name="extra-metadata", ) def extra_metadata(self, request, pk=None): - return common_extra_metadata_handler(request, self.get_object()) + return common_extra_metadata_handler(request, self.get_object(), self.queryset) diff --git a/geonode/layers/views.py b/geonode/layers/views.py index c780a21bd94..17fb33fa86f 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -1023,7 +1023,8 @@ def layer_metadata( layer.regions.add(*new_regions) layer.category = new_category - layer.extra_metadata = json.loads(layer_form.cleaned_data['extra_metadata']) + for _m in json.loads(layer_form.cleaned_data['extra_metadata']): + layer.metadata.add(**_m) up_sessions = UploadSession.objects.filter(layer=layer) if up_sessions.count() > 0 and up_sessions[0].user != layer.owner: diff --git a/geonode/settings.py b/geonode/settings.py index 0c7e60e2fd0..f6c306c008a 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2192,5 +2192,6 @@ def get_geonode_catalogue_service(): "map": os.getenv('MAP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), "layer": os.getenv('DATASET_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), "document": os.getenv('DOCUMENT_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), - "geoapp": os.getenv('GEOAPP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA) + "geoapp": os.getenv('GEOAPP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), + "dashboard": os.getenv('GEOAPP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA) }, **CUSTOM_METADATA_SCHEMA} From 0ba2ebd6b9eacfc331720ec913e39fcb1581215c Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Fri, 28 Jan 2022 16:23:16 +0100 Subject: [PATCH 09/24] [Fixes #8689] Fix views with new relation and prettify json on UI --- geonode/base/forms.py | 13 ++++++++++-- geonode/base/models.py | 5 ++++- geonode/documents/views.py | 27 ++++++++++++++++++++++--- geonode/geoapps/views.py | 4 +--- geonode/layers/views.py | 10 ++++++++- geonode/maps/views.py | 12 ++++++++++- geonode/templates/metadata_form_js.html | 6 ++++++ 7 files changed, 66 insertions(+), 11 deletions(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 226aa15be07..737288cc73c 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -29,7 +29,7 @@ from django.contrib.auth.models import Group from django.core import validators from django.db.models import Prefetch, Q -from django.forms import ModelForm, models +from django.forms import ModelForm, model_to_dict, models from django.forms.fields import ChoiceField, MultipleChoiceField from django.forms.utils import flatatt from django.utils.encoding import force_text @@ -479,10 +479,19 @@ class ResourceBaseForm(TranslationModelForm): regions.widget.attrs = {"size": 20} - extra_metadata = forms.CharField(required=False, widget=forms.Textarea) + extra_metadata = forms.CharField( + required=False, + widget=forms.Textarea, + help_text=_('Additional metadata, must be in format [\ + {"metadata_key": "metadata_value"},\ + {"metadata_key": "metadata_value"} \ + ]') +) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + if self.instance and self.instance.metadata.exists(): + self.fields['extra_metadata'].initial = [x.metadata for x in self.instance.metadata.all()] for field in self.fields: help_text = self.fields[field].help_text if help_text != '': diff --git a/geonode/base/models.py b/geonode/base/models.py index 23dad309db5..99a4d475af1 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -703,6 +703,8 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): data_quality_statement_help_text = _( 'general explanation of the data producer\'s knowledge about the lineage of a' ' dataset') + extra_metadata_help_text = _( + 'Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]') # internal fields uuid = models.CharField(max_length=36) title = models.CharField(_('title'), max_length=255, help_text=_( @@ -967,7 +969,8 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): "ExtraMetadata", verbose_name=_('Extra Metadata'), null=True, - blank=True) + blank=True, + help_text=extra_metadata_help_text) objects = ResourceBaseManager() diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 6b7cbe323bb..2f7cfa2d89e 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -45,6 +45,7 @@ from geonode.base.bbox_utils import BBOXHelper from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm from geonode.base.models import ( + ExtraMetadata, Thesaurus, TopicCategory) from geonode.documents.enumerations import DOCUMENT_TYPE_MAP, DOCUMENT_MIMETYPE_MAP @@ -229,8 +230,6 @@ def form_valid(self, form): if settings.RESOURCE_PUBLISHING: self.object.is_published = False self.object.was_published = False - - self.object.extra_metadata = json.loads(self.object.extra_metadata) self.object.save() form.save_many2many() @@ -465,6 +464,17 @@ def document_metadata( document.regions.clear() document.regions.add(*new_regions) document.category = new_category + + # deleting old metadata from the resource + document.metadata.all().delete() + # creating new metadata for the resource + for _m in json.loads(document_form.cleaned_data['extra_metadata']): + new_m = ExtraMetadata.objects.create( + resource=document, + metadata=_m + ) + document.metadata.add(new_m) + document.save(notify=True) document_form.save_many2many() @@ -500,7 +510,18 @@ def document_metadata( logger.error(tb) return HttpResponse(json.dumps({'message': message})) - + elif request.method == "POST" and (not document_form.is_valid( + ) or not category_form.is_valid() or not tkeywords_form.is_valid()): + errors_list = {**document_form.errors.as_data(), **category_form.errors.as_data(), **tkeywords_form.errors.as_data()} + logger.error(f"GeoApp Metadata form is not valid: {errors_list}") + out = { + 'success': False, + "errors": [f"{x}: {y[0].messages[0]}" for x, y in errors_list.items()] + } + return HttpResponse( + json.dumps(out), + content_type='application/json', + status=400) # - POST Request Ends here - # Request.GET diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py index d268fc7fdd8..467f0f41dbc 100644 --- a/geonode/geoapps/views.py +++ b/geonode/geoapps/views.py @@ -347,8 +347,6 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T else: geoapp_form = GeoAppForm(instance=geoapp_obj, prefix="resource") - if geoapp_obj.metadata.exists(): - geoapp_form.fields['extra_metadata'].initial = [json.dumps(x.metadata, indent=4) for x in geoapp_obj.metadata.all()] geoapp_form.disable_keywords_widget_for_non_superuser(request.user) category_form = CategoryForm( @@ -454,7 +452,7 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T geoapp_obj.save(notify=True) # clearing old metadata from the resource - geoapp_obj.metadata.clear() + geoapp_obj.metadata.all().delete() # creating new metadata for the resource for _m in json.loads(geoapp_form.cleaned_data['extra_metadata']): new_m = ExtraMetadata.objects.create( diff --git a/geonode/layers/views.py b/geonode/layers/views.py index 17fb33fa86f..2dd9177e891 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -60,6 +60,7 @@ from geonode.base.forms import CategoryForm, TKeywordForm, BatchPermissionsForm, ThesaurusAvailableForm from geonode.base.views import batch_modify, get_url_for_model from geonode.base.models import ( + ExtraMetadata, Thesaurus, TopicCategory) from geonode.base.enumerations import CHARSETS @@ -1023,8 +1024,15 @@ def layer_metadata( layer.regions.add(*new_regions) layer.category = new_category + # clearing old metadata from the resource + layer.metadata.all().delete() + # creating new metadata for the resource for _m in json.loads(layer_form.cleaned_data['extra_metadata']): - layer.metadata.add(**_m) + new_m = ExtraMetadata.objects.create( + resource=layer, + metadata=_m + ) + layer.metadata.add(new_m) up_sessions = UploadSession.objects.filter(layer=layer) if up_sessions.count() > 0 and up_sessions[0].user != layer.owner: diff --git a/geonode/maps/views.py b/geonode/maps/views.py index 2c069b9d684..9edcbef4520 100644 --- a/geonode/maps/views.py +++ b/geonode/maps/views.py @@ -59,6 +59,7 @@ from geonode.security.views import _perms_info_json from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm from geonode.base.models import ( + ExtraMetadata, Thesaurus, TopicCategory) from geonode import geoserver @@ -333,7 +334,16 @@ def map_metadata( map_obj.regions.clear() map_obj.regions.add(*new_regions) map_obj.category = new_category - map_obj.extra_metadata = json.loads(map_form.cleaned_data['extra_metadata']) + + # clearing old metadata from the resource + map_obj.metadata.all().delete() + # creating new metadata for the resource + for _m in json.loads(map_form.cleaned_data['extra_metadata']): + new_m = ExtraMetadata.objects.create( + resource=map_obj, + metadata=_m + ) + map_obj.metadata.add(new_m) register_event(request, EventType.EVENT_CHANGE_METADATA, map_obj) if not ajax: diff --git a/geonode/templates/metadata_form_js.html b/geonode/templates/metadata_form_js.html index 5e66f174d78..b4557a2c45b 100644 --- a/geonode/templates/metadata_form_js.html +++ b/geonode/templates/metadata_form_js.html @@ -16,6 +16,12 @@ $('#poc_form').modal(); } }); + var _area = $("#id_resource-extra_metadata").val() + if (_area) { + var areaAsObj = (0, eval)('(' + _area + ')'); + var _value = JSON.stringify(areaAsObj,null,2); + $("#id_resource-extra_metadata").val(_value); + } var metadata_uri = $(location).attr('pathname').split('/').pop(); var metadata_update_done = false; From 19319a256b2f72fa4d1ca7849458a051d4430918 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Mon, 31 Jan 2022 11:17:59 +0100 Subject: [PATCH 10/24] [Fixes #8689] Fix serializer --- geonode/base/api/serializers.py | 34 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 137945bf317..f5368a045a2 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -35,17 +35,9 @@ from avatar.templatetags.avatar_tags import avatar_url from geonode.favorite.models import Favorite -from geonode.base.models import ( - ResourceBase, - HierarchicalKeyword, - Region, - RestrictionCodeType, - License, - TopicCategory, - SpatialRepresentationType, - ThesaurusKeyword, - ThesaurusKeywordLabel -) +from geonode.base.models import (ExtraMetadata, HierarchicalKeyword, License, Region, ResourceBase, + RestrictionCodeType, SpatialRepresentationType, ThesaurusKeyword, ThesaurusKeywordLabel, + TopicCategory) from geonode.groups.models import ( GroupCategory, GroupProfile) @@ -199,7 +191,6 @@ class Meta: name = 'Region' fields = ('code', 'name') - class SimpleTopicCategorySerializer(DynamicModelSerializer): class Meta: @@ -263,15 +254,18 @@ def __init__(self, **kwargs): def get_attribute(self, instance): return build_absolute_uri(instance.detail_url) -class ExtraMetadataSerializer(DynamicComputedField): - - def __init__(self, **kwargs): - super().__init__(**kwargs) - def get_attribute(self, instance): - return json.loads(instance.extra_metadata) +class ExtraMetadataSerializer(DynamicModelSerializer): + class Meta: + model = ExtraMetadata + name = 'ExtraMetadata' + fields = ('metadata',) + def to_representation(self, obj): + return obj.metadata + + class ThumbnailUrlField(DynamicComputedField): def __init__(self, **kwargs): @@ -389,6 +383,8 @@ def __init__(self, *args, **kwargs): LicenseSerializer, embed=True, many=False) self.fields['spatial_representation_type'] = DynamicRelationField( SpatialRepresentationTypeSerializer, embed=True, many=False) + self.fields['metadata'] = DynamicRelationField( + ExtraMetadataSerializer, embed=False, many=True) class Meta: model = ResourceBase @@ -406,7 +402,7 @@ class Meta: 'popular_count', 'share_count', 'rating', 'featured', 'is_published', 'is_approved', 'detail_url', 'embed_url', 'created', 'last_updated', 'raw_abstract', 'raw_purpose', 'raw_constraints_other', - 'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed' + 'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed', "metadata" # TODO # csw_typename, csw_schema, csw_mdsource, csw_insert_date, csw_type, csw_anytext, csw_wkt_geometry, # metadata_uploaded, metadata_uploaded_preserve, metadata_xml, From f06c954ccf3ada31e29ae2f167fac825cbf8e6b4 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 1 Feb 2022 13:41:37 +0100 Subject: [PATCH 11/24] [Fixes #8689] Fix custom metadata endpoint, update metadata schema --- geonode/base/api/serializers.py | 22 ++++++---- geonode/base/api/views.py | 75 ++++++++++++++++++++++++--------- geonode/settings.py | 3 ++ 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index f5368a045a2..68e94ec6e1e 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -16,7 +16,7 @@ # along with this program. If not, see . # ######################################################################### -import json +from django.db.models.query import QuerySet from slugify import slugify from urllib.parse import urljoin @@ -256,15 +256,21 @@ def get_attribute(self, instance): class ExtraMetadataSerializer(DynamicModelSerializer): - class Meta: model = ExtraMetadata name = 'ExtraMetadata' - fields = ('metadata',) - - def to_representation(self, obj): - return obj.metadata + fields = ('pk', 'metadata') + def to_representation(self, obj): + + if isinstance(obj, QuerySet): + out = [] + for el in obj: + out.append({**{"id": el.id}, **el.metadata}) + return out + elif isinstance(obj, list): + return obj + return {**{"id": obj.id}, **obj.metadata} class ThumbnailUrlField(DynamicComputedField): @@ -383,8 +389,8 @@ def __init__(self, *args, **kwargs): LicenseSerializer, embed=True, many=False) self.fields['spatial_representation_type'] = DynamicRelationField( SpatialRepresentationTypeSerializer, embed=True, many=False) - self.fields['metadata'] = DynamicRelationField( - ExtraMetadataSerializer, embed=False, many=True) + + metadata = DynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True) class Meta: model = ResourceBase diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 5138a690d12..67ac5583b6e 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -38,7 +38,7 @@ from geonode.favorite.models import Favorite from geonode.thumbs.exceptions import ThumbnailError from geonode.thumbs.thumbnails import create_thumbnail -from geonode.base.models import HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword +from geonode.base.models import ExtraMetadata, HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter, FavoriteFilter from geonode.groups.models import GroupProfile, GroupMember from geonode.layers.models import Layer @@ -473,29 +473,66 @@ def set_thumbnail_from_bbox(self, request, resource_id): def common_extra_metadata_handler(request, _obj): if request.method == "GET": # get list of available metadata - return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata)) - - try: - extra_metadata = validate_extra_metadata(request.data, _obj) - except Exception as e: - return Response(status=500, data=e.args[0]) + queryset = _obj.metadata.all() + _filters = [{f"metadata__{key}": value} for key,value in request.query_params.items()] + if _filters: + queryset = queryset.filter(**_filters[0]) + return Response(ExtraMetadataSerializer().to_representation(queryset)) + if not request.method == "DELETE": + try: + extra_metadata = validate_extra_metadata(request.data, _obj) + except Exception as e: + return Response(status=500, data=e.args[0]) if request.method == "PUT": - # update all metadata - ResourceBase.objects.filter(id=_obj.id).update(extra_metadata=extra_metadata) + ''' + update specific metadata. The ID of the metadata is required to perform the update + [ + { + "id": 1, + "name": "foo_name", + "slug": "foo_sug", + "help_text": "object", + "field_type": "int", + "value": "object", + "category": "object" + } + ] + ''' + for _m in extra_metadata: + _id = _m.pop('id') + ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) logger.info("metadata updated for the selected resource") return Response(ExtraMetadataSerializer().to_representation(extra_metadata)) elif request.method == "DELETE": # delete single metadata - metadata_to_keep = [dict(sorted(x.items())) for x in _obj.extra_metadata] - old_metadata = metadata_to_keep.copy() - for _metadata in extra_metadata: - _m = dict(sorted(_metadata.items())) - if _m in old_metadata: - metadata_to_keep.remove(_m) - ResourceBase.objects.filter(id=_obj.id).update(extra_metadata=metadata_to_keep) - return Response(ExtraMetadataSerializer().to_representation(metadata_to_keep)) + ''' + Expect a payload with the IDs of the metadata that should be deleted. Payload be like: + [4, 3] + ''' + ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete() + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) elif request.method == "POST": # add new metadata - ResourceBase.objects.filter(id=_obj.id).update(extra_metadata=_obj.extra_metadata + extra_metadata) - return Response(ExtraMetadataSerializer().to_representation(_obj.extra_metadata + extra_metadata), status=201) + ''' + [ + { + "name": "foo_name", + "slug": "foo_sug", + "help_text": "object", + "field_type": "int", + "value": "object", + "category": "object" + } + ] + ''' + for _m in extra_metadata: + new_m = ExtraMetadata.objects.create( + resource=_obj, + metadata=_m + ) + new_m.save() + _obj.metadata.add(new_m) + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201) diff --git a/geonode/settings.py b/geonode/settings.py index f6c306c008a..32fb1238d85 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2167,7 +2167,10 @@ def get_geonode_catalogue_service(): ''' Default schema used to store extra and dynamic metadata for the resource ''' +from schema import Optional + DEFAULT_EXTRA_METADATA_SCHEMA = { + Optional("id"): int, "name": str, "slug": str, "help_text": str, From 43c77dfa13a79c7772da580138dd9a44c5a095eb Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 1 Feb 2022 13:51:42 +0100 Subject: [PATCH 12/24] [Fixes #8689] Fix flake8 issues --- geonode/base/api/serializers.py | 21 ++++++++++++++++----- geonode/base/api/views.py | 2 +- geonode/base/forms.py | 2 +- geonode/base/models.py | 1 - geonode/documents/api/tests.py | 4 ++-- geonode/documents/api/views.py | 1 - geonode/geoapps/api/tests.py | 4 ++-- geonode/geoapps/api/views.py | 1 - geonode/geoapps/views.py | 2 +- geonode/layers/api/tests.py | 3 +-- geonode/layers/api/views.py | 1 - geonode/layers/views.py | 2 +- geonode/maps/api/tests.py | 2 +- geonode/maps/api/views.py | 1 - geonode/settings.py | 2 +- 15 files changed, 27 insertions(+), 22 deletions(-) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 68e94ec6e1e..4ed9121a9a9 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -35,9 +35,18 @@ from avatar.templatetags.avatar_tags import avatar_url from geonode.favorite.models import Favorite -from geonode.base.models import (ExtraMetadata, HierarchicalKeyword, License, Region, ResourceBase, - RestrictionCodeType, SpatialRepresentationType, ThesaurusKeyword, ThesaurusKeywordLabel, - TopicCategory) +from geonode.base.models import ( + ExtraMetadata, + HierarchicalKeyword, + License, + Region, + ResourceBase, + RestrictionCodeType, + SpatialRepresentationType, + ThesaurusKeyword, + ThesaurusKeywordLabel, + TopicCategory, +) from geonode.groups.models import ( GroupCategory, GroupProfile) @@ -191,6 +200,7 @@ class Meta: name = 'Region' fields = ('code', 'name') + class SimpleTopicCategorySerializer(DynamicModelSerializer): class Meta: @@ -262,7 +272,7 @@ class Meta: fields = ('pk', 'metadata') def to_representation(self, obj): - + if isinstance(obj, QuerySet): out = [] for el in obj: @@ -272,6 +282,7 @@ def to_representation(self, obj): return obj return {**{"id": obj.id}, **obj.metadata} + class ThumbnailUrlField(DynamicComputedField): def __init__(self, **kwargs): @@ -389,7 +400,7 @@ def __init__(self, *args, **kwargs): LicenseSerializer, embed=True, many=False) self.fields['spatial_representation_type'] = DynamicRelationField( SpatialRepresentationTypeSerializer, embed=True, many=False) - + metadata = DynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True) class Meta: diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 67ac5583b6e..a279d21b08a 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -474,7 +474,7 @@ def common_extra_metadata_handler(request, _obj): if request.method == "GET": # get list of available metadata queryset = _obj.metadata.all() - _filters = [{f"metadata__{key}": value} for key,value in request.query_params.items()] + _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()] if _filters: queryset = queryset.filter(**_filters[0]) return Response(ExtraMetadataSerializer().to_representation(queryset)) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 737288cc73c..51aae7d78b7 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -29,7 +29,7 @@ from django.contrib.auth.models import Group from django.core import validators from django.db.models import Prefetch, Q -from django.forms import ModelForm, model_to_dict, models +from django.forms import ModelForm, models from django.forms.fields import ChoiceField, MultipleChoiceField from django.forms.utils import flatatt from django.utils.encoding import force_text diff --git a/geonode/base/models.py b/geonode/base/models.py index 99a4d475af1..1e80b419ef3 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -53,7 +53,6 @@ from polymorphic.models import PolymorphicModel from polymorphic.managers import PolymorphicManager from pinax.ratings.models import OverallRating -from sqlalchemy import JSON from taggit.models import TagBase, ItemBase from taggit.managers import TaggableManager, _TaggableManager diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index 47d1850637d..f239014e018 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -102,10 +102,10 @@ def setUp(self): "field_type": "str", "value": "my value", "category": "cat1" - } + } self.doc.extra_metadata = [self.metadata] self.doc.save() - + def test_get_will_return_the_list_of_extra_metadata(self): self.client.login(username="admin", password="admin") url = reverse('documents-extra_metadata', args=[self.doc.id]) diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index 3fa53498e4e..a56d09e3dc9 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -76,7 +76,6 @@ def linked_resources(self, request, pk=None): serializer = ResourceBaseSerializer(result_page, embed=True, many=True) return paginator.get_paginated_response({"resources": serializer.data}) - @extend_schema( methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" ) diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index 3db4e3f64ed..6ee4000b4a5 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -203,10 +203,10 @@ def setUp(self): "field_type": "str", "value": "my value", "category": "cat1" - } + } self.geo_app.extra_metadata = [self.metadata] self.geo_app.save() - + def test_get_will_return_the_list_of_extra_metadata(self): self.client.login(username="admin", password="admin") url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) diff --git a/geonode/geoapps/api/views.py b/geonode/geoapps/api/views.py index 5d4a350aa47..00d64fc48af 100644 --- a/geonode/geoapps/api/views.py +++ b/geonode/geoapps/api/views.py @@ -53,7 +53,6 @@ class GeoAppViewSet(DynamicModelViewSet): serializer_class = GeoAppSerializer pagination_class = GeoNodeApiPagination - @extend_schema( methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" ) diff --git a/geonode/geoapps/views.py b/geonode/geoapps/views.py index 467f0f41dbc..95c0809a733 100644 --- a/geonode/geoapps/views.py +++ b/geonode/geoapps/views.py @@ -460,7 +460,7 @@ def geoapp_metadata(request, geoappid, template='apps/app_metadata.html', ajax=T metadata=_m ) geoapp_obj.metadata.add(new_m) - + register_event(request, EventType.EVENT_CHANGE_METADATA, geoapp_obj) if not ajax: return HttpResponseRedirect( diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py index 06496581255..3b0375d7f9e 100644 --- a/geonode/layers/api/tests.py +++ b/geonode/layers/api/tests.py @@ -27,7 +27,6 @@ from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend from geonode.base.populate_test_data import create_models, create_single_layer -from geonode.geoserver.createlayer.utils import create_layer logger = logging.getLogger(__name__) @@ -112,7 +111,7 @@ def setUp(self): "category": "cat1" } Layer.objects.filter(id=self.layer.id).update(extra_metadata=[self.metadata]) - + def test_get_will_return_the_list_of_extra_metadata(self): self.client.login(username="admin", password="admin") url = reverse('layers-extra-metadata', args=[self.layer.id]) diff --git a/geonode/layers/api/views.py b/geonode/layers/api/views.py index f0555160f08..17d71291d81 100644 --- a/geonode/layers/api/views.py +++ b/geonode/layers/api/views.py @@ -52,7 +52,6 @@ class LayerViewSet(DynamicModelViewSet): serializer_class = LayerSerializer pagination_class = GeoNodeApiPagination - @extend_schema( methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" ) diff --git a/geonode/layers/views.py b/geonode/layers/views.py index 2dd9177e891..598e73feb7d 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -1004,7 +1004,7 @@ def layer_metadata( la.visible = form["visible"] la.display_order = form["display_order"] la.featureinfo_type = form["featureinfo_type"] - + la.save() if new_poc is not None or new_author is not None: diff --git a/geonode/maps/api/tests.py b/geonode/maps/api/tests.py index 668528ddbb2..a49db25b7c4 100644 --- a/geonode/maps/api/tests.py +++ b/geonode/maps/api/tests.py @@ -132,7 +132,7 @@ def setUp(self): "category": "cat1" } Map.objects.filter(id=self.map.id).update(extra_metadata=[self.metadata]) - + def test_get_will_return_the_list_of_extra_metadata(self): self.client.login(username="admin", password="admin") url = reverse('maps-extra-metadata', args=[self.map.id]) diff --git a/geonode/maps/api/views.py b/geonode/maps/api/views.py index 3f794c1567b..4287a1213c8 100644 --- a/geonode/maps/api/views.py +++ b/geonode/maps/api/views.py @@ -72,7 +72,6 @@ def local_layers(self, request, pk=None): resources = map.local_layers return Response(LayerSerializer(embed=True, many=True).to_representation(resources)) - @extend_schema( methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" ) diff --git a/geonode/settings.py b/geonode/settings.py index 32fb1238d85..4202c10ea6a 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -37,6 +37,7 @@ from kombu import Queue, Exchange from kombu.serialization import register +from schema import Optional from . import serializer SILENCED_SYSTEM_CHECKS = [ @@ -2167,7 +2168,6 @@ def get_geonode_catalogue_service(): ''' Default schema used to store extra and dynamic metadata for the resource ''' -from schema import Optional DEFAULT_EXTRA_METADATA_SCHEMA = { Optional("id"): int, From 4685630a01b11cd3cdeb7dd9d879e36367733d7a Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 1 Feb 2022 15:35:00 +0100 Subject: [PATCH 13/24] [Fixes #8689] Remove endpoint from each resorce, keep it only on base resource --- geonode/base/api/tests.py | 61 +++++++++++++- geonode/base/api/views.py | 146 ++++++++++++++++++--------------- geonode/documents/api/tests.py | 62 +------------- geonode/documents/api/views.py | 18 +--- geonode/geoapps/api/tests.py | 63 -------------- geonode/geoapps/api/views.py | 20 +---- geonode/layers/api/tests.py | 61 +------------- geonode/layers/api/views.py | 20 +---- geonode/maps/api/tests.py | 61 +------------- geonode/maps/api/views.py | 18 +--- 10 files changed, 146 insertions(+), 384 deletions(-) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 99f0b86d6d0..3a73fda3b57 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -36,12 +36,13 @@ from geonode import geoserver from geonode.layers.models import Layer +from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend, set_resource_default_links from geonode.favorite.models import Favorite from geonode.documents.models import Document from geonode.base.utils import build_absolute_uri from geonode.thumbs.exceptions import ThumbnailError -from geonode.base.populate_test_data import create_models +from geonode.base.populate_test_data import create_models, create_single_layer from geonode.security.utils import get_resources_with_perms from geonode.base.models import ( @@ -1021,3 +1022,61 @@ def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset_raise_exp } self.assertEqual(response.status_code, 500) self.assertEqual(expected, response.json()) + + +class TestExtraMetadataLayersApi(GeoNodeBaseTestSupport): + def setUp(self): + self.layer = create_single_layer('single_layer') + self.metadata = { + "name": "metadata-name", + "slug": "metadata-slug", + "help_text": "this is the help text", + "field_type": "str", + "value": "my value", + "category": "cat1" + } + Layer.objects.filter(id=self.layer.id).update(extra_metadata=[self.metadata]) + + def test_get_will_return_the_list_of_extra_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('resources-extra-metadata', args=[self.layer.id]) + response = self.client.get(url, content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([self.metadata], response.json()) + + def test_put_will_update_the_whole_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('resources-extra-metadata', args=[self.layer.id]) + input_metadata = { + "name": "metadata-updated", + "slug": "metadata-slug-updated", + "help_text": "this is the help text-updated", + "field_type": "str-updated", + "value": "my value-updated", + "category": "cat1-updated" + } + response = self.client.put(url, data=[input_metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([input_metadata], response.json()) + + def test_post_will_add_new_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('resources-extra-metadata', args=[self.layer.id]) + input_metadata = { + "name": "metadata-new", + "slug": "metadata-slug-new", + "help_text": "this is the help text-new", + "field_type": "str-new", + "value": "my value-new", + "category": "cat1-new" + } + response = self.client.post(url, data=[input_metadata], content_type='application/json') + self.assertTrue(201, response.status_code) + self.assertEqual(2, len(response.json())) + + def test_delete_will_delete_single_metadata(self): + self.client.login(username="admin", password="admin") + url = reverse('resources-extra-metadata', args=[self.layer.id]) + response = self.client.delete(url, data=[self.metadata], content_type='application/json') + self.assertTrue(200, response.status_code) + self.assertEqual([], response.json()) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index a279d21b08a..f2060fe9ca9 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -469,70 +469,82 @@ def set_thumbnail_from_bbox(self, request, resource_id): logger.error(e) return Response(data={"message": e.args[0], "success": False}, status=500, exception=True) - -def common_extra_metadata_handler(request, _obj): - if request.method == "GET": - # get list of available metadata - queryset = _obj.metadata.all() - _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()] - if _filters: - queryset = queryset.filter(**_filters[0]) - return Response(ExtraMetadataSerializer().to_representation(queryset)) - if not request.method == "DELETE": - try: - extra_metadata = validate_extra_metadata(request.data, _obj) - except Exception as e: - return Response(status=500, data=e.args[0]) - - if request.method == "PUT": - ''' - update specific metadata. The ID of the metadata is required to perform the update - [ - { - "id": 1, - "name": "foo_name", - "slug": "foo_sug", - "help_text": "object", - "field_type": "int", - "value": "object", - "category": "object" - } - ] - ''' - for _m in extra_metadata: - _id = _m.pop('id') - ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) - logger.info("metadata updated for the selected resource") - return Response(ExtraMetadataSerializer().to_representation(extra_metadata)) - elif request.method == "DELETE": - # delete single metadata - ''' - Expect a payload with the IDs of the metadata that should be deleted. Payload be like: - [4, 3] - ''' - ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete() - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) - elif request.method == "POST": - # add new metadata - ''' - [ - { - "name": "foo_name", - "slug": "foo_sug", - "help_text": "object", - "field_type": "int", - "value": "object", - "category": "object" - } - ] - ''' - for _m in extra_metadata: - new_m = ExtraMetadata.objects.create( - resource=_obj, - metadata=_m - ) - new_m.save() - _obj.metadata.add(new_m) - _obj.refresh_from_db() - return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201) + @extend_schema( + methods=["get", "put", "delete", "post"], description="Get/Update/Delete/Add extra metadata for resource" + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + permission_classes=[ + IsOwnerOrAdmin, + ], + url_path=r"extra_metadata", # noqa + url_name="extra-metadata", + ) + def extra_metadata(self, request, pk=None): + _obj = self.get_object() + if request.method == "GET": + # get list of available metadata + queryset = _obj.metadata.all() + _filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()] + if _filters: + queryset = queryset.filter(**_filters[0]) + return Response(ExtraMetadataSerializer().to_representation(queryset)) + if not request.method == "DELETE": + try: + extra_metadata = validate_extra_metadata(request.data, _obj) + except Exception as e: + return Response(status=500, data=e.args[0]) + + if request.method == "PUT": + ''' + update specific metadata. The ID of the metadata is required to perform the update + [ + { + "id": 1, + "name": "foo_name", + "slug": "foo_sug", + "help_text": "object", + "field_type": "int", + "value": "object", + "category": "object" + } + ] + ''' + for _m in extra_metadata: + _id = _m.pop('id') + ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) + logger.info("metadata updated for the selected resource") + return Response(ExtraMetadataSerializer().to_representation(extra_metadata)) + elif request.method == "DELETE": + # delete single metadata + ''' + Expect a payload with the IDs of the metadata that should be deleted. Payload be like: + [4, 3] + ''' + ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete() + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) + elif request.method == "POST": + # add new metadata + ''' + [ + { + "name": "foo_name", + "slug": "foo_sug", + "help_text": "object", + "field_type": "int", + "value": "object", + "category": "object" + } + ] + ''' + for _m in extra_metadata: + new_m = ExtraMetadata.objects.create( + resource=_obj, + metadata=_m + ) + new_m.save() + _obj.metadata.add(new_m) + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201) diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index f239014e018..7c5e0c7dd33 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -27,9 +27,8 @@ from geonode.documents.models import Document from geonode import geoserver -from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend -from geonode.base.populate_test_data import create_models, create_single_doc +from geonode.base.populate_test_data import create_models logger = logging.getLogger(__name__) @@ -90,62 +89,3 @@ def test_documents(self): # import json # logger.error(f"{json.dumps(layers_data)}") - - -class TestExtraMetadataUploadApi(GeoNodeBaseTestSupport): - def setUp(self): - self.doc = create_single_doc('document') - self.metadata = { - "name": "metadata-name", - "slug": "metadata-slug", - "help_text": "this is the help text", - "field_type": "str", - "value": "my value", - "category": "cat1" - } - self.doc.extra_metadata = [self.metadata] - self.doc.save() - - def test_get_will_return_the_list_of_extra_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('documents-extra_metadata', args=[self.doc.id]) - response = self.client.get(url, content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([self.metadata], response.json()) - - def test_put_will_update_the_whole_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('documents-extra_metadata', args=[self.doc.id]) - input_metadata = { - "name": "metadata-updated", - "slug": "metadata-slug-updated", - "help_text": "this is the help text-updated", - "field_type": "str-updated", - "value": "my value-updated", - "category": "cat1-updated" - } - response = self.client.put(url, data=[input_metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([input_metadata], response.json()) - - def test_post_will_add_new_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('documents-extra_metadata', args=[self.doc.id]) - input_metadata = { - "name": "metadata-new", - "slug": "metadata-slug-new", - "help_text": "this is the help text-new", - "field_type": "str-new", - "value": "my value-new", - "category": "cat1-new" - } - response = self.client.post(url, data=[input_metadata], content_type='application/json') - self.assertTrue(201, response.status_code) - self.assertEqual(2, len(response.json())) - - def test_delete_will_delete_single_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('documents-extra_metadata', args=[self.doc.id]) - response = self.client.delete(url, data=[self.metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([], response.json()) diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index a56d09e3dc9..cd80505faff 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -27,9 +27,8 @@ from oauth2_provider.contrib.rest_framework import OAuth2Authentication from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination -from geonode.base.api.views import common_extra_metadata_handler from geonode.documents.models import Document from geonode.base.models import ResourceBase @@ -75,18 +74,3 @@ def linked_resources(self, request, pk=None): result_page = paginator.paginate_queryset(resources, request) serializer = ResourceBaseSerializer(result_page, embed=True, many=True) return paginator.get_paginated_response({"resources": serializer.data}) - - @extend_schema( - methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" - ) - @action( - detail=True, - methods=["get", "put", "delete", "post"], - permission_classes=[ - IsOwnerOrAdmin, - ], - url_path=r"extra_metadata", # noqa - url_name="extra_metadata", - ) - def extra_metadata(self, request, pk=None): - return common_extra_metadata_handler(request, self.get_object()) diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index 6ee4000b4a5..c86219aa3f2 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -27,7 +27,6 @@ from geonode.geoapps.models import GeoApp, GeoAppData from geonode import geoserver -from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend from geonode.base.populate_test_data import create_models @@ -188,65 +187,3 @@ def test_geoapps_crud(self): response = self.client.get( f"{url}?include[]=data", format='json') self.assertEqual(response.status_code, 404) # 404 - Not Found - - -class TestExtraMetadataGeoAppApi(GeoNodeBaseTestSupport): - def setUp(self): - self.geo_app = GeoApp.objects.create( - title="Test GeoApp", - owner=get_user_model().objects.get(username='admin') - ) - self.metadata = { - "name": "metadata-name", - "slug": "metadata-slug", - "help_text": "this is the help text", - "field_type": "str", - "value": "my value", - "category": "cat1" - } - self.geo_app.extra_metadata = [self.metadata] - self.geo_app.save() - - def test_get_will_return_the_list_of_extra_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) - response = self.client.get(url, content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([self.metadata], response.json()) - - def test_put_will_update_the_whole_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) - input_metadata = { - "name": "metadata-updated", - "slug": "metadata-slug-updated", - "help_text": "this is the help text-updated", - "field_type": "str-updated", - "value": "my value-updated", - "category": "cat1-updated" - } - response = self.client.put(url, data=[input_metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([input_metadata], response.json()) - - def test_post_will_add_new_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) - input_metadata = { - "name": "metadata-new", - "slug": "metadata-slug-new", - "help_text": "this is the help text-new", - "field_type": "str-new", - "value": "my value-new", - "category": "cat1-new" - } - response = self.client.post(url, data=[input_metadata], content_type='application/json') - self.assertTrue(201, response.status_code) - self.assertEqual(2, len(response.json())) - - def test_delete_will_delete_single_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('geoapps-extra-metadata', args=[self.geo_app.id]) - response = self.client.delete(url, data=[self.metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([], response.json()) diff --git a/geonode/geoapps/api/views.py b/geonode/geoapps/api/views.py index 00d64fc48af..aa444b450cf 100644 --- a/geonode/geoapps/api/views.py +++ b/geonode/geoapps/api/views.py @@ -22,12 +22,9 @@ from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.authentication import SessionAuthentication, BasicAuthentication from oauth2_provider.contrib.rest_framework import OAuth2Authentication -from drf_spectacular.utils import extend_schema -from rest_framework.decorators import action -from geonode.base.api.views import common_extra_metadata_handler from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination from geonode.geoapps.models import GeoApp @@ -52,18 +49,3 @@ class GeoAppViewSet(DynamicModelViewSet): queryset = GeoApp.objects.all() serializer_class = GeoAppSerializer pagination_class = GeoNodeApiPagination - - @extend_schema( - methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" - ) - @action( - detail=True, - methods=["get", "put", "delete", "post"], - permission_classes=[ - IsOwnerOrAdmin, - ], - url_path=r"extra_metadata", # noqa - url_name="extra-metadata", - ) - def extra_metadata(self, request, pk=None): - return common_extra_metadata_handler(request, self.get_object()) diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py index 3b0375d7f9e..08451f3f882 100644 --- a/geonode/layers/api/tests.py +++ b/geonode/layers/api/tests.py @@ -24,9 +24,8 @@ from geonode import geoserver from geonode.layers.models import Layer -from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend -from geonode.base.populate_test_data import create_models, create_single_layer +from geonode.base.populate_test_data import create_models logger = logging.getLogger(__name__) @@ -97,61 +96,3 @@ def test_raw_HTML_stripped_properties(self): self.assertEqual(response.data['layer']['raw_constraints_other'], "None") self.assertEqual(response.data['layer']['raw_supplemental_information'], "No information provided í £682m") self.assertEqual(response.data['layer']['raw_data_quality_statement'], "OK 1 2 a b") - - -class TestExtraMetadataLayersApi(GeoNodeBaseTestSupport): - def setUp(self): - self.layer = create_single_layer('single_layer') - self.metadata = { - "name": "metadata-name", - "slug": "metadata-slug", - "help_text": "this is the help text", - "field_type": "str", - "value": "my value", - "category": "cat1" - } - Layer.objects.filter(id=self.layer.id).update(extra_metadata=[self.metadata]) - - def test_get_will_return_the_list_of_extra_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('layers-extra-metadata', args=[self.layer.id]) - response = self.client.get(url, content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([self.metadata], response.json()) - - def test_put_will_update_the_whole_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('layers-extra-metadata', args=[self.layer.id]) - input_metadata = { - "name": "metadata-updated", - "slug": "metadata-slug-updated", - "help_text": "this is the help text-updated", - "field_type": "str-updated", - "value": "my value-updated", - "category": "cat1-updated" - } - response = self.client.put(url, data=[input_metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([input_metadata], response.json()) - - def test_post_will_add_new_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('layers-extra-metadata', args=[self.layer.id]) - input_metadata = { - "name": "metadata-new", - "slug": "metadata-slug-new", - "help_text": "this is the help text-new", - "field_type": "str-new", - "value": "my value-new", - "category": "cat1-new" - } - response = self.client.post(url, data=[input_metadata], content_type='application/json') - self.assertTrue(201, response.status_code) - self.assertEqual(2, len(response.json())) - - def test_delete_will_delete_single_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('layers-extra-metadata', args=[self.layer.id]) - response = self.client.delete(url, data=[self.metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([], response.json()) diff --git a/geonode/layers/api/views.py b/geonode/layers/api/views.py index 17d71291d81..8751af1df6f 100644 --- a/geonode/layers/api/views.py +++ b/geonode/layers/api/views.py @@ -22,11 +22,8 @@ from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.authentication import SessionAuthentication, BasicAuthentication from oauth2_provider.contrib.rest_framework import OAuth2Authentication -from drf_spectacular.utils import extend_schema -from rest_framework.decorators import action -from geonode.base.api.views import common_extra_metadata_handler from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination from geonode.layers.models import Layer @@ -51,18 +48,3 @@ class LayerViewSet(DynamicModelViewSet): queryset = Layer.objects.all() serializer_class = LayerSerializer pagination_class = GeoNodeApiPagination - - @extend_schema( - methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" - ) - @action( - detail=True, - methods=["get", "put", "delete", "post"], - permission_classes=[ - IsOwnerOrAdmin, - ], - url_path=r"extra_metadata", # noqa - url_name="extra-metadata", - ) - def extra_metadata(self, request, pk=None): - return common_extra_metadata_handler(request, self.get_object(), self.queryset) diff --git a/geonode/maps/api/tests.py b/geonode/maps/api/tests.py index a49db25b7c4..fe655a8c3f6 100644 --- a/geonode/maps/api/tests.py +++ b/geonode/maps/api/tests.py @@ -28,9 +28,8 @@ from geonode.maps.models import Map from geonode import geoserver -from geonode.tests.base import GeoNodeBaseTestSupport from geonode.utils import check_ogc_backend -from geonode.base.populate_test_data import create_models, create_single_map +from geonode.base.populate_test_data import create_models logger = logging.getLogger(__name__) @@ -118,61 +117,3 @@ def test_maps(self): self.assertTrue(len(response.data) > 0) self.assertTrue('data' in response.data) self.assertTrue('attributes' in response.data) - - -class TestExtraMetadataMapsApi(GeoNodeBaseTestSupport): - def setUp(self): - self.map = create_single_map('single_map') - self.metadata = { - "name": "metadata-name", - "slug": "metadata-slug", - "help_text": "this is the help text", - "field_type": "str", - "value": "my value", - "category": "cat1" - } - Map.objects.filter(id=self.map.id).update(extra_metadata=[self.metadata]) - - def test_get_will_return_the_list_of_extra_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('maps-extra-metadata', args=[self.map.id]) - response = self.client.get(url, content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([self.metadata], response.json()) - - def test_put_will_update_the_whole_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('maps-extra-metadata', args=[self.map.id]) - input_metadata = { - "name": "metadata-updated", - "slug": "metadata-slug-updated", - "help_text": "this is the help text-updated", - "field_type": "str-updated", - "value": "my value-updated", - "category": "cat1-updated" - } - response = self.client.put(url, data=[input_metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([input_metadata], response.json()) - - def test_post_will_add_new_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('maps-extra-metadata', args=[self.map.id]) - input_metadata = { - "name": "metadata-new", - "slug": "metadata-slug-new", - "help_text": "this is the help text-new", - "field_type": "str-new", - "value": "my value-new", - "category": "cat1-new" - } - response = self.client.post(url, data=[input_metadata], content_type='application/json') - self.assertTrue(201, response.status_code) - self.assertEqual(2, len(response.json())) - - def test_delete_will_delete_single_metadata(self): - self.client.login(username="admin", password="admin") - url = reverse('maps-extra-metadata', args=[self.map.id]) - response = self.client.delete(url, data=[self.metadata], content_type='application/json') - self.assertTrue(200, response.status_code) - self.assertEqual([], response.json()) diff --git a/geonode/maps/api/views.py b/geonode/maps/api/views.py index 4287a1213c8..dbac2140d43 100644 --- a/geonode/maps/api/views.py +++ b/geonode/maps/api/views.py @@ -28,9 +28,8 @@ from oauth2_provider.contrib.rest_framework import OAuth2Authentication from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter -from geonode.base.api.permissions import IsOwnerOrAdmin, IsOwnerOrReadOnly +from geonode.base.api.permissions import IsOwnerOrReadOnly from geonode.base.api.pagination import GeoNodeApiPagination -from geonode.base.api.views import common_extra_metadata_handler from geonode.layers.api.serializers import LayerSerializer from geonode.maps.models import Map @@ -71,18 +70,3 @@ def local_layers(self, request, pk=None): map = self.get_object() resources = map.local_layers return Response(LayerSerializer(embed=True, many=True).to_representation(resources)) - - @extend_schema( - methods=["get", "put", "delete", "post"], description="Get or update extra metadata for each resource" - ) - @action( - detail=True, - methods=["get", "put", "delete", "post"], - permission_classes=[ - IsOwnerOrAdmin, - ], - url_path=r"extra_metadata", # noqa - url_name="extra-metadata", - ) - def extra_metadata(self, request, pk=None): - return common_extra_metadata_handler(request, self.get_object()) From 8cb942f870896c3fb4852fd8571d5538b91ba72b Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 1 Feb 2022 17:10:12 +0100 Subject: [PATCH 14/24] [Fixes #8689] Fix broken tests --- geonode/base/api/tests.py | 26 ++++++++++++++++++-------- geonode/base/api/views.py | 3 ++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 3a73fda3b57..0bb18f756e2 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -47,6 +47,7 @@ from geonode.base.models import ( CuratedThumbnail, + ExtraMetadata, HierarchicalKeyword, Region, ResourceBase, @@ -1024,7 +1025,7 @@ def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset_raise_exp self.assertEqual(expected, response.json()) -class TestExtraMetadataLayersApi(GeoNodeBaseTestSupport): +class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport): def setUp(self): self.layer = create_single_layer('single_layer') self.metadata = { @@ -1035,19 +1036,28 @@ def setUp(self): "value": "my value", "category": "cat1" } - Layer.objects.filter(id=self.layer.id).update(extra_metadata=[self.metadata]) + m = ExtraMetadata.objects.create( + resource=self.layer, + metadata=self.metadata + ) + self.layer.metadata.add(m) + self.mdata = ExtraMetadata.objects.first() def test_get_will_return_the_list_of_extra_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('resources-extra-metadata', args=[self.layer.id]) + url = reverse('base-resources-extra-metadata', args=[self.layer.id]) response = self.client.get(url, content_type='application/json') self.assertTrue(200, response.status_code) - self.assertEqual([self.metadata], response.json()) + expected = [ + {**{"id": self.mdata.id}, **self.metadata} + ] + self.assertEqual(expected, response.json()) def test_put_will_update_the_whole_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('resources-extra-metadata', args=[self.layer.id]) + url = reverse('base-resources-extra-metadata', args=[self.layer.id]) input_metadata = { + "id": self.mdata.id, "name": "metadata-updated", "slug": "metadata-slug-updated", "help_text": "this is the help text-updated", @@ -1061,7 +1071,7 @@ def test_put_will_update_the_whole_metadata(self): def test_post_will_add_new_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('resources-extra-metadata', args=[self.layer.id]) + url = reverse('base-resources-extra-metadata', args=[self.layer.id]) input_metadata = { "name": "metadata-new", "slug": "metadata-slug-new", @@ -1076,7 +1086,7 @@ def test_post_will_add_new_metadata(self): def test_delete_will_delete_single_metadata(self): self.client.login(username="admin", password="admin") - url = reverse('resources-extra-metadata', args=[self.layer.id]) - response = self.client.delete(url, data=[self.metadata], content_type='application/json') + url = reverse('base-resources-extra-metadata', args=[self.layer.id]) + response = self.client.delete(url, data=[self.mdata.id], content_type='application/json') self.assertTrue(200, response.status_code) self.assertEqual([], response.json()) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index f2060fe9ca9..691bdd3a65d 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -515,7 +515,8 @@ def extra_metadata(self, request, pk=None): _id = _m.pop('id') ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m) logger.info("metadata updated for the selected resource") - return Response(ExtraMetadataSerializer().to_representation(extra_metadata)) + _obj.refresh_from_db() + return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all())) elif request.method == "DELETE": # delete single metadata ''' From 499ea10b82489e00899a5897cd94913e4e6e04e8 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 1 Feb 2022 18:02:51 +0100 Subject: [PATCH 15/24] [Fixes #8689] Add metadata filtering in API v1 --- geonode/api/resourcebase_api.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index 44ecf59f6e1..8192515a1bc 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -95,7 +95,8 @@ class CommonMetaApi: 'date': ALL, 'purpose': ALL, 'uuid': ALL_WITH_RELATIONS, - 'abstract': ALL + 'abstract': ALL, + 'metadata': ALL_WITH_RELATIONS } ordering = ['date', 'title', 'popular_count'] max_limit = None @@ -168,6 +169,9 @@ def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): orm_filters.update({'polymorphic_ctype__model__in': [filt.lower() for filt in filters.getlist('app_type__in')]}) if 'extent' in filters: orm_filters.update({'extent': filters['extent']}) + _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith('metadata')} + if _metadata: + orm_filters.update({"metadata_filters": _metadata}) orm_filters['f_method'] = filters['f_method'] if 'f_method' in filters else 'and' if not settings.SEARCH_RESOURCES_EXTENDED: return self._remove_additional_filters(orm_filters) @@ -186,6 +190,9 @@ def apply_filters(self, request, applicable_filters): metadata_only = applicable_filters.pop('metadata_only', False) filtering_method = applicable_filters.pop('f_method', 'and') polyphormic_model = applicable_filters.pop('polymorphic_ctype__model__in', None) + + metadata_filters = applicable_filters.pop('metadata_filters', None) + if filtering_method == 'or': filters = Q() for f in applicable_filters.items(): @@ -234,6 +241,9 @@ def apply_filters(self, request, applicable_filters): if keywords: filtered = self.filter_h_keywords(filtered, keywords) + if metadata_filters: + filtered = filtered.filter(**metadata_filters) + # return filtered return get_visible_resources( filtered, @@ -589,6 +599,9 @@ def format_objects(self, objects): except Exception as e: logger.exception(e) + if formatted_obj.get('metadata', None): + formatted_obj['metadata'] = [model_to_dict(_m) for _m in formatted_obj['metadata']] + formatted_objects.append(formatted_obj) return formatted_objects From bf819b48a6c775d9c1cd99f7e738104e9d01bf89 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 1 Feb 2022 18:29:41 +0100 Subject: [PATCH 16/24] [Fixes #8689] Add test for metadata filtering in API v1 --- geonode/api/resourcebase_api.py | 8 ++++---- geonode/api/tests.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index 8192515a1bc..9fb6415b799 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -190,9 +190,9 @@ def apply_filters(self, request, applicable_filters): metadata_only = applicable_filters.pop('metadata_only', False) filtering_method = applicable_filters.pop('f_method', 'and') polyphormic_model = applicable_filters.pop('polymorphic_ctype__model__in', None) - + metadata_filters = applicable_filters.pop('metadata_filters', None) - + if filtering_method == 'or': filters = Q() for f in applicable_filters.items(): @@ -243,7 +243,7 @@ def apply_filters(self, request, applicable_filters): if metadata_filters: filtered = filtered.filter(**metadata_filters) - + # return filtered return get_visible_resources( filtered, @@ -601,7 +601,7 @@ def format_objects(self, objects): if formatted_obj.get('metadata', None): formatted_obj['metadata'] = [model_to_dict(_m) for _m in formatted_obj['metadata']] - + formatted_objects.append(formatted_obj) return formatted_objects diff --git a/geonode/api/tests.py b/geonode/api/tests.py index 8c85a2fa373..d748bc826e3 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -30,6 +30,7 @@ from guardian.shortcuts import get_anonymous_user from geonode import geoserver +from geonode.base.models import ExtraMetadata from geonode.maps.models import Map from geonode.layers.models import Layer from geonode.utils import check_ogc_backend @@ -360,6 +361,35 @@ def test_category_filters(self): self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 5) + def test_metadata_filters(self): + """Test category filtering""" + _r = Layer.objects.first() + _m = ExtraMetadata.objects.create( + resource=_r, + metadata={ + "name": "metadata-updated", + "slug": "metadata-slug-updated", + "help_text": "this is the help text-updated", + "field_type": "str-updated", + "value": "my value-updated", + "category": "category" + } + ) + _r.metadata.add(_m) + # check we get the correct layers number returnered filtering on one + # and then two different categories + filter_url = f"{self.list_url}?metadata__category=category" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 1) + + filter_url = f"{self.list_url}?metadata__category=not-existing-category" + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 0) + def test_tag_filters(self): """Test keywords filtering""" From 17cf50e832fc8cb433915275f2b1cdc0c25c57fb Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 10:32:40 +0100 Subject: [PATCH 17/24] [Fixes #8689] Fix some of broken tests --- geonode/base/forms.py | 2 +- geonode/documents/tests.py | 24 +++++++++++++++++++++--- geonode/layers/tests.py | 4 ++-- geonode/maps/tests.py | 10 +++++++++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 51aae7d78b7..ab220ceef9a 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -490,7 +490,7 @@ class ResourceBaseForm(TranslationModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.instance and self.instance.metadata.exists(): + if self.instance and self.instance.id and self.instance.metadata.exists(): self.fields['extra_metadata'].initial = [x.metadata for x in self.instance.metadata.all()] for field in self.fields: help_text = self.fields[field].help_text diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 1ad4b404797..42dbd5f1c09 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -807,10 +807,18 @@ def test_that_non_admin_user_can_create_write_to_map_without_keyword(self): self.client.login(username=self.not_admin.username, password='very-secret') url = reverse('document_metadata', args=(self.test_doc.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=True): - response = self.client.post(url) + response = self.client.post(url, data={ + "resource-owner": self.not_admin.id, + "resource-title": "doc", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + }) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) - + self.test_doc.refresh_from_db() + self.assertEqual('doc', self.test_doc.title) + def test_that_non_admin_user_cannot_create_edit_keyword(self): """ Test that non admin users cannot edit/create keywords when FREETEXT_KEYWORDS_READONLY=True @@ -843,9 +851,19 @@ def test_that_non_admin_user_can_create_edit_keyword_when_freetext_keywords_read self.client.login(username=self.not_admin.username, password='very-secret') url = reverse('document_metadata', args=(self.test_doc.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=False): - response = self.client.post(url, data={'resource-keywords': 'wonderful-keyword'}) + response = self.client.post(url, data={ + "resource-owner": self.not_admin.id, + "resource-title": "doc", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + 'resource-keywords': 'wonderful-keyword' + }) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) + self.test_doc.refresh_from_db() + self.assertEqual("doc", self.test_doc.title) + def test_document_link_with_permissions(self): self.test_doc.set_permissions(self.perm_spec) diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index c7c97a5c64b..8b15c1e3ee2 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -1964,7 +1964,7 @@ def test_resource_form_is_invalid_extra_metadata_not_json_format(self): "resource-language": "eng", "resource-extra_metadata": "not-a-json" }) - expected = {"success": False, "errors": ["extra_metadata: The value insered for the Extra metadata field is not a valid JSON"]} + expected = {"success": False, "errors": ["extra_metadata: The value provided for the Extra metadata field is not a valid JSON"]} self.assertDictEqual(expected, response.json()) @override_settings(EXTRA_METADATA_SCHEMA={"key": "value"}) @@ -1997,7 +1997,7 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): self.assertIn(expected, response.json()['errors'][0]) def test_resource_form_is_valid_extra_metadata(self): - form = self.sut(data={ + form = self.sut(instance=self.layer, data={ "owner": self.layer.owner.id, "title": "layer_title", "date": "2022-01-24 16:38 pm", diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index 8e26ab11eff..5dd7b3da897 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -478,9 +478,17 @@ def test_that_non_admin_user_can_create_write_to_map_without_keyword(self): url = reverse('map_metadata', args=(test_map.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=True): - response = self.client.post(url) + response = self.client.post(url, data={ + "resource-owner": self.not_admin.id, + "resource-title": "doc", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng" + }) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) + test_map.refresh_from_db() + self.assertEqual("doc", test_map.title) def test_that_keyword_multiselect_is_enabled_for_non_admin_users_when_freetext_keywords_readonly_istrue(self): """ From 0d425b1a0c450b63b1922090ab03e26cc981f083 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 11:05:08 +0100 Subject: [PATCH 18/24] [Fixes #8689] fix flake8 --- geonode/documents/tests.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 42dbd5f1c09..4bdd83cc5af 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -818,7 +818,7 @@ def test_that_non_admin_user_can_create_write_to_map_without_keyword(self): self.assertEqual(response.status_code, 200) self.test_doc.refresh_from_db() self.assertEqual('doc', self.test_doc.title) - + def test_that_non_admin_user_cannot_create_edit_keyword(self): """ Test that non admin users cannot edit/create keywords when FREETEXT_KEYWORDS_READONLY=True @@ -862,8 +862,7 @@ def test_that_non_admin_user_can_create_edit_keyword_when_freetext_keywords_read self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) self.test_doc.refresh_from_db() - self.assertEqual("doc", self.test_doc.title) - + self.assertEqual("doc", self.test_doc.title) def test_document_link_with_permissions(self): self.test_doc.set_permissions(self.perm_spec) From 2f197f11405930df866174314a4eb8e1d17688be Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 11:47:00 +0100 Subject: [PATCH 19/24] [Fixes #8689] fix tests --- geonode/geoapps/tests.py | 4 ++-- geonode/maps/tests.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/geonode/geoapps/tests.py b/geonode/geoapps/tests.py index 2b9b1bec8b0..31baabe1a5d 100644 --- a/geonode/geoapps/tests.py +++ b/geonode/geoapps/tests.py @@ -38,7 +38,7 @@ def setUp(self) -> None: self.sut = GeoAppForm def test_update_geoapp_metadata(self): - bobby = get_user_model().objects.get(username='bobby') + bobby = get_user_model().objects.get(username='admin') gep_app = GeoApp.objects.create( title="App", thumbnail_url='initial', @@ -53,7 +53,7 @@ def test_update_geoapp_metadata(self): "resource-date_type": 'publication', 'resource-language': gep_app.language } - self.client.login(username=bobby.username, password='bob') + self.client.login(username=bobby.username, password='admin') response = self.client.post(url, data=data, format='json') self.assertEqual(response.status_code, 200) self.assertEqual(gep_app.thumbnail_url, GeoApp.objects.get(id=gep_app.id).thumbnail_url) diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index 5dd7b3da897..15c752bcabb 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -557,8 +557,15 @@ def test_map_metadata(self, thumbnail_mock): self.assertEqual(response.status_code, 200) # Now test with a valid user using POST method + user = get_user_model().objects.first() self.client.login(username=self.user, password=self.passwd) - response = self.client.post(url) + response = self.client.post(url, data={ + "resource-owner": user.id, + "resource-title": "map_title", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + }) self.assertEqual(response.status_code, 200) # TODO: only invalid mapform is tested From d7b5c550d37a1c11f9632720132fa891bdc91b35 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 11:55:35 +0100 Subject: [PATCH 20/24] [Fixes #8689] removed typo on settings.py --- geonode/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/geonode/settings.py b/geonode/settings.py index 4202c10ea6a..c51d59cbb6d 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2195,6 +2195,5 @@ def get_geonode_catalogue_service(): "map": os.getenv('MAP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), "layer": os.getenv('DATASET_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), "document": os.getenv('DOCUMENT_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), - "geoapp": os.getenv('GEOAPP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA), - "dashboard": os.getenv('GEOAPP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA) + "geoapp": os.getenv('GEOAPP_EXTRA_METADATA_SCHEMA', DEFAULT_EXTRA_METADATA_SCHEMA) }, **CUSTOM_METADATA_SCHEMA} From a6498e2b0290469d569d3fcd169d7dd6de8437fd Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 13:23:11 +0100 Subject: [PATCH 21/24] [Fixes #8689] fix broken build --- geonode/maps/tests.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index 15c752bcabb..ea6bc8d3287 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -515,9 +515,19 @@ def test_that_non_admin_user_can_create_edit_keyword_when_freetext_keywords_read url = reverse('map_metadata', args=(test_map.pk,)) with self.settings(FREETEXT_KEYWORDS_READONLY=False): - response = self.client.post(url, data={'resource-keywords': 'wonderful-keyword'}) + response = self.client.post(url, data={ + "resource-owner": self.not_admin.id, + "resource-title": "map", + "resource-date": "2022-01-24 16:38 pm", + "resource-date_type": "creation", + "resource-language": "eng", + 'resource-keywords': 'wonderful-keyword' + }) self.assertFalse(self.not_admin.is_superuser) self.assertEqual(response.status_code, 200) + test_map.refresh_from_db() + self.assertEqual("map", test_map.title) + @patch('geonode.thumbs.thumbnails.create_thumbnail') def test_map_metadata(self, thumbnail_mock): @@ -557,7 +567,7 @@ def test_map_metadata(self, thumbnail_mock): self.assertEqual(response.status_code, 200) # Now test with a valid user using POST method - user = get_user_model().objects.first() + user = get_user_model().objects.filter(username='admin').first() self.client.login(username=self.user, password=self.passwd) response = self.client.post(url, data={ "resource-owner": user.id, From 9a911f6b8ed80556b3c0a1991ef47abec73669bf Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 15:58:15 +0100 Subject: [PATCH 22/24] [Fixes #8689] fix minor error on filter convertion --- geonode/api/resourcebase_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index 9fb6415b799..ba72c21689f 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -169,7 +169,7 @@ def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): orm_filters.update({'polymorphic_ctype__model__in': [filt.lower() for filt in filters.getlist('app_type__in')]}) if 'extent' in filters: orm_filters.update({'extent': filters['extent']}) - _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith('metadata')} + _metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith('metadata__')} if _metadata: orm_filters.update({"metadata_filters": _metadata}) orm_filters['f_method'] = filters['f_method'] if 'f_method' in filters else 'and' From cf7c4ec137630c67aff88e7167f2e75a12404dac Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 17:07:21 +0100 Subject: [PATCH 23/24] [Fixes #8689] fix flake8 --- geonode/maps/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index ea6bc8d3287..80e81a48d7f 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -528,7 +528,6 @@ def test_that_non_admin_user_can_create_edit_keyword_when_freetext_keywords_read test_map.refresh_from_db() self.assertEqual("map", test_map.title) - @patch('geonode.thumbs.thumbnails.create_thumbnail') def test_map_metadata(self, thumbnail_mock): """Test that map metadata can be properly rendered From 8d8480a4df2033a58922091f6ad9ad6f117bde41 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 2 Feb 2022 18:12:59 +0100 Subject: [PATCH 24/24] [Fixes #8689] Update default schema structure --- geonode/base/api/tests.py | 30 ++++++++++++------------------ geonode/geoapps/tests.py | 21 ++++++++++----------- geonode/layers/tests.py | 6 +++--- geonode/maps/tests.py | 6 +++--- geonode/settings.py | 12 +++++------- 5 files changed, 33 insertions(+), 42 deletions(-) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 0bb18f756e2..207dfaa755a 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -1029,12 +1029,10 @@ class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport): def setUp(self): self.layer = create_single_layer('single_layer') self.metadata = { - "name": "metadata-name", - "slug": "metadata-slug", - "help_text": "this is the help text", - "field_type": "str", - "value": "my value", - "category": "cat1" + "filter_header": "Foo Filter header", + "field_name": "metadata-name", + "field_label": "this is the help text", + "field_value": "foo" } m = ExtraMetadata.objects.create( resource=self.layer, @@ -1058,12 +1056,10 @@ def test_put_will_update_the_whole_metadata(self): url = reverse('base-resources-extra-metadata', args=[self.layer.id]) input_metadata = { "id": self.mdata.id, - "name": "metadata-updated", - "slug": "metadata-slug-updated", - "help_text": "this is the help text-updated", - "field_type": "str-updated", - "value": "my value-updated", - "category": "cat1-updated" + "filter_header": "Foo Filter header", + "field_name": "metadata-updated", + "field_label": "this is the help text", + "field_value": "foo" } response = self.client.put(url, data=[input_metadata], content_type='application/json') self.assertTrue(200, response.status_code) @@ -1073,12 +1069,10 @@ def test_post_will_add_new_metadata(self): self.client.login(username="admin", password="admin") url = reverse('base-resources-extra-metadata', args=[self.layer.id]) input_metadata = { - "name": "metadata-new", - "slug": "metadata-slug-new", - "help_text": "this is the help text-new", - "field_type": "str-new", - "value": "my value-new", - "category": "cat1-new" + "filter_header": "Foo Filter header", + "field_name": "metadata-updated", + "field_label": "this is the help text", + "field_value": "foo" } response = self.client.post(url, data=[input_metadata], content_type='application/json') self.assertTrue(201, response.status_code) diff --git a/geonode/geoapps/tests.py b/geonode/geoapps/tests.py index 31baabe1a5d..971f7a0d7da 100644 --- a/geonode/geoapps/tests.py +++ b/geonode/geoapps/tests.py @@ -99,21 +99,20 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): "resource-date": "2022-01-24 16:38 pm", "resource-date_type": "creation", "resource-language": "eng", - "resource-extra_metadata": '[{"key": "value"},{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + "resource-extra_metadata": '[{"key": "value"},{"id": "int", "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' }) - expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" + expected = "extra_metadata: Missing keys: \'field_label\', \'field_name\', \'field_value\', \'filter_header\' at index 0 " self.assertIn(expected, response.json()['errors'][0]) @override_settings(EXTRA_METADATA_SCHEMA={ "geoapp": { - "name": str, - "slug": str, - "help_text": str, - "field_type": object, - "value": object, - "category": str - } - }) + "id": int, + "filter_header": object, + "field_name": object, + "field_label": object, + "field_value": object + } + }) def test_resource_form_is_valid_extra_metadata(self): form = self.sut(data={ "owner": self.geoapp.owner.id, @@ -121,6 +120,6 @@ def test_resource_form_is_valid_extra_metadata(self): "date": "2022-01-24 16:38 pm", "date_type": "creation", "language": "eng", - "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' }) self.assertTrue(form.is_valid()) diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index 8b15c1e3ee2..6ce7afbca35 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -1991,9 +1991,9 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): "resource-date": "2022-01-24 16:38 pm", "resource-date_type": "creation", "resource-language": "eng", - "resource-extra_metadata": '[{"key": "value"},{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + "resource-extra_metadata": '[{"key": "value"},{"id": "int", "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' }) - expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" + expected = "extra_metadata: Missing keys: \'field_label\', \'field_name\', \'field_value\', \'filter_header\' at index 0 " self.assertIn(expected, response.json()['errors'][0]) def test_resource_form_is_valid_extra_metadata(self): @@ -2003,6 +2003,6 @@ def test_resource_form_is_valid_extra_metadata(self): "date": "2022-01-24 16:38 pm", "date_type": "creation", "language": "eng", - "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' }) self.assertTrue(form.is_valid()) diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index 80e81a48d7f..b7529b86fd7 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -1170,9 +1170,9 @@ def test_resource_form_is_invalid_extra_metadata_invalids_schema_entry(self): "resource-date": "2022-01-24 16:38 pm", "resource-date_type": "creation", "resource-language": "eng", - "resource-extra_metadata": '[{"key": "value"},{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + "resource-extra_metadata": '[{"key": "value"},{"id": "int", "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' }) - expected = "extra_metadata: Missing keys: \'category\', \'field_type\', \'help_text\', \'name\', \'slug\', \'value\' at index 0" + expected = "extra_metadata: Missing keys: \'field_label\', \'field_name\', \'field_value\', \'filter_header\' at index 0 " self.assertIn(expected, response.json()['errors'][0]) def test_resource_form_is_valid_extra_metadata(self): @@ -1182,6 +1182,6 @@ def test_resource_form_is_valid_extra_metadata(self): "date": "2022-01-24 16:38 pm", "date_type": "creation", "language": "eng", - "extra_metadata": '[{"name": "object", "slug": "object", "help_text": "object", "field_type": "object", "value": "object", "category": "object"}]' + "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]' }) self.assertTrue(form.is_valid()) diff --git a/geonode/settings.py b/geonode/settings.py index c51d59cbb6d..63ff4c37f53 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2170,13 +2170,11 @@ def get_geonode_catalogue_service(): ''' DEFAULT_EXTRA_METADATA_SCHEMA = { - Optional("id"): int, - "name": str, - "slug": str, - "help_text": str, - "field_type": object, - "value": object, - "category": str + Optional("id"): int, + "filter_header": object, + "field_name": object, + "field_label": object, + "field_value": object, } '''