Merge branch 'develop' into project_detail_v2

Author: harminius

server/Pipfile

@@ -26,7 +26,7 @@ flask-migrate = "==2.6.0" # 3.1.0
 wtforms-json = "==0.3.5"
 pytz = "==2022.2.1"
 scikit-build = "==0.18.1"
-pygeodiff = "==1.0.6"
+pygeodiff = "==2.0.4"
 pathvalidate = "==3.2.0"
 celery= "==5.4.0"
 redis= "==5.0.1"
@@ -40,6 +40,7 @@ psycogreen = "==1.0.2"
 importlib-metadata = "==8.4.0"  # https://github.com/pallets/flask/issues/4502
 typing_extensions = "==4.12.2"
 python-magic = "==0.4.27"
+click = "==8.2.0"
 # requirements for development on windows
 colorama = "==0.4.5"
 

server/Pipfile.lock

@@ -83,14 +83,14 @@
     get_user_agent,
     generate_location,
     is_valid_uuid,
-    gpkg_wkb_to_wkt,
     is_versioned_file,
     get_project_path,
     get_device_id,
     is_valid_path,
     is_supported_type,
     is_supported_extension,
     get_mimetype,
+    wkb2wkt,
 )
 from .errors import StorageLimitHit, ProjectLocked
 from ..utils import format_time_delta
@@ -1410,7 +1410,10 @@ def get_resource_changeset(project_name, namespace, version_id, path):  # noqa:
                 if key not in geom_change:
                     continue
                 gpkg_wkb = base64.b64decode(geom_change[key], validate=True)
-                wkt = gpkg_wkb_to_wkt(gpkg_wkb)
+                wkb = version.project.storage.geodiff.create_wkb_from_gpkg_header(
+                    gpkg_wkb
+                )
+                wkt = wkb2wkt(wkb)
                 if wkt:
                     geom_change[key] = wkt
         except (binascii.Error, TypeError, ValueError):

server/mergin/sync/public_api_controller.py

@@ -184,38 +184,8 @@ def is_valid_uuid(uuid):
         return False
 
 
-# inspired by C++ implementation https://github.com/lutraconsulting/geodiff/blob/master/geodiff/src/drivers/sqliteutils.cpp
-# in geodiff lib (MIT licence)
-def parse_gpkgb_header_size(gpkg_wkb):
-    """Parse header of geopackage wkb and return its size"""
-    # some constants
-    no_envelope_header_size = 8
-    flag_byte_pos = 3
-    envelope_size_mask = 14
-
-    try:
-        flag_byte = gpkg_wkb[flag_byte_pos]
-    except IndexError:
-        return -1  # probably some invalid input
-    envelope_byte = (flag_byte & envelope_size_mask) >> 1
-    envelope_size = 0
-
-    if envelope_byte == 1:
-        envelope_size = 32
-    elif envelope_byte == 2:
-        envelope_size = 48
-    elif envelope_byte == 3:
-        envelope_size = 48
-    elif envelope_byte == 4:
-        envelope_size = 64
-
-    return no_envelope_header_size + envelope_size
-
-
-def gpkg_wkb_to_wkt(gpkg_wkb):
-    """Convert WKB (with gpkg header) to WKT"""
-    wkb_header_length = parse_gpkgb_header_size(gpkg_wkb)
-    wkb_geom = gpkg_wkb[wkb_header_length:]
+def wkb2wkt(wkb_geom: bytes) -> str | None:
+    """Convert WKB to WKT"""
     try:
         wkt = wkb.loads(wkb_geom).wkt
     except ShapelyError:

server/mergin/sync/utils.py

@@ -12,18 +12,18 @@
 import os
 from unittest.mock import patch
 from pathvalidate import sanitize_filename
+from pygeodiff import GeoDiff
 
 from ..utils import save_diagnostic_log_file
 
 from ..sync.utils import (
-    parse_gpkgb_header_size,
-    gpkg_wkb_to_wkt,
     is_reserved_word,
     has_valid_characters,
     has_valid_first_character,
     check_filename,
     is_valid_path,
     get_x_accel_uri,
+    wkb2wkt,
 )
 from ..auth.models import LoginHistory, User
 from . import json_headers
@@ -86,30 +86,28 @@ def test_maintenance_mode(client):
 
 
 def test_parse_gpkg():
+    geodiff = GeoDiff()
     # Point
     gpkg_wkb = base64.b64decode(
         "R1AAAeYQAAABAQAAAID8bic0LLE/RlTr7Iuo1j8=", validate=True
     )
-    header_len = parse_gpkgb_header_size(gpkg_wkb)
-    assert header_len == 8
-    wkt = gpkg_wkb_to_wkt(gpkg_wkb)
+    wkb = geodiff.create_wkb_from_gpkg_header(gpkg_wkb)
+    wkt = wkb2wkt(wkb)
     assert "POINT" in wkt
 
     # Linestring
     gpkg_wkb = base64.b64decode(
         "R1AAA+YQAABA0VaD5wjkv+R3O6FhidC/hn8DkdjL0z8Iwc3FqhvlPwECAAAABAAAAGhDvPBvINK/CMHNxaob5T9I139WMEjZv+DthIYlQuA/5Hc7oWGJ0L+GfwOR2MvTP0DRVoPnCOS/zB9sPJo/2D8=",
         validate=True,
     )
-    header_len = parse_gpkgb_header_size(gpkg_wkb)
-    assert header_len == 40
-    wkt = gpkg_wkb_to_wkt(gpkg_wkb)
+
+    wkb = geodiff.create_wkb_from_gpkg_header(gpkg_wkb)
+    wkt = wkb2wkt(wkb)
     assert "LINESTRING" in wkt
 
     # Invalid
     gpkg_wkb = base64.b64decode("aaaa", validate=True)
-    header_len = parse_gpkgb_header_size(gpkg_wkb)
-    assert header_len == -1
-    wkt = gpkg_wkb_to_wkt(gpkg_wkb)
+    wkt = wkb2wkt(gpkg_wkb)
     assert not wkt