Merge pull request #542 from MerginMaps/develop

Release 2025.8.2

Author: MarcelGeo

server/.test.env

@@ -24,3 +24,4 @@ SECURITY_BEARER_SALT='bearer'
 SECURITY_EMAIL_SALT='email'
 SECURITY_PASSWORD_SALT='password'
 DIAGNOSTIC_LOGS_DIR=/tmp/diagnostic_logs
+GEVENT_WORKER=0

server/mergin/auth/forms.py

@@ -1,6 +1,7 @@
 # Copyright (C) Lutra Consulting Limited
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+
 import re
 import safe
 from flask_wtf import FlaskForm
@@ -48,18 +49,22 @@ class ExtendedEmail(Email):
     1. spaces,
     2. special characters ,:;()<>[]\"
     3, multiple @ symbols,
-    4, leading, trailing, or consecutive dots in the local part
-    5, invalid domain part - missing top level domain (user@example), consecutive dots
-    Custom check for additional invalid characters disallows |'— because they make our email sending service to fail
+    4, leading, trailing, or consecutive dots in the local part,
+    5, invalid domain part - missing top level domain (user@example), consecutive dots,
+    The extended validation checks email addresses using the regex provided by Brevo,
+    so that we stay consistent with their validation rules and avoid API failures.
     """
 
     def __call__(self, form, field):
         super().__call__(form, field)
 
-        if re.search(r"[|'—]", field.data):
-            raise ValidationError(
-                f"Email address '{field.data}' contains an invalid character."
-            )
+        email = field.data.strip()
+
+        pattern = r"^[\x60#&*\/=?^{!}~'+\w-]+(\.[\x60#&*\/=?^{!}~'+\w-]+)*\.?@([_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*\.)[a-zA-Z0-9-]*[a-zA-Z0-9]{2,}$"
+        email_regexp = re.compile(pattern, re.IGNORECASE)
+
+        if not email_regexp.match(email):
+            raise ValidationError(f"Email address '{email}' is invalid.")
 
 
 class PasswordValidator:

server/mergin/sync/files.py

@@ -13,10 +13,10 @@
 
 from .utils import (
     is_file_name_blacklisted,
-    is_qgis,
     is_supported_extension,
     is_valid_path,
     is_versioned_file,
+    has_trailing_space,
 )
 from ..app import DateTimeWithZ, ma
 
@@ -212,14 +212,21 @@ def validate(self, data, **kwargs):
 
             if not is_valid_path(file_path):
                 raise ValidationError(
-                    f"Unsupported file name detected: {file_path}. Please remove the invalid characters."
+                    f"Unsupported file name detected: '{file_path}'. Please remove the invalid characters."
                 )
 
             if not is_supported_extension(file_path):
                 raise ValidationError(
-                    f"Unsupported file type detected: {file_path}. "
+                    f"Unsupported file type detected: '{file_path}'. "
                     f"Please remove the file or try compressing it into a ZIP file before uploading.",
                 )
+        # new checks must restrict only new files not to block existing projects
+        for file in data["added"]:
+            file_path = file["path"]
+            if has_trailing_space(file_path):
+                raise ValidationError(
+                    f"Folder name contains a trailing space. Please remove the space in: '{file_path}'."
+                )
 
 
 class ProjectFileSchema(FileSchema):
@@ -230,5 +237,5 @@ class ProjectFileSchema(FileSchema):
     def patch_field(self, data, **kwargs):
         # drop 'diff' key entirely if empty or None as clients would expect
         if not data.get("diff"):
-            data.pop("diff")
+            data.pop("diff", None)
         return data

server/mergin/sync/permissions.py

@@ -209,7 +209,21 @@ def require_project(ws, project_name, permission) -> Project:
     return project
 
 
-def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled=False):
+def require_project_by_uuid(
+    uuid: str, permission: ProjectPermissions, scheduled=False, expose=True
+) -> Project:
+    """
+    Retrieves a project by UUID after validating existence, workspace status, and permissions.
+
+    Args:
+        uuid (str): The unique identifier of the project.
+        permission (ProjectPermissions): The permission level required to access the project.
+        scheduled (bool, optional): If ``True``, bypasses the check for projects marked for deletion.
+        expose (bool, optional): Controls security disclosure behavior on permission failure.
+            - If `True`: Returns 403 Forbidden (reveals project exists but access is denied).
+            - If `False`: Returns 404 Not Found (hides project existence for security).
+            Standard is that reading results in 404, while writing results in 403
+    """
     if not is_valid_uuid(uuid):
         abort(404)
 
@@ -219,13 +233,18 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled
     if not scheduled:
         project = project.filter(Project.removed_at.is_(None))
     project = project.first_or_404()
+    if not expose and current_user.is_anonymous and not project.public:
+        # we don't want to tell anonymous user if a private project exists
+        abort(404)
+
     workspace = project.workspace
     if not workspace:
         abort(404)
     if not is_active_workspace(workspace):
         abort(404, "Workspace doesn't exist")
     if not permission.check(project, current_user):
         abort(403, "You do not have permissions for this project")
+
     return project
 
 

server/mergin/sync/public_api_controller.py

@@ -55,9 +55,7 @@
 from .files import (
     ProjectFileChange,
     ChangesSchema,
-    UploadFileSchema,
     ProjectFileSchema,
-    FileSchema,
     files_changes_from_upload,
     mergin_secure_filename,
 )
@@ -83,17 +81,11 @@
     generate_checksum,
     Toucher,
     get_x_accel_uri,
-    is_file_name_blacklisted,
     get_ip,
     get_user_agent,
     generate_location,
     is_valid_uuid,
-    is_versioned_file,
-    get_project_path,
     get_device_id,
-    is_valid_path,
-    is_supported_type,
-    is_supported_extension,
     get_mimetype,
     wkb2wkt,
 )
@@ -980,7 +972,7 @@ def push_finish(transaction_id):
         if len(unsupported_files):
             abort(
                 400,
-                f"Unsupported file type detected: {unsupported_files[0]}. "
+                f"Unsupported file type detected: '{unsupported_files[0]}'. "
                 f"Please remove the file or try compressing it into a ZIP file before uploading.",
             )
 
@@ -1036,14 +1028,6 @@ def push_finish(transaction_id):
         # let's move uploaded files where they are expected to be
         os.renames(files_dir, version_dir)
 
-        # remove used chunks
-        for file in upload.changes["added"] + upload.changes["updated"]:
-            file_chunks = file.get("chunks", [])
-            for chunk_id in file_chunks:
-                chunk_file = os.path.join(upload.upload_dir, "chunks", chunk_id)
-                if os.path.exists(chunk_file):
-                    move_to_tmp(chunk_file)
-
         logging.info(
             f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}."
         )

server/mergin/sync/public_api_v2.yaml

@@ -76,6 +76,34 @@ paths:
         "409":
           $ref: "#/components/responses/Conflict"
       x-openapi-router-controller: mergin.sync.public_api_v2_controller
+    get:
+      tags:
+        - project
+      summary: Get project info
+      operationId: get_project
+      parameters:
+        - name: files_at_version
+          in: query
+          description: Include list of files at specific version
+          required: false
+          schema:
+            $ref: "#/components/schemas/VersionName"
+      responses:
+        "200":
+          description: Success
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ProjectDetail"
+        "400":
+          $ref: "#/components/responses/BadRequest"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          $ref: "#/components/responses/Forbidden"
+        "404":
+          $ref: "#/components/responses/NotFound"
+      x-openapi-router-controller: mergin.sync.public_api_v2_controller
   /projects/{id}/scheduleDelete:
     post:
       tags:
@@ -276,9 +304,7 @@ paths:
                   default: false
                   example: true
                 version:
-                  type: string
-                  pattern: '^$|^v\d+$'
-                  example: v2
+                  $ref: "#/components/schemas/VersionName"
                 changes:
                   type: object
                   required:
@@ -502,6 +528,72 @@ components:
           $ref: "#/components/schemas/ProjectRole"
         role:
           $ref: "#/components/schemas/Role"
+    ProjectDetail:
+      type: object
+      required:
+        - id
+        - name
+        - workspace
+        - role
+        - version
+        - created_at
+        - updated_at
+        - public
+        - size
+      properties:
+        id:
+          type: string
+          description: project uuid
+          example: c1ae6439-0056-42df-a06d-79cc430dd7df
+        name:
+          type: string
+          example: survey
+        workspace:
+          type: object
+          properties:
+            id:
+              type: integer
+              example: 123
+            name:
+              type: string
+              example: mergin
+        role:
+          $ref: "#/components/schemas/ProjectRole"
+        version:
+          type: string
+          description: latest project version
+          example: v2
+        created_at:
+          type: string
+          format: date-time
+          description: project creation timestamp
+          example: 2025-10-24T08:27:56Z
+        updated_at:
+          type: string
+          format: date-time
+          description: last project update timestamp
+          example: 2025-10-24T08:28:00.279699Z
+        public:
+          type: boolean
+          description: whether the project is public
+          example: false
+        size:
+          type: integer
+          description: project size in bytes for this version
+          example: 17092380
+        files:
+          type: array
+          description: List of files in the project
+          items:
+            allOf:
+              - $ref: '#/components/schemas/File'
+              - type: object
+                properties:
+                  mtime:
+                    type: string
+                    format: date-time
+                    description: File modification timestamp
+                    example: 2024-11-19T13:50:00Z
     File:
       type: object
       description: Project file metadata
@@ -754,3 +846,7 @@ components:
             - editor
             - writer
             - owner
+    VersionName:
+      type: string
+      pattern: '^$|^v\d+$'
+      example: v2

server/mergin/sync/public_api_v2_controller.py

@@ -14,6 +14,9 @@
 from marshmallow import ValidationError
 from sqlalchemy.exc import IntegrityError
 
+from mergin.sync.tasks import remove_transaction_chunks
+
+from .schemas_v2 import ProjectSchema as ProjectSchemaV2
 from ..app import db
 from ..auth import auth_required
 from ..auth.models import User
@@ -26,7 +29,7 @@
     StorageLimitHit,
     UploadError,
 )
-from .files import ChangesSchema
+from .files import ChangesSchema, ProjectFileSchema
 from .forms import project_name_validation
 from .models import (
     Project,
@@ -41,7 +44,6 @@
 from .public_api_controller import catch_sync_failure
 from .schemas import (
     ProjectMemberSchema,
-    ProjectVersionSchema,
     UploadChunkSchema,
     ProjectSchema,
 )
@@ -162,6 +164,22 @@ def remove_project_collaborator(id, user_id):
     return NoContent, 204
 
 
+def get_project(id, files_at_version=None):
+    """Get project info. Include list of files at specific version if requested."""
+    project = require_project_by_uuid(id, ProjectPermissions.Read, expose=False)
+    data = ProjectSchemaV2().dump(project)
+    if files_at_version:
+        pv = ProjectVersion.query.filter_by(
+            project_id=project.id, name=ProjectVersion.from_v_name(files_at_version)
+        ).first()
+        if pv:
+            data["files"] = ProjectFileSchema(
+                only=("path", "mtime", "size", "checksum"), many=True
+            ).dump(pv.files)
+
+    return data, 200
+
+
 @auth_required
 @catch_sync_failure
 def create_project_version(id):
@@ -302,12 +320,12 @@ def create_project_version(id):
             os.renames(temp_files_dir, version_dir)
 
             # remove used chunks
+            # get chunks from added and updated files
+            chunks_ids = []
             for file in to_be_added_files + to_be_updated_files:
                 file_chunks = file.get("chunks", [])
-                for chunk_id in file_chunks:
-                    chunk_file = get_chunk_location(chunk_id)
-                    if os.path.exists(chunk_file):
-                        move_to_tmp(chunk_file)
+                chunks_ids.extend(file_chunks)
+            remove_transaction_chunks.delay(chunks_ids)
 
         logging.info(
             f"Push finished for project: {project.id}, project version: {v_next_version}, upload id: {upload.id}."
@@ -360,7 +378,6 @@ def upload_chunk(id: str):
         # we could have used request.data here, but it could eventually cause OOM issue
         save_to_file(request.stream, dest_file, current_app.config["MAX_CHUNK_SIZE"])
     except IOError:
-        move_to_tmp(dest_file, chunk_id)
         return BigChunkError().response(413)
     except Exception as e:
         return UploadError(error="Error saving chunk").response(400)