Introduce multiple file upload app enhancements

.github/workflows/main.yml

@@ -0,0 +1,29 @@
+name: Documentation deployment workflow
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**/*
+  pull_request:
+    branches:
+      - main
+    paths:
+      - docs/**/*
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install curl
+        run: sudo apt-get update && sudo apt-get install -y curl
+
+      - name: Deploying the docs
+        run: |
+          echo "Deploying the docs"
+          curl --request POST ${{ secrets.ACINT_URL }} -H "Content-Type: application/json" -d "{\"action\": \"${{ secrets.ACINT_ACTION }}\", \"token\": \"${{ secrets.ACINT_TOKEN }}\"}"

.vscode/settings.json

@@ -10,7 +10,7 @@
     "editor.defaultFormatter": "ms-python.black-formatter",
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
-      "source.organizeImports": false
+      "source.organizeImports": "never"
     }
   },
   "python.testing.pytestArgs": ["api/app"],

api/django-common/django_common/models.py

@@ -22,7 +22,8 @@ class OwnedModel(models.Model):
     A model that has a relationship to the owner in the user model.
     """
 
-    owner = models.ForeignKey(get_user_model(), on_delete=models.RESTRICT)
+    # Null is allowed for the case that the owner is anonymous.
+    owner = models.ForeignKey(get_user_model(), on_delete=models.RESTRICT, null=True, blank=True)
 
     class Meta:
         abstract = True

api/django-fileupload/django_fileupload/admin.py

@@ -9,21 +9,29 @@
 
 
 class FileUploadAdmin(admin.ModelAdmin):
-    list_display = ("name", "uploaded_by", "uploaded_on", "detected_mime_type", "hr_size", "checksum")
-    readonly_fields = ("file_upload_batch", "position", "file", "detected_mime_type", "checksum")
+    list_display = ("name", "uploaded_by", "uploaded_on", "deleted_on", "detected_mime_type", "hr_size", "checksum")
+    readonly_fields = ("deleted_on", "file_upload_batch", "position", "file", "detected_mime_type", "checksum")
 
     def hr_size(self, file_upload: FileUpload):
         return hr_size(file_upload.size)
 
     hr_size.short_description = "Size"
 
     def uploaded_on(self, file_upload: FileUpload):
-        return dt.strftime(file_upload.batch.uploaded_on, DATE_TIME_FORMAT)
+        return dt.strftime(file_upload.file_upload_batch.uploaded_on, DATE_TIME_FORMAT)
 
     uploaded_on.short_description = "Uploaded on"
 
+    def deleted_on(self, file_upload: FileUpload):
+        if file_upload.deleted_on:
+            return dt.strftime(file_upload.deleted_on, DATE_TIME_FORMAT)
+        else:
+            return "Not deleted"
+
+    deleted_on.short_description = "Deleted on"
+
     def uploaded_by(self, file_upload: FileUpload):
-        return file_upload.batch.owner
+        return file_upload.file_upload_batch.owner
 
     uploaded_by.short_description = "Uploaded by"
 

api/django-fileupload/django_fileupload/apps.py

@@ -3,3 +3,4 @@
 
 class CoreConfig(AppConfig):
     name = "django_fileupload"
+    verbose_name = "File Uploads"

api/django-fileupload/django_fileupload/migrations/0007_alter_fileuploadbatch_owner.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.0 on 2024-07-02 13:16
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("django_fileupload", "0006_rename_mime_type_fileupload_detected_mime_type"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="fileuploadbatch",
+            name="owner",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.RESTRICT,
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

api/django-fileupload/django_fileupload/migrations/0008_fileupload_deleted_on.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2024-07-04 19:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("django_fileupload", "0007_alter_fileuploadbatch_owner"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="fileupload",
+            name="deleted_on",
+            field=models.DateTimeField(editable=False, null=True),
+        ),
+    ]

api/django-fileupload/django_fileupload/models.py

@@ -23,9 +23,12 @@ def __str__(self):
 
 
 class FileUploadFileStorage(FileSystemStorage):
-
     def get_alternative_name(self, file_root, file_ext):
-        raise FileExistsError
+        # raise FileExistsError
+        index = 1
+        while self.exists(f"{file_root}_{index}{file_ext}"):
+            index += 1
+        return f"{file_root}_{index}{file_ext}"
 
 
 def generate_file_path(instance):
@@ -50,6 +53,7 @@ class FileUpload(models.Model):
     file = models.FileField(upload_to=_generate_complete_file_path, storage=FileUploadFileStorage())
     detected_mime_type = models.CharField(max_length=100, editable=False)
     checksum = models.CharField(max_length=64, editable=False)
+    deleted_on = models.DateTimeField(null=True, editable=False)
 
     def __str__(self):
         return self.file.path

api/django-fileupload/django_fileupload/serializers.py

@@ -10,7 +10,7 @@ class FileUploadSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = FileUpload
-        fields = ("id", "name", "checksum")
+        fields = ("id", "name", "checksum", "deleted_on")
 
 
 class FileUploadBatchSerializer(serializers.ModelSerializer):

api/django-fileupload/django_fileupload/views.py

@@ -9,6 +9,7 @@
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from rest_framework.serializers import ValidationError
+from django.utils import timezone
 
 from django_common.postgresql import exclusive_insert_table_lock
 from django_common.renderers import PassthroughRenderer
@@ -26,6 +27,10 @@ class FileUploadBatchViewSet(
     queryset = FileUploadBatch.objects.all()
     serializer_class = FileUploadBatchSerializer
     parser_classes = (MultiPartParser,)
+    has_owner = True
+
+    def get_max_file_size(self, request):
+        return None
 
     # Workaround for "drf-yasg" (see https://github.com/axnsan12/drf-yasg/issues/503).
     def get_serializer_class(self):
@@ -50,6 +55,10 @@ def create(self, request, *args, **kwargs):
             files = request.FILES.getlist("files")
             if self.verify_file_count(request, len(files)):
                 for file_position, file in enumerate(files):
+
+                    if self.get_max_file_size(request) and file.size > self.get_max_file_size(request):
+                        raise ValidationError(_(f"File size exceeds the maximum allowed size of {self.get_max_file_size(request) / (1024 ** 2)} MB."))
+
                     file_name_parts = os.path.splitext(file.name)
                     if self.verify_file_extension(request, file_position, file_name_parts):
                         if self.verify_file_checksum(request, file_position, file_name_parts,
@@ -59,7 +68,10 @@ def create(self, request, *args, **kwargs):
                     raise ValidationError(_("Files with incorrect extension in the request."))
                 response = []
                 with exclusive_insert_table_lock(FileUploadBatch):
-                    file_upload_batch = FileUploadBatch.objects.create(owner=request.user)
+                    if self.has_owner:
+                        file_upload_batch = FileUploadBatch.objects.create(owner=request.user)
+                    else:
+                        file_upload_batch = FileUploadBatch.objects.create()
                     # Metadata needs to be added here as FileUpload.objects.create(...) may depend on it.
                     self.add_metadata(request, file_upload_batch)
                     for file_position, file in enumerate(files):
@@ -96,4 +108,16 @@ class FileUploadViewSet(
     mixins.DestroyModelMixin,
     viewsets.GenericViewSet,
 ):
+    def keep_after_deletion(self):
+        return False
+
+    def destroy(self, request, *args, **kwargs):
+        file_upload = self.get_object()
+        if self.keep_after_deletion():
+            file_upload.deleted_on = timezone.now()
+            file_upload.save()
+        else:
+            file_upload.delete()
+
+        return Response(status=status.HTTP_204_NO_CONTENT)
     pass

docker-compose.yml

@@ -0,0 +1,9 @@
+services:
+  mkdocs:
+    build:
+      context: docs
+      dockerfile: Dockerfile
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./docs/:/docs/:z

docs/Dockerfile

@@ -0,0 +1,4 @@
+FROM squidfunk/mkdocs-material
+WORKDIR /docs
+COPY ./mkdocs.yml /docs/mkdocs.yml
+RUN pip install mkdocs-glightbox mkdocs-awesome-pages-plugin

docs/docs/.pages

@@ -0,0 +1,5 @@
+nav:
+    - index.md
+    - setup.md
+    - django_apps
+    - vue_components

docs/docs/django_apps/.pages

@@ -0,0 +1,4 @@
+nav:
+    - index.md
+    - file_upload.md
+    - notifications.md

docs/docs/django_apps/file_upload.md

@@ -0,0 +1,101 @@
+---
+title: File Upload App
+---
+
+### Features
+The file upload app enables you to easily handle file uploads in your application. Whenever one or more files are uploaded using the file upload app a single batch entry is created and a set of file entries are created in their respective models.
+
+The batches keep information about the `owner`, `upload date` and `references to the associated files`. The files, on the other hand, keep information about the `deletion date`, `referece to the file upload batch`, `position in the batch`, `file location`, `file type` and a `checksum`.
+
+Uploading multiple files with the same name will cause the latest file with that name to be retrieved when retrieving files. Deleting the file will only delete the latest version of the file but the previous versions will be kept and will also be returned. Custom implementations are required to enable a different behavior (for example deleting files by name).
+
+#### Anonymous Owners
+In cases where an owner is not needed you simply extend the FileUploadBatchViewSet and set the `has_owner` property to `False`. This will result in the owner value being set to `None` in the database entries.
+
+```python title="views.py"
+from django_fileupload.views import FileUploadBatchViewSet
+
+class CustomFileUploadBatchViewSet(FileUploadBatchViewSet):
+    has_owner = True
+```
+
+!!! note
+
+    In this case you also have to use your newly defined view class in your route definitions.
+
+
+#### Max File Size
+There might be cases where you want to provide a max file upload size per upload request. In such cases you can take a similar approach to the one above but instead of setting a flag you can implement the `get_max_file_size` method.
+
+```python title="views.py"
+from django_fileupload.views import FileUploadBatchViewSet
+
+class CustomFileUploadBatchViewSet(FileUploadBatchViewSet):
+    def get_max_file_size(self, request):
+
+        # Extract data from the request object or do some other calculations
+
+        if some_condition:
+            # The max_file_size is in megabytes
+            # The get_max_file_size should return bytes or None
+            return max_file_size * 1024 * 1024
+        else:
+            return None
+```
+
+!!! warning "Large file protection"
+
+    If a user uploads a very large file, our application may get overwhelmed while checking its size (as it would attempt to load it onto the file system before reading the size). Therefore, we need to introduce additional measures as described below.
+
+Introduce custom file upload handlers by adding them to your `settings.py` using the following snippet:
+
+```python title="settings.py"
+FILE_UPLOAD_HANDLERS = [
+    "django_common.uploadhandlers.HardLimitMemoryFileUploadHandler",
+    "django_common.uploadhandlers.HardLimitTemporaryFileUploadHandler",
+]
+```
+
+Furthermore, you must set the following argument and environment variable in your backend `Dockerfile`:
+
+```dockerfile title="api/Dockerfile"
+ARG HARD_UPLOAD_SIZE_LIMIT=524288000
+ENV DJANGO_HARD_UPLOAD_SIZE_LIMIT $HARD_UPLOAD_SIZE_LIMIT
+```
+This will ensure that there is a global django upper size limit that will prevent users from uploading files larger than it.
+
+#### Keeping Files After Deletion
+To keep files after deletion we must overwrite the method `keep_after_deletion` of the `FileUploadViewSet` class:
+
+```python title="views.py"
+from django_fileupload.views import FileUploadViewSet
+
+class CustomFileUploadViewSet(FileUploadViewSet):
+    def keep_after_deletion(self):
+        return True
+```
+!!! info "Listing only files that are not marked as deleted"
+
+    To retrieve only files that have not been marked as deleted you need to provide custom logic in the list method of the FileUploadViewSet class.
+
+
+#### Other Customizations
+Other methods of the `FileUploadBatchViewSet` class and the `FileUploadViewSet` class may be overwriten or extended in order to provide more custom behavior and also for providing additional metadata.
+
+### Setup
+1. Add the following two apps under INSTALLED_APPS in settings.py:
+```python
+"django_common",
+"django_fileupload",
+```
+2. The following three packages need to be added to the `requirements.txt` of your django project:
+```text
+-e /nexus-app-stack-contrib/cli/python-utilities
+-e /nexus-app-stack-contrib/api/django-common
+-e /nexus-app-stack-contrib/api/django-fileupload
+```
+3. Make sure to setup the `FileUploadBatchViewSet` and `FileUploadViewSet` class endpoints by importing their routers / urls and defining them in your own routes.
+
+!!! info "Use Swagger"
+
+    To help you with your debugging process and to enable you to better understand what are the available methods and routes use swagger in your project.

docs/docs/django_apps/index.md

@@ -0,0 +1,5 @@
+---
+title: "Django Apps"
+---
+
+These are helpful django apps that can be included in your project. Generally the `django_common` app is used by the others and should therfore be always included.

docs/docs/django_apps/notifications.md

@@ -0,0 +1,10 @@
+---
+title: Notifications App
+---
+
+### Features
+Describe all the features as well as the class flags
+### Setup
+Describe any file upload app specific config (for example what are the dependencies and what apps need to be added where to make it run)
+### Customizations
+Describe how to customize urls, models etc.

docs/docs/index.md

@@ -0,0 +1,7 @@
+---
+title: App Stack Contrib
+---
+
+# App Stack Contrib 🧰
+The NEXUS App Stack Contrib project provides a versatile set of django apps, vue components and cli tools that can speed up the development of new projects. Furthermore, it also provides a standardized approach to problems such as file uploads in django.
+