Merge branch 'organization-collective' into staging

Author: mitchelljkotler

.github/workflows/lambda.yml

@@ -0,0 +1,24 @@
+name: Post-Deploy Lambda
+
+on:
+  deployment:
+
+jobs:
+  deploy-lambdas:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Show deployment info
+        run: |
+          echo "Deployment environment: $DEPLOYMENT_ENVIRONMENT"
+
+      - name: Run Lambda deploy
+        run: |
+          if [[ "$DEPLOYMENT_ENVIRONMENT" == "documentcloud-staging" ]]; then
+            echo "Deploying staging lambda updates"
+            bash config/aws/lambda/codeship_deploy_lambdas.sh staging-lambda --staging
+          else
+            echo "Deploying production lambda updates"
+            bash config/aws/lambda/codeship_deploy_lambdas.sh prod-lambda
+          fi

.gitignore

@@ -300,4 +300,7 @@ embedding/
 # solr
 !config/solr/lib/*
 
-scratch/
+scratch/
+
+CLAUDE.md
+.claude

Procfile

@@ -1,4 +1,5 @@
+release: ./postdeploy.sh
 web: bin/start-nginx gunicorn -c config/gunicorn.conf config.wsgi:application
 worker: REMAP_SIGTERM=SIGQUIT celery --app=config.celery_app worker --loglevel=info
 solr_worker: REMAP_SIGTERM=SIGQUIT celery --app=config.celery_app worker --loglevel=info -Q solr,celery
-beat: REMAP_SIGTERM=SIGQUIT celery --app=config.celery_app beat --loglevel=info
+beat: REMAP_SIGTERM=SIGQUIT celery --app=config.celery_app beat --loglevel=info

config/settings/base.py

@@ -290,6 +290,9 @@
 ADMINS = [("Mitchell Kotler", "mitch@muckrock.com")]
 # https://docs.djangoproject.com/en/dev/ref/settings/#managers
 MANAGERS = ADMINS
+# Chunk size for CSV exports using .iterator() to process large querysets
+# without loading all records into memory at once
+CSV_EXPORT_CHUNK_SIZE = env.int("CSV_EXPORT_CHUNK_SIZE", default=2000)
 
 # LOGGING
 # ------------------------------------------------------------------------------
@@ -389,8 +392,14 @@
         "task": "documentcloud.addons.tasks.dispatch_events",
         "schedule": crontab(minute="*/5"),
     },
+    "permission_digest": {
+        "task": "documentcloud.users.tasks.permission_digest",
+        "schedule": crontab(day_of_week="mon", hour=7, minute=0),
+    },
 }
 
+PERMISSIONS_DIGEST_EMAILS = env.list("PERMISSIONS_DIGEST_EMAILS", default=[])
+
 # django-compressor
 # ------------------------------------------------------------------------------
 # https://django-compressor.readthedocs.io/en/latest/quickstart/#installation
@@ -614,6 +623,7 @@
 CLOUDFLARE_API_EMAIL = env("CLOUDFLARE_API_EMAIL", default="")
 CLOUDFLARE_API_KEY = env("CLOUDFLARE_API_KEY", default="")
 CLOUDFLARE_API_ZONE = env("CLOUDFLARE_API_ZONE", default="")
+CLOUDFLARE_HOSTS = env.list("CLOUDFLARE_HOSTS", default=[])
 
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 

config/settings/production.py

@@ -4,7 +4,6 @@
 
 # Third Party
 import sentry_sdk
-from ipware import get_client_ip
 from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
@@ -74,6 +73,11 @@
 # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings
 AWS_S3_REGION_NAME = env("DJANGO_AWS_S3_REGION_NAME", default=None)
 
+CLOUDFLARE_HOSTS = env.list(
+    "CLOUDFLARE_HOSTS",
+    default=["https://www.documentcloud.org", "https://embed.documentcloud.org"],
+)
+
 # STATIC
 # ------------------------
 STATICFILES_STORAGE = "config.settings.production.StaticRootS3Boto3Storage"

documentcloud/addons/admin.py

@@ -1,6 +1,7 @@
 # Django
+from django.conf import settings
 from django.contrib import admin, messages
-from django.db.models import JSONField
+from django.db.models import JSONField, Q
 from django.forms import widgets
 from django.http import HttpResponse
 from django.http.response import HttpResponseRedirect
@@ -124,11 +125,27 @@ class AddOnRunAdmin(admin.ModelAdmin):
     actions = ["export_runs_as_csv"]
     readonly_fields = [f.name for f in AddOnRun._meta.fields]
 
+    def get_search_results(self, request, queryset, search_term):
+        """Avoid collation issue when searching add-on runs"""
+        if search_term:
+            queryset = queryset.filter(
+                Q(addon__name__icontains=search_term)
+                | Q(user__email__icontains=search_term)
+                | Q(status__icontains=search_term)
+            )
+            use_distinct = False
+        else:
+            use_distinct = False
+        return queryset, use_distinct
+
     def export_runs_as_csv(self, request, queryset):
         """Export selected Add-On Runs to CSV."""
         field_names = [
             "addon_id",
+            "addon_name",
             "user_id",
+            "user_name",
+            "user_email",
             "run_id",
             "status",
             "rating",
@@ -143,11 +160,28 @@ def export_runs_as_csv(self, request, queryset):
         writer = csv.writer(response)
         writer.writerow(field_names)
 
-        for run in queryset:
+        limited_queryset = queryset.select_related("addon", "user").only(
+            "addon_id",
+            "user_id",
+            "run_id",
+            "status",
+            "rating",
+            "credits_spent",
+            "created_at",
+            "updated_at",
+            "addon__name",
+            "user__name",
+            "user__email",
+        )
+
+        for run in limited_queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE):
             writer.writerow(
                 [
                     run.addon_id,
+                    run.addon.name,
                     run.user_id,
+                    run.user.name,
+                    run.user.email,
                     run.run_id,
                     run.status,
                     run.rating,

documentcloud/documents/models/document.py

@@ -702,9 +702,14 @@ def page_filter(text):
         return solr_document
 
     def invalidate_cache(self):
-        """Invalidate public CDN cache for this document's underlying file"""
+        """
+        Invalidate public CDN cache for this document's underlying file,
+        plus frontend URLs in Cloudflare
+        """
         logger.info("Invalidating cache for %s", self.pk)
         doc_path = self.doc_path[self.doc_path.index("/") :]
+
+        # cloudfront
         distribution_id = settings.CLOUDFRONT_DISTRIBUTION_ID
         if distribution_id:
             # we want the doc path without the s3 bucket name
@@ -716,15 +721,25 @@ def invalidate_cache(self):
                     "CallerReference": str(uuid.uuid4()),
                 },
             )
+
+        # cloudflare
         cloudflare_email = settings.CLOUDFLARE_API_EMAIL
         cloudflare_key = settings.CLOUDFLARE_API_KEY
         cloudflare_zone = settings.CLOUDFLARE_API_ZONE
-        url = settings.PUBLIC_ASSET_URL + doc_path[1:]
+        asset_url = settings.PUBLIC_ASSET_URL + doc_path[1:]
+
+        if self.access == Access.public:
+            public_urls = [
+                host + self.get_absolute_url() for host in settings.CLOUDFLARE_HOSTS
+            ] + [asset_url]
+        else:
+            public_urls = [asset_url]
+
         if cloudflare_zone:
             requests.post(
                 "https://api.cloudflare.com/client/v4/zones/"
                 f"{cloudflare_zone}/purge_cache",
-                json={"files": [url]},
+                json={"files": public_urls},
                 headers={
                     "X-Auth-Email": cloudflare_email,
                     "X-Auth-Key": cloudflare_key,

documentcloud/documents/oembed.py

@@ -166,7 +166,7 @@ def response(self, request, query, max_width=None, max_height=None, **kwargs):
         context = {
             "src": src,
             "title": note.title,
-            "style": self.get_style(document, note, max_width, max_height),
+            "style": self.get_style(max_width, max_height),
             "width": note_width,
             "height": note_height,
             "resize_script": RESIZE_SCRIPT,
@@ -183,13 +183,11 @@ def get_dimensions(self, document, note):
 
         return (note_width, note_height)
 
-    def get_style(self, document, note, max_width=None, max_height=None):
-
-        note_width, note_height = self.get_dimensions(document, note)
+    def get_style(self, max_width=None, max_height=None):
 
         style = (
-            f"border: 1px solid #d8dee2; border-radius: 0.5rem; width: 100%;"
-            f" height: 100%; aspect-ratio: {note_width} / {note_height};"
+            "border: 1px solid #d8dee2; border-radius: 0.5rem;"
+            " width: 100%; height: 300px;"
         )
         if max_width:
             style += f" max-width: {max_width}px;"

documentcloud/documents/tests/test_oembed.py

@@ -354,9 +354,6 @@ def test_note_oembed_response(self):
         response = self.note_oembed.response(
             request, query, max_width=600, max_height=None, doc_pk=123, pk=456
         )
-        note_width, note_height = self.note_oembed.get_dimensions(
-            self.document, self.note
-        )
 
         # Check response properties
         assert response["version"] == "1.0"
@@ -370,10 +367,9 @@ def test_note_oembed_response(self):
             "?embed=1&responsive=1"
         ) in response["html"]
         assert 'title="Test Note (Hosted by DocumentCloud)"' in response["html"]
-        assert f'width="{note_width}" height="{note_height}"' in response["html"]
         assert (
-            f"border: 1px solid #d8dee2; border-radius: 0.5rem; width: 100%;"
-            f" height: 100%; aspect-ratio: {note_width} / {note_height};"
+            "border: 1px solid #d8dee2; border-radius: 0.5rem;"
+            " width: 100%; height: 300px;"
         ) in response["html"]
         assert 'sandbox="allow-scripts allow-same-origin' in response["html"]
 

documentcloud/organizations/admin.py

@@ -1,4 +1,5 @@
 # Django
+from django.conf import settings
 from django.contrib import admin
 from django.http import HttpResponse
 
@@ -41,7 +42,7 @@ def export_ai_credit_logs(self, request, queryset):
         writer = csv.writer(response)
         writer.writerow(field_names)
 
-        for log in queryset:
+        for log in queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE):
             writer.writerow(
                 [
                     str(log.organization),