Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions .github/workflows/deploy-staging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,11 @@ jobs:
- '.github/workflows/deploy-staging.yml'
- '.github/workflows/test.yml'

# ====================== RUN TESTS ======================
run-tests:
name: Run Tests
needs: detect-changes
if: "!contains(github.event.head_commit.message, '[skip ci]')"
uses: ./.github/workflows/test.yml

# ====================== BUILD FRONTEND TEST ======================
build-frontend-test:
name: Build Frontend (Test)
runs-on: ubuntu-latest
needs: [detect-changes, run-tests]
needs: [detect-changes]
if: |
!contains(github.event.head_commit.message, '[skip ci]') &&
success() &&
Expand Down Expand Up @@ -105,7 +98,7 @@ jobs:
build-backend-test:
name: Build Backend (Test)
runs-on: ubuntu-latest
needs: [detect-changes, run-tests]
needs: [detect-changes]
if: |
!contains(github.event.head_commit.message, '[skip ci]') &&
success() &&
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-frontend-wrapper.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ fi

echo "→ Running TypeScript (type checking)..."
if command -v bun &> /dev/null; then
bun run tsc --noEmit || {
bun run tsc -b || {
echo "TypeScript errors found. Please fix before committing."
exit 1
}
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
[![Tests](https://github.com/dbca-wa/science-projects/actions/workflows/test.yml/badge.svg)](https://github.com/dbca-wa/science-projects/actions/workflows/test.yml)
![Frontend Coverage](https://img.shields.io/badge/frontend--coverage-66%25-orange)
![Backend Coverage](https://img.shields.io/badge/backend--coverage-83%25-green)
![WCAG 2.2 AA](https://img.shields.io/badge/WCAG%202.2-AA%20Compliant-green)


A project management and approval system for scientific research projects.
Expand Down Expand Up @@ -207,7 +208,7 @@ The CI/CD pipeline uses a modular approach with reusable workflows:

**test.yml** (reusable workflow):
- Called by deploy-staging.yml and deploy-prod.yml
- Frontend tests (2-way sharding, ~2 min)
- Frontend tests (2-way sharding, ~2 min) - includes accessibility tests
- Backend tests (4-way sharding, ~10 min)
- Coverage combining and validation
- Path-based execution (only test changed code)
Expand Down
2 changes: 1 addition & 1 deletion backend/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -184,4 +184,4 @@ EXPOSE 8000
# CMD curl -f http://127.0.0.1:8000/health/ || exit 1

# Launch production server
CMD ["gunicorn", "config.wsgi", "--bind", "0.0.0.0:8000", "--timeout", "300", "--graceful-timeout", "90", "--max-requests", "2048", "--workers", "4", "--preload"]
CMD ["gunicorn", "config.wsgi", "--bind", "0.0.0.0:8000", "--timeout", "300", "--graceful-timeout", "90", "--max-requests", "2048", "--workers", "4", "--preload", "--worker-tmp-dir", "/tmp"]
20 changes: 7 additions & 13 deletions backend/caretakers/management/commands/migrate_caretaker_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

from django.core.management.base import BaseCommand
from django.db import connection, transaction
from psycopg2 import sql


class Command(BaseCommand):
Expand Down Expand Up @@ -139,10 +138,9 @@ def _table_exists(self, table_name):
def _get_table_count(self, table_name):
"""Get the number of records in a table."""
with connection.cursor() as cursor:
# Use SQL identifier to prevent injection (table_name is hardcoded but bandit flags it)
cursor.execute(
sql.SQL("SELECT COUNT(*) FROM {}").format(sql.Identifier(table_name))
)
# Use string formatting with validated table name (table_name is hardcoded in callers)
# This is safe because table_name only comes from hardcoded strings in this file
cursor.execute(f"SELECT COUNT(*) FROM {table_name}") # nosec B608
return cursor.fetchone()[0]

def _migrate_data(self):
Expand All @@ -152,8 +150,7 @@ def _migrate_data(self):
"""
with connection.cursor() as cursor:
# Insert new records that don't exist in destination
cursor.execute(
"""
cursor.execute("""
INSERT INTO caretakers_caretaker
(id, user_id, caretaker_id, end_date, reason, notes, created_at, updated_at)
SELECT
Expand All @@ -172,13 +169,11 @@ def _migrate_data(self):
AND dest.caretaker_id = src.caretaker_id
)
ON CONFLICT (user_id, caretaker_id) DO NOTHING;
"""
)
""")
copied_count = cursor.rowcount

# Update existing records to ensure data is current
cursor.execute(
"""
cursor.execute("""
UPDATE caretakers_caretaker dest
SET
end_date = src.end_date,
Expand All @@ -196,8 +191,7 @@ def _migrate_data(self):
dest.created_at IS DISTINCT FROM src.created_at OR
dest.updated_at IS DISTINCT FROM src.updated_at
);
"""
)
""")
updated_count = cursor.rowcount

return copied_count, updated_count
125 changes: 106 additions & 19 deletions backend/config/cache_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,30 +6,117 @@
Note: Organisational Edge proxy (Nginx/Varnish/Fastly) handles HTTP-level caching.
This Redis cache is for application-level caching of user-specific data that cannot
be cached at the HTTP level.

Redis is optional:
- If REDIS_URL is set and Redis is reachable: Caching and throttling enabled
- If Redis unavailable: Caching and throttling disabled (graceful degradation)
- Tests always use dummy cache (no Redis required)
"""

import logging
import os

# Redis Cache Configuration
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": os.environ.get("REDIS_URL", "redis://127.0.0.1:6379/1"),
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"SOCKET_CONNECT_TIMEOUT": 5, # seconds
"SOCKET_TIMEOUT": 5, # seconds
"RETRY_ON_TIMEOUT": True,
"MAX_CONNECTIONS": 50,
"CONNECTION_POOL_KWARGS": {
"max_connections": 50,
"retry_on_timeout": True,
},
},
"KEY_PREFIX": "spms",
"TIMEOUT": 300, # 5 minutes default TTL
logger = logging.getLogger(__name__)

# Track Redis availability globally for other modules
REDIS_AVAILABLE = False


def get_cache_config():
"""
Get cache configuration - Redis if available, dummy cache otherwise.

Behavior:
- Tests: Always use dummy cache (no Redis needed)
- Production/Staging/Development: Use Redis if REDIS_URL set and connectable
- If Redis unavailable: Use dummy cache and log clearly

Sets global REDIS_AVAILABLE flag for other components (e.g., throttling).

Returns:
dict: Django CACHES configuration
"""
global REDIS_AVAILABLE

redis_url = os.environ.get("REDIS_URL")
is_testing = os.environ.get("PYTEST_RUNNING", "0") == "1"
environment = os.environ.get("ENVIRONMENT", "development")

# Always use dummy cache for tests
if is_testing:
logger.info("CACHE: Using dummy cache for tests (Redis not required)")
REDIS_AVAILABLE = False
return {
"default": {
"BACKEND": "django.core.cache.backends.dummy.DummyCache",
}
}

# Try Redis if URL is provided
if redis_url:
try:
# Test Redis connection with short timeout
import redis

client = redis.from_url(redis_url, socket_connect_timeout=2)
client.ping()
client.close()

logger.info(f"CACHE: Redis connected successfully ({redis_url})")
logger.info("CACHE: Caching ENABLED")
logger.info("CACHE: Rate limiting ENABLED")
REDIS_AVAILABLE = True

return {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": redis_url,
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"SOCKET_CONNECT_TIMEOUT": 5,
"SOCKET_TIMEOUT": 5,
"RETRY_ON_TIMEOUT": True,
"MAX_CONNECTIONS": 50,
"CONNECTION_POOL_KWARGS": {
"max_connections": 50,
"retry_on_timeout": True,
},
},
"KEY_PREFIX": "spms",
"TIMEOUT": 300, # 5 minutes default TTL
}
}
except Exception as e:
logger.warning(f"CACHE: Redis connection failed ({redis_url}): {e}")
logger.warning("CACHE: Caching DISABLED (Redis unavailable)")
logger.warning("CACHE: Rate limiting DISABLED (Redis unavailable)")
if environment in ["staging", "production"]:
logger.warning(
f"CACHE: Redis is recommended for {environment}. "
"See documentation for setup instructions."
)
REDIS_AVAILABLE = False
else:
logger.info("CACHE: REDIS_URL not set")
logger.info("CACHE: Caching DISABLED (no Redis configured)")
logger.info("CACHE: Rate limiting DISABLED (no Redis configured)")
if environment in ["staging", "production"]:
logger.info(
f"CACHE: Redis is recommended for {environment}. "
"See documentation for setup instructions."
)
REDIS_AVAILABLE = False

# Use dummy cache when Redis unavailable
return {
"default": {
"BACKEND": "django.core.cache.backends.dummy.DummyCache",
}
}
}


# Export cache configuration
CACHES = get_cache_config()

# Cache Key Patterns
# Use these patterns for consistent cache key generation across the application
Expand Down
83 changes: 46 additions & 37 deletions backend/config/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,14 +278,52 @@
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]

TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]

WSGI_APPLICATION = "config.wsgi.application"

# endregion ========================================================================================

# region Cache Configuration ======================================================
# Import cache keys, TTL values, and Redis availability flag
from config.cache_settings import ( # noqa: E402, F401
CACHE_KEYS,
CACHE_TTL,
CACHES,
REDIS_AVAILABLE,
)

# endregion ========================================================================================

# region Logs and Tracking =======================================================================
# Initialize logger early for use in configuration
LOGGER = logging.getLogger(__name__)

# endregion ========================================================================================

# region REST Framework Configuration =============================================
REST_FRAMEWORK = {
"DEFAULT_PERMISSION_CLASSES": [
"rest_framework.permissions.IsAuthenticated",
],
"DEFAULT_AUTHENTICATION_CLASSES": [
"rest_framework.authentication.SessionAuthentication",
],
# Throttle configuration (always present, but only enforced if Redis is available)
"DEFAULT_THROTTLE_CLASSES": [
"rest_framework.throttling.AnonRateThrottle",
"rest_framework.throttling.UserRateThrottle",
Expand All @@ -301,45 +339,18 @@
},
}

TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]

WSGI_APPLICATION = "config.wsgi.application"

# endregion ========================================================================================

# region Cache Configuration ======================================================
# Import cache keys and TTL values (needed by services)
from config.cache_settings import CACHE_KEYS, CACHE_TTL # noqa: E402, F401

# Use dummy cache for tests, Redis for production/development
if os.environ.get("PYTEST_RUNNING"):
# Dummy cache for tests (no Redis required)
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.dummy.DummyCache",
}
}
# Log throttling status based on Redis availability
if REDIS_AVAILABLE:
LOGGER.info("THROTTLING: Rate limiting ENABLED (Redis available)")
else:
# Redis cache for production/development
from config.cache_settings import CACHES # noqa: E402, F401
# Throttling configuration is present but won't be enforced without Redis
LOGGER.info(
"THROTTLING: Rate limiting DISABLED (Redis unavailable - throttling requires cache backend)"
)

# endregion ========================================================================================

# region Logs and Tracking =======================================================================
# region Sentry Configuration =====================================================
# Initialize Sentry only if SENTRY_URL is provided (optional)
SENTRY_URL = env("SENTRY_URL", default=None)
if ENVIRONMENT != "development" and SENTRY_URL:
Expand Down Expand Up @@ -435,8 +446,6 @@ def format(self, record: LogRecord) -> str:
},
}

LOGGER = logging.getLogger(__name__)

# endregion ========================================================================================

# region Django Debug Toolbar (Development Only) ======================================
Expand Down
Loading