diff --git a/mysql/docker-compose.test.yml b/mysql/docker-compose.test.yml index 4724fb1..1e49d3a 100644 --- a/mysql/docker-compose.test.yml +++ b/mysql/docker-compose.test.yml @@ -1,10 +1,11 @@ services: db: - image: mysql:8 + image: mariadb:10.5 platform: linux/amd64 - command: --default-authentication-plugin=mysql_native_password environment: MYSQL_ROOT_PASSWORD: password + volumes: + - db_data:/var/lib/mysql s3: image: localstack/localstack platform: linux/amd64 @@ -25,3 +26,6 @@ services: AWS_ENDPOINT_URL: http://s3:4566 AWS_ACCESS_KEY_ID: foo AWS_SECRET_ACCESS_KEY: bar + +volumes: + db_data: diff --git a/postgres/Dockerfile b/postgres/Dockerfile index ff7b975..7385bdd 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,9 +1,37 @@ -FROM --platform=linux/amd64 python:3.12-alpine3.19 +FROM debian:bookworm-slim ENV AWS_CONFIG_FILE=/.aws_config -RUN set -ex && \ - apk add --no-cache postgresql16-client bash && \ - pip install --no-cache-dir awscli && \ - aws configure set default.s3.multipart_chunksize 200MB +ENV PATH="/opt/postgresql/bin:/opt/awscli/bin:$PATH" + +# Add the PostgreSQL Apt repository +RUN apt-get update && apt-get install -y wget gnupg && \ + echo "deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +# Install PostgreSQL client tools for versions 11–17 and other dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + postgresql-client-11 \ + postgresql-client-12 \ + postgresql-client-13 \ + postgresql-client-14 \ + postgresql-client-15 \ + postgresql-client-16 \ + postgresql-client-17 \ + python3-pip && \ + mkdir -p /opt/postgresql && \ + for version in 11 12 13 14 15 16 17; do \ + ln -s /usr/lib/postgresql/$version/bin/pg_dump /usr/bin/pg_dump-$version && \ + ln -s /usr/lib/postgresql/$version/bin/pg_restore /usr/bin/pg_restore-$version; \ + done && \ + # Install AWS CLI globally + rm -f /usr/lib/python3.11/EXTERNALLY-MANAGED && \ + pip3 install --no-cache-dir awscli && \ + ln -s /usr/local/bin/aws /usr/bin/aws && \ + aws configure set default.s3.multipart_chunksize 200MB && \ + # Cleanup cache + apt-get clean && rm -rf /var/lib/apt/lists/* + COPY ./bin/ /bin/ ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/postgres/bin/dump-to-s3.sh b/postgres/bin/dump-to-s3.sh index ae74031..e25618e 100755 --- a/postgres/bin/dump-to-s3.sh +++ b/postgres/bin/dump-to-s3.sh @@ -1,19 +1,40 @@ #!/bin/bash # Usage: dump-to-s3.sh [dbname] -# Expects a DATABASE_URL environment variable that is the DB to dump +# Expects a DATABASE_URL environment variable that is the DB +# to dump and optionally SERVER_VERSION exported by the entrypoint. # Optionally, a dbname can be supplied as the second argument -# to override the name from the DATABASE_URL +# to override the name from DATABASE_URL. + set -euf -o pipefail cleanup() { rv=$?; if [ -f /tmp/db.dump ]; then shred -u /tmp/db.dump; fi; exit $rv; } trap cleanup EXIT +# Extract database name from DATABASE_URL if not provided as an argument NAME=${2:-$NAME} CONNECT_DB_URL="postgres://$USER@$HOST:$PORT/$NAME" -echo "Dumping $CONNECT_DB_URL to $1..." +# Try to detect server version if not provided +if [ -z "${SERVER_VERSION:-}" ]; then + echo "SERVER_VERSION not set. Trying to detect using psql..." + SERVER_VERSION=$(psql "$CONNECT_DB_URL" -tAc "SHOW server_version;" | cut -d '.' -f 1 || true) +fi + +if [ -z "$SERVER_VERSION" ]; then + echo "Warning: SERVER_VERSION not detected. Defaulting to version 17 (latest)." + SERVER_VERSION="17" +fi + +PG_DUMP="pg_dump-$SERVER_VERSION" + +if ! command -v "$PG_DUMP" &>/dev/null; then + echo "ERROR: $PG_DUMP not found in PATH. You must install it or override SERVER_VERSION." >&2 + exit 1 +fi + +echo "Dumping $CONNECT_DB_URL to $1 using $PG_DUMP..." set -x -pg_dump --no-privileges --no-owner --format=custom "$CONNECT_DB_URL" --file=/tmp/db.dump +"$PG_DUMP" --no-privileges --no-owner --format=custom "$CONNECT_DB_URL" --file=/tmp/db.dump aws s3 cp --acl=private --no-progress /tmp/db.dump "$1" { set +x; } 2>/dev/null echo "Done!" diff --git a/postgres/bin/entrypoint.sh b/postgres/bin/entrypoint.sh index f785fd6..e94afcc 100755 --- a/postgres/bin/entrypoint.sh +++ b/postgres/bin/entrypoint.sh @@ -4,15 +4,52 @@ set -euf -o pipefail export PGSSLMODE=require +wait_for_db() { + local retries=30 + local sleep_time=2 + + echo "Waiting for PostgreSQL server to be ready..." + until psql "$DATABASE_URL" -c '\q' 2>/dev/null || [ "$retries" -eq 0 ]; do + echo "PostgreSQL is unavailable - ($((retries--)) retries left)..." + sleep "$sleep_time" + done + + if [ "$retries" -eq 0 ]; then + echo "ERROR: PostgreSQL server did not respond." + exit 1 + fi +} + if [ -z "${DATABASE_URL:-""}" ]; then echo "WARNING: DATABASE_URL not found in environment." else + # Extract connection details from DATABASE_URL # shellcheck disable=SC2046 export $(parse_database_url.py | xargs) + # Setup PGSERVICE so `psql` just does the right thing /bin/echo -e "[$NAME]\nhost=$HOST\nport=$PORT\ndbname=$NAME\nuser=$USER" > ~/.pg_service.conf export PGSERVICE="$NAME" + + # Wait for PostgreSQL to be ready + wait_for_db + + # Detect PostgreSQL server version + if [ -z "${SERVER_VERSION:-}" ]; then + echo "Attempting to detect PostgreSQL server version..." + SERVER_VERSION=$(psql "$DATABASE_URL" -tAc "SHOW server_version;" | cut -d '.' -f 1 || true) + fi + + if [ -z "$SERVER_VERSION" ]; then + echo "WARNING: Unable to detect PostgreSQL version. Defaulting to latest (17)." + SERVER_VERSION="17" + else + echo "Detected PostgreSQL version: $SERVER_VERSION" + fi + + export SERVER_VERSION + fi exec "$@" diff --git a/postgres/bin/load-from-s3.sh b/postgres/bin/load-from-s3.sh index 9760c27..b336408 100755 --- a/postgres/bin/load-from-s3.sh +++ b/postgres/bin/load-from-s3.sh @@ -11,12 +11,29 @@ S3_PATH=$1 echo "Downloading $S3_PATH ..." aws s3 cp --no-progress "$S3_PATH" /tmp/db.dump -echo "Dropping $NAME..." +# Detect server version if not already available +if [ -z "${SERVER_VERSION:-}" ]; then + echo "SERVER_VERSION not set. Trying to detect using psql..." + SERVER_VERSION=$(psql "$DATABASE_URL" -tAc "SHOW server_version;" | cut -d '.' -f 1 || true) +fi +if [ -z "$SERVER_VERSION" ]; then + echo "Warning: SERVER_VERSION not detected. Defaulting to version 17 (latest)." + SERVER_VERSION="17" +fi + +PG_RESTORE="pg_restore-$SERVER_VERSION" + +if ! command -v "$PG_RESTORE" &>/dev/null; then + echo "Error: $PG_RESTORE not found in PATH." >&2 + exit 1 +fi + +echo "Dropping all objects owned by \"$USER\" in the database..." psql --echo-all -c "DROP OWNED BY \"$USER\" CASCADE;" -echo "Loading dump from S3..." +echo "Loading dump from S3 using $PG_RESTORE..." set -x -pg_restore --jobs="${PG_RESTORE_JOBS:-2}" --no-owner --no-privileges --dbname="$NAME" /tmp/db.dump +"$PG_RESTORE" --jobs="${PG_RESTORE_JOBS:-2}" --no-owner --no-privileges --dbname="$NAME" /tmp/db.dump { set +x; } 2>/dev/null echo "Done!" diff --git a/postgres/bin/parse_database_url.py b/postgres/bin/parse_database_url.py index 14dabe5..221425b 100755 --- a/postgres/bin/parse_database_url.py +++ b/postgres/bin/parse_database_url.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """Dump out a DATABASE_URL provided in the environment as individual variables""" import os -import sys from urllib.parse import urlparse parsed = urlparse(os.environ["DATABASE_URL"]) diff --git a/postgres/docker-compose.test.yml b/postgres/docker-compose.test.yml index 82a8edd..82dd2db 100644 --- a/postgres/docker-compose.test.yml +++ b/postgres/docker-compose.test.yml @@ -2,6 +2,8 @@ services: db: image: postgres:14 command: -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key + volumes: + - ./init.sql:/docker-entrypoint-initdb.d/init.sql environment: POSTGRES_PASSWORD: password s3: diff --git a/postgres/docs/support-newer-postgres-versions.md b/postgres/docs/support-newer-postgres-versions.md new file mode 100644 index 0000000..30ca9c0 --- /dev/null +++ b/postgres/docs/support-newer-postgres-versions.md @@ -0,0 +1,36 @@ +# How to Update for New PostgreSQL Versions + +Follow these steps to add support for a new PostgreSQL version (e.g., version 18): + +## 1. Install the New Client Tools + +In the `Dockerfile`: + +- Add the new version of `postgresql-client`, e.g., `postgresql-client-18` +- Update the `for` loop that creates version-specific symlinks: + +```dockerfile +for version in 11 12 13 14 15 16 17 18; do \ + ln -s /usr/lib/postgresql/$version/bin/pg_dump /usr/bin/pg_dump-$version && \ + ln -s /usr/lib/postgresql/$version/bin/pg_restore /usr/bin/pg_restore-$version; \ +done +``` + +These changes ensure the tooling for the new version is available. + +## 2. Update the Default Version (If Applicable) + +If the new version should become the default: + +- Update both `dump-to-s3.sh` and `load-from-s3.sh` + +This ensures the scripts default to the latest version when `SERVER_VERSION` is not set or detection fails. + +--- + +✅ Fallback behavior and override logic are already tested automatically in `tests.sh`, including: + +- Ensuring correct `pg_dump-*` is used when `SERVER_VERSION` is set +- Falling back to the detected or default version when unset + +Re-run `tests.sh` after changes to confirm compatibility across versions. diff --git a/postgres/init.sql b/postgres/init.sql new file mode 100644 index 0000000..ee6f0cf --- /dev/null +++ b/postgres/init.sql @@ -0,0 +1,2 @@ +CREATE ROLE test WITH LOGIN PASSWORD 'password'; +CREATE DATABASE test OWNER test; diff --git a/postgres/tests/tests.sh b/postgres/tests/tests.sh index 2b362f7..516e4b0 100755 --- a/postgres/tests/tests.sh +++ b/postgres/tests/tests.sh @@ -16,7 +16,7 @@ echo "###### Setup test state" aws s3api create-bucket --bucket "$BUCKET" aws s3 rm --recursive "s3://$BUCKET/" psql postgres -U postgres -c "DROP DATABASE IF EXISTS test" -psql postgres -U postgres -c "DROP DATABASE IF EXISTS \"test-clone\"" +psql postgres -U postgres -c "DROP DATABASE IF EXISTS \"test-clone\"" psql postgres -U postgres -c "DROP ROLE IF EXISTS test" psql postgres -U postgres -c "CREATE ROLE test WITH LOGIN PASSWORD 'password'" @@ -25,18 +25,31 @@ psql test -c "CREATE TABLE tbl (id SERIAL PRIMARY KEY, name CHAR(255) NOT NULL)" psql test -c "INSERT INTO tbl (name) VALUES ('name1')" psql test -c "INSERT INTO tbl (name) VALUES ('name2')" +printf "\n###### Testing SERVER_VERSION override...\n" +export SERVER_VERSION=17 +DUMP_LOG=$(dump-to-s3.sh "s3://$BUCKET/explicit.dump" test 2>&1) +echo "$DUMP_LOG" | grep "using pg_dump-17" > /dev/null +aws s3 ls "s3://$BUCKET/" | grep explicit.dump > /dev/null +echo "✅ pg_dump-17 used as expected with SERVER_VERSION=17" + +printf "\n###### Testing fallback with unset SERVER_VERSION...\n" +unset SERVER_VERSION +DUMP_LOG=$(dump-to-s3.sh "s3://$BUCKET/default.dump" test 2>&1) +echo "$DUMP_LOG" | grep "using pg_dump-14" > /dev/null +aws s3 ls "s3://$BUCKET/" | grep default.dump > /dev/null +echo "✅ pg_dump-14 used as expected when SERVER_VERSION was unset" + printf "\n###### Starting tests...\n" dump-to-s3.sh "s3://$BUCKET/dump.dump" test printf "\n###### Verify dump file exists...\n" aws s3 ls "s3://$BUCKET/" | grep dump.dump - psql test -c "INSERT INTO tbl (name) VALUES ('name3')" printf "\n###### Verify 3 records exist before load...\n" psql test -c "SELECT COUNT(*) FROM tbl" | grep "3" load-from-s3.sh "s3://$BUCKET/dump.dump" -printf "\n###### Verify 2 record exists after load...\n" +printf "\n###### Verify 2 records exist after load...\n" psql test -c "SELECT COUNT(*) FROM tbl" | grep "2" printf "\n###### Verify dump file does not exist after load...\n"