diff --git a/.env b/.env new file mode 100644 index 0000000..f923c05 --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +POSTGRES_USER=ayon +POSTGRES_PASSWORD=ayon +POSTGRES_DB=ayon + +AYON_STACK_SERVER_TAG=latest +AYON_STACK_SERVER_PORT=5001 + +AYON_STACK_WORKER_TAG=latest +AYON_API_KEY=veryinsecurapikey \ No newline at end of file diff --git a/Makefile b/Makefile index b0ae30d..5f283bd 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,9 @@ AYON_STACK_SETTINGS_FILE ?= settings/template.json AYON_STACK_SERVER_NAME ?= ynput/ayon AYON_STACK_SERVER_TAG ?= latest +POSTGRES_USER ?= ayon +POSTGRES_PASSWORD ?= ayon +POSTGRES_DB ?= ayon # # Variables # @@ -35,6 +38,8 @@ default: @echo " demo Create demo projects based on settings in demo directory" @echo " dump Use 'make dump projectname=' to backup a project" @echo " restore Use 'make restore projectname=' to restore a project from previous dump" + @echo " dump-entire Use 'make dump-entire targetname=' to backup the entire database, targetname is optional, no targetname uses 'backup_TIMESTAMP.sql', '_TIMESTAMP.sql' is always appended, targetname=test.sql == test_TIMESTAMP.sql" + @echo " restore-entire Use 'make restore-entire targetfilename=' to restore the entire database" @echo "" @echo "Development:" @echo " backend Download / update backend" @@ -58,7 +63,7 @@ endif @docker compose exec $(SERVER_CONTAINER) ./reload.sh dbshell: - @docker compose exec postgres psql -U ayon ayon + @docker compose exec -e PGPASSWORD=$(POSTGRES_PASSWORD) postgres psql -U $(POSTGRES_USER) -d $(POSTGRES_DB) reload: @docker compose exec $(SERVER_CONTAINER) ./reload.sh @@ -86,23 +91,21 @@ dump: @# Dump project data from public.projects table - docker compose exec -t postgres pg_dump --table=public.projects --column-inserts ayon -U ayon | \ + docker compose exec -e PGPASSWORD=$(POSTGRES_PASSWORD) -T postgres pg_dump --table=public.projects --column-inserts -d $(POSTGRES_DB) -U $(POSTGRES_USER) | \ grep "^INSERT INTO" | grep \'$(projectname)\' >> dump.$(projectname).sql @# Get all product types used in the project @# and insert them into the product_types table @# (if they don't exist yet) - docker compose exec postgres psql -U ayon ayon -Atc "SELECT DISTINCT(product_type) from project_$(projectname).products;" | \ + docker compose exec -e PGPASSWORD=$(POSTGRES_PASSWORD) postgres psql -U $(POSTGRES_USER) -d $(POSTGRES_DB) -Atc "SELECT DISTINCT(product_type) from project_$(projectname).products;" | \ while read -r product_type; do \ echo "INSERT INTO public.product_types (name) VALUES ('$${product_type}') ON CONFLICT DO NOTHING;"; \ done >> dump.$(projectname).sql @# Dump project schema (tables, views, etc.) - docker compose exec postgres pg_dump --schema=project_$(projectname) ayon -U ayon >> dump.$(projectname).sql - - + docker compose exec -e PGPASSWORD=$(POSTGRES_PASSWORD) postgres pg_dump --schema=project_$(projectname) -d $(POSTGRES_DB) -U $(POSTGRES_USER) >> dump.$(projectname).sql restore: @if [ -z "$(projectname)" ]; then \ @@ -114,8 +117,28 @@ restore: echo "Error: Dump file dump.$(projectname).sql not found"; \ exit 1; \ fi - docker compose exec -T postgres psql -U ayon ayon < dump.$(projectname).sql - + docker compose exec -e PGPASSWORD=$(POSTGRES_PASSWORD) -T postgres psql -U $(POSTGRES_USER) -d $(POSTGRES_DB) < dump.$(projectname).sql + +dump-entire: + $(eval TIMESTAMP := $(shell date +%y%m%d%H%M)) + $(eval CLEAN_NAME := $(if $(targetname),$(patsubst %.sql,%,$(targetname)),backup)) + $(eval TARGET := $(CLEAN_NAME)_$(TIMESTAMP).sql) + @echo "Dumping entire database to $(TARGET)..." + @docker compose exec -e PGPASSWORD=$(POSTGRES_PASSWORD) -T postgres pg_dump -U $(POSTGRES_USER) -d $(POSTGRES_DB) > $(TARGET) + @echo "Done." + +restore-entire: + @if [ -z "$(targetfilename)" ]; then \ + echo "Error: targetfilename is required. Usage: make restore-entire targetfilename="; \ + exit 1; \ + fi + @if [ ! -f $(targetfilename) ]; then \ + echo "Error: File $(targetfilename) not found"; \ + exit 1; \ + fi + @echo "Restoring entire database from $(targetfilename)..." + @docker compose exec -e PGPASSWORD=$(POSTGRES_PASSWORD) -T postgres psql -U $(POSTGRES_USER) -d $(POSTGRES_DB) < $(targetfilename) + @echo "Restore complete." # # The following targets are for development purposes only. # diff --git a/docker-compose.worker.yml b/docker-compose.worker.yml index d8d5fb1..c27c8f8 100644 --- a/docker-compose.worker.yml +++ b/docker-compose.worker.yml @@ -13,5 +13,5 @@ services: - "/var/run/docker.sock:/var/run/docker.sock" environment: - - "AYON_API_KEY=${AYON_API_KEY-veryinsecurapikey}" - - "AYON_SERVER_URL=${AYON_SERVER_URL-http://server:5000}" + - "AYON_API_KEY=${AYON_API_KEY:-veryinsecurapikey}" + - "AYON_SERVER_URL=${AYON_SERVER_URL:-http://server:5000}" diff --git a/docker-compose.yml b/docker-compose.yml index d5eaa43..b23c7f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,13 +11,15 @@ services: expose: [5432] volumes: + # comment out the following line on Windows - "/etc/localtime:/etc/localtime:ro" + - "db:/var/lib/postgresql/data" environment: - - "POSTGRES_USER=ayon" - - "POSTGRES_PASSWORD=ayon" - - "POSTGRES_DB=ayon" + - "POSTGRES_USER=${POSTGRES_USER:-ayon}" + - "POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-ayon}" + - "POSTGRES_DB=${POSTGRES_DB:-ayon}" redis: image: redis:alpine diff --git a/manage.ps1 b/manage.ps1 index 6d6642a..405897a 100644 --- a/manage.ps1 +++ b/manage.ps1 @@ -7,9 +7,39 @@ if ($ARGS.Length -gt 1) { # Settings $SCRIPT_DIR = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent Set-Location "$($SCRIPT_DIR)" + +# Load Environment Variables +function Load-Env { + $envFile = Join-Path $SCRIPT_DIR ".env" + if (Test-Path $envFile) { + Get-Content $envFile | ForEach-Object { + $line = $_.Trim() + if ($line -and -not $line.StartsWith("#")) { + $key, $value = $line -split '=', 2 + if ($key -and $value) { + $key = $key.Trim() + $value = $value.Trim().Trim('"').Trim("'") + if (-not (Get-Variable $key -ErrorAction SilentlyContinue)) { + New-Variable -Name $key -Value $value -Scope Script -Force + } + } + } + } + } +} + +# Initialize Defaults +$POSTGRES_USER = "ayon" +$POSTGRES_PASSWORD = "ayon" +$POSTGRES_DB = "ayon" +$AYON_STACK_SERVER_TAG = "latest" + +# Load overrides from .env +Load-Env + $SETTINGS_FILE = "settings/template.json" $IMAGE_NAME = "ynput/ayon" -$DEFAULT_IMAGE = "$($IMAGE_NAME):latest" +$DEFAULT_IMAGE = "$($IMAGE_NAME):$($AYON_STACK_SERVER_TAG)" $SERVER_CONTAINER = "server" # Variables @@ -28,10 +58,12 @@ function defaultfunc { Write-Host "Usage: ./manage.ps1 [target]" Write-Host "" Write-Host "Runtime targets:" - Write-Host " setup Apply settings template from the settings/template.json" - Write-Host " dbshell Open a PostgreSQL shell" - Write-Host " reload Reload the running server" - Write-Host " demo Create demo projects based on settings in demo directory" + Write-Host " setup Apply settings template from the settings/template.json" + Write-Host " dbshell Open a PostgreSQL shell" + Write-Host " reload Reload the running server" + Write-Host " demo Create demo projects based on settings in demo directory" + Write-Host " dump-entire [FILE] Backup the whole database (auto-named if empty), no FILE uses 'backup_TIMESTAMP.sql', '_TIMESTAMP.sql' is always appended, dump-entire test.sql == test_TIMESTAMP.sql" + Write-Host " restore-entire FILE Restore the whole database from file" Write-Host "" Write-Host "Development:" Write-Host " backend Download / update backend" @@ -49,7 +81,7 @@ function defaultfunc { # Gotta love makefiles function setup { - Write-Host "Server container: $($SERVER_CONTAINER)" + Write-Host "Server container: $($SERVER_CONTAINER)" if (!(Test-Path "$($SCRIPT_DIR)/settings/template.json")) { & "$($COMPOSE)" exec -T "$($SERVER_CONTAINER)" python -m setup } else { @@ -59,7 +91,9 @@ function setup { } function dbshell { - & "$($COMPOSE)" exec postgres psql -U ayon ayon + $env:PGPASSWORD = $POSTGRES_PASSWORD + & "$($COMPOSE)" exec postgres psql -U $POSTGRES_USER -d $POSTGRES_DB + $env:PGPASSWORD = $null } function reload { @@ -77,18 +111,110 @@ function update { & "$($COMPOSE)" up --detach --build "$($SERVER_CONTAINER)" } +function dump { + $projectname = $args[0] + if ($projectname -eq $null) { + Write-Error "Error: Project name is required. Usage: ./manage.ps1 dump [PROJECT]" + exit 1 + } + + Write-Host "Dumping project '$projectname'" + $dumpFile = "dump.$projectname.sql" + "DROP SCHEMA IF EXISTS project_$projectname CASCADE;" | Out-File -FilePath $dumpFile -Encoding utf8 + "DELETE FROM public.projects WHERE name = '$projectname';" | Out-File -FilePath $dumpFile -Append -Encoding utf8 + + $env:PGPASSWORD = $POSTGRES_PASSWORD + + # Project data dump (table public.projects) + & "$($COMPOSE)" exec -T postgres pg_dump --table=public.projects --column-inserts -U $POSTGRES_USER -d $POSTGRES_DB | + Select-String -Pattern "^INSERT INTO" | + Select-String -Pattern "'$projectname'" | + ForEach-Object { $_.Line } | Out-File -FilePath $dumpFile -Append -Encoding utf8 + + # Get all product types on a project + $types = & "$($COMPOSE)" exec -T postgres psql -U $POSTGRES_USER -d $POSTGRES_DB -Atc "SELECT DISTINCT(product_type) from project_$projectname.products;" + foreach ($product_type in $types) { + if ($product_type.Trim()) { + "INSERT INTO public.product_types (name) VALUES ('$($product_type.Trim())') ON CONFLICT DO NOTHING;" | Out-File -FilePath $dumpFile -Append -Encoding utf8 + } + } + + # Project schema dump + & "$($COMPOSE)" exec -T postgres pg_dump --schema=project_$projectname -U $POSTGRES_USER -d $POSTGRES_DB | Out-File -FilePath $dumpFile -Append -Encoding utf8 + + $env:PGPASSWORD = $null + Write-Host "Project dump saved to $dumpFile" +} + +function restore { + $projectname = $arguments[0] + if ($projectname -eq $null) { + Write-Error "Error: Project name is required. Usage: ./manage.ps1 restore [PROJECT]" + exit 1 + } + + $dumpfile = "dump.$projectname.sql" + + # Check if the dump file exists. + if (-not (Test-Path $dumpfile)) { + Write-Error "Error: Dump file $dumpfile not found" + exit 1 + } + + # Restore the database from the dump file. + $env:PGPASSWORD = $POSTGRES_PASSWORD + Get-Content $dumpfile | & "$($COMPOSE)" exec -T postgres psql -U $POSTGRES_USER -d $POSTGRES_DB + $env:PGPASSWORD = $null +} + +function dump-entire { + $baseName = $arguments[0] + $timestamp = Get-Date -Format "yyMMddHHmm" + + if ($baseName -eq $null) { + $target = "backup_$timestamp.sql" + } else { + $cleanName = $baseName -replace "\.sql$", "" + $target = "$($cleanName)_$timestamp.sql" + } + + Write-Host "Dumping entire database to $target..." + $env:PGPASSWORD = $POSTGRES_PASSWORD + & "$($COMPOSE)" exec -T postgres pg_dump -U $POSTGRES_USER -d $POSTGRES_DB | Out-File -FilePath $target -Encoding utf8 + $env:PGPASSWORD = $null + Write-Host "Done." +} + +function restore-entire { + $target = $arguments[0] + if ($target -eq $null) { + Write-Error "Error: targetfilename is required. Usage: ./manage.ps1 restore-entire [FILENAME]" + exit 1 + } + + if (-not (Test-Path $target)) { + Write-Error "Error: File $target not found" + exit 1 + } + + Write-Host "CAUTION: This will overwrite existing data in $POSTGRES_DB!" + $env:PGPASSWORD = $POSTGRES_PASSWORD + Get-Content $target | & "$($COMPOSE)" exec -T postgres psql -U $POSTGRES_USER -d $POSTGRES_DB + $env:PGPASSWORD = $null + Write-Host "Restore complete." +} + function relinfo { - $backend_dir = "$($SCRIPT_DIR)\backend" - $frontend_dir = "$($SCRIPT_DIR)\frontend" - $output_file = "$($SCRIPT_DIR)\RELEASE" + $backend_dir = Join-Path $SCRIPT_DIR "backend" + $frontend_dir = Join-Path $SCRIPT_DIR "frontend" + $output_file = Join-Path $SCRIPT_DIR "RELEASE" $cur_date = Get-Date - $backend_version = Invoke-Expression -Command "python -c ""import os;import sys;content={};f=open(r'$($backend_dir)\ayon_server\version.py');exec(f.read(),content);f.close();print(content['__version__'])""" $build_date = Get-Date -Date $cur_date -Format "yyyyMMdd" $build_time = Get-Date -Date $cur_date -Format "HHmm" - $cur_cwd = Get-Location + $cur_cwd = Get-Location Set-Location $backend_dir $backend_branch = Invoke-Expression -Command "git branch --show-current" $backend_commit = Invoke-Expression -Command "git rev-parse --short HEAD" @@ -140,84 +266,25 @@ function frontend { } } -function dump { - $projectname = $args[0] - if ($projectname -eq $null) { - Write-Error "Error: Project name is required. Usage: ./manage.ps1 dump [PROJECT]" - exit 1 - } - - Write-Host "Dumping project '$projectname'" - $dumpFile = "dump.$projectname.sql" - "DROP SCHEMA IF EXISTS project_$projectname CASCADE;" | Out-File -FilePath $dumpFile -Encoding utf8 - "DELETE FROM public.projects WHERE name = '$projectname';" | Out-File -FilePath $dumpFile -Append -Encoding utf8 - - # Project data dump (table public.projects) - docker compose exec -t postgres pg_dump --table=public.projects --column-inserts ayon -U ayon | - Select-String -Pattern "^INSERT INTO" | - Select-String -Pattern "'$projectname'" | - ForEach-Object { $_.Line } | Out-File -FilePath $dumpFile -Append -Encoding utf8 - - # Get all product types on a project - $types = docker compose exec postgres psql -U ayon ayon -Atc "SELECT DISTINCT(product_type) from project_$projectname.products;" - foreach ($product_type in $types) { - "INSERT INTO public.product_types (name) VALUES ('$product_type') ON CONFLICT DO NOTHING;" | Out-File -FilePath $dumpFile -Append -Encoding utf8 - } - - # Project schema dump - docker compose exec postgres pg_dump --schema=project_$projectname ayon -U ayon | Out-File -FilePath $dumpFile -Append -Encoding utf8 -} - -function restore { - $projectname = $arguments[0] - if ($projectname -eq $null) { - Write-Error "Error: Project name is required. Usage: ./manage.ps1 restore [PROJECT]" - exit 1 - } - - $dumpfile = "dump.$projectname.sql" - - # Check if the dump file exists. - if (-not (Test-Path $dumpfile)) { - Write-Error "Error: Dump file $SCRIPT_DIR\$dumpfile not found" - exit 1 - } - - # Restore the database from the dump file. - Get-Content $dumpfile | docker-compose exec -T postgres psql -U ayon ayon -} - function main { - if ($FunctionName -eq "setup") { - setup - } elseif ($FunctionName -eq "dbshell") { - dbshell - } elseif ($FunctionName -eq "reload") { - reload - } elseif ($FunctionName -eq "demo") { - demo - } elseif ($FunctionName -eq "update") { - update - } elseif ($FunctionName -eq "build") { - build - } elseif ($FunctionName -eq "relinfo") { - relinfo - } elseif ($FunctionName -eq "dist") { - dist - } elseif ($FunctionName -eq "backend") { - backend - } elseif ($FunctionName -eq "frontend") { - frontend - } elseif ($FunctionName -eq "dump") { - dump @arguments - } elseif ($FunctionName -eq "restore") { - restore @arguments - } elseif ($null -eq $FunctionName) { - defaultfunc - } else { - Write-Host "Unknown function ""$FunctionName""" - defaultfunc + switch ($FunctionName) { + "setup" { setup } + "dbshell" { dbshell } + "reload" { reload } + "demo" { demo } + "update" { update } + "build" { build } + "relinfo" { relinfo } + "dist" { dist } + "backend" { backend } + "frontend" { frontend } + "dump" { dump @arguments } + "restore" { restore @arguments } + "dump-entire" { dump-entire @arguments } + "restore-entire" { restore-entire @arguments } + $null { defaultfunc } + Default { Write-Host "Unknown function ""$FunctionName"""; defaultfunc } } } -main +main \ No newline at end of file