migrate project dependencies to poetry

.devcontainer/devcontainer.json

@@ -0,0 +1,74 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3
+{
+	"name": "Pdp Compose Dev Container",
+	"dockerComposeFile": ["../docker/local-run/docker-compose.yaml", "./docker-compose.override.yaml"],
+	"service": "frontend",
+	"workspaceFolder": "/workspace",
+	// Configure tool-specific properties.
+	"customizations": {
+	  // Configure properties specific to VS Code.
+	  "vscode": {
+		// Set *default* container specific settings.json values on container create.
+		"settings": {
+			"terminal.integrated.defaultProfile.linux": "bash",
+			"terminal.integrated.profiles.linux": { "bash": { "path": "/bin/bash" } },
+		  "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
+		  "python.linting.enabled": true,
+		  "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		  "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		  "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		  "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		  "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		  "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		  "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		  "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+		},
+
+		// VSCODE ONLY: Add the IDs of extensions you want installed when the container is created.
+		"extensions": [
+			"ms-python.python",
+			"ms-python.vscode-pylance",
+			"tamasfe.even-better-toml",
+			"github.vscode-github-actions",
+			"ms-vscode.makefile-tools"
+		]
+	  }
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	//"forwardPorts": [48423],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "bash ./.devcontainer/post-install.sh",
+	"postStartCommand": "bash ./.devcontainer/post-start.sh",
+
+	"features": {
+	  "docker-in-docker": "latest",
+	  //"ghcr.io/devcontainers/features/git:1": {},
+	  // add python to container
+	  "ghcr.io/devcontainers/features/python:1": {
+		"version": "3.11"
+	  }
+	},
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	// PDP specific: this is the default user in the pdp-base-minimal container, I can't seem to work out
+	// how to override. It may not be necessary but it might be nice to centralize on a standard user.
+	"remoteUser": "dockremap",
+
+	"remoteEnv": {
+	  //		"PATH": "${containerEnv:PATH}:${containerEnv:HOME}/.local/bin"
+	},
+	"mounts": [
+	  // Re-use local SSH keys (useful if you use SSH keys for Git repo access or other SSH work)
+	  "source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached",
+	  // Re-use local Git configuration
+	  "source=${localEnv:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached"
+	],
+
+	// "runArgs": [
+	// // allow container to be treated with no network isolation
+	// 	"--network=host",
+	// 	// give a nicer name to the container
+	// 	"--name", "${localEnv:USER}_pdp_devcontainer"]
+  }

.devcontainer/docker-compose.override.yaml

@@ -0,0 +1,6 @@
+version: '3'
+services:
+  frontend:
+    entrypoint: ["/workspace/.devcontainer/entry-fe.sh"]
+    volumes:
+      - ../../:/workspace

.devcontainer/entry-fe.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+/bin/sh -c "while sleep 1000; do :; done"

.devcontainer/post-install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -ex
+
+##
+## Create some aliases
+##
+echo 'alias ll="ls -alF"' >> $HOME/.bashrc
+echo 'alias la="ls -A"' >> $HOME/.bashrc
+echo 'alias l="ls -CF"' >> $HOME/.bashrc
+
+# Convenience workspace directory for later use
+WORKSPACE_DIR=$(pwd)
+
+# Set current workspace as safe for git
+git config --global --add safe.directory ${WORKSPACE_DIR}
+
+poetry install
+
+echo "Done!"

.devcontainer/post-start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -ex
+
+# Convenience workspace directory for later use
+WORKSPACE_DIR=$(pwd)

.env

@@ -0,0 +1,3 @@
+DSN=postgresql://********:********@pgbouncer-dev:5432/pcic_meta
+PCDS_DSN=postgresql://********:********@pgbouncer-dev:5432/crmp
+ORCA_ROOT=https://services.pacificclimate.org/dev/orca

.gitignore

@@ -65,3 +65,6 @@ target/
 
 # Passwords
 *password*
+.venv
+.cache
+.env.test

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Pytest: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "pytest",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}",
+            "justMyCode": false,
+            "envFile": "${workspaceFolder}/.env.test",
+            "args": [
+                "${file}",
+                "-k",
+                "test_nc_raster_response"
+            ]
+        }
+    ]
+}

docker/local-run/Dockerfile

@@ -1,21 +1,64 @@
-FROM pcic/pdp-base-minimal:pdp-python3
-LABEL Maintainer="Rod Glover <rglover@uvic.ca>"
+FROM ubuntu:20.04
 
-USER root
+ARG USERNAME=dockremap
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+ARG CODEBASE_DIR=/codebase
 
-# TODO: Move into pdp-base-minimal?
-RUN apt-get update && apt-get install -yq wget
-# Create directory for supervisord logs
-RUN mkdir etc/
+# from https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user
+# Create the user
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
+    #
+    # [Optional] Add sudo support. Omit if you don't need to install software after connecting.
+    && apt-get update \
+    && apt-get install -y sudo \
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME
 
-EXPOSE 8000
+RUN apt-get update && \
+        DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+        build-essential \
+        libhdf5-dev \
+        libgdal-dev \
+        libnetcdf-dev \
+        python3 \
+        python3-dev \
+        python3-pip \
+        git \
+        curl \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+# [Optional] Set the default user. Omit if you want to keep the default as root.
+USER $USERNAME
 
-USER ${USERNAME}
+# Install poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="/home/$USERNAME/.local/bin:$PATH"
 
 # You must mount the local codebase to /codebase
-WORKDIR /codebase
-ADD *requirements.txt /codebase/
+WORKDIR $CODEBASE_DIR
+
+# Change some Poetry settings to better deal with working in a container
+RUN poetry config cache-dir $CODEBASE_DIR/.cache
+RUN poetry config virtualenvs.in-project true
+
+# Apply gdal environment variables
+ENV CPLUS_INCLUDE_PATH /usr/include/gdal
+ENV C_INCLUDE_PATH /usr/include/gdal
 
-RUN pip3 install -r requirements.txt -r test_requirements.txt -r deploy_requirements.txt
 
-ENTRYPOINT ./docker/local-run/entrypoint.sh
+# Copy project files into /codebase
+ADD --chown=${USERNAME}:${GROUPNAME} ../.. .
+
+# Pre-install initial packages (pip, setuptools, numpy)
+# RUN poetry install --only initial
+# # Install gdal using pip3 into the Poetry virtualenv
+# RUN poetry run pip3 install gdal==3.0.4
+# # Install rest of project
+# RUN poetry install
+
+EXPOSE 8000
+ENTRYPOINT [ "poetry", "run", "gunicorn"]
+CMD ["--config", "docker/local-run/gunicorn.conf", "--log-config", "docker/local-run/logging.conf", "pdp.wsgi:frontend"]

docker/local-run/docker-compose.yaml

@@ -1,14 +1,53 @@
-version: '3.2'
+version: '3.8'
 services:
-  image:
-    # This service only builds the image common to both frontend and backend.
+    #image: pcic/pdp-local-run
+  backend:
     build:
       context: ../..
       dockerfile: ./docker/local-run/Dockerfile
     image: pcic/pdp-local-run
+    container_name: pdp-local-run-be
+    stdin_open: true
+    tty: true
+    env_file:
+      - common-with-passwords.env
+    environment:
+      # Specific to backend
+      - ORCA_ROOT=http://pdp.localhost:5000/orca
+      - APP_MODULE=pdp.wsgi:backend
+      - GUNICORN_WORKER_CLASS=gevent
+    entrypoint: /codebase/docker/local-run/entrypoint-be.sh
+#    ports:
+#      - "30446:8000"  # backend
+    volumes:
+      # This is to enable installing local versions of other packages from
+      # your own filesystem. You'll need to update this. Source and target
+      # could probably be made environment variables. Or it could be
+      # a relative source path and an impersonal target path such as
+      # `/local-packages`.
+      # - type: bind
+      #   source: /home/rglover/code
+      #   target: /home/rglover/code
+      - type: bind
+        source: ../..
+        target: /codebase
+      # These bindings are for testing the solution but I've had problems with them when attempting
+      # to use them in a docker desktop context (They seem to cause the container to hang on startup)
+      # Alternatively to binding, if using the .devcontainer you can nfs mount them directly.
+      # - /storage/data/climate/downscale:/storage/data/climate/downscale:ro
+      # - /storage/data/climate/hydrology:/storage/data/climate/hydrology:ro
+      # - /storage/data/climate/observations:/storage/data/climate/observations:ro
+      # - /storage/data/climate/PRISM:/storage/data/climate/PRISM:ro
+      # - /storage/data/projects/dataportal:/storage/data/projects/dataportal:ro
+      # - /storage/data/projects/hydrology:/storage/data/projects/hydrology:ro
+      # - /storage/data/projects/PRISM:/storage/data/projects/PRISM:ro
+
+
 
   frontend:
     image: pcic/pdp-local-run
+    depends_on:
+      - backend
     container_name: pdp-local-run-fe
     stdin_open: true
     tty: true
@@ -35,51 +74,16 @@ services:
       # could probably be made environment variables. Or it could be
       # a relative source path and an impersonal target path such as
       # `/local-packages`.
-      - type: bind
-        source: /home/rglover/code
-        target: /home/rglover/code
+      # - type: bind
+      #   source: /home/rglover/code
+      #   target: /home/rglover/code
       - type: bind
         source: ../..
         target: /codebase
       - type: bind
         source: ./prism_demo_config.js
         target: /codebase/pdp/static/js/prism_demo_config.js
 
-  backend:
-    image: pcic/pdp-local-run
-    container_name: pdp-local-run-be
-    stdin_open: true
-    tty: true
-    env_file:
-      - common-with-passwords.env
-    environment:
-      # Specific to backend
-      - ORCA_ROOT=http://pdp.localhost:5000/orca
-      - APP_MODULE=pdp.wsgi:backend
-      - GUNICORN_WORKER_CLASS=gevent
-    entrypoint: /codebase/docker/local-run/entrypoint-be.sh
-#    ports:
-#      - "30446:8000"  # backend
-    volumes:
-      # This is to enable installing local versions of other packages from
-      # your own filesystem. You'll need to update this. Source and target
-      # could probably be made environment variables. Or it could be
-      # a relative source path and an impersonal target path such as
-      # `/local-packages`.
-      - type: bind
-        source: /home/rglover/code
-        target: /home/rglover/code
-      - type: bind
-        source: ../..
-        target: /codebase
-      - /storage/data/climate/downscale:/storage/data/climate/downscale:ro
-      - /storage/data/climate/hydrology:/storage/data/climate/hydrology:ro
-      - /storage/data/climate/observations:/storage/data/climate/observations:ro
-      - /storage/data/climate/PRISM:/storage/data/climate/PRISM:ro
-      - /storage/data/projects/dataportal:/storage/data/projects/dataportal:ro
-      - /storage/data/projects/hydrology:/storage/data/projects/hydrology:ro
-      - /storage/data/projects/PRISM:/storage/data/projects/PRISM:ro
-
   pgbouncer:
     image: edoburu/pgbouncer
     container_name: pgbouncer-dev

docker/local-run/pgbouncer.ini

@@ -1,5 +1,5 @@
 [databases]
-metnorth_8fd8f556c548 = host=dbnorth.pcic.uvic.ca port=5432 user=metnorth_ro dbname=metnorth_8fd8f556c548
+#metnorth_8fd8f556c548 = host=dbnorth.pcic.uvic.ca port=5432 user=metnorth_ro dbname=metnorth_8fd8f556c548
 crmp = host=db.pcic.uvic.ca port=5432 user=httpd dbname=crmp
 pcic_meta = host=db.pcic.uvic.ca port=5432 user=httpd_meta dbname=pcic_meta
 pcic_meta_test = host=db.pcic.uvic.ca port=5432 user=httpd_meta dbname=pcic_meta