From 7df0a7e004959726f605a59eb51f02ba6c94a463 Mon Sep 17 00:00:00 2001 From: Oliver Pain <43761992+opain@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:32:43 +0100 Subject: [PATCH 1/7] Update build_deploy_container.yaml --- .github/workflows/build_deploy_container.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build_deploy_container.yaml b/.github/workflows/build_deploy_container.yaml index 48290ea..8cfa49a 100644 --- a/.github/workflows/build_deploy_container.yaml +++ b/.github/workflows/build_deploy_container.yaml @@ -30,6 +30,13 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + - name: Maximize build disk space + uses: easimon/maximize-build-space@v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: 'true' + - name: Export Secrets to Environment run: | echo "SINGULARITY_LIBRARY_TOKEN=${{ secrets.SINGULARITY_LIBRARY_TOKEN }}" >> $GITHUB_ENV From 20c03a5becde068422559144bf9b19f849620e41 Mon Sep 17 00:00:00 2001 From: Oliver Pain <43761992+opain@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:43:38 +0100 Subject: [PATCH 2/7] Update build_deploy_container.yaml --- .github/workflows/build_deploy_container.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_deploy_container.yaml b/.github/workflows/build_deploy_container.yaml index 8cfa49a..39db604 100644 --- a/.github/workflows/build_deploy_container.yaml +++ b/.github/workflows/build_deploy_container.yaml @@ -27,16 +27,16 @@ jobs: DOCKERFILE_PATH: pipeline/misc/docker/Dockerfile steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Maximize build disk space uses: easimon/maximize-build-space@v10 with: root-reserve-mb: 512 swap-size-mb: 1024 remove-dotnet: 'true' - + + - name: Checkout code + uses: actions/checkout@v2 + - name: Export Secrets to Environment run: | echo "SINGULARITY_LIBRARY_TOKEN=${{ secrets.SINGULARITY_LIBRARY_TOKEN }}" >> $GITHUB_ENV From d5f12a6e570d412da1d680d64467495c2b15b9ef Mon Sep 17 00:00:00 2001 From: Oliver Pain <43761992+opain@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:52:32 +0100 Subject: [PATCH 3/7] Update Dockerfile --- pipeline/misc/docker/Dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pipeline/misc/docker/Dockerfile b/pipeline/misc/docker/Dockerfile index 58fe319..5273464 100644 --- a/pipeline/misc/docker/Dockerfile +++ b/pipeline/misc/docker/Dockerfile @@ -10,10 +10,10 @@ RUN apt-get update && \ echo 'exec /bin/tar --no-same-owner "$@"' >> /usr/local/bin/tar && \ chmod +x /usr/local/bin/tar -# Install Mambaforge -RUN wget https://github.com/conda-forge/miniforge/releases/download/4.10.3-10/Mambaforge-4.10.3-10-Linux-x86_64.sh && \ - bash Mambaforge-4.10.3-10-Linux-x86_64.sh -b -p /opt/mambaforge && \ - rm Mambaforge-4.10.3-10-Linux-x86_64.sh +# Install Miniforge +RUN wget https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-Linux-x86_64.sh && \ + bash Miniforge3-24.11.3-0-Linux-x86_64.sh -b -p /opt/miniforge && \ + rm Miniforge3-24.11.3-0-Linux-x86_64.sh # Download GenoPred RUN mkdir -p /tools/GenoPred && \ @@ -22,20 +22,20 @@ RUN mkdir -p /tools/GenoPred && \ git fetch --tags # Create genopred env -RUN /opt/mambaforge/bin/conda init bash && \ - echo "source /opt/mambaforge/etc/profile.d/conda.sh" >> /etc/bash.bashrc && \ - . /opt/mambaforge/etc/profile.d/conda.sh && \ +RUN /opt/miniforge/bin/conda init bash && \ + echo "source /opt/miniforge/etc/profile.d/conda.sh" >> /etc/bash.bashrc && \ + . /opt/miniforge/etc/profile.d/conda.sh && \ mamba env update -n genopred -f /tools/GenoPred/pipeline/envs/pipeline.yaml && \ mamba clean -a -y # Run initial snakemake command to setup dependencies -RUN /opt/mambaforge/bin/conda init bash && \ - echo "source /opt/mambaforge/etc/profile.d/conda.sh" >> /etc/bash.bashrc && \ - . /opt/mambaforge/etc/profile.d/conda.sh && \ +RUN /opt/miniforge/bin/conda init bash && \ + echo "source /opt/miniforge/etc/profile.d/conda.sh" >> /etc/bash.bashrc && \ + . /opt/miniforge/etc/profile.d/conda.sh && \ conda activate genopred && \ cd /tools/GenoPred/pipeline && \ snakemake --restart-times 3 -j1 --use-conda --conda-frontend mamba install_r_packages resources/software/pgscatalog_utils/download_pgscatalog_utils.done && \ mamba clean -a -y ENTRYPOINT ["/bin/bash"] -CMD ["-c", "source /opt/mambaforge/etc/profile.d/conda.sh && conda init && source ~/.bashrc && conda activate genopred && cd /tools/GenoPred/pipeline && exec /bin/bash"] +CMD ["-c", "source /opt/miniforge/etc/profile.d/conda.sh && conda init && source ~/.bashrc && conda activate genopred && cd /tools/GenoPred/pipeline && exec /bin/bash"] From fb01408bdd21560dbb8755183aab8954547b1608 Mon Sep 17 00:00:00 2001 From: Oliver Pain <43761992+opain@users.noreply.github.com> Date: Thu, 17 Jul 2025 09:01:52 +0100 Subject: [PATCH 4/7] Update build_deploy_container.yaml --- .github/workflows/build_deploy_container.yaml | 54 +++++++++---------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build_deploy_container.yaml b/.github/workflows/build_deploy_container.yaml index 39db604..668f930 100644 --- a/.github/workflows/build_deploy_container.yaml +++ b/.github/workflows/build_deploy_container.yaml @@ -27,13 +27,6 @@ jobs: DOCKERFILE_PATH: pipeline/misc/docker/Dockerfile steps: - - name: Maximize build disk space - uses: easimon/maximize-build-space@v10 - with: - root-reserve-mb: 512 - swap-size-mb: 1024 - remove-dotnet: 'true' - - name: Checkout code uses: actions/checkout@v2 @@ -56,34 +49,33 @@ jobs: echo "::set-output name=tag::${{ github.event.inputs.tag || github.sha }}" fi - - name: Build docker container - run: | - docker buildx build --platform linux/amd64 -f $DOCKERFILE_PATH -t $DOCKER_IMAGE_NAME:latest -t $DOCKER_IMAGE_NAME:${{ steps.set_tag.outputs.tag }} --push . +# - name: Build docker container +# run: | +# docker buildx build --platform linux/amd64 -f $DOCKERFILE_PATH -t $DOCKER_IMAGE_NAME:latest -t $DOCKER_IMAGE_NAME:${{ steps.set_tag.outputs.tag }} --push . - - name: Clean up Docker build artifacts - run: | - docker system prune -af - + - name: Maximize build disk space + uses: easimon/maximize-build-space@v10 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: 'true' + - name: Setup Apptainer uses: eWaterCycle/setup-apptainer@v2.0.0 - name: Configure Singularity temp directories run: | - # create a temp area inside the GitHub runner's large temp folder - mkdir -p $RUNNER_TEMP/singularity + mkdir -p "$RUNNER_TEMP/singularity/cache" echo "APPTAINER_TMPDIR=$RUNNER_TEMP/singularity" >> $GITHUB_ENV - # optional: keep a separate cache dir - mkdir -p $RUNNER_TEMP/singularity/cache echo "APPTAINER_CACHEDIR=$RUNNER_TEMP/singularity/cache" >> $GITHUB_ENV - - - name: Check free space - run: | - df -h - + - name: Convert Docker Image to Singularity SIF run: | - apptainer pull --disable-cache --tmpdir $RUNNER_TEMP/singularity /mnt/genopred_image.sif docker://${DOCKER_IMAGE_NAME}:latest - + apptainer pull --disable-cache \ + --tmpdir "$RUNNER_TEMP/singularity" \ + "$GITHUB_WORKSPACE/genopred_image.sif" \ + docker://${DOCKER_IMAGE_NAME}:latest + - name: Install expect run: | sudo apt-get update @@ -105,14 +97,16 @@ jobs: send -- "${{ secrets.SIGNING_KEY_PASSWORD }}\r" expect eof ' + - name: Sign the Singularity Image run: | expect -c ' - spawn apptainer sign /mnt/genopred_image.sif - expect "Enter your key passphrase : " - send -- "${{ secrets.SIGNING_KEY_PASSWORD }}\r" - expect eof + spawn apptainer sign "$GITHUB_WORKSPACE/genopred_image.sif" + expect "Enter your key passphrase : " + send -- "${{ secrets.SIGNING_KEY_PASSWORD }}\r" + expect eof ' + - name: Create Token File run: echo "${{ secrets.SINGULARITY_LIBRARY_TOKEN }}" > tokenfile @@ -124,7 +118,7 @@ jobs: - name: Push Singularity Image to Singularity Library run: | - apptainer push /mnt/genopred_image.sif $SINGULARITY_IMAGE_NAME:latest + apptainer push "$GITHUB_WORKSPACE/genopred_image.sif" "$SINGULARITY_IMAGE_NAME:latest" - name: Setup tmate session if: ${{ failure() }} From 31d4ec5ae68c28b9d27da366e8b72c238e0df64f Mon Sep 17 00:00:00 2001 From: Oliver Pain <43761992+opain@users.noreply.github.com> Date: Thu, 17 Jul 2025 09:06:48 +0100 Subject: [PATCH 5/7] Update build_deploy_container.yaml --- .github/workflows/build_deploy_container.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_deploy_container.yaml b/.github/workflows/build_deploy_container.yaml index 668f930..246dc65 100644 --- a/.github/workflows/build_deploy_container.yaml +++ b/.github/workflows/build_deploy_container.yaml @@ -65,14 +65,14 @@ jobs: - name: Configure Singularity temp directories run: | - mkdir -p "$RUNNER_TEMP/singularity/cache" - echo "APPTAINER_TMPDIR=$RUNNER_TEMP/singularity" >> $GITHUB_ENV - echo "APPTAINER_CACHEDIR=$RUNNER_TEMP/singularity/cache" >> $GITHUB_ENV + mkdir -p "$GITHUB_WORKSPACE/.singularity/cache" + echo "APPTAINER_TMPDIR=$GITHUB_WORKSPACE/.singularity" >> $GITHUB_ENV + echo "APPTAINER_CACHEDIR=$GITHUB_WORKSPACE/.singularity/cache" >> $GITHUB_ENV - name: Convert Docker Image to Singularity SIF run: | apptainer pull --disable-cache \ - --tmpdir "$RUNNER_TEMP/singularity" \ + --tmpdir "$GITHUB_WORKSPACE/.singularity" \ "$GITHUB_WORKSPACE/genopred_image.sif" \ docker://${DOCKER_IMAGE_NAME}:latest From b8393a0c4f680b28cbad07a190947002c4ad5a9d Mon Sep 17 00:00:00 2001 From: Oliver Pain <43761992+opain@users.noreply.github.com> Date: Thu, 17 Jul 2025 09:23:19 +0100 Subject: [PATCH 6/7] Update build_deploy_container.yaml --- .github/workflows/build_deploy_container.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_deploy_container.yaml b/.github/workflows/build_deploy_container.yaml index 246dc65..8138121 100644 --- a/.github/workflows/build_deploy_container.yaml +++ b/.github/workflows/build_deploy_container.yaml @@ -100,12 +100,13 @@ jobs: - name: Sign the Singularity Image run: | - expect -c ' - spawn apptainer sign "$GITHUB_WORKSPACE/genopred_image.sif" - expect "Enter your key passphrase : " - send -- "${{ secrets.SIGNING_KEY_PASSWORD }}\r" + SIF_PATH="$GITHUB_WORKSPACE/genopred_image.sif" + expect -c " + spawn apptainer sign \"$SIF_PATH\" + expect \"Enter your key passphrase : \" + send -- \"${{ secrets.SIGNING_KEY_PASSWORD }}\r\" expect eof - ' + " - name: Create Token File run: echo "${{ secrets.SINGULARITY_LIBRARY_TOKEN }}" > tokenfile From 35cf09cbf413f90f0aa987d7171e00bfe5e45c67 Mon Sep 17 00:00:00 2001 From: Oliver Pain <43761992+opain@users.noreply.github.com> Date: Thu, 17 Jul 2025 09:47:50 +0100 Subject: [PATCH 7/7] Update build_deploy_container.yaml --- .github/workflows/build_deploy_container.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_deploy_container.yaml b/.github/workflows/build_deploy_container.yaml index 8138121..2d3eb93 100644 --- a/.github/workflows/build_deploy_container.yaml +++ b/.github/workflows/build_deploy_container.yaml @@ -49,9 +49,9 @@ jobs: echo "::set-output name=tag::${{ github.event.inputs.tag || github.sha }}" fi -# - name: Build docker container -# run: | -# docker buildx build --platform linux/amd64 -f $DOCKERFILE_PATH -t $DOCKER_IMAGE_NAME:latest -t $DOCKER_IMAGE_NAME:${{ steps.set_tag.outputs.tag }} --push . + - name: Build docker container + run: | + docker buildx build --platform linux/amd64 -f $DOCKERFILE_PATH -t $DOCKER_IMAGE_NAME:latest -t $DOCKER_IMAGE_NAME:${{ steps.set_tag.outputs.tag }} --push . - name: Maximize build disk space uses: easimon/maximize-build-space@v10