Ak/revise docker builds

.github/workflows/CI-Linux-CUDA-Docker.yml

@@ -10,26 +10,27 @@ env:
   NOTE_TO_SELF: "environments can not be passed from here to reused workflows!"
 
 jobs:
-  # linux-cuda-build-prep:
+  linux-cuda-docker-build-prep:
   # this is in caching, not using it for now
-  #   uses: ./.github/workflows/build-test-lin-container.yml
-  #   with:
-  #     runner: 'ubuntu-20.04'
-  #     flav: 'Linux-Cuda'
-  #     is-selfhosted: false
-  #     cuda: 'ON'
-  #     prep-cmd: 'echo skipping builder prep as I can not sudo'
-  #     is-prep-phase: true
-  #     cache-path: '/github/home/.cache/vcpkg/archives'      
-  #     nProc: 3
+    uses: ./.github/workflows/build-test-lin-container.yml
+    with:
+      runner: 'ubuntu-latest'
+      flav: 'Linux-Cuda'
+      is-selfhosted: false
+      cuda: 'ON'
+#      prep-cmd: 'echo skipping builder prep as I can not sudo'
+      is-prep-phase: true
+      cache-path: '/github/home/.cache/vcpkg/archives'      
+      nProc: 3
   linux-cuda-docker-build-no-test:
+    needs: linux-cuda-docker-build-prep
     uses: ./.github/workflows/build-test-lin-container.yml
     with:
-      runner: 'ubuntu-20.04'
+      runner: 'ubuntu-latest'
       flav: Linux-Cuda
       is-selfhosted: false      
       cuda: 'ON'
-      prep-cmd: 'echo skipping builder prep as I can not sudo'
+#      prep-cmd: 'echo skipping builder prep as I can not sudo'
       skip-test: true 
       cache-path: '/github/home/.cache/vcpkg/archives'
       nProc: 3

.github/workflows/build-test-lin-container.yml

@@ -30,7 +30,7 @@ on:
       prep-cmd:
         type: string
         description: 'commands required to be run on a builder to prep it for build'
-        default: 'sudo apt-get update -qq && sudo apt-get -y install ca-certificates curl zip unzip tar  autoconf   automake  autopoint build-essential  flex git-core   libass-dev   libfreetype6-dev   libgnutls28-dev   libmp3lame-dev   libsdl2-dev   libtool   libsoup-gnome2.4-dev   libva-dev   libvdpau-dev   libvorbis-dev   libxcb1-dev   libxcb-shm0-dev   libxcb-xfixes0-dev libncurses5-dev libncursesw5-dev  ninja-build   pkg-config   texinfo   wget   yasm   zlib1g-dev   nasm   gperf  bison python3 python3-pip dos2unix libx11-dev libgles2-mesa-dev libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev python3-jinja2 && pip3 install meson'
+        default: 'apt-get update -qq && apt-get -y install ca-certificates curl zip unzip tar autoconf automake autopoint build-essential flex git-core git-lfs libass-dev libfreetype6-dev libgnutls28-dev libmp3lame-dev libsdl2-dev libtool libsoup-gnome2.4-dev libva-dev libvdpau-dev libvorbis-dev libxdamage-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev libx11-dev libxft-dev libxext-dev libncurses5-dev libncursesw5-dev ninja-build pkg-config texinfo wget apt-transport-https software-properties-common yasm zlib1g-dev nasm gperf bison python3 python3-pip dos2unix libx11-dev libgles2-mesa-dev libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev python3-jinja2 libssl-dev libssl1.1 && pip3 install meson && pip3 install Jinja2 && pip3 install cmake==3.29.6 && wget -q https://packages.microsoft.com/keys/microsoft.asc -O- | apt-key add - && add-apt-repository "deb [arch=amd64] https://packages.microsoft.com/ubuntu/$(lsb_release -rs)/prod $(lsb_release -cs) main" && apt-get update && apt-get install -y powershell'
         required: false
       prep-check-cmd:
         type: string
@@ -73,7 +73,9 @@ jobs:
     env:
       TEST_EXE: build/aprapipesut
       CMAKE_TC_FILE: '../vcpkg/scripts/buildsystems/vcpkg.cmake' # Note: naming this variable as CMAKE_TOOLCHAIN_FILE can cause havoc!!!
-    container: ghcr.io/kumaakh/aprapipes-build-x86-ubutu18.04-cuda:last-good
+    container: 
+      image: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04
+      options: --user root --env DEBIAN_FRONTEND=noninteractive
     defaults:
       run:
         shell: bash  
@@ -82,15 +84,13 @@ jobs:
     - name: Prepare builder
       run: |
         ${{ inputs.prep-cmd }} 
-        apt purge cmake || true
-        pip3 install cmake==3.29.6 || echo 'did not update cmake'
         ${{ inputs.prep-check-cmd }} 
 
     - name: Check builder for CUDA
       if: ${{ contains(inputs.cuda,'ON')}} 
       run: |
         (PATH=/usr/local/cuda/bin:$PATH && nvcc --version) || echo 'please install cuda and add /usr/local/cuda/bin into path'
-        test -f /usr/local/cuda/include/cudnn.h || echo 'install cudnn as described in the readme.md'
+        test -f /usr/include/cudnn.h || echo 'install cudnn as described in the readme.md' # Note: on nvidia supplied image, cudnn is already installed in /usr/include
 
     - name: Cleanup workspace on self hosted runners
       if: inputs.is-selfhosted
@@ -151,6 +151,13 @@ jobs:
       working-directory: ${{github.workspace}}/build
       run: '${{ inputs.cmake-conf-cmd }} -DCMAKE_TOOLCHAIN_FILE=${{env.CMAKE_TC_FILE}} -DCMAKE_BUILD_TYPE=${{inputs.buildConf}} -DENABLE_CUDA=${{inputs.cuda}}'
       continue-on-error: ${{inputs.is-prep-phase}} # in prep phase we expect an error here due to missing OpenCV
+    - name: Remove files not needed for the build
+      if: ${{!inputs.is-selfhosted}} # we have space crunch on hosted builder only
+      working-directory: ${{github.workspace}}
+      run: remove-item vcpkg/downloads -Recurse -Force && remove-item * -Recurse -Force -Include *.pdb,*.ilk
+      shell: pwsh
+      continue-on-error: true
+
 
     - name: Build
       if: ${{!inputs.is-prep-phase}}
@@ -185,7 +192,7 @@ jobs:
       if:  ${{ always() }} # only upload logs when we have a failure above
       uses: actions/upload-artifact@v4
       with:
-        name: BuildLogs_${{ inputs.flav }}
+        name: BuildLogs_${{ inputs.flav }}_${{ inputs.is-prep-phase && '0' || '1' }}
         path: |
           ${{ github.workspace }}/vcpkg/buildtrees/**/*.log
           ${{ github.workspace }}/vcpkg/buildtrees/**/*.txt

.gitignore

@@ -1,4 +1,5 @@
 _build*
+build
 _debug*
 data/testOutput
 thirdparty/gst-build/gst-build-1.16

docker/dockerfile

@@ -0,0 +1,11 @@
+FROM nvidia/cuda:11.8.0-devel-ubuntu20.04
+RUN apt-get update -qq && apt-get -y install \
+    ca-certificates curl zip unzip tar autoconf automake autopoint build-essential flex git-core \
+    libass-dev libfreetype6-dev libgnutls28-dev libmp3lame-dev libsdl2-dev libtool libsoup-gnome2.4-dev \
+    libva-dev libvdpau-dev libvorbis-dev libxdamage-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev \
+    libncurses5-dev libncursesw5-dev ninja-build pkg-config texinfo wget yasm zlib1g-dev nasm gperf \
+    bison python3 python3-pip dos2unix libgles2-mesa-dev libxcursor-dev \
+    libglu1-mesa-dev python3-jinja2 && pip3 install jinja2 meson cmake==3.29.6 && \
+    apt-get purge -y man-db && \
+    rm -rf /usr/share/man /usr/share/doc /usr/share/doc-base && \
+    rm -rf /var/lib/apt/lists/*

docker/readme.md

@@ -0,0 +1 @@
+This folder contains definition of docker images which are required to build or run ApraPipes 

thirdparty/custom-overlay/cudnn/portfile.cmake

@@ -8,7 +8,7 @@ find_path(CUDNN_INCLUDE_DIR NAMES cudnn.h cudnn_v8.h cudnn_v7.h
   PATH_SUFFIXES cuda/include include include/12.3)
 message(STATUS "CUDNN_INCLUDE_DIR: ${CUDNN_INCLUDE_DIR}")
 find_library(CUDNN_LIBRARY NAMES cudnn cudnn8 cudnn7 libcudnn libcudnn8 libcudnn7
-  HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/lib/aarch64-linux-gnu/ /usr/include/aarch64-linux-gnu/ /usr/
+HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/ /usr/include/aarch64-linux-gnu/ /usr/
   PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64 cuda/lib/x64 lib/12.3/x64)
 message(STATUS "CUDNN_LIBRARY: ${CUDNN_LIBRARY}")
 if(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn.h")