diff --git a/.github/workflows/CI-Linux-ARM64.yml b/.github/workflows/CI-Linux-ARM64.yml deleted file mode 100644 index 289185b2f..000000000 --- a/.github/workflows/CI-Linux-ARM64.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: CI-Linux-ARM64 - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - NOTE_TO_SELF: "environments can not be passed from here to reused workflows!" - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - -jobs: - jetson-build-test: - uses: ./.github/workflows/build-test-lin.yml - with: - runner: AGX - flav: Linux_ARM64 - is-selfhosted: true - cuda: 'ON' - prep-cmd: 'echo skipping builder prep as I can not sudo' - bootstrap-cmd: 'export PATH="$HOME/.local/bin/gcc-8:$PATH" && export VCPKG_FORCE_SYSTEM_BINARIES=1 && ./vcpkg/bootstrap-vcpkg.sh' - cache-path: './none' - cmake-conf-cmd: 'export VCPKG_OVERLAY_PORTS=../thirdparty/custom-overlay && cmake -B . -DENABLE_ARM64=ON ../base' - nProc: 6 - jetson-publish: - needs: jetson-build-test - permissions: - checks: write - pull-requests: write - uses: ./.github/workflows/publish-test.yml - with: - flav: Linux_ARM64 - secrets: - GIST_TOKEN: ${{ secrets.GIST_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/CI-Linux-CUDA-Docker.yml b/.github/workflows/CI-Linux-CUDA-Docker.yml index 7b077cc1b..ee44f1de6 100644 --- a/.github/workflows/CI-Linux-CUDA-Docker.yml +++ b/.github/workflows/CI-Linux-CUDA-Docker.yml @@ -10,26 +10,27 @@ env: NOTE_TO_SELF: "environments can not be passed from here to reused workflows!" jobs: - # linux-cuda-build-prep: + linux-cuda-docker-build-prep: # this is in caching, not using it for now - # uses: ./.github/workflows/build-test-lin-container.yml - # with: - # runner: 'ubuntu-20.04' - # flav: 'Linux-Cuda' - # is-selfhosted: false - # cuda: 'ON' - # prep-cmd: 'echo skipping builder prep as I can not sudo' - # is-prep-phase: true - # cache-path: '/github/home/.cache/vcpkg/archives' - # nProc: 3 + uses: ./.github/workflows/build-test-lin-container.yml + with: + runner: 'ubuntu-latest' + flav: 'Linux-Cuda' + is-selfhosted: false + cuda: 'ON' +# prep-cmd: 'echo skipping builder prep as I can not sudo' + is-prep-phase: true + cache-path: '/github/home/.cache/vcpkg/archives' + nProc: 3 linux-cuda-docker-build-no-test: + needs: linux-cuda-docker-build-prep uses: ./.github/workflows/build-test-lin-container.yml with: - runner: 'ubuntu-20.04' + runner: 'ubuntu-latest' flav: Linux-Cuda is-selfhosted: false cuda: 'ON' - prep-cmd: 'echo skipping builder prep as I can not sudo' +# prep-cmd: 'echo skipping builder prep as I can not sudo' skip-test: true cache-path: '/github/home/.cache/vcpkg/archives' nProc: 3 diff --git a/.github/workflows/CI-Linux-CUDA-wsl.yml b/.github/workflows/CI-Linux-CUDA-wsl.yml deleted file mode 100644 index f08c2b618..000000000 --- a/.github/workflows/CI-Linux-CUDA-wsl.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: CI-Linux-CUDA-WSL - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - NOTE_TO_SELF: "environments can not be passed from here to reused workflows!" - -jobs: - wsl-cuda-build-test: - uses: ./.github/workflows/build-test-lin-wsl.yml - with: - runner: windows-cuda - flav: WSL - is-selfhosted: true - cuda: 'ON' - cache-path: './none' - nProc: 6 - wsl-cuda-publish: - needs: wsl-cuda-build-test - permissions: - checks: write - pull-requests: write - uses: ./.github/workflows/publish-test.yml - with: - flav: WSL - secrets: - GIST_TOKEN: ${{ secrets.GIST_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/CI-Linux-CUDA.yml b/.github/workflows/CI-Linux-CUDA.yml deleted file mode 100644 index 02af3fbad..000000000 --- a/.github/workflows/CI-Linux-CUDA.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: CI-Linux-CUDA - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - NOTE_TO_SELF: "environments can not be passed from here to reused workflows!" - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - -jobs: - linux-cuda-build-test: - uses: ./.github/workflows/build-test-lin.yml - with: - runner: 'linux-cuda' - flav: Linux-CudaT - is-selfhosted: true - cuda: 'ON' - prep-cmd: 'echo skipping builder prep as I can not sudo' - bootstrap-cmd: 'export PATH="/usr/bin/gcc-8:$PATH" && ./vcpkg/bootstrap-vcpkg.sh' - cache-path: './none' - nProc: 6 - linux-cuda-publish: - needs: linux-cuda-build-test - permissions: - checks: write - pull-requests: write - uses: ./.github/workflows/publish-test.yml - with: - flav: Linux-CudaT - secrets: - GIST_TOKEN: ${{ secrets.GIST_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/CI-Linux-NoCUDA.yml b/.github/workflows/CI-Linux-NoCUDA.yml deleted file mode 100644 index c6b6cbc58..000000000 --- a/.github/workflows/CI-Linux-NoCUDA.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: CI-Linux-NoCUDA - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - NOTE_TO_SELF: "environments can not be passed from here to reused workflows!" - -jobs: - linux-nocuda-build-test: - uses: ./.github/workflows/build-test-lin.yml - with: - runner: 'ubuntu-20.04' - flav: Linux - is-selfhosted: false - cuda: 'OFF' - nProc: 3 - linux-nocuda-publish: - needs: linux-nocuda-build-test - permissions: - checks: write - pull-requests: write - uses: ./.github/workflows/publish-test.yml - with: - flav: Linux - secrets: - GIST_TOKEN: ${{ secrets.GIST_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/CI-Win-CUDA.yml b/.github/workflows/CI-Win-CUDA.yml deleted file mode 100644 index 19cdf2d8f..000000000 --- a/.github/workflows/CI-Win-CUDA.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: CI-Win-CUDA - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - NOTE_TO_SELF: "environments can not be passed from here to reused workflows!" - -jobs: - win-cuda-build-test: - uses: ./.github/workflows/build-test-win.yml - with: - runner: windows-cuda - flav: 'Windows-cuda' - cuda: 'ON' - is-selfhosted: true - nProc: 8 - nTestTimeoutMins: 20 - win-cuda-publish: - needs: win-cuda-build-test - permissions: - checks: write - pull-requests: write - uses: ./.github/workflows/publish-test.yml - with: - flav: 'Windows-cuda' - secrets: - GIST_TOKEN: ${{ secrets.GIST_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/CI-Win-NoCUDA.yml b/.github/workflows/CI-Win-NoCUDA.yml deleted file mode 100644 index 68afe1f43..000000000 --- a/.github/workflows/CI-Win-NoCUDA.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: CI-Win-NoCUDA - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - NOTE_TO_SELF: "environments can not be passed from here to reused workflows!" - -jobs: - win-nocuda-build-prep: - uses: ./.github/workflows/build-test-win.yml - with: - runner: 'windows-2019' - flav: 'Win-nocuda' - cuda: 'OFF' - is-selfhosted: false - is-prep-phase: true - nProc: 3 - win-nocuda-build-test: - needs: win-nocuda-build-prep - uses: ./.github/workflows/build-test-win.yml - with: - runner: 'windows-2019' - flav: 'Win-nocuda' - is-selfhosted: false - cuda: 'OFF' - is-prep-phase: false - nProc: 3 - win-nocuda-publish: - needs: win-nocuda-build-test - permissions: - checks: write - pull-requests: write - uses: ./.github/workflows/publish-test.yml - with: - flav: 'Win-nocuda' - secrets: - GIST_TOKEN: ${{ secrets.GIST_TOKEN }} diff --git a/.github/workflows/build-test-lin-container.yml b/.github/workflows/build-test-lin-container.yml index acd2a0209..f77670972 100644 --- a/.github/workflows/build-test-lin-container.yml +++ b/.github/workflows/build-test-lin-container.yml @@ -30,7 +30,7 @@ on: prep-cmd: type: string description: 'commands required to be run on a builder to prep it for build' - default: 'sudo apt-get update -qq && sudo apt-get -y install ca-certificates curl zip unzip tar autoconf automake autopoint build-essential flex git-core libass-dev libfreetype6-dev libgnutls28-dev libmp3lame-dev libsdl2-dev libtool libsoup-gnome2.4-dev libva-dev libvdpau-dev libvorbis-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev libncurses5-dev libncursesw5-dev ninja-build pkg-config texinfo wget yasm zlib1g-dev nasm gperf bison python3 python3-pip dos2unix libx11-dev libgles2-mesa-dev libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev python3-jinja2 && pip3 install meson' + default: 'apt-get update -qq && apt-get -y install ca-certificates curl zip unzip tar autoconf automake autopoint build-essential flex git-core git-lfs libass-dev libfreetype6-dev libgnutls28-dev libmp3lame-dev libsdl2-dev libtool libsoup-gnome2.4-dev libva-dev libvdpau-dev libvorbis-dev libxdamage-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev libx11-dev libxft-dev libxext-dev libncurses5-dev libncursesw5-dev ninja-build pkg-config texinfo wget apt-transport-https software-properties-common yasm zlib1g-dev nasm gperf bison python3 python3-pip dos2unix libx11-dev libgles2-mesa-dev libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev python3-jinja2 libssl-dev libssl1.1 && pip3 install meson && pip3 install Jinja2 && pip3 install cmake==3.29.6 && wget -q https://packages.microsoft.com/keys/microsoft.asc -O- | apt-key add - && add-apt-repository "deb [arch=amd64] https://packages.microsoft.com/ubuntu/$(lsb_release -rs)/prod $(lsb_release -cs) main" && apt-get update && apt-get install -y powershell' required: false prep-check-cmd: type: string @@ -73,7 +73,9 @@ jobs: env: TEST_EXE: build/aprapipesut CMAKE_TC_FILE: '../vcpkg/scripts/buildsystems/vcpkg.cmake' # Note: naming this variable as CMAKE_TOOLCHAIN_FILE can cause havoc!!! - container: ghcr.io/kumaakh/aprapipes-build-x86-ubutu18.04-cuda:last-good + container: + image: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 + options: --user root --env DEBIAN_FRONTEND=noninteractive defaults: run: shell: bash @@ -82,15 +84,13 @@ jobs: - name: Prepare builder run: | ${{ inputs.prep-cmd }} - apt purge cmake || true - pip3 install cmake==3.29.6 || echo 'did not update cmake' ${{ inputs.prep-check-cmd }} - name: Check builder for CUDA if: ${{ contains(inputs.cuda,'ON')}} run: | (PATH=/usr/local/cuda/bin:$PATH && nvcc --version) || echo 'please install cuda and add /usr/local/cuda/bin into path' - test -f /usr/local/cuda/include/cudnn.h || echo 'install cudnn as described in the readme.md' + test -f /usr/include/cudnn.h || echo 'install cudnn as described in the readme.md' # Note: on nvidia supplied image, cudnn is already installed in /usr/include - name: Cleanup workspace on self hosted runners if: inputs.is-selfhosted @@ -151,6 +151,13 @@ jobs: working-directory: ${{github.workspace}}/build run: '${{ inputs.cmake-conf-cmd }} -DCMAKE_TOOLCHAIN_FILE=${{env.CMAKE_TC_FILE}} -DCMAKE_BUILD_TYPE=${{inputs.buildConf}} -DENABLE_CUDA=${{inputs.cuda}}' continue-on-error: ${{inputs.is-prep-phase}} # in prep phase we expect an error here due to missing OpenCV + - name: Remove files not needed for the build + if: ${{!inputs.is-selfhosted}} # we have space crunch on hosted builder only + working-directory: ${{github.workspace}} + run: remove-item vcpkg/downloads -Recurse -Force && remove-item * -Recurse -Force -Include *.pdb,*.ilk + shell: pwsh + continue-on-error: true + - name: Build if: ${{!inputs.is-prep-phase}} @@ -185,7 +192,7 @@ jobs: if: ${{ always() }} # only upload logs when we have a failure above uses: actions/upload-artifact@v4 with: - name: BuildLogs_${{ inputs.flav }} + name: BuildLogs_${{ inputs.flav }}_${{ inputs.is-prep-phase && '0' || '1' }} path: | ${{ github.workspace }}/vcpkg/buildtrees/**/*.log ${{ github.workspace }}/vcpkg/buildtrees/**/*.txt diff --git a/.gitignore b/.gitignore index 91196770c..0e986fede 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ _build* +build _debug* data/testOutput thirdparty/gst-build/gst-build-1.16 diff --git a/docker/dockerfile b/docker/dockerfile new file mode 100644 index 000000000..0a7068c3e --- /dev/null +++ b/docker/dockerfile @@ -0,0 +1,11 @@ +FROM nvidia/cuda:11.8.0-devel-ubuntu20.04 +RUN apt-get update -qq && apt-get -y install \ + ca-certificates curl zip unzip tar autoconf automake autopoint build-essential flex git-core \ + libass-dev libfreetype6-dev libgnutls28-dev libmp3lame-dev libsdl2-dev libtool libsoup-gnome2.4-dev \ + libva-dev libvdpau-dev libvorbis-dev libxdamage-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev \ + libncurses5-dev libncursesw5-dev ninja-build pkg-config texinfo wget yasm zlib1g-dev nasm gperf \ + bison python3 python3-pip dos2unix libgles2-mesa-dev libxcursor-dev \ + libglu1-mesa-dev python3-jinja2 && pip3 install jinja2 meson cmake==3.29.6 && \ + apt-get purge -y man-db && \ + rm -rf /usr/share/man /usr/share/doc /usr/share/doc-base && \ + rm -rf /var/lib/apt/lists/* diff --git a/docker/readme.md b/docker/readme.md new file mode 100644 index 000000000..0483b3c54 --- /dev/null +++ b/docker/readme.md @@ -0,0 +1 @@ +This folder contains definition of docker images which are required to build or run ApraPipes \ No newline at end of file diff --git a/thirdparty/custom-overlay/cudnn/portfile.cmake b/thirdparty/custom-overlay/cudnn/portfile.cmake index f33292fa5..b75df484d 100644 --- a/thirdparty/custom-overlay/cudnn/portfile.cmake +++ b/thirdparty/custom-overlay/cudnn/portfile.cmake @@ -8,7 +8,7 @@ find_path(CUDNN_INCLUDE_DIR NAMES cudnn.h cudnn_v8.h cudnn_v7.h PATH_SUFFIXES cuda/include include include/12.3) message(STATUS "CUDNN_INCLUDE_DIR: ${CUDNN_INCLUDE_DIR}") find_library(CUDNN_LIBRARY NAMES cudnn cudnn8 cudnn7 libcudnn libcudnn8 libcudnn7 - HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/lib/aarch64-linux-gnu/ /usr/include/aarch64-linux-gnu/ /usr/ +HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/ /usr/include/aarch64-linux-gnu/ /usr/ PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64 cuda/lib/x64 lib/12.3/x64) message(STATUS "CUDNN_LIBRARY: ${CUDNN_LIBRARY}") if(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn.h")