channel routing data extraction #52
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test Forcing Processor Docker Containers on ARM | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - fp_workflow | |
| - trivy_scan | |
| paths: | |
| - 'docker/**' | |
| - 'src/forcingprocessor/**' | |
| - '!src/forcingprocessor/README.md' | |
| pull_request: | |
| paths: | |
| - 'docker/**' | |
| - 'src/forcingprocessor/**' | |
| - '!src/forcingprocessor/README.md' | |
| permissions: | |
| contents: read | |
| security-events: write | |
| jobs: | |
| build-arm64: | |
| runs-on: ubuntu-22.04-arm-aws | |
| steps: | |
| - name: Pre-cleanup workspace | |
| continue-on-error: true | |
| run: | | |
| sudo rm -rf ${{ github.workspace }}/* || true | |
| sudo rm -rf ${{ github.workspace }}/.* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS | |
| run: | | |
| aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws configure set region us-east-1 | |
| - name: Build forcingprocessor-deps | |
| run: | | |
| cd docker | |
| ARCH=aarch64 TAG=latest-arm64 \ | |
| docker compose -f docker-compose.yml build forcingprocessor-deps | |
| - name: Build forcingprocessor | |
| run: | | |
| cd docker | |
| ARCH=aarch64 TAG=latest-arm64 \ | |
| docker compose -f docker-compose.yml build forcingprocessor | |
| - name: List built images | |
| run: docker images | grep forcingprocessor | |
| - name: Save Docker images | |
| run: | | |
| docker save awiciroh/forcingprocessor:latest-arm64 | gzip > /tmp/forcingprocessor-arm64.tar.gz | |
| - name: Upload forcingprocessor image | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: forcingprocessor-arm64 | |
| path: /tmp/forcingprocessor-arm64.tar.gz | |
| retention-days: 1 | |
| - name: Clean up local tar files | |
| if: always() | |
| run: rm -f /tmp/forcingprocessor-arm64.tar.gz | |
| test-arm64: | |
| needs: build-arm64 | |
| runs-on: ubuntu-22.04-arm-aws | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 10 | |
| matrix: | |
| test-suite: | |
| - name: "NOMADS" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k test_nomads" | |
| - name: "Google Cloud Storage" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_nwm_google_apis or test_google or test_gs or test_gcs'" | |
| - name: "NOAA NWM PDS - Base HTTPS" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https and not short_range and not medium_range and not analysis_assim'" | |
| - name: "NOAA NWM - Short Range" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_short_range'" | |
| - name: "NOAA NWM - Medium Range" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_medium_range'" | |
| - name: "NOAA NWM - Analysis Assim" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_analysis_assim and not extend'" | |
| - name: "NOAA NWM - Analysis Assim Extend" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_analysis_assim_extend'" | |
| - name: "NOAA NWM - S3 Access" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_s3'" | |
| - name: "Retrospective v2.1" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_retro_2_1'" | |
| - name: "Retrospective v3.0" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_retro_3_0'" | |
| - name: "Output Formats" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_csv_output_type or test_parquet_output_type or test_tar_output_type or test_netcdf_output_type'" | |
| - name: "Plotting & S3 Output" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_plotting or test_s3_output'" | |
| - name: "Weight Generation" | |
| image: "forcingprocessor" | |
| command: "pytest -q --tb=short --disable-warnings --maxfail=1 tests/test_hf2ds.py" | |
| name: "${{ matrix.test-suite.name }}" | |
| steps: | |
| - name: Pre-cleanup workspace | |
| continue-on-error: true | |
| run: | | |
| sudo rm -rf ${{ github.workspace }}/* || true | |
| sudo rm -rf ${{ github.workspace }}/.* || true | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download Docker image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: forcingprocessor-arm64 | |
| path: /tmp/artifacts | |
| - name: Verify artifact and load | |
| run: | | |
| set -euo pipefail | |
| ls -lh /tmp/artifacts | |
| # Load gzipped image | |
| for i in 1 2 3; do | |
| if gunzip -c /tmp/artifacts/forcingprocessor-arm64.tar.gz | docker load; then | |
| echo "Successfully loaded image on attempt $i" | |
| docker images | grep forcingprocessor | |
| break | |
| else | |
| echo "Failed to load image on attempt $i" | |
| if [ $i -lt 3 ]; then | |
| echo "Retrying in 10 seconds..." | |
| sleep 10 | |
| else | |
| exit 1 | |
| fi | |
| fi | |
| done | |
| - name: Run ${{ matrix.test-suite.name }} | |
| run: | | |
| docker run --rm \ | |
| -v $(pwd):/forcingprocessor \ | |
| -w /forcingprocessor \ | |
| -e AWS_REGION=us-east-1 \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| awiciroh/${{ matrix.test-suite.image }}:latest-arm64 \ | |
| ${{ matrix.test-suite.command }} | |
| # - name: Clean up Docker image | |
| # if: always() | |
| # run: | | |
| # docker rmi awiciroh/${{ matrix.test-suite.image }}:latest-arm64 || true | |
| # docker system prune -f | |
| cleanup-artifacts: | |
| needs: test-arm64 | |
| runs-on: ubuntu-22.04-arm-aws | |
| if: always() | |
| steps: | |
| - name: Delete artifacts | |
| uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: forcingprocessor-arm64 |