From 1b0fb34552175add4d4ff950a4997b5a0ec09858 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sat, 15 Feb 2025 22:23:25 +0100 Subject: [PATCH 01/17] reusable workflow to distribute multi-platform builds Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/distribute.yml | 405 +++++++++++++++++++++++++++++++ 1 file changed, 405 insertions(+) create mode 100644 .github/workflows/distribute.yml diff --git a/.github/workflows/distribute.yml b/.github/workflows/distribute.yml new file mode 100644 index 00000000..8b6f4904 --- /dev/null +++ b/.github/workflows/distribute.yml @@ -0,0 +1,405 @@ +name: distribute + +on: + workflow_call: + inputs: + # inputs specific to this reusable worklow + runner: + type: string + description: "Runner instance" + required: false + default: 'ubuntu-latest' + target: + type: string + description: "Multi-platform target to build" + required: true + push: + type: boolean + description: "Push image to registry" + required: false + default: false + # same as docker/metadata-action inputs + meta-image: + type: string + description: "Image to use as base name for tags" + required: true + meta-tags: + type: string + description: 'List of tags as key-value pair attributes' + required: false + meta-flavor: + type: string + description: 'Flavors to apply' + required: false + meta-labels: + type: string + description: 'List of custom labels' + required: false + meta-annotations: + type: string + description: 'List of custom annotations' + required: false + # same as docker/login-action inputs (minus logout) + login-registry: + type: string + description: 'Server address of Docker registry. If not set then will default to Docker Hub' + required: false + login-username: + type: string + description: 'Username used to log against the Docker registry' + required: false + login-ecr: + type: string + description: 'Specifies whether the given registry is ECR (auto, true or false)' + default: 'auto' + required: false + # same as docker/bake-action inputs (minus builder, targets, load, push) + bake-source: + type: string + description: "Context to build from. Can be either local or a remote bake definition" + required: false + bake-allow: + type: string + description: "Allow build to access specified resources (e.g., network.host)" + required: false + bake-files: + type: string + description: "List of bake definition files" + required: false + bake-workdir: + type: string + description: "Working directory of bake execution" + required: false + default: '.' + bake-no-cache: + type: boolean + description: "Do not use cache when building the image" + required: false + bake-pull: + type: boolean + description: "Always attempt to pull a newer version of the image" + required: false + bake-provenance: + type: string + description: "Provenance is a shorthand for --set=*.attest=type=provenance" + required: false + bake-sbom: + type: string + description: "SBOM is a shorthand for --set=*.attest=type=sbom" + bake-set: + type: string + description: "List of targets values to override (eg. targetpattern.key=value)" + required: false + secrets: + login-password: + description: "Password or personal access token used to log against the Docker registry" + required: false + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + includes: ${{ steps.set.outputs.includes }} + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Set includes + id: set + uses: actions/github-script@v7 + with: + script: | + let def; + const source = `${{ inputs.bake-source }}`; + const files = `${{ inputs.bake-files }}` ? `${{ inputs.bake-files }}`.split(/[\r?\n,]+/).filter(Boolean) : []; + const target = `${{ inputs.target }}`; + + const metaImage = `${{ inputs.meta-image }}` ? `${{ inputs.meta-image }}`.split(/[\r?\n,]+/).filter(Boolean) : []; + if (metaImage.length > 1) { + throw new Error('Only one meta-image is allowed'); + } + + await core.group(`Validating definition`, async () => { + let args = ['buildx', 'bake']; + for (const file of files) { + args.push('--file', file); + } + if (source && source !== '.') { + args.push(source); + } + args.push(target, '--print'); + + const res = await exec.getExecOutput('docker', args, { + ignoreReturnCode: true, + silent: true, + cwd: `${{ inputs.bake-workdir }}` + }); + if (res.stderr.length > 0 && res.exitCode != 0) { + throw new Error(res.stderr); + } + def = JSON.parse(res.stdout.trim()); + core.info(JSON.stringify(def, null, 2)); + }); + + await core.group(`Set includes`, async () => { + const platforms = def.target[target].platforms; + if (platforms.length > 100) { + throw new Error('Too many platforms'); + } else if (platforms.length <= 1) { + throw new Error('At least 2 platforms are required'); + } + let includes = []; + platforms.forEach((platform, index) => { + includes.push({ + index: index, + platform: platform + }); + }); + core.info(JSON.stringify(includes, null, 2)); + core.setOutput('includes', JSON.stringify(includes)); + }); + + build: + runs-on: ${{ inputs.runner }} + needs: + - prepare + outputs: + # needs predefined outputs as we can't use dynamic ones atm: https://github.com/actions/runner/pull/2477 + # 100 is the maximum number of platforms supported by the matrix strategy + digest_0: ${{ steps.digest.outputs.digest_0 }} + digest_1: ${{ steps.digest.outputs.digest_1 }} + digest_2: ${{ steps.digest.outputs.digest_2 }} + digest_3: ${{ steps.digest.outputs.digest_3 }} + digest_4: ${{ steps.digest.outputs.digest_4 }} + digest_5: ${{ steps.digest.outputs.digest_5 }} + digest_6: ${{ steps.digest.outputs.digest_6 }} + digest_7: ${{ steps.digest.outputs.digest_7 }} + digest_8: ${{ steps.digest.outputs.digest_8 }} + digest_9: ${{ steps.digest.outputs.digest_9 }} + digest_10: ${{ steps.digest.outputs.digest_10 }} + digest_11: ${{ steps.digest.outputs.digest_11 }} + digest_12: ${{ steps.digest.outputs.digest_12 }} + digest_13: ${{ steps.digest.outputs.digest_13 }} + digest_14: ${{ steps.digest.outputs.digest_14 }} + digest_15: ${{ steps.digest.outputs.digest_15 }} + digest_16: ${{ steps.digest.outputs.digest_16 }} + digest_17: ${{ steps.digest.outputs.digest_17 }} + digest_18: ${{ steps.digest.outputs.digest_18 }} + digest_19: ${{ steps.digest.outputs.digest_19 }} + digest_20: ${{ steps.digest.outputs.digest_20 }} + digest_21: ${{ steps.digest.outputs.digest_21 }} + digest_22: ${{ steps.digest.outputs.digest_22 }} + digest_23: ${{ steps.digest.outputs.digest_23 }} + digest_24: ${{ steps.digest.outputs.digest_24 }} + digest_25: ${{ steps.digest.outputs.digest_25 }} + digest_26: ${{ steps.digest.outputs.digest_26 }} + digest_27: ${{ steps.digest.outputs.digest_27 }} + digest_28: ${{ steps.digest.outputs.digest_28 }} + digest_29: ${{ steps.digest.outputs.digest_29 }} + digest_30: ${{ steps.digest.outputs.digest_30 }} + digest_31: ${{ steps.digest.outputs.digest_31 }} + digest_32: ${{ steps.digest.outputs.digest_32 }} + digest_33: ${{ steps.digest.outputs.digest_33 }} + digest_34: ${{ steps.digest.outputs.digest_34 }} + digest_35: ${{ steps.digest.outputs.digest_35 }} + digest_36: ${{ steps.digest.outputs.digest_36 }} + digest_37: ${{ steps.digest.outputs.digest_37 }} + digest_38: ${{ steps.digest.outputs.digest_38 }} + digest_39: ${{ steps.digest.outputs.digest_39 }} + digest_40: ${{ steps.digest.outputs.digest_40 }} + digest_41: ${{ steps.digest.outputs.digest_41 }} + digest_42: ${{ steps.digest.outputs.digest_42 }} + digest_43: ${{ steps.digest.outputs.digest_43 }} + digest_44: ${{ steps.digest.outputs.digest_44 }} + digest_45: ${{ steps.digest.outputs.digest_45 }} + digest_46: ${{ steps.digest.outputs.digest_46 }} + digest_47: ${{ steps.digest.outputs.digest_47 }} + digest_48: ${{ steps.digest.outputs.digest_48 }} + digest_49: ${{ steps.digest.outputs.digest_49 }} + digest_50: ${{ steps.digest.outputs.digest_50 }} + digest_51: ${{ steps.digest.outputs.digest_51 }} + digest_52: ${{ steps.digest.outputs.digest_52 }} + digest_53: ${{ steps.digest.outputs.digest_53 }} + digest_54: ${{ steps.digest.outputs.digest_54 }} + digest_55: ${{ steps.digest.outputs.digest_55 }} + digest_56: ${{ steps.digest.outputs.digest_56 }} + digest_57: ${{ steps.digest.outputs.digest_57 }} + digest_58: ${{ steps.digest.outputs.digest_58 }} + digest_59: ${{ steps.digest.outputs.digest_59 }} + digest_60: ${{ steps.digest.outputs.digest_60 }} + digest_61: ${{ steps.digest.outputs.digest_61 }} + digest_62: ${{ steps.digest.outputs.digest_62 }} + digest_63: ${{ steps.digest.outputs.digest_63 }} + digest_64: ${{ steps.digest.outputs.digest_64 }} + digest_65: ${{ steps.digest.outputs.digest_65 }} + digest_66: ${{ steps.digest.outputs.digest_66 }} + digest_67: ${{ steps.digest.outputs.digest_67 }} + digest_68: ${{ steps.digest.outputs.digest_68 }} + digest_69: ${{ steps.digest.outputs.digest_69 }} + digest_70: ${{ steps.digest.outputs.digest_70 }} + digest_71: ${{ steps.digest.outputs.digest_71 }} + digest_72: ${{ steps.digest.outputs.digest_72 }} + digest_73: ${{ steps.digest.outputs.digest_73 }} + digest_74: ${{ steps.digest.outputs.digest_74 }} + digest_75: ${{ steps.digest.outputs.digest_75 }} + digest_76: ${{ steps.digest.outputs.digest_76 }} + digest_77: ${{ steps.digest.outputs.digest_77 }} + digest_78: ${{ steps.digest.outputs.digest_78 }} + digest_79: ${{ steps.digest.outputs.digest_79 }} + digest_80: ${{ steps.digest.outputs.digest_80 }} + digest_81: ${{ steps.digest.outputs.digest_81 }} + digest_82: ${{ steps.digest.outputs.digest_82 }} + digest_83: ${{ steps.digest.outputs.digest_83 }} + digest_84: ${{ steps.digest.outputs.digest_84 }} + digest_85: ${{ steps.digest.outputs.digest_85 }} + digest_86: ${{ steps.digest.outputs.digest_86 }} + digest_87: ${{ steps.digest.outputs.digest_87 }} + digest_88: ${{ steps.digest.outputs.digest_88 }} + digest_89: ${{ steps.digest.outputs.digest_89 }} + digest_90: ${{ steps.digest.outputs.digest_90 }} + digest_91: ${{ steps.digest.outputs.digest_91 }} + digest_92: ${{ steps.digest.outputs.digest_92 }} + digest_93: ${{ steps.digest.outputs.digest_93 }} + digest_94: ${{ steps.digest.outputs.digest_94 }} + digest_95: ${{ steps.digest.outputs.digest_95 }} + digest_96: ${{ steps.digest.outputs.digest_96 }} + digest_97: ${{ steps.digest.outputs.digest_97 }} + digest_98: ${{ steps.digest.outputs.digest_98 }} + digest_99: ${{ steps.digest.outputs.digest_99 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.includes) }} + steps: + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.meta-image }} + tags: ${{ inputs.meta-tags }} + flavor: ${{ inputs.meta-flavor }} + labels: ${{ inputs.meta-labels }} + annotations: ${{ inputs.meta-annotations }} + - + name: Login to registry + uses: docker/login-action@v3 + if: ${{ inputs.push }} + with: + registry: ${{ inputs.login-registry }} + username: ${{ inputs.login-username }} + password: ${{ secrets.login-password }} + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build + id: bake + uses: docker/bake-action@v6 + with: + source: ${{ inputs.bake-source }} + files: | + ${{ inputs.bake-files }} + cwd://${{ steps.meta.outputs.bake-file }} + targets: ${{ inputs.target }} + allow: ${{ inputs.bake-allow }} + no-cache: ${{ inputs.bake-no-cache }} + pull: ${{ inputs.bake-pull }} + provenance: ${{ inputs.bake-provenance }} + sbom: ${{ inputs.bake-sbom }} + set: | + ${{ inputs.bake-set }} + *.tags= + *.platform=${{ matrix.platform }} + *.output=type=image,"name=${{ inputs.meta-image }}",push-by-digest=true,name-canonical=true,push=${{ inputs.push }} + - + name: Set digest output + id: digest + uses: actions/github-script@v7 + with: + script: | + const metadata = JSON.parse(`${{ steps.bake.outputs.metadata }}`); + const digest = metadata[`${{ inputs.target }}`]['containerimage.digest']; + const outputKey = `digest_${{ matrix.index }}`; + core.info(`Setting digest output: ${outputKey}=${digest}`); + core.setOutput(outputKey, digest); + + merge: + runs-on: ubuntu-latest + needs: + - build + steps: + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.meta-image }} + tags: ${{ inputs.meta-tags }} + flavor: ${{ inputs.meta-flavor }} + labels: ${{ inputs.meta-labels }} + annotations: ${{ inputs.meta-annotations }} + - + name: Login to registry + uses: docker/login-action@v3 + if: ${{ inputs.push }} + with: + registry: ${{ inputs.login-registry }} + username: ${{ inputs.login-username }} + password: ${{ secrets.login-password }} + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: ${{ inputs.push }} + - + name: Create manifest list + uses: actions/github-script@v7 + with: + script: | + let digests = []; + await core.group(`Digests`, async () => { + digests = Object.values(JSON.parse(`${{ toJSON(needs.build.outputs) }}`)); + core.info(JSON.stringify(digests, null, 2)); + }); + + let tags = []; + await core.group(`Tags`, async () => { + tags = `${{ steps.meta.outputs.tags }}`.split('\n').filter(Boolean); + core.info(JSON.stringify(tags, null, 2)); + }); + + let createArgs = ['buildx', 'imagetools', 'create']; + for (const tag of tags) { + createArgs.push(`-t`, tag); + } + for (const digest of digests) { + createArgs.push(`${{ inputs.meta-image }}@${digest}`); + } + + if (${{ inputs.push }}) { + await exec.getExecOutput('docker', createArgs, { + ignoreReturnCode: true + }).then(res => { + if (res.stderr.length > 0 && res.exitCode != 0) { + throw new Error(res.stderr); + } + }); + await core.group(`Inspect image`, async () => { + await exec.getExecOutput('docker', ['buildx', 'imagetools', 'inspect', `${{ inputs.meta-image }}:${tags[0]}`], { + ignoreReturnCode: true + }).then(res => { + if (res.stderr.length > 0 && res.exitCode != 0) { + throw new Error(res.stderr); + } + }); + }); + } else { + await core.group(`Generated imagetools create command`, async () => { + core.info(`docker ${createArgs.join(' ')}`); + }); + core.info(`Push is disabled, skipping manifest list creation`); + } From 5e4cf03cf93ce70b14bb3c261476d2e9a2e30968 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sat, 15 Feb 2025 23:04:16 +0100 Subject: [PATCH 02/17] distribute: use the most performant runner based on the platform to build Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/distribute.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/distribute.yml b/.github/workflows/distribute.yml index 8b6f4904..e01175a1 100644 --- a/.github/workflows/distribute.yml +++ b/.github/workflows/distribute.yml @@ -8,7 +8,7 @@ on: type: string description: "Runner instance" required: false - default: 'ubuntu-latest' + default: 'auto' target: type: string description: "Multi-platform target to build" @@ -151,9 +151,14 @@ jobs: } let includes = []; platforms.forEach((platform, index) => { + let runner = `${{ inputs.runner }}`; + if (runner === 'auto') { + runner = platform.startsWith('linux/arm') ? 'ubuntu-24.04-arm' : 'ubuntu-latest'; + } includes.push({ index: index, - platform: platform + platform: platform, + runner: runner }); }); core.info(JSON.stringify(includes, null, 2)); @@ -161,7 +166,7 @@ jobs: }); build: - runs-on: ${{ inputs.runner }} + runs-on: ${{ matrix.runner }} needs: - prepare outputs: From d56a1c8a4ec372b4400ee8254405ebd82200b9c1 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sat, 15 Feb 2025 23:29:29 +0100 Subject: [PATCH 03/17] distribute: qemu opts Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/distribute.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/distribute.yml b/.github/workflows/distribute.yml index e01175a1..b91a2c95 100644 --- a/.github/workflows/distribute.yml +++ b/.github/workflows/distribute.yml @@ -18,6 +18,11 @@ on: description: "Push image to registry" required: false default: false + setup-qemu: + type: boolean + description: "Install QEMU static binaries" + required: false + default: true # same as docker/metadata-action inputs meta-image: type: string @@ -53,6 +58,11 @@ on: description: 'Specifies whether the given registry is ECR (auto, true or false)' default: 'auto' required: false + # same as docker/setup-qemu-action inputs (minus platforms, cache-image) + qemu-image: + type: string + description: 'QEMU static binaries Docker image (e.g. tonistiigi/binfmt:latest)' + required: false # same as docker/bake-action inputs (minus builder, targets, load, push) bake-source: type: string @@ -298,6 +308,9 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 + if: ${{ inputs.setup-qemu }} + with: + image: ${{ inputs.qemu-image }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 400ed06e020f203ae39a7d50ed50abd74431683c Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sat, 15 Feb 2025 23:45:20 +0100 Subject: [PATCH 04/17] ci: test distribute workflow Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2fb79a1..a7575891 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -733,3 +733,11 @@ jobs: ./test/config.hcl allow: network.host targets: app-entitlements + + distribute: + uses: ./.github/workflows/distribute.yml + with: + target: app-plus + push: false + meta-image: user/app + bake-files: ./test/config.hcl From 7fe874197b4c88fe2ea685901b4795a39fc63861 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sun, 16 Feb 2025 11:35:07 +0100 Subject: [PATCH 05/17] distribute: missing bake-target for metadata-action and verify tags Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/distribute.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/distribute.yml b/.github/workflows/distribute.yml index b91a2c95..cd5aca33 100644 --- a/.github/workflows/distribute.yml +++ b/.github/workflows/distribute.yml @@ -23,7 +23,7 @@ on: description: "Install QEMU static binaries" required: false default: true - # same as docker/metadata-action inputs + # same as docker/metadata-action inputs (minus sep-tags, sep-labels, sep-annotations) meta-image: type: string description: "Image to use as base name for tags" @@ -44,6 +44,10 @@ on: type: string description: 'List of custom annotations' required: false + meta-bake-target: + type: string + description: 'Bake target name (default docker-metadata-action)' + required: false # same as docker/login-action inputs (minus logout) login-registry: type: string @@ -297,6 +301,7 @@ jobs: flavor: ${{ inputs.meta-flavor }} labels: ${{ inputs.meta-labels }} annotations: ${{ inputs.meta-annotations }} + bake-target: ${{ inputs.meta-bake-target }} - name: Login to registry uses: docker/login-action@v3 @@ -361,6 +366,7 @@ jobs: flavor: ${{ inputs.meta-flavor }} labels: ${{ inputs.meta-labels }} annotations: ${{ inputs.meta-annotations }} + bake-target: ${{ inputs.meta-bake-target }} - name: Login to registry uses: docker/login-action@v3 @@ -399,6 +405,9 @@ jobs: } if (${{ inputs.push }}) { + if (tags.length === 0) { + throw new Error('No tags to create manifest list'); + } await exec.getExecOutput('docker', createArgs, { ignoreReturnCode: true }).then(res => { From 1f247ae16a0597c80f345c0cbc36cfde98d425e8 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Sun, 16 Feb 2025 23:37:02 +0100 Subject: [PATCH 06/17] distribute: use toolkit to parse bake definition Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/distribute.yml | 48 +++++++++++++++++++------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/distribute.yml b/.github/workflows/distribute.yml index cd5aca33..cea14d03 100644 --- a/.github/workflows/distribute.yml +++ b/.github/workflows/distribute.yml @@ -100,6 +100,7 @@ on: bake-sbom: type: string description: "SBOM is a shorthand for --set=*.attest=type=sbom" + required: false bake-set: type: string description: "List of targets values to override (eg. targetpattern.key=value)" @@ -108,6 +109,9 @@ on: login-password: description: "Password or personal access token used to log against the Docker registry" required: false + github-token: + description: "API token used to authenticate to a Git repository for remote definitions" + required: false jobs: prepare: @@ -125,7 +129,6 @@ jobs: with: script: | let def; - const source = `${{ inputs.bake-source }}`; const files = `${{ inputs.bake-files }}` ? `${{ inputs.bake-files }}`.split(/[\r?\n,]+/).filter(Boolean) : []; const target = `${{ inputs.target }}`; @@ -134,26 +137,32 @@ jobs: throw new Error('Only one meta-image is allowed'); } - await core.group(`Validating definition`, async () => { - let args = ['buildx', 'bake']; - for (const file of files) { - args.push('--file', file); - } - if (source && source !== '.') { - args.push(source); - } - args.push(target, '--print'); + await core.group(`Install docker/actions-toolkit`, async () => { + await exec.exec('npm', ['install', '@docker/actions-toolkit']); + }); - const res = await exec.getExecOutput('docker', args, { - ignoreReturnCode: true, - silent: true, - cwd: `${{ inputs.bake-workdir }}` - }); - if (res.stderr.length > 0 && res.exitCode != 0) { - throw new Error(res.stderr); + await core.group(`Validating definition`, async () => { + const { Bake } = require('@docker/actions-toolkit/lib/buildx/bake'); + const bake = new Bake(); + def = await bake.getDefinition( + { + allow: `${{ inputs.bake-allow }}` ? `${{ inputs.bake-allow }}`.split(/[\r?\n,]+/).filter(Boolean) : [], + files: `${{ inputs.bake-files }}` ? `${{ inputs.bake-files }}`.split(/[\r?\n,]+/).filter(Boolean) : [], + noCache: ${{ inputs.bake-no-cache }}, + overrides: `${{ inputs.bake-set }}` ? `${{ inputs.bake-set }}`.split(/[\r?\n,]+/).filter(Boolean) : [], + provenance: `${{ inputs.bake-provenance }}`, + sbom: `${{ inputs.bake-sbom }}`, + source: `${{ inputs.bake-source }}`, + targets: [`${{ inputs.target }}`], + githubToken: `${{ secrets.github-token || github.token }}` + }, + { + cwd: `${{ inputs.bake-workdir }}` + } + ); + if (!def) { + throw new Error('Bake definition not set'); } - def = JSON.parse(res.stdout.trim()); - core.info(JSON.stringify(def, null, 2)); }); await core.group(`Set includes`, async () => { @@ -339,6 +348,7 @@ jobs: *.tags= *.platform=${{ matrix.platform }} *.output=type=image,"name=${{ inputs.meta-image }}",push-by-digest=true,name-canonical=true,push=${{ inputs.push }} + github-token: ${{ secrets.github-token || github.token }} - name: Set digest output id: digest From 5f340452d924b03c5c755c9a63f244e15fd99f98 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Feb 2025 02:21:08 +0100 Subject: [PATCH 07/17] distribute: handle git context Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/distribute.yml | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/distribute.yml b/.github/workflows/distribute.yml index cea14d03..de03c49b 100644 --- a/.github/workflows/distribute.yml +++ b/.github/workflows/distribute.yml @@ -67,7 +67,7 @@ on: type: string description: 'QEMU static binaries Docker image (e.g. tonistiigi/binfmt:latest)' required: false - # same as docker/bake-action inputs (minus builder, targets, load, push) + # same as docker/bake-action inputs (minus workdir, builder, targets, load, push) bake-source: type: string description: "Context to build from. Can be either local or a remote bake definition" @@ -80,11 +80,6 @@ on: type: string description: "List of bake definition files" required: false - bake-workdir: - type: string - description: "Working directory of bake execution" - required: false - default: '.' bake-no-cache: type: boolean description: "Do not use cache when building the image" @@ -119,9 +114,6 @@ jobs: outputs: includes: ${{ steps.set.outputs.includes }} steps: - - - name: Checkout - uses: actions/checkout@v4 - name: Set includes id: set @@ -137,29 +129,37 @@ jobs: throw new Error('Only one meta-image is allowed'); } - await core.group(`Install docker/actions-toolkit`, async () => { - await exec.exec('npm', ['install', '@docker/actions-toolkit']); + await core.group(`Install npm dependencies`, async () => { + await exec.exec('npm', ['install', '@docker/actions-toolkit', 'handlebars']); }); await core.group(`Validating definition`, async () => { + const handlebars = require('handlebars'); + const { Context } = require('@docker/actions-toolkit/lib/context'); const { Bake } = require('@docker/actions-toolkit/lib/buildx/bake'); + + let source = handlebars.compile(`${{ inputs.bake-source }}`)({ + defaultContext: Context.gitContext() + }); + if (!source) { + source = Context.gitContext(); + } + if (source === '.') { + source = ''; + } + const bake = new Bake(); - def = await bake.getDefinition( - { - allow: `${{ inputs.bake-allow }}` ? `${{ inputs.bake-allow }}`.split(/[\r?\n,]+/).filter(Boolean) : [], - files: `${{ inputs.bake-files }}` ? `${{ inputs.bake-files }}`.split(/[\r?\n,]+/).filter(Boolean) : [], - noCache: ${{ inputs.bake-no-cache }}, - overrides: `${{ inputs.bake-set }}` ? `${{ inputs.bake-set }}`.split(/[\r?\n,]+/).filter(Boolean) : [], - provenance: `${{ inputs.bake-provenance }}`, - sbom: `${{ inputs.bake-sbom }}`, - source: `${{ inputs.bake-source }}`, - targets: [`${{ inputs.target }}`], - githubToken: `${{ secrets.github-token || github.token }}` - }, - { - cwd: `${{ inputs.bake-workdir }}` - } - ); + def = await bake.getDefinition({ + allow: `${{ inputs.bake-allow }}` ? `${{ inputs.bake-allow }}`.split(/[\r?\n,]+/).filter(Boolean) : [], + files: `${{ inputs.bake-files }}` ? `${{ inputs.bake-files }}`.split(/[\r?\n,]+/).filter(Boolean) : [], + noCache: ${{ inputs.bake-no-cache }}, + overrides: `${{ inputs.bake-set }}` ? `${{ inputs.bake-set }}`.split(/[\r?\n,]+/).filter(Boolean) : [], + provenance: `${{ inputs.bake-provenance }}`, + sbom: `${{ inputs.bake-sbom }}`, + source: source, + targets: [`${{ inputs.target }}`], + githubToken: `${{ secrets.github-token || github.token }}` + }); if (!def) { throw new Error('Bake definition not set'); } From b7a398645c7e3b1ac50aaf3f16a03ff84df7825f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Feb 2025 10:02:25 +0100 Subject: [PATCH 08/17] distribute: pin dependencies and sanitize inputs Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/distribute.yml | 109 +++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 36 deletions(-) diff --git a/.github/workflows/distribute.yml b/.github/workflows/distribute.yml index de03c49b..d1507404 100644 --- a/.github/workflows/distribute.yml +++ b/.github/workflows/distribute.yml @@ -84,10 +84,12 @@ on: type: boolean description: "Do not use cache when building the image" required: false + default: false bake-pull: type: boolean description: "Always attempt to pull a newer version of the image" required: false + default: false bake-provenance: type: string description: "Provenance is a shorthand for --set=*.attest=type=provenance" @@ -108,57 +110,84 @@ on: description: "API token used to authenticate to a Git repository for remote definitions" required: false +env: + ACTIONS_TOOLKIT_VERSION: "0.54.0" + HANDLEBARS_VERSION: "4.7.8" + jobs: prepare: runs-on: ubuntu-latest outputs: includes: ${{ steps.set.outputs.includes }} steps: + - + name: Install npm dependencies + uses: actions/github-script@v7 + with: + script: | + await exec.exec('npm', ['install', + '@docker/actions-toolkit@${{ env.ACTIONS_TOOLKIT_VERSION }}', + 'handlebars@${{ env.HANDLEBARS_VERSION }}' + ]); - name: Set includes id: set uses: actions/github-script@v7 + env: + INPUT_RUNNER: ${{ inputs.runner }} + INPUT_TARGET: ${{ inputs.target }} + INPUT_META-IMAGE: ${{ inputs.meta-image }} + INPUT_BAKE-ALLOW: ${{ inputs.bake-allow }} + INPUT_BAKE-FILES: ${{ inputs.bake-files }} + INPUT_BAKE-NO-CACHE: ${{ inputs.bake-no-cache }} + INPUT_BAKE-PROVENANCE: ${{ inputs.bake-provenance }} + INPUT_BAKE-SBOM: ${{ inputs.bake-sbom }} + INPUT_BAKE-SET: ${{ inputs.bake-set }} + INPUT_BAKE-SOURCE: ${{ inputs.bake-source }} + GITHUB_TOKEN: ${{ secrets.github-token || github.token }} with: script: | - let def; - const files = `${{ inputs.bake-files }}` ? `${{ inputs.bake-files }}`.split(/[\r?\n,]+/).filter(Boolean) : []; - const target = `${{ inputs.target }}`; + const handlebars = require('handlebars'); + const { Bake } = require('@docker/actions-toolkit/lib/buildx/bake'); + const { Build } = require('@docker/actions-toolkit/lib/buildx/build'); + const { Context } = require('@docker/actions-toolkit/lib/context'); + const { Util } = require('@docker/actions-toolkit/lib/util'); - const metaImage = `${{ inputs.meta-image }}` ? `${{ inputs.meta-image }}`.split(/[\r?\n,]+/).filter(Boolean) : []; - if (metaImage.length > 1) { + if (Util.getInputList('meta-image').length > 1) { throw new Error('Only one meta-image is allowed'); } - await core.group(`Install npm dependencies`, async () => { - await exec.exec('npm', ['install', '@docker/actions-toolkit', 'handlebars']); + const inpRunner = core.getInput('runner'); + const inpTarget = core.getInput('target'); + const inpBakeAllow = Util.getInputList('bake-allow'); + const inpBakeFiles = Util.getInputList('bake-files'); + const inpBakeNoCache = core.getBooleanInput('bake-no-cache'); + const inpBakeProvenance = Build.getProvenanceInput('bake-provenance'); + const inpBakeSbom = core.getInput('bake-sbom'); + const inpBakeSet = Util.getInputList('bake-set', {ignoreComma: true, quote: false}); + let inpBakeSource = handlebars.compile(core.getInput('source'))({ + defaultContext: Context.gitContext() }); + if (!inpBakeSource) { + inpBakeSource = Context.gitContext(); + } + if (inpBakeSource === '.') { + inpBakeSource = ''; + } + let def; await core.group(`Validating definition`, async () => { - const handlebars = require('handlebars'); - const { Context } = require('@docker/actions-toolkit/lib/context'); - const { Bake } = require('@docker/actions-toolkit/lib/buildx/bake'); - - let source = handlebars.compile(`${{ inputs.bake-source }}`)({ - defaultContext: Context.gitContext() - }); - if (!source) { - source = Context.gitContext(); - } - if (source === '.') { - source = ''; - } - const bake = new Bake(); def = await bake.getDefinition({ - allow: `${{ inputs.bake-allow }}` ? `${{ inputs.bake-allow }}`.split(/[\r?\n,]+/).filter(Boolean) : [], - files: `${{ inputs.bake-files }}` ? `${{ inputs.bake-files }}`.split(/[\r?\n,]+/).filter(Boolean) : [], - noCache: ${{ inputs.bake-no-cache }}, - overrides: `${{ inputs.bake-set }}` ? `${{ inputs.bake-set }}`.split(/[\r?\n,]+/).filter(Boolean) : [], - provenance: `${{ inputs.bake-provenance }}`, - sbom: `${{ inputs.bake-sbom }}`, - source: source, - targets: [`${{ inputs.target }}`], - githubToken: `${{ secrets.github-token || github.token }}` + allow: inpBakeAllow, + files: inpBakeFiles, + noCache: inpBakeNoCache, + overrides: inpBakeSet, + provenance: inpBakeProvenance, + sbom: inpBakeSbom, + source: inpBakeSource, + targets: [inpTarget], + githubToken: process.env.GITHUB_TOKEN }); if (!def) { throw new Error('Bake definition not set'); @@ -166,7 +195,7 @@ jobs: }); await core.group(`Set includes`, async () => { - const platforms = def.target[target].platforms; + const platforms = def.target[inpTarget].platforms; if (platforms.length > 100) { throw new Error('Too many platforms'); } else if (platforms.length <= 1) { @@ -174,7 +203,7 @@ jobs: } let includes = []; platforms.forEach((platform, index) => { - let runner = `${{ inputs.runner }}`; + let runner = inpRunner; if (runner === 'auto') { runner = platform.startsWith('linux/arm') ? 'ubuntu-24.04-arm' : 'ubuntu-latest'; } @@ -353,10 +382,12 @@ jobs: name: Set digest output id: digest uses: actions/github-script@v7 + env: + INPUT_TARGET: ${{ inputs.target }} with: script: | const metadata = JSON.parse(`${{ steps.bake.outputs.metadata }}`); - const digest = metadata[`${{ inputs.target }}`]['containerimage.digest']; + const digest = metadata[core.getInput('target')]['containerimage.digest']; const outputKey = `digest_${{ matrix.index }}`; core.info(`Setting digest output: ${outputKey}=${digest}`); core.setOutput(outputKey, digest); @@ -392,8 +423,14 @@ jobs: - name: Create manifest list uses: actions/github-script@v7 + env: + INPUT_PUSH: ${{ inputs.push }} + INPUT_META-IMAGE: ${{ inputs.meta-image }} with: script: | + const inpPush = core.getBooleanInput('push'); + const inpMetaImage = core.getInput('meta-image'); + let digests = []; await core.group(`Digests`, async () => { digests = Object.values(JSON.parse(`${{ toJSON(needs.build.outputs) }}`)); @@ -411,10 +448,10 @@ jobs: createArgs.push(`-t`, tag); } for (const digest of digests) { - createArgs.push(`${{ inputs.meta-image }}@${digest}`); + createArgs.push(`${inpMetaImage}@${digest}`); } - if (${{ inputs.push }}) { + if (inpPush) { if (tags.length === 0) { throw new Error('No tags to create manifest list'); } @@ -426,7 +463,7 @@ jobs: } }); await core.group(`Inspect image`, async () => { - await exec.getExecOutput('docker', ['buildx', 'imagetools', 'inspect', `${{ inputs.meta-image }}:${tags[0]}`], { + await exec.getExecOutput('docker', ['buildx', 'imagetools', 'inspect', `${inpMetaImage}:${tags[0]}`], { ignoreReturnCode: true }).then(res => { if (res.stderr.length > 0 && res.exitCode != 0) { From fc79706edd7afa6740c69bd8f60b9c93ec3810a9 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Feb 2025 10:36:48 +0100 Subject: [PATCH 09/17] distribute: rename workflow Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/{distribute.yml => reusable-distribute.yml} | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) rename .github/workflows/{distribute.yml => reusable-distribute.yml} (99%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7575891..ff9bbeb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -735,7 +735,7 @@ jobs: targets: app-entitlements distribute: - uses: ./.github/workflows/distribute.yml + uses: ./.github/workflows/reusable-distribute.yml with: target: app-plus push: false diff --git a/.github/workflows/distribute.yml b/.github/workflows/reusable-distribute.yml similarity index 99% rename from .github/workflows/distribute.yml rename to .github/workflows/reusable-distribute.yml index d1507404..8df4631f 100644 --- a/.github/workflows/distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -1,4 +1,5 @@ -name: distribute +# Reusable workflow to distribute multi-platform builds efficiently +name: reusable-distribute on: workflow_call: From 989409fc5a7d25d4d30a8879256c433614fc3c5f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Feb 2025 11:34:31 +0100 Subject: [PATCH 10/17] distribute: opt to set meta annotations and labels Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/reusable-distribute.yml | 56 +++++++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute.yml index 8df4631f..5be2d609 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -1,4 +1,5 @@ # Reusable workflow to distribute multi-platform builds efficiently +# https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners name: reusable-distribute on: @@ -19,6 +20,16 @@ on: description: "Push image to registry" required: false default: false + set-meta-annotations: + type: boolean + description: "Set metadata-action annotations" + required: false + default: false + set-meta-labels: + type: boolean + description: "Set metadata-action labels" + required: false + default: false setup-qemu: type: boolean description: "Install QEMU static binaries" @@ -358,15 +369,52 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - + name: Set bake files + id: bake-files + uses: actions/github-script@v7 + env: + INPUT_SET-META-ANNOTATIONS: ${{ inputs.set-meta-annotations }} + INPUT_SET-META-LABELS: ${{ inputs.set-meta-labels }} + INPUT_BAKE-FILES: ${{ inputs.bake-files }} + with: + result-encoding: string + script: | + await core.group(`Installing npm dependencies`, async () => { + await exec.exec('npm', ['install', + '@docker/actions-toolkit@${{ env.ACTIONS_TOOLKIT_VERSION }}' + ]); + }); + + const os = require('os'); + const { Util } = require('@docker/actions-toolkit/lib/util'); + + let bakeFiles = Util.getInputList('bake-files'); + const inpSetMetaAnnotations = core.getBooleanInput('set-meta-annotations'); + const inpSetMetaLabels = core.getBooleanInput('set-meta-labels'); + + await core.group(`Set bake files`, async () => { + if (bakeFiles.length === 0) { + bakeFiles = ['docker-bake.hcl']; + } + bakeFiles.push(`cwd://${{ steps.meta.outputs.bake-file-tags }}`); + if (inpSetMetaAnnotations) { + bakeFiles.push(`cwd://${{ steps.meta.outputs.bake-file-annotations }}`); + } + if (inpSetMetaLabels) { + bakeFiles.push(`cwd://${{ steps.meta.outputs.bake-file-labels }}`); + } + core.info(JSON.stringify(bakeFiles, null, 2)); + }); + + return bakeFiles.join(os.EOL); - name: Build id: bake uses: docker/bake-action@v6 with: source: ${{ inputs.bake-source }} - files: | - ${{ inputs.bake-files }} - cwd://${{ steps.meta.outputs.bake-file }} + files: ${{ steps.bake-files.outputs.result }} targets: ${{ inputs.target }} allow: ${{ inputs.bake-allow }} no-cache: ${{ inputs.bake-no-cache }} @@ -406,8 +454,6 @@ jobs: images: ${{ inputs.meta-image }} tags: ${{ inputs.meta-tags }} flavor: ${{ inputs.meta-flavor }} - labels: ${{ inputs.meta-labels }} - annotations: ${{ inputs.meta-annotations }} bake-target: ${{ inputs.meta-bake-target }} - name: Login to registry From bd6e8b619b2558fa84e7cacb1ccb175d48cbb8c3 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:25:54 +0100 Subject: [PATCH 11/17] distribute: allow login-username as secret Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/reusable-distribute.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute.yml index 5be2d609..7d85aada 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -115,6 +115,9 @@ on: description: "List of targets values to override (eg. targetpattern.key=value)" required: false secrets: + login-username: + description: 'Username used to log against the Docker registry' + required: false login-password: description: "Password or personal access token used to log against the Docker registry" required: false @@ -358,7 +361,7 @@ jobs: if: ${{ inputs.push }} with: registry: ${{ inputs.login-registry }} - username: ${{ inputs.login-username }} + username: ${{ inputs.login-username || secrets.login-username }} password: ${{ secrets.login-password }} - name: Set up QEMU @@ -461,7 +464,7 @@ jobs: if: ${{ inputs.push }} with: registry: ${{ inputs.login-registry }} - username: ${{ inputs.login-username }} + username: ${{ inputs.login-username || secrets.login-username }} password: ${{ secrets.login-password }} - name: Set up Docker Buildx From 99087c3155f10a1e82337bb85d3dac469def85ea Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 19 Feb 2025 02:02:20 +0100 Subject: [PATCH 12/17] distribute: setup-buildx opts Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/reusable-distribute.yml | 41 ++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute.yml index 7d85aada..59702bff 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -74,6 +74,32 @@ on: description: 'Specifies whether the given registry is ECR (auto, true or false)' default: 'auto' required: false + # same as docker/setup-buildx-action inputs (minus driver, install, use, endpoint, append, cleanup) + buildx-version: + type: string + description: 'Buildx version. (eg. v0.3.0)' + required: false + buildx-cache-binary: + type: boolean + description: 'Cache buildx binary to GitHub Actions cache backend' + default: true + required: false + buildx-driver-opts: + type: string + description: 'List of additional docker-container options. (eg. image=moby/buildkit:master)' + required: false + buildkitd-flags: + type: string + description: 'BuildKit daemon flags' + required: false + buildkitd-config: + type: string + description: 'BuildKit daemon config file' + required: false + buildkitd-config-inline: + type: string + description: 'Inline BuildKit daemon config' + required: false # same as docker/setup-qemu-action inputs (minus platforms, cache-image) qemu-image: type: string @@ -372,6 +398,13 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + version: ${{ inputs.buildx-version }} + driver-opts: ${{ inputs.buildx-driver-opts }} + buildkitd-flags: ${{ inputs.buildkitd-flags }} + buildkitd-config: ${{ inputs.buildkitd-config }} + buildkitd-config-inline: ${{ inputs.buildkitd-config-inline }} + cache-binary: ${{ inputs.buildx-cache-binary }} - name: Set bake files id: bake-files @@ -469,7 +502,13 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - if: ${{ inputs.push }} + with: + version: ${{ inputs.buildx-version }} + driver-opts: ${{ inputs.buildx-driver-opts }} + buildkitd-flags: ${{ inputs.buildkitd-flags }} + buildkitd-config: ${{ inputs.buildkitd-config }} + buildkitd-config-inline: ${{ inputs.buildkitd-config-inline }} + cache-binary: ${{ inputs.buildx-cache-binary }} - name: Create manifest list uses: actions/github-script@v7 From 1e4f4ca53400259b8b30bc2fc66c783da252866b Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 19 Feb 2025 02:28:06 +0100 Subject: [PATCH 13/17] distribute: gha cache Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/ci.yml | 10 +++ .github/workflows/reusable-distribute.yml | 82 +++++++++++++++++++++-- test/config.hcl | 3 + test/go/docker-bake.hcl | 12 ++++ 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff9bbeb2..afd8a60f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -741,3 +741,13 @@ jobs: push: false meta-image: user/app bake-files: ./test/config.hcl + + distribute-cache: + uses: ./.github/workflows/reusable-distribute.yml + with: + target: image-all + push: false + cache: true + cache-scope: go-app + meta-image: user/goapp + bake-source: "{{defaultContext}}:test/go" diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute.yml index 59702bff..99ea3dfc 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -20,6 +20,20 @@ on: description: "Push image to registry" required: false default: false + cache: + type: boolean + description: "Enable cache to GitHub Actions cache backend" + required: false + default: false + cache-scope: + type: string + description: "Which scope cache object belongs to if cache enabled (default is target name)" + required: false + cache-mode: + type: string + description: "Cache layers to export if cache enabled (min or max)" + required: false + default: 'min' set-meta-annotations: type: boolean description: "Set metadata-action annotations" @@ -258,10 +272,48 @@ jobs: core.setOutput('includes', JSON.stringify(includes)); }); + warmup: + runs-on: ${{ inputs.runner == 'auto' && 'ubuntu-latest' || inputs.runner }} + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + if: ${{ inputs.setup-qemu }} + with: + image: ${{ inputs.qemu-image }} + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: ${{ inputs.buildx-version }} + driver-opts: ${{ inputs.buildx-driver-opts }} + buildkitd-flags: ${{ inputs.buildkitd-flags }} + buildkitd-config: ${{ inputs.buildkitd-config }} + buildkitd-config-inline: ${{ inputs.buildkitd-config-inline }} + cache-binary: ${{ inputs.buildx-cache-binary }} + - + name: Warm up cache + uses: docker/bake-action@v6 + if: ${{ inputs.cache }} + with: + source: ${{ inputs.bake-source }} + files: ${{ inputs.bake-files }} + targets: ${{ inputs.target }} + provenance: false + sbom: false + set: | + ${{ inputs.bake-set }} + *.platform= + *.output=type=cacheonly + *.cache-from=type=gha,scope=${{ inputs.cache-scope || inputs.target }}-warmup + *.cache-to=type=gha,scope=${{ inputs.cache-scope || inputs.target }}-warmup,mode=${{ inputs.cache-mode }} + github-token: ${{ secrets.github-token || github.token }} + build: runs-on: ${{ matrix.runner }} needs: - prepare + - warmup outputs: # needs predefined outputs as we can't use dynamic ones atm: https://github.com/actions/runner/pull/2477 # 100 is the maximum number of platforms supported by the matrix strategy @@ -406,15 +458,19 @@ jobs: buildkitd-config-inline: ${{ inputs.buildkitd-config-inline }} cache-binary: ${{ inputs.buildx-cache-binary }} - - name: Set bake files - id: bake-files + name: Set bake files and overrides + id: bake-opts uses: actions/github-script@v7 env: + PLATFORM: ${{ matrix.platform }} + INPUT_TARGET: ${{ inputs.target }} + INPUT_CACHE: ${{ inputs.cache }} + INPUT_CACHE-SCOPE: ${{ inputs.cache-scope }} + INPUT_CACHE-MODE: ${{ inputs.cache-mode }} INPUT_SET-META-ANNOTATIONS: ${{ inputs.set-meta-annotations }} INPUT_SET-META-LABELS: ${{ inputs.set-meta-labels }} INPUT_BAKE-FILES: ${{ inputs.bake-files }} with: - result-encoding: string script: | await core.group(`Installing npm dependencies`, async () => { await exec.exec('npm', ['install', @@ -425,6 +481,11 @@ jobs: const os = require('os'); const { Util } = require('@docker/actions-toolkit/lib/util'); + const platformPair = process.env.PLATFORM.replace(/\//g, '-'); + const inpTarget = core.getInput('target'); + const inpCache = core.getBooleanInput('cache'); + const inpCacheScope = core.getInput('cache-scope'); + const inpCacheMode = core.getInput('cache-mode'); let bakeFiles = Util.getInputList('bake-files'); const inpSetMetaAnnotations = core.getBooleanInput('set-meta-annotations'); const inpSetMetaLabels = core.getBooleanInput('set-meta-labels'); @@ -441,16 +502,26 @@ jobs: bakeFiles.push(`cwd://${{ steps.meta.outputs.bake-file-labels }}`); } core.info(JSON.stringify(bakeFiles, null, 2)); + core.setOutput('files', bakeFiles.join(os.EOL)); }); - return bakeFiles.join(os.EOL); + await core.group(`Set bake overrides`, async () => { + let bakeOverrides = []; + if (inpCache) { + bakeOverrides.push(`*.cache-from=type=gha,scope=${inpCacheScope || inpTarget}-warmup`); + bakeOverrides.push(`*.cache-from=type=gha,scope=${inpCacheScope || inpTarget}-${platformPair}`); + bakeOverrides.push(`*.cache-to=type=gha,scope=${inpCacheScope || inpTarget}-${platformPair},mode=${inpCacheMode}`); + } + core.info(JSON.stringify(bakeOverrides, null, 2)); + core.setOutput('overrides', bakeOverrides.join(os.EOL)); + }); - name: Build id: bake uses: docker/bake-action@v6 with: source: ${{ inputs.bake-source }} - files: ${{ steps.bake-files.outputs.result }} + files: ${{ steps.bake-opts.outputs.files }} targets: ${{ inputs.target }} allow: ${{ inputs.bake-allow }} no-cache: ${{ inputs.bake-no-cache }} @@ -459,6 +530,7 @@ jobs: sbom: ${{ inputs.bake-sbom }} set: | ${{ inputs.bake-set }} + ${{ steps.bake-opts.outputs.overrides }} *.tags= *.platform=${{ matrix.platform }} *.output=type=image,"name=${{ inputs.meta-image }}",push-by-digest=true,name-canonical=true,push=${{ inputs.push }} diff --git a/test/config.hcl b/test/config.hcl index 6e48771b..47303bf6 100644 --- a/test/config.hcl +++ b/test/config.hcl @@ -1,3 +1,5 @@ +target "docker-metadata-action" {} + group "default" { targets = ["db", "app"] } @@ -12,6 +14,7 @@ target "db" { } target "app" { + inherits = ["docker-metadata-action"] context = "./test" dockerfile = "Dockerfile" args = { diff --git a/test/go/docker-bake.hcl b/test/go/docker-bake.hcl index 639f5c47..52008482 100644 --- a/test/go/docker-bake.hcl +++ b/test/go/docker-bake.hcl @@ -1,3 +1,5 @@ +target "docker-metadata-action" {} + variable "DESTDIR" { default = "/tmp/bake-build" } @@ -15,3 +17,13 @@ target "image" { target = "image" tags = ["localhost:5000/name/app:latest"] } + +target "image-all" { + inherits = ["binary"] + platforms = [ + "linux/amd64", + "linux/arm64", + "linux/arm/v7", + "linux/arm/v6" + ] +} From 85ad91474ddc9db0bd6590db39123d7fda1ea57f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 19 Feb 2025 02:48:52 +0100 Subject: [PATCH 14/17] distribute: fix typo Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/reusable-distribute.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute.yml index 99ea3dfc..19699bf4 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -220,7 +220,7 @@ jobs: const inpBakeProvenance = Build.getProvenanceInput('bake-provenance'); const inpBakeSbom = core.getInput('bake-sbom'); const inpBakeSet = Util.getInputList('bake-set', {ignoreComma: true, quote: false}); - let inpBakeSource = handlebars.compile(core.getInput('source'))({ + let inpBakeSource = handlebars.compile(core.getInput('bake-source'))({ defaultContext: Context.gitContext() }); if (!inpBakeSource) { From d5c7867b65908e3d10028277b343c0ccd50e2e54 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 19 Feb 2025 03:05:01 +0100 Subject: [PATCH 15/17] distribute: only git context Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/reusable-distribute.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute.yml index 19699bf4..cc1fcb4c 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -122,7 +122,7 @@ on: # same as docker/bake-action inputs (minus workdir, builder, targets, load, push) bake-source: type: string - description: "Context to build from. Can be either local or a remote bake definition" + description: "Git context to build from for a remote bake definition" required: false bake-allow: type: string From 3f088f611cdd58449e495b1f12223182910e183f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 5 Mar 2025 10:34:42 +0100 Subject: [PATCH 16/17] readme: examples Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/reusable-distribute.yml | 32 +++++++++++------------ README.md | 8 ++++++ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute.yml index cc1fcb4c..fe9a0d98 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute.yml @@ -56,68 +56,68 @@ on: required: true meta-tags: type: string - description: 'List of tags as key-value pair attributes' + description: "List of tags as key-value pair attributes" required: false meta-flavor: type: string - description: 'Flavors to apply' + description: "Flavors to apply" required: false meta-labels: type: string - description: 'List of custom labels' + description: "List of custom labels" required: false meta-annotations: type: string - description: 'List of custom annotations' + description: "List of custom annotations" required: false meta-bake-target: type: string - description: 'Bake target name (default docker-metadata-action)' + description: "Bake target name (default docker-metadata-action)" required: false # same as docker/login-action inputs (minus logout) login-registry: type: string - description: 'Server address of Docker registry. If not set then will default to Docker Hub' + description: "Server address of Docker registry. If not set then will default to Docker Hub" required: false login-username: type: string - description: 'Username used to log against the Docker registry' + description: "Username used to log against the Docker registry" required: false login-ecr: type: string - description: 'Specifies whether the given registry is ECR (auto, true or false)' + description: "Specifies whether the given registry is ECR (auto, true or false)" default: 'auto' required: false # same as docker/setup-buildx-action inputs (minus driver, install, use, endpoint, append, cleanup) buildx-version: type: string - description: 'Buildx version. (eg. v0.3.0)' + description: "Buildx version. (eg. v0.3.0)" required: false buildx-cache-binary: type: boolean - description: 'Cache buildx binary to GitHub Actions cache backend' + description: "Cache buildx binary to GitHub Actions cache backend" default: true required: false buildx-driver-opts: type: string - description: 'List of additional docker-container options. (eg. image=moby/buildkit:master)' + description: "List of additional docker-container options. (eg. image=moby/buildkit:master)" required: false buildkitd-flags: type: string - description: 'BuildKit daemon flags' + description: "BuildKit daemon flags" required: false buildkitd-config: type: string - description: 'BuildKit daemon config file' + description: "BuildKit daemon config file" required: false buildkitd-config-inline: type: string - description: 'Inline BuildKit daemon config' + description: "Inline BuildKit daemon config" required: false # same as docker/setup-qemu-action inputs (minus platforms, cache-image) qemu-image: type: string - description: 'QEMU static binaries Docker image (e.g. tonistiigi/binfmt:latest)' + description: "QEMU static binaries Docker image (e.g. tonistiigi/binfmt:latest)" required: false # same as docker/bake-action inputs (minus workdir, builder, targets, load, push) bake-source: @@ -156,7 +156,7 @@ on: required: false secrets: login-username: - description: 'Username used to log against the Docker registry' + description: "Username used to log against the Docker registry" required: false login-password: description: "Password or personal access token used to log against the Docker registry" diff --git a/README.md b/README.md index 67a27440..e512e6a5 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,14 @@ jobs: *.tags=user/app:latest ``` +## Examples + +* [Distribute multi-platform build across runners](https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-multi-platform-build-across-runners) +* [Annotations](https://docs.docker.com/build/ci/github-actions/annotations/) +* [Secrets](https://docs.docker.com/build/ci/github-actions/secrets/) +* [Build checks](https://docs.docker.com/build/ci/github-actions/checks/#run-checks-with-dockerbake-action) +* [Reproducible builds](https://docs.docker.com/build/ci/github-actions/reproducible-builds/) + ## Summaries This action generates a [job summary](https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/) From a3d86a00cd43adce9cc184c0f30f11422fc11f5c Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 5 Mar 2025 12:30:39 +0100 Subject: [PATCH 17/17] distribute: rename workflow and check target Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- .../{reusable-distribute.yml => reusable-distribute-mp.yml} | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) rename .github/workflows/{reusable-distribute.yml => reusable-distribute-mp.yml} (99%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afd8a60f..2bea55c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -735,7 +735,7 @@ jobs: targets: app-entitlements distribute: - uses: ./.github/workflows/reusable-distribute.yml + uses: ./.github/workflows/reusable-distribute-mp.yml with: target: app-plus push: false @@ -743,7 +743,7 @@ jobs: bake-files: ./test/config.hcl distribute-cache: - uses: ./.github/workflows/reusable-distribute.yml + uses: ./.github/workflows/reusable-distribute-mp.yml with: target: image-all push: false diff --git a/.github/workflows/reusable-distribute.yml b/.github/workflows/reusable-distribute-mp.yml similarity index 99% rename from .github/workflows/reusable-distribute.yml rename to .github/workflows/reusable-distribute-mp.yml index fe9a0d98..24e5b022 100644 --- a/.github/workflows/reusable-distribute.yml +++ b/.github/workflows/reusable-distribute-mp.yml @@ -1,6 +1,6 @@ # Reusable workflow to distribute multi-platform builds efficiently # https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners -name: reusable-distribute +name: reusable-distribute-mp on: workflow_call: @@ -211,6 +211,9 @@ jobs: if (Util.getInputList('meta-image').length > 1) { throw new Error('Only one meta-image is allowed'); } + if (Util.getInputList('target').length > 1) { + throw new Error('Only one target is allowed'); + } const inpRunner = core.getInput('runner'); const inpTarget = core.getInput('target');