From f1010cd4c75efd8b1ee1a2f588c6132944d5f6ca Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 21 Jun 2024 14:01:44 +0100 Subject: [PATCH 1/2] consolidate image generation matrices --- .github/data/matrix-images-nap.json | 85 +++++++++++++++++++++++++ .github/data/matrix-images-oss.json | 15 +++++ .github/data/matrix-images-plus.json | 21 ++++++ .github/workflows/build-base-images.yml | 55 +++------------- .github/workflows/build-oss.yml | 2 +- .github/workflows/cache-update.yml | 75 +++------------------- .github/workflows/ci.yml | 77 ++++------------------ .github/workflows/image-promotion.yml | 74 +++------------------ 8 files changed, 161 insertions(+), 243 deletions(-) create mode 100644 .github/data/matrix-images-nap.json create mode 100644 .github/data/matrix-images-oss.json create mode 100644 .github/data/matrix-images-plus.json diff --git a/.github/data/matrix-images-nap.json b/.github/data/matrix-images-nap.json new file mode 100644 index 0000000000..47da890cd9 --- /dev/null +++ b/.github/data/matrix-images-nap.json @@ -0,0 +1,85 @@ +{ + "image": [ + "debian-plus-nap" + ], + "platforms": [ + "linux/amd64" + ], + "target": [ + "goreleaser", + "aws" + ], + "nap_modules": [ + "dos", + "waf", + "waf,dos" + ], + "include": [ + { + "image": "ubi-9-plus-nap", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "waf" + }, + { + "image": "ubi-8-plus-nap", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "dos" + }, + { + "image": "ubi-8-plus-nap", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "waf,dos" + }, + { + "image": "ubi-9-plus-nap", + "target": "aws", + "platforms": "linux/amd64", + "nap_modules": "waf" + }, + { + "image": "ubi-8-plus-nap", + "target": "aws", + "platforms": "linux/amd64", + "nap_modules": "dos" + }, + { + "image": "ubi-8-plus-nap", + "target": "aws", + "platforms": "linux/amd64", + "nap_modules": "waf,dos" + }, + { + "image": "alpine-plus-nap-fips", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "waf" + }, + { + "image": "alpine-plus-nap-v5-fips", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "waf" + }, + { + "image": "debian-plus-nap-v5", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "waf" + }, + { + "image": "ubi-9-plus-nap-v5", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "waf" + }, + { + "image": "ubi-8-plus-nap-v5", + "target": "goreleaser", + "platforms": "linux/amd64", + "nap_modules": "waf" + } + ] +} diff --git a/.github/data/matrix-images-oss.json b/.github/data/matrix-images-oss.json new file mode 100644 index 0000000000..237c3014fb --- /dev/null +++ b/.github/data/matrix-images-oss.json @@ -0,0 +1,15 @@ +{ + "image": [ + "debian", + "alpine" + ], + "platforms": [ + "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + ], + "include": [ + { + "image": "ubi", + "platforms": "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + } + ] +} diff --git a/.github/data/matrix-images-plus.json b/.github/data/matrix-images-plus.json new file mode 100644 index 0000000000..f9f02f865d --- /dev/null +++ b/.github/data/matrix-images-plus.json @@ -0,0 +1,21 @@ +{ + "image": [ + "debian-plus", + "alpine-plus", + "alpine-plus-fips" + ], + "platforms": [ + "linux/arm64, linux/amd64" + ], + "target": [ + "goreleaser", + "aws" + ], + "include": [ + { + "image": "ubi-plus", + "platforms": "linux/arm64, linux/amd64, linux/s390x", + "target": "goreleaser" + } + ] +} diff --git a/.github/workflows/build-base-images.yml b/.github/workflows/build-base-images.yml index ff0d203344..1490969a10 100644 --- a/.github/workflows/build-base-images.yml +++ b/.github/workflows/build-base-images.yml @@ -25,6 +25,9 @@ jobs: outputs: docker_md5: ${{ steps.vars.outputs.docker_md5 }} ic_version: ${{ steps.vars.outputs.ic_version }} + image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }} + image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }} + image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -35,6 +38,9 @@ jobs: ./.github/scripts/variables.sh docker_md5 >> $GITHUB_OUTPUT source .github/data/version.txt echo "ic_version=${IC_VERSION}" >> $GITHUB_OUTPUT + echo "image_matrix_oss=$(cat .github/data/matrix-images-oss.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_plus=$(cat .github/data/matrix-images-plus.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_nap=$(cat .github/data/matrix-images-nap.json | jq -c)" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT build-oss: @@ -46,13 +52,7 @@ jobs: pull-requests: write # for scout report strategy: fail-fast: false - matrix: - image: [debian, alpine] - platforms: - ["linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"] - include: - - image: ubi - platforms: "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_oss ) }} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -116,12 +116,7 @@ jobs: pull-requests: write # for scout report strategy: fail-fast: false - matrix: - image: [debian-plus, alpine-plus, alpine-plus-fips] - platforms: ["linux/arm64, linux/amd64"] - include: - - image: ubi-plus - platforms: "linux/arm64, linux/amd64, linux/s390x" + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_plus ) }} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -188,39 +183,7 @@ jobs: pull-requests: write # for scout report strategy: fail-fast: false - matrix: - image: [debian-plus-nap] - platforms: ["linux/amd64"] - nap_modules: [dos, waf, "waf,dos"] - include: - - image: ubi-9-plus-nap - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap - platforms: "linux/amd64" - nap_modules: dos - - image: ubi-8-plus-nap - platforms: "linux/amd64" - nap_modules: "waf,dos" - - image: alpine-plus-nap-fips - platforms: "linux/amd64" - nap_modules: waf - - image: alpine-plus-nap-v5-fips - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: debian-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-9-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_nap ) }} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 diff --git a/.github/workflows/build-oss.yml b/.github/workflows/build-oss.yml index 9d5d96abb7..afde1eb656 100644 --- a/.github/workflows/build-oss.yml +++ b/.github/workflows/build-oss.yml @@ -210,7 +210,7 @@ jobs: summary: true if: ${{ inputs.authenticated && steps.build-push.conclusion == 'success' }} - - name: Upload Scan Results to the cache + - name: Upload Scan Results to Github Artifacts uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 continue-on-error: true with: diff --git a/.github/workflows/cache-update.yml b/.github/workflows/cache-update.yml index 78d85b9646..9e12b59f56 100644 --- a/.github/workflows/cache-update.yml +++ b/.github/workflows/cache-update.yml @@ -21,6 +21,9 @@ jobs: outputs: go_code_md5: ${{ steps.vars.outputs.go_code_md5 }} docker_md5: ${{ steps.vars.outputs.docker_md5 }} + image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }} + image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }} + image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -30,7 +33,9 @@ jobs: run: | ./.github/scripts/variables.sh go_code_md5 >> $GITHUB_OUTPUT ./.github/scripts/variables.sh docker_md5 >> $GITHUB_OUTPUT - source .github/data/version.txt + echo "image_matrix_oss=$(cat .github/data/matrix-images-oss.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_plus=$(cat .github/data/matrix-images-plus.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_nap=$(cat .github/data/matrix-images-nap.json | jq -c)" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT build-docker: @@ -38,13 +43,7 @@ jobs: needs: [checks] strategy: fail-fast: false - matrix: - image: [debian, alpine] - platforms: - ["linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"] - include: - - image: ubi - platforms: "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_oss ) }} uses: ./.github/workflows/build-oss.yml with: platforms: ${{ matrix.platforms }} @@ -68,14 +67,7 @@ jobs: needs: [checks] strategy: fail-fast: false - matrix: - image: [debian-plus, alpine-plus, alpine-plus-fips] - platforms: ["linux/arm64, linux/amd64"] - target: [goreleaser, aws] - include: - - image: ubi-plus - platforms: "linux/arm64, linux/amd64, linux/s390x" - target: goreleaser + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_plus ) }} uses: ./.github/workflows/build-plus.yml with: platforms: ${{ matrix.platforms }} @@ -98,56 +90,7 @@ jobs: needs: [checks] strategy: fail-fast: false - matrix: - image: [debian-plus-nap] - platforms: ["linux/amd64"] - target: [goreleaser, aws] - nap_modules: [dos, waf, "waf,dos"] - include: - - image: ubi-9-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: dos - - image: ubi-8-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: "waf,dos" - - image: ubi-9-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: dos - - image: ubi-8-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: "waf,dos" - - image: alpine-plus-nap-fips - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: alpine-plus-nap-v5-fips - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: debian-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-9-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_nap ) }} uses: ./.github/workflows/build-plus.yml with: platforms: ${{ matrix.platforms }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76fa0885b9..801a4f20bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,9 @@ jobs: forked_workflow: ${{ steps.vars.outputs.forked_workflow }} stable_image_exists: ${{ steps.stable_exists.outputs.exists }} additional_tag: ${{ steps.vars.outputs.additional_tag }} + image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }} + image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }} + image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -98,6 +101,9 @@ jobs: additional_tag="${ref//\//-}" fi echo "additional_tag=${additional_tag}" >> $GITHUB_OUTPUT + echo "image_matrix_oss=$(cat .github/data/matrix-images-oss.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_plus=$(cat .github/data/matrix-images-plus.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_nap=$(cat .github/data/matrix-images-nap.json | jq -c)" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT - name: Fetch Cached Binary Artifacts @@ -148,6 +154,9 @@ jobs: echo forked_workflow: ${{ steps.vars.outputs.forked_workflow }} echo stable_image_exists: ${{ steps.stable_exists.outputs.exists }} echo additional_tag: ${{ steps.vars.outputs.additional_tag }} + echo image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }} + echo image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }} + echo image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }} verify-codegen: name: Verify generated code @@ -283,13 +292,7 @@ jobs: needs: [binaries, checks] strategy: fail-fast: false - matrix: - image: [debian, alpine] - platforms: - ["linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"] - include: - - image: ubi - platforms: "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_oss ) }} uses: ./.github/workflows/build-oss.yml with: platforms: ${{ matrix.platforms }} @@ -314,14 +317,7 @@ jobs: needs: [binaries, checks] strategy: fail-fast: false - matrix: - image: [debian-plus, alpine-plus, alpine-plus-fips] - platforms: ["linux/arm64, linux/amd64"] - target: [goreleaser, aws] - include: - - image: ubi-plus - platforms: "linux/arm64, linux/amd64, linux/s390x" - target: goreleaser + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_plus ) }} uses: ./.github/workflows/build-plus.yml with: platforms: ${{ matrix.platforms }} @@ -345,56 +341,7 @@ jobs: needs: [binaries, checks] strategy: fail-fast: false - matrix: - image: [debian-plus-nap] - platforms: ["linux/amd64"] - target: [goreleaser, aws] - nap_modules: [dos, waf, "waf,dos"] - include: - - image: ubi-9-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: dos - - image: ubi-8-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: "waf,dos" - - image: ubi-9-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: dos - - image: ubi-8-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: "waf,dos" - - image: alpine-plus-nap-fips - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: alpine-plus-nap-v5-fips - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: debian-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-9-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_nap ) }} uses: ./.github/workflows/build-plus.yml with: platforms: ${{ matrix.platforms }} diff --git a/.github/workflows/image-promotion.yml b/.github/workflows/image-promotion.yml index bba93a719b..34f4771b82 100644 --- a/.github/workflows/image-promotion.yml +++ b/.github/workflows/image-promotion.yml @@ -41,6 +41,9 @@ jobs: build_tag: ${{ steps.vars.outputs.build_tag }} stable_tag: ${{ steps.vars.outputs.stable_tag }} stable_image_exists: ${{ steps.stable_exists.outputs.exists }} + image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }} + image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }} + image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -61,6 +64,9 @@ jobs: ./.github/scripts/variables.sh docker_md5 >> $GITHUB_OUTPUT ./.github/scripts/variables.sh build_tag >> $GITHUB_OUTPUT ./.github/scripts/variables.sh stable_tag >> $GITHUB_OUTPUT + echo "image_matrix_oss=$(cat .github/data/matrix-images-oss.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_plus=$(cat .github/data/matrix-images-plus.json | jq -c)" >> $GITHUB_OUTPUT + echo "image_matrix_nap=$(cat .github/data/matrix-images-nap.json | jq -c)" >> $GITHUB_OUTPUT - name: Fetch Cached Binary Artifacts id: binary-cache @@ -152,13 +158,7 @@ jobs: needs: [checks, binaries] strategy: fail-fast: false - matrix: - image: [debian, alpine] - platforms: - ["linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"] - include: - - image: ubi - platforms: "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_oss ) }} uses: ./.github/workflows/build-oss.yml with: platforms: ${{ matrix.platforms }} @@ -183,14 +183,7 @@ jobs: needs: [checks, binaries] strategy: fail-fast: false - matrix: - image: [debian-plus, alpine-plus, alpine-plus-fips] - platforms: ["linux/arm64, linux/amd64"] - target: [goreleaser, aws] - include: - - image: ubi-plus - platforms: "linux/arm64, linux/amd64, linux/s390x" - target: goreleaser + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_plus ) }} uses: ./.github/workflows/build-plus.yml with: platforms: ${{ matrix.platforms }} @@ -216,56 +209,7 @@ jobs: needs: [checks, binaries] strategy: fail-fast: false - matrix: - image: [debian-plus-nap] - platforms: ["linux/amd64"] - target: [goreleaser, aws] - nap_modules: [dos, waf, "waf,dos"] - include: - - image: ubi-9-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: dos - - image: ubi-8-plus-nap - target: goreleaser - platforms: "linux/amd64" - nap_modules: "waf,dos" - - image: ubi-9-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: dos - - image: ubi-8-plus-nap - target: aws - platforms: "linux/amd64" - nap_modules: "waf,dos" - - image: alpine-plus-nap-fips - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: alpine-plus-nap-v5-fips - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: debian-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-9-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf - - image: ubi-8-plus-nap-v5 - target: goreleaser - platforms: "linux/amd64" - nap_modules: waf + matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_nap ) }} uses: ./.github/workflows/build-plus.yml with: platforms: ${{ matrix.platforms }} From b82a7c6a00ac7aee71572dc0200962960581c647 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 21 Jun 2024 14:13:48 +0100 Subject: [PATCH 2/2] add quotes when echoing json --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 801a4f20bf..dfda4599cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,9 +154,9 @@ jobs: echo forked_workflow: ${{ steps.vars.outputs.forked_workflow }} echo stable_image_exists: ${{ steps.stable_exists.outputs.exists }} echo additional_tag: ${{ steps.vars.outputs.additional_tag }} - echo image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }} - echo image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }} - echo image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }} + echo 'image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }}' + echo 'image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }}' + echo 'image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }}' verify-codegen: name: Verify generated code