update release manifest (sourcegraph#4307)

burmudar · web-flow · commit d18c74885364 · 2024-03-28T10:53:38.000+02:00
* update release manifest

the following branches get created:

- internal release: internal/release-vX.Y.Z
- promote to public
  - promote/release-vX.Y.Z
  - release/vX.Y.Z &lt;--- The tag for the release will live on this branch

* update pipeline for release process

* update branch names for qa

* fix find expressions

* add github cli to tool-versions
diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
@@ -60,6 +60,23 @@ steps:
     command: .buildkite/cleanup-disks.sh
     agents: { queue: standard }
 
+  # Please keep in mind that the release manifest uses specific branch names when creating releases.
+  # Therefore, if you update them, or if you decide to change how we detect what kind of build we're dealing
+  # with, please update this file as well.
+  - label: "(internal) Release: test"
+    if: build.branch =~ /^internal\/release-.*/
+    plugins:
+      - ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
+    command: |
+      sg release run test --workdir=. --config-from-commit
+
+  - label: "(promote) Release: test"
+    if: build.branch =~ /^promote\/release-.*/
+    plugins:
+      - ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
+    command: |
+      sg release run test --workdir=. --config-from-commit
+
   - label: "Release: test"
     if: "build.branch =~ /^wip_/"
     plugins:
@@ -69,14 +86,15 @@ steps:
 
   - wait
 
-  - label: "Release: finalize"
-    if: "build.branch =~ /^wip_/"
+  - label: "(internal) Release: finalize"
+    if: build.branch =~ /^internal\/release-.*/
     plugins:
       - ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
     command: |
       sg release run internal finalize --workdir=. --config-from-commit
-  - label: "Promote to public: finalize"
-    if: build.message =~ /^promote_release/ && build.branch =~ /^wip-release/
+
+  - label: "(promote) Release: finalize"
+    if: build.branch =~ /^promote\/release-.*/
     plugins:
       - ssh://git@github.com/sourcegraph/sg-buildkite-plugin.git#main: ~
     command: |
diff --git a/.tool-versions b/.tool-versions
@@ -6,3 +6,4 @@ kustomize 4.5.7
 shfmt 3.1.0
 nodejs 20.8.1
 python system
+github-cli 2.46.0
diff --git a/release.yaml b/release.yaml
@@ -31,7 +31,7 @@ internal:
               base/
         - name: "sg ops (overlays)"
           cmd: |
-            folders=$(find overlays -type d -d 1 \! -name "low-resource")
+            folders=$(find overlays -maxdepth 1 overlays -type d \! -name "low-resource")
 
             for path in $folders; do
               echo "updating ${path}"
@@ -45,7 +45,7 @@ internal:
             done
         - name: "sg ops (configure)"
           cmd: |
-            folders=$(find configure -type d -d 1)
+            folders=$(find configure -maxdepth 1 -type d)
 
             for path in $folders; do
               echo "updating ${path}"
@@ -59,23 +59,32 @@ internal:
             done
         - name: "git:branch"
           cmd: |
-            echo "Creating branch wip_{{version}}"
-            release_branch="wip_{{version}}"
-            git checkout -b $release_branch
+            set -eu
+            branch="internal/release-{{version}}"
+            echo "Creating branch $branch"
+            git checkout -b $branch
         - name: "git:commit"
           cmd: |
             find . -name "*.yaml" | xargs git add
             find . -name "*.yml" | xargs git add
+
             # Careful with the quoting for the config, using double quotes will lead
             # to the shell dropping out all quotes from the json, leading to failed
             # parsing.
             git commit -m "release_patch: {{version}}" -m '{{config}}'
         - name: "git:push"
           cmd: |
-            git push origin wip_{{version}}
-        - name: "gh cli"
+            branch="internal/release-{{version}}"
+            git push origin "$branch"
+        - name: "github:pr"
           cmd: |
-            gh pr create -f -t "PRETEND RELEASE WIP: release_patch: build {{version}}" --body "Test plan: automated release PR, CI will perform additional checks"  # -l "wip_release"
+            set -eu
+            gh pr create \
+              --fill \
+              --draft \
+              --title "(internal) release_patch: build {{version}}" \
+              --body "Test plan: automated release PR, CI will perform additional checks"
+            echo "🚢 Please check the associated CI build to ensure the process completed".
       minor:
         - name: "sg ops (base)"
           cmd: |
@@ -88,7 +97,7 @@ internal:
               base/
         - name: "sg ops (overlays)"
           cmd: |
-            folders=$(find overlays -type d -d 1 \! -name "low-resource")
+            folders=$(find overlays -maxdepth 1 -type d \! -name "low-resource")
 
             for path in $folders; do
               echo "updating ${path}"
@@ -102,7 +111,7 @@ internal:
             done
         - name: "sg ops (configure)"
           cmd: |
-            folders=$(find configure -type d -d 1)
+            folders=$(find configure -maxdepth 1 -type d)
 
             for path in $folders; do
               echo "updating ${path}"
@@ -116,23 +125,32 @@ internal:
             done
         - name: "git:branch"
           cmd: |
-            echo "Creating branch wip_{{version}}"
-            release_branch="wip_{{version}}"
-            git checkout -b $release_branch
+            set -eu
+            branch="internal/release-{{version}}"
+            echo "Creating branch $branch"
+            git checkout -b $branch
         - name: "git:commit"
           cmd: |
             find . -name "*.yaml" | xargs git add
             find . -name "*.yml" | xargs git add
+
             # Careful with the quoting for the config, using double quotes will lead
             # to the shell dropping out all quotes from the json, leading to failed
             # parsing.
             git commit -m "release_minor: {{version}}" -m '{{config}}'
         - name: "git:push"
           cmd: |
-            git push origin wip_{{version}}
-        - name: "gh cli"
+            branch="internal/release-{{version}}"
+            git push origin "$branch"
+        - name: "github:pr"
           cmd: |
-            gh pr create -f -t "PRETEND RELEASE WIP: release_minor: build {{version}}" --body "Test plan: automated release PR, CI will perform additional checks"  # -l "wip_release"
+            set -eu
+            gh pr create \
+              --fill \
+              --draft \
+              --title "(internal) release_patch: build {{version}}" \
+              --body "Test plan: automated release PR, CI will perform additional checks"
+            echo "🚢 Please check the associated CI build to ensure the process completed".
       major:
         - name: "sg ops (base)"
           cmd: |
@@ -145,7 +163,7 @@ internal:
               base/
         - name: "sg ops (overlays)"
           cmd: |
-            folders=$(find overlays -type d -d 1 \! -name "low-resource")
+            folders=$(find overlays -maxdepth 1 -type d \! -name "low-resource")
 
             for path in $folders; do
               echo "updating ${path}"
@@ -159,7 +177,7 @@ internal:
             done
         - name: "sg ops (configure)"
           cmd: |
-            folders=$(find configure -type d -d 1)
+            folders=$(find configure -maxdepth 1 -type d)
 
             for path in $folders; do
               echo "updating ${path}"
@@ -173,31 +191,49 @@ internal:
             done
         - name: "git:branch"
           cmd: |
-            echo "Creating branch wip_{{version}}"
-            release_branch="wip_{{version}}"
-            git checkout -b $release_branch
+            set -eu
+            branch="internal/release-{{version}}"
+            echo "Creating branch $branch"
+            git checkout -b $branch
         - name: "git:commit"
           cmd: |
             find . -name "*.yaml" | xargs git add
             find . -name "*.yml" | xargs git add
+
             # Careful with the quoting for the config, using double quotes will lead
             # to the shell dropping out all quotes from the json, leading to failed
             # parsing.
-            git commit -m "release_patch: {{version}}" -m '{{config}}'
+            git commit -m "release_major: {{version}}" -m '{{config}}'
         - name: "git:push"
           cmd: |
-            git push origin wip_{{version}}
-        - name: "gh cli"
+            branch="internal/release-{{version}}"
+            git push origin "$branch"
+        - name: "github:pr"
           cmd: |
-            gh pr create -f -t "PRETEND RELEASE WIP: release_major: build {{version}}" --body "Test plan: automated release PR, CI will perform additional checks"  # -l "wip_release"
+            set -eu
+            gh pr create \
+              --fill \
+              --draft \
+              --title "(internal) release_patch: build {{version}}" \
+              --body "Test plan: automated release PR, CI will perform additional checks"
+            echo "🚢 Please check the associated CI build to ensure the process completed".
   finalize:
     steps:
-      - name: "git"
+      - name: "notifications"
         cmd: |
-          set -e
-          git checkout -b wip-release-{{version}}
-          git push origin wip-release-{{version}}
-          git checkout -
+          set -eu
+
+          branch="internal/release-{{version}}"
+
+          # Post a comment on the PR.
+          cat << EOF | gh pr comment "$branch" --body-file -
+          - :green_circle: Internal release is ready for promotion!
+          - :warning: Do not close/merge the pull request or delete the associated branch if you intend to promote it.
+          EOF
+          # Post an annotation.
+          cat << EOF | buildkite-agent annotate --style info
+          Internal release is ready for promotion under the branch [\`$branch\`](https://github.com/sourcegraph/deploy-sourcegraph/tree/$branch).
+          EOF
 
 test:
   steps:
@@ -210,21 +246,23 @@ promoteToPublic:
     steps:
       - name: "git"
         cmd: |
-          echo "Checking out origin/wip-release-{{version}}"
-          git fetch origin
-          git checkout origin/wip-release-{{version}}
+          set -eu
+          branch="internal/release-{{version}}"
+          echo "Checking out origin/${branch}"
+          git fetch origin "${branch}"
+          git switch "${branch}"
       - name: "sg ops (base)"
         cmd: |
           sg ops update-images \
             --kind k8s \
-            --registry us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal \
+            --registry us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-public \
             --docker-username=$DOCKER_USERNAME \
             --docker-password=$DOCKER_PASSWORD \
             --pin-tag {{inputs.server.tag}} \
             base/
       - name: "sg ops (overlays)"
         cmd: |
-          folders=$(find overlays -type d -d 1 \! -name "low-resource")
+          folders=$(find overlays -maxdepth 1 -type d \! -name "low-resource")
 
           for path in $folders; do
             echo "updating ${path}"
@@ -238,7 +276,7 @@ promoteToPublic:
           done
       - name: "sg ops (configure)"
         cmd: |
-          folders=$(find configure -type d -d 1)
+          folders=$(find configure -maxdepth 1 -type d )
 
           for path in $folders; do
             echo "updating ${path}"
@@ -252,29 +290,67 @@ promoteToPublic:
           done
       - name: "git:branch"
         cmd: |
-          echo "Creating branch promote-release_{{version}}"
-          branch="promote-release_{{version}}"
-          git checkout -b $branch
+          set -eu
+          branch="promote/release-{{version}}"
+          git switch -c "${branch}"
       - name: "git:commit"
         cmd: |
+          set -eu
+          branch="promote/release-{{version}}"
           find . -name "*.yaml" | xargs git add
           find . -name "*.yml" | xargs git add
+
           # Careful with the quoting for the config, using double quotes will lead
           # to the shell dropping out all quotes from the json, leading to failed
           # parsing.
-          git commit -m "promote_release: {{version}}" -m '{{config}}'
-      - name: "github"
+          git commit -am 'promote-release: {{version}}' -m '{{config}}'
+          git push origin "${branch}"
+      - name: "github:pr"
         cmd: |
-          set -e
-          git push origin promote-release_{{version}}
-          gh pr create -f -t "PRETEND PROMOTE RELEASE WIP: promote-release: build {{version}}" --base wip-release-{{version}} --body "Test plan: automated release PR, CI will perform additional checks"
+          set -eu
+          internal_branch="internal/release-{{version}}"
+          gh pr create \
+            --fill \
+            --draft \
+            --base "$internal_branch" \
+            --title "(promote) release: build {{version}}" \
+            --body "Test plan: automated release PR, CI will perform additional checks"
+          echo "🚢 Please check the associated CI build to ensure the process completed".
   finalize:
     # These steps should only really run once the pr created in the create step is merged
     steps:
       - name: git:tag
         cmd: |
-          set -e
-          branch="wip-release-{{version}}"
-          git checkout ${branch}
-          git tag {{version}}
-          git push origin ${branch} --tags
+          set -eu
+
+          # Branches
+          internal_branch="internal/release-{{version}}"
+          promote_branch="promote/release-{{version}}"
+          release_branch="qa-release-{{version}}"
+
+          # Create the final branch holding the tagged commit
+          git checkout "${promote_branch}"
+          git switch -c "${release_branch}"
+          git tag qa-{{version}}
+          git push origin ${release_branch} --tags
+
+          # Web URL to the tag
+          tag_url="https://github.com/sourcegraph/deploy-sourcegraph/tree/{{version}}"
+
+          # Annotate PRs
+          cat << EOF | gh pr comment "$internal_branch" --body-file -
+          - :green_circle: Release has been promoted, see tag: $tag_url.
+          - :no_entry: Do not under any circumstance delete the branch holding the tagged commit (i.e. \`$release_branch\`).
+          - :arrow_right: You can safely close the PR and delete its a associated branch.
+          EOF
+
+          cat << EOF | gh pr comment "$promote_branch" --body-file -
+          - :green_circle: Release has been promoted, see tag: $tag_url.
+          - :no_entry: Do not under any circumstance delete the branch holding the tagged commit (i.e. \`$release_branch\`).
+          - :arrow_right: You can safely close that PR and delete its a associated branch.
+          EOF
+
+          # Annotate build
+          cat << EOF | buildkite-agent annotate --style info
+          Promoted release is **publicly available** through a git tag at [\`{{version}}\`](https://github.com/sourcegraph/deploy-sourcegraph/tree/{{version}}).
+          EOF
