ci: update release flow and publish list.json (paritytech#276)

alvicsam · web-flow · commit 0dafc779ce2c · 2025-04-07T13:39:56.000+02:00
This pull request adds changes described in **this comment**: - `.github/scripts/json_generator.py` - a small script that generates several json files for different platforms. - [generate_versions.yml](https://github.com/paritytech/revive/compare/as-release-json?expand=1#diff-2aee05b96020ac60943e6dfcb30597e53898f31542aeb570468b970d9a13a5a6) - the workflow that runs when release is published, creates info.json files and pushes them into resolc-bin repo - `js/build.js` is adjusted in order to set `RESOLC_WASM_URI` from env variable - ⚠️ Release workflow is changed: - In PRs and main branch it'll only build artifacts - Release will happen automatically only on the `v*` tag push. This is needed for revive_web.js to have the necessary `RESOLC_WASM_URI` - workflow will check that version in Cargo.toml is the same as the tag when the new tag is pushed cc paritytech#162 cc paritytech/devops#3890
diff --git a/.github/scripts/json_generator.py b/.github/scripts/json_generator.py
@@ -0,0 +1,119 @@
+import os
+import sys
+import json
+import requests
+from datetime import datetime
+
+def validate_github_token():
+    """Validate that GITHUB_TOKEN environment variable is set."""
+    if 'GITHUB_TOKEN' not in os.environ:
+        print("Error: GITHUB_TOKEN environment variable is not set.")
+        sys.exit(1)
+
+def fetch_release_data(repo, tag):
+    """Fetch release data from GitHub API."""
+    url = f"https://api.github.com/repos/{repo}/releases/tags/{tag}"
+    headers = {
+        'Authorization': f"Bearer {os.environ['GITHUB_TOKEN']}",
+        'Accept': 'application/vnd.github+json',
+        'X-GitHub-Api-Version': '2022-11-28'
+    }
+
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+        return response.json()
+    except requests.RequestException as e:
+        print(f"Error fetching release data: {e}")
+        sys.exit(1)
+
+def extract_build_hash(target_commitish):
+    """Extract the first 8 characters of the commit hash."""
+    return f"commit.{target_commitish[:8]}"
+
+def generate_asset_json(release_data, asset):
+    """Generate JSON for a specific asset."""
+    version = release_data['tag_name'].lstrip('v')
+    build = extract_build_hash(release_data['target_commitish'])
+    long_version = f"{version}+{build}"
+    path = f"{asset['name']}+{long_version}"
+
+    return {
+        "path": path,
+        "name": asset['name'],
+        "version": version,
+        "build": build,
+        "longVersion": long_version,
+        "url": asset['browser_download_url'],
+        "firstSolcVersion": os.environ["FIRST_SOLC_VERSION"],
+        "lastSolcVersion": os.environ["LAST_SOLC_VERSION"]
+    }
+
+def save_platform_json(platform_folder, asset_json, tag):
+    """Save asset JSON and update list.json for a specific platform."""
+    # Create platform folder if it doesn't exist
+    os.makedirs(platform_folder, exist_ok=True)
+
+    # Update or create list.json
+    list_file_path = os.path.join(platform_folder, "list.json")
+
+    if os.path.exists(list_file_path):
+        with open(list_file_path, 'r') as f:
+            try:
+                list_data = json.load(f)
+            except json.JSONDecodeError:
+                list_data = {"builds": [], "releases": {}, "latestRelease": ""}
+    else:
+        list_data = {"builds": [], "releases": {}, "latestRelease": ""}
+
+    # Remove any existing entry with the same path
+    list_data['builds'] = [
+        build for build in list_data['builds']
+        if build['path'] != asset_json['path']
+    ]
+    # Add the new build
+    list_data['builds'].append(asset_json)
+
+    # Update releases
+    version = asset_json['version']
+    list_data['releases'][version] = asset_json['path']
+
+    # Update latest release
+    list_data['latestRelease'] = version
+
+    with open(list_file_path, 'w') as f:
+        json.dump(list_data, f, indent=4)
+
+def main():
+    # Validate arguments
+    if len(sys.argv) != 3:
+        print("Usage: python script.py <repo> <tag>")
+        sys.exit(1)
+
+    repo, tag = sys.argv[1], sys.argv[2]
+
+    # Validate GitHub token
+    validate_github_token()
+
+    # Fetch release data
+    release_data = fetch_release_data(repo, tag)
+
+    # Mapping of asset names to platform folders
+    platform_mapping = {
+        'resolc-x86_64-unknown-linux-musl': 'linux',
+        'resolc-universal-apple-darwin': 'macos',
+        'resolc-x86_64-pc-windows-msvc.exe': 'windows',
+        'resolc_web.js': 'wasm'
+    }
+
+    # Process each asset
+    for asset in release_data['assets']:
+        platform_name = platform_mapping.get(asset['name'])
+        if platform_name:
+            platform_folder = os.path.join(platform_name)
+            asset_json = generate_asset_json(release_data, asset)
+            save_platform_json(platform_folder, asset_json, tag)
+            print(f"Processed {asset['name']} for {platform_name}")
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/generate_versions.yml b/.github/workflows/generate_versions.yml
@@ -0,0 +1,64 @@
+name: Generate JSON for resolc-bin
+on:
+  release:
+    types: [published]
+
+jobs:
+  generateJson:
+    runs-on: ubuntu-latest
+    if: contains(github.event.release.tag_name, 'llvm') == false
+    environment: tags
+    env:
+      # the token is needed for json_generator.py
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: tmp
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: paritytech/resolc-bin
+          path: resolc-bin
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.REVIVE_JSON_APP_ID }}
+          private-key: ${{ secrets.REVIVE_JSON_APP_KEY }}
+          owner: paritytech
+          repositories: resolc-bin
+
+      - name: Generate json and push
+        env:
+          TOKEN: ${{ steps.app-token.outputs.token }}
+          APP_NAME: "paritytech-revive-json"
+          Green: "\e[32m"
+          NC: "\e[0m"
+        run: |
+          sudo apt-get update && sudo apt-get install -y wget
+          wget https://github.com/${GITHUB_REPOSITORY}/releases/download/${GITHUB_REF_NAME}/resolc-x86_64-unknown-linux-musl
+          chmod +x resolc-x86_64-unknown-linux-musl
+          export FIRST_SOLC_VERSION=$(./resolc-x86_64-unknown-linux-musl --supported-solc-versions | cut -f 1 -d "," | tr -d ">=")
+          export LAST_SOLC_VERSION=$(./resolc-x86_64-unknown-linux-musl --supported-solc-versions | cut -f 2 -d "," | tr -d "<=")
+
+          cd resolc-bin
+          python ../tmp/.github/scripts/json_generator.py ${GITHUB_REPOSITORY} ${{ github.event.release.tag_name }}
+
+          echo "${Green}Add new remote with gh app token${NC}"
+          git remote set-url origin $(git config remote.origin.url | sed "s/github.com/${APP_NAME}:${TOKEN}@github.com/g")
+
+          echo "${Green}Remove http section that causes issues with gh app auth token${NC}"
+          sed -i.bak '/\[http/d' ./.git/config
+          sed -i.bak '/extraheader/d' ./.git/config
+
+          git config user.email "ci@parity.io"
+          git config user.name "${APP_NAME}"
+
+          git add .
+          git commit -m "Update json"
+          git push origin main
+
+          echo "::notice::info.list files were successfully published to https://github.com/paritytech/resolc-bin"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,11 +1,12 @@
-name: Release
+name: Build & Release
 on:
   push:
     branches: ["main"]
+    tags:
+      - "v*"
   pull_request:
     branches: ["main"]
     types: [opened, synchronize, labeled, unlabeled]
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -17,41 +18,44 @@ env:
 
 jobs:
   check-version-changed:
-    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'release-test')
     runs-on: ubuntu-24.04
     permissions:
       contents: write
+    env:
+      CURRENT_TAG: ${{ github.ref_name }}
     outputs:
-      TAG: ${{ steps.versions.outputs.TAG }}
-      PKG_VER: ${{ steps.versions.outputs.PKG_VER }}
       RELEASE_NOTES: ${{ steps.versions.outputs.RELEASE_NOTES }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-          fetch-depth: 0
 
-      - name: Check Versions
+      # Check that tag and version in Cargo.toml match
+      - name: Check versions
         id: versions
         run: |
-          export CURRENT_TAG=$(git describe --tags --abbrev=0 --exclude "llvm-*")
+          if [[ $CURRENT_TAG == 'main' ]];
+            then
+              echo "::notice::Tag $CURRENT_TAG is not a release tag, skipping the check in the main branch";
+              exit 0
+            fi
+
+          if [[ $CURRENT_TAG != "v"* ]];
+            then
+              echo "::notice::Tag $CURRENT_TAG is not a release tag, skipping the check in a PR";
+              exit 0
+            fi
+
           export PKG_VER=v$(cat Cargo.toml | grep -A 5 package] | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d " ")
           echo "Current tag $CURRENT_TAG"
           echo "Package version $PKG_VER"
           #
-          echo "PKG_VER=$PKG_VER" >> $GITHUB_OUTPUT
-          if [[ $CURRENT_TAG == $PKG_VER  ]];
+          if [[ $CURRENT_TAG != $PKG_VER  ]];
             then
-              echo "Tag is up to date. Nothing to do.";
-              export TAG=old;
-            else
-              echo "Tag was updated.";
-              export TAG=new;
+              echo "::error::Tag $CURRENT_TAG doesn't match package version $PKG_VER in Cargo.toml, please fix";
+              exit 1
             fi
-          echo "TAG=$TAG" >> $GITHUB_OUTPUT
 
           # Generating release notes early, in order to avoid checkout at the last step
-          export RELEASE_NOTES="$(sed '/^## '${PKG_VER}'/,/^## v/!d' CHANGELOG.md | sed -e '1d' -e '$d')"
+          export RELEASE_NOTES="$(sed '/^## '${CURRENT_TAG}'/,/^## v/!d' CHANGELOG.md | sed -e '1d' -e '$d')"
 
           echo "Release notes:"
           echo "$RELEASE_NOTES"
@@ -83,7 +87,6 @@ jobs:
           - target: x86_64-pc-windows-msvc
             type: native
             runner: windows-2022
-    if: ${{ needs.check-version-changed.outputs.TAG == 'new' }}
     runs-on: ${{ matrix.runner }}
     needs: [check-version-changed]
     steps:
@@ -137,9 +140,10 @@ jobs:
           retention-days: 1
 
   build-wasm:
-    if: ${{ needs.check-version-changed.outputs.TAG == 'new' }}
     runs-on: ubuntu-24.04
     needs: [check-version-changed]
+    env:
+      RELEASE_RESOLC_WASM_URI: https://github.com/paritytech/revive-workflow-test/releases/download/${{ github.ref_name }}/resolc.wasm
     steps:
       - uses: actions/checkout@v4
       - uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -226,11 +230,10 @@ jobs:
           retention-days: 1
 
   create-release:
-    if: github.event_name != 'pull_request'
+    if: startsWith(github.ref_name, 'v')
     needs: [check-version-changed, build-wasm]
     runs-on: macos-14
-    permissions:
-      contents: write
+    environment: tags
     steps:
       - name: Download Artifacts
         uses: actions/download-artifact@v4
@@ -246,6 +249,12 @@ jobs:
           chmod +x resolc-x86_64-unknown-linux-musl
           chmod +x resolc-universal-apple-darwin
 
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.REVIVE_RELEASE_APP_ID }}
+          private-key: ${{ secrets.REVIVE_RELEASE_APP_KEY }}
+
       - name: create-release
         uses: softprops/action-gh-release@v2
         with:
@@ -255,9 +264,10 @@ jobs:
 
             ## Note for macOS Users
             The macOS binary is unsigned and it needs to be made runnable using `xattr -c resolc-universal-apple-darwin`.
-          tag_name: ${{ needs.check-version-changed.outputs.PKG_VER }}
-          name: ${{ needs.check-version-changed.outputs.PKG_VER }}
-          draft: true
+          tag_name: ${{ github.ref_name }}
+          name: ${{ github.ref_name }}
+          prerelease: true
+          token: ${{ steps.app-token.outputs.token }}
           target_commitish: ${{ github.sha }}
           files: |
             resolc-x86_64-unknown-linux-musl
diff --git a/Makefile b/Makefile
@@ -23,7 +23,7 @@
 install: install-bin install-npm
 
 install-bin:
-	cargo install --path crates/solidity
+	cargo install --locked --path crates/solidity
 
 install-npm:
 	npm install && npm fund
diff --git a/RELEASE.md b/RELEASE.md
@@ -4,11 +4,14 @@ Prior to the first stable release we neither have formal release processes nor d
 
 To create a new pre-release:
 
-1. Create a release PR which updates the `-dev.X` versions in the workspace `Cargo.toml` and updates the `CHANGELOG.md` accordingly. Add the `release-test` label to trigger the release workflows.
-2. If the CI passes, merge the release PR. The release workflow will attempt to build and publish a new release whenever the latest git tag does not match the cargo package version.
-3. Wait for the `Release` workflow to finish. If the workflow fails after the `build-linux-all` step, check if a tag has been created and delete it before restarting or pushing updates. Note: It's more convenient to debug the release workflow in a fork (the fork has to be under the `paritytech` org to access `parity-large` runners).
-4. Check draft release on [Releases page](https://github.com/paritytech/revive/releases) and publish (should contain `resolc.js`, `resolc.wasm`, `resolc-web.js`, and `resolc-static-linux` release assets)
-5. Update the [contract-docs](https://github.com/paritytech/contract-docs/) accordingly
+1. Create a release PR which updates the `-dev.X` versions in the workspace `Cargo.toml` and updates the `CHANGELOG.md` accordingly.
+2. If the CI passes, merge the release PR.
+3. Push a tag that has the same `-dev.X` version as in `Cargo.toml`
+4. The release workflow will attempt to build and publish a new pre-release if the latest tag does match the cargo package version.
+5. Wait for the `Release` workflow to finish. It should create the pre-release with the same `-dev.X` name.
+6. Check that pre-release was created on the [Releases page](https://github.com/paritytech/revive/releases) with all artifacts.
+7. After the release is published, another workflow should start automatically and update json files in https://github.com/paritytech/resolc-bin. Check the changes.
+8. Update the [contract-docs](https://github.com/paritytech/contract-docs/) accordingly
 
 # LLVM release
 
diff --git a/js/build.js b/js/build.js
@@ -4,7 +4,8 @@ const { minify } = require("terser");
 
 const SOLJSON_URI =
   "https://binaries.soliditylang.org/wasm/soljson-v0.8.29+commit.ab55807c.js";
-const RESOLC_WASM_URI = "http://127.0.0.1:8080/resolc.wasm";
+const RESOLC_WASM_URI =
+  process.env.RELEASE_RESOLC_WASM_URI || "http://127.0.0.1:8080/resolc.wasm";
 const RESOLC_WASM_TARGET_DIR = path.join(
   __dirname,
   "../target/wasm32-unknown-emscripten/release",
