Improve docker image publish and runtime upgrade test workflow

.github/workflows/joystream-node-docker-publish.yml

@@ -0,0 +1,43 @@
+# Publish production runtime build joystream/node:latest
+name: joystream-node-docker-publish
+
+on:
+  - workflow_dispatch
+
+env:
+  REPOSITORY: joystream/node
+
+jobs:
+  main:
+    name: Publish joystream/node:latest docker image
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '14.x'
+
+      - id: compute_shasum
+        name: Compute runtime code shasum
+        run: |
+          export TAG_SHASUM=`scripts/runtime-code-shasum.sh`
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Publish manifest
+        run: |
+          IMAGE=${{ env.REPOSITORY }}:${{ env.TAG_SHASUM }}
+          LATEST_TAG=${{ env.REPOSITORY }}:latest
+          docker pull $IMAGE-amd64
+          docker pull $IMAGE-arm64
+          docker manifest create $LATEST_TAG $IMAGE-amd64 $IMAGE-arm64
+          docker manifest annotate $LATEST_TAG $IMAGE-amd64 --arch amd64
+          docker manifest annotate $LATEST_TAG $IMAGE-arm64 --arch arm64
+          docker manifest push $LATEST_TAG

.github/workflows/joystream-node-docker.yml

@@ -162,12 +162,3 @@ jobs:
           docker manifest annotate $IMAGE $IMAGE-arm64 --arch arm64
           docker manifest push $IMAGE
 
-      - name: Create manifest with latest tag for master
-        if: github.ref == 'refs/heads/master'
-        run: |
-          IMAGE=${{ env.REPOSITORY }}:${{ env.TAG_SHASUM }}
-          LATEST_TAG=${{ env.REPOSITORY }}:latest
-          docker manifest create $LATEST_TAG $IMAGE-amd64 $IMAGE-arm64
-          docker manifest annotate $LATEST_TAG $IMAGE-amd64 --arch amd64
-          docker manifest annotate $LATEST_TAG $IMAGE-arm64 --arch arm64
-          docker manifest push $LATEST_TAG

.github/workflows/run-network-tests.yml

@@ -45,6 +45,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       use_artifact: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
+      runtime: ${{ steps.compute_shasum.outputs.shasum }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -65,7 +66,7 @@ jobs:
       - name: Cache docker images
         uses: actions/cache@v3
         env:
-          cache-name: joystream-node-docker
+          cache-name: joystream-node-docker-1
         with:
           path: ~/docker-images
           key: ${{ env.cache-name }}-${{ steps.compute_shasum.outputs.shasum }}
@@ -83,8 +84,7 @@ jobs:
         run: |
           if ! [ -f joystream-node-docker-image.tar.gz ]; then
             docker pull joystream/node:${{ steps.compute_shasum.outputs.shasum }}
-            docker image tag joystream/node:${{ steps.compute_shasum.outputs.shasum }} joystream/node:latest
-            docker save --output joystream-node-docker-image.tar joystream/node:latest
+            docker save --output joystream-node-docker-image.tar joystream/node:${{ steps.compute_shasum.outputs.shasum }}
             gzip joystream-node-docker-image.tar
             cp joystream-node-docker-image.tar.gz ~/docker-images/
           fi
@@ -117,7 +117,7 @@ jobs:
             CARGO_FEATURES=testing-runtime
             CODE_SHASUM=${{ steps.compute_shasum.outputs.shasum }}
           push: false
-          tags: joystream/node
+          tags: joystream/node:${{ steps.compute_shasum.outputs.shasum }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
           builder: ${{ steps.builder1.outputs.name }}
@@ -126,7 +126,7 @@ jobs:
 
       - name: Save joystream/node image to cache
         run: |
-          docker save --output joystream-node-docker-image.tar joystream/node
+          docker save --output joystream-node-docker-image.tar joystream/node:${{ steps.compute_shasum.outputs.shasum }}
           gzip joystream-node-docker-image.tar
           cp joystream-node-docker-image.tar.gz ~/docker-images/
         if: steps.check_files.outputs.files_exists == 'false'
@@ -171,125 +171,74 @@ jobs:
         run: yarn workspace network-tests build
       - name: Execute network tests
         run: |
-          export RUNTIME=latest
+          export RUNTIME=${{ needs.build_images.outputs.runtime }}
           export NO_STORAGE=${{ matrix.no_storage }}
           tests/network-tests/run-tests.sh ${{ matrix.scenario }}
 
-  build_master_testing:
-    name: Build joystream node images for mainnet-testing
+  runtime_upgrade:
+    name: Runtime Upgrade from production runtime
+    needs: build_images
     runs-on: ubuntu-latest
     if: github.ref != 'refs/heads/master'
-    outputs:
-      use_artifact: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
     steps:
+      # Checkout master branch
       - name: check out master repo
         uses: actions/checkout@v3
         with:
           repository: Joystream/joystream
           ref: master
 
-      - id: compute_shasum
+      - name: pull base runtime image
+        id: pull_base_image
         env:
           RUNTIME_PROFILE: 'TESTING'
         run: |
-          export RUNTIME_CODE_SHASUM=`scripts/runtime-code-shasum.sh`
-          echo "::set-output name=shasum::${RUNTIME_CODE_SHASUM}"
-
-      - name: Setup cache directory
-        run: mkdir ~/docker-images
-
-      - name: Cache docker images
-        uses: actions/cache@v3
-        env:
-          cache-name: joystream-node-docker-image-master
-        with:
-          path: ~/docker-images
-          key: ${{ env.cache-name }}-${{ steps.compute_shasum.outputs.shasum }}
-
-      - name: Check if we have cached image
-        continue-on-error: true
-        run: |
-          if [ -f ~/docker-images/joystream-node-docker-image-master.tar.gz ]; then
-            docker load --input ~/docker-images/joystream-node-docker-image-master.tar.gz
-            cp ~/docker-images/joystream-node-docker-image-master.tar.gz .
-          fi
-
-      - name: Check if we have pre-built image on Dockerhub
-        continue-on-error: true
-        run: |
-          if ! [ -f joystream-node-docker-image-master.tar.gz ]; then
-            docker pull joystream/node:${{ steps.compute_shasum.outputs.shasum }}
-            docker image tag joystream/node:${{ steps.compute_shasum.outputs.shasum }} joystream/node:mainnetDev
-            docker save --output joystream-node-docker-image-master.tar joystream/node:mainnetDev
-            gzip joystream-node-docker-image.tar
-            cp joystream-node-docker-image-master.tar.gz ~/docker-images/
-          fi
-
-      - name: Check we now have an image
-        id: check_master_image
-        uses: andstor/file-existence-action@v1
-        with:
-          files: 'joystream-node-docker-image-master.tar.gz'
-
-      - name: Set up Docker Buildx master
-        uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1.7
-        id: builder2
-        if: steps.check_master_image.outputs.files_exists == 'false'
+          export RUNTIME=`scripts/runtime-code-shasum.sh`
+          echo "::set-output name=shasum::${RUNTIME}"
+          docker pull joystream/node:$RUNTIME
+          docker images
 
-      - name: Build master image
-        uses: docker/build-push-action@v3
+      # Get new runtime built for this workflow (target runtime)
+      # tagged in local repo as joystream/node:latest
+      - name: Get artifacts
+        uses: actions/download-artifact@v3
         with:
-          context: .
-          file: joystream-node.Dockerfile
-          platforms: linux/amd64
-          build-args: |
-            CARGO_FEATURES=testing-runtime
-            CODE_SHASUM=${{ steps.compute_shasum_master.outputs.shasum }}
-          push: false
-          tags: joystream/node:mainnetDev
-          builder: ${{ steps.builder2.outputs.name }}
-          load: true
-        if: steps.check_master_image.outputs.files_exists == 'false'
-
-      - name: Save joystream/node image to cache
+          name: ${{ needs.build_images.outputs.use_artifact }}
+      - name: Install artifacts
         run: |
-          docker save --output joystream-node-docker-image-master.tar joystream/node:mainnetDev
-          gzip joystream-node-docker-image-master.tar
-          cp joystream-node-docker-image-master.tar.gz ~/docker-images/
-        if: steps.check_master_image.outputs.files_exists == 'false'
+          docker load --input joystream-node-docker-image.tar.gz
+          docker images
 
-      - name: Save joystream/node image (master) to Artifacts
-        uses: actions/upload-artifact@v3
+      # Checkout workflow's branch/tag/commit
+      - name: checkout workflow branch
+        uses: actions/checkout@v3
+      - name: Check for runtime code changes
+        uses: technote-space/get-diff-action@v3
         with:
-          name: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
-          path: joystream-node-docker-image-master.tar.gz
-
-  runtime_upgrade:
-    name: Runtime Upgrade From production runtime
-    needs: [build_master_testing, build_images]
-    if: github.ref != 'refs/heads/master'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+          PREFIX_FILTER: |
+            runtime
+            runtime-modules
+          SUFFIX_FILTER: |
+            .rs
+          FILES: |
+            Cargo.lock
+            Cargo.toml
+      - name: setup node
+        uses: actions/setup-node@v3
         with:
           node-version: '14.x'
-      - name: Install artifacts
-        uses: actions/download-artifact@v3
-      - name: load docker images
-        run: |
-          docker load --input ${{needs.build_images.outputs.use_artifact}}/*
-          docker load --input ${{needs.build_master_testing.outputs.use_artifact}}/*
-          docker images
       - name: Install packages and dependencies
+        if: env.GIT_DIFF
         run: |
           yarn build:packages
       - name: Ensure tests are runnable
+        if: env.GIT_DIFF
         run: yarn workspace network-tests build
       - name: Execute network tests
+        if: env.GIT_DIFF
         env:
-          RUNTIME: mainnetDev
-          TARGET_RUNTIME: latest
+          TARGET_RUNTIME: ${{ needs.build_images.outputs.runtime }}
+          RUNTIME: ${{ steps.pull_base_image.outputs.shasum }}
         run: |
           export HOME=${PWD}
           mkdir -p ${HOME}/.local/share/joystream-cli

devops/ansible/roles/common/tasks/install-tools.yml

@@ -1,16 +1,10 @@
 ---
 # Install developer tools
 
-- name: Install aptitude
-  become: true
-  apt:
-    name: aptitude
-    state: latest
-    update_cache: true
-
 - name: Install required system packages
   become: true
   apt:
+    update_cache: yes
     pkg:
       - coreutils
       - apt-transport-https

joystream-node.Dockerfile

@@ -8,7 +8,9 @@ RUN rustup default nightly-2022-05-11
 RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2022-05-11
 RUN rustup component add --toolchain nightly-2022-05-11 clippy
 RUN rustup install nightly
-RUN cargo +nightly install cargo-chef
+# Temporary build fix by using older version of cargo-chef
+# https://github.com/Joystream/joystream/issues/4745
+RUN cargo +nightly install cargo-chef@0.1.57
 
 FROM rust AS planner
 LABEL description="Cargo chef prepare"

tests/network-tests/run-runtime-upgrade-tests.sh

@@ -150,16 +150,16 @@ function cleanup() {
 
 # entrypoint
 function main {
-    # Start a query-node
-    if [ "${NO_QN}" != true ]; then
-        ../../query-node/start.sh
-    fi
-
     CONTAINER_ID=""
     export JOYSTREAM_NODE_TAG=${RUNTIME}
     if [ $TARGET_RUNTIME == $RUNTIME ]; then
         echo >&2 "Same tag for runtime and target runtime aborting..."
-        exit 1
+        exit 0
+    fi
+
+    # Start a query-node
+    if [ "${NO_QN}" != true ]; then
+        ../../query-node/start.sh
     fi
 
     # 0. Generate config files