diff --git a/.github/workflows/joystream-node-docker-publish.yml b/.github/workflows/joystream-node-docker-publish.yml new file mode 100644 index 0000000000..9b930964ef --- /dev/null +++ b/.github/workflows/joystream-node-docker-publish.yml @@ -0,0 +1,43 @@ +# Publish production runtime build joystream/node:latest +name: joystream-node-docker-publish + +on: + - workflow_dispatch + +env: + REPOSITORY: joystream/node + +jobs: + main: + name: Publish joystream/node:latest docker image + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: '14.x' + + - id: compute_shasum + name: Compute runtime code shasum + run: | + export TAG_SHASUM=`scripts/runtime-code-shasum.sh` + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Publish manifest + run: | + IMAGE=${{ env.REPOSITORY }}:${{ env.TAG_SHASUM }} + LATEST_TAG=${{ env.REPOSITORY }}:latest + docker pull $IMAGE-amd64 + docker pull $IMAGE-arm64 + docker manifest create $LATEST_TAG $IMAGE-amd64 $IMAGE-arm64 + docker manifest annotate $LATEST_TAG $IMAGE-amd64 --arch amd64 + docker manifest annotate $LATEST_TAG $IMAGE-arm64 --arch arm64 + docker manifest push $LATEST_TAG diff --git a/.github/workflows/joystream-node-docker.yml b/.github/workflows/joystream-node-docker.yml index c6ba36dcab..aafa3816a7 100644 --- a/.github/workflows/joystream-node-docker.yml +++ b/.github/workflows/joystream-node-docker.yml @@ -162,12 +162,3 @@ jobs: docker manifest annotate $IMAGE $IMAGE-arm64 --arch arm64 docker manifest push $IMAGE - - name: Create manifest with latest tag for master - if: github.ref == 'refs/heads/master' - run: | - IMAGE=${{ env.REPOSITORY }}:${{ env.TAG_SHASUM }} - LATEST_TAG=${{ env.REPOSITORY }}:latest - docker manifest create $LATEST_TAG $IMAGE-amd64 $IMAGE-arm64 - docker manifest annotate $LATEST_TAG $IMAGE-amd64 --arch amd64 - docker manifest annotate $LATEST_TAG $IMAGE-arm64 --arch arm64 - docker manifest push $LATEST_TAG diff --git a/.github/workflows/run-network-tests.yml b/.github/workflows/run-network-tests.yml index bf0c106d99..b51c9b5937 100644 --- a/.github/workflows/run-network-tests.yml +++ b/.github/workflows/run-network-tests.yml @@ -45,6 +45,7 @@ jobs: runs-on: ubuntu-latest outputs: use_artifact: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz + runtime: ${{ steps.compute_shasum.outputs.shasum }} steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 @@ -65,7 +66,7 @@ jobs: - name: Cache docker images uses: actions/cache@v3 env: - cache-name: joystream-node-docker + cache-name: joystream-node-docker-1 with: path: ~/docker-images key: ${{ env.cache-name }}-${{ steps.compute_shasum.outputs.shasum }} @@ -83,8 +84,7 @@ jobs: run: | if ! [ -f joystream-node-docker-image.tar.gz ]; then docker pull joystream/node:${{ steps.compute_shasum.outputs.shasum }} - docker image tag joystream/node:${{ steps.compute_shasum.outputs.shasum }} joystream/node:latest - docker save --output joystream-node-docker-image.tar joystream/node:latest + docker save --output joystream-node-docker-image.tar joystream/node:${{ steps.compute_shasum.outputs.shasum }} gzip joystream-node-docker-image.tar cp joystream-node-docker-image.tar.gz ~/docker-images/ fi @@ -117,7 +117,7 @@ jobs: CARGO_FEATURES=testing-runtime CODE_SHASUM=${{ steps.compute_shasum.outputs.shasum }} push: false - tags: joystream/node + tags: joystream/node:${{ steps.compute_shasum.outputs.shasum }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache builder: ${{ steps.builder1.outputs.name }} @@ -126,7 +126,7 @@ jobs: - name: Save joystream/node image to cache run: | - docker save --output joystream-node-docker-image.tar joystream/node + docker save --output joystream-node-docker-image.tar joystream/node:${{ steps.compute_shasum.outputs.shasum }} gzip joystream-node-docker-image.tar cp joystream-node-docker-image.tar.gz ~/docker-images/ if: steps.check_files.outputs.files_exists == 'false' @@ -171,125 +171,74 @@ jobs: run: yarn workspace network-tests build - name: Execute network tests run: | - export RUNTIME=latest + export RUNTIME=${{ needs.build_images.outputs.runtime }} export NO_STORAGE=${{ matrix.no_storage }} tests/network-tests/run-tests.sh ${{ matrix.scenario }} - build_master_testing: - name: Build joystream node images for mainnet-testing + runtime_upgrade: + name: Runtime Upgrade from production runtime + needs: build_images runs-on: ubuntu-latest if: github.ref != 'refs/heads/master' - outputs: - use_artifact: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz steps: + # Checkout master branch - name: check out master repo uses: actions/checkout@v3 with: repository: Joystream/joystream ref: master - - id: compute_shasum + - name: pull base runtime image + id: pull_base_image env: RUNTIME_PROFILE: 'TESTING' run: | - export RUNTIME_CODE_SHASUM=`scripts/runtime-code-shasum.sh` - echo "::set-output name=shasum::${RUNTIME_CODE_SHASUM}" - - - name: Setup cache directory - run: mkdir ~/docker-images - - - name: Cache docker images - uses: actions/cache@v3 - env: - cache-name: joystream-node-docker-image-master - with: - path: ~/docker-images - key: ${{ env.cache-name }}-${{ steps.compute_shasum.outputs.shasum }} - - - name: Check if we have cached image - continue-on-error: true - run: | - if [ -f ~/docker-images/joystream-node-docker-image-master.tar.gz ]; then - docker load --input ~/docker-images/joystream-node-docker-image-master.tar.gz - cp ~/docker-images/joystream-node-docker-image-master.tar.gz . - fi - - - name: Check if we have pre-built image on Dockerhub - continue-on-error: true - run: | - if ! [ -f joystream-node-docker-image-master.tar.gz ]; then - docker pull joystream/node:${{ steps.compute_shasum.outputs.shasum }} - docker image tag joystream/node:${{ steps.compute_shasum.outputs.shasum }} joystream/node:mainnetDev - docker save --output joystream-node-docker-image-master.tar joystream/node:mainnetDev - gzip joystream-node-docker-image.tar - cp joystream-node-docker-image-master.tar.gz ~/docker-images/ - fi - - - name: Check we now have an image - id: check_master_image - uses: andstor/file-existence-action@v1 - with: - files: 'joystream-node-docker-image-master.tar.gz' - - - name: Set up Docker Buildx master - uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1.7 - id: builder2 - if: steps.check_master_image.outputs.files_exists == 'false' + export RUNTIME=`scripts/runtime-code-shasum.sh` + echo "::set-output name=shasum::${RUNTIME}" + docker pull joystream/node:$RUNTIME + docker images - - name: Build master image - uses: docker/build-push-action@v3 + # Get new runtime built for this workflow (target runtime) + # tagged in local repo as joystream/node:latest + - name: Get artifacts + uses: actions/download-artifact@v3 with: - context: . - file: joystream-node.Dockerfile - platforms: linux/amd64 - build-args: | - CARGO_FEATURES=testing-runtime - CODE_SHASUM=${{ steps.compute_shasum_master.outputs.shasum }} - push: false - tags: joystream/node:mainnetDev - builder: ${{ steps.builder2.outputs.name }} - load: true - if: steps.check_master_image.outputs.files_exists == 'false' - - - name: Save joystream/node image to cache + name: ${{ needs.build_images.outputs.use_artifact }} + - name: Install artifacts run: | - docker save --output joystream-node-docker-image-master.tar joystream/node:mainnetDev - gzip joystream-node-docker-image-master.tar - cp joystream-node-docker-image-master.tar.gz ~/docker-images/ - if: steps.check_master_image.outputs.files_exists == 'false' + docker load --input joystream-node-docker-image.tar.gz + docker images - - name: Save joystream/node image (master) to Artifacts - uses: actions/upload-artifact@v3 + # Checkout workflow's branch/tag/commit + - name: checkout workflow branch + uses: actions/checkout@v3 + - name: Check for runtime code changes + uses: technote-space/get-diff-action@v3 with: - name: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz - path: joystream-node-docker-image-master.tar.gz - - runtime_upgrade: - name: Runtime Upgrade From production runtime - needs: [build_master_testing, build_images] - if: github.ref != 'refs/heads/master' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + PREFIX_FILTER: | + runtime + runtime-modules + SUFFIX_FILTER: | + .rs + FILES: | + Cargo.lock + Cargo.toml + - name: setup node + uses: actions/setup-node@v3 with: node-version: '14.x' - - name: Install artifacts - uses: actions/download-artifact@v3 - - name: load docker images - run: | - docker load --input ${{needs.build_images.outputs.use_artifact}}/* - docker load --input ${{needs.build_master_testing.outputs.use_artifact}}/* - docker images - name: Install packages and dependencies + if: env.GIT_DIFF run: | yarn build:packages - name: Ensure tests are runnable + if: env.GIT_DIFF run: yarn workspace network-tests build - name: Execute network tests + if: env.GIT_DIFF env: - RUNTIME: mainnetDev - TARGET_RUNTIME: latest + TARGET_RUNTIME: ${{ needs.build_images.outputs.runtime }} + RUNTIME: ${{ steps.pull_base_image.outputs.shasum }} run: | export HOME=${PWD} mkdir -p ${HOME}/.local/share/joystream-cli diff --git a/devops/ansible/roles/common/tasks/install-tools.yml b/devops/ansible/roles/common/tasks/install-tools.yml index c9301b8b2f..094b3a014f 100644 --- a/devops/ansible/roles/common/tasks/install-tools.yml +++ b/devops/ansible/roles/common/tasks/install-tools.yml @@ -1,16 +1,10 @@ --- # Install developer tools -- name: Install aptitude - become: true - apt: - name: aptitude - state: latest - update_cache: true - - name: Install required system packages become: true apt: + update_cache: yes pkg: - coreutils - apt-transport-https diff --git a/joystream-node.Dockerfile b/joystream-node.Dockerfile index d94b42fa34..70c6c18528 100644 --- a/joystream-node.Dockerfile +++ b/joystream-node.Dockerfile @@ -8,7 +8,9 @@ RUN rustup default nightly-2022-05-11 RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2022-05-11 RUN rustup component add --toolchain nightly-2022-05-11 clippy RUN rustup install nightly -RUN cargo +nightly install cargo-chef +# Temporary build fix by using older version of cargo-chef +# https://github.com/Joystream/joystream/issues/4745 +RUN cargo +nightly install cargo-chef@0.1.57 FROM rust AS planner LABEL description="Cargo chef prepare" diff --git a/tests/network-tests/run-runtime-upgrade-tests.sh b/tests/network-tests/run-runtime-upgrade-tests.sh index 9d54cfee64..6929f8c9e5 100755 --- a/tests/network-tests/run-runtime-upgrade-tests.sh +++ b/tests/network-tests/run-runtime-upgrade-tests.sh @@ -150,16 +150,16 @@ function cleanup() { # entrypoint function main { - # Start a query-node - if [ "${NO_QN}" != true ]; then - ../../query-node/start.sh - fi - CONTAINER_ID="" export JOYSTREAM_NODE_TAG=${RUNTIME} if [ $TARGET_RUNTIME == $RUNTIME ]; then echo >&2 "Same tag for runtime and target runtime aborting..." - exit 1 + exit 0 + fi + + # Start a query-node + if [ "${NO_QN}" != true ]; then + ../../query-node/start.sh fi # 0. Generate config files