From 87b1b4728e29f68c6b2bafde7f3d0f4c8af9b302 Mon Sep 17 00:00:00 2001 From: Denis Pisarev Date: Mon, 22 May 2023 14:36:20 +0200 Subject: [PATCH] Release pipeline for multiarch binaries and an image (#432) * add release binaries * test * Test builds * Install protobuff * Install protobuff for mac and windows * use sudo * workaround windows warning * typo * typo2 * github vars at their best * remove RUSTFLAGS * remove windows instead * mint release * undebug * more docs; build binary for an image * Containerfile and joint pipeline to build-publish it * debug workspace * debug workspace 2 * debug workspace 3 * undebug, add dependabot * expose more ports * edit README; remove old CI * edit github context variables * fix COPY * fix COPY chown * fix GLIBCXX error * optimize Containerfile * don't cache the release * Update README.md --- .github/dependabot.yml | 11 ++ .github/workflows/build-publish-image.yml | 53 -------- .github/workflows/check.yml | 19 ++- .github/workflows/release.yml | 155 ++++++++++++++++++++++ Containerfile | 31 +++++ README.md | 37 ++++-- 6 files changed, 235 insertions(+), 71 deletions(-) create mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/build-publish-image.yml create mode 100644 .github/workflows/release.yml create mode 100644 Containerfile diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1116676 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "daily" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/build-publish-image.yml b/.github/workflows/build-publish-image.yml deleted file mode 100644 index d2ee88a..0000000 --- a/.github/workflows/build-publish-image.yml +++ /dev/null @@ -1,53 +0,0 @@ -# This is an example GitHub action that will build and publish a Docker image to DockerHub -# You need to add the following secrets to your GitHub Repository or Organization to make this work -# - DOCKER_USERNAME: The username of the DockerHub account. E.g. parity -# - DOCKER_TOKEN: Access token for DockerHub, see https://docs.docker.com/docker-hub/access-tokens/. E.g. VVVVVVVV-WWWW-XXXXXX-YYYY-ZZZZZZZZZ -# The following are setup as an environment variable below -# - DOCKER_REPO: The unique name of the DockerHub repository. E.g. parity/polkadot - -name: Build & Publish Docker Image - -# Controls when the action will run. -on: - # Triggers the workflow on push events but only for the main branch - # push: - # branches: [ main ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Set an environment variable (that can be overriden) for the Docker Repo -env: - DOCKER_REPO: parity/polkadot - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - build: - # The type of runner that the job will run on - runs-on: ubuntu-22.04 - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - name: Check out the repo - uses: actions/checkout@v2.5.0 - - # Login to Docker hub using the credentials stored in the repository secrets - - name: Log in to Docker Hub - uses: docker/login-action@v2.1.0 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - # Get the commit short hash, to use as the rev - - name: Calculate rev hash - id: rev - run: echo "value=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - # Build and push 2 images, One with the version tag and the other with latest tag - - name: Build and push Docker images - uses: docker/build-push-action@v3.2.0 - with: - context: . - push: true - tags: ${{ env.DOCKER_REPO }}:v${{ steps.rev.outputs.value }}, ${{ env.DOCKER_REPO }}:latest diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c735640..143cca4 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -20,19 +20,18 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Set-Up - run: sudo apt install -y git clang curl libssl-dev llvm libudev-dev protobuf-compiler + - name: Install linux dependencies + run: sudo apt-get install -y clang libssl-dev llvm libudev-dev protobuf-compiler - - name: Install Rustup + - name: Install Rust run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source ~/.cargo/env - rustup default stable - rustup update nightly - rustup update stable - rustup target add wasm32-unknown-unknown --toolchain nightly + rustup update stable --no-self-update + rustup target add wasm32-unknown-unknown + + # Rust cache + - uses: Swatinem/rust-cache@v2 - name: Check Build run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a600578 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,155 @@ +# This is an example GitHub action that will build and publish the binaries and a Docker image +# You need to add the following secrets to your GitHub Repository or Organization to make this work +# - DOCKERHUB_USERNAME: The username of the DockerHub account. E.g. parity +# - DOCKERHUB_TOKEN: Access token for DockerHub, see https://docs.docker.com/docker-hub/access-tokens/. +# The following is set up as an environment variable below +# - DOCKER_REPO: The unique name of the DockerHub repository. E.g. parity/polkadot + +name: Release + +permissions: + contents: read + +# Controls when the action will run. +on: + push: + # Triggers the workflow on tag push events + tags: + - v[0-9]+.* + +env: + RUST_BACKTRACE: 1 + CARGO_INCREMENTAL: 0 + CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_NET_RETRY: 10 + RUSTFLAGS: -D warnings + RUSTUP_MAX_RETRIES: 10 + CARGO_TERM_COLOR: always + # Set an environment variable (that can be overriden) for the Docker Repo + DOCKER_REPO: tripleight/node-template + +defaults: + run: + shell: bash + + +jobs: + create-release: + # The type of runner that the job will run on + runs-on: ubuntu-22.04 + timeout-minutes: 60 + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + - uses: taiki-e/create-gh-release-action@v1 + with: + title: $version + branch: main + token: ${{ secrets.GITHUB_TOKEN }} + + upload-assets: + name: ${{ matrix.target }} + needs: + - create-release + strategy: + matrix: + # The list of architechture and OS to build for + # You can add or remove targets here if you want + # + # When updating this list, remember to update the target list in tests too + include: + # - target: aarch64-unknown-linux-gnu + - target: x86_64-unknown-linux-gnu + - target: aarch64-apple-darwin + os: macos-11 + - target: x86_64-apple-darwin + os: macos-11 + # - target: universal-apple-darwin + # os: macos-11 + + # The type of runner that the job will run on + # Runs on Ubuntu if other os is not specified above + runs-on: ${{ matrix.os || 'ubuntu-22.04' }} + timeout-minutes: 90 + permissions: + contents: write + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + with: + persist-credentials: false + + - name: Install Rust + run: | + rustup update stable --no-self-update + rustup target add wasm32-unknown-unknown + + - name: Install linux dependencies + if: (matrix.os == '' || startsWith(matrix.os, 'ubuntu')) + run: | + sudo apt-get -qq update + sudo apt-get install -y protobuf-compiler + + - name: Install mac dependencies + if: startsWith(matrix.os, 'macos') + run: brew install protobuf + - uses: taiki-e/setup-cross-toolchain-action@v1 + if: (matrix.os == '' || startsWith(matrix.os, 'ubuntu')) + with: + target: ${{ matrix.target }} + + # Build and upload the binary to the new release + - uses: taiki-e/upload-rust-binary-action@v1 + with: + bin: node-template + target: ${{ matrix.target }} + tar: all + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload x86_64 linux binary to workflow + if: (matrix.target == 'x86_64-unknown-linux-gnu') + uses: actions/upload-artifact@v3 + with: + name: node-template + path: ${{ github.workspace }}/target/x86_64-unknown-linux-gnu/release/node-template + + build-image: + # The type of runner that the job will run on + needs: + - upload-assets + runs-on: ubuntu-22.04 + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Login to Docker hub using the credentials stored in the repository secrets + - name: Log in to Docker Hub + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Check out the repo + uses: actions/checkout@v3 + + # Download the binary from the previous job + - name: Download x86_64 linux binary + uses: actions/download-artifact@v3 + with: + name: node-template + path: ${{ github.workspace }} + + # Build and push 2 images, One with the version tag and the other with latest tag + - name: Build and push Docker images + uses: docker/build-push-action@v4 + with: + context: . + file: ./Containerfile + push: true + build-args: | + DOCKER_REPO=${{ env.DOCKER_REPO }} + tags: | + ${{ env.DOCKER_REPO }}:${{ github.ref_name }} + ${{ env.DOCKER_REPO }}:latest diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..01a4a54 --- /dev/null +++ b/Containerfile @@ -0,0 +1,31 @@ +FROM docker.io/library/ubuntu:22.04 + +# show backtraces +ENV RUST_BACKTRACE 1 + +# install tools and dependencies +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates && \ +# apt cleanup + apt-get autoremove -y && \ + apt-get clean && \ + find /var/lib/apt/lists/ -type f -not -name lock -delete; \ +# add user and link ~/.local/share/polkadot to /data + useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ + mkdir -p /data /polkadot/.local/share && \ + chown -R polkadot:polkadot /data && \ + ln -s /data /polkadot/.local/share/node-template + +USER polkadot + +# copy the compiled binary to the container +COPY --chown=polkadot:polkadot --chmod=774 node-template /usr/bin/node-template + +# check if executable works in this container +RUN /usr/bin/node-template --version + +# ws_port +EXPOSE 9930 9333 9944 30333 30334 + +CMD ["/usr/bin/node-template"] diff --git a/README.md b/README.md index 7562b62..51399e5 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,14 @@ A fresh [Substrate](https://substrate.io/) node, ready for hacking :rocket: A standalone version of this template is available for each release of Polkadot in the [Substrate Developer Hub Parachain Template](https://github.com/substrate-developer-hub/substrate-parachain-template/) repository. The parachain template is generated directly at each Polkadot release branch from the [Node Template in Substrate](https://github.com/paritytech/substrate/tree/master/bin/node-template) upstream -It is usually best to use the stand-alone version to start a new project. +It is usually best to use the standalone version to start a new project. All bugs, suggestions, and feature requests should be made upstream in the [Substrate](https://github.com/paritytech/substrate/tree/master/bin/node-template) repository. ## Getting Started Depending on your operating system and Rust version, there might be additional packages required to compile this template. -Check the [Install](https://docs.substrate.io/install/) instructions for your platform for the most common dependencies. -Alternatively, you can use one of the [alternative installation](#alternatives-installations) options. +Check the [installation](https://docs.substrate.io/install/) instructions for your platform for the most common dependencies. +Alternatively, you can use one of the [alternative installation](#alternative-installations) options. ### Build @@ -63,7 +63,6 @@ Development chains: - Use the **Alice** account as the default `sudo` account. - Are preconfigured with a genesis state (`/node/src/chain_spec.rs`) that includes several prefunded development accounts. - To persist chain state between runs, specify a base path by running a command similar to the following: ```sh @@ -112,15 +111,13 @@ Take special note of the following: - [`chain_spec.rs`](./node/src/chain_spec.rs): A [chain specification](https://docs.substrate.io/build/chain-spec/) is a source code file that defines a Substrate chain's initial (genesis) state. Chain specifications are useful for development and testing, and critical when architecting the launch of a production chain. - Take note of the `development_config` and `testnet_genesis` functions,. + Take note of the `development_config` and `testnet_genesis` functions. These functions are used to define the genesis state for the local development chain configuration. These functions identify some [well-known accounts](https://docs.substrate.io/reference/command-line-tools/subkey/) and use them to configure the blockchain's initial state. - [`service.rs`](./node/src/service.rs): This file defines the node implementation. Take note of the libraries that this file imports and the names of the functions it invokes. In particular, there are references to consensus-related topics, such as the [block finalization and forks](https://docs.substrate.io/fundamentals/consensus/#finalization-and-forks) and other [consensus mechanisms](https://docs.substrate.io/fundamentals/consensus/#default-consensus-models) such as Aura for block authoring and GRANDPA for finality. - - ### Runtime In Substrate, the terms "runtime" and "state transition function" are analogous. @@ -147,10 +144,34 @@ A FRAME pallet is compromised of a number of blockchain primitives: - Errors: When a dispatchable fails, it returns an error. - Config: The `Config` configuration interface is used to define the types and parameters upon which a FRAME pallet depends. -## Alternatives Installations +## Alternative Installations Instead of installing dependencies and building this source directly, consider the following alternatives. +### CI + +#### Binary + +Check the [CI release workflow](./.github/workflows/release.yml) to see how the binary is built on CI. +You can modify the compilation targets depending on your needs. + +Allow GitHub actions in your forked repository to build the binary for you. + +Push a tag. For example, `v0.1.1`. Based on [Semantic Versioning](https://semver.org/), the supported tag format is `v?MAJOR.MINOR.PATCH(-PRERELEASE)?(+BUILD_METADATA)?` (the leading "v", pre-release version, and build metadata are optional and the optional prefix is also supported). + +After the pipeline is finished, you can download the binary from the releases page. + +#### Container + +Check the [CI release workflow](./.github/workflows/release.yml) to see how the Docker image is built on CI. + +Add your `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` secrets or other organization settings to your forked repository. +Change the `DOCKER_REPO` variable in the workflow to `[your DockerHub registry name]/[image name]`. + +Push a tag. + +After the image is built and pushed, you can pull it with `docker pull :`. + ### Nix Install [nix](https://nixos.org/), and optionally [direnv](https://github.com/direnv/direnv) and [lorri](https://github.com/nix-community/lorri) for a fully plug-and-play experience for setting up the development environment.