From 7e8f8bb0c9de6e651653a45fa169a49e27cd0d56 Mon Sep 17 00:00:00 2001 From: Scott Dixon Date: Fri, 16 Feb 2024 20:18:50 -0800 Subject: [PATCH] Making toxic multi-platform --- .github/workflows/toxic.yml | 56 ++++++++++++++++++++++++---------- README.md | 3 +- toolshed/llvm-install.sh | 7 +++-- toxic/README.md | 60 +++++++++++++++++++++++++++++++++---- 4 files changed, 100 insertions(+), 26 deletions(-) diff --git a/.github/workflows/toxic.yml b/.github/workflows/toxic.yml index 8bf3c95..c00c9e0 100644 --- a/.github/workflows/toxic.yml +++ b/.github/workflows/toxic.yml @@ -14,27 +14,51 @@ env: jobs: - build_and_publish: + dry_run: + if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'release' && startsWith( github.ref, 'refs/tags/tx') && github.event.release.prerelease ) }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - # Add QEMU to allow building non-native containers as + # part of multi-platform container builds. + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - # Setup buildx so we can do multi-platform container builds. + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: build + run: cd ${{ env.IMAGE_NAME }} && docker buildx build --platform linux/arm64,linux/amd64 . + + release: + if: ${{ github.event_name == 'release' && startsWith( github.ref, 'refs/tags/tx' ) && !github.event.release.prerelease }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: process event - id: process_event - uses: olegtarasov/get-tag@v2.1.2 + - # Add QEMU to allow building non-native containers as + # part of multi-platform container builds. + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - # Setup buildx so we can do multi-platform container builds. + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v2 with: - tagRegex: "(${{ env.IMAGE_VERSION_PREFIX }}).*" - - name: dryrun-build - if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'release' && startsWith( steps.process_event.outputs.tag, env.IMAGE_VERSION_PREFIX ) && github.event.release.prerelease ) }} - run: docker build ./${{ env.IMAGE_NAME }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: form image tag + id: image_tag + uses: olegtarasov/get-tag@v2.1.2 + - name: build-and-push - if: ${{ github.event_name == 'release' && startsWith( steps.process_event.outputs.tag, env.IMAGE_VERSION_PREFIX ) && !github.event.release.prerelease }} - uses: macbre/push-to-ghcr@master - with: - context: ./${{ env.IMAGE_NAME }} - dockerfile: ./${{ env.IMAGE_NAME }}/Dockerfile - image_name: ${{ env.ORG_NAMESPACE }}/${{ env.IMAGE_NAME }} - github_token: ${{ secrets.GITHUB_TOKEN }} - repository: ghcr.io + run: cd ${{ env.IMAGE_NAME }} && docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/${{ env.ORG_NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ steps.image_tag.outputs.tag }} . diff --git a/README.md b/README.md index 40e0ab6..a796c0c 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,8 @@ is based on Ubuntu and provides all modern Python3 distributions, pip, tox, and | tag | Base Python | Python Versions | Tox | Nox | pip | Platforms | Other Utilities | |----------|-------------|-----------------|-----|-----|-----|-----------|-----------------| | [tx20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | | | -| [tx22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | | | +| [tx22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | 2023.4.22 | 20.0.2 | | | +| [tx22.4.2](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.10 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.13.0 | 2023.4.22 | 20.0.2 | | | You can use this in your workflow yaml like this: diff --git a/toolshed/llvm-install.sh b/toolshed/llvm-install.sh index d057053..7c3ea85 100755 --- a/toolshed/llvm-install.sh +++ b/toolshed/llvm-install.sh @@ -20,7 +20,7 @@ usage() { exit 1; } -CURRENT_LLVM_STABLE=17 +CURRENT_LLVM_STABLE=18 BASE_URL="http://apt.llvm.org" # Check for required tools @@ -126,7 +126,8 @@ LLVM_VERSION_PATTERNS[14]="-14" LLVM_VERSION_PATTERNS[15]="-15" LLVM_VERSION_PATTERNS[16]="-16" LLVM_VERSION_PATTERNS[17]="-17" -LLVM_VERSION_PATTERNS[18]="" +LLVM_VERSION_PATTERNS[18]="-18" +LLVM_VERSION_PATTERNS[19]="" if [ ! ${LLVM_VERSION_PATTERNS[$LLVM_VERSION]+_} ]; then echo "This script does not support LLVM version $LLVM_VERSION" @@ -173,4 +174,4 @@ if [[ $ALL -eq 1 ]]; then PKG="$PKG libclang-rt-$LLVM_VERSION-dev libpolly-$LLVM_VERSION-dev" fi fi -apt-get install -y $PKG \ No newline at end of file +apt-get install -y $PKG diff --git a/toxic/README.md b/toxic/README.md index afd7e54..46878c5 100644 --- a/toxic/README.md +++ b/toxic/README.md @@ -29,22 +29,70 @@ export FGP = (fine-grained permission for OpenCyphal organization) echo $FGP | docker login ghcr.io -u (github username) --password-stdin ``` -... now build (where x is the next version number for the container): +... build a multi-platform image following the instructions [here](https://docs.docker.com/build/building/multi-platform/#multiple-native-nodes): ```bash -docker build -t ghcr.io/opencyphal/toxic:tx22.4.x . +docker buildx create --use --name cyphalbuild ``` -... and finally, push. +If you already created the `cyphalbuild` builder then just use it instead of creating it: ```bash -docker push ghcr.io/opencyphal/toxic:tx22.4.x +docker buildx use cyphalbuild ``` -## Testing out the container +... then build the container: -To login to an interactive session do: +```bash +docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/opencyphal/toxic:tx22.4.x . +``` + +(where x is the next version number for the container) + + +When the build completes you'll see the following warning: + +> WARNING: No output specified with docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load + +It's important you don't restart your docker build container before you load and/or push since it may blow away the cache you need to push or load from. + +### Load +Currently you can't use --load with docker desktop so the only way to test the container is to rebuild for one platform only: + +```bash +docker buildx build --platform linux/arm64 --load -t ghcr.io/opencyphal/toxic:tx22.4.x . +``` + +This will be operating off of the cache so it shouldn't take very long to complete. After it does you'll see your image using the classic `docker images` command. Now you can login to the container to test it out: ```bash docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toxic:tx22.4.x +``` + +### Push + +As with load, you need to re-build with a `--push` argument but you'll be using the cache so the build should be a no-op: + +```bash +docker buildx build --platform linux/amd64,linux/arm64 --push -t ghcr.io/opencyphal/toxic:tx22.4.x . +``` + +## More on Multi-Platform Builders + +The two commands above make some assumptions about defaults and capabilities that we haven't verified on all build hosts. First, the `buildx create` command is assumed to target the correct Docker context. You can see your contexts by doing: + +``` +docker context ls +``` + +... then target a specific context by adding it as an additional argument to the builder create command: + +``` +docker buildx create --use --name cyphalbuild desktop-linux +``` + +We also assume you are using a build that has our two supported host platforms `linux/amd64` and `linux/arm64`. You can verify this after creating the builder using the inspect command. This should also verify that your builder is now in effect: + +``` +docker buildx inspect --bootstrap ``` \ No newline at end of file