Fix for #35 and first multi-platform build (#37)

This builds llvm 18 and builds for both amd64 and arm64 hosts.

.github/workflows/texer.yml

@@ -3,7 +3,7 @@ name: ghcr.io/opencyphal/texer container build and publish.
 on:
   release:
     types: [ published ]
-    
+
   pull_request:
     branches: [ "main" ]
 

.github/workflows/toolshed.yml

@@ -3,7 +3,7 @@ name: ghcr.io/opencyphal/toolshed container build and publish.
 on:
   release:
     types: [ published ]
-    
+
   pull_request:
     branches: [ "main" ]
 
@@ -14,27 +14,47 @@ env:
 
 jobs:
 
-  build_and_publish:
+  dry_run:
+    if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'release' && startsWith( github.ref, 'ts') && github.event.release.prerelease ) }}
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - # Add QEMU to allow building non-native containers as
+      # part of multi-platform container builds.
+      name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - # Setup buildx so we can do multi-platform container builds.
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
 
+    - name: build
+      run: cd ${{ env.IMAGE_NAME}} && docker buildx build --platform linux/arm64,linux/amd64 .
+
+  release:
+    if: ${{ github.event_name == 'release' && startsWith( github.ref, 'ts' ) && !github.event.release.prerelease }}
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: process event
-      id: process_event
-      uses: olegtarasov/[email protected]
+    - # Add QEMU to allow building non-native containers as
+      # part of multi-platform container builds.
+      name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - # Setup buildx so we can do multi-platform container builds.
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to GHCR
+      uses: docker/login-action@v2
       with:
-        tagRegex: "(${{ env.IMAGE_VERSION_PREFIX }}).*"
-    - name: dryrun-build
-      if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'release' && startsWith( steps.process_event.outputs.tag, env.IMAGE_VERSION_PREFIX ) && github.event.release.prerelease ) }}
-      run: docker build ./${{ env.IMAGE_NAME }}
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
     - name: build-and-push
-      if: ${{ github.event_name == 'release' && startsWith( steps.process_event.outputs.tag, env.IMAGE_VERSION_PREFIX ) && !github.event.release.prerelease }}
-      uses: macbre/push-to-ghcr@master
-      with:
-        context: ./${{ env.IMAGE_NAME }}
-        dockerfile: ./${{ env.IMAGE_NAME }}/Dockerfile
-        image_name: ${{ env.ORG_NAMESPACE }}/${{ env.IMAGE_NAME }}
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        repository: ghcr.io
+      run: cd ${{ env.IMAGE_NAME }} && docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/${{ env.ORG_NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_VERSION_PREFIX }}${{ github.ref }} .

.github/workflows/toxic.yml

@@ -3,7 +3,7 @@ name: ghcr.io/opencyphal/toxic container build and publish.
 on:
   release:
     types: [ published ]
-    
+
   pull_request:
     branches: [ "main" ]
 

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "git-blame.gitWebUrl": ""
+}

README.md

@@ -23,12 +23,14 @@ like [libcanard](https://github.com/OpenCyphal/libcanard) and
 
 #### Supported Versions
 
-| tag      | Python | Gcc | Clang | Cmake | Other Utilities |
-|----------|--------|-----|-------|-------|-----------------|
-| [ts20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 9.3.0 | 10.0.0 | 3.16.3 | <ul><li>qemu</li><li>can-utils</li><li>sonar-scanner</li></ul> |
-| [ts22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | <ul><li>qemu</li><li>can-utils</li><li>sonar-scanner</li><li>doxygen 1.9.6</li></ul>
-| [ts22.4.2](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | <ul><li>qemu</li><li>can-utils</li><li>~~sonar-scanner~~</li><li>doxygen 1.9.6</li><li>nvm</li><li>node 16.20.0</li></ul>
-| [ts22.4.3](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | <ul><li>qemu</li><li>can-utils</li><li>doxygen 1.9.6</li><li>nvm</li><li>node 16.20.0</li><li>nox</li><li>govr</li></ul>
+| tag      | Python | Gcc | Clang | Cmake | Platforms | Other Utilities |
+|----------|--------|-----|-------|-------|-----------|-----------------|
+| [ts20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 9.3.0 | 10.0.0 | 3.16.3 | <ul><li>linux/amd64</li></ul> | <ul><li>qemu</li><li>can-utils</li><li>sonar-scanner</li></ul> |
+| [ts22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | <ul><li>linux/amd64</li></ul> | <ul><li>qemu</li><li>can-utils</li><li>sonar-scanner</li><li>doxygen 1.9.6</li></ul>
+| [ts22.4.2](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | <ul><li>linux/amd64</li></ul> | <ul><li>qemu</li><li>can-utils</li><li>~~sonar-scanner~~</li><li>doxygen 1.9.6</li><li>nvm</li><li>node 16.20.0</li></ul>
+| [ts22.4.3](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.3.0 | 15.0.7 | 3.22.1 | <ul><li>linux/amd64</li></ul> | <ul><li>qemu</li><li>can-utils</li><li>doxygen 1.9.6</li><li>nvm</li><li>node 16.20.0</li><li>nox</li><li>govr</li></ul>
+| ts22.4.4 - An experimental release that shouldn't be used. |
+| [ts22.4.5](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toolshed) | 3.10 | 11.4.0 | 18.0.0 | 3.22.1 | <ul><li>linux/amd64</li><li>linux/arm64</li></ul> | <ul><li>qemu</li><li>can-utils</li><li>doxygen 1.9.6</li><li>nvm</li><li>node 16.20.0</li><li>nox</li><li>govr</li></ul>
 
 
 You can use this in your workflow yaml like this:
@@ -49,10 +51,10 @@ is based on Ubuntu and provides the necessary compilers and utilities to author
 
 #### Supported Versions
 
-| tag      | Python | Tex Live | git | Other Utilities |
-|----------|--------|----------|-----|-----------------|
-| [te20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/texer) | 3.11 | 2019 | 2.25.1 | <ul><li>python pygments</li><li>lyx</li><li>inkscape</li></ul> |
-| [te22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/texer) | 3.11 | 2022 | 2.34.1 | <ul><li>python pygments</li><li>lyx</li><li>inkscape</li></ul> |
+| tag      | Python | Tex Live | git | Platforms | Other Utilities |
+|----------|--------|----------|-----|-----------|-----------------|
+| [te20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/texer) | 3.11 | 2019 | 2.25.1 | <ul><li>linux/amd64</li></ul> | <ul><li>python pygments</li><li>lyx</li><li>inkscape</li></ul> |
+| [te22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/texer) | 3.11 | 2022 | 2.34.1 | <ul><li>linux/amd64</li></ul> | <ul><li>python pygments</li><li>lyx</li><li>inkscape</li></ul> |
 
 You can use this in your workflow yaml like this:
 
@@ -74,10 +76,10 @@ is based on Ubuntu and provides all modern Python3 distributions, pip, tox, and
 
 #### Supported Versions
 
-| tag      | Base Python | Python Versions | Tox | Nox | pip | Other Utilities |
-|----------|-------------|-----------------|-----|-----|-----|-----------------|
-| [tx20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | <ul><li>sonar-scanner</li></ul> |
-| [tx22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | |
+| tag      | Base Python | Python Versions | Tox | Nox | pip | Platforms | Other Utilities |
+|----------|-------------|-----------------|-----|-----|-----|-----------|-----------------|
+| [tx20.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | <ul><li>linux/amd64</li></ul> | <ul><li>sonar-scanner</li></ul> |
+| [tx22.4.1](https://github.com/OpenCyphal/docker_toolchains/pkgs/container/toxic) | 3.8 | 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | 4.4.5 | (not available) | 20.0.2 | <ul><li>linux/amd64</li></ul> | |
 
 You can use this in your workflow yaml like this:
 

toolshed/Dockerfile

@@ -1,7 +1,7 @@
 #
 # Builds the toolshed/ubuntu-22.04 toolchain container.
 #
-FROM ubuntu:22.04
+FROM ubuntu:22.04 AS base
 
 LABEL org.opencontainers.image.description "Provides a consistent build and test environment for development, continuous-integration, and test automation of OpenCyphal C and C++ based projects."
 
@@ -19,53 +19,60 @@ ENV LC_ALL=en_US.UTF-8
 COPY provision.sh /
 RUN /provision.sh
 
+
 # +---------------------------------------------------------------------------+
 # | NVM/nodejs
 # +---------------------------------------------------------------------------+
+FROM base AS node
+
 WORKDIR /tmp
 ADD nvm-install.sh ./nvm-install.sh
 RUN . ./nvm-install.sh
 ENV NVM_DIR="~/.nvm"
 RUN . ~/.nvm/nvm.sh && nvm install v16.20.0
 
+
+# +---------------------------------------------------------------------------+
+# | INSTALL PYTHON STUFF
+# +---------------------------------------------------------------------------+
+FROM node AS python
+WORKDIR /tmp
+
+RUN pip3 install virtualenv
+RUN pip3 install tox
+RUN pip3 install nox
+RUN pip3 install gcovr
+RUN pip3 list
+
+
 # +---------------------------------------------------------------------------+
 # | LLVM
 # +---------------------------------------------------------------------------+
+FROM python AS llvm
 WORKDIR /tmp
+
 ADD llvm-install.sh ./llvm-install.sh
 ADD llvm-select.sh ./llvm-select.sh
-RUN ./llvm-install.sh 15 all
-RUN ./llvm-select.sh 15
+RUN ./llvm-install.sh 18 all
+RUN ./llvm-select.sh 18
+
 
 # +---------------------------------------------------------------------------+
-# | DOXYGEN BUILD
+# | GCC
 # +---------------------------------------------------------------------------+
-ARG DOXYGEN_VERSION=1_9_6
-
-WORKDIR /tmp
-COPY doxygen-Release_${DOXYGEN_VERSION}.sha512.txt .
-ADD https://github.com/doxygen/doxygen/archive/refs/tags/Release_${DOXYGEN_VERSION}.tar.gz doxygen-Release_${DOXYGEN_VERSION}.tar.gz
-RUN sha512sum -c doxygen-Release_${DOXYGEN_VERSION}.sha512.txt
-RUN mkdir doxygen-Release
-RUN tar -xvf doxygen-Release_${DOXYGEN_VERSION}.tar.gz -C doxygen-Release --strip-components 1
-RUN rm doxygen-Release_${DOXYGEN_VERSION}.tar.gz
-RUN mkdir doxygen-Release/build
+FROM llvm AS gcc-arm64
 
-WORKDIR /tmp/doxygen-Release/build
-RUN cmake -G "Unix Makefiles"  -Dbuild_parse:BOOL=ON -Duse_libclang:BOOL=ON ..
-RUN make
-RUN make install
+ARG GCC_HOST_PLATFORM=aarch64
 
-WORKDIR /tmp
-RUN rm -rf doxygen-Release
+FROM llvm AS gcc-amd64
 
+ARG GCC_HOST_PLATFORM=x86_64
 
-# +---------------------------------------------------------------------------+
-# | INSTALL TOOLCHAINS
-# +---------------------------------------------------------------------------+
+FROM gcc-$TARGETARCH AS gcc
+WORKDIR /tmp
 
 # ARM: M-Cores, Bare Metal
-ARG GCC_ARM_NONE_EABI_PLATFORM=x86_64
+ARG GCC_ARM_NONE_EABI_PLATFORM=${GCC_HOST_PLATFORM}
 ARG GCC_ARM_NONE_EABI_VERSION=11.2-2022.02
 ARG GCC_ARM_NONE_EABI_TRIPLE=arm-none-eabi
 ARG GCC_ARM_NONE_EABI_BASENAME=gcc-arm-${GCC_ARM_NONE_EABI_VERSION}-${GCC_ARM_NONE_EABI_PLATFORM}-${GCC_ARM_NONE_EABI_TRIPLE}
@@ -84,7 +91,7 @@ ENV PATH="/usr/share/${GCC_ARM_NONE_EABI_BASENAME}/bin:${PATH}"
 RUN ${GCC_ARM_NONE_EABI_TRIPLE}-gcc --version
 
 # ARM: A-Cores, Linux
-ARG GCC_ARM_NONE_LINUX_PLATFORM=x86_64
+ARG GCC_ARM_NONE_LINUX_PLATFORM=${GCC_HOST_PLATFORM}
 ARG GCC_ARM_NONE_LINUX_VERSION=11.2-2022.02
 ARG GCC_ARM_NONE_LINUX_TRIPLE=arm-none-linux-gnueabihf
 ARG GCC_ARM_NONE_LINUX_BASENAME=gcc-arm-${GCC_ARM_NONE_LINUX_VERSION}-${GCC_ARM_NONE_LINUX_PLATFORM}-${GCC_ARM_NONE_LINUX_TRIPLE}
@@ -101,21 +108,47 @@ ENV PATH="/usr/share/${GCC_ARM_NONE_LINUX_BASENAME}/bin:${PATH}"
 
 RUN ${GCC_ARM_NONE_LINUX_TRIPLE}-gcc --version
 
-# +---------------------------------------------------------------------------+
-# | INSTALL THE PATH FOR INTERATIVE SESSIONS
-# +---------------------------------------------------------------------------+
-RUN echo "export PATH=$PATH" >> ~/.bashrc
 
 # +---------------------------------------------------------------------------+
-# | INSTALL PYTHON STUFF
+# | DOXYGEN BUILD
 # +---------------------------------------------------------------------------+
-RUN pip3 install virtualenv
-RUN pip3 install tox
-RUN pip3 install nox
-RUN pip3 install gcovr
-RUN pip3 list
+FROM gcc AS doxygen-arm64
+
+ARG USE_LIB_CLANG=OFF
+
+FROM gcc AS doxygen-amd64
+
+ARG USE_LIB_CLANG=OFF
+
+FROM doxygen-$TARGETARCH AS doxygen
+WORKDIR /tmp
+
+ARG DOXYGEN_VERSION=1_9_6
+
+COPY doxygen-Release_${DOXYGEN_VERSION}.sha512.txt .
+ADD https://github.com/doxygen/doxygen/archive/refs/tags/Release_${DOXYGEN_VERSION}.tar.gz doxygen-Release_${DOXYGEN_VERSION}.tar.gz
+RUN sha512sum -c doxygen-Release_${DOXYGEN_VERSION}.sha512.txt
+RUN mkdir doxygen-Release
+RUN tar -xvf doxygen-Release_${DOXYGEN_VERSION}.tar.gz -C doxygen-Release --strip-components 1
+RUN rm doxygen-Release_${DOXYGEN_VERSION}.tar.gz
+RUN mkdir doxygen-Release/build
+
+WORKDIR /tmp/doxygen-Release/build
+RUN cmake -G "Unix Makefiles"  -Dbuild_parse:BOOL=ON -Duse_libclang:BOOL=${USE_LIB_CLANG} ..
+RUN make
+RUN make install
+
+WORKDIR /tmp
+RUN rm -rf doxygen-Release
+
 
 # +---------------------------------------------------------------------------+
-# | LEAVE THE WORKDIR AS /repo
+# | FINALIZE
 # +---------------------------------------------------------------------------+
+FROM doxygen AS all
+
+# INSTALL THE PATH FOR INTERATIVE SESSIONS
+RUN echo "export PATH=$PATH" >> ~/.bashrc
+
+# LEAVE THE WORKDIR AS /repo
 WORKDIR /repo

toolshed/README.md

@@ -29,22 +29,69 @@ export FGP = (fine-grained permission for OpenCyphal organization)
 echo $FGP | docker login ghcr.io -u (github username) --password-stdin
 ```
 
-... now build (where x is the next version number for the container):
+... build a multi-platform image following the instructions [here](https://docs.docker.com/build/building/multi-platform/#multiple-native-nodes):
 
 ```bash
-docker build -t ghcr.io/opencyphal/toolshed:ts22.4.x .
+docker buildx create --use --name cyphalbuild
 ```
 
-... and finally, push.
+If you already created the `cyphalbuild` builder then just use it instead of creating it:
 
 ```bash
-docker push ghcr.io/opencyphal/toolshed:ts22.4.x
+docker buildx use cyphalbuild
 ```
 
-## Testing out the container
+... then build the container:
 
-To login to an interactive session do:
+```bash
+docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/opencyphal/toolshed:ts22.4.x .
+```
+
+(where x is the next version number for the container)
+
+When the build completes you'll see the following warning:
+
+> WARNING: No output specified with docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load
+
+It's important you don't restart your docker build container before you load and/or push since it may blow away the cache you need to push or load from.
+
+### Load
+Currently you can't use --load with docker desktop so the only way to test the container is to rebuild for one platform only:
+
+```bash
+docker buildx build --platform linux/arm64 --load -t ghcr.io/opencyphal/toolshed:ts22.4.x .
+```
+
+This will be operating off of the cache so it shouldn't take very long to complete. After it does you'll see your image using the classic `docker images` command. Now you can login to the container to test it out:
 
 ```bash
 docker run --rm -it -v ${PWD}:/repo ghcr.io/opencyphal/toolshed:ts22.4.x
 ```
+
+### Push
+
+As with load, you need to re-build with a `--push` argument but you'll be using the cache so the build should be a no-op:
+
+```bash
+docker buildx build --platform linux/amd64,linux/arm64 --push -t ghcr.io/opencyphal/toolshed:ts22.4.x .
+```
+
+## More on Multi-Platform Builders
+
+The two commands above make some assumptions about defaults and capabilities that we haven't verified on all build hosts. First, the `buildx create` command is assumed to target the correct Docker context. You can see your contexts by doing:
+
+```
+docker context ls
+```
+
+... then target a specific context by adding it as an additional argument to the builder create command:
+
+```
+docker buildx create --use --name cyphalbuild desktop-linux
+```
+
+We also assume you are using a build that has our two supported host platforms `linux/amd64` and `linux/arm64`. You can verify this after creating the builder using the inspect command. This should also verify that your builder is now in effect:
+
+```
+docker buildx inspect --bootstrap
+```

toolshed/gcc-arm-11.2-2022.02-aarch64-arm-none-eabi.sha512.txt

@@ -0,0 +1 @@
+be19622e61d320427143c8492d8955447b75de080f3228766942e3529874adcf1f174a84e8f1f3ec92d14f72e9a93ba3fc43a347000fecb1c4bf1c3c7cf0667f  gcc-arm-11.2-2022.02-aarch64-arm-none-eabi.tar.xz

toolshed/gcc-arm-11.2-2022.02-aarch64-arm-none-linux-gnueabihf.sha512.txt

@@ -0,0 +1 @@
+f2acde25fbd5a0e5fcf2eb796942a7fa988dfe2a6190a0505f3b43b57ea3a4eb3b23e51aa35894d551a07a85561374040456826e407a02443daab132937f3be4  gcc-arm-11.2-2022.02-aarch64-arm-none-linux-gnueabihf.tar.xz