From 27a68e13878844f7b003cd5a15866374aed8ca77 Mon Sep 17 00:00:00 2001 From: Josua Mayer Date: Thu, 4 Jul 2024 14:34:34 +0200 Subject: [PATCH] add ci using github actions Signed-off-by: Josua Mayer --- .github/workflows/build.yml | 242 ++++++++++++++++++++++++++++++++++++ conf/docker/Dockerfile | 60 +++++++++ conf/includes/ci.conf | 13 ++ 3 files changed, 315 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 conf/docker/Dockerfile create mode 100644 conf/includes/ci.conf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..8ffbc13 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,242 @@ +name: build + +on: + push: + branches: [ "kirkstone" ] + pull_request: + branches: [ "kirkstone" ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + prepare_container: + runs-on: self-hosted + outputs: + uid: ${{ steps.uid_step.outputs.userid }} + gid: ${{ steps.uid_step.outputs.groupid }} + steps: + # Checks-out your repository under ${{ github.host-workspace }}, so your job can access it + - uses: actions/checkout@v4 + + - name: Get user id/group + id: uid_step + run: | + echo "userid=$(id -u)" >> "$GITHUB_OUTPUT" + echo "groupid=$(id -g)" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config: /etc/docker/cibuilder.toml + + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ciserver.ci:5000 + username: ${{ secrets.CI_CACHE_REGISTRY_LOGIN }} + password: ${{ secrets.CI_CACHE_REGISTRY_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ciserver.ci:5000/${{ github.repository_id }}:latest + cache-from: type=registry,ref=ciserver.ci:5000/${{ github.repository_id }}:cache + cache-to: type=registry,ref=ciserver.ci:5000/${{ github.repository_id }}:cache,mode=max + file: conf/docker/Dockerfile + build-args: | + USER_ID=${{ steps.uid_step.outputs.userid }} + GROUP_ID=${{ steps.uid_step.outputs.groupid }} + + build_images: + needs: prepare_container + runs-on: self-hosted + timeout-minutes: 1080 + container: + image: ciserver.ci:5000/${{ github.repository_id }}:latest + credentials: + username: ${{ secrets.CI_CACHE_REGISTRY_LOGIN }} + password: ${{ secrets.CI_CACHE_REGISTRY_PASSWORD }} + options: --user "${{ needs.prepare_container.outputs.uid }}:${{ needs.prepare_container.outputs.gid }}" + outputs: + build_tag: ${{ steps.tag_step.outputs.build_tag }} + steps: + - name: Checkout dependency layers + shell: bash {0} + run: | + git clone -b kirkstone git://git.yoctoproject.org/poky + git clone -b kirkstone git://git.yoctoproject.org/git/meta-freescale + git clone -b kirkstone https://git.openembedded.org/meta-openembedded + git clone -b kirkstone https://git.yoctoproject.org/meta-virtualization + git clone -b kirkstone https://github.com/aws4embeddedlinux/meta-aws.git + pwd + echo ${{ github.host-workspace }} + ls -lh + find meta-aws/* + + - name: Checkout pull-request version of meta-solidrun-arm-imx6 + uses: actions/checkout@v4 + with: + path: meta-solidrun-arm-imx6 + + - name: Get build tag + shell: bash {0} + id: tag_step + run: | + build_tag=$(date +%Y-%m-%d)_$(cd meta-solidrun-arm-imx6; git rev-parse --short HEAD) + echo "build_tag=$build_tag" >> "$GITHUB_OUTPUT" + + - name: Create cache dir + run: mkdir -p buildcache + + - name: Fetch cache from server + uses: tespkg/actions-cache/restore@v1 + with: + endpoint: ciserver.ci + port: 9000 + insecure: true + accessKey: ${{ secrets.CI_CACHE_MINIO_ACCESS }} + secretKey: ${{ secrets.CI_CACHE_MINIO_SECRET }} + bucket: cicache + use-fallback: false + key: ${{ github.repository }} + path: | + buildcache + + - name: Configure + shell: bash {0} + run: | + cd poky + . ./oe-init-build-env ../build + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-freescale >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-solidrun-arm-imx6 >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-openembedded/meta-filesystems >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-openembedded/meta-multimedia >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-openembedded/meta-networking >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-oe >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-python >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-virtualization >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-aws >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/meta-freescale >> conf/bblayers.conf + printf 'BBLAYERS += "%s"\n' ${{ github.host-workspace }}/sources/meta-solidrun-arm-imx6 >> conf/bblayers.conf + printf '# Enable Docker\nIMAGE_INSTALL:append = " docker-ce python3-docker-compose kernel-modules"\nDISTRO_FEATURES:append = " virtualization"\n' >> conf/local.conf + printf '# Enable Corretto JRE\nIMAGE_INSTALL:append = " corretto-11-bin"\n' >> conf/local.conf + printf '# CI Config\nrequire conf/includes/ci.conf\n' >> conf/local.conf + # patch meta-aws + sed -i "s;greengrass-bin;greengrass;g" ${{ github.host-workspace }}/meta-aws/recipes-devtools/amazon-corretto/corretto-11-bin_*.bb + + - name: Download Sources (ignore errors) + shell: bash {0} + run: | + cd poky + . ./oe-init-build-env ../build + export CACHE_DIR="${{ github.host-workspace }}/buildcache" + export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS CACHE_DIR" + MACHINE=imx6qdlcubox bitbake core-image-full-cmdline core-image-weston-sdk --runall=fetch + continue-on-error: true + + - name: Build (ignore errors) + shell: bash {0} + run: | + cd poky + . ./oe-init-build-env ../build + export CACHE_DIR="${{ github.host-workspace }}/buildcache" + export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS CACHE_DIR" + MACHINE=imx6qdlcubox bitbake core-image-full-cmdline core-image-weston-sdk + continue-on-error: true + + - name: Update cache on the server (build may have failed) +# if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + uses: tespkg/actions-cache/save@v1 + with: + endpoint: ciserver.ci + port: 9000 + insecure: true + accessKey: ${{ secrets.CI_CACHE_MINIO_ACCESS }} + secretKey: ${{ secrets.CI_CACHE_MINIO_SECRET }} + bucket: cicache + use-fallback: false + key: ${{ github.repository }} + path: | + buildcache + + - name: Build + shell: bash {0} + run: | + cd poky + . ./oe-init-build-env ../build + export CACHE_DIR="${{ github.host-workspace }}/buildcache" + export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS CACHE_DIR" + MACHINE=imx6qdlcubox bitbake core-image-full-cmdline core-image-weston-sdk + + - name: Update cache on the server (build has succeeded) +# if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + uses: tespkg/actions-cache/save@v1 + with: + endpoint: ciserver.ci + port: 9000 + insecure: true + accessKey: ${{ secrets.CI_CACHE_MINIO_ACCESS }} + secretKey: ${{ secrets.CI_CACHE_MINIO_SECRET }} + bucket: cicache + use-fallback: false + key: ${{ github.repository }} + path: | + buildcache + + - name: Select artifacts for publishing + shell: bash {0} + run: | + mkdir deploy + DEPLOY_PATH=build/tmp/deploy/images/imx6qdlcubox/ + cp -L $DEPLOY_PATH/core-image-full-cmdline-imx6qdlcubox.wic.gz deploy/ + cp -L $DEPLOY_PATH/core-image-full-cmdline-imx6qdlcubox.wic.bmap deploy/ + cp -L $DEPLOY_PATH/core-image-full-cmdline-imx6qdlcubox.manifest deploy/ + cp -L $DEPLOY_PATH/core-image-weston-sdk-imx6qdlcubox.wic.gz deploy/ + cp -L $DEPLOY_PATH/core-image-weston-sdk-imx6qdlcubox.wic.bmap deploy/ + cp -L $DEPLOY_PATH/core-image-weston-sdk-imx6qdlcubox.manifest deploy/ + cp -L $DEPLOY_PATH/SPL deploy/ + cp -L $DEPLOY_PATH/u-boot.img deploy/ + tar -C $DEPLOY_PATH -cpzf deploy/device-tree.tgz + cp -L $DEPLOY_PATH/zImage deploy/ + cp -L $DEPLOY_PATH/modules-imx6qdlcubox.tgz deploy/ + + - name: Deploy to the local minio storage + uses: yakubique/minio-upload@v1.1.3 + with: + endpoint: http://ciserver.ci:9000 + insecure: true + access_key: ${{ secrets.CI_CACHE_MINIO_ACCESS }} + secret_key: ${{ secrets.CI_CACHE_MINIO_SECRET }} + bucket: cipublish + source: ./deploy + target: "/${{ github.repository_id }}/${{ steps.tag_step.outputs.build_tag }}" + recursive: true + + publish_images: + needs: build_images + runs-on: self-hosted + if: github.event_name == 'push' + steps: + - name: Download an artifacts from MinIO + uses: yakubique/minio-download@v1.1.1 + with: + endpoint: http://ciserver.ci:9000 + insecure: true + access_key: ${{ secrets.CI_CACHE_MINIO_ACCESS }} + secret_key: ${{ secrets.CI_CACHE_MINIO_SECRET }} + bucket: cipublish + source: "/${{ github.repository_id }}/${{ needs.build_images.outputs.build_tag }}/" + target: "." + recursive: true + + - name: Upload to S3 + uses: shallwefootball/upload-s3-action@v1.3.3 + with: + aws_key_id: ${{ secrets.IMAGES_S3_ACCESS }} + aws_secret_access_key: ${{ secrets.IMAGES_S3_SECRET }} + aws_bucket: ${{ secrets.IMAGES_S3_BUCKET }} + endpoint: ${{ secrets.IMAGES_S3_HOST }} + source_dir: deploy + destination_dir: IMX6/meta-solidrun-arm-imx6/${{ needs.build_images.outputs.build_tag }} diff --git a/conf/docker/Dockerfile b/conf/docker/Dockerfile new file mode 100644 index 0000000..47e0cfc --- /dev/null +++ b/conf/docker/Dockerfile @@ -0,0 +1,60 @@ +FROM ubuntu:22.04 + +# Set the default shell to bash +SHELL ["/bin/bash", "-c"] + +# Create and set the working directory +WORKDIR /workspace + +# Install necessary packages and dependencies for Yocto Project builds. +# This includes all the recommended packages listed in the Yocto Project Quick Start +# guide for a Ubuntu-based host, as of my last update. + +ENV DEBIAN_FRONTEND noninteractive + +# Build tools +RUN apt-get update && apt-get install -y apt-transport-https \ + gawk wget git diffstat \ + unzip texinfo gcc build-essential chrpath \ + socat cpio python3 python3-pip python3-pexpect \ + xz-utils debianutils iputils-ping python3-git \ + python3-jinja2 libegl1-mesa libsdl1.2-dev \ + python3-subunit mesa-common-dev patchelf \ + zstd liblz4-tool file locales libacl1 \ + tree sudo locales ca-certificates \ + bash icecc + +RUN locale-gen en_US.UTF-8 + +RUN update-ca-certificates + +# KAS +RUN python3 -m pip install kas + +# Set environment variables +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +# Arguments to pass host user's UID and GID +ARG USER_ID=1000 +ARG GROUP_ID=1000 + +# Create a user 'developer' with the same UID/GID as the host user +RUN groupadd -g ${GROUP_ID} developer +RUN useradd -m -u ${USER_ID} -g developer -d /home/developer developer +RUN mkdir -p /workspace +RUN chown -R developer:developer /workspace +RUN mkdir -p /home/developer/.ssh +RUN touch /home/developer/.ssh/known_hosts +RUN chmod 700 /home/developer/.ssh +RUN chmod 600 /home/developer/.ssh/known_hosts +RUN chown -R developer:developer /home/developer/.ssh + + +RUN git config --add --system user.email "developer@example.com" +RUN git config --add --system user.name "Build Container" +RUN git config --add --system http.version HTTP/1.1 + +# The entry point is set to bash. This means that when the container starts, it will drop the user into bash +ENTRYPOINT ["/bin/bash"] diff --git a/conf/includes/ci.conf b/conf/includes/ci.conf new file mode 100644 index 0000000..7dde455 --- /dev/null +++ b/conf/includes/ci.conf @@ -0,0 +1,13 @@ +CACHE_DIR ??= "${TOPDIR}" + +BB_GENERATE_MIRROR_TARBALLS = "1" +BB_GIT_SHALLOW ?= "1" +BB_GIT_SHALLOW_DEPTH ?= "1" + +DL_DIR = "${CACHE_DIR}/downloads" +SSTATE_DIR = "${CACHE_DIR}/sstate-cache" +CCACHE_TOP_DIR = "${CACHE_DIR}/ccache" + +INHERIT += "ccache" + +INHERIT += "rm_work"