Image.yaml

name: Image
run-name: 'Image - ${{ inputs.oci-image-name }} - ${{ github.ref }}'

on:
  push:
    paths:
      - "oci/*/image.y*ml"
  pull_request:
    paths:
      - "oci/*/image.y*ml"
  workflow_dispatch:
    inputs:
      oci-image-name:
        description: "OCI image to build and test"
        required: true
      upload:
        description: "Upload image to GHCR"
        required: true
        type: boolean
        default: false
      external_ref_id: #(1)
        description: 'Optional ID for unique run detection'
        required: false
        type: string
        default: "default-id"

env:
  VULNERABILITY_REPORT_SUFFIX: ".vulnerability-report.json"

jobs:
  prepare-build:
    runs-on: ubuntu-22.04
    name: Prepare build
    outputs:
      build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
      release-to: ${{ steps.prepare-matrix.outputs.release-to }}
      oci-img-path: ${{ steps.validate-image.outputs.img-path }}
      oci-img-name: ${{ steps.validate-image.outputs.img-name }}
      revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
    steps:
      - name: ${{ inputs.external_ref_id }} #(2)
        if: ${{ github.event_name == 'workflow_dispatch' }}
        run: echo 'Started by ${{ inputs.external_ref_id }}' >> "$GITHUB_STEP_SUMMARY"

      - uses: actions/checkout@v3

      - name: Infer number of image triggers
        uses: tj-actions/changed-files@v35
        id: changed-files
        if: github.event_name != 'workflow_dispatch'
        with:
          separator: ","
          dir_names: "true"
          files: |
            oci/*/image.y*ml

      - name: Validate image from dispatch
        id: validate-image
        run: |
          set -ex

          if [[ "${{ github.event_name }}" != "workflow_dispatch" ]]
          then
            img_path="${{ steps.changed-files.outputs.all_changed_files }}"
            occurrences="${img_path//[^,]}"
            if [ ${#occurrences} -ne 0 ]
            then
              echo "ERR: can only build 1 image at a time, but trying to trigger ${img_path}"
              exit 1
            fi
          else
            img_path="oci/${{ inputs.oci-image-name }}"
          fi
          test -d "${img_path}"

          echo "img-name=$(basename ${img_path})" >> "$GITHUB_OUTPUT"
          echo "img-path=${img_path}" >> "$GITHUB_OUTPUT"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.x"

      - run: pip install -r src/image/requirements.txt

      - name: Get next revision number
        id: get-next-revision
        env:
          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
          IMAGE_NAME: ${{ steps.validate-image.outputs.img-name }}
          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
        run: ./src/image/define_image_revision.sh

      - name: Validate and prepare builds matrix
        id: prepare-matrix
        env:
          DATA_DIR: "revision-data"
        run: |
          mkdir ${{ env.DATA_DIR }}

          ./src/image/prepare_single_image_build_matrix.py \
            --oci-path ${{ steps.validate-image.outputs.img-path }} \
            --revision-data-dir ${{ env.DATA_DIR }} \
            --next-revision ${{ steps.get-next-revision.outputs.revision }}

          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"

      - uses: actions/cache/save@v3
        with:
          path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
          key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}

  run-build:
    runs-on: ubuntu-22.04
    needs: [prepare-build]
    name: Build
    strategy:
      fail-fast: true
      matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
    env:
      OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}
    steps:
      - name: Clone GitHub image repository
        uses: actions/checkout@v3
        id: clone-image-repo
        continue-on-error: true
        with:
          repository: ${{ matrix.source }}
          fetch-depth: 0

      - name: Clone generic image repository
        if: ${{ steps.clone-image-repo.outcome == 'failure' }}
        run: |
          git clone ${{ matrix.source }} .

      - run: git checkout ${{ matrix.commit }}

      - name: Validate image naming and base
        working-directory: ${{ matrix.directory }}
        run: |
          if [ -z ${{ matrix.dockerfile-build }} ]
          then
            rock_name=`cat rockcraft.y*ml | yq -r .name`
            if [[ "${{ matrix.path }}" != *"${rock_name}"* ]]
            then
              echo "ERROR: the ROCK's name '${rock_name}' must match the OCI folder name!"
              exit 1
            fi
          else
            grep 'FROM' Dockerfile | tail -1 | grep ubuntu || \
              (echo "ERROR: the image '${rock_name}' must be based on Ubuntu!" \
                && exit 1)
          fi

      # If this is a ROCK...
      - name: Build ROCK ${{ matrix.name }}
        if: matrix.dockerfile-build == ''
        id: rockcraft
        uses: canonical/craft-actions/rockcraft-pack@main
        with:
          path: "${{ matrix.directory }}"
          verbosity: debug

      - name: Rename ROCK OCI archive
        if: matrix.dockerfile-build == ''
        run: |
          mv ${{ steps.rockcraft.outputs.rock }} ${{ env.OCI_ARCHIVE_NAME }}

      # If this is a Dockerfile-based image...
      - name: Set up QEMU
        if: matrix.dockerfile-build != ''
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        if: matrix.dockerfile-build != ''
        id: buildx
        uses: docker/setup-buildx-action@v2

      - name: Form the platforms string
        if: matrix.dockerfile-build != ''
        id: buildx-platforms
        run: |
          sudo apt install -y jq
          platforms=$(echo linux/$(echo '${{ toJSON(matrix.dockerfile-build.platforms) }}' | jq -r 'join(",linux/")'))
          echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"

      - name: Build OCI image ${{ matrix.name }}
        if: matrix.dockerfile-build != ''
        uses: docker/build-push-action@v4
        with:
          context: "${{ matrix.directory }}"
          outputs: "type=oci,dest=${{ env.OCI_ARCHIVE_NAME }}"
          platforms: ${{ steps.buildx-platforms.outputs.platforms }}
          push: false
          cache-from: type=gha
          cache-to: type=gha,mode=max
          provenance: false
          sbom: false

      - uses: actions/cache/save@v3
        with:
          path: ${{ env.OCI_ARCHIVE_NAME }}
          key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}

  test:
    needs: [prepare-build, run-build]
    name: Test
    strategy:
      fail-fast: true
      matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
    uses: canonical/oci-factory/.github/workflows/Tests.yaml@main
    with:
      oci-image-name: "${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}"
      oci-image-path: "oci/${{ matrix.name }}"
      is-a-rock: ${{ matrix.dockerfile-build == '' && true || false }}
      test-from: "cache"
      cache-key: "${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}"
    secrets: inherit

  upload:
    runs-on: ubuntu-22.04
    needs: [prepare-build, run-build, test]
    name: Upload
    if: ${{ inputs.upload || (github.ref_name == 'main' && github.event_name == 'push') }}
    strategy:
      fail-fast: true
      matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
    env:
      OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}
    outputs:
      artefacts-hashes: ${{ steps.artefacts-hashes.outputs.hashes }}
    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Setup environment
        env:
          ROCKS_DEV_LP_SSH_PRIVATE: ${{ secrets.ROCKS_DEV_LP_SSH_PRIVATE }}
          ROCKS_DEV_LP_USERNAME: ${{ secrets.ROCKS_DEV_LP_USERNAME }}
          CPC_BUILD_TOOLS_REPO: git.launchpad.net/~cloudware/cloudware/+git/cpc_build_tools
          # CPC_BUILD_TOOLS_REPO_REF: 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
          SKOPEO_BRANCH: "v1.9.1"
          SKOPEO_URL: "https://github.com/containers/skopeo"
        run: |
          ./src/image/requirements.sh
          ./src/uploads/requirements.sh
          pip install -r src/uploads/requirements.txt -r src/image/requirements.txt

      - name: Clone GitHub image repository
        uses: actions/checkout@v3
        id: clone-image-repo
        continue-on-error: true
        with:
          repository: ${{ matrix.source }}
          fetch-depth: 0
          path: source

      - name: Clone generic image repository
        if: ${{ steps.clone-image-repo.outcome == 'failure' }}
        run: |
          git clone ${{ matrix.source }} source

      - run: cd source && git checkout ${{ matrix.commit }}

      - uses: actions/cache/restore@v3
        with:
          path: ${{ env.OCI_ARCHIVE_NAME }}
          key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}
          fail-on-cache-miss: true

      - name: Infer track name
        id: get-track
        env:
          DOCKERFILE_IMAGE_VERSION: ${{ matrix.dockerfile-build.version }}
        run: |
          ./src/uploads/infer_image_track.py --recipe-dirname source/${{ matrix.directory }}

      - name: Name output artefact
        id: rename-oci-archive
        run: |
          # Rename the OCI archive tarball
          canonical_tag="${{ steps.get-track.outputs.track }}_${{ matrix.revision }}"
          name="${{ matrix.name }}_${canonical_tag}"
          mv ${{ env.OCI_ARCHIVE_NAME }} $name

          echo "name=${name}" >> "$GITHUB_OUTPUT"
          echo "canonical-tag=${canonical_tag}" >> "$GITHUB_OUTPUT"

      - uses: actions/cache/save@v3
        with:
          path: ${{ steps.rename-oci-archive.outputs.name }}
          key: ${{ github.run_id }}-${{ steps.rename-oci-archive.outputs.name }}

      - name: Install Syft
        uses: anchore/sbom-action/download-syft@v0
        with:
          syft-version: "v0.75.0"

      - name: Infer architectures
        id: get-arches
        run: |
          set -ex
          arches=$(skopeo inspect --raw \
            oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
            | jq -r 'if has("manifests") then .manifests[].platform.architecture else "${{ runner.arch }}" end' \
            | jq -nRcr '[inputs] | join(",")')

          echo "Detected architectures: ${arches}"
          echo "arches='${arches}'" >> "$GITHUB_OUTPUT"

      - name: Generate SBOMs
        id: generate-sboms
        run: |
          set -ex
          echo "Generating SBOMs for ${{ steps.get-arches.outputs.arches }}"

          IFS=',' read -ra arch_list <<< ${{ steps.get-arches.outputs.arches }}
          for arch in "${arch_list[@]}"; do
            if [[ "${arch}" == "unknown" ]]; then
              continue
            fi

            echo "Generate SBOM for ${arch}"

            skopeo --override-arch $arch copy \
              oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
              oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch}

            syft oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch} \
              -o spdx-json \
              --file ${{ steps.rename-oci-archive.outputs.name }}.${arch}.sbom.spdx.json
          done

          all_sboms_zip="${{ steps.rename-oci-archive.outputs.name }}.sbom.spdx.zip"
          zip "${all_sboms_zip}" ${{ steps.rename-oci-archive.outputs.name }}.*.sbom.spdx.json

          echo "sboms=${all_sboms_zip}" >> "$GITHUB_OUTPUT"

      - name: Fetch vulnerability artifacts for hashing
        uses: actions/cache/restore@v3
        with:
          path: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
          key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
          fail-on-cache-miss: true

      # https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md
      - name: Calculate artefacts hashes
        id: artefacts-hashes
        env:
          VULN_REPORT: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
          SBOMS: ${{ steps.generate-sboms.outputs.sboms }}
          OCI_IMAGE_ARCHIVE: ${{ steps.rename-oci-archive.outputs.name }}
        run: |
          set -ex
          echo "hashes=$(sha256sum ${{ env.VULN_REPORT }} ${{ env.OCI_IMAGE_ARCHIVE }} ${{ env.SBOMS }} | base64 -w0)"

      - name: Login to GHCR
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Upload SBOM
        uses: actions/upload-artifact@v3
        with:
          name: ${{ steps.generate-sboms.outputs.sboms }}
          path: ${{ steps.generate-sboms.outputs.sboms }}
          if-no-files-found: error

      - name: Upload image
        uses: actions/upload-artifact@v3
        with:
          name: ${{ steps.rename-oci-archive.outputs.name }}
          path: ${{ steps.rename-oci-archive.outputs.name }}
          if-no-files-found: error

      - name: Upload to GHCR
        id: upload-image
        env:
          GHCR_REPO: ghcr.io/${{ github.repository_owner }}/oci-factory
          GHCR_USERNAME: ${{ github.actor }}
          GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
        run: |
          oci_images="${PWD}/images-oci"
          rm -fr $oci_images
          mkdir -p $oci_images  

          tar -xf ${{ steps.rename-oci-archive.outputs.name }} -C $oci_images

          source="oci:${oci_images}"

          digest="$(skopeo inspect $source | jq -r .Digest)"
          echo "digest=$digest" >> $GITHUB_OUTPUT

          ./src/image/tag_and_publish.sh "${source}" \
            ${{ matrix.name }} \
            ${{ steps.rename-oci-archive.outputs.canonical-tag }}

      - name: Upload build metadata to Swift
        env:
          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
          IMAGE_NAME: ${{ matrix.name }}
          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
        run: |
          jq --arg base "${{ steps.get-track.outputs.base }}" \
            --arg digest "${{ steps.upload-image.outputs.digest }}" \
            '. + {base: $base, digest: $digest}' <<< '${{ toJSON(matrix) }}' > build_metadata.json
          ./src/uploads/upload_to_swift.sh \
            ${{ matrix.name }} \
            ${{ steps.get-track.outputs.track }} \
            ${{ matrix.revision }} \
            build_metadata.json \
            ${{ steps.generate-sboms.outputs.sboms }} \
            ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}

      - name: Create Git tag
        uses: rickstaa/action-create-tag@v1
        with:
          tag: "${{ steps.rename-oci-archive.outputs.name }}"
          message: "upload(${{ matrix.name }}): Build and upload new image revision ${{ matrix.revision }}"
          force_push_tag: true
          github_token: ${{ secrets.ROCKSBOT_TOKEN }}

  # We need this job because we want to make the releases all in one go,
  # so that we know which revisions to release, and so that we can update
  # and commit the _releases.json file in a single commit, outside a matrix job
  dispatch-releases:
    name: Dispatch releases
    needs: [prepare-build, upload]
    runs-on: ubuntu-22.04
    if: ${{ needs.prepare-build.outputs.release-to != '' }}
    concurrency:
      group: ${{ needs.prepare-build.outputs.oci-img-path }}
      cancel-in-progress: true
    env:
      REVISION_DATA_DIR: revision-data
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: "3.x"

      - uses: actions/checkout@v3
        with:
          persist-credentials: false

      - uses: actions/cache/restore@v3
        with:
          path: ${{ env.REVISION_DATA_DIR }}
          key: ${{ needs.prepare-build.outputs.revision-data-cache-key }}
          fail-on-cache-miss: true

      - run: pip install -r src/image/requirements.txt

      - name: Merge release requests
        id: merge-release-requests
        env:
          PYTHONUNBUFFERED: 1
        run: |
          set -ex
          for revision_file in `ls ${{ env.REVISION_DATA_DIR }}`
          do
            ret=0
            release=$(jq -er .release < ${{ env.REVISION_DATA_DIR }}/$revision_file) || ret=1
            
            if [ $ret -eq 1 ]
            then
              echo "Revision $revision_file not marked for release"
              continue
            fi

            echo "Merge revision $revision_file with requested releases"

            ./src/image/merge_release_info.py \
              --image-trigger "${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml" \
              --revision-data-file "${{ env.REVISION_DATA_DIR }}/${revision_file}"
          done

      - uses: actions/cache/save@v3
        with:
          path: ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
          key: ${{ github.run_id }}-image-trigger

      - name: Dispatch Releases workflow
        # Using this action because others can have this problem:
        # https://github.com/convictional/trigger-workflow-and-wait/issues/61
        uses: mathze/workflow-dispatch-action@v1.2.0
        id: run-releases
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ github.ref_name }}
          fail-on-error: true
          workflow-name: Release.yaml
          payload: '{ "oci-image-name": "${{ needs.prepare-build.outputs.oci-img-name }}", "image-trigger-cache-key": "${{ github.run_id }}-image-trigger"}'
          run-id: dummy
          trigger-timeout: "3m"
          use-marker-step: true

      - name: Write step summary
        run: |
          url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-releases.outputs.run-id }}'
          echo " - Triggered releases for '${{ needs.prepare-build.outputs.oci-img-name }}' at [${url}](${url})" >> "$GITHUB_STEP_SUMMARY"

      - name: Enforce release conclusion
        if: ${{ steps.run-releases.outputs.run-conclusion != 'success' }}
        # The previous step doesn't always raise an error
        run: |
          url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-releases.outputs.run-id }}'
          echo "Failed to release '${{ needs.prepare-build.outputs.oci-img-name }}' at [${url}](${url})."
          exit 1

  generate-provenance:
    name: Generate SLSA provenance
    needs: [upload]
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/download-artifact@v3
        id: download
        with:
          path: artifacts

      - name: Generate provenance
        uses: philips-labs/slsa-provenance-action@v0.8.0
        with:
          command: generate
          subcommand: files
          arguments: --artifact-path ${{ steps.download.outputs.download-path }} --output-path provenance.json

      # - uses: sigstore/cosign-installer@v3.0.1
      #   with:
      #     cosign-release: 'v2.0.0'

      # - name: Sign provenance file
      #   env:
      #     PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
      #     COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
      #     PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
      #   run: |
      #     echo "${PRIVATE_KEY}" > cosign.key
      #     echo "${PUBLIC_KEY}" > cosign.pub
      #     cosign sign-blob --key cosign.key --output-signature provenance.json.sig provenance.json

      # - name: Upload verification keys
      #   uses: actions/upload-artifact@v3
      #   with:
      #     name: verify
      #     path: |
      #       cosign.pub
      #       provenance.json.sig
      #     if-no-files-found: error

      # - name: Generate checksums
      #   working-directory: ${{ steps.download.outputs.download-path }}
      #   run: sha256sum * > SHA256SUMS || true

      # - name: Upload SHA256SUMS file
      #   uses: actions/upload-artifact@v3
      #   with:
      #     name: SHA256SUMS
      #     path: SHA256SUMS
      #     if-no-files-found: error

      - name: Upload provenance file
        uses: actions/upload-artifact@v3
        with:
          name: provenance
          path: provenance.json
          if-no-files-found: error

  # generate-provenance:
  #   name: Generate SLSA provenance
  #   needs: [upload]
  #   permissions:
  #     actions: read # Needed for detection of GitHub Actions environment.
  #     id-token: write # Needed for provenance signing and ID
  #     contents: write # Needed for release uploads
  #   uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.5.0
  #   with:
  #     base64-subjects: "${{ needs.upload.outputs.artefacts-hashes }}"

  notify:
    runs-on: ubuntu-22.04
    name: Notify
    needs: [prepare-build, run-build, upload, dispatch-releases, generate-provenance]
    if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        with:
          python-version: '3.x'
          
      - name: Summarize workflow failure message
        id: get-summary
        run: |
          echo '${{ toJson(needs) }}' > jobs.json
          ./src/notifications/summarize_workflow_results.py --jobs-file jobs.json

      - name: Get contacts for ${{ needs.prepare-build.outputs.oci-img-name }}
        id: get-contacts
        working-directory: ${{ needs.prepare-build.outputs.oci-img-path }}
        run: |
          mm_channels=$(yq -r '.notify | ."mattermost-channels" | join(",")' < contacts.y*ml)
          echo "mattermost-channels=${mm_channels}" >> "$GITHUB_OUTPUT"

      - name: Notify via Mattermost
        env:
          MM_BOT_TOKEN: ${{ secrets.MM_BOT_TOKEN }}
          FINAL_STATUS: failure
          MM_SERVER: ${{ secrets.MM_SERVER }}
          URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          SUMMARY: ${{ steps.get-summary.outputs.summary }}
          FOOTER: "Triggered by ${{ github.triggering_actor }}. Ref: ${{ github.ref }}. Attempts: ${{ github.run_attempt }}"
          TITLE: '${{ needs.prepare-build.outputs.oci-img-name }}: failed to build->upload->release'
        run: |
          for channel in $(echo ${{ steps.get-contacts.outputs.mattermost-channels }} | tr ',' ' ')
          do
            MM_CHANNEL_ID="${channel}" ./src/notifications/send_to_mattermost.sh
          done