From 04f55a733a26c7039c2a680410e5c2d2d4a48972 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 11 Apr 2024 14:31:47 +0200 Subject: [PATCH] Align image builds with other projects Signed-off-by: Evan Lezar --- .github/workflows/images.yaml | 19 +- Makefile | 173 ++++++++---------- .../container/Dockerfile.ubi8 | 12 +- deployments/container/Makefile | 109 +++++++++++ deployments/container/multi-arch.mk | 19 ++ deployments/container/native-only.mk | 31 ++++ versions.mk | 6 + 7 files changed, 256 insertions(+), 113 deletions(-) rename Dockerfile.ubi8 => deployments/container/Dockerfile.ubi8 (86%) create mode 100644 deployments/container/Makefile create mode 100644 deployments/container/multi-arch.mk create mode 100644 deployments/container/native-only.mk diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 7fda6b01..7cb42a57 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Images +# Run this workflow on pull requests or merge to main/release branches +name: Image -# Run this workflow on pull requests on: pull_request: types: @@ -29,12 +29,14 @@ on: - release-* jobs: - build-image: + build: runs-on: ubuntu-latest + strategy: + matrix: + dist: [ubi8] steps: - uses: actions/checkout@v4 - name: Checkout code - + name: Check out code - name: Calculate build vars id: vars run: | @@ -54,21 +56,20 @@ jobs: fi echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV echo "BUILD_MULTI_ARCH_IMAGES=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV - + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build image env: IMAGE_NAME: ghcr.io/${LOWERCASE_REPO_OWNER}/k8s-kata-manager VERSION: ${COMMIT_SHORT_SHA} run: | echo "${VERSION}" - make build-image + make -f deployments/container/Makefile build-${{ matrix.dist }} diff --git a/Makefile b/Makefile index 310442b1..6b0e13b7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,113 +12,96 @@ # See the License for the specific language governing permissions and # limitations under the License. +DOCKER ?= docker +MKDIR ?= mkdir +DIST_DIR ?= $(CURDIR)/dist + include $(CURDIR)/versions.mk -GIT_COMMIT := $(shell git describe --tags --dirty --always) +ifeq ($(IMAGE_NAME),) +REGISTRY ?= nvidia +IMAGE_NAME = $(REGISTRY)/$(NAME) +endif -##### Go variables ##### -MODULE := github.com/NVIDIA/k8s-kata-manager -GOOS ?= linux -GO_CMD ?= go -GO_FMT ?= gofmt -GO_TEST_FLAGS ?= -race -LDFLAGS = -ldflags "-s -w -X github.com/NVIDIA/k8s-kata-manager/internal/version.version=$(GIT_COMMIT)" +EXAMPLES := $(patsubst ./examples/%/,%,$(sort $(dir $(wildcard ./examples/*/)))) +EXAMPLE_TARGETS := $(patsubst %,example-%, $(EXAMPLES)) -##### General make targets ##### CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/)))) CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS)) -CHECK_TARGETS := assert-fmt lint -MAKE_TARGETS := cmds build install fmt test coverage generate $(CHECK_TARGETS) +CHECK_TARGETS := lint +MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate vendor check-vendor $(CHECK_TARGETS) -TARGETS := $(MAKE_TARGETS) -DOCKER_TARGETS := $(patsubst %, docker-%, $(TARGETS)) +TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS) + +DOCKER_TARGETS := $(patsubst %,docker-%, $(TARGETS)) .PHONY: $(TARGETS) $(DOCKER_TARGETS) -##### Container image variables ##### -BUILD_MULTI_ARCH_IMAGES ?= no -DOCKER ?= docker -BUILDX = -ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) -BUILDX = buildx +ifeq ($(VERSION),) +CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG)) +else +CLI_VERSION = $(VERSION) endif +CLI_VERSION_PACKAGE = github.com/NVIDIA/$(NAME)/internal/info -ifeq ($(IMAGE_NAME),) -REGISTRY ?= nvidia -IMAGE_NAME := $(REGISTRY)/k8s-kata-manager +binaries: cmds +ifneq ($(PREFIX),) +cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*) endif +ifneq ($(shell uname),Darwin) +EXTLDFLAGS = -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files +else +EXTLDFLAGS = -Wl,-undefined,dynamic_lookup +endif +BUILDFLAGS = -ldflags "-s -w '-extldflags=$(EXTLDFLAGS)' -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" +build: + go build $(BUILDFLAGS) ./... -IMAGE_VERSION := $(VERSION) - -DIST ?= ubi8 - -# Note: currently there is no need to build images for different distributions, -# so the distribution is omitted from the tag -#IMAGE_TAG ?= $(IMAGE_VERSION)-$(DIST) -IMAGE_TAG ?= $(IMAGE_VERSION) -IMAGE = $(IMAGE_NAME):$(IMAGE_TAG) - -OUT_IMAGE_NAME ?= $(IMAGE_NAME) -OUT_IMAGE_VERSION ?= $(IMAGE_VERSION) -#OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST) -OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION) -OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG) - -##### Container image make targets ##### -# Note: currently there is no need to build images for different distributions. -IMAGE_BUILD_TARGETS := build-image -IMAGE_PUSH_TARGETS := push-image -IMAGE_PULL_TARGETS := pull-image -.PHONY: $(IMAGE_BUILD_TARGETS) $(IMAGE_PUSH_TARGETS) - -###### Target definitions ##### cmds: $(CMD_TARGETS) $(CMD_TARGETS): cmd-%: - @mkdir -p bin - GOOS=$(GOOS) $(GO_CMD) build -v -o bin $(LDFLAGS) $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*) + go build $(BUILDFLAGS) $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*) -build: - GOOS=$(GOOS) $(GO_CMD) build -v $(LDFLAGS) $(MODULE)/... +examples: $(EXAMPLE_TARGETS) +$(EXAMPLE_TARGETS): example-%: + go build $(BUILDFLAGS) $(COMMAND_BUILD_OPTIONS) ./examples/$(*) -install: - $(GO_CMD) install -v $(LDFLAGS) $(MODULE)/cmd/... +all: check test build binary +check: $(CHECK_TARGETS) +# Apply go fmt to the codebase fmt: - $(GO_CMD) list -f '{{.Dir}}' $(MODULE)/... \ - | xargs $(GO_FMT) -s -l -w - -assert-fmt: - $(GO_CMD) list -f '{{.Dir}}' $(MODULE)/... \ - | xargs $(GO_FMT) -s -l | ( grep -v /vendor/ || true ) > fmt.out - @if [ -s fmt.out ]; then \ - echo "\nERROR: The following files are not formatted:\n"; \ - cat fmt.out; \ - rm fmt.out; \ - exit 1; \ - else \ - rm fmt.out; \ - fi + go list -f '{{.Dir}}' $(MODULE)/... \ + | xargs gofmt -s -l -w + +goimports: + go list -f {{.Dir}} $(MODULE)/... \ + | xargs goimports -local $(MODULE) -w lint: golangci-lint run ./... +vendor: + go mod tidy + go mod vendor + go mod verify + +check-vendor: vendor + git diff --quiet HEAD -- go.mod go.sum vendor + COVERAGE_FILE := coverage.out -test: build - $(GO_CMD) test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/... +test: build cmds + go test -coverprofile=$(COVERAGE_FILE) $(MODULE)/cmd/... $(MODULE)/internal/... $(MODULE)/api/... coverage: test cat $(COVERAGE_FILE) | grep -v "_mock.go" > $(COVERAGE_FILE).no-mocks - $(GO_CMD) tool cover -func=$(COVERAGE_FILE).no-mocks + go tool cover -func=$(COVERAGE_FILE).no-mocks -# Generate code -generate: controller-gen - $(CONTROLLER_GEN) object object:headerFile="hack/boilerplate.go.txt" paths="./api/..." +generate: .generate-api + go generate $(MODULE)/... -# Download controller-gen locally if necessary -PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) -CONTROLLER_GEN = $(PROJECT_DIR)/bin/controller-gen -controller-gen: - @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.10.0 +CONTROLLER_GEN ?= controller-gen +.generate-api: + $(CONTROLLER_GEN) object object:headerFile="hack/boilerplate.go.txt" paths="./api/..." $(DOCKER_TARGETS): docker-%: @echo "Running 'make $(*)' in container image $(BUILDIMAGE)" @@ -132,25 +115,15 @@ $(DOCKER_TARGETS): docker-%: $(BUILDIMAGE) \ make $(*) - -##### Image build and push targets ##### -build-image: - DOCKER_BUILDKIT=1 \ - $(DOCKER) $(BUILDX) build --pull \ - $(DOCKER_BUILD_OPTIONS) \ - $(DOCKER_BUILD_PLATFORM_OPTIONS) \ - --tag $(IMAGE) \ - --build-arg BASE_DIST="$(DIST)" \ - --build-arg CUDA_VERSION="$(CUDA_VERSION)" \ - --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \ - --build-arg VERSION="$(VERSION)" \ - --build-arg CVE_UPDATES="$(CVE_UPDATES)" \ - --file Dockerfile.ubi8 \ - $(CURDIR) - -push-image: - $(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)" - $(DOCKER) push "$(OUT_IMAGE)" - -pull-image: - $(DOCKER) pull "$(IMAGE)" +# Start an interactive shell using the development image. +PHONY: .shell +.shell: + $(DOCKER) run \ + --rm \ + -ti \ + -e GOCACHE=/tmp/.cache/go \ + -e GOMODCACHE=/tmp/.cache/gomod \ + -v $(PWD):/work \ + -w /work \ + --user $$(id -u):$$(id -g) \ + $(BUILDIMAGE) diff --git a/Dockerfile.ubi8 b/deployments/container/Dockerfile.ubi8 similarity index 86% rename from Dockerfile.ubi8 rename to deployments/container/Dockerfile.ubi8 index 7caf7bf2..fb291213 100644 --- a/Dockerfile.ubi8 +++ b/deployments/container/Dockerfile.ubi8 @@ -28,7 +28,7 @@ RUN set -eux; \ case "${arch##*-}" in \ x86_64 | amd64) ARCH='amd64' ;; \ ppc64el | ppc64le) ARCH='ppc64le' ;; \ - aarch64) ARCH='arm64' ;; \ + aarch64 | arm64) ARCH='arm64' ;; \ *) echo "unsupported architecture" ; exit 1 ;; \ esac; \ wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \ @@ -39,11 +39,15 @@ ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH WORKDIR /build COPY . . -RUN GOOS=linux make cmd-k8s-kata-manager + +RUN mkdir /artifacts +ARG VERSION="N/A" +ARG GIT_COMMIT="unknown" +RUN make PREFIX=/artifacts cmds FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} -COPY --from=build /build/bin/k8s-kata-manager /usr/local/bin/k8s-kata-manager +COPY --from=build /artifacts/k8s-kata-manager /usr/bin/k8s-kata-manager RUN dnf install -y kmod @@ -55,4 +59,4 @@ LABEL name="NVIDIA Kata Manager for Kubernetes" LABEL summary="NVIDIA Kata Manager for Kubernetes" LABEL description="See summary" -ENTRYPOINT [ "/usr/local/bin/k8s-kata-manager" ] +ENTRYPOINT [ "/usr/bin/k8s-kata-manager" ] diff --git a/deployments/container/Makefile b/deployments/container/Makefile new file mode 100644 index 00000000..86d4a85c --- /dev/null +++ b/deployments/container/Makefile @@ -0,0 +1,109 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BUILD_MULTI_ARCH_IMAGES ?= no +DOCKER ?= docker +BUILDX = +ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) +BUILDX = buildx +endif +MKDIR ?= mkdir + +##### Global variables ##### +include $(CURDIR)/versions.mk + +ifeq ($(IMAGE_NAME),) +IMAGE_NAME := $(REGISTRY)/$(NAME) +endif + +IMAGE_VERSION := $(VERSION) + +IMAGE_TAG ?= $(IMAGE_VERSION)-$(DIST) +IMAGE = $(IMAGE_NAME):$(IMAGE_TAG) + +OUT_IMAGE_NAME ?= $(IMAGE_NAME) +OUT_IMAGE_VERSION ?= $(IMAGE_VERSION) +OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST) +OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG) + +##### Public rules ##### +DEFAULT_PUSH_TARGET := ubi8 +DISTRIBUTIONS = $(DEFAULT_PUSH_TARGET) + +IMAGE_TARGETS := $(patsubst %,image-%,$(DISTRIBUTIONS)) +BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS)) +PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS)) +TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS)) + +.PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) $(BUILD_TARGETS) + +ifneq ($(BUILD_MULTI_ARCH_IMAGES),true) +include $(CURDIR)/deployments/container/native-only.mk +else +include $(CURDIR)/deployments/container/multi-arch.mk +endif + +# For the default push target we also push a short tag equal to the version. +# We skip this for the development release +DEVEL_RELEASE_IMAGE_VERSION ?= devel +PUSH_MULTIPLE_TAGS ?= true +ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION)) +PUSH_MULTIPLE_TAGS = false +endif +ifeq ($(PUSH_MULTIPLE_TAGS),true) +push-$(DEFAULT_PUSH_TARGET): push-short +endif + +push-%: DIST = $(*) +push-short: DIST = $(DEFAULT_PUSH_TARGET) + +build-%: DIST = $(*) +build-%: DOCKERFILE = $(CURDIR)/deployments/container/Dockerfile.$(DOCKERFILE_SUFFIX) + +# Use a generic build target to build the relevant images +$(IMAGE_TARGETS): image-%: + DOCKER_BUILDKIT=1 \ + $(DOCKER) $(BUILDX) build --pull \ + $(DOCKER_BUILD_OPTIONS) \ + $(DOCKER_BUILD_PLATFORM_OPTIONS) \ + --tag $(IMAGE) \ + --build-arg BASE_DIST="$(DIST)" \ + --build-arg CUDA_VERSION="$(CUDA_VERSION)" \ + --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \ + --build-arg VERSION="$(VERSION)" \ + --build-arg GIT_COMMIT="$(GIT_COMMIT)" \ + $(if $(LABEL_IMAGE_SOURCE),--label "org.opencontainers.image.source=$(LABEL_IMAGE_SOURCE)",) \ + -f $(DOCKERFILE) \ + $(CURDIR) + +build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu + +build-ubi8: DOCKERFILE_SUFFIX := ubi8 + +# Handle the default build target. +.PHONY: build +build: $(DEFAULT_PUSH_TARGET) +$(DEFAULT_PUSH_TARGET): build-$(DEFAULT_PUSH_TARGET) +$(DEFAULT_PUSH_TARGET): DIST = $(DEFAULT_PUSH_TARGET) + +REGCTL ?= regctl +$(PUSH_TARGETS): push-%: + $(REGCTL) \ + image copy \ + $(IMAGE) $(OUT_IMAGE) + +push-short: + $(REGCTL) \ + image copy \ + $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION) diff --git a/deployments/container/multi-arch.mk b/deployments/container/multi-arch.mk new file mode 100644 index 00000000..7e624554 --- /dev/null +++ b/deployments/container/multi-arch.mk @@ -0,0 +1,19 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PUSH_ON_BUILD ?= false +DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD) +DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64 + +$(BUILD_TARGETS): build-%: image-% diff --git a/deployments/container/native-only.mk b/deployments/container/native-only.mk new file mode 100644 index 00000000..1cb86b76 --- /dev/null +++ b/deployments/container/native-only.mk @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PUSH_ON_BUILD ?= false +DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 + +ifeq ($(PUSH_ON_BUILD),true) +$(BUILD_TARGETS): build-%: image-% + $(DOCKER) push "$(IMAGE)" +else +$(BUILD_TARGETS): build-%: image-% +endif + +# For the default distribution we also retag the image. +# Note: This needs to be updated for multi-arch images. +ifeq ($(IMAGE_TAG),$(VERSION)-$(DIST)) +$(DEFAULT_PUSH_TARGET): + $(DOCKER) image inspect $(IMAGE) > /dev/null || $(DOCKER) pull $(IMAGE) + $(DOCKER) tag $(IMAGE) $(subst :$(IMAGE_TAG),:$(VERSION),$(IMAGE)) +endif diff --git a/versions.mk b/versions.mk index 686f62b7..23885bff 100644 --- a/versions.mk +++ b/versions.mk @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +NAME := k8s-kata-manager +MODULE := github.com/NVIDIA/$(NAME) VERSION ?= v0.1.2 +REGISTRY ?= nvcr.io/nvidia + vVERSION := v$(VERSION:v%=%) CUDA_VERSION := 12.3.1 @@ -22,3 +26,5 @@ GOLANG_VERSION ?= 1.21.5 BUILDIMAGE_TAG ?= devel-go$(GOLANG_VERSION) BUILDIMAGE ?= ghcr.io/nvidia/k8s-test-infra:$(BUILDIMAGE_TAG) + +GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always --abbrev=40 2> /dev/null || echo "")