[PATCH v3 6/6] containers/Makefile: Makefile to automate builds

[ohilyard at iol.unh.edu]

From: Owen Hilyard <ohilyard at iol.unh.edu>

The Makefile that can be used to build all of the container images using
"make build", and can also be used to push them to a remote repository
(for use in CI).

Signed-off-by: Owen Hilyard <ohilyard at iol.unh.edu>
---
 containers/Makefile | 166 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)
 create mode 100644 containers/Makefile

diff --git a/containers/Makefile b/containers/Makefile
new file mode 100644
index 0000000..41209ab
--- /dev/null
+++ b/containers/Makefile
@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022 University of New Hampshire
+
+# Recommended Resources:
+#	All:
+# 		~100 GB of disk space: These are not minimal containers, they 
+# 							   have full distros in them minus the kernel.
+#							   With ABI images, expect 10 GB per distro per 
+#							   platform. Without, it's closer to 2 GB per distro
+#							   per platform. 
+#
+#	No ABI:
+#		Serial Build (make -j 1): Should run on any post 2010 system reasonably well.
+# 		Parallel build (make -j $(nproc)): 1 GB of free ram per job should be good enough.
+#	ABI:
+#		Time: If you are building more than four distros, or you have one distro that is built under
+#			  emulation (x86 host + arm container or the inverse), it may be best to run it overnight.
+#			  Some versions of qemu + podman will try to compile with a single thread under emulation,
+#			  which can cause what should be an hour-long compile to turn into a 8+ hour compile.
+#		Serial Build (make -j 1): 6 GB of free ram should be fine
+#   	Parallel build (make -j $(nproc)): 6 GB of free ram per job. You may be compiling ~20 versions of DPDK at the same time, with some under emulation.
+#
+
+
+############ Arguments #############
+# Set to 'Y' to override distro detection
+DPDK_CI_CONTAINERS_ON_RHEL?=$(shell (test -f /etc/redhat-release && grep -q 'Red Hat Enterprise Linux' /etc/redhat-release && echo 'Y') || echo 'N')
+
+# If set to 'Y' and any container is detected as unbuildable, fail the build. 
+DPDK_CI_CONTAINERS_FAIL_ON_UNBUILDABLE?=N
+
+# If set to 'Y', build the ABIs for specificed DPDK versions and embed them
+# in the container
+DPDK_CI_CONTAINERS_BUILD_ABI?=N
+
+# If set to 'Y', build libabigail from source for distros that do not have it.
+# If DPDK_CI_CONTAINERS_BUILD_ABI='Y', then this is always enabled.
+ifeq ($(DPDK_CI_CONTAINERS_BUILD_ABI), Y)
+DPDK_CI_CONTAINERS_BUILD_LIBABIGAIL=Y
+else
+DPDK_CI_CONTAINERS_BUILD_LIBABIGAIL?=N
+endif
+
+# Used to set the python interpreter
+DPDK_CI_CONTAINERS_PYTHON3_CMD?=python3.8
+
+# The base program to use to build individual containers.
+DPDK_CI_CONTAINER_BUILDER_PROGRAM?=podman
+
+# The url of the git repository to pull libabigail from
+DPDK_CI_CONTAINERS_LIBABIGAIL_CLONE_URL?=git://sourceware.org/git/libabigail.git
+
+# The url of the git repository to pull DPDK from
+DPDK_CI_CONTAINERS_DPDK_CLONE_URL?=https://dpdk.org/git/dpdk
+
+# The tag to apply to the built container builder image
+DPDK_CI_CONTAINERS_CONTAINER_BUILDER_TAG?=dpdk_ci_container_builder
+
+# Extra arguments to add to the push command, can be used for credentials
+DPDK_CI_CONTAINERS_EXTRA_PUSH_ARGS?=
+
+DPDK_CI_CONTAINERS_REGISTRY_HOSTNAME?=localhost
+
+# The path to a directory to be recursively copied to $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY). Not used if unset.
+# DPDK_CI_CONTAINERS_EXTRA_SCRIPTS_PATH
+
+DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY?=$(CURDIR)/container_context
+############ End Arguments #############
+
+############ Internal Variables #############
+SCRIPT_ARGS=--output-dir "$(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)"
+
+ifeq ($(DPDK_CI_CONTAINERS_ON_RHEL), Y)
+	SCRIPT_ARGS +=--rhel
+endif
+
+ifeq ($(DPDK_CI_CONTAINERS_FAIL_ON_UNBUILDABLE), Y)
+	SCRIPT_ARGS +=--fail-on-unbuildable
+endif
+
+ifeq ($(DPDK_CI_CONTAINERS_BUILD_ABI), Y)
+	SCRIPT_ARGS +=--build-abi
+endif
+
+ifeq ($(DPDK_CI_CONTAINERS_BUILD_LIBABIGAIL), Y)
+	SCRIPT_ARGS +=--build-libabigail
+endif
+
+SCRIPT_DIRECTORY=$(CURDIR)/template_engine
+
+LIBABIGAIL_DIRECTORY=$(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)/libabigail
+DPDK_DIRECTORY=$(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)/dpdk
+
+DOCKERFILE=$(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)/Dockerfile
+
+MAKE_DOCKERFILE_SCRIPT=$(SCRIPT_DIRECTORY)/make_dockerfile.py
+MAKE_DOCKERFILE_COMMAND=$(DPDK_CI_CONTAINERS_PYTHON3_CMD) $(MAKE_DOCKERFILE_SCRIPT)
+
+TEMPLATE_FILE_DIRECTORY=$(SCRIPT_DIRECTORY)/templates
+DOCKER_CONTAINER_TEMPLATE_DIR=$(TEMPLATE_FILE_DIRECTORY)/containers
+
+INVENTORY_FILE=$(SCRIPT_DIRECTORY)/inventory.yaml
+
+EXTRA_SCRIPTS_DIRECTORY=$(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)/scripts
+
+GIT_FETCH_ARGS=--all --tags
+
+CONTAINER_BUILDER_DOCKERFILE=$(CURDIR)/container_builder.dockerfile
+
+export
+############ End Internal Variables #############
+
+.PHONY: build push make_docker_files_in_container build_builder_container docker_deps extra_scripts libabigail dpdk clean_container_files clean
+
+build: make_docker_files_in_container extra_scripts
+	$(MAKE) -C $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY) build_all
+
+push: make_docker_files_in_container extra_scripts
+	$(MAKE) -C $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY) push_all
+
+make_docker_files_in_container: build_builder_container $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)
+	$(DPDK_CI_CONTAINER_BUILDER_PROGRAM) run -v $(CURDIR):/container_workspace -v $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY):/container_workspace/container_context \
+		-e DPDK_CI_CONTAINERS_ON_RHEL=$(DPDK_CI_CONTAINERS_ON_RHEL) \
+		-e DPDK_CI_CONTAINERS_BUILD_LIBABIGAIL=$(DPDK_CI_CONTAINERS_BUILD_LIBABIGAIL) \
+		-e DPDK_CI_CONTAINERS_BUILD_ABI=$(DPDK_CI_CONTAINERS_BUILD_ABI) \
+		-e DPDK_CI_CONTAINERS_FAIL_ON_UNBUILDABLE=$(DPDK_CI_CONTAINERS_FAIL_ON_UNBUILDABLE) \
+		-e DPDK_CI_CONTAINERS_REGISTRY_HOSTNAME='$(DPDK_CI_CONTAINERS_REGISTRY_HOSTNAME)' \
+		$(DPDK_CI_CONTAINERS_CONTAINER_BUILDER_TAG) make docker_deps
+
+build_builder_container:
+	$(DPDK_CI_CONTAINER_BUILDER_PROGRAM) build -f $(CONTAINER_BUILDER_DOCKERFILE) -t $(DPDK_CI_CONTAINERS_CONTAINER_BUILDER_TAG) $(CURDIR)
+
+docker_deps: $(DOCKERFILE) deps extra_scripts
+	chmod 666 $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)/*.dockerfile
+
+$(DOCKERFILE): $(INVENTORY_FILE) $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY) deps $(MAKE_DOCKERFILE_SCRIPT) $(shell find $(DOCKER_CONTAINER_TEMPLATE_DIR) -type f)
+	cd $(SCRIPT_DIRECTORY) && $(MAKE_DOCKERFILE_COMMAND) $(SCRIPT_ARGS)
+
+deps: $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY) dpdk libabigail
+
+extra_scripts: $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY) $(EXTRA_SCRIPTS_DIRECTORY) $(DPDK_CI_CONTAINERS_EXTRA_SCRIPTS_PATH)
+ifdef $(DPDK_CI_CONTAINERS_EXTRA_SCRIPTS_PATH)
+	cp -pur $(DPDK_CI_CONTAINERS_EXTRA_SCRIPTS_PATH) $(EXTRA_SCRIPTS_DIRECTORY)
+endif
+	mkdir -p $(EXTRA_SCRIPTS_DIRECTORY)
+
+# Clone libabigail source
+libabigail: $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)
+	git -C $(LIBABIGAIL_DIRECTORY) fetch $(GIT_FETCH_ARGS) || git clone $(DPDK_CI_CONTAINERS_LIBABIGAIL_CLONE_URL) $(LIBABIGAIL_DIRECTORY)
+	git -C $(LIBABIGAIL_DIRECTORY) describe --tags `git -C $(LIBABIGAIL_DIRECTORY) rev-list --tags --max-count=1` | xargs -n 1 git -C $(LIBABIGAIL_DIRECTORY) checkout
+
+# Clone DPDK source
+dpdk: $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)
+	git -C $(DPDK_DIRECTORY) fetch $(GIT_FETCH_ARGS) || git clone $(DPDK_CI_CONTAINERS_DPDK_CLONE_URL) $(DPDK_DIRECTORY)
+
+$(EXTRA_SCRIPTS_DIRECTORY):
+	mkdir -p $(EXTRA_SCRIPTS_DIRECTORY)
+
+$(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY):
+	mkdir -p $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)
+
+clean_container_files:
+	rm $(DOCKERFILE)
+
+clean:
+	rm -rf $(DPDK_CI_CONTAINERS_CONTEXT_DIRECTORY)
-- 
2.34.1