diff --git a/.oebuild/features/k3s.yaml b/.oebuild/features/k3s.yaml index b91f8a5250bbda0fcd4288c2efc980348c7cb651..abb55a878e67bda3bf385658e00ba955c7ebc958 100644 --- a/.oebuild/features/k3s.yaml +++ b/.oebuild/features/k3s.yaml @@ -3,4 +3,4 @@ type: feature support: qemu-aarch64 local_conf: | - DISTRO_FEATURES:append = " k3s isulad " + DISTRO_FEATURES:append = "k3s-node " diff --git a/meta-openeuler/recipes-containers/k3s/README.md b/meta-openeuler/recipes-containers/k3s/README.md index 8212e8de19705a15d5ef128a1fe0f264e202d6a4..a0ff9f637f4d378c2341e8fcf31810936f15ea8f 100644 --- a/meta-openeuler/recipes-containers/k3s/README.md +++ b/meta-openeuler/recipes-containers/k3s/README.md @@ -1,89 +1,18 @@ -# issues - -## isulad 问题 - -启动isulad.service 后， 会发现isula找不到 engine 相关文件; - -## 版本问题 - -考虑到大文件托管不便，现在暂时移除 airgap images; 相关的脚本、设置也都已经作为yocto文件了，暂时将上游设置为k3s-io official -TODO src-openeuler 上游版本和 k3s-io有小出入，他们提供的是1.24+ 的 k3s, 当时QEMU试验部署没有很好工作，所以暂时用1.22.17 - - -## package - -目前的packagegroup-k3s其实是一个单包, 是用变量来控制 full k3s(server) 或者 agent, 这样不好，下一次PR时改成packagegroup的形式； - -## do patch - -k3s check config 脚本在oee镜像中是无用的；因为 oee 镜像没有 kernel cfg 列表 -待解决 - -## do compile - -### cni - -目前使用 oee `yocto-meta-openeuler/recipe-container/cni-plugin` 的cni -TODO: 移除 cni-plugin 配方的依赖，把 cni 也作为单独进程压缩到 multicall binary 中 - -### seccomp && selinux - -还未整合起来, 其中 seccomp 是安全上必需的 - -### containerd, runc, containerd shim - -目前 oee k3s 配方中提供了 containerd, isulad 两种 runtime endpoint 的部分支持，还有一些设置需要在runtime进行，已经整合到脚本/systemd service文件中； - -构建containerd以及相关组件的构建暂时从k3s配方中移除，分析后认为应该单独作为依赖配方构建;即暂时不可以使用 k3s containerd作为 runtime endpoint - -### go vendor & go modules mix compilation - - # ? 如果要构建非 isulad 作runtime的k3s,把 vendor和module混用起来比较好 - # 1. 不希望在do_compile时重复拉取依赖。 - # 2. 其他组件多数是vendor构建的，不保证编译结果差异 - -已经在本地尝试中，如果最终分析结果为没必要，那就不会提交; - -### multicall binary - -#### intro - -k3s实现 single binary , bunches of processes 的方式是压缩-解压, 在调用上，类似于busybox,将各组件做符号链接指向containerd, 识别command line argument来确定要启用的服务，并进行解压。 - -multicall k3s binary 在WORKDIR下一直编译不了,导致压缩二进制这一步无法进行，目前都是用无压缩地二进制测试 - -#### undefined data.Asset - # ? FIXME: - # cmd/k3s/main.go:181:16: undefined: data.AssetNames - # cmd/k3s/main.go:218:23: undefined: data.Asset - # under ${S}, pkg/data/data.go is empty indeed - # 位于${S}下编译会出现上面的错误 - # 源码是从 FROM=yocto-meta-openeuler/../k3s 获取,复制到${S} - # 在FROM用完全相同地编译指令，编译，则没有问题, go env 除了pwd地差别，其他完全没有差异； - # 使用sysroot-native的go, host的go,都一样 - # 从${FROM}前往${S}, 失败，从${S}返回到${FROM}, 成功 - # 将${S}复制到外部，依然失败,${FROM}复制到相同区域，依然成功 - # tree 差别为 ${S} 总是多一个 pkg/deploy/zz_generated... 文件，但将此文件mv到FROM, - # S依然失败，FROM依然成功；build 前总是 go clean -cache - # 编译指令如下： - # `CGO_ENABLED=0 ${GO} build -tags "urfave_cli_no_docs" -ldflags "${K3S_LDFLAGS} ${STATIC}"` - # -v -o "${PN}-${ARCH}" ./cmd/k3s/main.go - # 手动加打印也确认了用的是pkg/data/data.go - -解决中 - -### airgap images - -#### Why airgap images? - -airgap images使得agent node在仅联通 server node时即可加入k3s集群; -配方在调试时总是通过k3s-io官方prebuild的airgap image。 -**如果** 在配方中实现了 airgap images 的构建，那么会方便许多,而且不依赖外部托管数百MB的airgap images了 - - -airgap images 将k3s运行依赖地容器镜像组件通过docker build 打包，目前只能在host上打包，本地尝试中, 如果airgap并不比较，则不提交 -初步分析 没有发现比较容易地在无容器引擎环境中build 容器镜像; oebuild 容器环境内部目前也没有提供 containerd/docker/podman等。 - +# k3s oee yocto 说明 + +* 去除k3s标准构建中的静态资源嵌入,在系统运行时解压缩,比较gz,xz,lz4,zstd后选择zstd level16来压 +* 目前只能用符号链接进行multicall,接下来会同时支持 `k3s EXEC` 和 `k3s-EXEC` +* 在 k3s 配方和在`conf/local.conf` 中通过 `APPEND +=` 加入kernel command line arguments 没有生效， 于是我们放在 `k3s.cfg` 中 +* airgap镜像在oebuild无容器的环境中只能: + - LFS + - 手动下载 + 在提交本PR时, oee对LFS的支持已经在进行中，所以我们先移除掉了手动下载容器镜像layers blob的方式 +* 使用 k3s-io 上游风格的用环境变量来控制的部署脚本 +* TODO: 下一次提交根据 `meta-virtualization` 的 k3s 配方，以 `bbappend` 形式重写我们的更改 +* TODO: 用PACKAGES来控制 full k3s 和 k3s agent 的构建以及相关变量的override, 目前都是用`bb.utils.contain`判断，略显笨拙 +* TODO: 得到一个 k3s 专用镜像，考虑到标准镜像中dhcp等cni工具会被其他服务使用，我们把cni-plugin的产物都安装了，在k3s 专门镜像中，我们可以在使用 openEuler cni-plguin 时，裁掉比较大的几个无关组件 +* TODO: 支持 busybox init +* **FIXME**：启动`isulad.service`时提示`failed to list engines path`， 但是qemu aarch64上agent又可以和server打通并部署服务 # k3s: Lightweight Kubernetes diff --git a/meta-openeuler/recipes-containers/k3s/k3s.inc b/meta-openeuler/recipes-containers/k3s/k3s.inc new file mode 100644 index 0000000000000000000000000000000000000000..df25735655244ca9a604830771b6eed579279b98 --- /dev/null +++ b/meta-openeuler/recipes-containers/k3s/k3s.inc @@ -0,0 +1,491 @@ +# reference: https://git.yoctoproject.org/meta-virtualization/tree/recipes-containers/k3s/k3s?h=kirkstone +SUMMARY = "Production-Grade Container Scheduling and Management" +DESCRIPTION = "Lightweight Kubernetes, intended to be a fully compliant Kubernetes." +HOMEPAGE = "https://k3s.io/" +LICENSE = "Apache-2.0" +S = "${WORKDIR}/${BP}" +LIC_FILES_CHKSUM = "file://${S}/LICENSE;md5=2ee41112a44fe7014dce33e26468ba93" + +# TODO: after synchronized with src-openeuler k3s-1.24, we will parse the commit value +# from yocto-meta-openeuler/.oebuild/manifest.yaml +COMMIT = "3ed243df453edc27d20b8d5efa74eb0cbba4548b" +TAGREV = "v1.22.17+k3s1" +PV = "1.22.17-k3s1" + +python () { + archs = { + "aarch64" : "arm64", + "x86_64" : "amd64", + } + tune_arch = d.getVar('TUNE_ARCH') + d.setVar('ARCH', archs.get(tune_arch, tune_arch)) +} + +# Options +# MiB and MB are all considered as 1024B +MAX_K3S_BINARY_SIZE ?= "150MB" +install_commands = "k3s-agent crictl k3s-completion" +full_k3s ?= "false" +static_agent ?= "true" +enable_seccomp ?= "false" +container_runtime_endpoint ?= "isulad" +# if install_airgaped, the k3s rpm contains image package neeeded by kubelet +# the airgap image package will be located in ${asset_dir} +install_airgaped ?= "true" +downloaded_airgap ?= "false" +# TODO: we will provide a simple examination for BSP information to decide whether +# upx-compression is needed (higher RAM requirement, a little bit slower performance) +turnon_upx ?= "false" + +# full k3s: +# TODO: use override syntax to rewrite and split into k3s-server and k3s-agent +FULL_COMMANDS = "\ + crictl kubectl \ + k3s-completion \ +" +install_commands ?= "${FULL_COMMANDS}" + +REQUIRED_DISTRO_FEATURES = "seccomp" + + +SRC_URI = "\ + file://oee_archive/${BPN}/${BP}.tar.gz \ + file://k3s-agent.service \ + file://k3s.service \ + file://k3s-rootless.service \ + file://k3s-clean \ + file://k3s-killall.sh \ + file://k3s-install-agent \ + file://cni-demo-net.conf \ +" + +# file://0001-add-warning-for-rootless-k3s-on-a-hybrid-cgroup-host.patch +SRC_URI += "\ + file://0001-add-missing-required-item-to-check-in-check-config.patch \ + file://0001-modify-cli-commands-for-main-functions.patch \ + ${@bb.utils.contains("downloaded_airgap", 'true', 'file://k3s-airgap-images-${ARCH}.tar.gz;unpack=0', '', d)} \ + file://download-frozen-image-v2.sh \ +" + +DEPENDS += "\ + rsync-native \ + coreutils-native \ + go-native \ + ${@bb.utils.contains('full_k3s', 'true', 'zlib', '', d)} \ + ${@bb.utils.contains('enable_seccomp', 'true', 'libseccomp', '', d)} \ +" + +FULL_K3S_FILES = "" + +# ${unitfile_install_dir}/k3s.service +# ${asset_dir}/${executables_pkg} + +FILES:${PN}-agent = "\ + ${k3s_bindir}/k3s-install-agent \ + "${@bb.utils.contains('install_airgaped', 'true', '${airgap_path}', '', d)}" \ + ${unitfile_install_dir}/k3s-agent.service \ + ${cni_confdir}/10-flannel.conflist \ +" + +FILES:${PN} += "\ + ${k3s_bindir}/${corebin} \ + ${k3s_bindir}/k3s-clean \ + ${k3s_bindir}/k3s-killall.sh \ + "${@bb.utils.contains('install_airgaped', 'true', '${airgap_path}', '', d)}" \ + ${cni_confdir}/10-flannel.conflist \ +" + + +PACKAGES =+ "${PN}-server ${PN}-agent" + +SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}-server ${PN}-agent','',d)}" +SYSTEMD_SERVICE:${PN}-server = "${@bb.utils.contains('DISTRO_FEATURES','systemd','k3s.service','',d)}" +SYSTEMD_SERVICE:${PN}-agent = "${@bb.utils.contains('DISTRO_FEATURES','systemd','k3s-agent.service','',d)}" + +SYSTEMD_AUTO_ENABLE:${PN} = "disable" +SYSTEMD_AUTO_ENABLE:${PN}-agent = "disable" + +USE_PREBUILD_SHIM_V2 = "${@bb.utils.contains_any('ARCH', 'arm riscv64', '0', '1', d)}" + +RDEPENDS:${PN}-server = "${PN}" +RDEPENDS:${PN}-agent = "${PN}" + +RDEPENDS:${PN} = "\ + zstd \ + cni \ + conntrack-tools \ + coreutils \ + iproute2 \ + iptables \ + ${@bb.utils.contains('USE_PREBUILD_SHIM_V2', '1', 'lib-shim-v2-bin', 'lib-shim-v2', d)} \ +" + +# moved into feature/k3s/k3s.cfg: +RDEPENDS:${PN} += "\ + kernel-module-br-netfilter \ + kernel-module-bridge \ + kernel-module-veth \ + kernel-module-ipt-reject \ + kernel-module-iptable-mangle \ + kernel-module-iptable-filter \ + kernel-module-ip6table-mangle \ + kernel-module-ip6-tables \ + kernel-module-iptable-nat \ + kernel-module-nf-conntrack \ + kernel-module-nf-nat \ + kernel-module-xt-addrtype \ + kernel-module-xt-comment \ + kernel-module-xt-nat \ + kernel-module-xt-conntrack \ + kernel-module-xt-comment \ + kernel-module-xt-connmark \ + kernel-module-xt-ipvs \ + kernel-module-xt-multiport \ + kernel-module-xt-statistic \ + kernel-module-xt-masquerade \ + kernel-module-vxlan \ + kernel-module-ip-set \ + kernel-module-ip-vs \ + kernel-module-ip-vs-rr \ + kernel-module-ip-vs-sh \ + kernel-module-ip-vs-wrr \ +" + +INHIBIT_PACKAGE_STRIP = "1" +INSANE_SKIP:${PN} += "ldflags already-stripped" + + +# buildtime path definition +build_bindir = "${S}/build/bin" +vendor_path = "${S}/vendor" +modules_path = "${S}/modules" +corebin = "k3s" + +# runtime path definition +k3s_bindir = "${bindir}" +asset_dir = "${sysconfdir}/rancher/k3s/assets" +default_confdir = "${sysconfdir}/rancher/k3s/config" +executables_pkg = "assets.tar.zst" + +# criconfig is located in data_dir +# NOTICE: if `--selinux` flag is pass to k3s command line, `--data-dir` is unsupported. +# TODO: support changing the default data_dir hardcoded in the source file +# in case `--selinux` and `--data-dir` are present together +data_dir = "${localstatedir}/lib/rancher/k3s" +cni_bindir = "${libexecdir}/cni" +# cni_confdir = "${data_dir}/agent/etc/cni/net.d" +cni_confdir = "${sysconfdir}/cni/net.d" + +inherit go +inherit goarch +inherit systemd +inherit features_check +REQUIRED_DISTRO_FEATURES ?= "seccomp" + +PACKAGECONFIG = "" +PACKAGECONFIG[upx] = ",,upx-native" +GO_IMPORT = "import" + +# trag variables for do_compile +# osusergo is needed and ctrd is removed +TAGS = "no_btrfs netcgo osusergo providerless urfave_cli_no_docs" +SELINUX = "selinux" +APPARMOR = "apparmor" +STATIC_TAG = "static_build" +SQLITE_TAG = "${STATIC_TAG} libsqlite3" + +SERVER_TAGS = "${TAGS} ${SELINUX} ${STATIC_TAG} ${APPARMOR}" +SERVER_TAGS += "\ + ${@bb.utils.contains('enable_seccomp', 'true', 'seccomp', '', d)} \ +" + +AGENT_TAGS = "${TAGS} ${APPARMOR} ${SELINUX}" +AGENT_TAGS += "\ + ${@bb.utils.contains('static_agent', 'true', '${STATIC_TAG}', '', d)} \ + ${@bb.utils.contains('enable_seccomp', 'true', 'seccomp', '', d)} \ +" + +# variables for do_install +CNI_COMPONENTS = "bridge flannel host-local loopback portmap" +# here we use vxlan as the default flannel backend +# oee flannel as cni plugin +VXLANTEP = "flannel.1" +PLUGIN_IF = "${VXLANTEP}" + +# make sure user is allowed to delete readonly dirs +do_chmod() { + if [ -d ${modules_path} ]; then + chmod 777 -R ${modules_path} + fi + + if [ -d "${WORKDIR}/build/pkg/mod" ]; then + chmod 777 -R "${WORKDIR}/build/pkg/mod" + fi +} + +addtask do_chmod + +do_chmod[nostamp] = "1" +do_compile[network] = "1" + +do_compile() { + + #build K3S core binaries + export GOPROXY="https://goproxy.cn,direct" + export CGO_ENABLED="1" + export GOPATH="${modules_path}:${vendor_path}:${STAGING_DIR_TARGET}/${prefix}/lib64/go" + export CGO_CFLAGS="${CFLAGS} --sysroot=${STAGING_DIR_TARGET}" + export CGO_LDFLAGS="${LDFLAGS} --sysroot=${STAGING_DIR_TARGET}" + export CFLAGS="" + export LDFLAGS=" -w -s" + export CC="${CC}" + export LD="${LD}" + export OEE_YOCTO_VERSION="${PV}-oee+yocto" + + mkdir -p "${modules_path}" + mkdir -p "${build_bindir}" + mkdir -p "${WORKDIR}/build/pkg/mod" + + cd ${S} + PKG="github.com/k3s-io/k3s" + PKG_CRICTL="github.com/kubernetes-sigs/cri-tools/pkg" + VERSION_CRICTL=$(grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}') + + CRICTL=$(grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}') + if [ -z "$VERSION_CRICTL" ]; then + VERSION_CRICTL="v0.0.0" + fi + + # necessary for the Default config parser_raw + VERSION_FLAGS="\ + -X ${PKG}/pkg/version.Version=${PV} \ + -X ${PKG}/pkg/version.GitCommit=${COMMIT:0:8} \ + -X ${PKG_CRICTL}/version.Version=${VERSION_CRICTL} \ + " + + # build flags + STATIC=" -extldflags '-static' " + STATIC_SQLITE=" -extldflags '-static -lm -lz -ldl -lpthread' " + K3S_LDFLAGS=" ${VERSION_FLAGS} -w -s" + SERVER_LDFLAGS="${K3S_LDFLAGS} ${STATIC_SQLITE}" + AGENT_LDFLAGS="${K3S_LDFLAGS}" + if [ "${static_agent}" = true ]; then + AGENT_LDFLAGS+=" ${STATIC}" + fi + + # if we use go vendor to build, GO will search for pkgs in both recipes-sysroot and vendor dir. + # However, go vendor do not provider version info in dirs, some conflicts are inevitable + + rm -rf "${build_bindir}/${corebin}" + rm -rf "${build_bindir}/${executables_pkg}" + rm -rf "${builds}/data" + rm -rf "${builds}/out" + + cd ${S} + + # NOTICE: glibc static linking warnings + # * warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking + # * warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking + if [ "${full_k3s}" = true ]; then + GO111MODULE=on ${GO} build -mod=readonly -tags "${SERVER_TAGS}" -ldflags "${SERVER_LDFLAGS}" \ + -trimpath -v -o ${build_bindir}/${corebin} cmd/server/main.go \ + 2> ${WORKDIR}/temp/k3s_build_info + else + GO111MODULE=on ${GO} build -mod=readonly -tags "${AGENT_TAGS}" -ldflags "${AGENT_LDFLAGS}" \ + -trimpath -v -o ${build_bindir}/${corebin} main.go \ + 2> ${WORKDIR}/temp/k3s_build_info + fi + + if [[ "${turnon_upx}" = "true" ]] && command -v upx > /dev/null 2>&1; then + bbwarn "if the bsp RAM is in short, please turn off upx" + upx -9 ${build_bindir}/${corebin} + fi +} + +do_compress() { + + cd ${build_bindir} + if [ ! -f "${corebin}" ]; then + bbfatal "${corebin} is not found is it not built?" + fi + + chmod 755 ${corebin} + + for c in ${install_commands}; do + ln -sfr ${corebin} ${c} + done + + for i in ${CNI_COMPONENTS}; do + ln -sf "${cni_bindir}/${i}" ${i} + done + + tar czf executable_assets.tar $(find . -maxdepth 1 -type l -o -name ${corebin}) + + zstd --rm --no-progress -T0 -8 -f --long=25 executable_assets.tar -o ${executables_pkg} +} + +# addtask compress before do_install after do_compile + +do_install() { + + # create dirs + install -d "${D}${k3s_bindir}" + install -d "${D}${sysconfdir}" + install -d "${D}${asset_dir}" + install -d "${D}${default_confdir}" + install -d "${D}${cni_confdir}" + + # make lns + + install -m 755 "${build_bindir}/${corebin}" "${D}${k3s_bindir}/${corebin}" + for c in ${install_commands}; do + ln -sf "${k3s_bindir}/${corebin}" ${c} + done + + for c in ${CNI_COMPONENTS}; do + ln -sf "${cni_bindir}/${c}" ${c} + done + + # replaced some placeholders + sed -i 's#@DEFAULT_BIN_DIR@#${k3s_bindir}#g' "${WORKDIR}/k3s-install-agent" + sed -i 's#@DEFAULT_DATA_DIR@#${data_dir}#g' \ + "${WORKDIR}/k3s-install-agent" "${WORKDIR}/k3s-clean" + sed -i 's#@ASSET_DIR@#${asset_dir}#g' "${WORKDIR}/k3s-install-agent" + sed -i 's#@tar.zst@#${executables_pkg}#g' "${WORKDIR}/k3s-install-agent" + sed -i 's/@INSTALL_COMMANDS@/${INSTALL_COMMANDS}/g' "${WORKDIR}/k3s-install-agent" + sed -i 's#@IMAGE_LIST@#${image_list}#g' "${WORKDIR}/k3s-install-agent" + sed -i 's#@CNI_LOCALSTATE_DIR@#/var/lib/cni#g' \ + "${WORKDIR}/k3s-install-agent" "${WORKDIR}/k3s-clean" + # setup interface communicating with cni0 + sed -i 's/@CNI_PLUGIN_IF@/${PLUGIN_IF}/g' \ + "${WORKDIR}/k3s-install-agent" "${WORKDIR}/k3s-killall.sh" + sed -i 's#@rootless@#${asset_dir}/k3s-rootless.service#g' "${WORKDIR}/k3s-install-agent" + + install -m 755 "${S}/contrib/util/check-config.sh" "${D}${k3s_bindir}/check-config" + + # TODO modify the killer + install -m 755 "${WORKDIR}/k3s-clean" "${D}${k3s_bindir}" + install -m 755 "${WORKDIR}/k3s-killall.sh" "${D}${k3s_bindir}/k3s-killall" + + #install -m 644 "${build_bindir}/${executables_pkg}" "${D}${asset_dir}" + + # install script for deploying a k3s agent + install -m 644 "${WORKDIR}/cni-demo-net.conf" "${D}${cni_confdir}/10-flannel.conflist" + + install -D -m 0644 "${S}/k3s-rootless.service" "${D}${asset_dir}/k3s-rootless.service" + install_airgap_images + install -D -m 755 "${WORKDIR}/k3s-install-agent" "${D}${k3s_bindir}/k3s-install-agent" + # install systemd services unit files + unitfile_install_dir="${D}${sysconfdir}/systemd/system" + install -d "${unitfile_install_dir}" + # if [ "${full_k3s}" = "true" ]; then + install -D -m 0644 "${WORKDIR}/k3s.service" "${unitfile_install_dir}/k3s.service" + sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${k3s_bindir}/\4#g" "${unitfile_install_dir}/k3s.service" + # fi + + install -D -m 0644 "${WORKDIR}/k3s-rootless.service" "${D}${asset_dir}" + sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${k3s_bindir}/\4#g" "${D}${asset_dir}/k3s-rootless.service" + + install -D -m 0644 "${WORKDIR}/k3s-agent.service" "${unitfile_install_dir}/k3s-agent.service" + sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${k3s_bindir}/\4#g" "${unitfile_install_dir}/k3s-agent.service" + + # TODO: handle the traefik +} + + +airgap_target_dir = "${@bb.utils.contains('container_runtime_endpoint', \ + 'isulad', \ + '${asset_dir}', \ + '${localstatedir}/lib/rancher/k3s/agent/images', d)}\ +" +airgap_install_dir = "${D}${airgap_target_dir}" +airgap_path = "${airgap_target_dir}/k3s-airgap-images-${ARCH}.tar.gz" + +install_airgap_images() { + if [ "${install_airgaped}" = "true" ]; then + install -d "${airgap_install_dir}" + if [ "${downloaded_airgap}" != "true" ]; then + # makeairgap + bbwarn "\ + required images should be stored in gitee LFS(as airgap images) or pulled by isula. + If you do wanna load downloaded airgap images during yocto building process, which increase your + Openeuler Embedded image size greatly, you should fetch k3s/k3s-airgap-images-${ARCH}.tar.gz + in src/oee_archive first. + " + fi + if [ -f "${WORKDIR}/k3s-airgap-images-${ARCH}.tar.gz"]; then + install -D -m 644 "${WORKDIR}/k3s-airgap-images-${ARCH}.tar.gz" "${D}${airgap_path}" + fi + fi +} + +# DEPRECATED: +# * Images should be stored in gitee LFS(as airgap images) or pulled by isula. +# * If you do wanna build airgap images during yocto building process, which increase your +# Openeuler Embedded image size greatly, enable: +# install_airgaped = true and download_airgaped = false in k3s_%.bbappend +# INPROGRESS: +# * Bundle all images as one tar.gz. Make sure isula load -i k3s-airgap-images-${ARCH}.tar.gz works. +# * The docker registry is unreachable in most cases. We need to solve this problem. +makeairgap() { + if [ "${downloaded_airgap}" = "true" ]; then + bbplain "airgap image package already downloaded" + fi + mkdir -p ${WORKDIR}/build/airgap + cd ${WORKDIR}/build/airgap + + for fullname in ${image_list} + do + fetchname="${fullname#docker.io}" + _dir="${fetchname##*/}" + downloaddir=${_dir%:*} + rm -rf ${downloaddir} + mkdir -p ${downloaddir} + TARGETARCH=${ARCH} ${WORKDIR}/download-frozen-image-v2.sh ${downloaddir} ${fetchname} + tar -cf "${downloaddir}.tar" -C "${downloaddir}" . + rm -rf ${downloaddir} + done + + tar -cf "k3s-airgap-images-${ARCH}.tar.gz" ./*.tar + mv "k3s-airgap-images-${ARCH}.tar.gz" ${WORKDIR} +} + +def get_bytecount(d): + size_str = d.getVar('MAX_K3S_BINARY_SIZE') + if len(size_str) == 0 or not size_str[0].isdigit(): + return 0 + numeric_end = 0 + unit = None + for i, c in enumerate(size_str): + numeric_end = i + if not c.isdigit(): + unit = c.lower() + break + if unit is None or unit == 'b': + return int(size_str) + num_part = int(size_str[:numeric_end]) + # only consider k or m + factor = 1024 if unit == 'k' else 1024 * 1024 + return num_part * factor + + +do_sizecheck() { + bbplain "checking k3s binary size, threshold is ${MAX_K3S_BINARY_SIZE} defined" + SAMPLE="${build_bindir}/${corebin}" + if [ ! -f ${SAMPLE} ]; then + return + fi + + BIN_SIZE=$(stat -c '%s' ${SAMPLE}) + THRESHOLD=${@get_bytecount(d)} + if [ ${BIN_SIZE} -gt ${THRESHOLD} ]; then + bbplain "k3s binary ${SAMPLE} size ${BIN_SIZE} > the max acceptable size of ${THRESHOLD} bytes" + else + bbplain "k3s binary ${SAMPLE} size ${BIN_SIZE} < the max acceptable size of ${THRESHOLD} bytes" + fi +} + +addtask sizecheck after do_compile before do_install + +INHIBIT_PACKAGE_STRIP = "1" +INSANE_SKIP:${PN} += "ldflags already-stripped" \ No newline at end of file diff --git a/meta-openeuler/recipes-containers/k3s/k3s/0001-Finding-host-local-in-usr-libexec.patch b/meta-openeuler/recipes-containers/k3s/k3s/0001-Finding-host-local-in-usr-libexec.patch deleted file mode 100644 index 524ac2cb73f0140ad69f69bbc8aa6b931f960d59..0000000000000000000000000000000000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s/0001-Finding-host-local-in-usr-libexec.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 4faf68d68c97cfd10947e1152f711acc59f39647 Mon Sep 17 00:00:00 2001 -From: Erik Jansson -Date: Wed, 16 Oct 2019 15:07:48 +0200 -Subject: [PATCH] Finding host-local in /usr/libexec - -Upstream-status: Inappropriate [embedded specific] -Signed-off-by: ---- - pkg/agent/config/config.go | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -Index: import/pkg/agent/config/config.go -=================================================================== ---- import.orig/pkg/agent/config/config.go -+++ import/pkg/agent/config/config.go -@@ -445,7 +445,7 @@ - } - - if !nodeConfig.NoFlannel { -- hostLocal, err := exec.LookPath("host-local") -+ hostLocal, err := exec.LookPath("/usr/libexec/cni/host-local") - if err != nil { - return nil, errors.Wrapf(err, "failed to find host-local") - } diff --git a/meta-openeuler/recipes-containers/k3s/k3s/0001-add-missing-required-item-to-check-in-check-config.patch b/meta-openeuler/recipes-containers/k3s/k3s/0001-add-missing-required-item-to-check-in-check-config.patch new file mode 100644 index 0000000000000000000000000000000000000000..fcd2729df2fd87c5d8251c17654f7c354851494c --- /dev/null +++ b/meta-openeuler/recipes-containers/k3s/k3s/0001-add-missing-required-item-to-check-in-check-config.patch @@ -0,0 +1,36 @@ +From 7ed4d4b830c5749bfd9561007f51f98a176aca75 Mon Sep 17 00:00:00 2001 +From: xiaojunzhe +Date: Mon, 3 Mar 2025 15:31:20 +0800 +Subject: [PATCH] add missing required item to check in check-config + +--- + contrib/util/check-config.sh | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/contrib/util/check-config.sh b/contrib/util/check-config.sh +index 3f139fe28b..d9fb8186d9 100755 +--- a/contrib/util/check-config.sh ++++ b/contrib/util/check-config.sh +@@ -227,7 +227,7 @@ echo + wrap_warn "- $iptablesCmd" "unknown version: $iptablesInfo" + elif version_ge $iptablesVersion v1.8.0; then + iptablesMode=$(echo $iptablesInfo | awk '{ print $3 }') +- if [ "$iptablesMode" != "(legacy)" ] && version_less $iptablesVersion v1.8.4; then ++ if [ "$iptablesMode" != "(legacy)" ] && version_less $iptablesVersion v1.8.4; then + wrap_bad "- $label" 'should be older than v1.8.0, newer than v1.8.3, or in legacy mode' + else + wrap_good "- $label" 'ok' +@@ -375,8 +375,9 @@ flags=" + CGROUPS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG + KEYS + VETH BRIDGE BRIDGE_NETFILTER +- IP_NF_FILTER IP_NF_TARGET_MASQUERADE ++ IP_NF_FILTER IP_NF_TARGET_MASQUERADE IP_NF_TARGET_REJECT + NETFILTER_XT_MATCH_ADDRTYPE NETFILTER_XT_MATCH_CONNTRACK NETFILTER_XT_MATCH_IPVS ++ NETFILTER_XT_MATCH_COMMENT NETFILTER_XT_MATCH_MULTIPORT NETFILTER_XT_MATCH_STATISTIC + IP_NF_NAT NF_NAT + POSIX_MQUEUE + " +-- +2.43.0 + diff --git a/meta-openeuler/recipes-containers/k3s/k3s/0001-add-warning-for-rootless-k3s-on-a-hybrid-cgroup-host.patch b/meta-openeuler/recipes-containers/k3s/k3s/0001-add-warning-for-rootless-k3s-on-a-hybrid-cgroup-host.patch new file mode 100644 index 0000000000000000000000000000000000000000..5b424f62d6ace050500ac9009fbc54d2c6bf7ccd --- /dev/null +++ b/meta-openeuler/recipes-containers/k3s/k3s/0001-add-warning-for-rootless-k3s-on-a-hybrid-cgroup-host.patch @@ -0,0 +1,44 @@ +From 4774375f03eecd3f382caec6a6ad8298bdfe8d73 Mon Sep 17 00:00:00 2001 +From: xiaojunzhe +Date: Mon, 10 Mar 2025 20:34:44 +0800 +Subject: [PATCH] add warning for rootless k3s on a hybrid cgroup host + +--- + pkg/cgroups/cgroups_linux.go | 4 ++++ + pkg/daemons/agent/agent_linux.go | 4 +++- + 2 files changed, 7 insertions(+), 1 deletion(-) + +diff --git a/pkg/cgroups/cgroups_linux.go b/pkg/cgroups/cgroups_linux.go +index 2947c00060..fb82d0472a 100644 +--- a/pkg/cgroups/cgroups_linux.go ++++ b/pkg/cgroups/cgroups_linux.go +@@ -65,6 +65,10 @@ func validateCgroupsV2() error { + return nil + } + ++func HybridCgroups() bool { ++ return cgroups.Mode() == cgroups.Hybrid ++} ++ + func CheckCgroups() (kubeletRoot, runtimeRoot string, controllers map[string]bool) { + cgroupsModeV2 := cgroups.Mode() == cgroups.Unified + controllers = make(map[string]bool) +diff --git a/pkg/daemons/agent/agent_linux.go b/pkg/daemons/agent/agent_linux.go +index e06e2b8a91..9e2e994ff7 100644 +--- a/pkg/daemons/agent/agent_linux.go ++++ b/pkg/daemons/agent/agent_linux.go +@@ -23,8 +23,10 @@ func createRootlessConfig(argsMap map[string]string, controllers map[string]bool + // "/sys/fs/cgroup" is namespaced + cgroupfsWritable := unix.Access("/sys/fs/cgroup", unix.W_OK) == nil + if controllers["cpu"] && controllers["pids"] && cgroupfsWritable { ++ if cgroups.HybridCgroups() { ++ logrus.Warn("while the host cgroup mode is Hybrid, openEuler embedded is not well supported.") ++ } + logrus.Info("cgroup v2 controllers are delegated for rootless.") +- return + } + logrus.Fatal("delegated cgroup v2 controllers are required for rootless.") + } +-- +2.43.0 + diff --git a/meta-openeuler/recipes-containers/k3s/k3s/0001-modify-cli-commands-for-main-functions.patch b/meta-openeuler/recipes-containers/k3s/k3s/0001-modify-cli-commands-for-main-functions.patch new file mode 100644 index 0000000000000000000000000000000000000000..1460b9eda1fae2e76de1c24036d2a887d641eebe --- /dev/null +++ b/meta-openeuler/recipes-containers/k3s/k3s/0001-modify-cli-commands-for-main-functions.patch @@ -0,0 +1,152 @@ +From c7b939f6827458d5a98066673c0cbf69a2ee1254 Mon Sep 17 00:00:00 2001 +From: egg12138 +Date: Sat, 15 Mar 2025 18:13:41 +0800 +Subject: [PATCH] modify cli commands for main functions + +--- + cmd/server/main.go | 3 +++ + main.go | 44 ++++++++++++++++---------------------------- + pkg/util/run_cli.go | 28 ++++++++++++++++++++++++++++ + 3 files changed, 47 insertions(+), 28 deletions(-) + create mode 100644 pkg/util/run_cli.go + +diff --git a/cmd/server/main.go b/cmd/server/main.go +index f5e2c3c..39e7b2b 100644 +--- a/cmd/server/main.go ++++ b/cmd/server/main.go +@@ -21,6 +21,7 @@ import ( + "github.com/k3s-io/k3s/pkg/containerd" + ctr2 "github.com/k3s-io/k3s/pkg/ctr" + kubectl2 "github.com/k3s-io/k3s/pkg/kubectl" ++ "github.com/k3s-io/k3s/pkg/util" + crictl2 "github.com/kubernetes-sigs/cri-tools/cmd/crictl" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +@@ -42,6 +43,7 @@ func main() { + os.Args[0] = cmd + + app := cmds.NewApp() ++ app.EnableBashCompletion = true + app.Commands = []cli.Command{ + cmds.NewServerCommand(server.Run), + cmds.NewAgentCommand(agent.Run), +@@ -69,6 +71,7 @@ func main() { + cert.Run), + ), + cmds.NewCompletionCommand(completion.Run), ++ cmds.NewCheckConfigCommand(util.CheckConfig), + } + + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { +diff --git a/main.go b/main.go +index 8fbeeeb..927b58e 100644 +--- a/main.go ++++ b/main.go +@@ -10,52 +10,40 @@ import ( + "context" + "errors" + "os" ++ "path/filepath" + ++ "github.com/docker/docker/pkg/reexec" + "github.com/k3s-io/k3s/pkg/cli/agent" +- "github.com/k3s-io/k3s/pkg/cli/cert" + "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/cli/completion" + "github.com/k3s-io/k3s/pkg/cli/crictl" +- "github.com/k3s-io/k3s/pkg/cli/etcdsnapshot" +- "github.com/k3s-io/k3s/pkg/cli/kubectl" +- "github.com/k3s-io/k3s/pkg/cli/secretsencrypt" +- "github.com/k3s-io/k3s/pkg/cli/server" + "github.com/k3s-io/k3s/pkg/configfilearg" ++ "github.com/k3s-io/k3s/pkg/util" ++ crictl2 "github.com/kubernetes-sigs/cri-tools/cmd/crictl" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" + ) ++func init() { ++ reexec.Register("crictl", crictl2.Main) ++} + + func main() { ++ cmd := os.Args[0] ++ os.Args[0] = filepath.Base(os.Args[0]) ++ if reexec.Init() { ++ return ++ } ++ os.Args[0] = cmd + app := cmds.NewApp() ++ app.EnableBashCompletion = true + app.Commands = []cli.Command{ +- cmds.NewServerCommand(server.Run), + cmds.NewAgentCommand(agent.Run), +- cmds.NewKubectlCommand(kubectl.Run), + cmds.NewCRICTL(crictl.Run), +- cmds.NewEtcdSnapshotCommand(etcdsnapshot.Save, +- cmds.NewEtcdSnapshotSubcommands( +- etcdsnapshot.Delete, +- etcdsnapshot.List, +- etcdsnapshot.Prune, +- etcdsnapshot.Save), +- ), +- cmds.NewSecretsEncryptCommand(cli.ShowAppHelp, +- cmds.NewSecretsEncryptSubcommands( +- secretsencrypt.Status, +- secretsencrypt.Enable, +- secretsencrypt.Disable, +- secretsencrypt.Prepare, +- secretsencrypt.Rotate, +- secretsencrypt.Reencrypt), +- ), +- cmds.NewCertCommand( +- cmds.NewCertSubcommands( +- cert.Run), +- ), + cmds.NewCompletionCommand(completion.Run), ++ cmds.NewCheckConfigCommand(util.CheckConfig), + } + + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { + logrus.Fatal(err) + } +-} ++} +\ No newline at end of file +diff --git a/pkg/util/run_cli.go b/pkg/util/run_cli.go +new file mode 100644 +index 0000000..26ac090 +--- /dev/null ++++ b/pkg/util/run_cli.go +@@ -0,0 +1,28 @@ ++package util ++ ++import ( ++ "os" ++ "os/exec" ++ "syscall" ++ ++ "github.com/sirupsen/logrus" ++ "github.com/urfave/cli" ++) ++ ++func Main() { ++} ++ ++func CheckConfig(cli *cli.Context) error { ++ return simpleRunCLI("check-config", nil) ++} ++ ++func simpleRunCLI(cmd string, args []string) error { ++ cmd, err := exec.LookPath(cmd) ++ if err != nil { ++ return err ++ } ++ ++ logrus.Debugf("Running %s %v", cmd, args) ++ ++ return syscall.Exec(cmd, args, os.Environ()) ++} +\ No newline at end of file +-- +2.43.0 + diff --git a/meta-openeuler/recipes-containers/k3s/k3s/cni-containerd-net.conf b/meta-openeuler/recipes-containers/k3s/k3s/cni-containerd-net.conf deleted file mode 100644 index ca434d6fcdfdb656241bd83c927fb695972f3734..0000000000000000000000000000000000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s/cni-containerd-net.conf +++ /dev/null @@ -1,24 +0,0 @@ -{ - "cniVersion": "0.4.0", - "name": "containerd-net", - "plugins": [ - { - "type": "bridge", - "bridge": "cni0", - "isGateway": true, - "ipMasq": true, - "promiscMode": true, - "ipam": { - "type": "host-local", - "subnet": "10.88.0.0/16", - "routes": [ - { "dst": "0.0.0.0/0" } - ] - } - }, - { - "type": "portmap", - "capabilities": {"portMappings": true} - } - ] -} diff --git a/meta-openeuler/recipes-containers/k3s/k3s/cni-demo-net.conf b/meta-openeuler/recipes-containers/k3s/k3s/cni-demo-net.conf new file mode 100644 index 0000000000000000000000000000000000000000..fc60fd5509db2cca5775cec87ee9d0294618a5ab --- /dev/null +++ b/meta-openeuler/recipes-containers/k3s/k3s/cni-demo-net.conf @@ -0,0 +1,20 @@ +{ + "cniVersion": "1.2.0", + "name": "cbr", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "forceAddress": true, + "isDefaultGateway": true, + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] +} diff --git a/meta-openeuler/recipes-containers/k3s/k3s/download-frozen-image-v2.sh b/meta-openeuler/recipes-containers/k3s/k3s/download-frozen-image-v2.sh new file mode 100755 index 0000000000000000000000000000000000000000..e4abca2f6ac61e93de7e961ffb13c028500e0050 --- /dev/null +++ b/meta-openeuler/recipes-containers/k3s/k3s/download-frozen-image-v2.sh @@ -0,0 +1,448 @@ +#!/usr/bin/env bash +set -eo pipefail + +# This script was developed for use in Moby's CI, and as such the use cases and +# usability are (intentionally) limited. You may find this script useful for +# educational purposes, for example, to learn how pulling images works "under +# the hood", and you may be able to use it for other purposes, but it should not +# be considered a "general purpose" tool for pulling images. +# +# The project maintainers accept contributions to this script within its intended +# scope, but may not accept contributions beyond that. +# +# For users who have a similar need but require more flexibility/functionality, +# refer to the discussion on GitHub, which mentions various alternatives that +# are more suitable for other uses: https://github.com/moby/moby/issues/40857 + +# hello-world latest ef872312fe1b 3 months ago 910 B +# hello-world latest ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9 3 months ago 910 B + +# debian latest f6fab3b798be 10 weeks ago 85.1 MB +# debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB + +# check if essential commands are in our PATH +for cmd in curl jq; do + if ! command -v $cmd &> /dev/null; then + echo >&2 "error: \"$cmd\" not found!" + exit 1 + fi +done + +usage() { + echo "usage: $0 dir image[:tag][@digest] ..." + echo " $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7" + [ -z "$1" ] || exit "$1" +} + +dir="$1" # dir for building tar in +shift || usage 1 >&2 + +if ! [ $# -gt 0 ] && [ "$dir" ]; then + usage 2 >&2 +fi +mkdir -p "$dir" + +# hacky workarounds for Bash 3 support (no associative arrays) +images=() +rm -f "$dir"/tags-*.tmp +manifestJsonEntries=() +doNotGenerateManifestJson= +# repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."' + +# bash v4 on Windows CI requires CRLF separator... and linux doesn't seem to care either way +newlineIFS=$'\n' +major=$(echo "${BASH_VERSION%%[^0.9]}" | cut -d. -f1) +if [ "$major" -ge 4 ]; then + newlineIFS=$'\r\n' +fi + +registryBase='https://registry-1.docker.io' +authBase='https://auth.docker.io' +authService='registry.docker.io' + +# https://github.com/moby/moby/issues/33700 +fetch_blob() { + local token="$1" + shift + local image="$1" + shift + local digest="$1" + shift + local targetFile="$1" + shift + local curlArgs=("$@") + + local curlHeaders + curlHeaders="$( + curl -S "${curlArgs[@]}" \ + -H "Authorization: Bearer $token" \ + "$registryBase/v2/$image/blobs/$digest" \ + -o "$targetFile" \ + -D- + )" + curlHeaders="$(echo "$curlHeaders" | tr -d '\r')" + if grep -qE "^HTTP/[0-9].[0-9] 3" <<< "$curlHeaders"; then + rm -f "$targetFile" + + local blobRedirect + blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')" + if [ -z "$blobRedirect" ]; then + echo >&2 "error: failed fetching '$image' blob '$digest'" + echo "$curlHeaders" | head -1 >&2 + return 1 + fi + + curl -fSL "${curlArgs[@]}" \ + "$blobRedirect" \ + -o "$targetFile" + fi +} + +# handle 'application/vnd.docker.distribution.manifest.v2+json' manifest +handle_single_manifest_v2() { + local manifestJson="$1" + shift + + local configDigest + configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')" + local imageId="${configDigest#*:}" # strip off "sha256:" + + local configFile="$imageId.json" + fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s + + local layersFs + layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')" + local IFS="$newlineIFS" + local layers + mapfile -t layers <<< "$layersFs" + unset IFS + + echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." + local layerId= + local layerFiles=() + for i in "${!layers[@]}"; do + local layerMeta="${layers[$i]}" + + local layerMediaType + layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')" + local layerDigest + layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')" + + # save the previous layer's ID + local parentId="$layerId" + # create a new fake layer ID based on this layer's digest and the previous layer's fake ID + layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)" + # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value) + + mkdir -p "$dir/$layerId" + echo '1.0' > "$dir/$layerId/VERSION" + + if [ ! -s "$dir/$layerId/json" ]; then + local parentJson + parentJson="$(printf ', parent: "%s"' "$parentId")" + local addJson + addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")" + # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers + jq "$addJson + ." > "$dir/$layerId/json" <<- 'EOJSON' + { + "created": "0001-01-01T00:00:00Z", + "container_config": { + "Hostname": "", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": null, + "Image": "", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": null + } + } + EOJSON + fi + + case "$layerMediaType" in + application/vnd.oci.image.layer.v1.tar+gzip | application/vnd.docker.image.rootfs.diff.tar.gzip) + local layerTar="$layerId/layer.tar" + layerFiles=("${layerFiles[@]}" "$layerTar") + # TODO figure out why "-C -" doesn't work here + # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." + # "HTTP/1.1 416 Requested Range Not Satisfiable" + if [ -f "$dir/$layerTar" ]; then + # TODO hackpatch for no -C support :'( + echo "skipping existing ${layerId:0:12}" + continue + fi + local token + token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" + fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress-bar + ;; + + *) + echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'" + exit 1 + ;; + esac + done + + # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons) + imageId="$layerId" + + # munge the top layer image manifest to have the appropriate image configuration for older daemons + local imageOldConfig + imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")" + jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json" + + local manifestJsonEntry + manifestJsonEntry="$( + echo '{}' | jq --raw-output --compact-output '. + { + Config: "'"$configFile"'", + RepoTags: ["'"${image#library\/}:$tag"'"], + Layers: '"$(echo '[]' | jq --raw-output --compact-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"' + }' + )" + manifestJsonEntries=("${manifestJsonEntries[@]}" "$manifestJsonEntry") +} + +get_target_arch() { + if [ -n "${TARGETARCH:-}" ]; then + echo "${TARGETARCH}" + return 0 + fi + + if type go > /dev/null; then + go env GOARCH + return 0 + fi + + if type dpkg > /dev/null; then + debArch="$(dpkg --print-architecture)" + case "${debArch}" in + armel | armhf) + echo "arm" + return 0 + ;; + *64el) + echo "${debArch%el}le" + return 0 + ;; + *) + echo "${debArch}" + return 0 + ;; + esac + fi + + if type uname > /dev/null; then + uArch="$(uname -m)" + case "${uArch}" in + x86_64) + echo amd64 + return 0 + ;; + arm | armv[0-9]*) + echo arm + return 0 + ;; + aarch64) + echo arm64 + return 0 + ;; + mips*) + echo >&2 "I see you are running on mips but I don't know how to determine endianness yet, so I cannot select a correct arch to fetch." + echo >&2 "Consider installing \"go\" on the system which I can use to determine the correct arch or specify it explicitly by setting TARGETARCH" + exit 1 + ;; + *) + echo "${uArch}" + return 0 + ;; + esac + + fi + + # default value + echo >&2 "Unable to determine CPU arch, falling back to amd64. You can specify a target arch by setting TARGETARCH" + echo amd64 +} + +get_target_variant() { + echo "${TARGETVARIANT:-}" +} + +while [ $# -gt 0 ]; do + imageTag="$1" + shift + image="${imageTag%%[:@]*}" + imageTag="${imageTag#*:}" + digest="${imageTag##*@}" + tag="${imageTag%%@*}" + + # add prefix library if passed official image + if [[ "$image" != *"/"* ]]; then + image="library/$image" + fi + + imageFile="${image//\//_}" # "/" can't be in filenames :) + + token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" + + manifestJson="$( + curl -fsSL \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ + -H 'Accept: application/vnd.oci.image.index.v1+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ + "$registryBase/v2/$image/manifests/$digest" + )" + if [ "${manifestJson:0:1}" != '{' ]; then + echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:" + echo >&2 " $manifestJson" + exit 1 + fi + + imageIdentifier="$image:$tag@$digest" + + schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')" + case "$schemaVersion" in + 2) + mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')" + + case "$mediaType" in + application/vnd.oci.image.manifest.v1+json | application/vnd.docker.distribution.manifest.v2+json) + handle_single_manifest_v2 "$manifestJson" + ;; + application/vnd.oci.image.index.v1+json | application/vnd.docker.distribution.manifest.list.v2+json) + layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')" + IFS="$newlineIFS" + mapfile -t layers <<< "$layersFs" + unset IFS + + found="" + targetArch="$(get_target_arch)" + targetVariant="$(get_target_variant)" + # parse first level multi-arch manifest + for i in "${!layers[@]}"; do + layerMeta="${layers[$i]}" + maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')" + maniVariant="$(echo "$layerMeta" | jq --raw-output '.platform.variant')" + if [[ "$maniArch" = "${targetArch}" ]] && [[ -z "${targetVariant}" || "$maniVariant" = "${targetVariant}" ]]; then + digest="$(echo "$layerMeta" | jq --raw-output '.digest')" + # get second level single manifest + submanifestJson="$( + curl -fsSL \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ + -H 'Accept: application/vnd.oci.image.index.v1+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ + "$registryBase/v2/$image/manifests/$digest" + )" + handle_single_manifest_v2 "$submanifestJson" + found="found" + break + fi + done + if [ -z "$found" ]; then + echo >&2 "error: manifest for ${targetArch}${targetVariant:+/${targetVariant}} is not found" + exit 1 + fi + ;; + *) + echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'" + exit 1 + ;; + esac + ;; + + 1) + if [ -z "$doNotGenerateManifestJson" ]; then + echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'" + echo >&2 " this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)" + echo >&2 + doNotGenerateManifestJson=1 + fi + + layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')" + IFS="$newlineIFS" + mapfile -t layers <<< "$layersFs" + unset IFS + + history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')" + imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')" + + echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." + for i in "${!layers[@]}"; do + imageJson="$(echo "$history" | jq --raw-output ".[${i}]")" + layerId="$(echo "$imageJson" | jq --raw-output '.id')" + imageLayer="${layers[$i]}" + + mkdir -p "$dir/$layerId" + echo '1.0' > "$dir/$layerId/VERSION" + + echo "$imageJson" > "$dir/$layerId/json" + + # TODO figure out why "-C -" doesn't work here + # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." + # "HTTP/1.1 416 Requested Range Not Satisfiable" + if [ -f "$dir/$layerId/layer.tar" ]; then + # TODO hackpatch for no -C support :'( + echo "skipping existing ${layerId:0:12}" + continue + fi + token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" + fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress-bar + done + ;; + + *) + echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'" + exit 1 + ;; + esac + + echo + + if [ -s "$dir/tags-$imageFile.tmp" ]; then + echo -n ', ' >> "$dir/tags-$imageFile.tmp" + else + images=("${images[@]}" "$image") + fi + echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp" +done + +echo -n '{' > "$dir/repositories" +firstImage=1 +for image in "${images[@]}"; do + imageFile="${image//\//_}" # "/" can't be in filenames :) + image="${image#library\/}" + + [ "$firstImage" ] || echo -n ',' >> "$dir/repositories" + firstImage= + echo -n $'\n\t' >> "$dir/repositories" + echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories" +done +echo -n $'\n}\n' >> "$dir/repositories" + +rm -f "$dir"/tags-*.tmp + +if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then + echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json" +else + rm -f "$dir/manifest.json" +fi + +echo "Download of images into '$dir' complete." +echo "Use something like the following to load the result into a Docker daemon:" +echo " tar -cC '$dir' . | docker load" diff --git a/meta-openeuler/recipes-containers/k3s/k3s/isulad-daemon-setting.patch b/meta-openeuler/recipes-containers/k3s/k3s/isulad-daemon-setting.patch deleted file mode 100644 index 775f8a5b308318156096cf8485e7e59d1fd5bc77..0000000000000000000000000000000000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s/isulad-daemon-setting.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/etc/isulad/daemon.json b/./daemon.json -index 55ba5401619..8dc1e8e71c0 100644 ---- a/etc/isulad/daemon.json -+++ b/./daemon.json -@@ -37,11 +37,11 @@ - ], - "insecure-registries": [ - ], -- "pod-sandbox-image": "", -+ "pod-sandbox-image": "docker.io/rancher/mirrored-pause:3.6", - "native.umask": "normal", - "network-plugin": "", -- "cni-bin-dir": "", -- "cni-conf-dir": "", -+ "cni-bin-dir": "/var/lib/rancher/k3s/data/current/bin", -+ "cni-conf-dir": "/var/lib/rancher/k3s/agent/etc/cni/net.d", - "image-layer-check": false, - "use-decrypted-key": true, - "insecure-skip-verify-enforce": false, diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent b/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent deleted file mode 100755 index ca32b409fabb6e14c93b6528ab4902da0048ada6..0000000000000000000000000000000000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/sh -eu -# -# Copyright (C) 2020 Axis Communications AB -# -# SPDX-License-Identifier: Apache-2.0 - -# -# Simplified from k3s-install.sh - -set -x - -ENV_CONF=/etc/systemd/system/k3s-agent.service.d/10-env.conf - -usage() { - echo " -USAGE: - ${0##*/} [OPTIONS] -OPTIONS: - --token value, -t value Token to use for authentication [\$K3S_TOKEN] - --token-file value Token file to use for authentication [\$K3S_TOKEN_FILE] - --server value, -s value Server to connect to [\$K3S_URL] - --node-name value Node name [\$K3S_NODE_NAME] - --resolv-conf value Kubelet resolv.conf file [\$K3S_RESOLV_CONF] - --cluster-secret value Shared secret used to bootstrap a cluster [\$K3S_CLUSTER_SECRET] - --isula-setup, -i Setup container runtime endpoint as iSulad - -h print this -" -} - -isulad_daemon_set() { - sed -i 's/"cni-bin-dir": "*",/"cni-bin-dir": "\/var\/lib\/rancher\/k3s\/data\/current\/bin",/' /etc/isulad/daemon.json - sed -i 's/"cni-conf-dir": "*",/"cni-conf-dir": "\/var\/lib\/rancher\/k3s\/agent\/etc\/cni\/net.d",/' /etc/isulad/daemon.json - sed -i 's/"pod-sandbox-image": "*",/"pod-sandbox-image": "docker.io\/rancher\/mirrored-pause:3.6",/' /etc/isulad/daemon.json -} - -isulad_preload_images() { - if [ ! -e "/etc/k3s/tools/k3s-airgap-images-arm64.tar.gz" ]; then - echo "failed to find k3s-airgap-images tarball!" - exit -1 - fi - isula load -i /etc/k3s/tools/k3s-airgap-images-arm64.tar.gz -} - -isula_set() { - isulad_preload_images - isulad_daemon_set -} - -[ $# -gt 0 ] || { - usage - exit 1 -} - -rm -f $ENV_CONF -mkdir -p ${ENV_CONF%/*} -echo [Service] > $ENV_CONF - -while getopts "t:s:ih-:" opt; do - case $opt in - h) - usage - exit - ;; - i) - isulad_daemon_set - exit - ;; - t) - VAR_NAME=K3S_TOKEN - ;; - s) - VAR_NAME=K3S_URL - ;; - -) - [ $# -ge $OPTIND ] || { - usage - exit 1 - } - opt=$OPTARG - case $opt in - token) - VAR_NAME=K3S_TOKEN - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - token-file) - VAR_NAME=K3S_TOKEN_FILE - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - server) - VAR_NAME=K3S_URL - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - node-name) - VAR_NAME=K3S_NODE_NAME - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - resolv-conf) - VAR_NAME=K3S_RESOLV_CONF - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - cluster-secret) - VAR_NAME=K3S_CLUSTER_SECRET - eval OPTARG='$'$OPTIND - OPTIND=$(($OPTIND + 1)) - ;; - isula-setup) - isula_set - exit - ;; - help) - usage - exit - ;; - *) - usage - exit 1 - ;; - esac - ;; - *) - usage - exit 1 - ;; - esac -echo Environment=$VAR_NAME=$OPTARG >> $ENV_CONF -done - -chmod 0644 $ENV_CONF -rm -rf /var/lib/rancher/k3s/agent -systemctl daemon-reload -systemctl restart k3s-agent -systemctl enable k3s-agent.service diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent.service b/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent.service index a613de2311cbc36d8f902c702f1d5227969d14ef..f2610badc124f0fa6e7cb056e8248012485f424b 100644 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent.service +++ b/meta-openeuler/recipes-containers/k3s/k3s/k3s-agent.service @@ -1,13 +1,17 @@ -# Derived from the k3s install.sh's create_systemd_service_file() function [Unit] Description=Lightweight Kubernetes Agent Documentation=https://k3s.io +Wants=network-online.target +After=network-online.target +Requires=systemd-timesyncd.service +Requires=isulad.service +After=isulad.service [Install] WantedBy=multi-user.target [Service] -Type=notify +Type=exec KillMode=control-group Delegate=yes LimitNOFILE=infinity @@ -17,8 +21,12 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s +EnvironmentFile=/etc/systemd/system/k3s-agent.service.env ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/bin/k3s agent -ExecStopPost=/usr/bin/k3s-kill-agent +ExecStart=/usr/bin/k3s \ + agent \ + --with-node-id \ + --container-runtime-endpoint=unix:///var/run/isulad.sock +ExecStopPost=/usr/bin/k3s-clean diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-kill-agent b/meta-openeuler/recipes-containers/k3s/k3s/k3s-clean old mode 100755 new mode 100644 similarity index 79% rename from meta-openeuler/recipes-containers/k3s/k3s/k3s-kill-agent rename to meta-openeuler/recipes-containers/k3s/k3s/k3s-clean index 8eca918c05184976220f5982eb9e04ff4f4a2e3a..9467dc2c00d0a061967da43f71a22ba03eda00b4 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s-kill-agent +++ b/meta-openeuler/recipes-containers/k3s/k3s/k3s-clean @@ -17,14 +17,14 @@ do_unmount() { [ -z "$mounts" ] || umount $mounts } -do_unmount /run/k3s /var/lib/rancher/k3s +do_unmount /run/k3s @DEFAULT_DATA_DIR@ -# The lines below come from install.sh's create_killall() function: ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do iface=${iface%%@*} [ -z "$iface" ] || ip link delete $iface done ip link delete cni0 -ip link delete flannel.1 +ip link delete @PLUGIN_IF@ rm -rf /var/lib/cni/ +rm -rf @CNI_LOCALSTATE_DIR@ diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent b/meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent index b5c60e628dca12246f484b1c606c10eb382e24f7..63fce7726d71ab904db09bc193dd856aaf7c46ec 100644 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent +++ b/meta-openeuler/recipes-containers/k3s/k3s/k3s-install-agent @@ -1,21 +1,11 @@ -#!/bin/sh +#!/bin/bash set -e -# OpenEuler Embedded K3s Agent Deploy Script -# Modified from k3s-io install.sh -# # Usage: # curl ... | ENV_VAR=... sh - # or # ENV_VAR=... ./install.sh # -# **Only if you know what you are doing, you can use this script to install k3s agent on OpenEuler Embedded. -# -# What is hardcoded: -# INSTALL_K3S_CHANNEL=stable -# INSTALL_K3S_SKIP_SELINUX_RPM=true -# INSTALL_K3S_TYPE=exec (not notify for server) -# INSTALL_K3S_EXEC="agent " # Example: # Installing an agent with airgap images: # AIRGAP_IMAGES_DIR=/path/to/images K3S_URL=https://server-url:6443 K3S_TOKEN=xxx k3s-install @@ -29,52 +19,51 @@ set -e # - K3S_TOKEN/K3S_TOKEN_FILE (REQUIRED) # Cluster secret token for agent authentication. At least one must be set. # CLUSTER_SECRETE is deprecated -# -# - AIRGAP_IMAGES_DIR -# Directory containing k3s-airgap-images-ARCH.tar.gz for offline installation -# -# - AIRGAP_IMAGES -# Directly specify k3s-airgap images for offline installation -# -# - ARGS -# ARGS will be after `k3s agent`, e.g.: -# ARGS="--disable=traefik" => "k3s agent --disable=traefik" -# -# - CONTAINER_RT_EP -# Container runtime endpoint type -# default CONTAINER_RT_EP=isulad => --container-runtime-endpoint unix:///var/run/isulad.sock instead of containerd.sock) -# if CONTAINER_RT_EP=embedded => use embedded k3s containerd instead. -# otherwise => --container-runtime=endpint "${CONTAINER_RT_EP}" -# +# # - INSTALL_K3S_BIN_DIR # Directory to install k3s binary and scripts (default: /usr/bin) # +# - INSTALL_K3S_SKIP_ENABLE +# If set to true will not enable or start k3s service. +# +# - INSTALL_K3S_SKIP_START +# If set to true will not start k3s service. +# # - INSTALL_K3S_SYSTEMD_DIR # Directory to install systemd service files (default: /etc/systemd/system) # # - INSTALL_K3S_BIN_DIR_READ_ONLY # If set to true, prevents writing to INSTALL_K3S_BIN_DIR # -# - SKIP_LOAD_AIRGAP -# If set to true, prevents loading airgap images to isula +# - INSTALL_K3S_ROOTLESS (as a server) +# If set to true, deploy a rootless k3s agent service instead of the default one, +# but cgroup v2 is needed. +# +# - INSTALL_K3S_EXEC or script arguments +# Command flags to use for launching k3s in the systemd service, if +# The final systemd command resolves to a combination +# of k3s-agent "EXEC" and script args ($@). +# +# The following commands result in the same behavior: +# INSTALL_K3S_EXEC="--flannel-backend=none" k3s-install-agent +# k3s-install-agent - --flannel-backend=none +# # -# - ONLY_SET_ISULAD -# If set to true, only isulad configuration will take place +# - INSTALL_K3S_AIRGAP +# +# - AIRGAP_DIR +# The directory containing k3s airgap images for offline installation +# (default: /etc/k3s/k3s-airgap-images-${ARCH}.tar.gz) # -# Notes: -# 1. This version ONLY supports agent mode and requires: -# - systemd as process supervisor (SysV is in progress) -# - isulad as container runtime endpoint -# - Preloaded pause image (docker.io/rancher/mirrored-pause:3.6) -# 2. Automatically configures isulad: -# - Configures cni-conf-dir and cni-bin-dir for k3s -# 3. Airgap installation requires: -# - k3s-airgap-images-ARCH.tar.gz in AIRGAP_IMAGES_DIR -# - Properly configured container registry +# - ONLY_CONFIG_ISULAD +# Only config isulad, skipping script install, service file creation and service enable. +# (default: true because the k3s-agent is configured after yocto build.) # -# For detailed documentation about k3s, see: https://docs.k3s.io/ +# - DATA_DIR +# override options: +# - OVERRIDE_UNINSTALLER +# - OVERRIDE_KILLER -# --- helper functions for logs --- info() { echo '[INFO] ' "$@" } @@ -82,20 +71,15 @@ warn() { echo '[WARN] ' "$@" >&2 } +note() +{ + echo '[>>>>] ' "$@" >&2 +} fatal() { echo '[ERROR] ' "$@" >&2 exit 1 } -# --- fatal if not system (future: or SysV) --- -verify_system() { - if [ -x /sbin/systemd ]; then - HAS_SYSTEMD=true - return - fi - fatal 'Can not find systemd to use as a process supervisor for k3s' -} - # --- add quotes to command arguments --- quote() { for arg in "$@"; do @@ -134,67 +118,62 @@ verify_k3s_url() { ;; esac } +# --- fatal if not systemd (busy init will be supported in weeks) +verify_system() { + if [ -d /run/systemd ]; then + HAS_SYSTEMD=true + return + fi + fatal 'Can not find systemd to use as a process supervisor for k3s' +} -# --- setup basic environment --- +# --- define needed environment variables --- setup_env() { - # always set OEE as agent mode - AIRGAP_DIR="${AIRGAP_IMAGES_DIR:-/etc/k3s/tools}" - rawARCH="$(uname -m)" - case "$rawARCH" in + ARCH_raw="$(uname -m)" + case "$ARCH_raw" in arm) ARCH=arm;; aarch64|arm64) ARCH=arm64 ;; x86_64) ARCH=amd64 ;; - s390x) - ARCH=s390x ;; *) - fatal "unsupported architecture ${rawARCH}" + fatal "unsupported architecture ${ARCH_raw}" ;; esac - info "got arch=${ARCH}" - if [ ! -n "${AIRGAP_IMAGES}" ]; then - AIRGAP_IMAGES="${AIRGAP_DIR}/k3s-airgap-images-${ARCH}.tar.gz" + AIRGAP_DIR=${AIRGAP_DIR:-@ASSET_DIR@} + if [[ $INSTALL_K3S_AIRGAP = 'true' ]]; then + IMAGES=$AIRGAP_DIR/k3s-airgap-images-${ARCH}.tar.gz + # NOTICE: 此处用的airgap包假设为上游的规范包，但我们在yocto构建中提供了手动下载打包 + # 的airgap包，k3s-airgap-images-${ARCH}-yocto.tar.gz，可以更改为使用此包而不用预先下载好的airgap fi - - - if [ "$ONLY_SET_ISULAD" = "true" ]; then - config_isulad - exit 0 + ONLY_CONFIG_ISULAD=${ONLY_CONFIG_ISULAD:-false} + if [ "$ONLY_CONFIG_ISULAD" = "true" ]; then + isulad_daemon_conf + exit 0 fi verify_k3s_url - - CONTAINER_RT_EP=${CONTAINER_RT_EP:-"isulad"} - case $CONTAINER_RT_EP in - "isulad") - CONTAINER_RUNTIME_ENDPOINT="--container-runtime-endpoint=unix:///var/run/isulad.sock" - ;; - "embedded") - CONTAINER_RUNTIME_ENDPOINT="" - ;; - *) - CONTAINER_RUNTIME_ENDPOINT="--container-runtime-endpoint=${CONTAINER_RT_EP}" - ;; - esac - CMD_K3S=agent - CMD_K3S_EXEC="${CMD_K3S}$(quote_indent ""$@""${CONTAINER_RUNTIME_ENDPOINT}"")" - SYSTEM_NAME=k3s-${CMD_K3S} + CONTAINER_RUNTIME_ENDPOINT="--container-runtime-endpoint=unix:///var/run/isulad.sock" + CMD_K3S_EXEC="$(quote_indent ""$@"")" + # TODO: modify the source codes. we will implementr a mini multicall without + # building cli/k3s/main.go + SYSTEM_NAME="k3s-${CMD_K3S}" SUDO=sudo if [ $(id -u) -eq 0 ]; then SUDO= fi + # NOTICE: is notify SYSTEM_TYPE=exec # --- set binary and service directories - BIN_DIR=${INSTALL_K3S_BIN_DIR:-/usr/bin} + BIN_DIR=@DEFAULT_BIN_DIR@ if ! $SUDO sh -c "touch ${BIN_DIR}/k3s-ro-test && rm -rf ${BIN_DIR}/k3s-ro-test"; then if [ -d /opt/bin ]; then BIN_DIR=/opt/bin @@ -212,53 +191,64 @@ setup_env() { SERVICE_K3S=${SYSTEM_NAME}.service UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh} KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh} - AGENT_KILLER=${KILL_AGENT:-${BIN_DIR}/k3s-kill-agent} - FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S} FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env - + #TODO: handle logic about /var/lib/rancher/k3s/data/HASH/ many things in /var/lib/rancher/k3s + DATA_DIR=${DATA_DIR:-@DEFAULT_DATA_DIR@} } -# --- create killall script --- -create_agent_killer() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating killall script ${AGENT_KILLER}" - $SUDO tee ${AGENT_KILLER} >/dev/null << \EOF -#!/bin/sh -eu -# -# Copyright (C) 2020 Axis Communications AB -# -# SPDX-License-Identifier: Apache-2.0 - -do_unmount() { - [ $# -eq 2 ] || return - local mounts= - while read ignore mount ignore; do - case $mount in - $1/*|$2/*) - mounts="$mount $mounts" - ;; - esac - done /dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface -done -ip link delete cni0 -ip link delete flannel.1 -rm -rf /var/lib/cni/ -EOF - $SUDO chmod 755 ${AGENT_KILLER} - $SUDO chown root:root ${AGENT_KILLER} +create_scripts() { + insert_completion + [ $OVERRIDE_KILLER=true ] && create_killall + [ $OVERRIDE_UNINSTALLER=true ] && create_uninstall } +# --- install auto-completion for bash or zsh --- +insert_completion() { + myshell=$(basename $SHELL) + rc="$HOME/.${myshell}rc" + touch $rc + if grep -q "<(k3s completion bash)" $rc; then + info "k3s $myshell completion script is already inserted" + else + k3s completion $myshell -i + fi +} # --- create killall script --- create_killall() { @@ -316,10 +306,13 @@ do_unmount_and_remove() { awk -v path="$1" '$2 ~ ("^" path) { print $2 }' /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' } +do_unmount_and_remove '/run/netns/cni-' +do_unmount_and_remove '/run/netns/isulacni-' + +# for k3s server do_unmount_and_remove '/run/k3s' do_unmount_and_remove '/var/lib/rancher/k3s' do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/run/netns/cni-' # Remove CNI namespaces ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete @@ -330,7 +323,7 @@ ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; [ -z "$iface" ] || ip link delete $iface done ip link delete cni0 -ip link delete flannel.1 +ip link delete @CNI_PLUGIN_IF@ rm -rf /var/lib/cni/ iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore EOF @@ -340,8 +333,7 @@ EOF # --- create uninstall script --- create_uninstall() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating uninstall script ${UNINSTALL_K3S_SH}" +info "Creating uninstall script ${UNINSTALL_K3S_SH}" $SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF #!/bin/sh set -x @@ -371,20 +363,29 @@ if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then exit fi -for cmd in kubectl crictl ctr; do +for cmd in @INSTALL_COMMANDS@; do if [ -L ${BIN_DIR}/\$cmd ]; then rm -f ${BIN_DIR}/\$cmd fi done -rm -rf /etc/rancher/k3s +rm -rf /etc/k3s rm -rf /run/k3s rm -rf /run/flannel -rm -rf /var/lib/rancher/k3s +rm -rf ${DATA_DIR} rm -rf /var/lib/kubelet rm -f ${BIN_DIR}/k3s +rm -f @ASSET_DIR@/@tar.zst@ +rm -f @ASSET_DIR@/k3s-assets.tar rm -f ${KILLALL_K3S_SH} +for fullname in @IMAGE_LIST@; do + name_and_tag="\${fullname#docker.io/}" + $SUDO isula rmi \$name_and_tag +done + +# NOTICE: openeuler embedded may support online package management in future +# this script will update when it comes if type yum >/dev/null 2>&1; then yum remove -y k3s-selinux rm -f /etc/yum.repos.d/rancher-k3s-common*.repo @@ -394,39 +395,45 @@ EOF $SUDO chown root:root ${UNINSTALL_K3S_SH} } -create_scripts() { - create_agent_killer - create_uninstall -} - -systemd_disable() { - $SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true -} - # --- create environment file --- create_env_file() { - info "Creating environment file ${FILE_K3S_ENV}" - UMASK=$(umask) - umask 0377 - if [ -z "${K3S_TOKEN}" ]; then - fatal "missing K3S_TOKEN." - exit -1 + create_k3s_env_files + create_timesync_env_files +} + +create_k3s_env_files() { + + if [ -f "${FILE_K3S_ENV}" ]; then + info "${FILE_K3S_ENV} already exists" + return fi + info "env: Creating environment file ${FILE_K3S_ENV}" + $SUDO touch ${FILE_K3S_ENV} + $SUDO chmod 0600 ${FILE_K3S_ENV} + [ -z ${K3S_URL} ] && fatal "missing K3S_TOKEN" \ + || [ -z ${K3S_TOKEN} ] && fatal "missing K3S_TOKEN" env | grep '^K3S_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null - env | grep -E '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null - umask $UMASK + env | grep -Ei '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null } -# --- verify an executable k3s binary is installed --- -verify_k3s_is_executable() { - if [ ! -x ${BIN_DIR}/k3s ]; then - fatal "Executable k3s binary not found at ${BIN_DIR}/k3s" +create_timesync_env_files() { + set -x + if which timedatectl > /dev/null 2>&1; then + TIME_CONF=/etc/systemd/timesyncd.conf.d + $SUDO mkdir -p $TIME_CONF + $SUDO echo "NTP=ntp.ntsc.ac.cn cn.ntp.org.cn" > "${TIME_CONF}/cn_ntp_pool" + + $SUDO timedatectl set-timezone Asia/Shanghai + $SUDO timedatectl set-ntp true + $SUDO systemctl restart systemd-timesyncd + $SUDO timedatectl + else + warn "cannot find timedatectl." + note "use alternatives like ntpdate to synchronize the system clock" fi } -# --- create systemd service file --- +# --- create a new systemd service file (only for agent)--- create_systemd_service_file() { info "Creating service file ${FILE_K3S_SERVICE}" $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF @@ -435,6 +442,9 @@ Description=Lightweight Kubernetes Documentation=https://k3s.io Wants=network-online.target After=network-online.target +Requires=systemd-timesyncd.service +Requires=isulad.service +After=isulad.service [Install] WantedBy=multi-user.target @@ -442,7 +452,7 @@ WantedBy=multi-user.target [Service] Type=exec EnvironmentFile=${FILE_K3S_ENV} -KillMode=process +KillMode=control-group Delegate=yes LimitNOFILE=1048576 LimitNPROC=infinity @@ -451,62 +461,144 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s -ExecStart=${BIN_DIR}/k3s \\ - ${CMD_K3S_EXEC} +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=${BIN_DIR}/k3s-agent \\ + ${CMD_K3S_EXEC} \\ + --container-runtime-endpoint=unix:///var/run/isulad.sock \\ + --flannel-backend=none EOF } +check_cgroup_mode() { + cg_dir=/sys/fs/cgroup + v2_dir=/sys/fs/cgroup/unified + [ $(stat -fc %T $cg_dir) = "cgroup2fs" ] \ + && (info "cgroup mode=v2 unified") \ + || [ -e $v2_dir ] && ( + warn "cgroup mode=v2/v1 hybrid;" + note "add \`cgroup_no_v1=all\` to kernel boot command line to disable v1 cgroup" + fatal "k3s-rootless does not support the cgroup hybrid mode" + ) \ + || fatal "cgroup mode=v1, which is not recommended for k3s rootless" +} + +deploy_rootless_service() { + warn "k3s rootless service is experimental, deploy it manually by reading @rootless@" + [ $(ps --no-headers -o comm 1) != systemd ] && fatal "systemd is required for rootless k3s" + check_cgroup_mode +} # --- enable and start systemd service --- -systemd_enable_and_start() { +service_enable_and_start() { + + if [ -f "/proc/cgroups" ] && [ "$(grep memory /proc/cgroups | while read -r n n n enabled; do echo $enabled; done)" -eq 0 ]; + then + info 'Failed to find memory cgroup, you may need to add "cgroup_memory=1 cgroup_enable=memory" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)' + fi + + [ "${INSTALL_K3S_SKIP_ENABLE}" = true ] && return + info "Enabling ${SYSTEM_NAME} unit" - $SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null + + $SUDO systemctl enable isulad.service + $SUDO systemctl enable ${FILE_K3S_SERVICE} $SUDO systemctl daemon-reload >/dev/null + [ "${INSTALL_K3S_SKIP_START}" = true ] && return + + info "Starting iSulad systemd service" + $SUDO systemctl restart isulad.service info "Starting ${SYSTEM_NAME}" $SUDO systemctl restart ${SYSTEM_NAME} } -isulad_daemon_set() { - sed -i 's/"cni-bin-dir": "*",/"cni-bin-dir": "\/var\/lib\/rancher\/k3s\/data\/current\/bin",/' /etc/isulad/daemon.json - sed -i 's/"cni-conf-dir": "*",/"cni-conf-dir": "\/var\/lib\/rancher\/k3s\/agent\/etc\/cni\/net.d",/' /etc/isulad/daemon.json - sed -i 's/"pod-sandbox-image": "*",/"pod-sandbox-image": "docker.io\/rancher\/mirrored-pause:3.6",/' /etc/isulad/daemon.json - systemctl daemon-reload - systemctl restart isulad +isulad_daemon_conf() { + #NOTICE: keep snapshotter "overlayfs", DO NOT EDIT + SRC_DAEMON="/etc/isulad/daemon.json" + DAEMON=$(mktemp /tmp/daemon.XXXX) + cp -f $SRC_DAEMON $DAEMON + # support for rootless + # isula does not support mix instance of cgroup v1 and v2 + # TODO: consider add: network-plugin: "cni" + CNI_BIN=/usr/libexec/cni + CNI_CONF=/etc/cni/net.d + sed -i "s#\(\"default-runtime\": \"\)[^\"]*#\1runc#" "${DAEMON}" + if [ -d "${CNI_BIN}" ] && [ -d "${CNI_CONF}" ]; then + info "use openEuler cni-plguin as cni backend" + sed -i "s#\(\"cni-bin-dir\": \"\)[^\"]*#\1${CNI_BIN}#" "${DAEMON}" + sed -i "s#\(\"cni-conf-dir\": \"\)[^\"]*#\1${CNI_CONF}#" "${DAEMON}" + else + warn "cannot find openeuler cni plugin binary directory '${CNI_BIN}' \ + and conf directory '${CNI_CONF}' \ + use k3s embedded cni plugin instead. + " + # use embedded cni , no supported now because we have removed the compilcation of embedded cni + sed -i 's#\("cni-bin-dir": "\)[^"]*#\1/var/lib/rancher/k3s/data/current/bin#' "${DAEMON}" + sed -i 's#\("cni-conf-dir": "\)[^"]*#\1/var/lib/rancher/k3s/agent/etc/cni/net.d#' "${DAEMON}" + fi + # airgap + if [ "${INSTALL_K3S_AIRGAP}" = true ]; then + sed -i 's#\("pod-sandbox-image": "\)[^"]*#\1docker.io/rancher/mirrored-pause:3.6#' "${DAEMON}" + else + sed -i 's#\("pod-sandbox-image": "\)[^"]*#\1docker.io/rancher/pause:3.6#' "${DAEMON}" + fi + $SUDO cp $DAEMON $SRC_DAEMON --backup + $SUDO systemctl daemon-reload + $SUDO systemctl restart isulad + rm $DAEMON } -isulad_preload_images() { - if [ "${SKIP_LOAD_AIRGAP}" = true ]; then - info "skipped airgap images loading" - return 0 +# TODO remove airgap +isulad_load_images() { + set +e -x + if [ "${INSTALL_K3S_AIRGAP}" != true ]; then + info "skipped airgap images loading + kubelet will pull images" + return + fi + + if [ ! -f "${IMAGES}" ]; then + fatal "Unable to find ${IMAGES}" + fi + + $SUDO isula images | grep -q -E "rancher/mirrored" || \ + $SUDO isula load -i "${IMAGES}" > /dev/null || + set +x + return +} + +limit_vmem_for_high_concurrency() { + if [ ${LIMIT_VMEM} = true ]; then + init=$(ps --no-headers -o comm 1) + [ $init = systemd ] && ( + echo "MALLOC_ARENA_MAX=$" >> /etc/sysconf/iSulad + ) + [ $init = init ] && ( + # TODO: append to iSulad init file + echo "not implmented yet" + ) fi - info "searching for : ${AIRGAP_IMAGES}" - if ! test -f "${AIRGAP_IMAGES}"; then - fatal "Unable to find ${AIRGAP_IMAGES} tarball" - fi - isula load -i "${AIRGAP_IMAGES}" > /dev/null } config_isulad() { - isulad_daemon_set - isulad_preload_images + isulad_daemon_conf + isulad_load_images + limit_vmem_for_high_concurrency } -eval set -- $(escape "${ARGS}") $(quote "${@}") +eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "${@}") { - setup_env "$@" - create_scripts - create_env_file - if [ "${CONTAINER_RT_EP}" = "isulad" ]; then - config_isulad - elif [ "${CONTAINER_RT_EP}" = "embedded" ]; then - create_systemd_service_file - else - echo "customize k3s-agent.service manually for endpoint ${CONTAINER_RT_EP}" - fi - systemd_enable_and_start -} - + verify_system + setup_env "$@" + verify_k3s_file + create_scripts + create_env_file + create_systemd_service_file + config_isulad + [ "$INSTALL_K3S_ROOTLESS" = "true" ] && deploy_rootless_service + service_enable_and_start +} \ No newline at end of file diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-killall.sh b/meta-openeuler/recipes-containers/k3s/k3s/k3s-killall.sh index 9e726153e0604bdc2505205ccd114bc28137c848..9e7524aa6a7bb1303594079105ca6254a9364280 100644 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s-killall.sh +++ b/meta-openeuler/recipes-containers/k3s/k3s/k3s-killall.sh @@ -1,5 +1,4 @@ #!/bin/sh - # Based on: k3s-killall.sh installed when running Rancher Lab's K3S install.sh # In open-source project: https://github.com/k3s-io/k3s # @@ -12,8 +11,14 @@ # - Remove service stopping code (as this is intended to run as part of service # stop) # - Changes to resolve warnings from the ShellCheck static analysis tool +# # # SPDX-License-Identifier: Apache License 2.0 +# +# Modfications from openEuler: +# - Add cni removal for iSulad +# - Handle some path configs preventing hardcode in this file +# [ "$(id -u)" -eq 0 ] || exec sudo "$0" "$@" @@ -66,6 +71,7 @@ do_unmount_and_remove '/var/lib/rancher/k3s' do_unmount_and_remove '/var/lib/kubelet/pods' do_unmount_and_remove '/var/lib/kubelet/plugins' do_unmount_and_remove '/run/netns/cni-' +do_unmount_and_remove '/run/netns/isulacni-' # Remove CNI namespaces ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete @@ -76,7 +82,6 @@ ip link show 2>/dev/null | grep 'master cni0' | while read -r _ iface _; do [ -z "$iface" ] || ip link delete "$iface" done ip link delete cni0 -ip link delete flannel.1 -ip link delete flannel-v6.1 +ip link delete @CNI_PLUGIN_IF@ rm -rf /var/lib/cni/ iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s-rootless.service b/meta-openeuler/recipes-containers/k3s/k3s/k3s-rootless.service new file mode 100644 index 0000000000000000000000000000000000000000..fee7f7a4d749d219096f7bcfc2a9d76141281376 --- /dev/null +++ b/meta-openeuler/recipes-containers/k3s/k3s/k3s-rootless.service @@ -0,0 +1,45 @@ +# systemd unit file for k3s (rootless) +# +# Usage: +# - [Optional] Enable cgroup v2 delegation, see https://rootlesscontaine.rs/getting-started/common/cgroup2/ . +# This step is optional, but highly recommended for enabling CPU and memory resource limtitation. +# +# - Copy this file as `~/.config/systemd/user/k3s-rootless.service`. +# Installing this file as a system-wide service (`/etc/systemd/...`) is not supported. +# Depending on the path of `k3s` binary, you might need to modify the `ExecStart=/usr/local/bin/k3s ...` line of this file. +# +# - Run `systemctl --user daemon-reload` +# +# - Run `systemctl --user enable --now k3s-rootless` +# +# - Run `KUBECONFIG=~/.kube/k3s.yaml kubectl get pods -A`, and make sure the pods are running. +# +# Troubleshooting: +# - See `systemctl --user status k3s-rootless` to check the daemon status +# - See `journalctl --user -f -u k3s-rootless` to see the daemon log +# - See also https://rootlesscontaine.rs/ + +[Unit] +Description=k3s (Rootless) + +[Service] +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +# NOTE: Don't try to run `k3s server --rootless` on a terminal, as it doesn't enable cgroup v2 delegation. +# If you really need to try it on a terminal, prepend `systemd-run --user -p Delegate=yes --tty` to create a systemd scope. +ExecStart=/usr/bin/k3s-server --rootless --snapshotter=fuse-overlayfs --container-runtime-endpoint=unix:///var/run/isulad.sock +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +Delegate=yes +Type=simple +KillMode=mixed + +[Install] +WantedBy=default.target diff --git a/meta-openeuler/recipes-containers/k3s/k3s/k3s.service b/meta-openeuler/recipes-containers/k3s/k3s/k3s.service index 0917c3f2e368ba459389babb527af437a30dbf5c..35abd7fd24956bd383630a09f36af3d892c93b9a 100644 --- a/meta-openeuler/recipes-containers/k3s/k3s/k3s.service +++ b/meta-openeuler/recipes-containers/k3s/k3s/k3s.service @@ -2,6 +2,7 @@ [Unit] Description=Lightweight Kubernetes Documentation=https://k3s.io +Required=systemd-timesyncd.service After=network-online.target Wants=network-online.target @@ -27,7 +28,8 @@ RestartSec=5s ExecStartPre=/bin/sh -xc '! systemctl is-enabled --quiet nm-cloud-setup.service' ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/bin/k3s server +ExecStart=/usr/bin/k3s-server \ + --container-runtime-endpoint=unix//var/run/isulad.sock # Avoid any delay due to this service when the system is rebooting or shutting # down by using the k3s-killall.sh script to kill all of the running k3s # services and containers diff --git a/meta-openeuler/recipes-containers/k3s/k3s/modules.txt b/meta-openeuler/recipes-containers/k3s/k3s/modules.txt deleted file mode 100644 index ef53706779d58438ad5f61f033d11c960ce28989..0000000000000000000000000000000000000000 --- a/meta-openeuler/recipes-containers/k3s/k3s/modules.txt +++ /dev/null @@ -1,202 +0,0 @@ -# github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.20 -# github.com/benmoss/go-powershell => github.com/k3s-io/go-powershell v0.0.0-20201118222746-51f4c451fbd7 -# github.com/containerd/aufs => github.com/containerd/aufs v1.0.0 -# github.com/containerd/btrfs => github.com/containerd/btrfs v1.0.0 -# github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.1 -# github.com/containerd/console => github.com/containerd/console v1.0.2 -# github.com/containerd/containerd => github.com/k3s-io/containerd v1.5.9-k3s1 -# github.com/containerd/continuity => github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1 -# github.com/containerd/fifo => github.com/containerd/fifo v1.0.0 -# github.com/containerd/go-runc => github.com/containerd/go-runc v1.0.0 -# github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.0.2 -# github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.2 -# github.com/containerd/zfs => github.com/containerd/zfs v1.0.0 -# github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e -# github.com/docker/distribution => github.com/docker/distribution v2.7.1+incompatible -# github.com/docker/docker => github.com/docker/docker v20.10.2+incompatible -# github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34 -# github.com/golang/protobuf => github.com/golang/protobuf v1.5.2 -# github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.0.5 -# github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f -# github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.21.0-k3s1 -# github.com/matryer/moq => github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009 -# github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.3 -# github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210316141917-a8c4a9ee0f6b -# go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.1-k3s1 -# go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.1-k3s1 -# go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.1-k3s1 -# go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.1-k3s1 -# golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 -# golang.org/x/net => golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 -# golang.org/x/sys => golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e -# google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 -# google.golang.org/grpc => google.golang.org/grpc v1.38.0 -# gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.2.2 -# k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.22.6-k3s1 -# k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.22.6-k3s1 -# k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.22.6-k3s1 -# k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.22.6-k3s1 -# k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.22.6-k3s1 -# k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.22.6-k3s1 -# k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.22.6-k3s1 -# k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.22.6-k3s1 -# k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.22.6-k3s1 -# k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.22.6-k3s1 -# k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.22.6-k3s1 -# k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.22.6-k3s1 -# k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.22.6-k3s1 -# k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.22.6-k3s1 -# k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s2 -# k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s2 -# k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.6-k3s1 -# k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.22.6-k3s1 -# k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.22.6-k3s1 -# k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.22.6-k3s1 -# k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.22.6-k3s1 -# k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.22.6-k3s1 -# k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.22.6-k3s1 -# k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.22.6-k3s1 -# k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.22.6-k3s1 -# k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.22.6-k3s1 -# k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.22.6-k3s1 -# k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.22.6-k3s1 -# k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.22.6-k3s1 -# k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.22.6-k3s1 -# k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.22.6-k3s1 -# mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 -# github.com/Microsoft/hcsshim v0.9.2 -## explicit -# github.com/cloudnativelabs/kube-router v1.3.2 -## explicit -# github.com/containerd/cgroups v1.0.1 -## explicit -# github.com/containerd/containerd v1.6.0-beta.2.0.20211117185425-a776a27af54a -## explicit -# github.com/containerd/fuse-overlayfs-snapshotter v1.0.4 -## explicit -# github.com/containerd/stargz-snapshotter v0.10.1 -## explicit -# github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f -## explicit -# github.com/docker/docker v20.10.10+incompatible -## explicit -# github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 -## explicit -# github.com/flannel-io/flannel v0.16.3 -## explicit -# github.com/go-bindata/go-bindata v3.1.2+incompatible -## explicit -# github.com/go-sql-driver/mysql v1.6.0 -## explicit -# github.com/golangplus/testing v1.0.0 -## explicit -# github.com/google/cadvisor v0.39.3 -## explicit -# github.com/google/uuid v1.3.0 -## explicit -# github.com/gorilla/mux v1.8.0 -## explicit -# github.com/gorilla/websocket v1.4.2 -## explicit -# github.com/k3s-io/helm-controller v0.11.7 -## explicit -# github.com/k3s-io/kine v0.8.1 -## explicit -# github.com/klauspost/compress v1.13.6 -## explicit -# github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 -## explicit -# github.com/lib/pq v1.10.2 -## explicit -# github.com/mattn/go-sqlite3 v1.14.8 -## explicit -# github.com/minio/minio-go/v7 v7.0.7 -## explicit -# github.com/natefinch/lumberjack v2.0.0+incompatible -## explicit -# github.com/onsi/ginkgo v1.16.4 -## explicit -# github.com/onsi/gomega v1.11.0 -## explicit -# github.com/opencontainers/runc v1.0.3 -## explicit -# github.com/opencontainers/selinux v1.8.2 -## explicit -# github.com/otiai10/copy v1.6.0 -## explicit -# github.com/pkg/errors v0.9.1 -## explicit -# github.com/rancher/dynamiclistener v0.3.1 -## explicit -# github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 -## explicit -# github.com/rancher/remotedialer v0.2.0 -## explicit -# github.com/rancher/wharfie v0.5.1 -## explicit -# github.com/rancher/wrangler v0.8.10 -## explicit -# github.com/robfig/cron/v3 v3.0.1 -## explicit -# github.com/rootless-containers/rootlesskit v0.14.5 -## explicit -# github.com/sirupsen/logrus v1.8.1 -## explicit -# github.com/spf13/pflag v1.0.5 -## explicit -# github.com/stretchr/testify v1.7.0 -## explicit -# github.com/tchap/go-patricia v2.3.0+incompatible -## explicit -# github.com/urfave/cli v1.22.4 -## explicit -# github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 -## explicit -# go.etcd.io/etcd/api/v3 v3.5.1 -## explicit -# go.etcd.io/etcd/client/v3 v3.5.1 -## explicit -# go.etcd.io/etcd/etcdutl/v3 v3.5.1 -## explicit -# go.etcd.io/etcd/server/v3 v3.5.1 -## explicit -# golang.org/x/crypto v0.0.0-20211202192323-5770296d904e -## explicit -# golang.org/x/net v0.0.0-20211209124913-491a49abca63 -## explicit -# golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e -## explicit -# google.golang.org/grpc v1.42.0 -## explicit -# gopkg.in/yaml.v2 v2.4.0 -## explicit -# inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 -## explicit -# k8s.io/api v0.22.6 -## explicit -# k8s.io/apimachinery v0.22.6 -## explicit -# k8s.io/apiserver v0.22.6 -## explicit -# k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible -## explicit -# k8s.io/cloud-provider v0.22.6 -## explicit -# k8s.io/code-generator v0.22.6 -## explicit -# k8s.io/component-base v0.22.6 -## explicit -# k8s.io/controller-manager v0.22.6 -## explicit -# k8s.io/cri-api v0.23.0-alpha.4 -## explicit -# k8s.io/klog v1.0.0 -## explicit -# k8s.io/kubectl v0.22.6 -## explicit -# k8s.io/kubernetes v1.22.6 -## explicit -# k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a -## explicit -# sigs.k8s.io/yaml v1.2.0 -## explicit diff --git a/meta-openeuler/recipes-containers/k3s/k3s_%.bbappend b/meta-openeuler/recipes-containers/k3s/k3s_%.bbappend index 75cc276e4bbc554d7f9298c0ecf409864b5e59f4..38edf1085d22f4d662f725b01515a31639d31512 100644 --- a/meta-openeuler/recipes-containers/k3s/k3s_%.bbappend +++ b/meta-openeuler/recipes-containers/k3s/k3s_%.bbappend @@ -1,30 +1,4 @@ -# enable cgroup v2 -# isulad, embedded, ... -container_runtime_endpoint = "embedded" -PV = "1.22.17-k3s1" -#full_k3s = "false" -MAX_K3S_BINARY_SIZE ?= "61000000" -GO_BUILD_LDFLAGS:append = "" - -# overwrite install_other_endpoint() to use other container_runtime_points -install_other_endpoint() { - # customize install: - # 1. install airgap images - # 2. modifiy k3s-install-agent - # 3. modifiy k3s-agent.service and k3s.service - bbplain "customize your endpoint install script!" - bbwarn "rtest" -} - -do_sizecheck() { - bbplain "checking k3s binary size at WORKDIR, modified from ./scripts/binary_size_check.sh" - SAMPLE="${build_bindir}/k3s" - BIN_SIZE=$(stat -c '%s' ${SAMPLE}) - if [ ${BIN_SIZE} -gt ${MAX_K3S_BINARY_SIZE} ]; then - bbwarn "k3s binary ${SAMPLE} size ${BIN_SIZE} exceeds max accetable size of ${MAX_K3S_BINARY_SIZE} bytes" - else - bbplain "k3s binary ${SAMPLE} size ${BIN_SIZE} is less than max acceptaable size of ${MAX_K3S_BINARY_SIZE} bytes" - fi -} - -addtask do_sizecheck after do_install +full_k3s = "true" +downloaded_airgap = "false" +static_agent = "true" +turnon_upx = "true" \ No newline at end of file diff --git a/meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb b/meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb index 2ce51e556b81924b257ca2bb324376a6c6f94958..bc11ff129da8407ba0be1bbe0b514b5863c2dde4 100644 --- a/meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb +++ b/meta-openeuler/recipes-containers/k3s/k3s_v1.22.17.bb @@ -1,439 +1,16 @@ -SUMMARY = "Production-Grade Container Scheduling and Management" -DESCRIPTION = "Lightweight Kubernetes, intended to be a fully compliant Kubernetes." -HOMEPAGE = "https://k3s.io/" -LICENSE = "Apache-2.0" -S = "${WORKDIR}/${BP}" -LIC_FILES_CHKSUM = "file://${S}/LICENSE;md5=2ee41112a44fe7014dce33e26468ba93" -APPEND += " cgroup_no_v1=all" - -PV = "v1.22.17-k3s1" - -python() { - if d.getVar('TUNE_PKGARCH') == 'aarch64': - d.setVar('ARCH', 'arm64') - elif d.getVar('TUNE_PKGARCH') == 'x86_64': - d.setVar('ARCH', 'amd64') - else: - d.setVar('ARCH', d.getVar('TUNE_PKGARCH')) -} - -#file://0001-Finding-host-local-in-usr-libexec.patch;patchdir=src/import - -do_compile[network] = "1" -SRC_URI = "\ - file://oee_archive/${BP}.tar.gz \ - file://k3s-agent.service \ - file://k3s.service \ - file://k3s-kill-agent \ - file://k3s-killall.sh \ - file://modules.txt \ - file://k3s-install-agent \ - file://install.sh \ - file://k3s-rootless.service \ -" - -#include src_uri.inc - -inherit go -inherit goarch -inherit systemd -inherit features_check - -PACKAGECONFIG = "" -GO_IMPORT = "import" -full_k3s ?= "true" -# runtime_endpoint: isulad => isulad, embedded => k3s-embedded containerd, others -container_runtime_endpoint ?= "isulad" -PKG = "github.com/k3s-io/k3s" -PKG_CONTAINERD="github.com/containerd/containerd" -PKG_K3S_CONTAINERD="github.com/k3s-io/containerd" -PKG_CRICTL="github.com/kubernetes-sigs/cri-tools/pkg" -PKG_K8S_BASE="k8s.io/component-base" -PKG_K8S_CLIENT="k8s.io/client-go/pkg" -PKG_CNI_PLUGINS="github.com/containernetworking/plugins" - -# PKG = "github.com/k3s-io/k3s" -# PKG_CONTAINERD = "github.com/containerd/containerd" -# PKG_K3S_CONTAINERD = "github.com/k3s-io/containerd" -# PKG_CRICTL = "github.com/kubernetes-sigs/cri-tools/pkg" -# PKG_K8S_BASE = "k8s.io/component-base" -# PKG_K8S_CLIENT = "k8s.io/client-go/pkg" -# PKG_CNI_PLUGINS = "github.com/containernetworking/plugins" -COMMIT = "3ed243d" -#COMMIT = "3ed243d" -#COMMIT = "3d82902b" from k3s-io upstream - -# because of the limits of variable expansion in shell-style function, -# set k3s-building flags inside `do_compile` is troublesome -VERSION_FLAGS="\ - -X ${PKG}/pkg/version.Version=${PV} \ - -X ${PKG}/pkg/version.GitCommit=${COMMIT} \ -" - -STATIC = " -extldflags '-static' " -# `-lz` : adding zlib to sysroot -STATIC_SQLITE = " -extldflags '-static -lm -lz -ldl -lpthread' " -K3S_LDFLAGS = " ${VERSION_FLAGS} -w -s" - -# tags -# BASIC = "ctrd,seccomp,no_btrfs,netcgo,osusergo,providerless" -BASIC = "no_btrfs,ctrd,netcgo,osusergo,providerless" -SELINUX = "selinux" -APPARMOR = "apparmor" -STATIC_TAG = "static_build" -SECCOMP = "seccomp" -SQLITE_TAG = "${STATIC_TAG},libsqlite3" -TAGS = "${BASIC}" - -# TODO fix compilation abortion when applied sqlite to k3s server compilation -SERVER_LDFLAGS = "${K3S_LDFLAGS} ${STATIC_SQLITE}" -AGENT_LDFLAGS = "${K3S_LDFLAGS} ${STATIC}" -SERVER_TAGS = "${TAGS}" -AGENT_TAGS = "${TAGS}" - -AGENT_TAGS .= ",${STATIC_TAG},${APPARMOR}" - -k3s_bindir = "${exec_prefix}/bin" -build_dir = "${S}/build" -build_bindir = "${build_dir}/bin/${BPN}" -build_srcdir = "${build_dir}/pkg/mod" -vendor_path = "${S}/vendor" -modules_path = "${S}/modules" -REQUIRED_DISTRO_FEATURES = "seccomp" -DEPENDS += "\ - rsync-native \ - coreutils-native \ - go-native \ - zlib \ -" -export GOPROXY="https://goproxy.cn,direct" - - -do_compile() { - - #build K3S binary - # go module will cache module@ver under the dir: vendor_gopath/pkg/mod - # I use vendor dir under the module mod for convinence, but it is a confussing action - # TODO: module is module, vendor is vendor. - # download path: - # go mod vendor => ${S}/vendor/sites/packages - # GO111MODULE=on => ${S}/vendor/pkg/mod/sites/packages - export CGO_ENABLED="1" - export GO111MODULE=on - export _GOPATH=${GOPATH} - export GOPATH="${modules_path}:${vendor_path}:${STAGING_DIR_TARGET}/${prefix}/lib64/go" - export CGO_CFLAGS="${CFLAGS} --sysroot=${STAGING_DIR_TARGET}" - export CGO_LDFLAGS="${LDFLAGS} --sysroot=${STAGING_DIR_TARGET}" - export CFLAGS="" - export LDFLAGS=" -w -s" - export CC="${CC}" - export LD="${LD}" - export OEE_YOCTO_VERSION="${PV}-oee+yocto" - - cd "${S}" - # mkdir -p "${vendor_path}" - mkdir -p "${modules_path}" - mkdir -p "${build_bindir}" - mkdir -p "${WORKDIR}/build/pkg/mod" - - # if [ ! -e "${vendor_path}" ]; then - # ${GO} mod vendor 2> ${WORKDIR}/temp/k3s_vendor_info - # fi - - # useless building scripts - rm -f "Dockerfile.*" - # GO will find same package in both recipes-sysroot and vendor dir. - # pointing modules/pkg/mod/src to vendor dir - rm -f "${build_bindir}/k3s-agent" - rm -f "${build_bindir}/k3s-server" - rm -f "${build_bindir}/k3s-etcd-snapshot" - rm -f "${build_bindir}/k3s-secrets-encrypt" - rm -f "${build_bindir}/k3s-certificate" - rm -f "${build_bindir}/k3s-completion" - rm -f "${build_bindir}/kubectl" - rm -f "${build_bindir}/crictl" - rm -f "${build_bindir}/containerd" - rm -f "${build_bindir}/containerd-shim" - rm -f "${build_bindir}/containerd-shim-runc-v2" - rm -f "${build_bindir}/runc" - rm -rf "${builds}/data" - - # TODO: fixing linking warnings - # * warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking - # * warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking - # NOTICE In k3s version 1.22 and later, k3s-io has removed the building for the - # standalone k3s-agent binary,resulting in an increase of a little in the binary size. - # both two products are stored in /var/lib/rancher/k3s/data/SomeHash/bin - if [ "${full_k3s}" = "true" ]; then - ${GO} build -mod=readonly -tags "${SERVER_TAGS}" -ldflags "${SERVER_LDFLAGS}" \ - -trimpath -v -o ${build_bindir}/containerd ./cmd/server/main.go \ - 2> ${WORKDIR}/temp/k3s_build_info - else - ${GO} build -mod=readonly -tags "${AGENT_TAGS}" -ldflags "${AGENT_LDFLAGS}" \ - -trimpath -v -o ${build_bindir}/k3s-agent ./cmd/agent/main.go \ - 2> ${WORKDIR}/temp/k3s_build_info - fi - - # make sure cached package tree modificable without 'Permission Denided' - chmod 777 -R "${modules_path}/pkg/mod" - chmod 777 -R "${WORKDIR}/build/pkg/mod" - - # TODO move them into containerd recipe instead. - # build_runc - # build_containerd_shim_runc_v2 - - # ? 如果要构建非 isulad 作runtime的k3s,就需要把 vendor和module混用起来。 - # 1. 不希望在do_compile时重复拉取依赖。 - # 2. 其他组件多数是vendor构建的。 - # bbfatal "stop" - # ${GO} generate - export GO111MODULE=off - # FATAL: - # cmd/k3s/main.go:181:16: undefined: data.AssetNames - # cmd/k3s/main.go:218:23: undefined: data.Asset - # under ${S}, pkg/data/data.go is empty indeed - # ?? 位于${S}下，编译会出现上面的错误 - # 源码是从 FROM=yocto-meta-openeuler/../k3s 获取,复制到${S} - # 在FROM编译，则没有问题； - # 使用sysroot-native的go, host的go,都一样 - # 从${FROM}前往${S}, 失败，从${S}返回到${FROM}, 成功, - # CGO_ENABLED=0 ${GO} build -tags "urfave_cli_no_docs" -ldflags "${K3S_LDFLAGS} ${STATIC}"\ - # -v -o "${PN}-${ARCH}" ./cmd/k3s/main.go 2> ${WORKDIR}/temp/cli_build_info - # 手动加打印也确认了用的是pkg/data/data.go -} - -build_runc() { - RUNC_TAGS="${APPARMOR},${SECCOMP}" - RUNC_STATIC="static" - # go vendor build. - bbplain "[INFO] Building runc" - CGO_ENABLE=0 GO111MODULE=off make GOPATH=${build_dir} EXTRA_LDFLAGS=" -w -s" BUILDTAGS="$RUNC_TAGS" \ - -C "${vendor_path}/github.com/opencontainers/runc" ${RUNC_STATIC} - cp -f "${vendor_path}/github.com/opencontainers/runc/runc" ${build_bindir}/runc -} - -build_containerd_shim() { - bbplain "Building containerd-shim" - rm -r "${build_bindir}/containerd-shim" - cd "${vendor_path}" - CGO_ENABLE=0 GO111MODULE=off make -C ./github.com/containerd/containerd bin/containerd-shim - cp -f ./github.com/containerd/containerd/bin/containerd-shim ${build_bindir}/containerd-shim -} - -build_containerd_shim_runc_v2() { - bbplain "Building containerd-shim-runc-v2" - rm -r "${build_bindir}/containerd-shim-runc-v2" - cd "${modules_path}" - # CGO_ENABLE=0 GO111MODULE=off make -C ./github.com/containerd/containerd bin/containerd-shim-runc-v2 - CGO_ENABLE=0 GO111MODULE=off make -C "./github.com/k3s-io/containerd@*"/cmd/containerd-shim-runc-v2 bin/containerd-shim-runc-v2 - cp -f ./github.com/containerd/containerd/bin/containerd-shim-runc-v2 ${build_bindir}/containerd-shim-runc-v2 -} - -# compress and make a multicall binary (a single binary contains bunches of subprocess) -do_multicall() { - # mkdir -p "${build_dir}/data ${build_dir}/out" - # mkdir -p "${S}/dist/artifacts" - - # tar cvf "${build_dir}/out/data.tar" "${build_dir}" - # zstd --no-progress -T0 -16 -f --long=25 --rm "${build_dir}/out/data.tar.zst" - # HASH=$(sha256sum "${build_dir}/out/data.tar.zst" | awk '{print $1}') - # cp "${build_dir}/out/data.tar.zst" "${build_dir}/data/${HASH}.tar.zst" - - # build multicall k3s binary, this is what will be installed into usr/bin - bbplain "[INFO] multicall built" -} - -addtask do_multicall after do_compile before do_install - - -do_airgap() { - bbfatal "it is super troublesome to build airgap images in the oebuild docker environment, \ - if you wanna build k3s airgap image manually, you should run \ - `bitbake k3s-airgap` in the **native host** outside oebuild docker environment" -} - -# addtask do_airgap after do_multicall before do_install - -do_install() { - - install -d "${D}${k3s_bindir}" - install -d "${D}${sysconfdir}" - install -d "${D}${sysconfdir}/k3s" - install -d "${D}${sysconfdir}/k3s/config" - # airgap images for isulad will installed here - install -d "${D}${sysconfdir}/k3s/tools" - install -d "${D}${sysconfdir}/k3s/tools" - # install dir for airgap images when use embedded containerd as endpoint - install -d "${D}${localstatedir}/lib/rancher/k3s/agent/images" - - - bbplain "container runtime endpoint = ${container_runtime_endpoint}" - bbplain "full_k3s = ${full_k3s}" - - # install binaries - # only if `-k` option is passed, the search for talternatives works after the failure of do_compile - # you should add k3s prebuild binary to SRC_URI - if [ ! -f "${build_bindir}/containerd" ] && \ - [ ! -f "${build_bindir}/k3s" ] && \ - [ -f "${WORKDIR}/k3s"]; then - echo "in ${build_bindir} products does not exist, use a prebuild standard binary as alternative." - if [ -f "${WORKDIR}/k3s-${ARCH}-${PV}" ]; then - mv "${WORKDIR}/k3s-${ARCH}-${PV}" "${build_bindir}/k3s" - else - echo "missing prebuild binary" - exit -1 - fi - fi - - install -m 755 "${WORKDIR}/k3s-kill-agent" "${D}${k3s_bindir}" - # check-config, which is temporaliy useless for oee because currently oee image - # is production-level, does not contain a config file in rootfs - install -m 755 "${S}/contrib/util/check-config.sh" "${D}${k3s_bindir}/check-config" - - # next commit : remove full_k3s, support k3s agent only and make binary smaller - if [ "${full_k3s}" = "true" ]; then - install -m 755 "${build_bindir}/containerd" "${D}${k3s_bindir}" - #TODO crictl conflicts - #ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/crictl" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/ctr" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/kubectl" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-agent" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-server" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-completion" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-certificate" - ln -sr "${D}${k3s_bindir}/containerd" "${D}${k3s_bindir}/k3s-secrets-encrpyt" - # etcd does not well work in some k3s at lower version - #ln -sr "${D}${k3s_bindir}/k3s" "${D}${k3s_bindir}/k3s-etcd-snapshot" - install -m 755 "${WORKDIR}/k3s-killall.sh" "${D}${k3s_bindir}" - else - install -m 755 "${build_bindir}/k3s-agent" "${D}${k3s_bindir}" - fi - install -D -m 755 "${WORKDIR}/install.sh" "${D}${k3s_bindir}/k3s-official-install" - - # install script for tests - install -D -m 755 "${WORKDIR}/k3s-install-agent" "${D}${k3s_bindir}/k3s-install-agent" - - - # install systemd services unit files - systemd_launched=${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)} - if [ "$systemd_launched" = "true" ]; then - unitfile_destdir="${D}${sysconfdir}/systemd/system" - install -d "${unitfile_destdir}" - if [ "${full_k3s}" = "true" ]; then - install -D -m 0644 "${WORKDIR}/k3s.service" "${unitfile_destdir}/k3s.service" - sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${k3s_bindir}/\4#g" "${unitfile_destdir}/k3s.service" - fi - - install -D -m 0644 "${WORKDIR}/k3s-agent.service" "${unitfile_destdir}/k3s-agent.service" - sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${k3s_bindir}/\4#g" "${unitfile_destdir}/k3s-agent.service" - - if [ "${container_runtime_endpoint}" = "isulad" ]; then - sed -i "s#^ExecStart=\(.*\)#ExecStart=\1\n\t --container-runtime-endpoint unix:///var/run/isulad.sock #" "${unitfile_destdir}/k3s-agent.service" - sed -i "s/^Documentation=.*$/&\nRequires=isulad.service /" "${unitfile_destdir}/k3s-agent.service" - sed -i "s/^Requires=isulad.service/&\nAfter=isulad.service/" "${unitfile_destdir}/k3s-agent.service" - elif [ "${container_runtime_endpoint}" != "embedded" ]; then - # customize your endpoint setup functions in k3s_%.bbappend - install_other_endpoint - fi - - else - echo "Systemd-free k3s hasn't implemented!" - install -d "${D}${sysconfdir}/init.d" - install -d "${D}${sysconfdir}/rcS.d" - exit -1 - fi -} - - - -install_other_endpoint() { - bbwarn " \ - Using ${container_runtime_endpoint} as the container runtime endpoint \ - You should overwrite install_more_endpoint function in k3s_%.bbappend \ - to customize your endpoint setup functions \ - " -} - -compress_binaries() { - bbplain "compressing binaries" -} - -FULL_K3S_FILES = "\ - ${k3s_bindir}/crictl \ - ${k3s_bindir}/kubectl \ - ${k3s_bindir}/ctr \ - ${k3s_bindir}/k3s-secrets-encrypt \ - ${k3s_bindir}/k3s-certificate \ - ${k3s_bindir}/k3s-server \ - ${k3s_bindir}/k3s-killall.sh \ - ${unitfile_destdir}/k3s.service \ -" - -FILES:${PN} += "\ - ${k3s_bindir}/k3s-install-agent \ - ${k3s_bindir}/k3s-kill-agent \ - ${unitfile_destdir}/k3s-agent.service \ - ${bb.utils.contains('full_k3s', 'true', '${FULL_K3S_FILES}', '', d)} \ -" - -#${sysconfdir}/systemd/system/k3s-agent.service, -#${bb.utils.contains('full_k3s', 'true', 'k3s.service', '', d)} -SYSTEMD_SERVICE:k3s = " k3s-agent.service \ - k3s.service\ -" +require k3s.inc USE_PREBUILD_SHIM_V2 = "${@bb.utils.contains_any('ARCH', 'arm riscv64', '0', '1', d)}" -RDEPENDS:${PN} = "\ - conntrack-tools \ - coreutils \ - findutils \ - iproute2 \ - iptables \ - ${@bb.utils.contains('USE_PREBUILD_SHIM_V2', '1', 'lib-shim-v2-bin', 'lib-shim-v2', d)} \ +image_list = "\ + docker.io/rancher/klipper-helm:v0.7.3-build20220613 \ + docker.io/rancher/klipper-lb:v0.3.5 \ + docker.io/rancher/local-path-provisioner:v0.0.21 \ + docker.io/rancher/mirrored-coredns-coredns:1.9.1 \ + docker.io/rancher/mirrored-library-busybox:1.34.1 \ + docker.io/rancher/mirrored-library-traefik:2.6.2 \ + docker.io/rancher/mirrored-metrics-server:v0.5.2 \ + docker.io/rancher/mirrored-pause:3.6 \ " -RDEPENDS:${PN} += "\ - kernel-module-br-netfilter \ - kernel-module-bridge \ - kernel-module-iptable-mangle \ - kernel-module-ip6table-mangle \ - kernel-module-libcrc32c \ - kernel-module-nf-conntrack \ - kernel-module-nf-nat \ - kernel-module-nf-defrag-ipv4 \ - kernel-module-nf-defrag-ipv6 \ - kernel-module-stp \ - kernel-module-xt-addrtype \ - kernel-module-xt-comment \ - kernel-module-xt-nat \ - kernel-module-xt-tcpudp \ -" - -RRECOMMENDS:${PN} = "\ - kernel-module-nf-conntrack-netlink \ - kernel-module-nfnetlink-log \ - kernel-module-nfnetlink \ - kernel-module-nft-chain-nat \ - kernel-module-nft-compat \ - kernel-module-nft-counter \ - kernel-module-xt-addrtype \ - kernel-module-xt-comment \ - kernel-module-xt-connmark \ - kernel-module-xt-conntrack \ - kernel-module-xt-limit \ - kernel-module-xt-mark \ - kernel-module-xt-masquerade \ - kernel-module-xt-multiport \ - kernel-module-xt-nflog \ - kernel-module-xt-physdev \ - kernel-module-xt-nat \ - kernel-module-xt-statistic \ - kernel-module-xt-conntrack \ - kernel-module-xt-statistic \ - kernel-module-xt-physdev \ - kernel-module-vxlan \ -" -INHIBIT_PACKAGE_STRIP = "1" -INSANE_SKIP:${PN} += "ldflags already-stripped" diff --git a/meta-openeuler/recipes-core/images/openeuler-image.bb b/meta-openeuler/recipes-core/images/openeuler-image.bb index 7ec2a6679a10d126df0fce7f07e7eb3273b58f87..31a64f82bc3ff99e97bfea34cbe3d6b90093dbbe 100644 --- a/meta-openeuler/recipes-core/images/openeuler-image.bb +++ b/meta-openeuler/recipes-core/images/openeuler-image.bb @@ -25,7 +25,7 @@ ${@bb.utils.contains("DISTRO_FEATURES", "mcs", "packagegroup-mcs", "",d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "ros", "packagegroup-ros", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "hmi", "packagegroup-hmi", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "kubeedge isulad", "packagegroup-kubeedge", "", d)} \ -${@bb.utils.contains("DISTRO_FEATURES", "k3s isulad", "k3s packagegroup-k3s", "", d)} \ +${@bb.utils.contains("DISTRO_FEATURES", "k3s-node", "packagegroup-k3s-node", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "isulad", "packagegroup-isulad", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "preempt-rt", "packagegroup-preempt-rt", "", d)} \ ${@bb.utils.contains("DISTRO_FEATURES", "dsoftbus", "packagegroup-dsoftbus", "", d)} \ diff --git a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb b/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb index 80f7b06b6e4b31ae094b77a6e6e7529c1cbc2990..285b0717af9cc0dc6e53c23040205ea59f7ef92b 100644 --- a/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb +++ b/meta-openeuler/recipes-core/packagegroups/packagegroup-k3s.bb @@ -5,16 +5,30 @@ PACKAGE_ARCH = "${MACHINE_ARCH}" PR = "r1" inherit packagegroup +inherit features_check REQUIRED_DISTRO_FEATURES ?= "seccomp" -PACKAGES = " \ - ${PN} \ +CRI ?= "isulad" + +PACKAGES = " \ + packagegroup-k3s \ + packagegroup-k3s-host \ + packagegroup-k3s-node \ +" + + +RDEPENDS:packagegroup-k3s-base = " \ + packagegroup-${CRI} \ + cni \ " -RDEPENDS:${PN} = " \ -packagegroup-isulad \ -cni \ -cri-tools \ +RDEPENDS:packagegroup-k3s-host = " \ + packagegroup-k3s \ + k3s-agent \ " +RDEPENDS:packagegroup-k3s-node = " \ + packagegroup-k3s \ + k3s-server \ +" diff --git a/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s-arm64.cfg b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s-arm64.cfg new file mode 100644 index 0000000000000000000000000000000000000000..e7cbe774408ecd008073ba1169526ac2f9639741 --- /dev/null +++ b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s-arm64.cfg @@ -0,0 +1,35 @@ +# minimal extra cfg set for k3s +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_VETH=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK_MARK=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_CONNMARK=m +CONFIG_INET_ESP=y +CONFIG_CGROUP_NET_PRIO=m +CONFIG_IP_NF_TARGET_MASQUERADE=y +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_VXLAN=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_EXT4_FS_POSIX_ACL=m +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_IP_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_NFCT=m +CONFIG_IP_VS_PROTO_TCP=m +CONFIG_IP_VS_PROTO_UDP=m +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_CMDLINE="cgroup_no_v1=all" diff --git a/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.cfg b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.cfg index 687f5cdb8b8c626c19499a77319fd8bd92d4aaac..398a1f42c8f15bb0c604c3663bb8eb61c77b2de6 100644 --- a/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.cfg +++ b/meta-openeuler/recipes-kernel/linux/files/meta-data/features/k3s/k3s.cfg @@ -1,91 +1,34 @@ -# -# k3s runtime related kernel config -# -CONFIG_PSI=y -CONFIG_IP_NF_NAT=m -CONFIG_PSI_CGROUP_V1=y -CONFIG_TASKS_TRACE_RCU=y -CONFIG_MEMCG_MEMFS_INFO=y -CONFIG_CGROUP_V1_WRITEBACK=y -CONFIG_QOS_SCHED=y -CONFIG_QOS_SCHED_MULTILEVEL=y -CONFIG_QOS_SCHED_SMT_EXPELLER=y -CONFIG_QOS_SCHED_PRIO_LB=y -CONFIG_QOS_SCHED_SMART_GRID=y +# minimal extra cfg set for k3s CONFIG_CGROUP_PIDS=y -CONFIG_CGROUP_RDMA=y CONFIG_CGROUP_FREEZER=y -CONFIG_CGROUP_BPF=y -CONFIG_SOCK_CGROUP_DATA=y -CONFIG_CGROUP_FILES=y -CONFIG_TIME_NS=y -CONFIG_SCHED_STEAL=y -CONFIG_BPF_SYSCALL=y -CONFIG_SCHED_MC=y -CONFIG_SCHED_CLUSTER=y -CONFIG_SCHED_SMT=y -CONFIG_MEMCG_SWAP_QOS=y -CONFIG_XFRM_USER=m +CONFIG_VETH=m +CONFIG_BRIDGE=m CONFIG_BRIDGE_NETFILTER=m -CONFIG_NETFILTER_NETLINK=m -CONFIG_NETFILTER_FAMILY_BRIDGE=y -CONFIG_NETFILTER_NETLINK_LOG=m -CONFIG_NF_CONNTRACK_MARK=y -CONFIG_NF_CONNTRACK_EVENTS=y -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CT_NETLINK=m -CONFIG_NETFILTER_NETLINK_GLUE_CT=y -CONFIG_NF_NAT_FTP=m -CONFIG_NF_NAT_TFTP=m -CONFIG_NF_NAT_MASQUERADE=y -CONFIG_NF_TABLES=m -CONFIG_NFT_COUNTER=m -CONFIG_NFT_NAT=m -CONFIG_NFT_COMPAT=m -CONFIG_NETFILTER_XT_MARK=m -CONFIG_NETFILTER_XT_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NF_CONNTRACK_MARK=m CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m CONFIG_NETFILTER_XT_MATCH_IPVS=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m -CONFIG_NETFILTER_XT_MATCH_RECENT=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_IP_VS=m -CONFIG_IP_VS_RR=m -CONFIG_IP_VS_WRR=m -CONFIG_IP_VS_SH=m +CONFIG_NETFILTER_XT_CONNMARK=m +CONFIG_INET_ESP=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_IP_NF_TARGET_MASQUERADE=y CONFIG_IP_NF_MANGLE=m CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_IP_NF_TARGET_MASQUERADE=m -CONFIG_IP_NF_TARGET_REDIRECT=m CONFIG_IP6_NF_MANGLE=m -CONFIG_NF_TABLES_IPV4=y -CONFIG_NF_TABLES_IPV6=y -CONFIG_MRP=m -CONFIG_BRIDGE=m -CONFIG_BRIDGE_IGMP_SNOOPING=y -CONFIG_BRIDGE_VLAN_FILTERING=y -CONFIG_VLAN_8021Q_MVRP=y -CONFIG_NETLINK_DIAG=m -CONFIG_NET_L3_MASTER_DEV=y -CONFIG_DUMMY=m -CONFIG_IPVLAN_L3S=y -CONFIG_IPVLAN=m CONFIG_VXLAN=m -CONFIG_VETH=m -CONFIG_SCHEDSTATS=y -CONFIG_BPF_EVENTS=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_EXT4_FS_POSIX_ACL=m CONFIG_BLK_DEV_THROTTLING=y -CONFIG_NET_CLS_CGROUP=m -CONFIG_CGRUOP_NET_PRIO=y -CONFIG_IP_VS_PROTO_TCP=y -CONFIG_IP_VS_PROTO_UDP=y -CONFIG_IP_VS_NFCT=y -CONFIG_INET_XFRM_MODE_TRANSPORT=y +CONFIG_IP_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_NFCT=m +CONFIG_IP_VS_PROTO_TCP=m +CONFIG_IP_VS_PROTO_UDP=m +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_CMDLINE="cgroup_no_v1=all"