From 0b5c7dc22cfcefbaff549e4984eb8fbd9789d3c0 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Sun, 25 Feb 2024 11:11:22 +0800 Subject: [PATCH] build-extensions-container: add command to build the extensions container [upstream] 1f34f5d18662f56dcdcc0cd6e90acfa60f1dc258 642ccd309b57e7468e478b45019d022b3cacb5ef --- cmd/build-extensions-container.go | 214 ++++++++++++++++++++++++++++++ cmd/coreos-assembler.go | 4 +- internal/pkg/cosash/cosash.go | 12 +- src/build-extensions-container.sh | 39 ++++++ src/cosalib/builds.py | 2 +- src/vmdeps.txt | 2 + 6 files changed, 265 insertions(+), 8 deletions(-) create mode 100644 cmd/build-extensions-container.go create mode 100755 src/build-extensions-container.sh diff --git a/cmd/build-extensions-container.go b/cmd/build-extensions-container.go new file mode 100644 index 00000000..e0e938e3 --- /dev/null +++ b/cmd/build-extensions-container.go @@ -0,0 +1,214 @@ +package main + +import ( + "fmt" + "os/exec" + + cosamodel "github.com/coreos/coreos-assembler/internal/pkg/cosa" + "github.com/coreos/coreos-assembler/internal/pkg/cosash" + cosa "github.com/coreos/coreos-assembler/pkg/builds" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" + + "crypto/sha256" + "encoding/json" + "io" + "os" + "path/filepath" + "time" +) + +// hotfix is an element in hotfixes.yaml which is a repo-locked RPM set. +type hotfix struct { + // URL for associated bug + Link string `json:"link"` + // The operating system major version (e.g. 8 or 9) + OsMajor string `json:"osmajor"` + // Repo used to download packages + Repo string `json:"repo"` + // Names of associated packages + Packages []string `json:"packages"` +} + +type hotfixData struct { + Hotfixes []hotfix `json:"hotfixes"` +} + +// downloadHotfixes basically just accepts as input a declarative JSON file +// format describing hotfixes, which are repo-locked RPM packages we want to download +// but without any dependencies. +func downloadHotfixes(srcdir, configpath, destdir string) error { + contents, err := os.ReadFile(configpath) + if err != nil { + return err + } + + var h hotfixData + if err := yaml.Unmarshal(contents, &h); err != nil { + return fmt.Errorf("failed to deserialize hotfixes: %w", err) + } + + fmt.Println("Downloading hotfixes") + + for _, fix := range h.Hotfixes { + fmt.Printf("Downloading content for hotfix: %s\n", fix.Link) + // Only enable the repos required for download + reposdir := filepath.Join(srcdir, "yumrepos") + argv := []string{"--disablerepo=*", fmt.Sprintf("--enablerepo=%s", fix.Repo), "--setopt=reposdir=" + reposdir, "download"} + argv = append(argv, fix.Packages...) + cmd := exec.Command("dnf", argv...) + cmd.Dir = destdir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to invoke dnf download: %w", err) + } + } + + serializedHotfixes, err := json.Marshal(h) + if err != nil { + return err + } + err = os.WriteFile(filepath.Join(destdir, "hotfixes.json"), serializedHotfixes, 0o644) + if err != nil { + return err + } + + return nil +} + +func generateHotfixes() (string, error) { + hotfixesTmpdir, err := os.MkdirTemp("", "") + if err != nil { + return "", err + } + defer os.RemoveAll(hotfixesTmpdir) + + variant, err := cosamodel.GetVariant() + if err != nil { + return "", err + } + + wd, err := os.Getwd() + if err != nil { + return "", err + } + + srcdir := filepath.Join(wd, "src") + p := fmt.Sprintf("%s/config/hotfixes-%s.yaml", srcdir, variant) + if _, err := os.Stat(p); err == nil { + err := downloadHotfixes(srcdir, p, hotfixesTmpdir) + if err != nil { + return "", fmt.Errorf("failed to download hotfixes: %w", err) + } + } else { + fmt.Printf("No %s found\n", p) + } + + out := filepath.Join(wd, "tmp/hotfixes.tar") + + // Serialize the hotfix RPMs into a tarball which we can pass via a virtio + // device to the qemu process. + cmd := exec.Command("tar", "-c", "-C", hotfixesTmpdir, "-f", out, ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + return "", err + } + + return out, nil +} + +func buildExtensionContainer() error { + cosaBuild, buildPath, err := cosa.ReadBuild("builds", "", "") + if err != nil { + return err + } + buildID := cosaBuild.BuildID + fmt.Printf("Generating extensions container for build: %s\n", buildID) + + hotfixPath, err := generateHotfixes() + if err != nil { + return fmt.Errorf("generating hotfixes failed: %w", err) + } + + arch := cosa.BuilderArch() + sh, err := cosash.NewCosaSh() + if err != nil { + return err + } + if _, err := sh.PrepareBuild("extensions-container"); err != nil { + return errors.Wrapf(err, "calling prepare_build") + } + targetname := cosaBuild.Name + "-" + buildID + "-extensions-container" + "." + arch + ".ociarchive" + process := "runvm -chardev \"file,id=ociarchiveout,path=${tmp_builddir}/\"" + targetname + + " -device \"virtserialport,chardev=ociarchiveout,name=ociarchiveout\"" + + " -drive file=" + hotfixPath + ",if=none,id=hotfixes,format=raw,media=disk,read-only=on" + + " -device virtio-blk,serial=hotfixes,drive=hotfixes" + + " -- /usr/lib/coreos-assembler/build-extensions-container.sh " + arch + + " /dev/virtio-ports/ociarchiveout " + buildID + if err := sh.Process(process); err != nil { + return errors.Wrapf(err, "calling build-extensions-container.sh") + } + // Find the temporary directory allocated by the shell process, and put the OCI archive in its final place + tmpdir, err := sh.ProcessWithReply("echo $tmp_builddir>&3\n") + if err != nil { + return errors.Wrapf(err, "querying tmpdir") + } + targetPath := filepath.Join(buildPath, targetname) + if err := exec.Command("/usr/lib/coreos-assembler/finalize-artifact", filepath.Join(tmpdir, targetname), targetPath).Run(); err != nil { + return errors.Wrapf(err, "finalizing artifact") + } + + fmt.Printf("Built %s\n", targetPath) + + // Gather metadata of the OCI archive (sha256, size) + file, err := os.Open(targetPath) + if err != nil { + return errors.Wrapf(err, "opening %s", targetPath) + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return errors.Wrapf(err, "hashing %s", targetPath) + } + stat, err := file.Stat() + if err != nil { + return errors.Wrapf(err, "stat(%s)", targetPath) + } + sha256sum := fmt.Sprintf("%x", hash.Sum(nil)) + + cosaBuild.BuildArtifacts.ExtensionsContainer = &cosa.Artifact{ + Path: targetname, + Sha256: sha256sum, + SizeInBytes: float64(stat.Size()), + SkipCompression: true, + } + cosaBuild.MetaStamp = float64(time.Now().UnixNano()) + + newBytes, err := json.MarshalIndent(cosaBuild, "", " ") + if err != nil { + return err + } + extensions_container_meta_path := filepath.Join(buildPath, "meta.extensions-container.json") + err = os.WriteFile(extensions_container_meta_path, newBytes, 0644) + if err != nil { + return errors.Wrapf(err, "writing %s", extensions_container_meta_path) + } + defer os.Remove(extensions_container_meta_path) + workdir, err := filepath.Abs(".") + if err != nil { + return err + } + abs_new_json, err := filepath.Abs(extensions_container_meta_path) + if err != nil { + return err + } + // Calling `cosa meta` as it locks the file and we need to make sure no other process writes to the file at the same time. + // Golang does not appear to have a public api to lock files at the moment. https://github.com/coreos/coreos-assembler/issues/3149 + if err := exec.Command("cosa", "meta", "--workdir", workdir, "--build", buildID, "--artifact-json", abs_new_json).Run(); err != nil { + return errors.Wrapf(err, "calling `cosa meta`") + } + return nil +} diff --git a/cmd/coreos-assembler.go b/cmd/coreos-assembler.go index 62004dcc..4f981f55 100644 --- a/cmd/coreos-assembler.go +++ b/cmd/coreos-assembler.go @@ -15,7 +15,7 @@ import ( var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean", "list"} var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container", "upload-oscontainer"} var buildextendCommands = []string{"aliyun", "aws", "azure", "digitalocean", "exoscale", "gcp", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"} -var utilityCommands = []string{"aws-replicate", "compress", "generate-hashlist", "koji-upload", "kola", "remote-build-container", "remote-prune", "sign", "tag"} +var utilityCommands = []string{"aws-replicate", "build-extensions-container", "compress", "generate-hashlist", "koji-upload", "kola", "remote-build-container", "remote-prune", "sign", "tag"} var otherCommands = []string{"shell", "meta"} func init() { @@ -74,6 +74,8 @@ func run(argv []string) error { switch cmd { case "clean": return runClean(argv) + case "build-extensions-container": + return buildExtensionContainer() } target := fmt.Sprintf("/usr/lib/coreos-assembler/cmd-%s", cmd) diff --git a/internal/pkg/cosash/cosash.go b/internal/pkg/cosash/cosash.go index 6e64bb75..abde61b5 100644 --- a/internal/pkg/cosash/cosash.go +++ b/internal/pkg/cosash/cosash.go @@ -117,7 +117,7 @@ func NewCosaSh() (*CosaSh, error) { }() // Initialize the internal library - err = r.process(fmt.Sprintf("%s\n. /usr/lib/coreos-assembler/cmdlib.sh\n", bashexec.StrictMode)) + err = r.Process(fmt.Sprintf("%s\n. /usr/lib/coreos-assembler/cmdlib.sh\n", bashexec.StrictMode)) if err != nil { return nil, fmt.Errorf("failed to init cosash: %w", err) } @@ -126,7 +126,7 @@ func NewCosaSh() (*CosaSh, error) { } // write sends content to the shell's stdin, synchronously wait for the reply -func (r *CosaSh) processWithReply(buf string) (string, error) { +func (r *CosaSh) ProcessWithReply(buf string) (string, error) { // Inject code which writes the serial reply prefix cmd := fmt.Sprintf("echo -n \"%d \" >&3\n", r.ackserial) if _, err := io.WriteString(r.input, cmd); err != nil { @@ -146,9 +146,9 @@ func (r *CosaSh) processWithReply(buf string) (string, error) { } } -func (sh *CosaSh) process(buf string) error { +func (sh *CosaSh) Process(buf string) error { buf = fmt.Sprintf("%s\necho OK >&3\n", buf) - r, err := sh.processWithReply(buf) + r, err := sh.ProcessWithReply(buf) if err != nil { return err } @@ -160,14 +160,14 @@ func (sh *CosaSh) process(buf string) error { // PrepareBuild prepares for a build, returning the newly allocated build directory func (sh *CosaSh) PrepareBuild() (string, error) { - return sh.processWithReply(`prepare_build + return sh.ProcessWithReply(`prepare_build pwd >&3 `) } // HasPrivileges checks if we can use sudo func (sh *CosaSh) HasPrivileges() (bool, error) { - r, err := sh.processWithReply(` + r, err := sh.ProcessWithReply(` if has_privileges; then echo true >&3 else diff --git a/src/build-extensions-container.sh b/src/build-extensions-container.sh new file mode 100755 index 00000000..621600c5 --- /dev/null +++ b/src/build-extensions-container.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euo pipefail + +# Used by cmd/build-extensions-container.go. Runs via `runvm` via `cosash.go`. + +arch=$1; shift +filename=$1; shift +buildid=$1; shift + +workdir=$PWD +builddir="${workdir}/builds/latest/${arch}" +ostree_ociarchive=$(ls "${builddir}"/*-ostree*.ociarchive) + +ctx_dir=$(mktemp -d -p /var/tmp) +cp -aLT "${workdir}/src/config" "${ctx_dir}" + +if [ -d "${workdir}/src/yumrepos" ]; then + find "${workdir}/src/yumrepos/" -maxdepth 1 -type f -name '*.repo' -exec cp -t "${ctx_dir}" {} + +fi + +variant="" +if [[ -f "${workdir}/src/config.json" ]]; then + variant="$(jq --raw-output '."coreos-assembler.config-variant"' "${workdir}/src/config.json")" +fi + +mkdir "${ctx_dir}/hotfixes" +tar -xC "${ctx_dir}/hotfixes" -f /dev/disk/by-id/virtio-hotfixes + +# Build the image, replacing the FROM directive with the local image we have. +# The `variant` variable is explicitely unquoted to be skipped when empty. +# Mount in /etc/pki/ca-trust to match the CA roots used by the rest of cosa. +img=localhost/extensions-container +(set -x; podman build --from oci-archive:"$ostree_ociarchive" --network=host \ + --build-arg COSA=true --build-arg VARIANT="${variant}" --label version="$buildid" \ + --volume /etc/pki/ca-trust:/etc/pki/ca-trust:ro \ + -t "${img}" -f extensions/Dockerfile "${ctx_dir}") + +# Call skopeo to export it from the container storage to an oci-archive. +(set -x; skopeo copy "containers-storage:${img}" oci-archive:"$filename") diff --git a/src/cosalib/builds.py b/src/cosalib/builds.py index fd857a8e..3a437d9c 100644 --- a/src/cosalib/builds.py +++ b/src/cosalib/builds.py @@ -134,7 +134,7 @@ class Builds: # pragma: nocover with open(metapath) as f: previous_buildmeta = json.load(f) previous_commit = previous_buildmeta['ostree-commit'] - previous_image_genver = int(previous_buildmeta[genver_key]) + previous_image_genver = int(previous_buildmeta.get(genver_key, 0)) if previous_commit == ostree_commit: image_genver = previous_image_genver + 1 buildid = f"{version}-{image_genver}" diff --git a/src/vmdeps.txt b/src/vmdeps.txt index 50049fe0..0191a871 100644 --- a/src/vmdeps.txt +++ b/src/vmdeps.txt @@ -27,3 +27,5 @@ gdisk xfsprogs e2fsprogs dosfstools btrfs-progs # needed for basic CA support ca-certificates +# needed for extensions container build +podman -- Gitee