From b09cb8b598e2869691145059f070878a864dfcd1 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Fri, 12 Jul 2024 15:23:27 +0800 Subject: [PATCH] sync auxiliary build related code from upstream v0.16.0 [upstream] cmd-buildextend-extensions: a361f252b cmd-buildextend-extensions: fix import_ostree_commit arg 5c708b42e buildextend-extensions: Copy repofiles to tmpdir 509b60ff8 buildextend-extensions: Clean up temporary workdir when done cmd-build-fast: c1aa145ec build-fast: Use target/ if available 34c2ce55d build-fast: Also propagate `ostree.bootable` cmd-buildfetch: 52774c1da Stream GET request for HttpFetcher download, and write in 30 MiB chunks, or declared chunk encoding to reduce RAM usage bbbcd35b6 cmd-buildfetch: also symlink `builds/latest` if build is latest 2c8db986c buildfetch: Add file parameter 8b5bec7c8 src/cmd-buildfetch: dedupe --artifact list 68645661b cmd-buildfetch: add --find-build-for-arch option 1c970d6c3 cmd-buildfetch: bring the ostree ref delete into the loop 1ac86fb63 cmd-buildfetch: continue loop if arch missing 9d55a20a1 cmd-buildfetch: add --aws-config-file option cmd-buildupload: 9214b9bb5 cmd-buildupload: add s3 --aws-config-file option 5acca78a9 buildupload: add support for --arch option b0312359c buildupload: scrub skipped missing images from meta.json cmd-remote-build-container: e716a60c9 cmd-remote-build-container: enhance tag existence check 7826c5ca3 cmd-remote-build-container: query single tag instead of all tags fd58cb668 remote-build-container: add remote for origin f8f76718d src/cmd-remote-build-container: use os.environ consistently 2702c68a7 src/cmd-remote-build-container: add --from 754992190 src/cmd-remote-build-container: use standard labels for git info 7b3739aff src/cmd-remote-build-container: add git repo info as build labels 67675031b src/cmd-remote-build-container: support not pushing to a registry c8a1e04b4 src/cmd-remote-build-container: add --cache-ttl parameter 5769026fa src/cmd-remote-build-container: support building from subdirectories in git repos 4960f0f16 src/cmd-remote-build-container: update comment about building from commit fea3a3c06 src/cmd-remote-build-container: fix quay label in example 4a109ebd4 src/cmd-remote-build-container: force no caching for builds efbe4e091 src/cmd-remote-build-container: support building from commit e2aaa1802 src/cmd-remote-build-container: add --auth option 350ff98ba src/cmd-remote-build-container: remote unneeded `--remote` options 18691e6d5 src/cmd-remote-build-container: appease flake8 b7f39c823 cosalib: further consolidate run() functions 3942ea4bf src/cmd-remote-build-container: fix git short-hash determination 59a912eed src/cmd-remote-build-container: mark file as executable 3604abb19 src/cmd-remote-build-container: use tenacity for retrying registry check 605c5fea1 src/cmd-remote-build-container: add and use logging package 2a1121d28 cosalib: enhance run_cmd; rename to runcmd 1bc46cef8 src/cmd-remote-build-container: fixup some spacing; add comments/logs a5dbf8ef5 src/cmd-remote-build-container: check podman storage too 2888f8c6a src/cmd-remote-build-container: check registry up front a5403b303 src/cmd-remote-build-container: convert some raise statements 7cb91d45f src/cmd-remote-build-container: rename some functions 49d26beef src/cmd-remote-build-container: re-arrange tag var usage ccd07d30d src/cmd-remote-build-container: move some code to main() a15d3a093 cosalib: Add utils lib 7d74b2555 Add cmd-remote-build-container --- src/cmd-build-fast | 14 +- src/cmd-buildextend-extensions | 18 ++- src/cmd-buildfetch | 84 ++++++++--- src/cmd-buildupload | 27 +++- src/cmd-remote-build-container | 251 +++++++++++++++++++++++++++++++++ 5 files changed, 361 insertions(+), 33 deletions(-) create mode 100755 src/cmd-remote-build-container diff --git a/src/cmd-build-fast b/src/cmd-build-fast index c07d3072..2cc8fcbc 100755 --- a/src/cmd-build-fast +++ b/src/cmd-build-fast @@ -90,16 +90,22 @@ echo "Basing on previous build: ${previous_build:-none}" if [ -n "${projectdir}" ]; then cd "${projectdir}" - rm _install -rf + instroot=_install + # Rust projects will already have a handy directory for build artifacts + if test -d target; then + instroot=target/cosa-build-fast + fi + instroot=$(pwd)/${instroot} # Must be absolute for `make` + rm "${instroot}" -rf make - make install DESTDIR="$(pwd)/_install" + make install DESTDIR="${instroot}" fastref=cosa/fastbuild/"$(basename "${projectdir}")" version="$(git describe --tags --abbrev=10)" if ! git diff --quiet; then version="${version}+dirty" fi outdir=${projectdir}/.cosa - rootfsoverrides="${projectdir}/_install" + rootfsoverrides="${instroot}" else fastref=cosa/fastbuild/${name} version="$(date +"%s")" @@ -133,7 +139,7 @@ fi if ! ostree --repo="${tmprepo}" commit -b "${fastref}" --base="${previous_commit}" --tree=dir="${rootfsoverrides}" \ --owner-uid 0 --owner-gid 0 --selinux-policy-from-base --link-checkout-speedup --no-bindings --no-xattrs \ --add-metadata-string=version="${version}" --parent="${previous_commit}" --keep-metadata='coreos-assembler.basearch' \ - --keep-metadata='fedora-coreos.stream' --fsync=0 "${commit_args[@]}"; then + --keep-metadata='nestos.stream' --keep-metadata='ostree.bootable' --fsync=0 "${commit_args[@]}"; then restore_etc exit 1 fi diff --git a/src/cmd-buildextend-extensions b/src/cmd-buildextend-extensions index 4201acd0..8764a1fb 100755 --- a/src/cmd-buildextend-extensions +++ b/src/cmd-buildextend-extensions @@ -44,7 +44,7 @@ def main(): raise Exception(f"Missing {extensions_src}") commit = buildmeta['ostree-commit'] - cmdlib.import_ostree_commit('tmp/repo', builddir, buildmeta) + cmdlib.import_ostree_commit(workdir, builddir, buildmeta) tmpworkdir = prepare_tmpworkdir() changed = run_rpmostree(tmpworkdir, commit, treefile_src, extensions_src) @@ -73,6 +73,8 @@ def main(): shutil.move(extensions_tarball, builddir) buildmeta.write(artifact_name='extensions') + shutil.rmtree(tmpworkdir) + def parse_args(): parser = argparse.ArgumentParser() @@ -86,13 +88,25 @@ def prepare_tmpworkdir(): if os.path.exists(tmpworkdir): shutil.rmtree(tmpworkdir) os.mkdir(tmpworkdir) + configdir = 'src/config' + for f in os.listdir(configdir): + if os.path.isfile(f"{configdir}/{f}") and f.endswith('.repo'): + shutil.copyfile(f"{configdir}/{f}", f"{tmpworkdir}/{f}") + yumreposdir = 'src/yumrepos' + if os.path.exists(yumreposdir): + for f in os.listdir(yumreposdir): + if os.path.isfile(f"{yumreposdir}/{f}") and f.endswith('.repo'): + shutil.copyfile(f"{yumreposdir}/{f}", f"{tmpworkdir}/{f}") return tmpworkdir def run_rpmostree(workdir, commit, treefile, extensions): cmdlib.cmdlib_sh(f''' + cat > "{workdir}/manifest-override.yaml" < 0 and arch not in args.arch: + print(f"Skipping upload of arch {arch} upon user request") + continue s3_upload_build(s3_client, args, builds.get_build_dir(args.build, arch), bucket, f'{prefix}/{args.build}/{arch}') # if there's anything else in the build dir, just upload it too, - # e.g. pipelines might inject additional metadata + # e.g. release metadata is stored at this level for f in os.listdir(f'builds/{args.build}'): # arches already uploaded higher up if f in builds.get_build_arches(args.build): @@ -103,6 +112,7 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix): # Upload images with special handling for gzipped data. uploaded = set() + scrub = set() for imgname in build['images']: img = build['images'][imgname] bn = img['path'] @@ -121,15 +131,18 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix): s3_path = f'{prefix}/{nogz}' set_content_disposition = True + s3_target_exists = s3_check_exists(s3_client, bucket, s3_path, args.dry_run) + # Mark artifacts that we don't want to upload as already uploaded - # so they'll get ignored later. + # so they'll get ignored later. If they're not in S3, delete them + # so that meta.json doesn't reference non-existent objects. if len(args.artifact) > 0 and imgname not in args.artifact: print(f"Skipping upload of artifact {bn} upon user request") uploaded.add(bn) + if not s3_target_exists: + scrub.add(imgname) continue - s3_target_exists = s3_check_exists(s3_client, bucket, s3_path, args.dry_run) - if not os.path.exists(path) and not s3_target_exists: raise Exception(f"{path} not found locally or in the s3 destination!") @@ -163,8 +176,10 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix): dry_run=args.dry_run) # Now upload a modified version of the meta.json which has the fixed - # filenames without the .gz suffixes. We don't want to modify the local - # build dir. + # filenames without the .gz suffixes and the scrubbed artifacts. We don't + # want to modify the local build dir. + for imgname in scrub: + del build['images'][imgname] with tempfile.NamedTemporaryFile('w') as f: json.dump(build, f, indent=4) f.flush() diff --git a/src/cmd-remote-build-container b/src/cmd-remote-build-container new file mode 100755 index 00000000..34ffecba --- /dev/null +++ b/src/cmd-remote-build-container @@ -0,0 +1,251 @@ +#!/usr/bin/python3 -u + +import argparse +import logging +import os +import subprocess +import sys +import tempfile +import tenacity + +from cosalib.cmdlib import runcmd + +# Set up logging +logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(levelname)s - %(message)s") + + +def build_container_image(labels, buildDir, fromimage, cacheTTL, repo, tag): + ''' + Build the image using podman remote and push to the registry + @param labels list labels to add to image + @param buildDir str the location of the directory to build from + @param fromimage str value to pass to `podman build --from=` + @param cacheTTL str value to pass to `podman build --cache-ttl=` + @param repo str registry repository + @param tag str image tag + ''' + cmd = ["podman", "build", f"--cache-ttl={cacheTTL}", f"--tag={repo}:{tag}", buildDir] + for label in labels: + cmd.extend([f"--label={label}"]) + if fromimage: + cmd.extend([f"--from={fromimage}"]) + # Long running command. Send output to stdout for logging + runcmd(cmd) + + +def push_container_image(repo, tag): + ''' + Push image to registry + @param repo str registry repository + @param tag str image tag + ''' + cmd = ["podman", "push", f"{repo}:{tag}"] + # Long running command. Send output to stdout for logging + runcmd(cmd) + # Quay seems to take some time to publish images in some occasions. + # After the push let's wait for it to show up in the registry + # before moving on. + retryer = tenacity.Retrying( + stop=tenacity.stop_after_delay(600), + wait=tenacity.wait_fixed(15), + retry=tenacity.retry_if_result(lambda x: x is False), + before_sleep=tenacity.before_sleep_log(logging, logging.INFO)) + try: + in_repo = retryer(is_tag_in_registry, repo, tag) + except tenacity.RetryError: + in_repo = False + if in_repo: + print(f"Build and Push done successfully via tag: {tag}") + else: + raise Exception(f"Image pushed but not viewable in registry: tag: {tag}") + + +def pull_oci_archive_from_remote(repo, tag, file): + ''' + Retrieve the oci archive of the image and write it to a file + @param repo str registry repository (used to deduce image name) + @param tag str image tag (used to deduce image name) + @param file str The name of the file to write the image to + ''' + cmd = ["podman", "image", "save", + "--format=oci-archive", f"--output={file}", f"{repo}:{tag}"] + # Long running command. Send output to stdout for logging + runcmd(cmd) + + +def is_tag_in_podman_storage(repo, tag): + ''' + Search for a tag in the local podman storage + @param repo str registry repository + @param tag str image tag + ''' + cmd = ["podman", "image", "exists", f"{repo}:{tag}"] + return runcmd(cmd, check=False, capture_output=True).returncode == 0 + + +def is_tag_in_registry(repo, tag): + ''' + Search for a tag in the registry + @param repo str registry repository + @param tag str image tag + ''' + # Podman remote doesn't allow push using digestfile. That's why the tag check is done + # We're not using runcmd here because it's unnecessarily noisy since we + # expect failure in some cases. + cmd = ["skopeo", "inspect", "--raw", f"docker://{repo}:{tag}"] + try: + subprocess.check_output(cmd, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + # yuck; check if it's because the tag doesn't exist. This + # handles two different kinds of failure: + # $ skopeo inspect --raw # docker://quay.io/coreos-assembler/staging:aarch64-706fa53 + # FATA[0000] Error parsing image name "docker://quay.io/coreos-assembler/staging:aarch64-706fa53": reading manifest aarch64-706fa53 in quay.io/coreos-assembler/staging: manifest unknown + # $ skopeo inspect --raw docker://quay.io/coreos-assembler/staging:aarch64-706fa52 + # FATA[0000] Error parsing image name "docker://quay.io/coreos-assembler/staging:aarch64-706fa52": reading manifest aarch64-706fa52 in quay.io/coreos-assembler/staging: unknown: Tag aarch64-706fa52 was deleted or has expired. To pull, revive via time machine + if b'manifest' in e.stderr and b'unknown' in e.stderr: + return False + # any other error is unexpected; fail + logging.error(f" STDOUT: {e.stdout.decode()}") + logging.error(f" STDERR: {e.stderr.decode()}") + raise e + return True + + +def main(): + # Arguments + args = parse_args() + # Set the REGISTRY_AUTH_FILE env var if user passed --authfile + if args.authfile: + os.environ["REGISTRY_AUTH_FILE"] = args.authfile + # Check for requisite env vars + if os.environ.get('CONTAINER_HOST') is None or os.environ.get('CONTAINER_SSHKEY') is None: + sys.exit('You must have CONTAINER_HOST and CONTAINER_SSHKEY environment variables setup') + + # Podman supports building from a specific commit + # (https://github.com/containers/buildah/issues/4148), but the way + # we've set this up we don't know if the argument the user is + # passing to --git-ref is a commit or a ref. If we knew it was a + # ref then we could use `git ls-remote` to remotely determine + # the commit we wanted to build, but we don't. Just fetch the code + # into a tmpdir for now and use that as the git repo to build from. + with tempfile.TemporaryDirectory() as gitdir: + # fetch the git repo contents for the build and determine commit/shortcommit + cmd = ["git", "-C", gitdir, "init", "."] + runcmd(cmd, quiet=True, capture_output=True) + cmd = ["git", "-C", gitdir, "remote", "add", "origin", args.git_url] + runcmd(cmd, quiet=True, capture_output=True) + cmd = ["git", "-C", gitdir, "fetch", "--depth=1", "origin", args.git_ref] + runcmd(cmd, quiet=True, capture_output=True) + cmd = ["git", "-C", gitdir, "checkout", "FETCH_HEAD"] + runcmd(cmd, quiet=True, capture_output=True) + cmd = ["git", "-C", gitdir, "rev-parse", "FETCH_HEAD"] + commit = runcmd(cmd, quiet=True, capture_output=True).stdout.strip().decode() + shortcommit = commit[0:7] + logging.info(f"Translated {args.git_url}#{args.git_ref} into {shortcommit}") + # Add some information about the commit to labels for the container + args.labels.append(f"org.opencontainers.image.revision={commit}") + args.labels.append(f"org.opencontainers.image.source={args.git_url}") + # If a tag wasn't passed then use the arch + shortcommit + if not args.tag: + args.tag = f"{args.arch}-{shortcommit}" + logging.info(f"Targetting a container image for {args.repo}:{args.tag}") + # Sanity check the registry if asked to push to a registry + if args.push_to_registry and is_tag_in_registry(args.repo, args.tag): + logging.info(f"Container image at {args.repo}:{args.tag} exists.") + if args.force: + logging.info(f"--force was passed. Will overwrite container at {args.repo}:{args.tag}") + else: + logging.info("No work to do. You can force with --force. Skipping build/push.") + return + # Check first if the build already exists in local storage on the builder + if is_tag_in_podman_storage(args.repo, args.tag): + if args.force: + logging.info(f"--force was passed. Will overwrite built container with tag {args.repo}:{args.tag}") + needbuild = True + else: + logging.info(f"Re-using existing built container with tag {args.repo}:{args.tag}") + needbuild = False + else: + needbuild = True + # Build the container if needed. + if needbuild: + logging.info("Building container via podman") + builddir = os.path.join(gitdir, args.git_sub_dir) + build_container_image(args.labels, builddir, args.fromimage, + args.cache_ttl, args.repo, args.tag) + + # Push to the registry if needed, else save the image to a file + if args.push_to_registry: + logging.info("Pushing to remote registry") + push_container_image(args.repo, args.tag) + else: + logging.info("Archiving build container image from remote") + pull_oci_archive_from_remote(args.repo, args.tag, args.write_to_file) + + +def parse_args(): + parser = argparse.ArgumentParser( + prog="CoreOS Assembler Remote Build", + description="Build coreos-assembler remotely", + usage=""" +Run multi-arch builds using podman remote. +In order to get cmd-remote-build-container working the CONTAINER_SSHKEY and CONTAINER_HOST environment variables +must be defined + +Examples: + $ cosa remote-build-container \ + --arch aarch64 \ + --label quay.expires-after=4d \ + --git-ref main \ + --git-url https://github.com/coreos/coreos-assembler.git \ + --repo quay.io/coreos/coreos-assembler-staging \ + --push-to-registry """) + + parser.add_argument( + '--arch', required=True, + help='Build Architecture') + parser.add_argument( + '--authfile', required=False, + help='A file to use for registry auth') + parser.add_argument( + '--cache-ttl', default="0.1s", required=False, + help="""Pass along --cache-ttl= to `podman build`. + Defaults to 0.1s, which is effectively `--no-cache`""") + parser.add_argument( + '--label', dest="labels", default=[], action='append', + required=False, help='Add image label(s)') + parser.add_argument( + '--force', required=False, action='store_true', + help='Force image overwrite') + parser.add_argument( + '--from', dest="fromimage", required=False, + help='Pass along --from= to `podman build`.') + parser.add_argument( + '--git-ref', required=True, + help='Git branch or tag or commit') + parser.add_argument( + '--git-url', required=True, + help='Git URL') + parser.add_argument( + '--git-sub-dir', default='', required=False, + help='Git sub directory to use for container build') + parser.add_argument( + '--repo', default='localhost', required=False, + help='Registry repository') + parser.add_argument( + '--tag', required=False, + help='Force image tag. The default is arch-commit') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + '--push-to-registry', required=False, action='store_true', + help='Push image to registry. You must be logged in before pushing images') + group.add_argument( + '--write-to-file', required=False, + help='Write container oci archive to named file') + + return parser.parse_args() + + +if __name__ == '__main__': + sys.exit(main()) -- Gitee