From 9856165728e3890fb21c97202ec610aa9746c30a Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Fri, 12 Jul 2024 16:19:14 +0800 Subject: [PATCH] sync cmd-compress from upstream v0.16.0 The major change is that add a method to determine the compression method from image.json [upstream] d0a0874b9 src/cmd-compress: don't require image.json for decompression b60dc1f43 cmd-compress: support `compressor` field in `image.yaml` 2913176a2 cmd-compress: move default compressor logic out of argparse c31990c14 cmd-compress: move compressor-related logic lower down bc2a45539 cmd-compress: Clarify operation on Skipped artifacts output 171b52b9e Consolidate CPU-count logic into mantle and reimplement it --- src/cmd-compress | 78 ++++++++++++++++++++++++------------------- src/cosalib/cmdlib.py | 4 +++ 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/cmd-compress b/src/cmd-compress index 1e8dba54..1e01d126 100755 --- a/src/cmd-compress +++ b/src/cmd-compress @@ -4,16 +4,17 @@ # Compresses all images in a build. import os -import re import sys -import math import json import shutil import argparse sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cosalib.builds import Builds +from cosalib.meta import GenericBuildMeta from cosalib.cmdlib import ( + import_ostree_commit, + ncpu, rm_allow_noent, runcmd, sha256sum_file, @@ -31,10 +32,8 @@ parser = argparse.ArgumentParser() parser.add_argument("--build", help="Build ID") parser.add_argument("--artifact", default=[], action='append', help="Only compress given image ARTIFACT", metavar="ARTIFACT") -parser.add_argument("--compressor", - choices=['xz', 'gzip'], - default=DEFAULT_COMPRESSOR, - help=f"Compressor to use, default is {DEFAULT_COMPRESSOR}") +parser.add_argument("--compressor", choices=['xz', 'gzip'], + help="Override compressor to use") parser.add_argument("--mode", choices=['compress', 'uncompress'], default=DEFAULT_MODE, @@ -43,16 +42,6 @@ parser.add_argument("--fast", action='store_true', help="Override compression to `gzip -1`") args = parser.parse_args() -# that's the tool default -gzip_level = 6 -if args.fast: - args.compressor = 'gzip' - gzip_level = 1 - -# find extension for --compressor -ext_dict = {'xz': '.xz', 'gzip': '.gz'} -ext = ext_dict[args.compressor] - builds = Builds() # default to latest build if not specified @@ -63,6 +52,9 @@ else: print(f"Targeting build: {build}") +# common extensions for known compressors +ext_dict = {'xz': '.xz', 'gzip': '.gz'} + def get_cpu_param(param): with open(f'/sys/fs/cgroup/cpu/cpu.{param}') as f: @@ -73,21 +65,6 @@ def strip_ext(path): return path.rsplit(".", 1)[0] -# XXX: should dedupe this with logic in cmdlib and put in the shared lib -def xz_threads(): - with open("/proc/1/cgroup") as f: - in_k8s = re.search(r'.*kubepods.*', f.read()) - if in_k8s: - # see similar code in OpenJKD HotSpot: - # https://hg.openjdk.java.net/jdk/jdk/file/7b671e6b0d5b/src/hotspot/os/linux/osContainer_linux.cpp#l595 - quota = get_cpu_param('cfs_quota_us') - period = get_cpu_param('cfs_period_us') - # are k8s limits enforced? - if quota != -1 and period > 0: - return math.ceil(quota / period) - return 0 # no limits, let xz choose what it wants - - def compress_one_builddir(builddir): print(f"Compressing: {builddir}") buildmeta_path = os.path.join(builddir, 'meta.json') @@ -100,6 +77,9 @@ def compress_one_builddir(builddir): with open(buildmeta_path) as f: buildmeta = json.load(f) + # Find what extension to use based on the selected compressor + ext = ext_dict[args.compressor] + tmpdir = 'tmp/compress' if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) @@ -136,7 +116,7 @@ def compress_one_builddir(builddir): img['uncompressed-size'] = img['size'] with open(tmpfile, 'wb') as f: if args.compressor == 'xz': - t = xz_threads() + t = ncpu() runcmd(['xz', '-c9', f'-T{t}', filepath], stdout=f) else: runcmd(['gzip', f'-{gzip_level}', '-c', filepath], stdout=f) @@ -161,7 +141,7 @@ def compress_one_builddir(builddir): rm_allow_noent(filepath[:-3]) if len(skipped) > 0: - print(f"Skipped artifacts: {' '.join(skipped)}") + print(f"Skipped compressing artifacts: {' '.join(skipped)}") if at_least_one: print(f"Updated: {buildmeta_path}") return at_least_one @@ -214,7 +194,7 @@ def uncompress_one_builddir(builddir): # during 'build' with open(tmpfile, 'wb') as f: if file.endswith('xz'): - t = xz_threads() + t = ncpu() runcmd(['xz', '-dc', f'-T{t}', filepath], stdout=f) elif file.endswith('gz'): runcmd(['gzip', '-dc', filepath], stdout=f) @@ -251,7 +231,7 @@ def uncompress_one_builddir(builddir): rm_allow_noent(filepath[:-3]) if len(skipped) > 0: - print(f"Skipped artifacts: {' '.join(skipped)}") + print(f"Skipped uncompressing artifacts: {' '.join(skipped)}") if len(skipped_missing) > 0: print(f"Skipped missing artifacts: {' '.join(skipped_missing)}") if at_least_one: @@ -259,8 +239,36 @@ def uncompress_one_builddir(builddir): return at_least_one +def get_image_json(): + # All arches might not be local. Find one that has the info. + workdir = os.getcwd() + for arch in builds.get_build_arches(build): + builddir = builds.get_build_dir(build, arch) + if not os.path.exists(os.path.join(builddir, 'meta.json')): + continue + buildmeta = GenericBuildMeta(workdir=workdir, build=build, basearch=arch) + if not os.path.exists(os.path.join(builddir, buildmeta['images']['ostree']['path'])): + continue + import_ostree_commit(workdir, builddir, buildmeta) # runs extract_image_json() + with open(os.path.join(workdir, 'tmp/image.json')) as f: + return json.load(f) + # If we get here we were unsuccessful + print("Could not find/extract image.json. Please pass --compressor\n" + + "or make sure local ociarchive exists in the build directory.") + raise Exception("Could not find/extract image.json") + + changed = [] if args.mode == "compress": + # Find what compressor we should use, either picking it up from + # CLI args or from image.json + gzip_level = 6 + if args.fast: + args.compressor = 'gzip' + gzip_level = 1 + elif not args.compressor: + image_json = get_image_json() + args.compressor = image_json.get('compressor', DEFAULT_COMPRESSOR) for arch in builds.get_build_arches(build): builddir = builds.get_build_dir(build, arch) changed.append(compress_one_builddir(builddir)) diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 4d5765b8..361e0a8a 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -437,3 +437,7 @@ def ensure_glob(pathname, **kwargs): if not ret: raise Exception(f'No matches for {pathname}') return ret + +def ncpu(): + '''Return the number of usable CPUs we have for parallelism.''' + return int(subprocess.check_output(['kola', 'ncpu'])) -- Gitee