diff --git a/src/cmd-compress b/src/cmd-compress index 1e8dba54d2b787adf932026a03e588b734145a75..1e01d1263c871153e6b744fe405efad882786777 100755 --- a/src/cmd-compress +++ b/src/cmd-compress @@ -4,16 +4,17 @@ # Compresses all images in a build. import os -import re import sys -import math import json import shutil import argparse sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cosalib.builds import Builds +from cosalib.meta import GenericBuildMeta from cosalib.cmdlib import ( + import_ostree_commit, + ncpu, rm_allow_noent, runcmd, sha256sum_file, @@ -31,10 +32,8 @@ parser = argparse.ArgumentParser() parser.add_argument("--build", help="Build ID") parser.add_argument("--artifact", default=[], action='append', help="Only compress given image ARTIFACT", metavar="ARTIFACT") -parser.add_argument("--compressor", - choices=['xz', 'gzip'], - default=DEFAULT_COMPRESSOR, - help=f"Compressor to use, default is {DEFAULT_COMPRESSOR}") +parser.add_argument("--compressor", choices=['xz', 'gzip'], + help="Override compressor to use") parser.add_argument("--mode", choices=['compress', 'uncompress'], default=DEFAULT_MODE, @@ -43,16 +42,6 @@ parser.add_argument("--fast", action='store_true', help="Override compression to `gzip -1`") args = parser.parse_args() -# that's the tool default -gzip_level = 6 -if args.fast: - args.compressor = 'gzip' - gzip_level = 1 - -# find extension for --compressor -ext_dict = {'xz': '.xz', 'gzip': '.gz'} -ext = ext_dict[args.compressor] - builds = Builds() # default to latest build if not specified @@ -63,6 +52,9 @@ else: print(f"Targeting build: {build}") +# common extensions for known compressors +ext_dict = {'xz': '.xz', 'gzip': '.gz'} + def get_cpu_param(param): with open(f'/sys/fs/cgroup/cpu/cpu.{param}') as f: @@ -73,21 +65,6 @@ def strip_ext(path): return path.rsplit(".", 1)[0] -# XXX: should dedupe this with logic in cmdlib and put in the shared lib -def xz_threads(): - with open("/proc/1/cgroup") as f: - in_k8s = re.search(r'.*kubepods.*', f.read()) - if in_k8s: - # see similar code in OpenJKD HotSpot: - # https://hg.openjdk.java.net/jdk/jdk/file/7b671e6b0d5b/src/hotspot/os/linux/osContainer_linux.cpp#l595 - quota = get_cpu_param('cfs_quota_us') - period = get_cpu_param('cfs_period_us') - # are k8s limits enforced? - if quota != -1 and period > 0: - return math.ceil(quota / period) - return 0 # no limits, let xz choose what it wants - - def compress_one_builddir(builddir): print(f"Compressing: {builddir}") buildmeta_path = os.path.join(builddir, 'meta.json') @@ -100,6 +77,9 @@ def compress_one_builddir(builddir): with open(buildmeta_path) as f: buildmeta = json.load(f) + # Find what extension to use based on the selected compressor + ext = ext_dict[args.compressor] + tmpdir = 'tmp/compress' if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) @@ -136,7 +116,7 @@ def compress_one_builddir(builddir): img['uncompressed-size'] = img['size'] with open(tmpfile, 'wb') as f: if args.compressor == 'xz': - t = xz_threads() + t = ncpu() runcmd(['xz', '-c9', f'-T{t}', filepath], stdout=f) else: runcmd(['gzip', f'-{gzip_level}', '-c', filepath], stdout=f) @@ -161,7 +141,7 @@ def compress_one_builddir(builddir): rm_allow_noent(filepath[:-3]) if len(skipped) > 0: - print(f"Skipped artifacts: {' '.join(skipped)}") + print(f"Skipped compressing artifacts: {' '.join(skipped)}") if at_least_one: print(f"Updated: {buildmeta_path}") return at_least_one @@ -214,7 +194,7 @@ def uncompress_one_builddir(builddir): # during 'build' with open(tmpfile, 'wb') as f: if file.endswith('xz'): - t = xz_threads() + t = ncpu() runcmd(['xz', '-dc', f'-T{t}', filepath], stdout=f) elif file.endswith('gz'): runcmd(['gzip', '-dc', filepath], stdout=f) @@ -251,7 +231,7 @@ def uncompress_one_builddir(builddir): rm_allow_noent(filepath[:-3]) if len(skipped) > 0: - print(f"Skipped artifacts: {' '.join(skipped)}") + print(f"Skipped uncompressing artifacts: {' '.join(skipped)}") if len(skipped_missing) > 0: print(f"Skipped missing artifacts: {' '.join(skipped_missing)}") if at_least_one: @@ -259,8 +239,36 @@ def uncompress_one_builddir(builddir): return at_least_one +def get_image_json(): + # All arches might not be local. Find one that has the info. + workdir = os.getcwd() + for arch in builds.get_build_arches(build): + builddir = builds.get_build_dir(build, arch) + if not os.path.exists(os.path.join(builddir, 'meta.json')): + continue + buildmeta = GenericBuildMeta(workdir=workdir, build=build, basearch=arch) + if not os.path.exists(os.path.join(builddir, buildmeta['images']['ostree']['path'])): + continue + import_ostree_commit(workdir, builddir, buildmeta) # runs extract_image_json() + with open(os.path.join(workdir, 'tmp/image.json')) as f: + return json.load(f) + # If we get here we were unsuccessful + print("Could not find/extract image.json. Please pass --compressor\n" + + "or make sure local ociarchive exists in the build directory.") + raise Exception("Could not find/extract image.json") + + changed = [] if args.mode == "compress": + # Find what compressor we should use, either picking it up from + # CLI args or from image.json + gzip_level = 6 + if args.fast: + args.compressor = 'gzip' + gzip_level = 1 + elif not args.compressor: + image_json = get_image_json() + args.compressor = image_json.get('compressor', DEFAULT_COMPRESSOR) for arch in builds.get_build_arches(build): builddir = builds.get_build_dir(build, arch) changed.append(compress_one_builddir(builddir)) diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 4d5765b879ab14bce4b952188e30cd399c8d72df..361e0a8ac4d3753dda004d9fc161df8aa7f3f9c6 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -437,3 +437,7 @@ def ensure_glob(pathname, **kwargs): if not ret: raise Exception(f'No matches for {pathname}') return ret + +def ncpu(): + '''Return the number of usable CPUs we have for parallelism.''' + return int(subprocess.check_output(['kola', 'ncpu']))