diff --git a/src/cmd-buildextend-extensions b/src/cmd-buildextend-extensions index 3c436986b0d29bbfa8ad40b34069ceac0118e65b..bbd6bb9c2e38489378d6addeda8a7d6b66e5b6d9 100755 --- a/src/cmd-buildextend-extensions +++ b/src/cmd-buildextend-extensions @@ -88,7 +88,7 @@ def run_rpmostree(workdir, commit, treefile, extensions): def create_yumrepo(repodir): - cmdlib.run_verbose(['createrepo_c', repodir]) + cmdlib.runcmd(['createrepo_c', repodir]) # we could also have rpm-ostree output the pkglist for us, but meh... we # need to run createrepo_c anyway and it's nice that we're using it as the # source of truth, since that's what rpm-ostree clients will also use @@ -116,7 +116,7 @@ def create_tarball(buildmeta, srcdir, destdir): destdir = os.path.abspath(destdir) basearch = buildmeta['coreos-assembler.basearch'] tarfile = f'{destdir}/{buildmeta["name"]}-{buildmeta["buildid"]}-extensions.{basearch}.tar' - cmdlib.run_verbose(['tar', '-cf', tarfile, '.'], cwd=srcdir) + cmdlib.runcmd(['tar', '-cf', tarfile, '.'], cwd=srcdir) return tarfile diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index 70238a027e04df788e39d04648c9459e9ec3a807..d32a2d10ad536cea8ed66eab380db853ae22673d 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -137,7 +137,7 @@ def align_initrd_for_uncompressed_append(destf): def get_os_features(): features = {} - f = run_verbose(['/usr/bin/ostree', 'cat', '--repo', repo, + f = runcmd(['/usr/bin/ostree', 'cat', '--repo', repo, buildmeta_commit, '/usr/libexec/nestos-installer-service'], capture_output=True).stdout.decode() # eventually we can assume nestos-installer >= 0.12.0, delete this check, @@ -145,7 +145,7 @@ def get_os_features(): if '--config-file' in f: features['installer-config'] = True - f = run_verbose(['/usr/bin/ostree', 'cat', '--repo', repo, + f = runcmd(['/usr/bin/ostree', 'cat', '--repo', repo, buildmeta_commit, '/usr/lib/dracut/modules.d/35nestos-network/nestos-copy-firstboot-network.sh'], capture_output=True).stdout.decode() @@ -197,7 +197,7 @@ def cp_reflink(src, dest): # Make stream hash for `rdcore stream-hash` -# https://github.com/coreos/nestos-installer/blob/a8d6f50dea6e/src/bin/rdcore/stream_hash.rs#L26-L41 +# https://github.com/coreos/coreos-installer/blob/a8d6f50dea6e/src/bin/rdcore/stream_hash.rs#L26-L41 def make_stream_hash(src, dest): bufsize = 2 * 1024 * 1024 with open(src, 'rb') as inf: @@ -239,17 +239,17 @@ def generate_iso(): # Find the directory under `/usr/lib/modules/` where the # kernel/initrd live. It will be the 2nd entity output by # `ostree ls /usr/lib/modules` - process = run_verbose(['/usr/bin/ostree', 'ls', '--repo', repo, - '--nul-filenames-only', f"{buildmeta_commit}", - '/usr/lib/modules'], capture_output=True) + process = runcmd(['/usr/bin/ostree', 'ls', '--repo', repo, + '--nul-filenames-only', f"{buildmeta_commit}", + '/usr/lib/modules'], capture_output=True) moduledir = process.stdout.decode().split('\0')[1] # copy those files out of the ostree into the iso root dir initramfs_img = 'initramfs.img' for file in [kernel_img, initramfs_img]: - run_verbose(['/usr/bin/ostree', 'checkout', '--force-copy', '--repo', repo, - '--user-mode', '--subpath', os.path.join(moduledir, file), - f"{buildmeta_commit}", tmpisoimagespxe]) + runcmd(['/usr/bin/ostree', 'checkout', '--force-copy', '--repo', repo, + '--user-mode', '--subpath', os.path.join(moduledir, file), + f"{buildmeta_commit}", tmpisoimagespxe]) # initramfs isn't world readable by default so let's open up perms os.chmod(os.path.join(tmpisoimagespxe, file), 0o644) if file == initramfs_img: @@ -286,13 +286,13 @@ def generate_iso(): # Add osmet files tmp_osmet = os.path.join(tmpinitrd_rootfs, img_metal_obj['path'] + '.osmet') print('Generating osmet file for 512b metal image') - run_verbose(['/usr/lib/coreos-assembler/osmet-pack', + runcmd(['/usr/lib/coreos-assembler/osmet-pack', img_metal, '512', tmp_osmet, img_metal_checksum, 'fast' if args.fast else 'normal']) if img_metal4k_obj: tmp_osmet4k = os.path.join(tmpinitrd_rootfs, img_metal4k_obj['path'] + '.osmet') print('Generating osmet file for 4k metal image') - run_verbose(['/usr/lib/coreos-assembler/osmet-pack', + runcmd(['/usr/lib/coreos-assembler/osmet-pack', img_metal4k, '4096', tmp_osmet4k, img_metal4k_checksum, 'fast' if args.fast else 'normal']) @@ -301,8 +301,8 @@ def generate_iso(): # Name must be exactly "root.squashfs" because the 20live dracut module # makes assumptions about the length of the name in sysroot.mount tmp_squashfs = os.path.join(tmpinitrd_rootfs, 'root.squashfs') - run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs', - img_metal, tmp_squashfs, squashfs_compression]) + runcmd(['/usr/lib/coreos-assembler/gf-mksquashfs', + img_metal, tmp_squashfs, squashfs_compression]) # Generate rootfs image iso_rootfs = os.path.join(tmpisoimagespxe, rootfs_img) @@ -329,8 +329,8 @@ def generate_iso(): cp_reflink(iso_initramfs, pxe_initramfs) # Read and filter kernel arguments for substituting into ISO bootloader - result = run_verbose(['/usr/lib/coreos-assembler/gf-get-kargs', - img_metal], stdout=subprocess.PIPE, text=True) + result = runcmd(['/usr/lib/coreos-assembler/gf-get-kargs', + img_metal], stdout=subprocess.PIPE, text=True) kargs_array = [karg for karg in result.stdout.split() if karg.split('=')[0] not in live_exclude_kargs] kargs_array.append(f"nestos.liveiso={volid}") @@ -383,7 +383,7 @@ def generate_iso(): print(f'{srcfile} -> {dstfile}') if karg_embed_area_length > 0: - assert(karg_embed_area_length > len(cmdline)) + assert (karg_embed_area_length > len(cmdline)) kargs_json.update( size=karg_embed_area_length, default=cmdline.strip(), @@ -434,7 +434,7 @@ def generate_iso(): # safely remove things we don't need in the final ISO tree for d in ['EFI', 'isolinux', 'zipl.prm']: - run_verbose(['rm', '-rf', os.path.join(tmpisoroot, d)]) + runcmd(['rm', '-rf', os.path.join(tmpisoroot, d)]) # grub2-mkrescue is a wrapper around xorriso genisoargs = ['grub2-mkrescue', '-volid', volid] @@ -461,7 +461,7 @@ def generate_iso(): kernel_img = 'kernel.img' # combine kernel, initramfs and cmdline using the mk-s390image tool - run_verbose(['/usr/bin/mk-s390image', + runcmd(['/usr/bin/mk-s390image', kernel_dest, os.path.join(tmpisoimages, 'cdboot.img'), '-r', iso_initramfs, @@ -474,7 +474,7 @@ def generate_iso(): # safely remove things we don't need in the final ISO tree for d in ['EFI', 'isolinux', 'zipl.prm']: - run_verbose(['rm', '-rf', os.path.join(tmpisoroot, d)]) + runcmd(['rm', '-rf', os.path.join(tmpisoroot, d)]) genisoargs = ['/usr/bin/xorrisofs', '-verbose', '-volid', volid, @@ -503,10 +503,10 @@ def generate_iso(): return tarinfo tmpimageefidir = os.path.join(tmpdir, "efi") - run_verbose(['/usr/bin/ostree', 'checkout', '--repo', repo, - '--user-mode', '--subpath', - "/usr/lib/bootupd/updates/EFI", - buildmeta_commit, tmpimageefidir]) + runcmd(['/usr/bin/ostree', 'checkout', '--repo', repo, + '--user-mode', '--subpath', + "/usr/lib/bootupd/updates/EFI", + buildmeta_commit, tmpimageefidir]) # Install binaries from boot partition # Manually construct the tarball to ensure proper permissions and ownership @@ -525,8 +525,8 @@ def generate_iso(): # so set EFI-SYSTEM for consistency with the metal image. # This should not be needed on Fedora or RHEL 9, but seems like # a good thing to do anyway. - run_verbose(['virt-make-fs', '--type=vfat', '--label=EFI-SYSTEM', - efitarfile.name, efibootfile]) + runcmd(['virt-make-fs', '--type=vfat', '--label=EFI-SYSTEM', + efitarfile.name, efibootfile]) genisoargs += ['-eltorito-alt-boot', '-efi-boot', 'images/efiboot.img', @@ -573,12 +573,12 @@ def generate_iso(): for f in ensure_glob(os.path.join(tmpisoisolinux, '*.msg')): os.unlink(f) - run_verbose(genisoargs_final) + runcmd(genisoargs_final) # Add MBR, and GPT with ESP, for x86_64 BIOS/UEFI boot when ISO is # copied to a USB stick if basearch == "x86_64": - run_verbose(['/usr/bin/isohybrid', '--uefi', tmpisofile]) + runcmd(['/usr/bin/isohybrid', '--uefi', tmpisofile]) genisoargs_minimal = genisoargs + ['-o', f'{tmpisofile}.minimal', tmpisoroot] # The only difference with the miniso is that we drop these two files. @@ -588,11 +588,11 @@ def generate_iso(): # nestos-installer takes care of removing it. os.unlink(iso_rootfs) os.unlink(miniso_data) - run_verbose(genisoargs_minimal) + runcmd(genisoargs_minimal) if basearch == "x86_64": - run_verbose(['/usr/bin/isohybrid', '--uefi', f'{tmpisofile}.minimal']) + runcmd(['/usr/bin/isohybrid', '--uefi', f'{tmpisofile}.minimal']) # this consumes the minimal image - #run_verbose(['coreos-installer', 'iso', 'extract', 'minimal-iso', + #runcmd(['coreos-installer', 'iso', 'extract', 'minimal-iso', # tmpisofile, f'{tmpisofile}.minimal', "--consume"]) buildmeta['images'].update({ diff --git a/src/cmd-compress b/src/cmd-compress index 8d30cae09823dd580f7fe6fbffb89d8f3971568a..1e8dba54d2b787adf932026a03e588b734145a75 100755 --- a/src/cmd-compress +++ b/src/cmd-compress @@ -15,7 +15,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cosalib.builds import Builds from cosalib.cmdlib import ( rm_allow_noent, - run_verbose, + runcmd, sha256sum_file, write_json) @@ -137,9 +137,9 @@ def compress_one_builddir(builddir): with open(tmpfile, 'wb') as f: if args.compressor == 'xz': t = xz_threads() - run_verbose(['xz', '-c9', f'-T{t}', filepath], stdout=f) + runcmd(['xz', '-c9', f'-T{t}', filepath], stdout=f) else: - run_verbose(['gzip', f'-{gzip_level}', '-c', filepath], stdout=f) + runcmd(['gzip', f'-{gzip_level}', '-c', filepath], stdout=f) file_with_ext = file + ext filepath_with_ext = filepath + ext compressed_size = os.path.getsize(tmpfile) @@ -215,9 +215,9 @@ def uncompress_one_builddir(builddir): with open(tmpfile, 'wb') as f: if file.endswith('xz'): t = xz_threads() - run_verbose(['xz', '-dc', f'-T{t}', filepath], stdout=f) + runcmd(['xz', '-dc', f'-T{t}', filepath], stdout=f) elif file.endswith('gz'): - run_verbose(['gzip', '-dc', filepath], stdout=f) + runcmd(['gzip', '-dc', filepath], stdout=f) else: print(f"Unknown sufix of file {file}") file_without_ext = strip_ext(file) diff --git a/src/cmd-oc-adm-release b/src/cmd-oc-adm-release index a499e63e813df040f0b0209db99eba5de6e43358..3d148d015dbcd5bd1a74bf6f53f98cf0ee4d3d1a 100755 --- a/src/cmd-oc-adm-release +++ b/src/cmd-oc-adm-release @@ -27,7 +27,7 @@ sys.path.insert(0, COSA_PATH) from cosalib.meta import GenericBuildMeta as Meta from cosalib.cmdlib import ( get_basearch, - run_verbose + runcmd ) os.environ["PATH"] = f"{os.getcwd()}:{COSA_PATH}:{os.environ.get('PATH')}" @@ -93,7 +93,7 @@ def release_stream(meta, args, ocp_ver): if args.authfile != "": tag_cmd.extend(["--authfile", args.authfile]) tag_cmd.extend([f"docker://{args.from_url}"]) - tags = json.loads(run_verbose(tag_cmd, capture_output=True).stdout) + tags = json.loads(runcmd(tag_cmd, capture_output=True).stdout) log.info(f"Looking for latest release tag for {ocp_ver}") ctags = [] @@ -176,5 +176,5 @@ if __name__ == '__main__': log.info(f"Command: {' '.join(cmd)}") if not args.dry_run: - run_verbose(cmd) + runcmd(cmd) log.info(f"Payload released to: {payload_dest}") diff --git a/src/cmd-upload-oscontainer b/src/cmd-upload-oscontainer index eb642cb8322a213289d776bb2738b8167b8af023..98e90c3f95287d75d041d19be02799f46483afa8 100755 --- a/src/cmd-upload-oscontainer +++ b/src/cmd-upload-oscontainer @@ -46,7 +46,7 @@ with open(metapath) as f: # for backcompat, we auto-build extensions if they're missing if os.path.exists('src/config/extensions.yaml'): if 'extensions' not in meta: - cmdlib.run_verbose(['coreos-assembler', 'buildextend-extensions']) + cmdlib.runcmd(['coreos-assembler', 'buildextend-extensions']) with open(metapath) as f: meta = json.load(f) assert 'extensions' in meta @@ -82,9 +82,9 @@ if not os.path.exists(tmprepo): tmp_osreleasedir = 'tmp/usrlib-osrelease' subprocess.check_call(['rm', '-rf', tmp_osreleasedir]) -cmdlib.run_verbose(['/usr/bin/ostree', 'checkout', '--repo', tmprepo, - '--user-mode', '--subpath=/usr/lib/os-release', ostree_commit, - tmp_osreleasedir]) +cmdlib.runcmd(['/usr/bin/ostree', 'checkout', '--repo', tmprepo, + '--user-mode', '--subpath=/usr/lib/os-release', ostree_commit, + tmp_osreleasedir]) display_name = None with open(os.path.join(tmp_osreleasedir, "os-release")) as f: display_name = subprocess.check_output(['/bin/sh', '-c', 'set -euo pipefail; . /proc/self/fd/0 && echo $NAME'], stdin=f, encoding='UTF-8').strip() diff --git a/src/cmd-virt-install b/src/cmd-virt-install index 22c0ecdc18b35e6356ecf9bcfd514a01cecea8ae..7c95e8c932d240ec2ddc31068360846a17ca34f6 100755 --- a/src/cmd-virt-install +++ b/src/cmd-virt-install @@ -127,6 +127,6 @@ basevinstall_args = ['virt-install', f"--connect={args.connect}", f'--name={domname}', '--os-type=linux', '--os-variant=rhel8-unknown', f'--qemu-commandline={qemu_args}', '--noautoconsole'] -cmdlib.run_verbose(basevinstall_args + vinstall_args) +cmdlib.runcmd(basevinstall_args + vinstall_args) if args.console: os.execlp("virsh", "virsh", "console", domname) diff --git a/src/cosalib/aliyun.py b/src/cosalib/aliyun.py index ea5eafabf4cb70ab94b040e29e50918ef767a14b..bde71db12ac8040d889f29b3dc44d23bec68fc25 100644 --- a/src/cosalib/aliyun.py +++ b/src/cosalib/aliyun.py @@ -2,7 +2,7 @@ import subprocess import logging as log import json import sys -from cosalib.cmdlib import run_verbose +from cosalib.cmdlib import runcmd from tenacity import ( retry, stop_after_attempt @@ -12,7 +12,7 @@ from tenacity import ( def remove_aliyun_image(aliyun_id, region): print(f"aliyun: removing image {aliyun_id} in {region}") try: - run_verbose([ + runcmd([ 'ore', 'aliyun', '--log-level', 'debug', 'delete-image', '--id', aliyun_id, diff --git a/src/cosalib/azure.py b/src/cosalib/azure.py index f91964a2d7bf072efde6203b48088a97ae4dce94..24e073305988e6545bf91ace0b7e5d198f0d8c8e 100644 --- a/src/cosalib/azure.py +++ b/src/cosalib/azure.py @@ -1,6 +1,6 @@ import os import urllib -from cosalib.cmdlib import run_verbose +from cosalib.cmdlib import runcmd from tenacity import ( retry, stop_after_attempt @@ -11,7 +11,7 @@ from tenacity import ( def remove_azure_image(image, resource_group, auth, profile): print(f"Azure: removing image {image}") try: - run_verbose([ + runcmd([ 'ore', 'azure', '--azure-auth', auth, '--azure-profile', profile, @@ -51,7 +51,7 @@ def azure_run_ore(build, args): ] if args.force: ore_args.append('--overwrite') - run_verbose(ore_args) + runcmd(ore_args) url_path = urllib.parse.quote(( f"{args.storage_account}.blob.core.windows.net/" diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index ae5455b54248c99ac03344864853647d1a7eceb5..adeebf81574243ebf8e078bbb7e797de9736a5b6 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -5,6 +5,7 @@ Houses helper code for python based coreos-assembler commands. import glob import hashlib import json +import logging import os import shutil import subprocess @@ -29,6 +30,10 @@ from gi.repository import RpmOstree from datetime import datetime, timezone +# Set up logging +logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(levelname)s - %(message)s") + retry_stop = (stop_after_delay(10) | stop_after_attempt(5)) retry_boto_exception = (retry_if_exception_type(ConnectionClosedError) | retry_if_exception_type(ConnectTimeoutError) | @@ -42,32 +47,33 @@ def retry_callback(retry_state): print(f"Retrying after {retry_state.outcome.exception()}") -def run_verbose(args, **kwargs): - """ - Prints out the command being executed before executing a subprocess call. - - :param args: All non-keyword arguments - :type args: list - :param kwargs: All keyword arguments - :type kwargs: dict - :raises: CalledProcessError - """ - print("+ {}".format(subprocess.list2cmdline(args))) - - # default to throwing exception - if 'check' not in kwargs.keys(): - kwargs['check'] = True - # capture_output is only on python 3.7+. Provide convenience here - # until 3.7 is a baseline: - if kwargs.pop('capture_output', False): - kwargs['stdout'] = subprocess.PIPE - kwargs['stderr'] = subprocess.PIPE - +def runcmd(cmd: list, quiet: bool = False, **kwargs: int) -> subprocess.CompletedProcess: + ''' + Run the given command using subprocess.run and perform verification. + @param cmd: list that represents the command to be executed + @param kwargs: key value pairs that represent options to run() + ''' try: - process = subprocess.run(args, **kwargs) - except subprocess.CalledProcessError: - fatal("Error running command " + args[0]) - return process + # default to error on failed command + pargs = {"check": True} + pargs.update(kwargs) + # capture_output is only on python 3.7+. Provide convenience here + # until 3.7 is a baseline: + if pargs.pop('capture_output', False): + pargs['stdout'] = subprocess.PIPE + pargs['stderr'] = subprocess.PIPE + if not quiet: + logging.info(f"Running command: {cmd}") + cp = subprocess.run(cmd, **pargs) + except subprocess.CalledProcessError as e: + logging.error("Command returned bad exitcode") + logging.error(f"COMMAND: {cmd}") + if e.stdout: + logging.error(f" STDOUT: {e.stdout.decode()}") + if e.stderr: + logging.error(f" STDERR: {e.stderr.decode()}") + raise e + return cp # subprocess.CompletedProcess def get_lock_path(path): @@ -348,7 +354,7 @@ def get_timestamp(entry): def image_info(image): try: - out = json.loads(run_verbose( + out = json.loads(runcmd( ['qemu-img', 'info', '--output=json', image], capture_output=True).stdout ) diff --git a/src/cosalib/gcp.py b/src/cosalib/gcp.py index 1eb17874a8df1ebffe29d8bc551e2c3205f713fb..038636fb351622b6c09e8122b0f3032e870fbf2f 100644 --- a/src/cosalib/gcp.py +++ b/src/cosalib/gcp.py @@ -1,6 +1,6 @@ import os import re -from cosalib.cmdlib import run_verbose +from cosalib.cmdlib import runcmd from tenacity import ( retry, stop_after_attempt @@ -21,7 +21,7 @@ GCP_NAMING_RE = r"[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?|[1-9][0-9]{0,19}" def remove_gcp_image(gcp_id, json_key, project): print(f"GCP: removing image {gcp_id}") try: - run_verbose([ + runcmd([ 'ore', 'gcloud', 'delete-images', gcp_id, '--json-key', json_key, '--project', project @@ -75,7 +75,7 @@ def gcp_run_ore(build, args): ore_upload_cmd.extend(['--create-image=false']) for license in args.license or DEFAULT_LICENSES: ore_upload_cmd.extend(['--license', license]) - run_verbose(ore_upload_cmd) + runcmd(ore_upload_cmd) # Run deprecate image to deprecate if requested if args.deprecated: @@ -84,7 +84,7 @@ def gcp_run_ore(build, args): '--image', gcp_name, '--state', 'DEPRECATED' ] - run_verbose(ore_deprecate_cmd) + runcmd(ore_deprecate_cmd) # Run update-image to add to an image family if requested. # We run this as a separate API call because we want to run @@ -95,7 +95,7 @@ def gcp_run_ore(build, args): '--image', gcp_name, '--family', args.family ] - run_verbose(ore_update_cmd) + runcmd(ore_update_cmd) build.meta['gcp'] = { 'image': gcp_name, diff --git a/src/cosalib/ibmcloud.py b/src/cosalib/ibmcloud.py index c3263c57ad1ddaef3c3194d1e12ed752988a6825..1d6975b0bb353a35f34ccd037d0219b7ec7ccee6 100644 --- a/src/cosalib/ibmcloud.py +++ b/src/cosalib/ibmcloud.py @@ -8,7 +8,7 @@ import urllib import os.path import sys from cosalib.cmdlib import ( - run_verbose + runcmd ) from tenacity import ( retry, @@ -139,7 +139,7 @@ def ibmcloud_run_ore(build, args): if args.force: ore_args.extend(['--force']) - run_verbose(ore_args) + runcmd(ore_args) url_path = urllib.parse.quote(( f"s3.{region}.cloud-object-storage.appdomain.cloud/" f"{args.bucket}/{ibmcloud_object_name}" @@ -169,7 +169,7 @@ def ibmcloud_run_ore_replicate(build, args): # PowerVS insatnces are supported in all the regions where cloud object storage can be created. This list is common for # both IBMCloud and PowerVS. if not args.region: - args.region = ['au-syd', 'br-sao', 'ca-tor', 'eu-de', 'eu-gb', 'jp-osa', 'jp-tok', 'us-east', 'us-south'] + args.region = ['au-syd', 'br-sao', 'ca-tor', 'eu-de', 'eu-es', 'eu-gb', 'jp-osa', 'jp-tok', 'us-east', 'us-south'] log.info(("default: replicating to all regions. If this is not " " desirable, use '--regions'")) diff --git a/src/cosalib/qemuvariants.py b/src/cosalib/qemuvariants.py index e0692e180fd452244b0d3f9f238b9e1e43f38103..0f5520c1b81383331fa2575275a5180befb47e48 100644 --- a/src/cosalib/qemuvariants.py +++ b/src/cosalib/qemuvariants.py @@ -262,14 +262,14 @@ class QemuVariantImage(_Build): if self.virtual_size is not None: resize_cmd = ['qemu-img', 'resize', self.tmp_image, self.virtual_size] - run_verbose(resize_cmd) + runcmd(resize_cmd) cmd = ['qemu-img', 'convert', '-f', 'raw', '-O', self.image_format, self.tmp_image] for k, v in self.convert_options.items(): cmd.extend([k, v]) cmd.extend([work_img]) - run_verbose(cmd) + runcmd(cmd) img_info = image_info(work_img) if self.image_format != img_info.get("format"): @@ -300,7 +300,7 @@ class QemuVariantImage(_Build): tar_cmd.extend(self.tar_flags) tar_cmd.extend(['-f', final_img]) tar_cmd.extend(tarlist) - run_verbose(tar_cmd) + runcmd(tar_cmd) elif not self.mutate_callback_creates_final_image: log.info(f"Moving {work_img} to {final_img}") shutil.move(work_img, final_img) @@ -310,7 +310,7 @@ class QemuVariantImage(_Build): size = os.stat(final_img).st_size temp_path = f"{final_img}.tmp" with open(temp_path, "wb") as fh: - run_verbose(['gzip', '-9c', final_img], stdout=fh) + runcmd(['gzip', '-9c', final_img], stdout=fh) shutil.move(temp_path, final_img) meta_patch.update({ 'skip-compression': True, diff --git a/src/oscontainer.py b/src/oscontainer.py index 136648c87cb8f76b2924db3434468ced6c670dfa..bcb56ee0c8fd153ae1777807b6d8c85a6e0636c4 100755 --- a/src/oscontainer.py +++ b/src/oscontainer.py @@ -35,11 +35,6 @@ def run_get_string(args): return subprocess.check_output(args, encoding='UTF-8').strip() -def run_verbose(args, **kwargs): - print("+ {}".format(subprocess.list2cmdline(args))) - subprocess.check_call(args, **kwargs) - - def find_commit_from_oscontainer(repo): """Given an ostree repo, find exactly one commit object in it""" o = subprocess.check_output(['find', repo + '/objects', '-name', '*.commit'], encoding='UTF-8').strip().split('\n') @@ -74,8 +69,8 @@ def oscontainer_extract(containers_storage, tmpdir, src, dest, cmd.append("--cert-dir={}".format(cert_dir)) tmp_tarball = tmpdir + '/container.tar' cmd += ['copy', "docker://" + src, 'docker-archive://' + tmp_tarball] - run_verbose(cmd) - run_verbose(['tar', 'xf', tmp_tarball], cwd=tmpdir) + cmdlib.runcmd(cmd) + cmdlib.runcmd(['tar', 'xf', tmp_tarball], cwd=tmpdir) os.unlink(tmp_tarball) # This is a brutal hack to extract all the layers; we don't even bother with ordering # because we know we're not removing anything in higher layers. @@ -86,9 +81,9 @@ def oscontainer_extract(containers_storage, tmpdir, src, dest, repo = tmpdir + '/srv/repo' commit = find_commit_from_oscontainer(repo) print(f"commit: {commit}") - run_verbose(["ostree", "--repo=" + dest, "pull-local", repo, commit]) + cmdlib.runcmd(["ostree", "--repo=" + dest, "pull-local", repo, commit]) if ref is not None: - run_verbose([ + cmdlib.runcmd([ "ostree", "--repo=" + dest, "refs", '--create=' + ref, commit]) @@ -133,7 +128,7 @@ def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag, # because the repo will be put into a container image and the build # process should handle its own fsync (or choose not to). print("Copying ostree commit into container: {} ...".format(rev)) - run_verbose(["ostree", "--repo=" + dest_repo, "pull-local", "--disable-fsync", src, rev]) + cmdlib.runcmd(["ostree", "--repo=" + dest_repo, "pull-local", "--disable-fsync", src, rev]) for d in add_directories: with os.scandir(d) as it: @@ -186,7 +181,7 @@ def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag, tarball = os.path.abspath(os.path.join(builddir, meta['extensions']['path'])) dest_dir = os.path.join(mnt, 'extensions') os.makedirs(dest_dir, exist_ok=True) - run_verbose(["tar", "-xf", tarball], cwd=dest_dir) + cmdlib.runcmd(["tar", "-xf", tarball], cwd=dest_dir) with open(os.path.join(dest_dir, 'extensions.json')) as f: extensions = json.load(f) @@ -203,7 +198,7 @@ def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag, if display_name is not None: config += ['-l', 'io.openshift.build.version-display-names=machine-os=' + display_name, '-l', 'io.openshift.build.versions=machine-os=' + ostree_version] - run_verbose(buildah_base_argv + ['config'] + config + [bid]) + cmdlib.runcmd(buildah_base_argv + ['config'] + config + [bid]) print("Committing container...") iid = run_get_string(buildah_base_argv + ['commit', bid, image_name_and_tag]) print("{} {}".format(image_name_and_tag, iid)) @@ -234,7 +229,7 @@ def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag, podCmd.append(image_name_and_tag) - run_verbose(podCmd) + cmdlib.runcmd(podCmd) elif digestfile is not None: inspect = run_get_json(buildah_base_argv + ['inspect', image_name_and_tag])[0] with open(digestfile, 'w') as f: