diff --git a/src/cmd-build b/src/cmd-build index 6f00d326dddd74dd0ef83f0c872832a7332f6b84..299b184a7103f0ac466fba73e1748a8ca7e50436 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -385,22 +385,7 @@ if [ "${commit}" == "${previous_commit}" ] && \ else ostree_format=$(jq -r '.["ostree-format"]' < "${image_json}") case "${ostree_format}" in - null|tar) - ostree_tarfile_path=${name}-${buildid}-ostree.${basearch}.tar - ostree init --repo=repo --mode=archive - # Pass the ref if it's set - # shellcheck disable=SC2086 - if ! ostree pull-local --repo=repo "${tmprepo}" "${buildid}" ${ref}; then - echo '(maybe https://github.com/coreos/coreos-assembler/issues/972 ?)' - exit 1 - fi - # Don't compress; archive repos are already compressed individually and we'd - # gain ~20M at best. We could probably have better gains if we compress the - # whole repo in bare/bare-user mode, but that's a different story... - tar -cf "${ostree_tarfile_path}".tmp -C repo . - rm -rf repo - ;; - oci) + null|oci) ostree_tarfile_path="${name}-${buildid}-ostree.${basearch}.ociarchive" rpm-ostree ex-container 'export' --cmd /usr/bin/bash --repo="${tmprepo}" "${buildid}" oci-archive:"${ostree_tarfile_path}".tmp:latest ;; diff --git a/src/cmd-sign b/src/cmd-sign index be33e4e5f5168e4a4ff98cda22d7e398ea377407..4a69eb9010e38516eac4de5500461f158ba55211 100755 --- a/src/cmd-sign +++ b/src/cmd-sign @@ -11,7 +11,6 @@ import os import shutil import subprocess import sys -import tarfile import tempfile import boto3 @@ -179,24 +178,16 @@ def robosign_ostree(args, s3, build, gpgkey): # We've validated the commit, now re-export the repo ostree_image = build['images']['ostree'] exported_ostree_path = os.path.join(builddir, ostree_image['path']) - if exported_ostree_path.endswith('.ociarchive'): - # Files stored in the build directory are mode 0600 to prevent - # accidental mutation. Remove the existing one because otherwise - # we'll try to `open(O_TRUNC)` it and fail. - os.unlink(exported_ostree_path) - subprocess.check_call(['ostree', 'container', 'export', '--repo=tmp/repo', checksum, f'oci-archive:{exported_ostree_path}:latest']) + exported_ostree_ref = f'oci-archive:{exported_ostree_path}:latest' + # Detect and use the replace-detached-metadata API only if available + verb = "replace-detached-metadata" + tmp_image = os.path.join(d, 'tmp.ociarchive') + tmp_ref = f"oci-archive:{tmp_image}:latest" + if subprocess.check_output(['ostree', 'container', 'image', '--help'], encoding='UTF-8').find(verb) >= 0: + subprocess.check_call(['ostree', 'container', 'image', verb, f'--src={exported_ostree_ref}', f'--dest={tmp_ref}', metapath]) else: - tmp_tar = os.path.join(d, ostree_image['path']) - # To make things a bit more efficient, append the commitmeta at - # the end of the archive after reflinking. - subprocess.check_call(['cp-reflink', exported_ostree_path, tmp_tar]) - # Normally we make our artifacts 0600 to avoid accidental mutation, but since - # we can efficiently *append* to an existing tar archive, do that instead of - # copying and rewriting. We just need to temporarily make it writable - os.chmod(tmp_tar, 0o660) - with tarfile.open(tmp_tar, 'a:') as t: - t.add(metapath, arcname=f'objects/{checksum[:2]}/{checksum[2:]}.commitmeta') - shutil.move(tmp_tar, exported_ostree_path) + subprocess.check_call(['ostree', 'container', 'export', '--repo=tmp/repo', checksum, tmp_ref]) + os.rename(tmp_image, exported_ostree_path) # Finalize the export by making it not writable. os.chmod(exported_ostree_path, 0o400) ostree_image['size'] = os.path.getsize(exported_ostree_path) diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 120087aaadfbe2daa1f53d6f31e71a69d14962bc..53753e9f1fc0c5cfbf5d8f38260ec4b26449aebf 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -258,30 +258,26 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False): return print(f"Extracting {commit}") - # extract in a new tmpdir inside the repo itself so we can still hardlink - if tarfile.endswith('.tar'): - with tempfile.TemporaryDirectory(dir=repo) as d: - subprocess.check_call(['tar', '-C', d, '-xf', tarfile]) - subprocess.check_call(['ostree', 'pull-local', '--repo', repo, - d, commit]) - elif tarfile.endswith('.ociarchive'): - # We do this in two stages, because right now ex-container only writes to - # non-archive repos. Also, in the privileged case we need sudo to write - # to `repo-build`, though it might be good to change this by default. - if os.environ.get('COSA_PRIVILEGED', '') == '1': - build_repo = os.path.join(repo, '../../cache/repo-build') - subprocess.check_call(['sudo', 'ostree', 'container', 'import', '--repo', build_repo, - '--write-ref', buildmeta['buildid'], 'ostree-unverified-image:oci-archive:' + tarfile]) - subprocess.check_call(['sudo', 'ostree', f'--repo={repo}', 'pull-local', build_repo, buildmeta['buildid']]) - uid = os.getuid() - gid = os.getgid() - subprocess.check_call(['sudo', 'chown', '-hR', f"{uid}:{gid}", repo]) - else: - with tempfile.TemporaryDirectory() as tmpd: - subprocess.check_call(['ostree', 'init', '--repo', tmpd, '--mode=bare-user']) - subprocess.check_call(['ostree', 'container', 'import', '--repo', tmpd, - '--write-ref', buildmeta['buildid'], 'ostree-unverified-image:oci-archive:' + tarfile]) - subprocess.check_call(['ostree', f'--repo={repo}', 'pull-local', tmpd, buildmeta['buildid']]) + assert tarfile.endswith('.ociarchive') + # We do this in two stages, because right now ex-container only writes to + # non-archive repos. Also, in the privileged case we need sudo to write + # to `repo-build`, though it might be good to change this by default. + if os.environ.get('COSA_PRIVILEGED', '') == '1': + build_repo = os.path.join(repo, '../../cache/repo-build') + subprocess.check_call(['sudo', 'ostree', 'container', 'import', '--repo', build_repo, + '--write-ref', buildmeta['buildid'], + 'ostree-unverified-image:oci-archive:' + tarfile]) + subprocess.check_call(['sudo', 'ostree', f'--repo={repo}', 'pull-local', build_repo, buildmeta['buildid']]) + uid = os.getuid() + gid = os.getgid() + subprocess.check_call(['sudo', 'chown', '-hR', f"{uid}:{gid}", repo]) + else: + with tempfile.TemporaryDirectory(dir=tmpdir) as tmpd: + subprocess.check_call(['ostree', 'init', '--repo', tmpd, '--mode=bare-user']) + subprocess.check_call(['ostree', 'container', 'import', '--repo', tmpd, + '--write-ref', buildmeta['buildid'], + 'ostree-unverified-image:oci-archive:' + tarfile]) + subprocess.check_call(['ostree', f'--repo={repo}', 'pull-local', tmpd, buildmeta['buildid']]) def get_basearch(): diff --git a/tests/test_cosalib_cmdlib.py b/tests/test_cosalib_cmdlib.py index 0efa1c8d1416695a1aacbad21331a04b9a79f3f1..63f9833b9af6afafb92ddd0ca971e9170da33785 100644 --- a/tests/test_cosalib_cmdlib.py +++ b/tests/test_cosalib_cmdlib.py @@ -102,53 +102,6 @@ def test_rm_allow_noent(tmpdir): cmdlib.rm_allow_noent(test_path) -def test_import_ostree_commit(monkeypatch, tmpdir): - """ - Verify the correct ostree/tar commands are executed when - import_ostree_commit is called. - """ - repo_tmp = os.path.join(tmpdir, 'tmp') - os.mkdir(repo_tmp) - - class monkeyspcheck_call: - """ - Verifies each subprocess.check_call matches what is required. - """ - check_call_count = 0 - - def __call__(self, *args, **kwargs): - if self.check_call_count == 0: - assert args[0] == [ - 'ostree', 'init', '--repo', tmpdir, '--mode=archive'] - if self.check_call_count == 1: - assert args[0][0:2] == ['tar', '-C'] - assert args[0][3:5] == ['-xf', './tarfile.tar'] - if self.check_call_count == 2: - assert args[0][0:4] == [ - 'ostree', 'pull-local', '--repo', tmpdir] - assert args[0][5] == 'commit' - self.check_call_count += 1 - - def monkeyspcall(*args, **kwargs): - """ - Verifies suprocess.call matches what we need. - """ - assert args[0] == ['ostree', 'show', '--repo', tmpdir, 'commit'] - - # Monkey patch the subprocess function - monkeypatch.setattr(subprocess, 'check_call', monkeyspcheck_call()) - monkeypatch.setattr(subprocess, 'call', monkeyspcall) - build = { - 'ostree-commit': 'commit', - 'images': { - 'ostree': { - 'path': 'tarfile.tar' - } - } - } - cmdlib.import_ostree_commit(tmpdir, './', build) - - def test_image_info(tmpdir): cmdlib.run_verbose([ "qemu-img", "create", "-f", "qcow2", f"{tmpdir}/test.qcow2", "10M"])