diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf59be4..8be05f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ stages: variables: BUILDER: buildah - JUST_VERSION: "1.40.0" + JUST_VERSION: "1.49.0" .build-image: stage: build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b938ef9..1695c09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,3 +22,16 @@ FEDORA_VERSION=43 just test # different Fedora version BUILDER=podman just build # use podman instead of buildah just ci # full CI run (validate + test all tiers) ``` + +## Building a split image + +The Containerfile supports building a split (content-based layered) +image using [chunkah](https://github.com/coreos/chunkah) via the +`chunked` build target: + +```bash +just build --chunkah +``` + +Extra arguments can be passed to chunkah via the `CHUNKAH_ARGS` build +arg (e.g. `BUILDER_EXTRA='--build-arg CHUNKAH_ARGS="--max-layers 128"' just build --chunkah`). diff --git a/Containerfile b/Containerfile index 90bad06..1aa0330 100644 --- a/Containerfile +++ b/Containerfile @@ -2,9 +2,11 @@ # nested containerization, so you must build with e.g. # podman build --security-opt=label=disable --cap-add=all --device /dev/fuse <...> -# NOTE: This container build will output a single giant layer. It is strongly recommended -# to run the "rechunker" on the output of this build, see -# https://coreos.github.io/rpm-ostree/experimental-build-chunked-oci/ +# NOTE: This container build will output a single giant layer. You can either +# run the "rechunker" on the output of this build (see bootc-base-imagectl.md), +# or build the split version directly with `just build --chunkah` +# (or if using podman/buildah directly, add `--build-arg FINAL=chunked +# --skip-unused-stages=false -v $PWD:/run/src`). # Override this repos container to control the base image package versions. For # example, podman build --from=quay.io/fedora/fedora:41 will get you a system @@ -14,8 +16,10 @@ # since konflux doesn't yet support --from. ARG REPOS_IMAGE=quay.io/fedora/fedora:rawhide ARG BUILDER_IMAGE=quay.io/fedora/fedora:rawhide -FROM $REPOS_IMAGE as repos +# Either 'unchunked' or 'chunked'. Determines whether we take the chunkah path. +ARG FINAL=unchunked +FROM $REPOS_IMAGE as repos # BOOTSTRAPPING: This can be any image that has rpm-ostree, selinux-policy-targeted # and python3 (for bootc-base-imagectl). FROM $BUILDER_IMAGE as builder @@ -54,9 +58,19 @@ install -m 0755 -t /usr/libexec ./bootc-base-imagectl EORUN # This pulls in the rootfs generated in the previous step -FROM scratch +FROM scratch AS unchunked COPY --from=builder /target-rootfs/ / +FROM builder AS rechunker +RUN dnf -y install chunkah +ARG CHUNKAH_ARGS="" +RUN --mount=from=unchunked,src=/,target=/chunkah,ro \ + --mount=type=bind,target=/run/src,rw \ + /usr/libexec/bootc-base-imagectl rechunk --chunkah ${CHUNKAH_ARGS} \ + > /run/src/out.ociarchive +FROM oci-archive:out.ociarchive AS chunked + +FROM $FINAL LABEL containers.bootc 1 # This is an ad-hoc way for us to reference bootc-image-builder in # a way that in theory client tooling can inspect and find. Today diff --git a/Justfile b/Justfile index 3c954d9..eec0a0a 100644 --- a/Justfile +++ b/Justfile @@ -23,6 +23,7 @@ _build_cmd := builder + " build" _tag := image + if tier == "standard" { "" } else { ":" + tier } _base_image := "quay.io/fedora/fedora:" + fedora_version _version_args := if fedora_version == "rawhide" { "" } else { "--build-arg=REPOS_IMAGE=" + _base_image + " --build-arg=BUILDER_IMAGE=" + _base_image } +_chunkah_args := "--build-arg FINAL=chunked --skip-unused-stages=false -v " + justfile_directory() + ":/run/src" # ============================================================================ # Core targets @@ -30,10 +31,12 @@ _version_args := if fedora_version == "rawhide" { "" } else { "--build-arg=REPOS # Build the container image [group('core')] -build: _check-tier +[arg("chunkah", long, value="true")] +build chunkah="": _check-tier {{_build_cmd}} -f Containerfile --no-cache \ -t {{_tag}} {{priv_args}} \ {{_version_args}} {{builder_extra}} \ + {{if chunkah != "" { _chunkah_args } else { "" }}} \ --build-arg=MANIFEST=fedora-{{tier}} . # Build and test diff --git a/bootc-base-imagectl b/bootc-base-imagectl index 33f832a..2798273 100755 --- a/bootc-base-imagectl +++ b/bootc-base-imagectl @@ -135,22 +135,40 @@ def run_build_rootfs(args): shutil.copy('/' + f, dst) def run_rechunk(args): - argv = [ - 'rpm-ostree', - 'experimental', - 'compose', - 'build-chunked-oci'] - if args.max_layers is not None: - argv.append(f"--max-layers={args.max_layers}") - argv.extend(['--bootc', - '--format-version=1', - f'--from={args.from_image}', - f'--output=containers-storage:{args.to_image}']) - try: - subprocess.run(argv, check=True) - except subprocess.CalledProcessError as e: - print(f"Error executing command: {e}") - sys.exit(1) + if args.chunkah: + argv = ['chunkah', 'build', '--rootfs=/chunkah'] + if args.max_layers is not None: + argv.append(f"--max-layers={args.max_layers}") + # Strip OSTree data and labels for bootc compatibility; see + # https://github.com/coreos/chunkah#compatibility-with-bootable-bootc-images + argv.extend(['--prune', '/sysroot/', + '--label', 'ostree.commit-', + '--label', 'ostree.final-diffid-']) + try: + subprocess.run(argv, check=True) + except subprocess.CalledProcessError as e: + print(f"Error executing command: {e}", file=sys.stderr) + sys.exit(1) + else: + if not args.from_image or not args.to_image: + print("Error: from_image and to_image are required when not using --chunkah", file=sys.stderr) + sys.exit(1) + argv = [ + 'rpm-ostree', + 'experimental', + 'compose', + 'build-chunked-oci'] + if args.max_layers is not None: + argv.append(f"--max-layers={args.max_layers}") + argv.extend(['--bootc', + '--format-version=1', + f'--from={args.from_image}', + f'--output=containers-storage:{args.to_image}']) + try: + subprocess.run(argv, check=True) + except subprocess.CalledProcessError as e: + print(f"Error executing command: {e}") + sys.exit(1) def run_list(args): d = '/' + MANIFESTDIR @@ -190,9 +208,10 @@ if __name__ == "__main__": build_rootfs.set_defaults(func=run_build_rootfs) cmd_rechunk = subparsers.add_parser('rechunk', help="Generate a new container image with split, reproducible, chunked layers") + cmd_rechunk.add_argument("--chunkah", help="Use chunkah instead of rpm-ostree (reads rootfs from /chunkah, writes OCI archive to stdout)", action='store_true') cmd_rechunk.add_argument("--max-layers", help="Configure the number of output layers") - cmd_rechunk.add_argument("from_image", help="Operate on this image in the container storage") - cmd_rechunk.add_argument("to_image", help="Output a new image to the container storage") + cmd_rechunk.add_argument("from_image", help="Operate on this image in the container storage", nargs='?') + cmd_rechunk.add_argument("to_image", help="Output a new image to the container storage", nargs='?') cmd_rechunk.set_defaults(func=run_rechunk) cmd_list = subparsers.add_parser('list', help='List available manifests') diff --git a/bootc-base-imagectl.md b/bootc-base-imagectl.md index 311bb7a..b98d01c 100644 --- a/bootc-base-imagectl.md +++ b/bootc-base-imagectl.md @@ -47,6 +47,15 @@ This command takes just two arguments: - A path to the target root filesystem which will be generated as a directory. The target should not already exist (but its parent must exist). +### Implementation + +The current implementation uses `rpm-ostree` on a manifest (treefile) +embedded in the container image itself. These manifests are not intended +to be editable directly. + +To emphasize: the implementation of this command (especially the configuration +files that it reads) are subject to change. + ## Using bootc-base-imagectl rechunk This operation is strongly related to `build-rootfs` but is also orthogonal; @@ -95,6 +104,35 @@ tree (hence removed/overridden files are handled), and then splits it up Further, because bootc uses OSTree today, and OSTree canonializes all timestamps to zero on the client side, this tool does that at build time. +### Using chunkah instead of rpm-ostree + +The `--chunkah` flag switches rechunk to use [chunkah] instead of +rpm-ostree for layer splitting. In this mode, chunkah reads the rootfs +from `/chunkah` (its default) and writes an OCI archive to stdout. +The `from_image` and `to_image` positional arguments are not used. +The `--max-layers` option is respected and passed through to chunkah. + +This mode automatically passes `--prune /sysroot/` to strip OSTree data +and `--label ostree.commit-` / `--label ostree.final-diffid-` to remove +OSTree-specific labels. In other words, this produces plain OCI bootc images +without any OSTree content. + +To rechunk an existing image using chunkah: + +``` +IMG=quay.io/exampleos/exampleos:latest +podman run --rm --mount=type=image,src=$IMG,dest=/chunkah \ + -e CHUNKAH_CONFIG_STR="$(podman inspect $IMG)" \ + quay.io/fedora/fedora-bootc:rawhide \ + /usr/libexec/bootc-base-imagectl rechunk --chunkah | podman load +``` + +The `CHUNKAH_CONFIG_STR` environment variable passes the original +image's metadata (labels, environment, command, etc.) to chunkah so +that it is retained in the rechunked output. + +[chunkah]: https://github.com/coreos/chunkah + ### Other options `bootc-base-imagectl list` will enumerate available configurations that @@ -102,12 +140,9 @@ can be selected by passing `--manifest` to `build-rootfs`. ### Implementation -The current implementation uses `rpm-ostree` on a manifest (treefile) -embedded in the container image itself. These manifests are not intended -to be editable directly. - -To emphasize: the implementation of this command (especially the configuration -files that it reads) are subject to change. +The default rechunking implementation also uses `rpm-ostree`. The `--chunkah` +mode uses [chunkah] instead, which is content-agnostic and not tied to +rpm-ostree. ### Cross builds and the builder image diff --git a/standard/manifest.yaml b/standard/manifest.yaml index fdd3f75..6215270 100644 --- a/standard/manifest.yaml +++ b/standard/manifest.yaml @@ -45,6 +45,8 @@ packages: - python3-rpm # Used by admins interactively - man-db + # Content-based container layer splitting for rechunking + - chunkah # These are random architecture-specific packages packages-x86_64: diff --git a/tests/rootfs/cases/bootupd b/tests/rootfs/cases/bootupd index b03840b..7834227 100755 --- a/tests/rootfs/cases/bootupd +++ b/tests/rootfs/cases/bootupd @@ -1,7 +1,5 @@ #!/bin/bash set -xeuo pipefail -# Verify the bootupd EFI has more than one component installed -versions=$(cat /usr/lib/bootupd/updates/EFI.json | jq -r .version | tr ',' ' ') -array=($versions) -length=${#array[*]} -[ $length -gt 1 ] +# Verify the bootupd EFI has both grub2 and shim components installed. +# (Note we don't use jq here because that's not available in minimal.) +grep -q 'grub2-.*,shim-' /usr/lib/bootupd/updates/EFI.json