Merge branch 'pr/add-chunkah' into 'main'

Add chunkah integration

See merge request fedora/bootc/base-images!451

.gitlab-ci.yml

@@ -20,7 +20,7 @@ stages:
 
 variables:
   BUILDER: buildah
-  JUST_VERSION: "1.40.0"
+  JUST_VERSION: "1.49.0"
 
 .build-image:
   stage: build

CONTRIBUTING.md

@@ -22,3 +22,16 @@ FEDORA_VERSION=43 just test                   # different Fedora version
 BUILDER=podman just build                     # use podman instead of buildah
 just ci                                       # full CI run (validate + test all tiers)
 ```
+
+## Building a split image
+
+The Containerfile supports building a split (content-based layered)
+image using [chunkah](https://github.com/coreos/chunkah) via the
+`chunked` build target:
+
+```bash
+just build --chunkah
+```
+
+Extra arguments can be passed to chunkah via the `CHUNKAH_ARGS` build
+arg (e.g. `BUILDER_EXTRA='--build-arg CHUNKAH_ARGS="--max-layers 128"' just build --chunkah`).

Containerfile

@@ -2,9 +2,11 @@
 # nested containerization, so you must build with e.g.
 # podman build --security-opt=label=disable --cap-add=all --device /dev/fuse <...>
 
-# NOTE: This container build will output a single giant layer. It is strongly recommended
-# to run the "rechunker" on the output of this build, see
-# https://coreos.github.io/rpm-ostree/experimental-build-chunked-oci/
+# NOTE: This container build will output a single giant layer. You can either
+# run the "rechunker" on the output of this build (see bootc-base-imagectl.md),
+# or build the split version directly with `just build --chunkah`
+# (or if using podman/buildah directly, add `--build-arg FINAL=chunked
+# --skip-unused-stages=false -v $PWD:/run/src`).
 
 # Override this repos container to control the base image package versions. For
 # example, podman build --from=quay.io/fedora/fedora:41 will get you a system
@@ -14,8 +16,10 @@
 # since konflux doesn't yet support --from.
 ARG REPOS_IMAGE=quay.io/fedora/fedora:rawhide
 ARG BUILDER_IMAGE=quay.io/fedora/fedora:rawhide
-FROM $REPOS_IMAGE as repos
+# Either 'unchunked' or 'chunked'. Determines whether we take the chunkah path.
+ARG FINAL=unchunked
 
+FROM $REPOS_IMAGE as repos
 # BOOTSTRAPPING: This can be any image that has rpm-ostree, selinux-policy-targeted
 #                and python3 (for bootc-base-imagectl).
 FROM $BUILDER_IMAGE as builder
@@ -54,9 +58,19 @@ install -m 0755 -t /usr/libexec ./bootc-base-imagectl
 EORUN
 
 # This pulls in the rootfs generated in the previous step
-FROM scratch
+FROM scratch AS unchunked
 COPY --from=builder /target-rootfs/ /
 
+FROM builder AS rechunker
+RUN dnf -y install chunkah
+ARG CHUNKAH_ARGS=""
+RUN --mount=from=unchunked,src=/,target=/chunkah,ro \
+    --mount=type=bind,target=/run/src,rw \
+    /usr/libexec/bootc-base-imagectl rechunk --chunkah ${CHUNKAH_ARGS} \
+    > /run/src/out.ociarchive
+FROM oci-archive:out.ociarchive AS chunked
+
+FROM $FINAL
 LABEL containers.bootc 1
 # This is an ad-hoc way for us to reference bootc-image-builder in
 # a way that in theory client tooling can inspect and find. Today

Justfile

@@ -23,6 +23,7 @@ _build_cmd := builder + " build"
 _tag := image + if tier == "standard" { "" } else { ":" + tier }
 _base_image := "quay.io/fedora/fedora:" + fedora_version
 _version_args := if fedora_version == "rawhide" { "" } else { "--build-arg=REPOS_IMAGE=" + _base_image + " --build-arg=BUILDER_IMAGE=" + _base_image }
+_chunkah_args := "--build-arg FINAL=chunked --skip-unused-stages=false -v " + justfile_directory() + ":/run/src"
 
 # ============================================================================
 # Core targets
@@ -30,10 +31,12 @@ _version_args := if fedora_version == "rawhide" { "" } else { "--build-arg=REPOS
 
 # Build the container image
 [group('core')]
-build: _check-tier
+[arg("chunkah", long, value="true")]
+build chunkah="": _check-tier
     {{_build_cmd}} -f Containerfile --no-cache \
         -t {{_tag}} {{priv_args}} \
         {{_version_args}} {{builder_extra}} \
+        {{if chunkah != "" { _chunkah_args } else { "" }}} \
         --build-arg=MANIFEST=fedora-{{tier}} .
 
 # Build and test

bootc-base-imagectl

@@ -135,22 +135,40 @@ def run_build_rootfs(args):
             shutil.copy('/' + f, dst)
 
 def run_rechunk(args):
-    argv = [
-            'rpm-ostree',
-            'experimental',
-            'compose',
-            'build-chunked-oci']
-    if args.max_layers is not None:
-        argv.append(f"--max-layers={args.max_layers}")
-    argv.extend(['--bootc',
-            '--format-version=1',
-            f'--from={args.from_image}',
-            f'--output=containers-storage:{args.to_image}'])
-    try:
-        subprocess.run(argv, check=True)
-    except subprocess.CalledProcessError as e:
-        print(f"Error executing command: {e}")
-        sys.exit(1)
+    if args.chunkah:
+        argv = ['chunkah', 'build', '--rootfs=/chunkah']
+        if args.max_layers is not None:
+            argv.append(f"--max-layers={args.max_layers}")
+        # Strip OSTree data and labels for bootc compatibility; see
+        # https://github.com/coreos/chunkah#compatibility-with-bootable-bootc-images
+        argv.extend(['--prune', '/sysroot/',
+                '--label', 'ostree.commit-',
+                '--label', 'ostree.final-diffid-'])
+        try:
+            subprocess.run(argv, check=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Error executing command: {e}", file=sys.stderr)
+            sys.exit(1)
+    else:
+        if not args.from_image or not args.to_image:
+            print("Error: from_image and to_image are required when not using --chunkah", file=sys.stderr)
+            sys.exit(1)
+        argv = [
+                'rpm-ostree',
+                'experimental',
+                'compose',
+                'build-chunked-oci']
+        if args.max_layers is not None:
+            argv.append(f"--max-layers={args.max_layers}")
+        argv.extend(['--bootc',
+                '--format-version=1',
+                f'--from={args.from_image}',
+                f'--output=containers-storage:{args.to_image}'])
+        try:
+            subprocess.run(argv, check=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Error executing command: {e}")
+            sys.exit(1)
 
 def run_list(args):
     d = '/' + MANIFESTDIR
@@ -190,9 +208,10 @@ if __name__ == "__main__":
     build_rootfs.set_defaults(func=run_build_rootfs)
 
     cmd_rechunk = subparsers.add_parser('rechunk', help="Generate a new container image with split, reproducible, chunked layers")
+    cmd_rechunk.add_argument("--chunkah", help="Use chunkah instead of rpm-ostree (reads rootfs from /chunkah, writes OCI archive to stdout)", action='store_true')
     cmd_rechunk.add_argument("--max-layers", help="Configure the number of output layers")
-    cmd_rechunk.add_argument("from_image", help="Operate on this image in the container storage")
-    cmd_rechunk.add_argument("to_image", help="Output a new image to the container storage")
+    cmd_rechunk.add_argument("from_image", help="Operate on this image in the container storage", nargs='?')
+    cmd_rechunk.add_argument("to_image", help="Output a new image to the container storage", nargs='?')
     cmd_rechunk.set_defaults(func=run_rechunk)
 
     cmd_list = subparsers.add_parser('list', help='List available manifests')

bootc-base-imagectl.md

@@ -47,6 +47,15 @@ This command takes just two arguments:
 - A path to the target root filesystem which will be generated as
   a directory. The target should not already exist (but its parent must exist).
 
+### Implementation
+
+The current implementation uses `rpm-ostree` on a manifest (treefile)
+embedded in the container image itself. These manifests are not intended
+to be editable directly.
+
+To emphasize: the implementation of this command (especially the configuration
+files that it reads) are subject to change.
+
 ## Using bootc-base-imagectl rechunk
 
 This operation is strongly related to `build-rootfs` but is also orthogonal;
@@ -95,6 +104,35 @@ tree (hence removed/overridden files are handled), and then splits it up
 Further, because bootc uses OSTree today, and OSTree canonializes all timestamps
 to zero on the client side, this tool does that at build time.
 
+### Using chunkah instead of rpm-ostree
+
+The `--chunkah` flag switches rechunk to use [chunkah] instead of
+rpm-ostree for layer splitting. In this mode, chunkah reads the rootfs
+from `/chunkah` (its default) and writes an OCI archive to stdout.
+The `from_image` and `to_image` positional arguments are not used.
+The `--max-layers` option is respected and passed through to chunkah.
+
+This mode automatically passes `--prune /sysroot/` to strip OSTree data
+and `--label ostree.commit-` / `--label ostree.final-diffid-` to remove
+OSTree-specific labels. In other words, this produces plain OCI bootc images
+without any OSTree content.
+
+To rechunk an existing image using chunkah:
+
+```
+IMG=quay.io/exampleos/exampleos:latest
+podman run --rm --mount=type=image,src=$IMG,dest=/chunkah \
+  -e CHUNKAH_CONFIG_STR="$(podman inspect $IMG)" \
+  quay.io/fedora/fedora-bootc:rawhide \
+  /usr/libexec/bootc-base-imagectl rechunk --chunkah | podman load
+```
+
+The `CHUNKAH_CONFIG_STR` environment variable passes the original
+image's metadata (labels, environment, command, etc.) to chunkah so
+that it is retained in the rechunked output.
+
+[chunkah]: https://github.com/coreos/chunkah
+
 ### Other options
 
 `bootc-base-imagectl list` will enumerate available configurations that
@@ -102,12 +140,9 @@ can be selected by passing `--manifest` to `build-rootfs`.
 
 ### Implementation
 
-The current implementation uses `rpm-ostree` on a manifest (treefile)
-embedded in the container image itself. These manifests are not intended
-to be editable directly.
-
-To emphasize: the implementation of this command (especially the configuration
-files that it reads) are subject to change.
+The default rechunking implementation also uses `rpm-ostree`. The `--chunkah`
+mode uses [chunkah] instead, which is content-agnostic and not tied to
+rpm-ostree.
 
 ### Cross builds and the builder image
 

standard/manifest.yaml

@@ -45,6 +45,8 @@ packages:
   - python3-rpm
   # Used by admins interactively
   - man-db
+  # Content-based container layer splitting for rechunking
+  - chunkah
 
 # These are random architecture-specific packages
 packages-x86_64:

tests/rootfs/cases/bootupd

@@ -1,7 +1,5 @@
 #!/bin/bash
 set -xeuo pipefail
-# Verify the bootupd EFI has more than one component installed
-versions=$(cat /usr/lib/bootupd/updates/EFI.json | jq -r .version | tr ',' ' ')
-array=($versions)
-length=${#array[*]}
-[ $length -gt 1 ]
+# Verify the bootupd EFI has both grub2 and shim components installed.
+# (Note we don't use jq here because that's not available in minimal.)
+grep -q 'grub2-.*,shim-' /usr/lib/bootupd/updates/EFI.json