releng/pungi-fedora
13
3
Fork 12
Code Issues 8 Pull requests 1 Projects Releases Packages Wiki Activity Actions
pungi-fedora/nightly.sh
Kevin Fenzi bbcdc37944
All checks were successful
Fedora Pungi Compose CI Validation / validate (push) Successful in 27s
Details
nightly: stop posting rawhide summary reports to devel list 
There's some folks that really don't like it and to be fair there's not
much utility for it being there. Not much quoting it or using it to
start discussions, so lets only send to the test reports list.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>
2026-07-31 10:08:01 -07:00

244 lines
9.7 KiB
Bash
Executable file
Raw Permalink Blame History

#!/bin/sh
CONFIG="fedora.conf"
TARGET_DIR="/mnt/koji/compose/rawhide"
MD_ARCHIVE_DIR="/mnt/koji/compose/metadata-archive"
SERIES_DIR="$MD_ARCHIVE_DIR/series"
#OLD_COMPOSES_DIR="--old-composes=/mnt/fedora_koji/compose/f23 --old-composes=$TARGET_DIR"
NIGHTLY="--nightly"
SKIP_PHASES=""
DEST=$(pwd)
DATE=$(date "+%Y%m%d")
SHORT="Fedora"
RELEASE="rawhide"
RELEASE_TITLE="Rawhide"
COMPSFILE="comps-rawhide.xml"
TMPDIR=`mktemp -d /tmp/$RELEASE.$DATE.XXXX`
TOMAIL="test-reports@lists.fedoraproject.org"
FROM="Fedora Rawhide Report <rawhide@fedoraproject.org>"
RSYNCPREFIX="sudo -u ftpsync"
RSYNCTARGET="/pub/fedora/linux/development/$RELEASE"
RSYNCALTTARGET="/pub/alt/development/$RELEASE"
RSYNCSECTARGET="/pub/fedora-secondary/development/$RELEASE"
OSTREESRCREPO="/mnt/koji/compose/ostree/repo/"
OSTREEDESTREPO="/mnt/koji/ostree/repo/"
OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)
# uncomment and edit for resuming a failed compose
#COMPOSE_ID="Fedora-23-20150530.n.0"
# assume a releng dir is a git checkout of the releng tooling repo
# if it does not exist clone it
if [ -d releng ]; then
pushd releng
git pull --rebase
popd
else
git clone https://forge.fedoraproject.org/releng/tooling.git releng
fi
# Set up our fedora messaging function, using the releng repo definition
FEDMSG_MODNAME="compose"
FEDMSG_CERTPREFIX="releng"
. ./releng/infrastructure/messaging/fedora_messaging.sh
# Announce that we are starting, even though we don't know the compose_id yet..
fedora_message_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s"}' "$RELEASE" "$ARCH" "$SHORT")
send_fedora_message "${fedora_message_json_start}" ${RELEASE} start
#pushd $TMPDIR
#git clone https://forge.fedoraproject.org/releng/fedora-comps.git && {
# pushd fedora-comps
# make "${COMPSFILE}"
# cp "${COMPSFILE}" $DEST/
# popd
#}
#popd
CMD="pungi-koji --notification-script=/usr/bin/pungi-fedmsg-notification --notification-script=pungi-wait-for-signed-ostree-handler --config=$CONFIG --old-composes=$TARGET_DIR $OLD_COMPOSES_DIR $NIGHTLY $SKIP_PHASES"
if [ -z "$COMPOSE_ID" ]; then
CMD="$CMD --target-dir=$TARGET_DIR"
else
CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID"
fi
time $CMD "$@"
if [ "$?" != "0" ]; then
exit 1
fi
NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)
SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')
# Set this to use later for a few items include depcheck
DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID
# Public URLs the synced compose will wind up at, we put them in fedora-messaging
LOCATION="https://dl.fedoraproject.org$RSYNCTARGET"
ALT_LOCATION="https://dl.fedoraproject.org$RSYNCALTTARGET"
SECONDARY_LOCATION="https://dl.fedoraproject.org$RSYNCSECTARGET"
# Update fedora_message template
fedora_message_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s", "alt_location": "%s", "secondary_location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION", "$ALT_LOCATION", "$SECONDARY_LOCATION")
fedora_message_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s", "alt_location": "%s", "secondary_location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID" "$LOCATION" "$ALT_LOCATION" "$SECONDARY_LOCATION")
# Fix permissions on the grub efi files and fonts (they're 0600)
chmod -R go+r $DESTDIR/compose/*/*/os/EFI/
if ! compose-changelog -p "$DESTDIR/logs/" "$TARGET_DIR/$OLDCOMPOSE_ID/" "$DESTDIR/" 2>"$DESTDIR/logs/changelog.stderr"; then
# Generating changelog failed. We should not send an empty announcement to
# general public.
TOMAIL=""
VERSION="$(rpm -q compose-utils)"
# Instead report it to rel-eng@ list.
mutt -e "set from=\"$FROM\"" \
-e 'set envelope_from=yes' \
-s "Generating changelog for $NEWCOMPOSE_ID failed (with $VERSION)" \
rel-eng@lists.fedoraproject.org \
< "$DESTDIR/changelog.stderr"
fi
# Figure out a version for broken deps e-mail that goes to package maintainers.
# In Rawhide it's just rawhide, for branched versions we prepend F- to the number.
if [ "$RELEASE" = "rawhide" ]; then
TREENAME="$RELEASE"
else
TREENAME="F-$RELEASE"
fi
[ -z "$ARCH" ] && {
./releng/quality-assurance/critpath/critpath.py --composeurl file://$DESTDIR/compose -o $DESTDIR/logs/critpath.txt rawhide &> $DESTDIR/logs/critpath.log
}
# Tell interested persons that the rsync is starting (zomg!)
send_fedora_message "${fedora_message_json_start}" ${RELEASE} rsync.start
# Sync the content to /pub/fedora
if [ ! -d "$RSYNCTARGET" ]; then
mkdir "$RSYNCTARGET"
chown ftpsync:ftpsync "$RSYNCTARGET"
fi
# aarch64/x86_64 for Everything Cloud Container Server Workstation is primary
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=x86_64 --arch src \
"$DESTDIR" "$RSYNCTARGET/" \
--variant Everything \
--variant Cloud \
--variant Container \
--variant KDE \
--variant Server \
--variant Spins \
--variant Silverblue \
--variant Kinoite \
--variant Sericea \
--variant Onyx \
--variant COSMIC-Atomic \
--variant Workstation \
--link-dest="$RSYNCTARGET/Everything" --exclude=repodata
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=x86_64 --arch src \
"$DESTDIR" "$RSYNCTARGET/" \
--variant Everything \
--variant Cloud \
--variant Container \
--variant KDE \
--variant Server \
--variant Spins \
--variant Silverblue \
--variant Kinoite \
--variant Sericea \
--variant Onyx \
--variant COSMIC-Atomic \
--variant Workstation \
--link-dest="$RSYNCTARGET/Everything" --delete-after
$RSYNCPREFIX rm "$RSYNCTARGET/.composeinfo"
$RSYNCPREFIX ./releng/builds/compose/build_composeinfo.py "$RSYNCTARGET/" --name "$NEWCOMPOSE_ID"
# hardlink content (this will hardlink aarch64 with the other arches)
time hardlink -v "$RSYNCTARGET/.."
# Sync the content to /pub/alt
if [ ! -d "$RSYNCALTTARGET" ]; then
mkdir "$RSYNCALTTARGET"
chown ftpsync:ftpsync "$RSYNCALTTARGET"
fi
# labs artefacts are alternative
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=x86_64 \
"$DESTDIR" "$RSYNCALTTARGET/" \
--variant Labs \
--link-dest="$RSYNCTARGET/Everything/" --exclude=repodata
$RSYNCPREFIX compose-partial-copy --arch=aarch64 --arch=x86_64 \
"$DESTDIR" "$RSYNCALTTARGET/" \
--variant Labs \
--link-dest="$RSYNCTARGET/Everything/" --delete-after
$RSYNCPREFIX rm "$RSYNCALTTARGET/.composeinfo"
$RSYNCPREFIX ./releng/builds/compose/build_composeinfo.py "$RSYNCALTTARGET/" --name "$NEWCOMPOSE_ID"
# hardlink content (this will hardlink aarch64 with the other arches)
time hardlink -v "$RSYNCTARGET/.."
# Sync the content to /pub/fedora-secondary
if [ ! -d "$RSYNCSECTARGET" ]; then
mkdir "$RSYNCSECTARGET"
chown ftpsync:ftpsync "$RSYNCSECTARGET"
fi
# all ppc64le/s390x artefacts are secondary
$RSYNCPREFIX compose-partial-copy --arch=ppc64le --arch=s390x \
"$DESTDIR" "$RSYNCSECTARGET/" \
--variant Everything --variant Cloud --variant Container --variant KDE \
--variant Labs --variant Server --variant Spins --variant Workstation \
--link-dest="$RSYNCTARGET/Everything/" --link-dest="$RSYNCSECTARGET/Everything/" --exclude=repodata
$RSYNCPREFIX compose-partial-copy --arch=ppc64le --arch=s390x \
"$DESTDIR" "$RSYNCSECTARGET/" \
--variant Everything --variant Cloud --variant Container --variant KDE \
--variant Labs --variant Server --variant Spins --variant Workstation \
--link-dest="$RSYNCTARGET/Everything/" --link-dest="$RSYNCSECTARGET/Everything/" --delete-after
$RSYNCPREFIX rm "$RSYNCSECTARGET/.composeinfo"
$RSYNCPREFIX ./releng/builds/compose/build_composeinfo.py "$RSYNCSECTARGET/" --name "$NEWCOMPOSE_ID"
# hardlink content (this will hardlink ppc64le/s390x)
time hardlink -v "$RSYNCSECTARGET/.."
# sync Silverblue, Kinoite, Sericea, Onyx, and COSMIC-Atomic to the unified ostree repo
# Set the umask to so directories can get group write
# https://forge.fedoraproject.org/releng/tickets/issues/8811
old_umask=$(umask)
umask 0002
for variant in 'silverblue' 'kinoite' 'sericea' 'onyx' 'cosmic-atomic'; do
for arch in x86_64 aarch64; do
ref="fedora/rawhide/${arch}/${variant}"
if ! ostree --repo=$OSTREESRCREPO rev-parse "${ref}"; then continue; fi
ostree pull-local --repo=$OSTREEDESTREPO $OSTREESRCREPO --depth=-1 "${ref}"
ostree summary -u --repo=$OSTREEDESTREPO # update summary file
done
done
# Set to old umask
umask $old_umask
# Tell interested persons that the rsync is done.
send_fedora_message "${fedora_message_json_done}" ${RELEASE} rsync.complete
# Tell everyone by fedora_message about the compose
send_fedora_message "${fedora_message_json_done}" ${RELEASE} complete
# Tell everyone by email about the compose
# "$DESTDIR/logs/depcheck" lets not cat out depcheck for now as it does
# not understand rich dependencies
SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'
for tomail in $TOMAIL ; do
cat $DESTDIR/logs/*verbose | \
mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail
done
# Copy the metadata to the non-garbage-collected metadata archive
YEAR=$(echo "${SHORTCOMPOSE_ID}" | cut -c1-4)
mkdir -p "${MD_ARCHIVE_DIR}/${YEAR}/${NEWCOMPOSE_ID}"
cp "${DESTDIR}"/compose/metadata/*.json "${MD_ARCHIVE_DIR}/${YEAR}/${NEWCOMPOSE_ID}/"
# Add the compose ID to the 'series' file which records the
# compose IDs for each SHORT name in order
mkdir -p "${SERIES_DIR}"
echo "${NEWCOMPOSE_ID}" >> "${SERIES_DIR}/${SHORT}-${RELEASE}"
# Removed all the older than 14 days composes
find $TARGET_DIR -xdev -depth -maxdepth 2 -mtime +14 -exec rm -rf {} \;
send_fedora_message "${fedora_message_json_done}" ${RELEASE} cleanup.complete
Reference in a new issue View git blame Copy permalink