releng/tooling
10
0
Fork 14
Code Pull requests 5 Projects Releases Packages Wiki Activity Actions

Restore mass rebuild restart skips and fix live logging.
All checks were successful
/ test (push) Successful in 3s
Details

Browse source 
Skip packages that already have a checkout, so interrupted runs
Resume past cloned trees instead of deleting and recloning.
Line-buffer output for tee, we used to do it in the last script.

Signed-off-by: jnsamyak <samyak.jn11@gmail.com>
This commit is contained in:
Samyak Jain 2026-07-15 18:35:14 +00:00
commit 26278a8bf3
1 changed files with 57 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

110
release-process/mass-rebuilds/mass_rebuild.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/python3
#
# mass-rebuild.py - A utility to rebuild packages.
# mass_rebuild.py - A utility to rebuild packages.
#
# Copyright (C) 2009-2013 Red Hat, Inc.
# SPDX-License-Identifier: GPL-2.0+
@ -23,12 +23,11 @@ import random
# here
from mass_rebuilds_info import MASSREBUILDS
# Configuration for retry logic
MAX_ATTEMPTS = 3 # Total attempts (1 initial + 2 retries)
RETRY_DELAY_MIN = 2 # Minimum delay in seconds between retries
RETRY_DELAY_MAX = 10 # Maximum delay in seconds between retries
# Backward-compatible alias
MAX_RETRIES = MAX_ATTEMPTS
# Retry: total attempts and short backoff (no sleep after the last failure)
MAX_ATTEMPTS = 3
RETRY_DELAY_MIN = 2
RETRY_DELAY_MAX = 10
MAX_RETRIES = MAX_ATTEMPTS # alias
# Set some variables
# Some of these could arguably be passed in as args.
@ -42,10 +41,7 @@ enviro = os.environ
def retry(func, *args, retries=MAX_ATTEMPTS, delay_min=RETRY_DELAY_MIN,
delay_max=RETRY_DELAY_MAX, success_check=None, failure_value=1, **kwargs):
"""Retry logic wrapper function.
Does not sleep after the final failed attempt.
"""
"""Retry wrapper. Does not sleep after the final failed attempt."""
if success_check is None:
success_check = lambda result: result == 0
for attempt in range(retries):
@ -55,9 +51,10 @@ def retry(func, *args, retries=MAX_ATTEMPTS, delay_min=RETRY_DELAY_MIN,
if attempt + 1 >= retries:
break
delay = random.uniform(delay_min, delay_max)
print(f"Attempt {attempt + 1} failed. Retrying in {int(delay)} seconds...")
print('Attempt %s failed. Retrying in %s seconds...' % (
attempt + 1, int(delay)))
time.sleep(delay)
print(f"All {retries} attempts failed.")
print('All %s attempts failed.' % retries)
return failure_value
@ -67,11 +64,14 @@ def buildmeoutput(cmd, action, pkg, env, cwd=workdir, retries=1):
try:
output = subprocess.check_output(cmd, env=env, cwd=cwd).decode('utf-8').split()
if len(output) < 3:
sys.stderr.write('%s failed %s: unexpected output: %s\n' % (pkg, action, ' '.join(output)))
sys.stderr.write('%s failed %s: unexpected output: %s\n' % (
pkg, action, ' '.join(output)))
return 1
with open(workdir + "/taskID_file", 'a') as task_file:
with open(os.path.join(workdir, 'taskID_file'), 'a') as task_file:
task_file.write('%s %s\n' % (pkg, output[2]))
sys.stdout.write(' Successful submission: %s taskID: %s\n' % (pkg, output[2]))
sys.stdout.write(' Successful submission: %s taskID: %s\n' % (
pkg, output[2]))
sys.stdout.flush()
return 0
except subprocess.CalledProcessError as e:
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
@ -80,7 +80,7 @@ def buildmeoutput(cmd, action, pkg, env, cwd=workdir, retries=1):
def runme(cmd, action, pkg, env, cwd=workdir, retries=MAX_ATTEMPTS):
"""Run a command; retries for transient failures. Use retries=1 for non-idempotent ops."""
"""Run a command. Use retries=1 for non-idempotent ops (bumpspec, build)."""
def do_attempt():
try:
subprocess.check_call(cmd, env=env, cwd=cwd)
@ -100,51 +100,56 @@ def runmeoutput(cmd, action, pkg, env, cwd=workdir, retries=MAX_ATTEMPTS):
result = pid.communicate()[0].rstrip('\n')
if pid.returncode == 0:
return result
else:
return None
return None
except BaseException as e:
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
return None
def success_check(value):
return value is not None
return retry(do_attempt, retries=retries, success_check=success_check,
return retry(do_attempt, retries=retries,
success_check=lambda value: value is not None,
failure_value=None)
def main():
# Environment for using releng credentials for pushing and building
# Line-buffer so `... 2>&1 | tee logfile` updates live
if hasattr(sys.stdout, 'reconfigure'):
sys.stdout.reconfigure(line_buffering=True)
if hasattr(sys.stderr, 'reconfigure'):
sys.stderr.reconfigure(line_buffering=True)
enviro['GIT_SSH'] = '/usr/local/bin/relengpush'
koji_bin = '/usr/bin/compose-koji'
# Create a koji session
os.makedirs(workdir, exist_ok=True)
kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')
# Generate a list of packages to iterate over
pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True)
# reduce the list to those that are not blocked and sort by package name
pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']],
key=operator.itemgetter('package_name'))
print('mass_rebuild.py starting (rebuildid=%s)' % rebuildid)
print('script=%s' % os.path.abspath(__file__))
print('workdir=%s' % workdir)
print('Checking %s packages...' % len(pkgs))
# Loop over each package
for pkg in pkgs:
name = pkg['package_name']
id = pkg['package_id']
pkgdir = os.path.join(workdir, name)
# some package we just dont want to ever rebuild
if name in massrebuild['pkg_skip_list']:
print('Skipping %s, package is explicitely skipped')
print('Skipping %s, package is explicitely skipped' % name)
continue
# Query to see if a build has already been attempted
# Restart: skip packages that already have a checkout (old script
# behavior). Only packages without a local dir are processed.
if os.path.exists(pkgdir):
print('Skipping %s, already checked out.' % name)
continue
# Skip if a build was already attempted for this mass rebuild
builds = kojisession.listBuilds(id, createdAfter=massrebuild['epoch'])
newbuild = False
# Check the builds to make sure they were for the target we care about
for build in builds:
try:
buildtarget = kojisession.getTaskInfo(build['task_id'],
@ -152,54 +157,55 @@ def main():
if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']:
newbuild = True
break
except:
except Exception:
print('Skipping %s, no taskinfo.' % name)
continue
if newbuild:
print('Skipping %s, already attempted.' % name)
continue
# Remove leftover checkout so clone retries are clean
if os.path.exists(pkgdir):
shutil.rmtree(pkgdir)
# Check out git
fedpkgcmd = ['fedpkg', '--user', 'releng', 'clone', '--branch', 'rawhide', name]
print('Checking out %s' % name)
if runme(fedpkgcmd, 'fedpkg', name, enviro):
def do_clone():
# Clean partial tree left by a previous failed attempt
if os.path.exists(pkgdir):
shutil.rmtree(pkgdir)
try:
subprocess.check_call(fedpkgcmd, env=enviro, cwd=workdir)
return 0
except subprocess.CalledProcessError as e:
sys.stderr.write('%s failed fedpkg: %s\n' % (name, e))
return 1
if retry(do_clone):
if os.path.exists(pkgdir):
shutil.rmtree(pkgdir)
continue
# Check for a checkout
if not os.path.exists(pkgdir):
sys.stderr.write('%s failed checkout.\n' % name)
continue
# Check for a noautobuild file
if os.path.exists(os.path.join(pkgdir, 'noautobuild')):
print('Skipping %s due to opt-out' % name)
continue
# Find the spec file
files = os.listdir(pkgdir)
spec = ''
for file in files:
for file in os.listdir(pkgdir):
if file.endswith('.spec'):
spec = os.path.join(pkgdir, file)
break
if not spec:
sys.stderr.write('%s failed spec check\n' % name)
continue
# rpmdev-bumpspec — single attempt (not idempotent)
# Non-idempotent: single attempt
bumpspec = ['rpmdev-bumpspec', '-D', '-u', user, '-c', comment, spec]
print('Bumping %s' % spec)
if runme(bumpspec, 'bumpspec', name, enviro, retries=1):
continue
# Set the git user.name and user.email
set_name = ['git', 'config', 'user.name', 'Fedora Release Engineering']
set_mail = ['git', 'config', 'user.email', 'releng@fedoraproject.org']
print('Setting git user.name and user.email')
@ -208,26 +214,24 @@ def main():
if runme(set_mail, 'set_mail', name, enviro, cwd=pkgdir, retries=1):
continue
# git commit — single attempt
commit = ['git', 'commit', '-a', '-m', comment, '--allow-empty']
print('Committing changes for %s' % name)
if runme(commit, 'commit', name, enviro, cwd=pkgdir, retries=1):
continue
# git push — retries OK (mostly idempotent)
# Retries OK
push = ['git', 'push', '--no-verify']
print('Pushing changes for %s' % name)
if runme(push, 'push', name, enviro, cwd=pkgdir):
continue
# get git url
urlcmd = ['fedpkg', 'giturl']
print('Getting git url for %s' % name)
url = runmeoutput(urlcmd, 'giturl', name, enviro, cwd=pkgdir)
if not url:
continue
# build — single attempt (not idempotent)
# Non-idempotent: single attempt
build = [koji_bin, 'build', '--nowait', '--background', '--fail-fast',
massrebuild['target'], url]
print('Building %s' % name)