Jesús De Los Reyes Larraga
9802566ec9
All checks were successful
/ test (push) Successful in 2s
This PR addresses ticket #13425 by centralizing the release configuration for the mass rebuild orchestration tools, establishing a single source of truth for future Fedora release cycles. Previously, scripts like `find_failures.py` and `need_rebuild.py` relied on hardcoded variables (e.g., `rebuildid = 'f45'`) that required manual bumps in multiple files. **Changes Made:** * **Centralized Configuration:** Removed hardcoded variables from `find_failures.py` and `need_rebuild.py`. They now import the `MASSREBUILDS` dictionary from `mass_rebuilds_info.py` to pull `buildtag`, `desttag`, `epoch`, and `targets`. * **Dynamic Resolution:** By default, scripts will now dynamically calculate and target the highest release version available in the dictionary. * **Added CLI Flag:** Introduced a `--rebuildid` argument via `argparse` to allow operators to override the default and target specific older releases (e.g., `--rebuildid f44`). * **Documentation:** Appended a usage section to the `README.md` detailing the new flag and the centralized configuration source. **Testing Notes:** * Verified the scripts parse arguments and load the dictionary correctly locally via `--help`. * *Note:* The existing `test_mass_rebuild.py` test suite only covers the network retry mechanisms in `mass_rebuild.py` and does not currently mock or cover `find_failures.py` or `need_rebuild.py`. **Resolves:** #13425 Reviewed-on: #13045
400 lines
15 KiB
Python
Executable file
400 lines
15 KiB
Python
Executable file
#!/usr/bin/python3
|
|
#
|
|
# mass_rebuild.py - A utility to rebuild packages.
|
|
#
|
|
# Copyright (C) 2009-2013 Red Hat, Inc.
|
|
# SPDX-License-Identifier: GPL-2.0+
|
|
#
|
|
# Authors:
|
|
# Jesse Keating <jkeating@redhat.com>
|
|
#
|
|
|
|
from __future__ import print_function
|
|
import koji
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import operator
|
|
import time
|
|
import random
|
|
|
|
# contains info about all rebuilds, add new rebuilds there and update rebuildid
|
|
# here
|
|
from mass_rebuilds_info import get_rebuild_config
|
|
|
|
# Retry: total attempts and short backoff (no sleep after the last failure)
|
|
MAX_ATTEMPTS = 3
|
|
RETRY_DELAY_MIN = 2
|
|
RETRY_DELAY_MAX = 10
|
|
MAX_RETRIES = MAX_ATTEMPTS # alias
|
|
|
|
|
|
|
|
# Set some variables
|
|
current_rebuild = get_rebuild_config(description="Script to find failed builds")
|
|
user = 'Fedora Release Engineering <releng@fedoraproject.org>'
|
|
comment = 'Rebuilt for ' + current_rebuild['wikipage']
|
|
workdir = os.path.expanduser('~/massbuild')
|
|
enviro = os.environ
|
|
SKIP_REPORT_NAME = 'skipped_distgit_mismatch.txt'
|
|
RELENG_AUTHOR = user
|
|
REBUILD_SUBJECT_PREFIX = 'Rebuilt for '
|
|
|
|
|
|
def parse_git_commit_from_source(source):
|
|
"""Return the git commit hash from a Koji build source URL, or None."""
|
|
if not source or '#' not in source:
|
|
return None
|
|
commit = source.rsplit('#', 1)[-1].strip()
|
|
return commit or None
|
|
|
|
|
|
def is_releng_rebuild_commit(author, subject, releng_author=RELENG_AUTHOR):
|
|
"""True if this commit looks like a releng mass-rebuild bumpspec commit."""
|
|
if not author or not subject:
|
|
return False
|
|
return (author.strip() == releng_author and
|
|
subject.strip().startswith(REBUILD_SUBJECT_PREFIX))
|
|
|
|
|
|
def append_distgit_skip_report(report_path, desttag, package_name):
|
|
"""Append package to the headed skip report; warn but do not raise on I/O errors."""
|
|
try:
|
|
with open(report_path, 'a') as report:
|
|
if report.tell() == 0:
|
|
report.write(
|
|
'Packages skipped: dist-git does not match latest tagged '
|
|
'build in %s\n' % desttag)
|
|
report.write(
|
|
'(Only releng mass-rebuild bumpspec commits on top of '
|
|
'the tagged source are allowed.)\n\n')
|
|
report.write('%s\n' % package_name)
|
|
except OSError as exc:
|
|
sys.stderr.write(
|
|
'WARNING: could not write skip report %s: %s\n' % (report_path, exc))
|
|
|
|
|
|
def retry(func, *args, retries=MAX_ATTEMPTS, delay_min=RETRY_DELAY_MIN,
|
|
delay_max=RETRY_DELAY_MAX, success_check=None, failure_value=1, **kwargs):
|
|
"""Retry wrapper. Does not sleep after the final failed attempt."""
|
|
if success_check is None:
|
|
success_check = lambda result: result == 0
|
|
for attempt in range(retries):
|
|
result = func(*args, **kwargs)
|
|
if success_check(result):
|
|
return result
|
|
if attempt + 1 >= retries:
|
|
break
|
|
delay = random.uniform(delay_min, delay_max)
|
|
print('Attempt %s failed. Retrying in %s seconds...' % (
|
|
attempt + 1, int(delay)))
|
|
time.sleep(delay)
|
|
print('All %s attempts failed.' % retries)
|
|
return failure_value
|
|
|
|
|
|
def buildmeoutput(cmd, action, pkg, env, cwd=workdir, retries=1):
|
|
"""Submit a koji build. Defaults to a single attempt (not idempotent)."""
|
|
def do_attempt():
|
|
try:
|
|
output = subprocess.check_output(cmd, env=env, cwd=cwd).decode('utf-8').split()
|
|
if len(output) < 3:
|
|
sys.stderr.write('%s failed %s: unexpected output: %s\n' % (
|
|
pkg, action, ' '.join(output)))
|
|
return 1
|
|
with open(os.path.join(workdir, 'taskID_file'), 'a') as task_file:
|
|
task_file.write('%s %s\n' % (pkg, output[2]))
|
|
sys.stdout.write(' Successful submission: %s taskID: %s\n' % (
|
|
pkg, output[2]))
|
|
sys.stdout.flush()
|
|
return 0
|
|
except subprocess.CalledProcessError as e:
|
|
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
|
|
return 1
|
|
return retry(do_attempt, retries=retries)
|
|
|
|
|
|
def runme(cmd, action, pkg, env, cwd=workdir, retries=MAX_ATTEMPTS):
|
|
"""Run a command. Use retries=1 for non-idempotent ops (bumpspec, build)."""
|
|
def do_attempt():
|
|
try:
|
|
subprocess.check_call(cmd, env=env, cwd=cwd)
|
|
return 0
|
|
except subprocess.CalledProcessError as e:
|
|
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
|
|
return 1
|
|
return retry(do_attempt, retries=retries)
|
|
|
|
|
|
def runmeoutput(cmd, action, pkg, env, cwd=workdir, retries=MAX_ATTEMPTS):
|
|
"""Run a command and return stdout on success, None on failure."""
|
|
def do_attempt():
|
|
try:
|
|
pid = subprocess.Popen(cmd, env=env, cwd=cwd,
|
|
stdout=subprocess.PIPE, encoding='utf8')
|
|
result = pid.communicate()[0].rstrip('\n')
|
|
if pid.returncode == 0:
|
|
return result
|
|
return None
|
|
except BaseException as e:
|
|
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
|
|
return None
|
|
|
|
return retry(do_attempt, retries=retries,
|
|
success_check=lambda value: value is not None,
|
|
failure_value=None)
|
|
|
|
|
|
def should_rebuild_from_distgit(pkgdir, tagged_commit, pkg, env=None,
|
|
releng_author=RELENG_AUTHOR):
|
|
"""Allow rebuild if dist-git matches tagged source (or only releng bumps on top).
|
|
|
|
tagged_commit None means no build in desttag → allow.
|
|
Returns False (skip) when ancestry cannot be verified.
|
|
Logs a specific reason for every False return.
|
|
"""
|
|
if env is None:
|
|
env = enviro
|
|
if not tagged_commit:
|
|
return True
|
|
|
|
# Use runmeoutput (retries=1) for probe commands. merge-base --is-ancestor
|
|
# stays as subprocess.call: exit 1 means "not an ancestor" (normal skip),
|
|
# not a failed command, so runme/runmeoutput retry would be wrong.
|
|
head = runmeoutput(
|
|
['git', 'rev-parse', 'HEAD'], 'git-rev-parse-HEAD', pkg, env,
|
|
cwd=pkgdir, retries=1)
|
|
if not head:
|
|
print('Skipping %s, could not read git HEAD' % pkg)
|
|
return False
|
|
|
|
tagged_full = runmeoutput(
|
|
['git', 'rev-parse', '--verify', tagged_commit + '^{commit}'],
|
|
'git-rev-parse-tagged', pkg, env, cwd=pkgdir, retries=1)
|
|
if not tagged_full:
|
|
print('Skipping %s, tagged commit %s not found in clone' % (
|
|
pkg, tagged_commit))
|
|
return False
|
|
|
|
if head == tagged_full:
|
|
return True
|
|
|
|
rc = subprocess.call(
|
|
['git', 'merge-base', '--is-ancestor', tagged_full, head],
|
|
cwd=pkgdir, env=env,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
if rc != 0:
|
|
print(
|
|
'Skipping %s, tagged commit %s is not an ancestor of HEAD %s '
|
|
'(merge-base rc=%s)' % (pkg, tagged_full[:12], head[:12], rc))
|
|
return False
|
|
|
|
log = runmeoutput(
|
|
['git', 'log', '--format=%an <%ae>\t%s',
|
|
'%s..%s' % (tagged_full, head)],
|
|
'git-log-since-tagged', pkg, env, cwd=pkgdir, retries=1)
|
|
if log is None:
|
|
print('Skipping %s, could not list commits between tagged and HEAD' % pkg)
|
|
return False
|
|
|
|
for line in log.splitlines():
|
|
if not line.strip():
|
|
continue
|
|
if '\t' not in line:
|
|
print('Skipping %s, unexpected git log line: %r' % (pkg, line))
|
|
return False
|
|
author, subject = line.split('\t', 1)
|
|
if not is_releng_rebuild_commit(author, subject, releng_author):
|
|
print(
|
|
'Skipping %s, non-releng commit since tagged build: '
|
|
'%s - %s' % (pkg, author, subject))
|
|
return False
|
|
return True
|
|
|
|
|
|
def main():
|
|
# Line-buffer so `... 2>&1 | tee logfile` updates live
|
|
if hasattr(sys.stdout, 'reconfigure'):
|
|
sys.stdout.reconfigure(line_buffering=True)
|
|
if hasattr(sys.stderr, 'reconfigure'):
|
|
sys.stderr.reconfigure(line_buffering=True)
|
|
|
|
enviro['GIT_SSH'] = '/usr/local/bin/relengpush'
|
|
koji_bin = '/usr/bin/compose-koji'
|
|
|
|
os.makedirs(workdir, exist_ok=True)
|
|
|
|
kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')
|
|
pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True)
|
|
pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']],
|
|
key=operator.itemgetter('package_name'))
|
|
|
|
print('mass_rebuild.py starting (rebuildid=%s)' % rebuildid)
|
|
print('script=%s' % os.path.abspath(__file__))
|
|
print('workdir=%s' % workdir)
|
|
print('Checking %s packages...' % len(pkgs))
|
|
desttag = massrebuild['desttag']
|
|
skip_report = os.path.join(workdir, SKIP_REPORT_NAME)
|
|
distgit_skips = 0
|
|
|
|
for pkg in pkgs:
|
|
name = pkg['package_name']
|
|
id = pkg['package_id']
|
|
pkgdir = os.path.join(workdir, name)
|
|
|
|
if name in massrebuild['pkg_skip_list']:
|
|
print('Skipping %s, package is explicitely skipped' % name)
|
|
continue
|
|
|
|
# Restart: skip packages that already have a checkout (old script
|
|
# behavior). Only packages without a local dir are processed.
|
|
if os.path.exists(pkgdir):
|
|
print('Skipping %s, already checked out.' % name)
|
|
continue
|
|
|
|
# Skip if a build was already attempted for this mass rebuild
|
|
builds = kojisession.listBuilds(id, createdAfter=massrebuild['epoch'])
|
|
newbuild = False
|
|
for build in builds:
|
|
try:
|
|
buildtarget = kojisession.getTaskInfo(build['task_id'],
|
|
request=True)['request'][1]
|
|
if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']:
|
|
newbuild = True
|
|
break
|
|
except Exception:
|
|
print('Skipping %s, no taskinfo.' % name)
|
|
continue
|
|
if newbuild:
|
|
print('Skipping %s, already attempted.' % name)
|
|
continue
|
|
|
|
fedpkgcmd = ['fedpkg', '--user', 'releng', 'clone', '--branch', 'rawhide', name]
|
|
print('Checking out %s' % name)
|
|
|
|
def do_clone():
|
|
# Clean partial tree left by a previous failed attempt
|
|
if os.path.exists(pkgdir):
|
|
shutil.rmtree(pkgdir)
|
|
try:
|
|
subprocess.check_call(fedpkgcmd, env=enviro, cwd=workdir)
|
|
return 0
|
|
except subprocess.CalledProcessError as e:
|
|
sys.stderr.write('%s failed fedpkg: %s\n' % (name, e))
|
|
return 1
|
|
|
|
if retry(do_clone):
|
|
if os.path.exists(pkgdir):
|
|
shutil.rmtree(pkgdir)
|
|
continue
|
|
|
|
if not os.path.exists(pkgdir):
|
|
sys.stderr.write('%s failed checkout.\n' % name)
|
|
continue
|
|
|
|
if os.path.exists(os.path.join(pkgdir, 'noautobuild')):
|
|
print('Skipping %s due to opt-out' % name)
|
|
continue
|
|
|
|
# Skip if dist-git has moved past the latest build in desttag
|
|
# (except pure releng mass-rebuild bumpspec commits).
|
|
tagged_commit = None
|
|
tagged_nvr = None
|
|
try:
|
|
tagged_builds = kojisession.listTagged(
|
|
desttag, latest=True, package=name)
|
|
except Exception as exc:
|
|
print('Skipping %s, failed to query tagged builds in %s: %s' % (
|
|
name, desttag, exc))
|
|
append_distgit_skip_report(skip_report, desttag, name)
|
|
shutil.rmtree(pkgdir)
|
|
distgit_skips += 1
|
|
continue
|
|
|
|
if tagged_builds:
|
|
tagged_build = tagged_builds[0]
|
|
tagged_nvr = tagged_build.get('nvr')
|
|
source = tagged_build.get('source')
|
|
if not source:
|
|
try:
|
|
tagged_build = kojisession.getBuild(
|
|
tagged_build['build_id'])
|
|
source = tagged_build.get('source')
|
|
except Exception as exc:
|
|
sys.stderr.write(
|
|
'WARNING: getBuild(%s) failed for %s: %s\n' % (
|
|
tagged_build['build_id'], name, exc))
|
|
source = None
|
|
tagged_commit = parse_git_commit_from_source(source)
|
|
if not tagged_commit:
|
|
print('Skipping %s, no git source on tagged build %s in %s' % (
|
|
name, tagged_nvr or '(unknown)', desttag))
|
|
append_distgit_skip_report(skip_report, desttag, name)
|
|
shutil.rmtree(pkgdir)
|
|
distgit_skips += 1
|
|
continue
|
|
|
|
if not should_rebuild_from_distgit(
|
|
pkgdir, tagged_commit, name, enviro, user):
|
|
# Specific skip reason already printed by should_rebuild_from_distgit
|
|
append_distgit_skip_report(skip_report, desttag, name)
|
|
shutil.rmtree(pkgdir)
|
|
distgit_skips += 1
|
|
continue
|
|
|
|
spec = ''
|
|
for file in os.listdir(pkgdir):
|
|
if file.endswith('.spec'):
|
|
spec = os.path.join(pkgdir, file)
|
|
break
|
|
if not spec:
|
|
sys.stderr.write('%s failed spec check\n' % name)
|
|
continue
|
|
|
|
# Non-idempotent: single attempt
|
|
bumpspec = ['rpmdev-bumpspec', '-D', '-u', user, '-c', comment, spec]
|
|
print('Bumping %s' % spec)
|
|
if runme(bumpspec, 'bumpspec', name, enviro, retries=1):
|
|
continue
|
|
|
|
set_name = ['git', 'config', 'user.name', 'Fedora Release Engineering']
|
|
set_mail = ['git', 'config', 'user.email', 'releng@fedoraproject.org']
|
|
print('Setting git user.name and user.email')
|
|
if runme(set_name, 'set_name', name, enviro, cwd=pkgdir, retries=1):
|
|
continue
|
|
if runme(set_mail, 'set_mail', name, enviro, cwd=pkgdir, retries=1):
|
|
continue
|
|
|
|
commit = ['git', 'commit', '-a', '-m', comment, '--allow-empty']
|
|
print('Committing changes for %s' % name)
|
|
if runme(commit, 'commit', name, enviro, cwd=pkgdir, retries=1):
|
|
continue
|
|
|
|
# Retries OK
|
|
push = ['git', 'push', '--no-verify']
|
|
print('Pushing changes for %s' % name)
|
|
if runme(push, 'push', name, enviro, cwd=pkgdir):
|
|
continue
|
|
|
|
urlcmd = ['fedpkg', 'giturl']
|
|
print('Getting git url for %s' % name)
|
|
url = runmeoutput(urlcmd, 'giturl', name, enviro, cwd=pkgdir)
|
|
if not url:
|
|
continue
|
|
|
|
# Non-idempotent: single attempt
|
|
build = [koji_bin, 'build', '--nowait', '--background', '--fail-fast',
|
|
massrebuild['target'], url]
|
|
print('Building %s' % name)
|
|
buildmeoutput(build, 'build', name, enviro, cwd=pkgdir, retries=1)
|
|
|
|
if distgit_skips:
|
|
print('Skipped %s packages due to dist-git/tagged mismatch (see %s)' % (
|
|
distgit_skips, skip_report))
|
|
else:
|
|
print('No packages skipped for dist-git/tagged mismatch.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|