jnsamyak
26278a8bf3
All checks were successful
/ test (push) Successful in 3s
Skip packages that already have a checkout, so interrupted runs Resume past cloned trees instead of deleting and recloning. Line-buffer output for tee, we used to do it in the last script. Signed-off-by: jnsamyak <samyak.jn11@gmail.com>
242 lines
8.6 KiB
Python
Executable file
242 lines
8.6 KiB
Python
Executable file
#!/usr/bin/python3
|
|
#
|
|
# mass_rebuild.py - A utility to rebuild packages.
|
|
#
|
|
# Copyright (C) 2009-2013 Red Hat, Inc.
|
|
# SPDX-License-Identifier: GPL-2.0+
|
|
#
|
|
# Authors:
|
|
# Jesse Keating <jkeating@redhat.com>
|
|
#
|
|
|
|
from __future__ import print_function
|
|
import koji
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import operator
|
|
import time
|
|
import random
|
|
|
|
# contains info about all rebuilds, add new rebuilds there and update rebuildid
|
|
# here
|
|
from mass_rebuilds_info import MASSREBUILDS
|
|
|
|
# Retry: total attempts and short backoff (no sleep after the last failure)
|
|
MAX_ATTEMPTS = 3
|
|
RETRY_DELAY_MIN = 2
|
|
RETRY_DELAY_MAX = 10
|
|
MAX_RETRIES = MAX_ATTEMPTS # alias
|
|
|
|
# Set some variables
|
|
# Some of these could arguably be passed in as args.
|
|
rebuildid = 'f45'
|
|
massrebuild = MASSREBUILDS[rebuildid]
|
|
user = 'Fedora Release Engineering <releng@fedoraproject.org>'
|
|
comment = 'Rebuilt for ' + massrebuild['wikipage']
|
|
workdir = os.path.expanduser('~/massbuild')
|
|
enviro = os.environ
|
|
|
|
|
|
def retry(func, *args, retries=MAX_ATTEMPTS, delay_min=RETRY_DELAY_MIN,
|
|
delay_max=RETRY_DELAY_MAX, success_check=None, failure_value=1, **kwargs):
|
|
"""Retry wrapper. Does not sleep after the final failed attempt."""
|
|
if success_check is None:
|
|
success_check = lambda result: result == 0
|
|
for attempt in range(retries):
|
|
result = func(*args, **kwargs)
|
|
if success_check(result):
|
|
return result
|
|
if attempt + 1 >= retries:
|
|
break
|
|
delay = random.uniform(delay_min, delay_max)
|
|
print('Attempt %s failed. Retrying in %s seconds...' % (
|
|
attempt + 1, int(delay)))
|
|
time.sleep(delay)
|
|
print('All %s attempts failed.' % retries)
|
|
return failure_value
|
|
|
|
|
|
def buildmeoutput(cmd, action, pkg, env, cwd=workdir, retries=1):
|
|
"""Submit a koji build. Defaults to a single attempt (not idempotent)."""
|
|
def do_attempt():
|
|
try:
|
|
output = subprocess.check_output(cmd, env=env, cwd=cwd).decode('utf-8').split()
|
|
if len(output) < 3:
|
|
sys.stderr.write('%s failed %s: unexpected output: %s\n' % (
|
|
pkg, action, ' '.join(output)))
|
|
return 1
|
|
with open(os.path.join(workdir, 'taskID_file'), 'a') as task_file:
|
|
task_file.write('%s %s\n' % (pkg, output[2]))
|
|
sys.stdout.write(' Successful submission: %s taskID: %s\n' % (
|
|
pkg, output[2]))
|
|
sys.stdout.flush()
|
|
return 0
|
|
except subprocess.CalledProcessError as e:
|
|
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
|
|
return 1
|
|
return retry(do_attempt, retries=retries)
|
|
|
|
|
|
def runme(cmd, action, pkg, env, cwd=workdir, retries=MAX_ATTEMPTS):
|
|
"""Run a command. Use retries=1 for non-idempotent ops (bumpspec, build)."""
|
|
def do_attempt():
|
|
try:
|
|
subprocess.check_call(cmd, env=env, cwd=cwd)
|
|
return 0
|
|
except subprocess.CalledProcessError as e:
|
|
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
|
|
return 1
|
|
return retry(do_attempt, retries=retries)
|
|
|
|
|
|
def runmeoutput(cmd, action, pkg, env, cwd=workdir, retries=MAX_ATTEMPTS):
|
|
"""Run a command and return stdout on success, None on failure."""
|
|
def do_attempt():
|
|
try:
|
|
pid = subprocess.Popen(cmd, env=env, cwd=cwd,
|
|
stdout=subprocess.PIPE, encoding='utf8')
|
|
result = pid.communicate()[0].rstrip('\n')
|
|
if pid.returncode == 0:
|
|
return result
|
|
return None
|
|
except BaseException as e:
|
|
sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
|
|
return None
|
|
|
|
return retry(do_attempt, retries=retries,
|
|
success_check=lambda value: value is not None,
|
|
failure_value=None)
|
|
|
|
|
|
def main():
|
|
# Line-buffer so `... 2>&1 | tee logfile` updates live
|
|
if hasattr(sys.stdout, 'reconfigure'):
|
|
sys.stdout.reconfigure(line_buffering=True)
|
|
if hasattr(sys.stderr, 'reconfigure'):
|
|
sys.stderr.reconfigure(line_buffering=True)
|
|
|
|
enviro['GIT_SSH'] = '/usr/local/bin/relengpush'
|
|
koji_bin = '/usr/bin/compose-koji'
|
|
|
|
os.makedirs(workdir, exist_ok=True)
|
|
|
|
kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')
|
|
pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True)
|
|
pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']],
|
|
key=operator.itemgetter('package_name'))
|
|
|
|
print('mass_rebuild.py starting (rebuildid=%s)' % rebuildid)
|
|
print('script=%s' % os.path.abspath(__file__))
|
|
print('workdir=%s' % workdir)
|
|
print('Checking %s packages...' % len(pkgs))
|
|
|
|
for pkg in pkgs:
|
|
name = pkg['package_name']
|
|
id = pkg['package_id']
|
|
pkgdir = os.path.join(workdir, name)
|
|
|
|
if name in massrebuild['pkg_skip_list']:
|
|
print('Skipping %s, package is explicitely skipped' % name)
|
|
continue
|
|
|
|
# Restart: skip packages that already have a checkout (old script
|
|
# behavior). Only packages without a local dir are processed.
|
|
if os.path.exists(pkgdir):
|
|
print('Skipping %s, already checked out.' % name)
|
|
continue
|
|
|
|
# Skip if a build was already attempted for this mass rebuild
|
|
builds = kojisession.listBuilds(id, createdAfter=massrebuild['epoch'])
|
|
newbuild = False
|
|
for build in builds:
|
|
try:
|
|
buildtarget = kojisession.getTaskInfo(build['task_id'],
|
|
request=True)['request'][1]
|
|
if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']:
|
|
newbuild = True
|
|
break
|
|
except Exception:
|
|
print('Skipping %s, no taskinfo.' % name)
|
|
continue
|
|
if newbuild:
|
|
print('Skipping %s, already attempted.' % name)
|
|
continue
|
|
|
|
fedpkgcmd = ['fedpkg', '--user', 'releng', 'clone', '--branch', 'rawhide', name]
|
|
print('Checking out %s' % name)
|
|
|
|
def do_clone():
|
|
# Clean partial tree left by a previous failed attempt
|
|
if os.path.exists(pkgdir):
|
|
shutil.rmtree(pkgdir)
|
|
try:
|
|
subprocess.check_call(fedpkgcmd, env=enviro, cwd=workdir)
|
|
return 0
|
|
except subprocess.CalledProcessError as e:
|
|
sys.stderr.write('%s failed fedpkg: %s\n' % (name, e))
|
|
return 1
|
|
|
|
if retry(do_clone):
|
|
if os.path.exists(pkgdir):
|
|
shutil.rmtree(pkgdir)
|
|
continue
|
|
|
|
if not os.path.exists(pkgdir):
|
|
sys.stderr.write('%s failed checkout.\n' % name)
|
|
continue
|
|
|
|
if os.path.exists(os.path.join(pkgdir, 'noautobuild')):
|
|
print('Skipping %s due to opt-out' % name)
|
|
continue
|
|
|
|
spec = ''
|
|
for file in os.listdir(pkgdir):
|
|
if file.endswith('.spec'):
|
|
spec = os.path.join(pkgdir, file)
|
|
break
|
|
if not spec:
|
|
sys.stderr.write('%s failed spec check\n' % name)
|
|
continue
|
|
|
|
# Non-idempotent: single attempt
|
|
bumpspec = ['rpmdev-bumpspec', '-D', '-u', user, '-c', comment, spec]
|
|
print('Bumping %s' % spec)
|
|
if runme(bumpspec, 'bumpspec', name, enviro, retries=1):
|
|
continue
|
|
|
|
set_name = ['git', 'config', 'user.name', 'Fedora Release Engineering']
|
|
set_mail = ['git', 'config', 'user.email', 'releng@fedoraproject.org']
|
|
print('Setting git user.name and user.email')
|
|
if runme(set_name, 'set_name', name, enviro, cwd=pkgdir, retries=1):
|
|
continue
|
|
if runme(set_mail, 'set_mail', name, enviro, cwd=pkgdir, retries=1):
|
|
continue
|
|
|
|
commit = ['git', 'commit', '-a', '-m', comment, '--allow-empty']
|
|
print('Committing changes for %s' % name)
|
|
if runme(commit, 'commit', name, enviro, cwd=pkgdir, retries=1):
|
|
continue
|
|
|
|
# Retries OK
|
|
push = ['git', 'push', '--no-verify']
|
|
print('Pushing changes for %s' % name)
|
|
if runme(push, 'push', name, enviro, cwd=pkgdir):
|
|
continue
|
|
|
|
urlcmd = ['fedpkg', 'giturl']
|
|
print('Getting git url for %s' % name)
|
|
url = runmeoutput(urlcmd, 'giturl', name, enviro, cwd=pkgdir)
|
|
if not url:
|
|
continue
|
|
|
|
# Non-idempotent: single attempt
|
|
build = [koji_bin, 'build', '--nowait', '--background', '--fail-fast',
|
|
massrebuild['target'], url]
|
|
print('Building %s' % name)
|
|
buildmeoutput(build, 'build', name, enviro, cwd=pkgdir, retries=1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|