#!/usr/bin/python3 # # mass-rebuild.py - A utility to rebuild packages. # # Copyright (C) 2009-2013 Red Hat, Inc. # SPDX-License-Identifier: GPL-2.0+ # # Authors: # Jesse Keating # from __future__ import print_function import koji import os import subprocess import sys import operator import time import random # contains info about all rebuilds, add new rebuilds there and update rebuildid # here from mass_rebuilds_info import MASSREBUILDS # Configuration for retry logic MAX_RETRIES = 3 # Number of retries RETRY_DELAY_MIN = 30 # Minimum delay in seconds between retries RETRY_DELAY_MAX = 300 # Maximum delay in seconds between retries # Set some variables # Some of these could arguably be passed in as args. rebuildid = 'f44' massrebuild = MASSREBUILDS[rebuildid] user = 'Fedora Release Engineering ' comment = 'Rebuilt for ' + massrebuild['wikipage'] workdir = os.path.expanduser('~/massbuild') enviro = os.environ # Retry logic wrapper function def retry(func, *args, retries=MAX_RETRIES, delay_min=RETRY_DELAY_MIN, delay_max=RETRY_DELAY_MAX, success_check=None, failure_value=1, **kwargs): """Retry logic wrapper function.""" if success_check is None: success_check = lambda result: result == 0 for attempt in range(retries): result = func(*args, **kwargs) if success_check(result): # Success return result delay = random.uniform(delay_min, delay_max) print(f"Attempt {attempt + 1} failed. Retrying in {int(delay)} seconds...") time.sleep(delay) print(f"All {retries} attempts failed.") return failure_value # Updated buildmeoutput with retry logic def buildmeoutput(cmd, action, pkg, env, cwd=workdir): def do_attempt(): try: output = subprocess.check_output(cmd, env=env, cwd=cwd).decode('utf-8').split() if len(output) < 3: sys.stderr.write('%s failed %s: unexpected output: %s\n' % (pkg, action, ' '.join(output))) return 1 with open(workdir + "/taskID_file", 'a') as task_file: task_file.write('%s %s\n' % (pkg, output[2])) sys.stdout.write(' Successful submission: %s taskID: %s\n' % (pkg, output[2])) return 0 except subprocess.CalledProcessError as e: sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e)) return 1 return retry(do_attempt) # Updated runme with retry logic def runme(cmd, action, pkg, env, cwd=workdir): def do_attempt(): try: subprocess.check_call(cmd, env=env, cwd=cwd) return 0 except subprocess.CalledProcessError as e: sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e)) return 1 return retry(do_attempt) # Updated runmeoutput with retry logic def runmeoutput(cmd, action, pkg, env, cwd=workdir): def do_attempt(): try: pid = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, encoding='utf8') result = pid.communicate()[0].rstrip('\n') if pid.returncode == 0: return result else: return None except BaseException as e: sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e)) return None def success_check(value): return value is not None result = retry(do_attempt, success_check=success_check, failure_value=None) if result is None: print(f"All {MAX_RETRIES} attempts failed for {pkg}.") return 0 return result # Environment for using releng credentials for pushing and building enviro['GIT_SSH'] = '/usr/local/bin/relengpush' koji_bin = '/usr/bin/compose-koji' # Create a koji session kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub') # Generate a list of packages to iterate over pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True) # reduce the list to those that are not blocked and sort by package name pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']], key=operator.itemgetter('package_name')) print('Checking %s packages...' % len(pkgs)) # Loop over each package for pkg in pkgs: name = pkg['package_name'] id = pkg['package_id'] # some package we just dont want to ever rebuild if name in massrebuild['pkg_skip_list']: print('Skipping %s, package is explicitely skipped') continue # Query to see if a build has already been attempted builds = kojisession.listBuilds(id, createdAfter=massrebuild['epoch']) newbuild = False # Check the builds to make sure they were for the target we care about for build in builds: try: buildtarget = kojisession.getTaskInfo(build['task_id'], request=True)['request'][1] if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']: newbuild = True break except: print('Skipping %s, no taskinfo.' % name) continue if newbuild: print('Skipping %s, already attempted.' % name) continue # Check out git fedpkgcmd = ['fedpkg', '--user', 'releng', 'clone', '--branch', 'rawhide', name] print('Checking out %s' % name) if runme(fedpkgcmd, 'fedpkg', name, enviro): continue # Check for a checkout if not os.path.exists(os.path.join(workdir, name)): sys.stderr.write('%s failed checkout.\n' % name) continue # Check for a noautobuild file if os.path.exists(os.path.join(workdir, name, 'noautobuild')): print('Skipping %s due to opt-out' % name) continue # Find the spec file files = os.listdir(os.path.join(workdir, name)) spec = '' for file in files: if file.endswith('.spec'): spec = os.path.join(workdir, name, file) break if not spec: sys.stderr.write('%s failed spec check\n' % name) continue # rpmdev-bumpspec bumpspec = ['rpmdev-bumpspec', '-D', '-u', user, '-c', comment, os.path.join(workdir, name, spec)] print('Bumping %s' % spec) if runme(bumpspec, 'bumpspec', name, enviro): continue # Set the git user.name and user.email set_name = ['git', 'config', 'user.name', 'Fedora Release Engineering'] set_mail = ['git', 'config', 'user.email', 'releng@fedoraproject.org'] print('Setting git user.name and user.email') if runme(set_name, 'set_name', name, enviro, cwd=os.path.join(workdir, name)): continue if runme(set_mail, 'set_mail', name, enviro, cwd=os.path.join(workdir, name)): continue # git commit commit = ['git', 'commit', '-a', '-m', comment, '--allow-empty'] print('Committing changes for %s' % name) if runme(commit, 'commit', name, enviro, cwd=os.path.join(workdir, name)): continue # git push push = ['git', 'push', '--no-verify'] print('Pushing changes for %s' % name) if runme(push, 'push', name, enviro, cwd=os.path.join(workdir, name)): continue # get git url urlcmd = ['fedpkg', 'giturl'] print('Getting git url for %s' % name) url = runmeoutput(urlcmd, 'giturl', name, enviro, cwd=os.path.join(workdir, name)) if not url: continue # build build = [koji_bin, 'build', '--nowait', '--background', '--fail-fast', massrebuild['target'], url] print('Building %s' % name) buildmeoutput(build, 'build', name, enviro, cwd=os.path.join(workdir, name))