tooling/release-process/mass-rebuilds/mass_rebuild.py

#!/usr/bin/python3
#
# mass-rebuild.py - A utility to rebuild packages.
#
# Copyright (C) 2009-2013 Red Hat, Inc.
# SPDX-License-Identifier:      GPL-2.0+
#
# Authors:
#     Jesse Keating <jkeating@redhat.com>
#

from __future__ import print_function
import koji
import os
import subprocess
import sys
import operator
import time

# contains info about all rebuilds, add new rebuilds there and update rebuildid
# here
from mass_rebuilds_info import MASSREBUILDS

# Configuration for retry logic
MAX_RETRIES = 3  # Number of retries
RETRY_DELAY = 5  # Delay in seconds between retries

# Set some variables
# Some of these could arguably be passed in as args.
rebuildid = 'f44'
massrebuild = MASSREBUILDS[rebuildid]
user = 'Fedora Release Engineering <releng@fedoraproject.org>'
comment = 'Rebuilt for ' + massrebuild['wikipage']
workdir = os.path.expanduser('~/massbuild')
enviro = os.environ

# Retry logic wrapper function
def retry(func, *args, retries=MAX_RETRIES, delay=RETRY_DELAY, **kwargs):
    """Retry logic wrapper function."""
    for attempt in range(retries):
        result = func(*args, **kwargs)
        if result == 0:  # Success
            return 0
        print(f"Attempt {attempt + 1} failed. Retrying in {delay} seconds...")
        time.sleep(delay)
    print(f"All {retries} attempts failed.")
    return 1

# Updated buildmeoutput with retry logic
def buildmeoutput(cmd, action, pkg, env, cwd=workdir):
    def attempt():
        try:
            output = subprocess.check_output(cmd, env=env, cwd=cwd).decode('utf-8').split()
            with open(workdir + "/taskID_file", 'a') as task_file:
                task_file.write('%s %s\n' % (pkg, output[2]))
            sys.stdout.write('  Successful submission: %s  taskID: %s\n' % (pkg, output[2]))
            return 0
        except subprocess.CalledProcessError as e:
            sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
            return 1
    return retry(attempt)

# Updated runme with retry logic
def runme(cmd, action, pkg, env, cwd=workdir):
    def attempt():
        try:
            subprocess.check_call(cmd, env=env, cwd=cwd)
            return 0
        except subprocess.CalledProcessError as e:
            sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
            return 1
    return retry(attempt)

# Updated runmeoutput with retry logic
def runmeoutput(cmd, action, pkg, env, cwd=workdir):
    def attempt():
        try:
            pid = subprocess.Popen(cmd, env=env, cwd=cwd,
                                   stdout=subprocess.PIPE, encoding='utf8')
            result = pid.communicate()[0].rstrip('\n')
            if pid.returncode == 0:
                return result
            else:
                return 0
        except BaseException as e:
            sys.stderr.write('%s failed %s: %s\n' % (pkg, action, e))
            return 0
    for attempt_number in range(MAX_RETRIES):
        result = attempt()
        if result:
            return result
        print(f"Attempt {attempt_number + 1} failed. Retrying in {RETRY_DELAY} seconds...")
        time.sleep(RETRY_DELAY)
    print(f"All {MAX_RETRIES} attempts failed for {pkg}.")
    return 0

# Environment for using releng credentials for pushing and building
enviro['GIT_SSH'] = '/usr/local/bin/relengpush'
koji_bin = '/usr/bin/compose-koji'

# Create a koji session
kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')

# Generate a list of packages to iterate over
pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True)

# reduce the list to those that are not blocked and sort by package name
pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']],
              key=operator.itemgetter('package_name'))

print('Checking %s packages...' % len(pkgs))

# Loop over each package
for pkg in pkgs:
    name = pkg['package_name']
    id = pkg['package_id']

    # some package we just dont want to ever rebuild
    if name in massrebuild['pkg_skip_list']:
        print('Skipping %s, package is explicitely skipped')
        continue

    # Query to see if a build has already been attempted
    builds = kojisession.listBuilds(id, createdAfter=massrebuild['epoch'])
    newbuild = False
    # Check the builds to make sure they were for the target we care about
    for build in builds:
        try:
            buildtarget = kojisession.getTaskInfo(build['task_id'],
                                       request=True)['request'][1]
            if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']:
                newbuild = True
                break
        except:
            print('Skipping %s, no taskinfo.' % name)
            continue
    if newbuild:
        print('Skipping %s, already attempted.' % name)
        continue

    # Check out git
    fedpkgcmd = ['fedpkg', '--user', 'releng', 'clone', '--branch', 'rawhide', name]
    print('Checking out %s' % name)
    if runme(fedpkgcmd, 'fedpkg', name, enviro):
        continue

    # Check for a checkout
    if not os.path.exists(os.path.join(workdir, name)):
        sys.stderr.write('%s failed checkout.\n' % name)
        continue

    # Check for a noautobuild file
    if os.path.exists(os.path.join(workdir, name, 'noautobuild')):
        print('Skipping %s due to opt-out' % name)
        continue

    # Find the spec file
    files = os.listdir(os.path.join(workdir, name))
    spec = ''
    for file in files:
        if file.endswith('.spec'):
            spec = os.path.join(workdir, name, file)
            break

    if not spec:
        sys.stderr.write('%s failed spec check\n' % name)
        continue

    # rpmdev-bumpspec
    bumpspec = ['rpmdev-bumpspec', '-D', '-u', user, '-c', comment,
                os.path.join(workdir, name, spec)]
    print('Bumping %s' % spec)
    if runme(bumpspec, 'bumpspec', name, enviro):
        continue

    # Set the git user.name and user.email
    set_name = ['git', 'config', 'user.name', 'Fedora Release Engineering']
    set_mail = ['git', 'config', 'user.email', 'releng@fedoraproject.org']
    print('Setting git user.name and user.email')
    if runme(set_name, 'set_name', name, enviro,
                 cwd=os.path.join(workdir, name)):
        continue
    if runme(set_mail, 'set_mail', name, enviro,
                 cwd=os.path.join(workdir, name)):
        continue

    # git commit
    commit = ['git', 'commit', '-a', '-m', comment, '--allow-empty']
    print('Committing changes for %s' % name)
    if runme(commit, 'commit', name, enviro,
                 cwd=os.path.join(workdir, name)):
        continue

    # git push
    push = ['git', 'push', '--no-verify']
    print('Pushing changes for %s' % name)
    if runme(push, 'push', name, enviro,
                 cwd=os.path.join(workdir, name)):
        continue

    # get git url
    urlcmd = ['fedpkg', 'giturl']
    print('Getting git url for %s' % name)
    url = runmeoutput(urlcmd, 'giturl', name, enviro,
                 cwd=os.path.join(workdir, name))
    if not url:
        continue

    # build
    build = [koji_bin, 'build', '--nowait', '--background', '--fail-fast', massrebuild['target'], url]
    print('Building %s' % name)
    buildmeoutput(build, 'build', name, enviro,
          cwd=os.path.join(workdir, name))