fedora-nightlies/fedora_nightlies.py
Kamil Páral d800a953dc migrate to Forge
Update links Pagure -> Forge.

Related to #14
2026-01-22 14:27:44 +01:00

692 lines
29 KiB
Python
Executable file

#!/usr/bin/python3
# Copyright Red Hat
#
# This file is part of fedora_nightlies.
#
# fedora_nightlies is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Author: Adam Williamson <awilliam@redhat.com>
"""Fedora nightly image finder."""
from collections import OrderedDict
import datetime
import functools
import itertools
import json
import logging
import os
import sys
import fedfind.exceptions
import fedfind.release
import fedfind.helpers
import fedora_messaging.config
import openqa_client.client
import openqa_client.exceptions
from openqa_client import const as oqc
# HTML template
HTML_TEMPLATE = """
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head>
<style>
body {width: 95%; margin: auto;}
table, th, td, tr {border: 1px solid grey; border-collapse: collapse; font-size: 0.95em;}
table {float: left; margin-right: 10px; margin-bottom: 10px;}
th, td {padding: 5px;}
th {background-color: lightgrey;}
.relheader {background-color: black; color: white;}
.groupheader {background-color: lightskyblue;}
.failedlink {color: red;}
.failcell {position: relative; display: inline-block;}
.faillinks {display: none; border: 1px solid #c8c8c8; left: 100px; position: absolute; z-index: 1; white-space: nowrap; background-color: #f9f9f9; padding: 10px;}
.faillinks a {display: block;}
.failcell:hover .faillinks {display: block;}
.passedlink {color: green;}
.key {font-size: 0.9em; color: dimgrey;}
</style>
<meta charset="utf-8">
<title>Fedora nightly compose finder</title>
</head>
<body>
<h1>Fedora nightly compose finder</h1>
<p>This page helps you locate recent Fedora nightly composes. For each Fedora image, you can find the latest Branched and Rawhide nightly composes that completed. For images tested by <a href="https://openqa.fedoraproject.org">openQA</a>, you can also find the latest image for which all image-specific tests passed. This is not as strong an indication of quality as official Alpha or Beta status, but at least indicates that the image successfully boots, completes a system installation, and boots to the installed system, in a straightforward virtual machine configuration. These images are <b>NOT</b> official Fedora pre-releases. For assistance, contact <a href="https://matrix.to/#/#quality:fedoraproject.org">#quality:fedoraproject.org on Matrix</a> or the <a href="https://lists.fedoraproject.org/archives/list/test@lists.fedoraproject.org/">test@ mailing list</a>.</p>
<p class='key'>Red linked images are known to have failed tests; green linked images are known to have passed all tests; blue linked images are untested or testing status is unknown. Mouse over red links to display links to the failed tests.</p>
<p class='key'>Generated by <a href="https://forge.fedoraproject.org/quality/fedora-nightlies">Fedora nightlies</a> at: {{DATE}} (UTC)</p>
{{TABLES}}
</body>
</html>
"""
logger = logging.getLogger(__name__)
## FUNCTIONS
def current_releases():
"""Figure out what releases we care about: for now, that's Rawhide
plus Branched if there is one. Note: we don't cache this any more
as fedfind does it for us (it caches the collections data it uses
to figure this out, with a one day expiry).
"""
curr = fedfind.helpers.get_current_release(branched=False)
branched = fedfind.helpers.get_current_release(branched=True)
if branched > curr:
return [str(branched), 'Rawhide']
else:
return ['Rawhide']
def get_cell(image):
"""Build a results table cell for an image. This is a bit complex
and looks pretty messy in-line in add_group, so it's split out
here."""
# YAY partial, we all love partial. We want a link to the image
# whatever, but its class differs depending on test status.
lnpart = functools.partial(Element, 'a', image['compose'], href=image['url'])
# simple cases first: pass or none. Just add the link.
if image.testspass is True:
content = lnpart(cssclass='passedlink')
elif image.testspass is None:
content = lnpart()
else:
# fail. this case is more complex. we need the contents of the
# cell to include the failed test links popups.
# First build the links div:
faillinks = ['Failed tests:']
fails = [(str(job['id']), job['test']) for job in image['openqa']
if job['result'] not in (oqc.JOB_RESULT_PASSED, oqc.JOB_RESULT_SOFTFAILED)]
if fails:
faillinks.extend(
[Element('a', "openQA: {0} ({1})".format(jobid, jobtest),
href='https://openqa.fedoraproject.org/tests/{0}'.format(jobid))
for (jobid, jobtest) in fails])
faillinksdiv = Element('div', faillinks, cssclass='faillinks')
# now wrap it and the image link in the cell div:
imglink = lnpart(cssclass='failedlink')
content = Element('div', [imglink, faillinksdiv], cssclass='failcell')
return Element('td', content)
## ERROR CLASSES
class NightlyFileError(Exception):
"""Exception raised when there's a problem with access to one of
the data files.
"""
def __init__(self, path, access):
self.path = path
self.access = access
super(NightlyFileError, self).__init__()
def __str__(self):
"""Human-readable representation of the problem."""
if self.access == 'e':
return "Required directory {0} doesn't exist or can't be accessed!".format(self.path)
if self.access == 'r':
accstr = 'read'
elif self.access == 'w':
accstr = 'written'
return "Path {0} exists but could not be {1}!".format(self.path, accstr)
## HTML ELEMENT CLASSES
class Element(object):
"""A generic HTML element class. Probably missing all sorts of
stuff, but these are the bits I need. self.contents should always
wind up as an iterable of things that can be str()ed.
"""
def __init__(self, name, contents='', cssclass='', **kwargs):
self.name = name
# handle contents as string...
if isinstance(contents, str):
self.contents = [contents]
else:
# or iterable of string-able things...
try:
''.join(str(item) for item in contents)
self.contents = contents
except TypeError:
# or single non-iterable string-able thing (usually
# another Element)
self.contents = [contents]
self.attributes = dict()
if kwargs:
self.attributes.update(kwargs)
# can't use class as kwarg, grr
if cssclass:
self.attributes['class'] = cssclass
def __str__(self):
"""Write the opening tag, include the attributes, add each
item in 'contents' on a new line, write the closing tag.
"""
attrtext = ''
if self.attributes:
# sort the attributes, makes testing easier
for (attr, value) in sorted(self.attributes.items()):
attrtext += ' {0}="{1}"'.format(attr, value)
conts = ''.join(str(item) for item in self.contents)
if len(conts) > 80:
contents = '\n'.join(str(item) for item in self.contents)
else:
contents = conts
# fuzzy...
if len(contents) > 80:
contents = '\n{0}\n'.format(contents)
return '<{0}{1}>{2}</{0}>'.format(self.name, attrtext, contents)
class ReleaseTable(Element):
"""A per-release table in the format we use."""
def __init__(self, release, groups=None):
# build the release name header
header = Element('th', 'Fedora {0}'.format(release),
colspan=3, cssclass='relheader')
# do super.__init__ with just a row containing the header
super(ReleaseTable, self).__init__(
'table', Element('tr', header))
# add the rows for each group to the table contents
if groups:
for (name, group) in groups.items():
self.add_group(name, group)
def add_group(self, name, group):
"""Add table rows for a single image group to the table's
contents.
"""
# build the group name header
groupheader = Element('th', name, colspan=3, cssclass='groupheader')
# start contents list with a table row containing the header
contents = [Element('tr', groupheader)]
# add another row containing the column title headers
columntitles = [(Element('th', column)) for column in
('Arch', 'Last built', 'Last known good')]
contents.append(Element('tr', columntitles))
if not any(group[arch] for arch in group):
# we have no images for this group, skip it
return
for arch in group:
# add a row if we have at least one item for the arch
if group[arch]:
# first cell: arch name
cells = [Element('td', arch)]
for image in group[arch]:
cells.append(get_cell(image))
# pad the row with empty cells (this is when we don't
# have a 'last known good' as no image passed tests)
while len(cells) < 3:
cells.append(Element('td'))
# stuff the cells in a row and add it to the contents
row = Element('tr', cells)
contents.append(row)
# stuff our new rows into the table's contents
self.contents.extend(contents)
## MAIN CLASSES
class Image(dict):
"""Class representing a single image from the data. Just a dict
with a few extra methods and properties.
"""
def __init__(self, *args, **kwargs):
# we always want these keys to exist, so set default values
# *BEFORE* super __init__
self['openqa'] = []
super(Image, self).__init__(*args, **kwargs)
@property
def release(self):
"""The release for this image."""
rel = fedfind.helpers.parse_cid(self.get('compose', ''))[0]
# 'rawhide' -> 'Rawhide', if something weird went wrong
# rel may be None so we have to be careful
if rel:
rel = rel.capitalize()
return rel
@property
def group(self):
"""The 'image group' for this image. Images are grouped by
subvariant and type; within each group will be multiple images
for different arches.
"""
return "{0} {1}".format(self.get('subvariant', ''), self.get('type', ''))
@property
def sortid(self):
"""A property used for comparisons with other Image instances,
notably for sorting them. For now we just take all digits from
the compose ID; this works OK for sorting nightlies of the
same release against each other. Does not work for sorting
nightlies of *different* releases against each other, or other
types of compose.
"""
return int(''.join([char for char in self.get('compose', '') if char.isdigit()]))
@property
def openqa_flavor(self):
"""The openQA 'flavor' for this image. Used for finding the
right Image for an openQA test. We should make sure to always
keep this in sync with fedora_openqa_schedule.
"""
try:
return fedfind.helpers.identify_image(self, undersub=True, out='string')
except KeyError:
# just in case dict was missing some value
return ''
@property
def testspass(self):
"""Whether we have some tests for the image and they all
passed. Note this does *NOT* know when we have 'all' tests for
an image. That's a bit out of scope here: it's expected the
results should only be updated when all tests are done. False
if any tests failed, True if all tests passed, None if we have
no test results.
"""
res = None
if self.get('openqa'):
res = all(job['result'] in (oqc.JOB_RESULT_PASSED, oqc.JOB_RESULT_SOFTFAILED)
for job in self['openqa'])
return res
def matches(self, other):
"""Check if this Image represents the same image as another
Image instance or dict.
"""
ret = True
for key in ('compose', 'path'):
if not self.get(key) or not other.get(key):
raise ValueError("Image.matches: something bad happened!")
if not other[key] == self[key]:
ret = False
return ret
def validate(self):
"""Check this dict is usable."""
pass
class Nightlies(object):
"""Class representing the data and providing methods to update it
and generate output.
"""
def __init__(self, datafile=None, htmlfile=None):
self.datafile = datafile
self.htmlfile = htmlfile
if not self.datafile:
self.datafile = '/var/www/fedora_nightlies/nightlies.json'
if not self.htmlfile:
self.htmlfile = '/var/www/fedora_nightlies/nightlies.html'
logger.debug("Data file: %s", self.datafile)
logger.debug("HTML file: %s", self.htmlfile)
self._check_files()
self.data = self._read_data()
def _check_files(self):
"""Check the data and HTML files are readable and writeable.
Create their parent directories if necessary. Raises
NightlyFileError if there's any problem. May raise OSError
via `os.mkdir` if we try to create a directory and it fails.
"""
for fil in (self.datafile, self.htmlfile):
# check parent directory, creating if necessary
directory = os.path.dirname(fil)
if not os.path.exists(directory):
os.mkdir(directory)
if not os.access(directory, os.W_OK):
raise NightlyFileError(directory, 'w')
# check file can be read/written if present
if os.path.exists(fil):
if not os.access(fil, os.R_OK):
raise NightlyFileError(fil, 'r')
if not os.access(fil, os.W_OK):
raise NightlyFileError(fil, 'w')
def _read_data(self):
"""Read in the data from the data file."""
try:
with open(self.datafile, 'r') as datafh:
datatext = datafh.read()
if datatext:
data = json.loads(datatext)
# run the image dicts through correct_image to fix
# up older data
return [Image(fedfind.helpers.correct_image(img)) for img in data]
else:
# file exists but is empty
return []
except IOError:
# no such file - first run, probably
return []
def write_data(self):
"""Serialize the current data dict out to the data file."""
logger.debug("Writing data")
self._prune_data()
datajson = json.dumps(self.data, sort_keys=True, indent=4)
with open(self.datafile, 'w') as datafh:
datafh.write(datajson)
def _prune_data(self, rels=None):
"""Remove data for releases we no longer care about (not in
current_releases) and data we no longer need.
"""
logger.debug("Pruning data")
logger.debug("Current images:")
for image in self.data:
logger.debug(image['url'])
if not rels:
rels = current_releases()
# flatten the latest image set
latests = self.get_latests(flat=True)
# we need to know the date...
today = datetime.datetime.today()
# copy the list to iterate over it
for image in list(self.data):
imgdate = fedfind.helpers.parse_cid(image["compose"], dic=True)['date']
imgdate = fedfind.helpers.date_check(imgdate, fail_raise=False)
if imgdate and (today - imgdate).days > 180:
logger.debug("Removing: %s (aged out)", image["url"])
self.data.remove(image)
elif image.release not in rels:
logger.debug("Removing: %s (release gone)", image['url'])
self.data.remove(image)
elif image not in latests:
logger.debug("Removing: %s (no longer latest)", image['url'])
self.data.remove(image)
# remove any duped openQA job dicts. this should not
# happen any more, but may as well keep it just in case.
elif image['openqa']:
for job in list(image['openqa']):
while True:
matches = [gotjob for gotjob in image['openqa'] if job == gotjob]
if len(matches) > 1:
logger.debug("Removing duplicated job: %s", job['id'])
image['openqa'].remove(job)
else:
break
def seed_data(self, days=28):
"""Initialize the base data set by reading in the last 'days'
worth of composes. This is only ever going to be needed at
first run or if the stored data are lost.
"""
# get a YYYYMMDD 28 days in the past.
fromdate = datetime.date.today() - datetime.timedelta(days)
fromdate = fromdate.strftime('%Y%m%d')
composes = []
for rel in current_releases():
# this should be a unique ID in PDC
relid = "fedora-{0}".format(rel)
pdcrel = fedfind.helpers.pdc_query('releases', {'release_id': relid})[0]
# this is all compose IDs for the release
cidlist = pdcrel['compose_set']
for cid in cidlist:
(_, cdate, ctype, _) = fedfind.helpers.parse_cid(cid)
if cdate >= fromdate and ctype == 'nightly':
composes.append(cid)
self.add_images(composes)
self.update_openqa(composes)
self.write_data()
def add_images(self, composes):
"""Update the 'last built' data for all specified composes (an
iterable of compose IDs).
"""
for compose in composes:
logger.info("Updating 'last built' for: %s", compose)
try:
ffrel = fedfind.release.get_release(cid=compose)
except ValueError:
logger.warning("Could not find compose %s! Skipping...", compose)
continue
except fedfind.exceptions.UnsupportedComposeError:
logger.info("Compose %s is not supported by fedfind! Skipping...", compose)
continue
if ffrel.release not in current_releases():
logger.info("Compose %s is not for Branched or Rawhide! "
"Skipping...", compose)
continue
for imgdict in ffrel.all_images:
# append extra properties to the image dict
imgdict['compose'] = compose
imgdict['url'] = "{0}/{1}".format(ffrel.location, imgdict['path'])
# update existing entry for image if there is one, or
# else create an entry
matches = [currimg for currimg in self.data if
currimg.matches(imgdict)]
if len(matches) == 0:
self.data.append(Image(imgdict))
elif len(matches) == 1:
matches[0].update(imgdict)
else:
# FIXME: better logging / raise or something
logger.warning("add_images(): More than one "
"matching image found!")
logger.info("Last built update done")
def update_openqa(self, composes):
"""Update data based on openQA test results for all specified
composes (an iterable of compose IDs).
"""
client = openqa_client.client.OpenQA_Client('openqa.fedoraproject.org')
# dict to note which composes we've tried to update as we
# went along - we don't wanna do this over and over
updated = set()
for compose in composes:
logger.info("Updating openQA results for: %s", compose)
# get all openQA jobs for the compose
try:
jobs = client.get_jobs(build=compose)
except openqa_client.exceptions.OpenQAClientError as err:
logger.error("Error performing openQA request for %s! Error: %s", compose, err)
continue
# check for still-running jobs, exclude them (we can't do
# anything sensible with them) and warn
running = [job for job in jobs if job['state'] not in oqc.JOB_FINAL_STATES]
if running:
logger.warning("Some jobs still scheduled or running! Jobs: %s will be ignored",
', '.join(str(job['id']) for job in running))
jobs = [job for job in jobs if job not in running]
if not jobs:
logger.warning("No jobs found for %s!", compose)
continue
for job in jobs:
flavor = job['settings']['FLAVOR']
arch = job['settings']['ARCH']
if flavor == 'universal':
# not an image-specific test, skip
continue
# find the matching image in the data. if there isn't
# one and we haven't updated the built data for this
# compose, update it and try again.
matches = []
while True:
matches = [image for image in self.data if
image['compose'] == compose and
image.openqa_flavor == flavor and
image['arch'] == arch]
# we break *after* doing the match so the match
# gets re-done after the add_images
if matches or compose in updated:
break
else:
self.add_images([compose])
updated.add(compose)
# no match
if len(matches) == 0:
logger.warning("update_openqa: No Image found for flavor %s, "
"arch %s, compose %s", flavor, arch, compose)
# exactly one match! yay. add job to image's list...if
# we don't already have it
elif len(matches) == 1 and job not in matches[0]['openqa']:
matches[0]['openqa'].append(job)
elif len(matches) > 1:
logger.warning("update_openqa: More than one Image found for "
"flavor %s, arch %s, compose %s", flavor, arch, compose)
logger.info("openQA update done")
def find_latest(self, rel, group, arch):
"""Find 'last built' and 'last known good' for a specific
release, group and arch.
"""
imgs = [img for img in self.data if img.release == rel and
img.group == group and img['arch'] == arch]
passimgs = [img for img in imgs if img.testspass]
if imgs:
latest = sorted(imgs, key=lambda x: x.sortid)[-1]
if passimgs:
passed = sorted(passimgs, key=lambda x: x.sortid)[-1]
return [latest, passed]
else:
return [latest]
else:
return []
def get_latests(self, flat=False):
"""Find all 'last built' and 'last known good' images. Return
sorted by release, group and arch if flat is True, or just a
list if flat is False.
"""
# get the releases, arches and groups in the data and sort.
# 'Rawhide' will sort after numbers, groups are sorted by
# weight of any old image in the group, arches not yet sorted
rels = sorted(set(image.release for image in self.data))
arches = set(image['arch'] for image in self.data)
groups = dict((image.group, fedfind.helpers.get_weight(image, arch=False))
for image in self.data)
groups = sorted(groups.keys(), key=lambda x: groups[x], reverse=True)
if flat:
# this is pretty easy....
return list(itertools.chain.from_iterable(
(self.find_latest(rel, group, arch)
for rel in rels for group in groups for arch in arches)))
else:
# mother of god, what is this? OK, relax, it's just a
# dict grouping 'find_latest' results by rel->group->arch
return OrderedDict(
(rel, OrderedDict(
(group, dict(
(arch, self.find_latest(rel, group, arch)) for arch in arches))
for group in groups))
for rel in rels)
def write_html(self):
"""Write out the HTML representation of the data."""
# get the tables, via get_latests and ReleaseTable
logger.debug("Writing HTML")
tables = (ReleaseTable(rel, groups)
for (rel, groups) in self.get_latests().items())
# stick the tables together as a string
tables = '\n'.join(str(table) for table in tables)
# read in the template and sub out the tables and the time
html = HTML_TEMPLATE.replace('{{TABLES}}', tables)
html = html.replace('{{DATE}}', str(datetime.datetime.utcnow()))
# write out the file
with open(self.htmlfile, 'w') as htmlfh:
htmlfh.write(html)
## FEDORA-MESSAGING CONSUMER CLASS
class NightlyConsumer(object):
"""A fedora-messaging consumer that updates the data when a new
compose appears or when openQA testing for a compose completes.
"""
def __init__(self):
self.datafile = fedora_messaging.config.conf["consumer_config"].get("datafile")
self.htmlfile = fedora_messaging.config.conf["consumer_config"].get("htmlfile")
self.log = logging.getLogger(self.__class__.__name__)
def __call__(self, message):
"""If this is a 'compose completed' message or it's a "job
done" message and no more jobs remain for the compose,
update the data.
"""
try:
nightlies = Nightlies(datafile=self.datafile, htmlfile=self.htmlfile)
except NightlyFileError as err:
self.log.error(err)
return
if message.topic.endswith('pungi.compose.status.change'):
# compose status change
typ = ''
status = message.body.get('status', 'FINISHED')
compose = message.body.get('compose_id', '')
if compose.startswith("EPEL"):
# we know this is fine, don't log the error
return
if compose:
try:
typ = fedfind.helpers.parse_cid(compose)[2]
except ValueError as err:
self.log.error(f"Could not parse compose ID for message {message.id}: {err}")
return
if 'FINISHED' not in status or typ != "nightly":
return
# we've got a finished compose: update built data
self.log.info("Updating for completion of compose %s", compose)
# set the method to run
meth = functools.partial(nightlies.add_images, [compose])
elif message.topic.endswith('openqa.job.done'):
# openQA job done
compose = message.body.get('BUILD', message.body.get('build', ""))
if message.body.get('remaining', 1) != 0 or not compose:
return
# no use running on updates...
if compose.startswith("Update-") or compose.startswith("Kojitask-"):
return
# or Fedora CoreOS builds...
if "coreos" in compose.lower():
return
# ...or ELN builds
if "-eln-" in compose.lower():
return
# if we got here, we're good
self.log.info("Updating for openQA results of compose %s", compose)
# set the method to run
meth = functools.partial(nightlies.update_openqa, [compose])
# do the update
meth()
nightlies.write_data()
nightlies.write_html()
## MAIN LOOP
def main():
"""Init data and write HTML. Run if file is called directly, or
by the setuptools entrypoint script.
"""
try:
logging.basicConfig(level=logging.INFO)
nightlies = Nightlies()
nightlies.seed_data()
nightlies.write_html()
except KeyboardInterrupt:
sys.stderr.write("Interrupted, exiting...\n")
sys.exit(1)
except NightlyFileError as err:
sys.exit(str(err))
except OSError as err:
sys.exit("Could not create directory {0}!".format(err.filename))
if __name__ == "__main__":
main()