692 lines
29 KiB
Python
Executable file
692 lines
29 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
# Copyright Red Hat
|
|
#
|
|
# This file is part of fedora_nightlies.
|
|
#
|
|
# fedora_nightlies is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
# Author: Adam Williamson <awilliam@redhat.com>
|
|
|
|
"""Fedora nightly image finder."""
|
|
|
|
from collections import OrderedDict
|
|
import datetime
|
|
import functools
|
|
import itertools
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
import fedfind.exceptions
|
|
import fedfind.release
|
|
import fedfind.helpers
|
|
import fedora_messaging.config
|
|
import openqa_client.client
|
|
import openqa_client.exceptions
|
|
from openqa_client import const as oqc
|
|
|
|
# HTML template
|
|
HTML_TEMPLATE = """
|
|
<!DOCTYPE html>
|
|
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
|
|
<head>
|
|
<style>
|
|
body {width: 95%; margin: auto;}
|
|
table, th, td, tr {border: 1px solid grey; border-collapse: collapse; font-size: 0.95em;}
|
|
table {float: left; margin-right: 10px; margin-bottom: 10px;}
|
|
th, td {padding: 5px;}
|
|
th {background-color: lightgrey;}
|
|
.relheader {background-color: black; color: white;}
|
|
.groupheader {background-color: lightskyblue;}
|
|
.failedlink {color: red;}
|
|
.failcell {position: relative; display: inline-block;}
|
|
.faillinks {display: none; border: 1px solid #c8c8c8; left: 100px; position: absolute; z-index: 1; white-space: nowrap; background-color: #f9f9f9; padding: 10px;}
|
|
.faillinks a {display: block;}
|
|
.failcell:hover .faillinks {display: block;}
|
|
.passedlink {color: green;}
|
|
.key {font-size: 0.9em; color: dimgrey;}
|
|
</style>
|
|
<meta charset="utf-8">
|
|
<title>Fedora nightly compose finder</title>
|
|
</head>
|
|
<body>
|
|
<h1>Fedora nightly compose finder</h1>
|
|
<p>This page helps you locate recent Fedora nightly composes. For each Fedora image, you can find the latest Branched and Rawhide nightly composes that completed. For images tested by <a href="https://openqa.fedoraproject.org">openQA</a>, you can also find the latest image for which all image-specific tests passed. This is not as strong an indication of quality as official Alpha or Beta status, but at least indicates that the image successfully boots, completes a system installation, and boots to the installed system, in a straightforward virtual machine configuration. These images are <b>NOT</b> official Fedora pre-releases. For assistance, contact <a href="https://matrix.to/#/#quality:fedoraproject.org">#quality:fedoraproject.org on Matrix</a> or the <a href="https://lists.fedoraproject.org/archives/list/test@lists.fedoraproject.org/">test@ mailing list</a>.</p>
|
|
<p class='key'>Red linked images are known to have failed tests; green linked images are known to have passed all tests; blue linked images are untested or testing status is unknown. Mouse over red links to display links to the failed tests.</p>
|
|
<p class='key'>Generated by <a href="https://forge.fedoraproject.org/quality/fedora-nightlies">Fedora nightlies</a> at: {{DATE}} (UTC)</p>
|
|
{{TABLES}}
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
## FUNCTIONS
|
|
|
|
def current_releases():
|
|
"""Figure out what releases we care about: for now, that's Rawhide
|
|
plus Branched if there is one. Note: we don't cache this any more
|
|
as fedfind does it for us (it caches the collections data it uses
|
|
to figure this out, with a one day expiry).
|
|
"""
|
|
curr = fedfind.helpers.get_current_release(branched=False)
|
|
branched = fedfind.helpers.get_current_release(branched=True)
|
|
if branched > curr:
|
|
return [str(branched), 'Rawhide']
|
|
else:
|
|
return ['Rawhide']
|
|
|
|
def get_cell(image):
|
|
"""Build a results table cell for an image. This is a bit complex
|
|
and looks pretty messy in-line in add_group, so it's split out
|
|
here."""
|
|
# YAY partial, we all love partial. We want a link to the image
|
|
# whatever, but its class differs depending on test status.
|
|
lnpart = functools.partial(Element, 'a', image['compose'], href=image['url'])
|
|
# simple cases first: pass or none. Just add the link.
|
|
if image.testspass is True:
|
|
content = lnpart(cssclass='passedlink')
|
|
elif image.testspass is None:
|
|
content = lnpart()
|
|
else:
|
|
# fail. this case is more complex. we need the contents of the
|
|
# cell to include the failed test links popups.
|
|
# First build the links div:
|
|
faillinks = ['Failed tests:']
|
|
fails = [(str(job['id']), job['test']) for job in image['openqa']
|
|
if job['result'] not in (oqc.JOB_RESULT_PASSED, oqc.JOB_RESULT_SOFTFAILED)]
|
|
if fails:
|
|
faillinks.extend(
|
|
[Element('a', "openQA: {0} ({1})".format(jobid, jobtest),
|
|
href='https://openqa.fedoraproject.org/tests/{0}'.format(jobid))
|
|
for (jobid, jobtest) in fails])
|
|
faillinksdiv = Element('div', faillinks, cssclass='faillinks')
|
|
# now wrap it and the image link in the cell div:
|
|
imglink = lnpart(cssclass='failedlink')
|
|
content = Element('div', [imglink, faillinksdiv], cssclass='failcell')
|
|
return Element('td', content)
|
|
|
|
## ERROR CLASSES
|
|
|
|
|
|
class NightlyFileError(Exception):
|
|
"""Exception raised when there's a problem with access to one of
|
|
the data files.
|
|
"""
|
|
def __init__(self, path, access):
|
|
self.path = path
|
|
self.access = access
|
|
super(NightlyFileError, self).__init__()
|
|
|
|
def __str__(self):
|
|
"""Human-readable representation of the problem."""
|
|
if self.access == 'e':
|
|
return "Required directory {0} doesn't exist or can't be accessed!".format(self.path)
|
|
if self.access == 'r':
|
|
accstr = 'read'
|
|
elif self.access == 'w':
|
|
accstr = 'written'
|
|
return "Path {0} exists but could not be {1}!".format(self.path, accstr)
|
|
|
|
## HTML ELEMENT CLASSES
|
|
|
|
|
|
class Element(object):
|
|
"""A generic HTML element class. Probably missing all sorts of
|
|
stuff, but these are the bits I need. self.contents should always
|
|
wind up as an iterable of things that can be str()ed.
|
|
"""
|
|
def __init__(self, name, contents='', cssclass='', **kwargs):
|
|
self.name = name
|
|
# handle contents as string...
|
|
if isinstance(contents, str):
|
|
self.contents = [contents]
|
|
else:
|
|
# or iterable of string-able things...
|
|
try:
|
|
''.join(str(item) for item in contents)
|
|
self.contents = contents
|
|
except TypeError:
|
|
# or single non-iterable string-able thing (usually
|
|
# another Element)
|
|
self.contents = [contents]
|
|
self.attributes = dict()
|
|
if kwargs:
|
|
self.attributes.update(kwargs)
|
|
# can't use class as kwarg, grr
|
|
if cssclass:
|
|
self.attributes['class'] = cssclass
|
|
|
|
def __str__(self):
|
|
"""Write the opening tag, include the attributes, add each
|
|
item in 'contents' on a new line, write the closing tag.
|
|
"""
|
|
attrtext = ''
|
|
if self.attributes:
|
|
# sort the attributes, makes testing easier
|
|
for (attr, value) in sorted(self.attributes.items()):
|
|
attrtext += ' {0}="{1}"'.format(attr, value)
|
|
conts = ''.join(str(item) for item in self.contents)
|
|
if len(conts) > 80:
|
|
contents = '\n'.join(str(item) for item in self.contents)
|
|
else:
|
|
contents = conts
|
|
# fuzzy...
|
|
if len(contents) > 80:
|
|
contents = '\n{0}\n'.format(contents)
|
|
return '<{0}{1}>{2}</{0}>'.format(self.name, attrtext, contents)
|
|
|
|
class ReleaseTable(Element):
|
|
"""A per-release table in the format we use."""
|
|
def __init__(self, release, groups=None):
|
|
# build the release name header
|
|
header = Element('th', 'Fedora {0}'.format(release),
|
|
colspan=3, cssclass='relheader')
|
|
# do super.__init__ with just a row containing the header
|
|
super(ReleaseTable, self).__init__(
|
|
'table', Element('tr', header))
|
|
# add the rows for each group to the table contents
|
|
if groups:
|
|
for (name, group) in groups.items():
|
|
self.add_group(name, group)
|
|
|
|
def add_group(self, name, group):
|
|
"""Add table rows for a single image group to the table's
|
|
contents.
|
|
"""
|
|
# build the group name header
|
|
groupheader = Element('th', name, colspan=3, cssclass='groupheader')
|
|
# start contents list with a table row containing the header
|
|
contents = [Element('tr', groupheader)]
|
|
# add another row containing the column title headers
|
|
columntitles = [(Element('th', column)) for column in
|
|
('Arch', 'Last built', 'Last known good')]
|
|
contents.append(Element('tr', columntitles))
|
|
if not any(group[arch] for arch in group):
|
|
# we have no images for this group, skip it
|
|
return
|
|
for arch in group:
|
|
# add a row if we have at least one item for the arch
|
|
if group[arch]:
|
|
# first cell: arch name
|
|
cells = [Element('td', arch)]
|
|
for image in group[arch]:
|
|
cells.append(get_cell(image))
|
|
# pad the row with empty cells (this is when we don't
|
|
# have a 'last known good' as no image passed tests)
|
|
while len(cells) < 3:
|
|
cells.append(Element('td'))
|
|
# stuff the cells in a row and add it to the contents
|
|
row = Element('tr', cells)
|
|
contents.append(row)
|
|
# stuff our new rows into the table's contents
|
|
self.contents.extend(contents)
|
|
|
|
## MAIN CLASSES
|
|
|
|
|
|
class Image(dict):
|
|
"""Class representing a single image from the data. Just a dict
|
|
with a few extra methods and properties.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
# we always want these keys to exist, so set default values
|
|
# *BEFORE* super __init__
|
|
self['openqa'] = []
|
|
super(Image, self).__init__(*args, **kwargs)
|
|
|
|
@property
|
|
def release(self):
|
|
"""The release for this image."""
|
|
rel = fedfind.helpers.parse_cid(self.get('compose', ''))[0]
|
|
# 'rawhide' -> 'Rawhide', if something weird went wrong
|
|
# rel may be None so we have to be careful
|
|
if rel:
|
|
rel = rel.capitalize()
|
|
return rel
|
|
|
|
@property
|
|
def group(self):
|
|
"""The 'image group' for this image. Images are grouped by
|
|
subvariant and type; within each group will be multiple images
|
|
for different arches.
|
|
"""
|
|
return "{0} {1}".format(self.get('subvariant', ''), self.get('type', ''))
|
|
|
|
@property
|
|
def sortid(self):
|
|
"""A property used for comparisons with other Image instances,
|
|
notably for sorting them. For now we just take all digits from
|
|
the compose ID; this works OK for sorting nightlies of the
|
|
same release against each other. Does not work for sorting
|
|
nightlies of *different* releases against each other, or other
|
|
types of compose.
|
|
"""
|
|
return int(''.join([char for char in self.get('compose', '') if char.isdigit()]))
|
|
|
|
@property
|
|
def openqa_flavor(self):
|
|
"""The openQA 'flavor' for this image. Used for finding the
|
|
right Image for an openQA test. We should make sure to always
|
|
keep this in sync with fedora_openqa_schedule.
|
|
"""
|
|
try:
|
|
return fedfind.helpers.identify_image(self, undersub=True, out='string')
|
|
except KeyError:
|
|
# just in case dict was missing some value
|
|
return ''
|
|
|
|
@property
|
|
def testspass(self):
|
|
"""Whether we have some tests for the image and they all
|
|
passed. Note this does *NOT* know when we have 'all' tests for
|
|
an image. That's a bit out of scope here: it's expected the
|
|
results should only be updated when all tests are done. False
|
|
if any tests failed, True if all tests passed, None if we have
|
|
no test results.
|
|
"""
|
|
res = None
|
|
if self.get('openqa'):
|
|
res = all(job['result'] in (oqc.JOB_RESULT_PASSED, oqc.JOB_RESULT_SOFTFAILED)
|
|
for job in self['openqa'])
|
|
return res
|
|
|
|
def matches(self, other):
|
|
"""Check if this Image represents the same image as another
|
|
Image instance or dict.
|
|
"""
|
|
ret = True
|
|
for key in ('compose', 'path'):
|
|
if not self.get(key) or not other.get(key):
|
|
raise ValueError("Image.matches: something bad happened!")
|
|
if not other[key] == self[key]:
|
|
ret = False
|
|
return ret
|
|
|
|
def validate(self):
|
|
"""Check this dict is usable."""
|
|
pass
|
|
|
|
|
|
class Nightlies(object):
|
|
"""Class representing the data and providing methods to update it
|
|
and generate output.
|
|
"""
|
|
def __init__(self, datafile=None, htmlfile=None):
|
|
self.datafile = datafile
|
|
self.htmlfile = htmlfile
|
|
if not self.datafile:
|
|
self.datafile = '/var/www/fedora_nightlies/nightlies.json'
|
|
if not self.htmlfile:
|
|
self.htmlfile = '/var/www/fedora_nightlies/nightlies.html'
|
|
logger.debug("Data file: %s", self.datafile)
|
|
logger.debug("HTML file: %s", self.htmlfile)
|
|
self._check_files()
|
|
self.data = self._read_data()
|
|
|
|
def _check_files(self):
|
|
"""Check the data and HTML files are readable and writeable.
|
|
Create their parent directories if necessary. Raises
|
|
NightlyFileError if there's any problem. May raise OSError
|
|
via `os.mkdir` if we try to create a directory and it fails.
|
|
"""
|
|
for fil in (self.datafile, self.htmlfile):
|
|
# check parent directory, creating if necessary
|
|
directory = os.path.dirname(fil)
|
|
if not os.path.exists(directory):
|
|
os.mkdir(directory)
|
|
if not os.access(directory, os.W_OK):
|
|
raise NightlyFileError(directory, 'w')
|
|
|
|
# check file can be read/written if present
|
|
if os.path.exists(fil):
|
|
if not os.access(fil, os.R_OK):
|
|
raise NightlyFileError(fil, 'r')
|
|
if not os.access(fil, os.W_OK):
|
|
raise NightlyFileError(fil, 'w')
|
|
|
|
def _read_data(self):
|
|
"""Read in the data from the data file."""
|
|
try:
|
|
with open(self.datafile, 'r') as datafh:
|
|
datatext = datafh.read()
|
|
if datatext:
|
|
data = json.loads(datatext)
|
|
# run the image dicts through correct_image to fix
|
|
# up older data
|
|
return [Image(fedfind.helpers.correct_image(img)) for img in data]
|
|
else:
|
|
# file exists but is empty
|
|
return []
|
|
except IOError:
|
|
# no such file - first run, probably
|
|
return []
|
|
|
|
def write_data(self):
|
|
"""Serialize the current data dict out to the data file."""
|
|
logger.debug("Writing data")
|
|
self._prune_data()
|
|
datajson = json.dumps(self.data, sort_keys=True, indent=4)
|
|
with open(self.datafile, 'w') as datafh:
|
|
datafh.write(datajson)
|
|
|
|
def _prune_data(self, rels=None):
|
|
"""Remove data for releases we no longer care about (not in
|
|
current_releases) and data we no longer need.
|
|
"""
|
|
logger.debug("Pruning data")
|
|
logger.debug("Current images:")
|
|
for image in self.data:
|
|
logger.debug(image['url'])
|
|
if not rels:
|
|
rels = current_releases()
|
|
# flatten the latest image set
|
|
latests = self.get_latests(flat=True)
|
|
# we need to know the date...
|
|
today = datetime.datetime.today()
|
|
# copy the list to iterate over it
|
|
for image in list(self.data):
|
|
imgdate = fedfind.helpers.parse_cid(image["compose"], dic=True)['date']
|
|
imgdate = fedfind.helpers.date_check(imgdate, fail_raise=False)
|
|
if imgdate and (today - imgdate).days > 180:
|
|
logger.debug("Removing: %s (aged out)", image["url"])
|
|
self.data.remove(image)
|
|
elif image.release not in rels:
|
|
logger.debug("Removing: %s (release gone)", image['url'])
|
|
self.data.remove(image)
|
|
elif image not in latests:
|
|
logger.debug("Removing: %s (no longer latest)", image['url'])
|
|
self.data.remove(image)
|
|
# remove any duped openQA job dicts. this should not
|
|
# happen any more, but may as well keep it just in case.
|
|
elif image['openqa']:
|
|
for job in list(image['openqa']):
|
|
while True:
|
|
matches = [gotjob for gotjob in image['openqa'] if job == gotjob]
|
|
if len(matches) > 1:
|
|
logger.debug("Removing duplicated job: %s", job['id'])
|
|
image['openqa'].remove(job)
|
|
else:
|
|
break
|
|
|
|
def seed_data(self, days=28):
|
|
"""Initialize the base data set by reading in the last 'days'
|
|
worth of composes. This is only ever going to be needed at
|
|
first run or if the stored data are lost.
|
|
"""
|
|
# get a YYYYMMDD 28 days in the past.
|
|
fromdate = datetime.date.today() - datetime.timedelta(days)
|
|
fromdate = fromdate.strftime('%Y%m%d')
|
|
|
|
composes = []
|
|
for rel in current_releases():
|
|
# this should be a unique ID in PDC
|
|
relid = "fedora-{0}".format(rel)
|
|
pdcrel = fedfind.helpers.pdc_query('releases', {'release_id': relid})[0]
|
|
# this is all compose IDs for the release
|
|
cidlist = pdcrel['compose_set']
|
|
for cid in cidlist:
|
|
(_, cdate, ctype, _) = fedfind.helpers.parse_cid(cid)
|
|
if cdate >= fromdate and ctype == 'nightly':
|
|
composes.append(cid)
|
|
self.add_images(composes)
|
|
self.update_openqa(composes)
|
|
self.write_data()
|
|
|
|
def add_images(self, composes):
|
|
"""Update the 'last built' data for all specified composes (an
|
|
iterable of compose IDs).
|
|
"""
|
|
for compose in composes:
|
|
logger.info("Updating 'last built' for: %s", compose)
|
|
try:
|
|
ffrel = fedfind.release.get_release(cid=compose)
|
|
except ValueError:
|
|
logger.warning("Could not find compose %s! Skipping...", compose)
|
|
continue
|
|
except fedfind.exceptions.UnsupportedComposeError:
|
|
logger.info("Compose %s is not supported by fedfind! Skipping...", compose)
|
|
continue
|
|
if ffrel.release not in current_releases():
|
|
logger.info("Compose %s is not for Branched or Rawhide! "
|
|
"Skipping...", compose)
|
|
continue
|
|
for imgdict in ffrel.all_images:
|
|
# append extra properties to the image dict
|
|
imgdict['compose'] = compose
|
|
imgdict['url'] = "{0}/{1}".format(ffrel.location, imgdict['path'])
|
|
# update existing entry for image if there is one, or
|
|
# else create an entry
|
|
matches = [currimg for currimg in self.data if
|
|
currimg.matches(imgdict)]
|
|
if len(matches) == 0:
|
|
self.data.append(Image(imgdict))
|
|
elif len(matches) == 1:
|
|
matches[0].update(imgdict)
|
|
else:
|
|
# FIXME: better logging / raise or something
|
|
logger.warning("add_images(): More than one "
|
|
"matching image found!")
|
|
logger.info("Last built update done")
|
|
|
|
def update_openqa(self, composes):
|
|
"""Update data based on openQA test results for all specified
|
|
composes (an iterable of compose IDs).
|
|
"""
|
|
client = openqa_client.client.OpenQA_Client('openqa.fedoraproject.org')
|
|
# dict to note which composes we've tried to update as we
|
|
# went along - we don't wanna do this over and over
|
|
updated = set()
|
|
for compose in composes:
|
|
logger.info("Updating openQA results for: %s", compose)
|
|
# get all openQA jobs for the compose
|
|
try:
|
|
jobs = client.get_jobs(build=compose)
|
|
except openqa_client.exceptions.OpenQAClientError as err:
|
|
logger.error("Error performing openQA request for %s! Error: %s", compose, err)
|
|
continue
|
|
# check for still-running jobs, exclude them (we can't do
|
|
# anything sensible with them) and warn
|
|
running = [job for job in jobs if job['state'] not in oqc.JOB_FINAL_STATES]
|
|
if running:
|
|
logger.warning("Some jobs still scheduled or running! Jobs: %s will be ignored",
|
|
', '.join(str(job['id']) for job in running))
|
|
jobs = [job for job in jobs if job not in running]
|
|
if not jobs:
|
|
logger.warning("No jobs found for %s!", compose)
|
|
continue
|
|
|
|
for job in jobs:
|
|
flavor = job['settings']['FLAVOR']
|
|
arch = job['settings']['ARCH']
|
|
if flavor == 'universal':
|
|
# not an image-specific test, skip
|
|
continue
|
|
# find the matching image in the data. if there isn't
|
|
# one and we haven't updated the built data for this
|
|
# compose, update it and try again.
|
|
matches = []
|
|
while True:
|
|
matches = [image for image in self.data if
|
|
image['compose'] == compose and
|
|
image.openqa_flavor == flavor and
|
|
image['arch'] == arch]
|
|
# we break *after* doing the match so the match
|
|
# gets re-done after the add_images
|
|
if matches or compose in updated:
|
|
break
|
|
else:
|
|
self.add_images([compose])
|
|
updated.add(compose)
|
|
# no match
|
|
if len(matches) == 0:
|
|
logger.warning("update_openqa: No Image found for flavor %s, "
|
|
"arch %s, compose %s", flavor, arch, compose)
|
|
# exactly one match! yay. add job to image's list...if
|
|
# we don't already have it
|
|
elif len(matches) == 1 and job not in matches[0]['openqa']:
|
|
matches[0]['openqa'].append(job)
|
|
elif len(matches) > 1:
|
|
logger.warning("update_openqa: More than one Image found for "
|
|
"flavor %s, arch %s, compose %s", flavor, arch, compose)
|
|
logger.info("openQA update done")
|
|
|
|
def find_latest(self, rel, group, arch):
|
|
"""Find 'last built' and 'last known good' for a specific
|
|
release, group and arch.
|
|
"""
|
|
imgs = [img for img in self.data if img.release == rel and
|
|
img.group == group and img['arch'] == arch]
|
|
passimgs = [img for img in imgs if img.testspass]
|
|
if imgs:
|
|
latest = sorted(imgs, key=lambda x: x.sortid)[-1]
|
|
if passimgs:
|
|
passed = sorted(passimgs, key=lambda x: x.sortid)[-1]
|
|
return [latest, passed]
|
|
else:
|
|
return [latest]
|
|
else:
|
|
return []
|
|
|
|
def get_latests(self, flat=False):
|
|
"""Find all 'last built' and 'last known good' images. Return
|
|
sorted by release, group and arch if flat is True, or just a
|
|
list if flat is False.
|
|
"""
|
|
# get the releases, arches and groups in the data and sort.
|
|
# 'Rawhide' will sort after numbers, groups are sorted by
|
|
# weight of any old image in the group, arches not yet sorted
|
|
rels = sorted(set(image.release for image in self.data))
|
|
arches = set(image['arch'] for image in self.data)
|
|
groups = dict((image.group, fedfind.helpers.get_weight(image, arch=False))
|
|
for image in self.data)
|
|
groups = sorted(groups.keys(), key=lambda x: groups[x], reverse=True)
|
|
if flat:
|
|
# this is pretty easy....
|
|
return list(itertools.chain.from_iterable(
|
|
(self.find_latest(rel, group, arch)
|
|
for rel in rels for group in groups for arch in arches)))
|
|
else:
|
|
# mother of god, what is this? OK, relax, it's just a
|
|
# dict grouping 'find_latest' results by rel->group->arch
|
|
return OrderedDict(
|
|
(rel, OrderedDict(
|
|
(group, dict(
|
|
(arch, self.find_latest(rel, group, arch)) for arch in arches))
|
|
for group in groups))
|
|
for rel in rels)
|
|
|
|
def write_html(self):
|
|
"""Write out the HTML representation of the data."""
|
|
# get the tables, via get_latests and ReleaseTable
|
|
logger.debug("Writing HTML")
|
|
tables = (ReleaseTable(rel, groups)
|
|
for (rel, groups) in self.get_latests().items())
|
|
# stick the tables together as a string
|
|
tables = '\n'.join(str(table) for table in tables)
|
|
# read in the template and sub out the tables and the time
|
|
html = HTML_TEMPLATE.replace('{{TABLES}}', tables)
|
|
html = html.replace('{{DATE}}', str(datetime.datetime.utcnow()))
|
|
# write out the file
|
|
with open(self.htmlfile, 'w') as htmlfh:
|
|
htmlfh.write(html)
|
|
|
|
## FEDORA-MESSAGING CONSUMER CLASS
|
|
|
|
|
|
class NightlyConsumer(object):
|
|
"""A fedora-messaging consumer that updates the data when a new
|
|
compose appears or when openQA testing for a compose completes.
|
|
"""
|
|
def __init__(self):
|
|
self.datafile = fedora_messaging.config.conf["consumer_config"].get("datafile")
|
|
self.htmlfile = fedora_messaging.config.conf["consumer_config"].get("htmlfile")
|
|
self.log = logging.getLogger(self.__class__.__name__)
|
|
|
|
def __call__(self, message):
|
|
"""If this is a 'compose completed' message or it's a "job
|
|
done" message and no more jobs remain for the compose,
|
|
update the data.
|
|
"""
|
|
try:
|
|
nightlies = Nightlies(datafile=self.datafile, htmlfile=self.htmlfile)
|
|
except NightlyFileError as err:
|
|
self.log.error(err)
|
|
return
|
|
if message.topic.endswith('pungi.compose.status.change'):
|
|
# compose status change
|
|
typ = ''
|
|
status = message.body.get('status', 'FINISHED')
|
|
compose = message.body.get('compose_id', '')
|
|
if compose.startswith("EPEL"):
|
|
# we know this is fine, don't log the error
|
|
return
|
|
if compose:
|
|
try:
|
|
typ = fedfind.helpers.parse_cid(compose)[2]
|
|
except ValueError as err:
|
|
self.log.error(f"Could not parse compose ID for message {message.id}: {err}")
|
|
return
|
|
if 'FINISHED' not in status or typ != "nightly":
|
|
return
|
|
# we've got a finished compose: update built data
|
|
self.log.info("Updating for completion of compose %s", compose)
|
|
# set the method to run
|
|
meth = functools.partial(nightlies.add_images, [compose])
|
|
elif message.topic.endswith('openqa.job.done'):
|
|
# openQA job done
|
|
compose = message.body.get('BUILD', message.body.get('build', ""))
|
|
if message.body.get('remaining', 1) != 0 or not compose:
|
|
return
|
|
# no use running on updates...
|
|
if compose.startswith("Update-") or compose.startswith("Kojitask-"):
|
|
return
|
|
# or Fedora CoreOS builds...
|
|
if "coreos" in compose.lower():
|
|
return
|
|
# ...or ELN builds
|
|
if "-eln-" in compose.lower():
|
|
return
|
|
# if we got here, we're good
|
|
self.log.info("Updating for openQA results of compose %s", compose)
|
|
# set the method to run
|
|
meth = functools.partial(nightlies.update_openqa, [compose])
|
|
# do the update
|
|
meth()
|
|
nightlies.write_data()
|
|
nightlies.write_html()
|
|
|
|
## MAIN LOOP
|
|
|
|
def main():
|
|
"""Init data and write HTML. Run if file is called directly, or
|
|
by the setuptools entrypoint script.
|
|
"""
|
|
try:
|
|
logging.basicConfig(level=logging.INFO)
|
|
nightlies = Nightlies()
|
|
nightlies.seed_data()
|
|
nightlies.write_html()
|
|
except KeyboardInterrupt:
|
|
sys.stderr.write("Interrupted, exiting...\n")
|
|
sys.exit(1)
|
|
except NightlyFileError as err:
|
|
sys.exit(str(err))
|
|
except OSError as err:
|
|
sys.exit("Could not create directory {0}!".format(err.filename))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|