koji/koji
6
5
Fork 13
Code Issues 245 Pull requests 51 Projects Releases 57 Packages 1 Activity Actions 1

rpm v6 support

Browse source
This commit is contained in:
Mike McLean 2025-08-16 13:03:20 -04:00
commit f9c2e21c89
22 changed files with 661 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

24
builder/kojid
View file

@ -652,14 +652,6 @@ class BuildRoot(object):
- external_repo, optional for external rpm
- location, optional for external rpm
"""
fields = ('name',
'version',
'release',
'epoch',
'arch',
'sigmd5',
'size',
'buildtime')
# Determine db path
dbpath = "%s/usr/lib/sysimage/rpm" % self.rootdir()
if not os.path.exists(dbpath):
@ -678,13 +670,11 @@ class BuildRoot(object):
try:
ts = rpm.TransactionSet()
for h in ts.dbMatch():
pkg = koji.get_header_fields(h, fields)
pkg = koji.get_rpm_ident_fields(h)
# skip our fake packages
if pkg['name'] in ['buildsys-build', 'gpg-pubkey']:
# XXX config
continue
pkg['payloadhash'] = koji.hex_string(pkg['sigmd5'])
del pkg['sigmd5']
ret.append(pkg)
finally:
rpm.delMacro("_dbpath")
@ -851,7 +841,7 @@ class BuildRoot(object):
This mapping relies on the rpmlist.jsonl file for the repo. If this file is missing, the
code will fall back to querying the hub.
This function will raise an error if there is a sigmd5 mismatch for a given rpm.
This function will raise an error if there is a digest mismatch for a given rpm.
:param list rpmlist: rpm list fetched from local RPMDB.
:return: None
@ -3407,17 +3397,13 @@ class ImageTask(BaseTaskHandler):
"""
found = False
hdrlist = []
fields = ['name', 'version', 'release', 'epoch', 'arch',
'buildtime', 'sigmd5']
for root, dirs, files in os.walk(cachepath):
for f in files:
if fnmatch(f, '*.rpm'):
pkgfile = os.path.join(root, f)
hdr = koji.get_header_fields(pkgfile, fields)
hdr['size'] = os.path.getsize(pkgfile)
hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
del hdr['sigmd5']
hdrlist.append(hdr)
hdr = koji.get_rpm_header(pkgfile)
data = koji.get_rpm_ident_fields(hdr)
hdrlist.append(data)
found = True
if not found:
raise koji.LiveCDError('No repos found in yum cache!')

30
cli/koji_cli/commands.py
View file

@ -1380,8 +1380,10 @@ def handle_import(goptions, session, args):
activate_session(session, goptions)
to_import = {}
for path in args:
data = koji.get_header_fields(path, ('name', 'version', 'release', 'epoch',
'arch', 'sigmd5', 'sourcepackage', 'sourcerpm'))
hdr = koji.get_rpm_header(path)
data = koji.get_header_fields(hdr, ('name', 'version', 'release', 'epoch',
'arch', 'sourcepackage', 'sourcerpm'))
data['_ident'] = koji.get_rpm_ident(hdr)
if data['sourcepackage']:
data['arch'] = 'src'
nvr = "%(name)s-%(version)s-%(release)s" % data
@ -1410,13 +1412,13 @@ def handle_import(goptions, session, args):
rinfo = dict([(k, data[k]) for k in ('name', 'version', 'release', 'arch')])
prev = session.getRPM(rinfo)
if prev and not prev.get('external_repo_id', 0):
if prev['payloadhash'] == koji.hex_string(data['sigmd5']):
if prev['payloadhash'] == data['_ident']:
print("RPM already imported: %s" % path)
else:
warn("md5sum mismatch for %s" % path)
warn("digest mismatch for %s" % path)
warn(" A different rpm with the same name has already been imported")
warn(" Existing sigmd5 is %r, your import has %r" % (
prev['payloadhash'], koji.hex_string(data['sigmd5'])))
warn(" Existing rpm has %r, your import has %r" % (
prev['payloadhash'], data['_ident']))
print("Skipping import")
return
if options.test:
@ -3861,7 +3863,21 @@ def anon_handle_rpminfo(goptions, session, args):
print("SRPM Path: %s" % srpm_path)
print("Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z',
time.localtime(info['buildtime'])))
print("SIGMD5: %(payloadhash)s" % info)
got_digest = False
for key in ('sigmd5', 'sha1header', 'sha256header', 'sha3_256header'):
# hubs before 1.36 will not report these fields
# old rpms in the system may have null values
digest = info.get(key)
if digest:
got_digest = True
print("%s: %s" % (key.upper(), digest))
if not got_digest:
if '::' in info['payloadhash']:
# shouldn't happen?
print("Legacy digest: %(payloadhash)s" % info)
else:
# compat case
print("SIGMD5: %(payloadhash)s" % info)
print("Size: %(size)s" % info)
if not info.get('external_repo_id', 0):
headers = session.getRPMHeaders(rpmID=info['id'],

4
cli/koji_cli/lib.py
View file

@ -662,8 +662,8 @@ def download_rpm(build, rpm, topurl, sigkey=None, quiet=False, noprogress=False,
error("Downloaded rpm %s is not valid rpm file, deleting" % path)
# payload hash
sigmd5 = koji.get_header_fields(path, ['sigmd5'])['sigmd5']
if rpm['payloadhash'] != koji.hex_string(sigmd5):
hdr = koji.get_rpm_header(path)
if rpm['payloadhash'] != koji.get_rpm_ident(hdr):
os.unlink(path)
error("Downloaded rpm %s doesn't match db, deleting" % path)

43
docs/source/content_generator_metadata.rst
View file

@ -91,18 +91,37 @@ Each map in the buildroots list contains the following entries:
- version: Version of the tool used.
- components: List of maps containing information about content
installed in the build environment (if any). Each map is guaranteed
installed in the build environment (if any). Each map is required
to contain a **type** field, which determines what other fields are
present in the map. For maps where **type = rpm**, the following
fields will be present:
present in the map. The currently available types are detailed below.
- For maps where **type = rpm**, the following fields must be present:
- name: The rpm name.
- version: The rpm version.
- release: The rpm release.
- epoch: The rpm epoch.
- arch: The rpm arch.
- sigmd5: The SIGMD5 tag from the rpm header.
- signature: The signature used to sign the rpm (if any).
Additionally, the following checksums from the rpm header
must be included if they exist in the header.
If given, these values must be lowercase hex strings.
- sigmd5: The SIGMD5 tag value
- sha1header: The SHA1HEADER tag value
- sha256header: The SHA256HEADER tag value
- sha3_256header: The SHA3_256HEADER tag value
For backwards compatibility, if is sufficient to specify only the sigmd5
checksum for rpm files having v4 and earlier formats.
The rpm id may be specified. If given, this must be the id of the rpm in Koji.
- id: The Koji rpm id
Lastly for rpms, the following legacy field is accepted but ignored:
- signature: ignored
- For maps where **type = file**, the following fields will be present:
@ -198,31 +217,31 @@ The below JSON is based loosely on the output of a docker image build.
"epoch": null,
"arch": "x86_64",
"sigmd5": "a1b2c3...",
"signature": "fd431d51"},
"sha1header": "2a1c91..."},
{"type": "rpm",
"name": "openssl",
"version": "1.0.1e",
"release": "42.el7",
"epoch": null,
"arch": "x86_64",
"sigmd5": "d4e5f6...",
"signature": "fd431d51"},
"sigmd5": "f8e857...",
"sha1header": "ea4c84..."},
{"type": "rpm",
"name": "bind-libs",
"version": "9.9.4",
"release": "18.el7",
"epoch": 32,
"arch": "x86_64",
"sigmd5": "987abc...",
"signature": null},
"sigmd5": "5b2ca0...",
"sha1header": "e73c78..."},
{"type": "rpm",
"name": "python-urllib3",
"version": "1.5",
"release": "8.el7",
"epoch": null,
"arch": "noarch",
"sigmd5": "123hgf...",
"signature": null},
"sigmd5": "5f38e3...",
"sha1header": "25a47b..."},
{"type": "file",
"filename": "jboss-eap-6.3.3-full-build.zip",
"filesize": 12345678,

83
koji/__init__.py
View file

@ -106,6 +106,9 @@ RPM_TAG_HEADERSIGNATURES = 62
RPM_TAG_FILEDIGESTALGO = 5011
RPM_SIGTAG_DSA = 267
RPM_SIGTAG_RSA = 268
RPM_SIGTAG_SHA1 = 269
RPM_SIGTAG_SHA256 = 273
RPM_SIGTAG_SHA3_256 = 279
RPM_SIGTAG_PGP = 1002
RPM_SIGTAG_MD5 = 1004
RPM_SIGTAG_GPG = 1005
@ -881,7 +884,16 @@ class RawHeader(object):
else:
return value.decode(errors='surrogateescape')
def __contains__(self, key):
return key in self.index
def keys(self):
return self.index.keys()
def __getitem__(self, key):
if key not in self.index:
# mimic rpm header object behavior
return None
tag, dtype, offset, count = self.index[key]
assert tag == key
return self._getitem(dtype, offset, count)
@ -1184,6 +1196,72 @@ def get_rpm_header(f, ts=None):
return hdr
def get_rpm_ident(hdr):
"""Extract a unique identifier from rpm header"""
digests = get_rpm_digests(hdr)
return make_rpm_ident(digests)
def get_rpm_ident_fields(hdr):
"""Extract rpm header fields used for identification"""
# these base fields should always exist
fields = ('name', 'version', 'release', 'epoch', 'arch', 'buildtime')
data = get_header_fields(hdr, fields)
# digest fields may vary
digests = get_rpm_digests(hdr)
data.update(digests)
# and our preferred ident value
data['payloadhash'] = make_rpm_ident(digests)
return data
def make_rpm_ident(digests):
"""Extract a unique identifier from rpm digest data"""
# for backwards compatibility, use plain sigmd5 if present
if 'sigmd5' in digests:
return digests['sigmd5']
# otherwise, use the best of the newer options with a label
options = (
# label, key
('sha3-256', 'sha3_256header'),
('sha256', 'sha256header'),
('sha1', 'sha1header'),
)
for label, key in options:
if key in digests:
return '%s::%s' % (label, digests[key])
raise KeyError('No digests found: %r' % digests)
def get_rpm_digests(hdr):
"""Extract digest fields from an rpm header"""
digests = {}
tags = ('sha3_256header', 'sha256header', 'sha1header', 'sigmd5')
for tag in tags:
try:
val = hdr[tag]
except (KeyError, ValueError):
# rpm throws this if it doesn't know the header name
continue
if val is None:
# not present in header
continue
if tag == 'sigmd5':
# rpm returns this one as binary
val = hex_string(val)
# (the rest are already strings)
key = tag.lower()
digests[key] = val
if not digests:
raise KeyError('No known header digests found')
return digests
def _decode_item(item):
"""Decode rpm header byte strings to str in py3"""
if six.PY2:
@ -1249,6 +1327,9 @@ def _get_header_field(hdr, name):
hdr_key = 1051
elif name == "NOPATCH":
hdr_key = 1052
elif name == "SHA3_256HEADER":
# added in v6
hdr_key = RPM_SIGTAG_SHA3_256
else:
raise GenericError("No such rpm header field: %s" % name)
return hdr[hdr_key]
@ -1274,7 +1355,7 @@ def get_header_fields(X, fields=None, src_arch=False):
raise GenericError("rpm's python bindings are not installed")
# resolve the names of all the keys we found in the header
fields = [rpm.tagnames[k] for k in hdr.keys()]
fields = [rpm.tagnames[k] for k in hdr.keys() if k in rpm.tagnames]
for f in fields:
ret[f] = get_header_field(hdr, f, src_arch=src_arch)

3
koji/util.py
View file

@ -1077,6 +1077,9 @@ def check_sigmd5(filename):
assert (data_type == 7) # binary data
assert (count == 16) # 16 bytes of md5
break
else:
# rpm v6 format does not include this header
return True
# seek to location of md5
f.seek(o + 8 + indexcount * 16 + offset)
sigmd5 = f.read(16)

1
kojihub/__init__.py
View file

@ -39,7 +39,6 @@ from .kojihub import ( # noqa: F401
_pkglist_owner_add,
_pkglist_owner_remove,
_pkglist_remove,
_scan_sighdr,
_set_build_volume,
_tag_build,
_untag_build,

312
kojihub/kojihub.py
View file

@ -41,6 +41,7 @@ import re
import secrets
import shutil
import stat
import string
import sys
import tarfile
import tempfile
@ -1590,6 +1591,10 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late
('rpminfo.id', 'id'),
('rpminfo.epoch', 'epoch'),
('rpminfo.draft', 'draft'),
('rpminfo.sigmd5', 'sigmd5'),
('rpminfo.sha1header', 'sha1header'),
('rpminfo.sha256header', 'sha256header'),
('rpminfo.sha3_256header', 'sha3_256header'),
('rpminfo.payloadhash', 'payloadhash'),
('rpminfo.size', 'size'),
('rpminfo.buildtime', 'buildtime'),
@ -4917,7 +4922,11 @@ def get_rpm(rpminfo, strict=False, multi=False):
- arch
- draft
- epoch
- payloadhash
- sigmd5
- sha1header
- sha256header
- sha3_256header
- payloadhash (legacy)
- size
- buildtime
- build_id
@ -5006,6 +5015,10 @@ def _get_rpms(data):
('draft', 'draft'),
('external_repo_id', 'external_repo_id'),
('external_repo.name', 'external_repo_name'),
('sigmd5', 'sigmd5'),
('sha1header', 'sha1header'),
('sha256header', 'sha256header'),
('sha3_256header', 'sha3_256header'),
('payloadhash', 'payloadhash'),
('size', 'size'),
('buildtime', 'buildtime'),
@ -5046,6 +5059,10 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID
- arch
- epoch
- draft
- sigmd5
- sha1header
- sha256header
- sha3_256header
- payloadhash
- size
- buildtime
@ -5076,6 +5093,10 @@ def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID
('rpminfo.arch', 'arch'),
('rpminfo.epoch', 'epoch'),
('rpminfo.draft', 'draft'),
('rpminfo.sigmd5', 'sigmd5'),
('rpminfo.sha1header', 'sha1header'),
('rpminfo.sha256header', 'sha256header'),
('rpminfo.sha3_256header', 'sha3_256header'),
('rpminfo.payloadhash', 'payloadhash'),
('rpminfo.size', 'size'), ('rpminfo.buildtime', 'buildtime'),
('rpminfo.build_id', 'build_id'), ('rpminfo.buildroot_id', 'buildroot_id'),
@ -6838,7 +6859,8 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None):
rpminfo['id'] = nextval('rpminfo_id_seq')
rpminfo['build_id'] = buildinfo['id']
rpminfo['size'] = os.path.getsize(fn)
rpminfo['payloadhash'] = koji.hex_string(koji.get_header_field(hdr, 'sigmd5'))
rpminfo['payloadhash'] = koji.get_rpm_ident(hdr) # legacy field
rpminfo.update(koji.get_rpm_digests(hdr)) # e.g. sha256header
rpminfo['buildroot_id'] = brootid
rpminfo['external_repo_id'] = 0
@ -6866,6 +6888,40 @@ def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None):
return rpminfo
def update_rpm_digests(rpminfo):
"""Transitional function to add missing rpm digests to db"""
rinfo = get_rpm(rpminfo, strict=True)
binfo = get_build(rinfo['build_id'])
builddir = koji.pathinfo.build(binfo)
rpm_path = joinpath(builddir, koji.pathinfo.rpm(rinfo))
# read digests from the rpm
hdr = koji.get_rpm_header(rpm_path)
data = koji.get_rpm_digests(hdr)
# the ident should match
ident = koji.make_rpm_ident(data)
if rinfo['payloadhash'] != ident:
# should not happen
raise koji.GenericError(f'payloadhash mismatch for rpm {rpminfo["id"]}')
# determine updates
updates = {}
for key in data:
if rinfo[key] is not None:
# already set, should match
if rinfo[key] != data[key]:
raise koji.GenericError(f'{key} mismatch for rpm {rpminfo["id"]}')
else:
# not set yet
updates[key] = data[key]
if updates:
update = UpdateProcessor('rpminfo', values=rinfo, clauses=["id = %(id)s"], data=updates)
update.execute()
return updates
def generate_token(nbytes=32):
"""
Generate random hex-string token of length 2 * nbytes
@ -7420,28 +7476,70 @@ class CG_Importer(object):
log_error = functools.partialmethod(log, level=logging.ERROR)
def match_rpm(self, comp):
# TODO: centralize this logic
# TODO: do we allow inclusion of external rpms?
if 'location' in comp:
raise koji.GenericError("External rpms not allowed")
if 'id' in comp:
# not in metadata spec, and will confuse get_rpm
raise koji.GenericError("Unexpected 'id' field in component")
# rpm is no more unique with NVRA as draft build is introduced
rinfo = get_rpm(comp, strict=False)
# look up by rpm id, but other data must still match
rinfo = get_rpm(comp['id'], strict=True)
for key in ('name', 'version', 'release', 'arch'):
if rinfo[key] != comp[key]:
raise koji.GenericError(f"Mismatched {key} for component rpm {comp['id']}")
if 'epoch' in comp:
# The spec lists this field as required, but the code has not
# historically enforced it. If given, it should match.
if comp['epoch'] != rinfo['epoch']:
raise koji.GenericError(f"Mismatched epoch for component rpm {comp['id']}")
# digest fields are checked later on
else:
# note: this will look up by nvra, which may not be unique for rpms with the
# advent of draft builds
rinfo = get_rpm(comp, strict=False)
if not rinfo:
# XXX - this is a temporary workaround until we can better track external refs
self.log_warning("IGNORING unmatched rpm component: %r" % comp)
return None
# Draft rpms are not allowed yet
# TODO: we should consider how to handle them once draft build is enabled for CG
if not context.opts.get('AllowDraftComponents', False):
reject_draft(rinfo, is_rpm=True)
if rinfo['payloadhash'] != comp['sigmd5']:
# ident should match payloadhash
try:
ident = koji.make_rpm_ident(comp)
except KeyError:
raise koji.GenericError('No digest values for rpm component')
if ident != rinfo['payloadhash']:
if 'id' in comp:
# if client specifies an rpm id, we can expect the digest to match
raise koji.GenericError(f'Digest mismatch: {comp!r}')
# XXX - this is a temporary workaround until we can better track external refs
self.log_warning("IGNORING rpm component (md5 mismatch): %r" % comp)
# nvr = "%(name)s-%(version)s-%(release)s" % rinfo
return None
# raise koji.GenericError("md5sum mismatch for %s: %s != %s"
# % (nvr, comp['sigmd5'], rinfo['payloadhash']))
# TODO - should we check the signature field?
# all specified digests must match
digest_fields = ('sigmd5', 'sha1header', 'sha256header', 'sha3_256header')
for key in digest_fields:
digest = comp.get(key)
if digest is None:
# treat the same as unspecified
continue
db_digest = rinfo[key]
if db_digest is None:
# our import may predate tracking this field
# TODO fix digests?
self.log_warning(f"Missing {key} digest in db for component: {comp}")
continue
if digest != db_digest:
# since we already matched one digest, we expect all digests to match
raise koji.GenericError(f'{key} mismatch: {comp!r}')
return rinfo
def match_file(self, comp):
@ -7651,33 +7749,82 @@ def add_external_rpm(rpminfo, external_repo, strict=True):
- entry will not reference a build
- rpm not available to us -- the necessary data is passed in
The rpminfo arg should contain the following fields:
- name, version, release, epoch, arch, payloadhash, size, buildtime
The rpminfo arg MUST contain the following fields:
- name, version, release, epoch, arch, size, buildtime
The rpminfo SHOULD contain all non-null rpm header values for the following
digest fields:
- sigmd5, sha1header, sha256header, sha3_256header
But it MUST contain either at least one of the above or the legacy
payloadhash field.
Returns info as get_rpm
"""
# [!] Calling function should perform access checks
# sanity check rpminfo
# key required fields and types
dtypes = (
('name', str),
('version', str),
('release', str),
('epoch', (int, type(None))),
('arch', str),
('payloadhash', str),
('size', int),
('buildtime', int))
data = {}
for field, allowed in dtypes:
if field not in rpminfo:
raise koji.GenericError("%s field missing: %r" % (field, rpminfo))
if not isinstance(rpminfo[field], allowed):
# this will catch unwanted NULLs
raise koji.GenericError("Invalid value for %s: %r" % (field, rpminfo[field]))
# strip extra fields
rpminfo = dslice(rpminfo, [x[0] for x in dtypes])
# TODO: more sanity checks for payloadhash
data[field] = rpminfo[field]
# digests are more complicated
got_digest = False
digests = (
# key, length
('sigmd5', 32),
('sha1header', 40),
('sha256header', 64),
('sha3_256header', 64),
)
for key, hashlen in digests:
if key in rpminfo:
digest = rpminfo[key]
if digest is None:
# we treat this as if header is missing and caller did not filter out
continue
if not isinstance(digest, str):
# all our digests must be strings
raise koji.GenericError(f"Invalid value for {key}: {digest}")
if digest.strip(string.hexdigits):
raise koji.GenericError(f"Non-hex value for {key}: {digest}")
if len(digest) != hashlen:
raise koji.GenericError(f"Invalid hash length for {key}: {digest}")
# force lowercase
data[key] = digest.lower()
got_digest = True
if 'payloadhash' in rpminfo:
digest = rpminfo['payloadhash']
if not isinstance(digest, str):
raise koji.GenericError(f"Invalid value for payloadhash: {digest}")
# historically we have not been strict about the payloadhash value here
data['payloadhash'] = digest
if not got_digest:
# for backwards compatibility, specifying only payloadhash is allowed
if 'payloadhash' not in data:
raise koji.GenericError(f"Missing digest info: {rpminfo!r}")
else:
ident = koji.make_rpm_ident(data)
if 'payloadhash' in data:
# not required if digests are given, but must match if so
digest = rpminfo['payloadhash']
if digest != ident:
raise koji.GenericError(f"Mismatch for payloadhash: {digest} != {ident}")
data['payloadhash'] = ident
def check_dup():
# Check to see if we have it
@ -7688,9 +7835,20 @@ def add_external_rpm(rpminfo, external_repo, strict=True):
disp = "%(name)s-%(version)s-%(release)s.%(arch)s@%(external_repo_name)s" % previous
if strict:
raise koji.GenericError("external rpm already exists: %s" % disp)
elif data['payloadhash'] != previous['payloadhash']:
if data['payloadhash'] != previous['payloadhash']:
raise koji.GenericError("hash changed for external rpm: %s (%s -> %s)"
% (disp, previous['payloadhash'], data['payloadhash']))
# any other specified digests should match
for key, hashlen in digests:
if key not in data:
# not specified
continue
if previous[key] is None:
# ignore - likely imported before we started tracking
continue
if data[key] != previous[key]:
raise koji.GenericError("hash %s changed for external rpm: %s (%s -> %s)"
% (key, disp, previous[key], data[key]))
else:
return previous
@ -7699,7 +7857,6 @@ def add_external_rpm(rpminfo, external_repo, strict=True):
return previous
# add rpminfo entry
data = rpminfo.copy()
data['external_repo_id'] = get_external_repo_id(external_repo, strict=True)
data['id'] = nextval('rpminfo_id_seq')
data['build_id'] = None
@ -8294,6 +8451,7 @@ def add_rpm_sig(an_rpm, sighdr, sigkey=None):
if rinfo['external_repo_id']:
raise koji.GenericError("Not an internal rpm: %s (from %s)"
% (an_rpm, rinfo['external_repo_name']))
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rinfo
binfo = get_build(rinfo['build_id'])
builddir = koji.pathinfo.build(binfo)
if not os.path.isdir(builddir):
@ -8301,33 +8459,44 @@ def add_rpm_sig(an_rpm, sighdr, sigkey=None):
if sigkey is not None:
validate_sigkey_value(sigkey)
# verify sigmd5 matches rpm and pick sigkey if needed
rawhdr = koji.RawHeader(sighdr)
sigmd5 = koji.hex_string(rawhdr.get(koji.RPM_SIGTAG_MD5))
if sigmd5 != rinfo['payloadhash']:
# note: payloadhash is a misnomer, that field is populated with sigmd5.
# Double check using rpm in case we have somehow misread
rpm_path = "%s/%s" % (builddir, koji.pathinfo.rpm(rinfo))
sigmd5, rawsig = _scan_sighdr(sighdr, rpm_path)
sigmd5 = koji.hex_string(sigmd5)
if sigmd5 != rinfo['payloadhash']:
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rinfo
raise koji.GenericError("wrong md5 for %s: %s" % (nvra, sigmd5))
elif sigkey is None:
rawsig = rawhdr.get(koji.RPM_SIGTAG_GPG)
if not rawsig:
rawsig = rawhdr.get(koji.RPM_SIGTAG_PGP)
if not rawsig:
rawsig = rawhdr.get(koji.RPM_SIGTAG_DSA)
if not rawsig:
rawsig = rawhdr.get(koji.RPM_SIGTAG_RSA)
# verify sig matches existing checksum
rawsighdr = koji.RawHeader(sighdr)
rpm_path = "%s/%s" % (builddir, koji.pathinfo.rpm(rinfo))
orig_sig = koji.RawHeader(koji.rip_rpm_sighdr(rpm_path))
got_match = False
digests = (
('MD5', koji.RPM_SIGTAG_MD5),
('SHA1', koji.RPM_SIGTAG_SHA1),
('SHA256', koji.RPM_SIGTAG_SHA256),
('SHA3-256', koji.RPM_SIGTAG_SHA3_256),
)
for name, sigtag in digests:
if sigtag in rawsighdr and sigtag in orig_sig:
if rawsighdr[sigtag] != orig_sig[sigtag]:
# error on any mismatch
raise koji.GenericError(f'Wrong {name} in signature for {nvra}')
else:
got_match = True
if not got_match:
# new signatures must provide an overlapping checksum with old
raise koji.GenericError(f'Unable to validate signature for {nvra}')
# choose sigkey if needed
if sigkey is None:
if not rawsig:
sigtags = (
koji.RPM_SIGTAG_GPG,
koji.RPM_SIGTAG_PGP,
koji.RPM_SIGTAG_DSA,
koji.RPM_SIGTAG_RSA,
)
for sigtag in sigtags:
rawsig = rawsighdr.get(sigtag)
if rawsig:
sigkey = koji.get_sigpacket_key_id(rawsig)
break
else:
sigkey = ''
# we use the sigkey='' to represent unsigned in the db (so that uniqueness works)
else:
sigkey = koji.get_sigpacket_key_id(rawsig)
# do the insert
sighash = md5_constructor(sighdr).hexdigest()
@ -8338,7 +8507,6 @@ def add_rpm_sig(an_rpm, sighdr, sigkey=None):
try:
insert.execute()
except IntegrityError:
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rinfo
raise koji.GenericError("Signature already exists for package %s, key %s" % (nvra, sigkey))
# - write to fs
@ -8544,43 +8712,6 @@ def delete_rpm_sig(rpminfo, sigkey=None, all_sigs=False):
logger.warning("Deleted signatures %s for rpm %s", found_keys, rinfo['id'])
def _scan_sighdr(sighdr, fn):
"""Splices sighdr with other headers from fn and queries (no payload)"""
# This is hackish, but it works
if not os.path.exists(fn):
raise koji.GenericError("No such path: %s" % fn)
if not os.path.isfile(fn):
raise koji.GenericError("Not a regular file: %s" % fn)
# XXX should probably add an option to splice_rpm_sighdr to handle this instead
sig_start, sigsize = koji.find_rpm_sighdr(fn)
hdr_start = sig_start + sigsize
hdrsize = koji.rpm_hdr_size(fn, hdr_start)
inp = open(fn, 'rb')
outp = tempfile.TemporaryFile(mode='w+b')
# before signature
outp.write(inp.read(sig_start))
# signature
outp.write(sighdr)
inp.seek(sigsize, 1)
# main header
outp.write(inp.read(hdrsize))
inp.close()
outp.seek(0, 0)
ts = rpm.TransactionSet()
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS)
# (we have no payload, so verifies would fail otherwise)
hdr = ts.hdrFromFdno(outp.fileno())
outp.close()
sig = koji.get_header_field(hdr, 'siggpg')
if not sig:
sig = koji.get_header_field(hdr, 'sigpgp')
if not sig:
sig = koji.get_header_field(hdr, 'dsaheader')
if not sig:
sig = koji.get_header_field(hdr, 'rsaheader')
return koji.get_header_field(hdr, 'sigmd5'), sig
def query_rpm_sigs(rpm_id=None, sigkey=None, queryOpts=None):
"""Queries db for rpm signatures
@ -13063,6 +13194,10 @@ class RootExports(object):
- arch
- epoch
- draft
- sigmd5
- sha1header
- sha256header
- sha3_256header
- payloadhash
- size
- buildtime
@ -13154,13 +13289,13 @@ class RootExports(object):
results = []
hdr = koji.get_rpm_header(rpm_path)
fields = koji.get_header_fields(hdr, ['filenames', 'filemd5s', 'filesizes', 'fileflags',
fields = koji.get_header_fields(hdr, ['filenames', 'filedigests', 'filesizes', 'fileflags',
'fileusername', 'filegroupname', 'filemtimes',
'filemodes'])
digest_algo = koji.util.filedigestAlgo(hdr)
for (name, digest, size, flags, user, group, mtime, mode) \
in zip(fields['filenames'], fields['filemd5s'],
in zip(fields['filenames'], fields['filedigests'],
fields['filesizes'], fields['fileflags'],
fields['fileusername'], fields['filegroupname'],
fields['filemtimes'], fields['filemodes']):
@ -13168,7 +13303,8 @@ class RootExports(object):
results.append([name, digest, size, flags, digest_algo, user, group, mtime, mode])
else:
results.append({'name': name, 'digest': digest, 'digest_algo': digest_algo,
'md5': digest, 'size': size, 'flags': flags,
'size': size, 'flags': flags,
'md5': digest, # alias of digest for backwards compat
'user': user, 'group': group, 'mtime': mtime, 'mode': mode})
return _applyQueryOpts(results, queryOpts)
@ -13212,8 +13348,7 @@ class RootExports(object):
return {}
hdr = koji.get_rpm_header(rpm_path)
# use filemd5s for backward compatibility
fields = koji.get_header_fields(hdr, ['filenames', 'filemd5s', 'filesizes', 'fileflags',
fields = koji.get_header_fields(hdr, ['filenames', 'filedigests', 'filesizes', 'fileflags',
'fileusername', 'filegroupname', 'filemtimes',
'filemodes'])
digest_algo = koji.util.filedigestAlgo(hdr)
@ -13221,8 +13356,9 @@ class RootExports(object):
i = 0
for name in fields['filenames']:
if name == filename:
return {'rpm_id': rpm_info['id'], 'name': name, 'digest': fields['filemd5s'][i],
'digest_algo': digest_algo, 'md5': fields['filemd5s'][i],
return {'rpm_id': rpm_info['id'], 'name': name, 'digest': fields['filedigests'][i],
'digest_algo': digest_algo,
'md5': fields['filedigests'][i], # alias of digest for backwards compat
'size': fields['filesizes'][i], 'flags': fields['fileflags'][i],
'user': fields['fileusername'][i], 'group': fields['filegroupname'][i],
'mtime': fields['filemtimes'][i], 'mode': fields['filemodes'][i]}
@ -13392,6 +13528,10 @@ class RootExports(object):
context.session.assertPerm('sign')
return add_rpm_sig(an_rpm, base64.b64decode(data), sigkey=sigkey)
def updateRPMDigests(self, rpminfo):
context.session.assertPerm('admin')
return update_rpm_digests(rpminfo)
def renameRPMSig(self, rpminfo, oldkey, newkey):
"""Rename rpm signature

10
plugins/builder/dud.py
View file

@ -139,17 +139,13 @@ class DudCreateImageTask(BaseBuildTask):
found = False
hdrlist = {}
# For non scratch builds this is a must or it will not work
fields = ['name', 'version', 'release', 'epoch', 'arch',
'buildtime', 'sigmd5']
for root, dirs, files in os.walk(path):
for f in files:
if fnmatch(f, '*.rpm'):
pkgfile = os.path.join(root, f)
hdr = koji.get_header_fields(pkgfile, fields)
hdr['size'] = os.path.getsize(pkgfile)
hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
del hdr['sigmd5']
hdrlist[os.path.basename(pkgfile)] = hdr
hdr = koji.get_rpm_header(pkgfile)
data = koji.get_rpm_ident_fields(hdr)
hdrlist[os.path.basename(pkgfile)] = data
found = True
if not found:
raise koji.LiveCDError('No rpms found in root dir!')

10
plugins/builder/kiwi.py
View file

@ -267,17 +267,13 @@ class KiwiCreateImageTask(BaseBuildTask):
"""
found = False
hdrlist = {}
fields = ['name', 'version', 'release', 'epoch', 'arch',
'buildtime', 'sigmd5']
for root, dirs, files in os.walk(cachepath):
for f in files:
if fnmatch(f, '*.rpm'):
pkgfile = os.path.join(root, f)
hdr = koji.get_header_fields(pkgfile, fields)
hdr['size'] = os.path.getsize(pkgfile)
hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
del hdr['sigmd5']
hdrlist[os.path.basename(pkgfile)] = hdr
hdr = koji.get_rpm_header(pkgfile)
data = koji.get_rpm_ident_fields(hdr)
hdrlist[os.path.basename(pkgfile)] = data
found = True
if not found:
raise koji.LiveCDError('No repos found in yum cache!')

9
schemas/schema.sql
View file

@ -802,6 +802,11 @@ CREATE TABLE rpminfo (
draft BOOLEAN,
external_repo_id INTEGER NOT NULL REFERENCES external_repo(id),
payloadhash TEXT NOT NULL,
-- payloadhash is a legacy field and misnamed
sigmd5 TEXT,
sha1header TEXT,
sha256header TEXT,
sha3_256header TEXT,
size BIGINT NOT NULL,
buildtime BIGINT NOT NULL,
metadata_only BOOLEAN NOT NULL DEFAULT FALSE,
@ -816,6 +821,10 @@ CREATE INDEX rpminfo_build ON rpminfo(build_id);
CREATE UNIQUE INDEX rpminfo_unique_nvra_not_draft ON rpminfo(name,version,release,arch,external_repo_id)
WHERE draft IS NOT TRUE;
CREATE INDEX rpminfo_nvra ON rpminfo(name,version,release,arch,external_repo_id);
CREATE INDEX rpminfo_sigmd5 ON rpminfo(sigmd5);
CREATE INDEX rpminfo_sha1header ON rpminfo(sha1header);
CREATE INDEX rpminfo_sha256header ON rpminfo(sha256header);
CREATE INDEX rpminfo_sha3_256header ON rpminfo(sha3_256header);
-- index for default search method for rpms, PG11+ can benefit from new include method
DO $$
DECLARE version integer;

28
tests/test_cli/test_import.py
View file

@ -26,6 +26,9 @@ class TestImport(utils.CliTestCase):
self.md5 = '00112233445566778899aabbccddeeff'
self.fake_srv_dir = '/path/to/server/import'
self.get_rpm_header = mock.patch('koji.get_rpm_header').start()
self.get_rpm_header.return_value = {'sigmd5': md5_to_bytes(self.md5)}
#
# RPM header example (bash-4.4.12-5.fc26.x86_64.rpm):
# {
@ -110,11 +113,7 @@ class TestImport(utils.CliTestCase):
# check mock calls
activate_session_mock.assert_called_with(session, options)
get_header_fields_mock.assert_called_with(
arguments[0],
('name', 'version', 'release', 'epoch',
'arch', 'sigmd5', 'sourcepackage', 'sourcerpm')
)
get_header_fields_mock.assert_called_once()
session.getRPM.assert_called_with(
dict((k, rpm_header.get(k, ''))
@ -153,11 +152,7 @@ class TestImport(utils.CliTestCase):
# check mock calls
activate_session_mock.assert_called_with(session, options)
get_header_fields_mock.assert_called_with(
arguments[0],
('name', 'version', 'release', 'epoch',
'arch', 'sigmd5', 'sourcepackage', 'sourcerpm')
)
get_header_fields_mock.assert_called_once()
session.getRPM.assert_called_with(
dict((k, rpm_header.get(k, ''))
@ -278,11 +273,11 @@ class TestImport(utils.CliTestCase):
expected=expected)
# Case 2: build exists and status is 'COMPLETE', md5 mismatched
# reseult: import skipped
# result: import skipped
session.getRPM.return_value['payloadhash'] = false_md5
expected_warn = "md5sum mismatch for %s\n" % arguments[0]
expected_warn = "digest mismatch for %s\n" % arguments[0]
expected_warn += " A different rpm with the same name has already been imported\n"
expected_warn += " Existing sigmd5 is %r, your import has %r\n" % (false_md5, self.md5)
expected_warn += " Existing rpm has %r, your import has %r\n" % (false_md5, self.md5)
expected = "Skipping import\n"
self.__skip_import_test(
options, session, arguments,
@ -290,7 +285,7 @@ class TestImport(utils.CliTestCase):
expected=expected, expected_warn=expected_warn)
# Case 3: build exists and status is 'COMPLETE', has external_repo_id
# reseult: import will be performed
# result: import will be performed
session.getRPM.return_value['external_repo_id'] = 1
expected = "uploading %s... done\n" % arguments[0]
expected += "importing %s... done\n" % arguments[0]
@ -639,11 +634,6 @@ class TestImport(utils.CliTestCase):
handle_import(options, session, arguments)
expected = case['msg'] % (nvr, case['state']) + "\n"
self.assert_console_message(stdout, expected)
get_header_fields_mock.assert_called_with(
arguments[0],
('name', 'version', 'release', 'epoch',
'arch', 'sigmd5', 'sourcepackage', 'sourcerpm')
)
activate_session_mock.assert_called_with(session, options)
session.getRPM.assert_not_called()
session.importRPM.assert_not_called()

BIN
tests/test_hub/data/rpms/fake-1.1-37.src.rpm.v6 Normal file
View file

Binary file not shown.

BIN
tests/test_hub/data/rpms/header-unsigned.rpm Normal file
View file

Binary file not shown.

126
tests/test_hub/test_add_external_rpm.py
View file

@ -30,7 +30,7 @@ class TestAddExternalRPM(unittest.TestCase):
'release': 'RELEASE',
'epoch': None,
'arch': 'noarch',
'payloadhash': 'fakehash',
'payloadhash': 'f18f0605e998702af6e3fbea3a6d1c8e',
'size': 42,
'buildtime': 0,
}
@ -55,8 +55,36 @@ class TestAddExternalRPM(unittest.TestCase):
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]
self.assertEqual(insert.data['external_repo_id'], mock.sentinel.repo_id)
self.assertEqual(insert.data['id'], mock.sentinel.rpm_id)
expected = self.rpminfo.copy()
expected['id'] = mock.sentinel.rpm_id
expected['external_repo_id'] = mock.sentinel.repo_id
expected['build_id'] = None
expected['buildroot_id'] = None
self.assertEqual(insert.data, expected)
self.assertEqual(insert.table, 'rpminfo')
def test_add_ext_rpm2(self):
# data variation
self.get_rpm.return_value = None
self.get_external_repo_id.return_value = mock.sentinel.repo_id
self.nextval.return_value = mock.sentinel.rpm_id
rpminfo = self.rpminfo.copy()
rpminfo['sigmd5'] = rpminfo['payloadhash']
rpminfo['sha1header'] = 'a' * 40
rpminfo['sha256header'] = None
# call it
kojihub.add_external_rpm(rpminfo, self.repo)
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]
expected = rpminfo.copy()
expected['id'] = mock.sentinel.rpm_id
expected['external_repo_id'] = mock.sentinel.repo_id
expected['build_id'] = None
expected['buildroot_id'] = None
del expected['sha256header'] # the None value is omitted from insert
self.assertEqual(insert.data, expected)
self.assertEqual(insert.table, 'rpminfo')
def test_add_ext_rpm_bad_data(self):
@ -83,6 +111,52 @@ class TestAddExternalRPM(unittest.TestCase):
self.nextval.assert_not_called()
self.assertEqual(len(self.inserts), 0)
def test_add_ext_rpm_bad_digest(self):
rpminfo = self.rpminfo.copy()
rpminfo['sigmd5'] = [] # not a string
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(rpminfo, self.repo)
expected = "Invalid value for sigmd5: []"
self.assertEqual(expected, str(ex.exception))
rpminfo['sigmd5'] = 'NOT HEX'
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(rpminfo, self.repo)
expected = "Non-hex value for sigmd5: NOT HEX"
self.assertEqual(expected, str(ex.exception))
rpminfo['sigmd5'] = 'c0ffee' # too short
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(rpminfo, self.repo)
expected = "Invalid hash length for sigmd5: c0ffee"
self.assertEqual(expected, str(ex.exception))
rpminfo['sigmd5'] = '0'*32 # valid, but doesn't match payloadhash
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(rpminfo, self.repo)
assert str(ex.exception).startswith('Mismatch for payloadhash:')
rpminfo = self.rpminfo.copy()
rpminfo['payloadhash'] = [] # wrong type
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(rpminfo, self.repo)
expected = "Invalid value for payloadhash: []"
self.assertEqual(expected, str(ex.exception))
# no digest at all
del rpminfo['payloadhash']
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(rpminfo, self.repo)
assert str(ex.exception).startswith('Missing digest info:')
# none of these should have gotten to the end
self.get_external_repo_id.assert_not_called()
self.Savepoint.assert_not_called()
self.get_rpm.assert_not_called()
self.nextval.assert_not_called()
self.assertEqual(len(self.inserts), 0)
def test_add_ext_rpm_dup(self):
prev = self.rpminfo.copy()
prev['external_repo_id'] = mock.sentinel.repo_id
@ -113,12 +187,46 @@ class TestAddExternalRPM(unittest.TestCase):
disp = f"{nvra}@{self.repo}"
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(self.rpminfo, self.repo, strict=False)
self.assertEqual(f"hash changed for external rpm: {disp} (different hash -> fakehash)",
str(ex.exception))
expected = (f"hash changed for external rpm: {disp} "
f"(different hash -> {self.rpminfo['payloadhash']})")
self.assertEqual(expected, str(ex.exception))
self.assertEqual(len(self.inserts), 0)
self.nextval.assert_not_called()
def test_add_ext_rpm_dup2(self):
# data variation
rpminfo = self.rpminfo.copy()
rpminfo['sigmd5'] = rpminfo['payloadhash']
prev = rpminfo.copy()
prev['external_repo_id'] = mock.sentinel.repo_id
prev['external_repo_name'] = self.repo
self.get_rpm.return_value = prev
self.get_external_repo_id.return_value = mock.sentinel.repo_id
# call it without strict
ret = kojihub.add_external_rpm(rpminfo, self.repo, strict=False)
self.assertEqual(ret, self.get_rpm.return_value)
# previous didn't have extra digest
prev['sigmd5'] = None
ret = kojihub.add_external_rpm(rpminfo, self.repo, strict=False)
self.assertEqual(ret, self.get_rpm.return_value)
# different sigmd5
prev['sigmd5'] = 'different hash'
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpminfo
disp = f"{nvra}@{self.repo}"
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(rpminfo, self.repo, strict=False)
expected = (f"hash sigmd5 changed for external rpm: {disp} "
f"(different hash -> {rpminfo['sigmd5']})")
self.assertEqual(expected, str(ex.exception))
# none of these cases should have inserted an entry
self.assertEqual(len(self.inserts), 0)
self.nextval.assert_not_called()
def test_add_ext_rpm_dup_late(self):
prev = self.rpminfo.copy()
prev['external_repo_id'] = mock.sentinel.repo_id
@ -156,8 +264,9 @@ class TestAddExternalRPM(unittest.TestCase):
disp = f"{nvra}@{self.repo}"
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_external_rpm(self.rpminfo, self.repo, strict=False)
self.assertEqual(f"hash changed for external rpm: {disp} (different hash -> fakehash)",
str(ex.exception))
expected = (f"hash changed for external rpm: {disp} "
f"(different hash -> {self.rpminfo['payloadhash']})")
self.assertEqual(expected, str(ex.exception))
self.assertEqual(len(self.inserts), 1)
self.nextval.assert_called_once()
@ -172,3 +281,6 @@ class TestAddExternalRPM(unittest.TestCase):
self.assertEqual(len(self.inserts), 1)
self.nextval.assert_called_once()
# the end

106
tests/test_hub/test_add_rpm_sig.py
View file

@ -98,13 +98,84 @@ class TestAddRPMSig(unittest.TestCase):
rpm_path = DATADIR + '/rpms/header-signed.rpm'
sighdr = koji.rip_rpm_sighdr(rpm_path)
unsigned_path = DATADIR + '/rpms/header-unsigned.rpm'
orighdr = koji.rip_rpm_sighdr(unsigned_path)
with mock.patch('koji.rip_rpm_sighdr') as rip:
rip.return_value = orighdr
kojihub.add_rpm_sig(1, sighdr)
kojihub.add_rpm_sig(1, sighdr)
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]
self.assertEqual(insert.data['rpm_id'], 1)
self.assertEqual(insert.data['sigkey'], '15f712be')
def test_add_rpm_sig_mismatch(self):
"""Test addRPMSig with mismatched sig"""
self.query_execute.side_effect = [[]]
self.isdir.side_effect = [True]
self.get_rpm.side_effect = [{
'id': 1,
'name': 'testpkg',
'version': '1.0.0',
'release': '1',
'arch': 'noarch',
'epoch': None,
'payloadhash': '1706d0174aa29a5a3e5c60855a778c35',
'size': 123,
'external_repo_id': None,
'build_id': 1,
}]
self.open.side_effect = [mock.MagicMock()]
rpm_path = DATADIR + '/rpms/test-pkg-1.0.0-1.fc24.noarch.rpm'
sighdr = koji.rip_rpm_sighdr(rpm_path)
unsigned_path = DATADIR + '/rpms/header-unsigned.rpm'
orighdr = koji.rip_rpm_sighdr(unsigned_path)
with mock.patch('koji.rip_rpm_sighdr') as rip:
rip.return_value = orighdr
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_rpm_sig(1, sighdr)
expected = 'Wrong MD5 in signature for testpkg-1.0.0-1.noarch'
self.assertEqual(str(ex.exception), expected)
self.assertEqual(len(self.inserts), 0)
def test_add_rpm_sig_no_digest_overlap(self):
"""Test addRPMSig with no digest overlap"""
self.query_execute.side_effect = [[]]
self.isdir.side_effect = [True]
self.get_rpm.side_effect = [{
'id': 1,
'name': 'testpkg',
'version': '1.0.0',
'release': '1',
'arch': 'noarch',
'epoch': None,
'payloadhash': '1706d0174aa29a5a3e5c60855a778c35',
'size': 123,
'external_repo_id': None,
'build_id': 1,
}]
self.open.side_effect = [mock.MagicMock()]
# this v6 rpm has sha256 and sha3_256 digests only
rpm_path = DATADIR + '/rpms/fake-1.1-37.src.rpm.v6'
sighdr = koji.rip_rpm_sighdr(rpm_path)
# this v4 rpm is old enough to not have a sha256 digest
orig_path = DATADIR + '/rpms/test-pkg-1.0.0-1.fc24.noarch.rpm'
orighdr = koji.rip_rpm_sighdr(orig_path)
with mock.patch('koji.rip_rpm_sighdr') as rip:
rip.return_value = orighdr
with self.assertRaises(koji.GenericError) as ex:
kojihub.add_rpm_sig(1, sighdr)
expected = 'Unable to validate signature for testpkg-1.0.0-1.noarch'
self.assertEqual(str(ex.exception), expected)
self.assertEqual(len(self.inserts), 0)
def test_add_rpm_sig_external(self):
"""external rpm failure case"""
self.get_rpm.side_effect = [{
@ -134,7 +205,15 @@ class TestAddRPMSig(unittest.TestCase):
"""missing build dir failure case"""
sighdr = 'SIG HEADER 99'
self.isdir.side_effect = [False]
self.get_rpm.side_effect = [{'build_id': 100, 'external_repo_id': None}]
self.get_rpm.side_effect = [{
'id': 1,
'name': 'testpkg',
'version': '1.0.0',
'release': '1',
'arch': 'noarch',
'build_id': 100,
'external_repo_id': None,
}]
with self.assertRaises(koji.GenericError):
kojihub.add_rpm_sig(1, sighdr)
@ -147,7 +226,15 @@ class TestAddRPMSig(unittest.TestCase):
"""bad sigkey failure case"""
sighdr = 'SIG HEADER 99'
self.isdir.return_value = True
self.get_rpm.return_value = {'build_id': 100, 'external_repo_id': None}
self.get_rpm.return_value = {
'id': 1,
'name': 'testpkg',
'version': '1.0.0',
'release': '1',
'arch': 'noarch',
'build_id': 100,
'external_repo_id': None,
}
badkeys = [
'white space',
@ -165,17 +252,4 @@ class TestAddRPMSig(unittest.TestCase):
self.isdir.assert_called()
class TestScanHeaderOnly(unittest.TestCase):
def test_scan_sighdr_header_signed(self):
"""Test _scan_sighdr on a header-only signed package"""
rpm_path = DATADIR + '/rpms/header-signed.rpm'
sighdr = koji.rip_rpm_sighdr(rpm_path)
sigmd5, sig = kojihub._scan_sighdr(sighdr, rpm_path)
self.assertEqual(koji.hex_string(sigmd5), '1706d0174aa29a5a3e5c60855a778c35')
sigkey = koji.get_sigpacket_key_id(sig)
self.assertEqual(sigkey, '15f712be')
# the end

33
tests/test_hub/test_getRPM.py
View file

@ -17,7 +17,7 @@ class TestGetRPM(DBQueryTestCase):
self.context = mock.patch('kojihub.kojihub.context').start()
self.get_external_repo_id = mock.patch('kojihub.kojihub.get_external_repo_id').start()
self.find_build_id = mock.patch('kojihub.kojihub.find_build_id').start()
def tearDown(self):
mock.patch.stopall()
@ -27,6 +27,12 @@ class TestGetRPM(DBQueryTestCase):
kojihub.get_rpm(rpminfo)
self.assertEqual(f"Invalid type for rpminfo: {type(rpminfo)}", str(cm.exception))
rpm_columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release',
'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name',
'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra',
'sigmd5', 'sha1header', 'sha256header', 'sha3_256header']
def test_rpm_info_int(self):
rpminfo = 123
self.qp_execute_return_value = [{'rpminfo.id': 123}]
@ -36,10 +42,7 @@ class TestGetRPM(DBQueryTestCase):
self.assertEqual(len(self.queries), 1)
query = self.queries[0]
self.assertEqual(query.tables, ['rpminfo'])
columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release',
'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name',
'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra']
self.assertEqual(set(query.columns), set(columns))
self.assertEqual(set(query.columns), set(self.rpm_columns))
self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"])
self.assertEqual(query.joins,
['external_repo ON rpminfo.external_repo_id = external_repo.id'])
@ -54,10 +57,7 @@ class TestGetRPM(DBQueryTestCase):
self.assertEqual(len(self.queries), 1)
query = self.queries[0]
self.assertEqual(query.tables, ['rpminfo'])
columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release',
'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name',
'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra']
self.assertEqual(set(query.columns), set(columns))
self.assertEqual(set(query.columns), set(self.rpm_columns))
self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"])
self.assertEqual(query.joins,
['external_repo ON rpminfo.external_repo_id = external_repo.id'])
@ -73,10 +73,7 @@ class TestGetRPM(DBQueryTestCase):
self.assertEqual(len(self.queries), 1)
query = self.queries[0]
self.assertEqual(query.tables, ['rpminfo'])
columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release',
'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name',
'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra']
self.assertEqual(set(query.columns), set(columns))
self.assertEqual(set(query.columns), set(self.rpm_columns))
self.assertEqual(query.clauses, ["rpminfo.id=%(id)s"])
self.assertEqual(query.joins,
['external_repo ON rpminfo.external_repo_id = external_repo.id'])
@ -89,10 +86,7 @@ class TestGetRPM(DBQueryTestCase):
self.assertEqual(len(self.queries), 1)
query = self.queries[0]
self.assertEqual(query.tables, ['rpminfo'])
columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release',
'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name',
'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra']
self.assertEqual(set(query.columns), set(columns))
self.assertEqual(set(query.columns), set(self.rpm_columns))
self.assertEqual(query.clauses, ["rpminfo.name=%(name)s AND version=%(version)s "
"AND release=%(release)s AND arch=%(arch)s"])
self.assertEqual(query.joins,
@ -111,10 +105,7 @@ class TestGetRPM(DBQueryTestCase):
self.assertEqual(len(self.queries), 1)
query = self.queries[0]
self.assertEqual(query.tables, ['rpminfo'])
columns = ['rpminfo.id', 'build_id', 'buildroot_id', 'rpminfo.name', 'version', 'release',
'epoch', 'arch', 'draft', 'external_repo_id', 'external_repo.name',
'payloadhash', 'size', 'buildtime', 'metadata_only', 'extra']
self.assertEqual(set(query.columns), set(columns))
self.assertEqual(set(query.columns), set(self.rpm_columns))
self.assertEqual(query.clauses,
["external_repo_id = %(external_repo_id)s", "rpminfo.id=%(id)s"])
self.assertEqual(query.joins,

22
tests/test_hub/test_import_rpm.py
View file

@ -33,9 +33,12 @@ class TestImportRPM(unittest.TestCase):
self.context_db = mock.patch('kojihub.db.context').start()
self.cursor = mock.MagicMock()
self.md5 = '7061796c6f61642068617368'
self.rpm_header_retval = {
'filename': 'name-version-release.arch.rpm',
'sourcepackage': 2,
'sigmd5': bytearray.fromhex(self.md5),
1000: 'name',
1001: 'version',
1002: 'release',
@ -44,11 +47,13 @@ class TestImportRPM(unittest.TestCase):
1022: 'arch',
1044: 'name-version-release.src.rpm',
1106: 'sourcepackage',
261: 'payload hash',
261: self.md5,
}
self.get_build = mock.patch('kojihub.kojihub.get_build').start()
self.new_build = mock.patch('kojihub.kojihub.new_build').start()
self.get_rpm_header = mock.patch('koji.get_rpm_header').start()
self.get_rpm_ident = mock.patch('koji.get_rpm_ident').start()
self.get_rpm_ident.return_value = self.md5
self.new_typed_build = mock.patch('kojihub.kojihub.new_typed_build').start()
self.nextval = mock.patch('kojihub.kojihub.nextval').start()
self.os_path_exists = mock.patch('os.path.exists').start()
@ -98,7 +103,7 @@ class TestImportRPM(unittest.TestCase):
'arch': 'arch',
'buildtime': 'buildtime',
'draft': False,
'payloadhash': '7061796c6f61642068617368',
'payloadhash': self.md5,
'epoch': 'epoch',
'version': 'version',
'buildroot_id': None,
@ -106,6 +111,7 @@ class TestImportRPM(unittest.TestCase):
'external_repo_id': 0,
'id': 9876,
'size': 0,
'sigmd5': self.md5,
}
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]
@ -138,7 +144,7 @@ class TestImportRPM(unittest.TestCase):
'arch': 'src',
'buildtime': 'buildtime',
'draft': False,
'payloadhash': '7061796c6f61642068617368',
'payloadhash': self.md5,
'epoch': 'epoch',
'version': 'version',
'buildroot_id': None,
@ -146,6 +152,7 @@ class TestImportRPM(unittest.TestCase):
'external_repo_id': 0,
'id': 9876,
'size': 0,
'sigmd5': self.md5,
}
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]
@ -196,7 +203,7 @@ class TestImportRPM(unittest.TestCase):
'arch': 'arch',
'buildtime': 'buildtime',
'draft': False,
'payloadhash': '7061796c6f61642068617368',
'payloadhash': self.md5,
'epoch': 'epoch',
'version': 'version',
'buildroot_id': None,
@ -204,6 +211,7 @@ class TestImportRPM(unittest.TestCase):
'external_repo_id': 0,
'id': 9876,
'size': 0,
'sigmd5': self.md5,
}
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]
@ -257,7 +265,7 @@ class TestImportRPM(unittest.TestCase):
'arch': 'arch',
'buildtime': 'buildtime',
'draft': True,
'payloadhash': '7061796c6f61642068617368',
'payloadhash': self.md5,
'epoch': 'epoch',
'version': 'version',
'buildroot_id': None,
@ -265,6 +273,7 @@ class TestImportRPM(unittest.TestCase):
'external_repo_id': 0,
'id': 9876,
'size': 0,
'sigmd5': self.md5,
}
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]
@ -298,7 +307,7 @@ class TestImportRPM(unittest.TestCase):
'arch': 'src',
'buildtime': 'buildtime',
'draft': True,
'payloadhash': '7061796c6f61642068617368',
'payloadhash': self.md5,
'epoch': 'epoch',
'version': 'version',
'buildroot_id': None,
@ -306,6 +315,7 @@ class TestImportRPM(unittest.TestCase):
'external_repo_id': 0,
'id': 9876,
'size': 0,
'sigmd5': self.md5,
}
self.assertEqual(len(self.inserts), 1)
insert = self.inserts[0]

12
tests/test_hub/test_read_tagged_rpms.py
View file

@ -31,11 +31,14 @@ class TestReadTaggedRPMS(unittest.TestCase):
self.columns = ['rpminfo.name', 'rpminfo.version', 'rpminfo.release', 'rpminfo.arch',
'rpminfo.id', 'rpminfo.epoch', 'rpminfo.draft', 'rpminfo.payloadhash',
'rpminfo.size', 'rpminfo.buildtime', 'rpminfo.buildroot_id',
'rpminfo.build_id', 'rpminfo.metadata_only']
'rpminfo.build_id', 'rpminfo.metadata_only',
'rpminfo.sigmd5', 'rpminfo.sha1header',
'rpminfo.sha256header', 'rpminfo.sha3_256header']
self.joins = ['tag_listing ON rpminfo.build_id = tag_listing.build_id']
self.aliases = ['name', 'version', 'release', 'arch', 'id', 'epoch', 'draft',
'payloadhash', 'size', 'buildtime', 'buildroot_id', 'build_id',
'metadata_only']
'metadata_only',
'sigmd5', 'sha1header', 'sha256header', 'sha3_256header']
self.clauses = ['(tag_listing.active = TRUE)',
'tag_id=%(tagid)s']
self.tables = ['rpminfo']
@ -118,4 +121,7 @@ class TestReadTaggedRPMS(unittest.TestCase):
self.assertEqual(set(query.joins), set(self.joins))
self.assertEqual(set(query.aliases), set(self.aliases))
self.assertEqual(set(query.clauses), set(clauses))
self.assertEqual(query.values, {})
self.assertEqual(query.values, {})
# the end

6
vm/kojivmd
View file

@ -766,10 +766,10 @@ class VMExecTask(BaseTaskHandler):
for chunk in response.iter_content(chunk_size=65536):
f.write(chunk)
if type == 'rpm':
# rpm, check sigmd5. It is enough, as if content is broken,
# check digest if we can. It is enough, as if content is broken,
# rpm will fail later
hdr = koji.get_rpm_header(localpath)
payloadhash = koji.hex_string(koji.get_header_field(hdr, 'sigmd5'))
payloadhash = koji.get_rpm_ident(hdr)
if fileinfo['payloadhash'] != payloadhash:
raise koji.BuildError(
"Downloaded rpm %s doesn't match checksum (expected: %s, got %s)" %
@ -777,6 +777,8 @@ class VMExecTask(BaseTaskHandler):
fileinfo['payloadhash'],
payloadhash))
if not koji.util.check_sigmd5(localpath):
# note that check will trivially pass if there is no sigmd5 header present
# TODO handle other digests
raise koji.BuildError("Downloaded rpm %s doesn't match sigmd5" %
os.path.basename(fileinfo['localpath']))
else:

4
www/kojiweb/index.py
View file

@ -1632,6 +1632,10 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i
values['build'] = build
values['builtInRoot'] = builtInRoot
values['buildroots'] = buildroots
for key in ('sigmd5', 'sha1header', 'sha256header', 'sha3_256header'):
if rpm.get(key):
values['got_digest'] = True
break
kojiweb.util.paginateMethod(server, values, 'listRPMFiles', args=[rpm['id']],
start=fileStart, dataName='files', prefix='file', order=fileOrder)

13
www/kojiweb/templates/rpminfo.html.j2
View file

@ -68,12 +68,21 @@
<tr>
<th>Size</th><td><span title="{{ util.formatThousands(rpm.size) }}">{{ util.formatNatural(rpm.size) }}</span></td>
</tr>
#for key in ('sigmd5', 'sha1header', 'sha256header', 'sha3_256header')
#if rpm.get(key)
<tr>
<th><label title="The MD5 digest of the combined header and payload contents. You can query it by `rpmkeys -Kv foo.rpm`">SIGMD5</label></th><td>{{ rpm.payloadhash }}</td>
<th><label title="{{ key }} digest value for the rpm. Check `rpmkeys -Kv foo.rpm`">{{ key|upper }}</label></th><td>{{ rpm.get(key) }}</td>
</tr>
#endif
#endfor
#if not got_digest
<tr>
<th><label title="Digest value for the rpm. Check `rpmkeys -Kv foo.rpm`">Digest</label></th><td>{{ rpm.payloadhash }}</td>
</tr>
#endif
#if rpm.external_repo_id == 0
<tr>
<th>License</th><td>{{ license }}</td>
<th>License</th><td>{{ license }} {{ got_digest }}</td>
</tr>
#if vcs
<tr>