initial revision

This commit is contained in:
Jay Greguske 2015-12-31 14:08:25 -05:00
commit fcdda361d9
8 changed files with 664 additions and 1 deletions

1
.gitignore vendored
View file

@ -27,7 +27,6 @@ var/
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt

30
conf/multilib.conf Normal file
View file

@ -0,0 +1,30 @@
[devel]
white =
black = dmraid-devel
httpd-devel
kdeutils-devel
mkinitrd-devel
java-1.5.0-gcj-devel
java-1.6.0-openjdk-devel
java-1.7.0-icedtea-devel
java-1.7.0-openjdk-devel
java-1.8.0-openjdk-devel
php-devel
[runtime]
white = libflashsupport
libgnat
lmms-vst
nspluginwrapper
perl-libs
redhat-lsb
syslinux-extlinux-nonlinux
syslinux-nonlinux
syslinux-tftpboot
valgrind
wine
yaboot
black = httpd
php
tomcat-native

0
multilib/__init__.py Normal file
View file

31
multilib/fakepo.py Normal file
View file

@ -0,0 +1,31 @@
class FakePackageObject(object):
"""
fake package object that contains enough data to run through the testing
framework herein
"""
def __init__(self, po=None, d=None):
# FPO's can be created from yum Package Objects or dictionaries
if po:
self.name = po.name
self.arch = po.arch
self.provides = po.provides
self.files = po.returnFileEntries()
elif d:
self.name = d['name']
self.arch = d['arch']
self.provides = d['provides']
self.files = d['files']
else:
raise RuntimeError('fake package objects must come from a real yum object or dictionary')
def convert(self):
return {
'name': self.name,
'arch': self.arch,
'provides': self.provides,
'files': self.files
}
def returnFileEntries(self):
return self.files

94
multilib/gentestdata.py Executable file
View file

@ -0,0 +1,94 @@
#!/usr/bin/python -tt
from glob import glob
import logging
from optparse import OptionParser
import os
import yum.packages
try:
# RHEL 6 and earlier
import simplejson as json
except ImportError:
# RHEL 7 and later
import json
import fakepo
# generate test data for multilib
# accepts a path to a compose and writes out a bigass json file with multlib
# stuff
logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.INFO)
ARCHES = ('ppc64', 'x86_64')
def get_options():
parser = OptionParser(usage='%prog [options] path-to-compose')
parser.add_option('-d', '--debug', default=False, action='store_true')
parser.add_option('-o', '--outfile', default='multilib_data.json')
opts, args = parser.parse_args()
if len(args) != 1:
parser.error('you must provide a path to a compose')
if opts.debug:
log.setLevel(logging.DEBUG)
return opts, args[0]
def get_fpo(rpmfile):
"""return a fake yum PackageObject for a given RPM"""
po = yum.packages.YumLocalPackage(filename=rpmfile)
return fakepo.FakePackageObject(po=po)
def find_repos(cpath):
"""generator to recursively go down a path and find yum repositories"""
for (dirpath, dirnames, _) in os.walk(cpath):
if 'repodata' in dirnames:
# make sure this is a repo for a multilib-supporting arch
for arch in ARCHES:
if arch in dirpath:
ppath = dirpath
rpms = glob(os.path.join(dirpath, '*.rpm'))
if len(rpms) == 0:
log.debug('no rpms in base dir')
ppath = os.path.join(dirpath, 'Packages')
rpms = glob(os.path.join(ppath, '*.rpm'))
if len(rpms) == 0:
log.debug('no rpms in Packages')
continue
yield ppath
for deeperdir in dirnames:
find_repos(os.path.join(dirpath, deeperdir))
if __name__ == '__main__':
opts, compose_path = get_options()
data = {}
for repo in find_repos(compose_path):
log.info('processing %s' % repo)
for rpmf in glob(os.path.join(repo, '*.rpm')):
fpo = get_fpo(rpmf)
if 'debuginfo' in fpo.name:
# debuginfos never have multilib, skip for brevity's sake
continue
key = '%s.%s' % (fpo.name, fpo.arch)
if fpo.arch not in ARCHES:
# we have found a 32-bit rpm
if data.has_key(key):
# we have seen the 64-bit rpm already, this is multilib
data[key]['multi'] = True
else:
# have not seen the 64-bit rpm yet
data[key] = {'details': None, 'multi': True}
else:
# we have found a 64-bit rpm (or something we don't care about)
if data.has_key(key):
# we have seen the 32-bit rpm already, this is multilib
data[key]['details'] = fpo.convert()
else:
# we have not seen the 32-bit rpm
data[key] = {'details': fpo.convert(), 'multi': False}
fd = open(opts.outfile, 'w')
json.dump(data, fd, indent=2, sort_keys=True)
fd.close()
log.info('data written to %s' % opts.outfile)

240
multilib/multilib.py Normal file
View file

@ -0,0 +1,240 @@
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
from fnmatch import fnmatch
from ConfigParser import ConfigParser
class MultilibMethod(object):
PREFER_64 = frozenset(
('gdb', 'frysk', 'systemtap', 'systemtap-runtime', 'ltrace', 'strace'))
def __init__(self, config):
self.name = 'base'
def select(self, po):
if po.arch.find('64') != -1:
if po.name in self.PREFER_64:
return True
if po.name.startswith('kernel'):
for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:
if p_name == 'kernel' or p_name == 'kernel-devel':
return True
return False
class NoMultilibMethod(object):
def __init__(self, config):
self.name = 'none'
def select(self, po):
return False
class AllMultilibMethod(MultilibMethod):
def __init__(self, config):
self.name = 'all'
def select(self, po):
return True
class FileMultilibMethod(MultilibMethod):
def __init__(self, section):
self.config = '/etc/multilib.conf'
self.name = 'file'
cp = ConfigParser()
cp.read(self.config)
self.list = cp.get(section, 'white')
def select(self, po):
for item in self.list:
if fnmatch(po.name, item):
return True
return False
class KernelMultilibMethod(object):
def __init__(self, config):
self.name = 'base'
def select(self, po):
if po.arch.find('64') != -1:
if po.name.startswith('kernel'):
for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:
if p_name == 'kernel' or p_name == 'kernel-devel':
return True
return False
class YabootMultilibMethod(object):
def __init__(self, config):
self.name = 'base'
def select(self, po):
if po.arch in ['ppc']:
if po.name.startswith('yaboot'):
return True
return False
class RuntimeMultilibMethod(MultilibMethod):
ROOTLIBDIRS = frozenset(('/lib', '/lib64'))
USRLIBDIRS = frozenset(('/usr/lib', '/usr/lib64'))
LIBDIRS = ROOTLIBDIRS.union(USRLIBDIRS)
OPROFILEDIRS = frozenset(('/usr/lib/oprofile', '/usr/lib64/oprofile'))
WINEDIRS = frozenset(('/usr/lib/wine', '/usr/lib64/wine'))
SANEDIRS = frozenset(('/usr/lib/sane', '/usr/lib64/sane'))
by_dir = set()
# alsa, dri, gtk-accessibility, scim-bridge-gtk, krb5, sasl, vdpau
by_dir.update(frozenset(os.path.join('/usr/lib', p) for p in ('alsa-lib',
'dri', 'gtk-2.0/modules', 'gtk-2.0/immodules', 'krb5/plugins',
'sasl2', 'vdpau')))
by_dir.update(frozenset(os.path.join('/usr/lib64', p) for p in ('alsa-lib',
'dri', 'gtk-2.0/modules', 'gtk-2.0/immodules', 'krb5/plugins',
'sasl2', 'vdpau')))
# pam
by_dir.update(frozenset(os.path.join(p, 'security') for p in ROOTLIBDIRS))
# lsb
by_dir.add('/etc/lsb-release.d')
def __init__(self, config):
self.name = 'runtime'
self.config = '/etc/multilib.conf'
cp = ConfigParser()
cp.read(self.config)
self.whitelist = cp.get(self.name, 'white')
self.blacklist = cp.get(self.name, 'black')
def select(self, po):
if po.name in self.blacklist:
return False
if po.name in self.whitelist:
return True
if MultilibMethod.select(self, po):
return True
if po.name.startswith('kernel'):
for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:
if p_name == 'kernel':
return False
for file in po.returnFileEntries():
(dirname, filename) = file.rsplit('/', 1)
# libraries in standard dirs
if dirname in self.LIBDIRS and fnmatch(filename, '*.so.*'):
return True
if dirname in self.by_dir:
return True
# mysql, qt, etc.
if dirname == '/etc/ld.so.conf.d' and filename.endswith('.conf'):
return True
# nss (Some nss modules end in .so instead of .so.X)
# db (db modules end in .so instead of .so.X)
if dirname in self.ROOTLIBDIRS and (filename.startswith('libnss_') or filename.startswith('libdb-')):
return True
# Optimization:
# All tests beyond here are for things in USRLIBDIRS
if not dirname.startswith(tuple(self.USRLIBDIRS)):
# The dirname does not start with a USRLIBDIR so we can move
# on to the next file
continue
if dirname.startswith(('/usr/lib/gtk-2.0', '/usr/lib64/gtk-2.0')):
# gtk2-engines
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/engines'):
return True
# accessibility
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/modules'):
return True
# scim-bridge-gtk
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/immodules'):
return True
# images
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/loaders'):
return True
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/printbackends'):
return True
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/filesystems'):
return True
# Optimization:
# No tests beyond here for things in /usr/lib*/gtk-2.0
continue
# gstreamer
if dirname.startswith(('/usr/lib/gstreamer-', '/usr/lib64/gstreamer-')):
return True
# qt/kde fun
if fnmatch(dirname, '/usr/lib*/qt*/plugins/*'):
return True
if fnmatch(dirname, '/usr/lib*/kde*/plugins/*'):
return True
# qml
if fnmatch(dirname, '/usr/lib*/qt5/qml/*'):
return True
# images
if fnmatch(dirname, '/usr/lib*/gdk-pixbuf-2.0/*/loaders'):
return True
# xine-lib
if fnmatch(dirname, '/usr/lib*/xine/plugins/*'):
return True
# oprofile
if dirname in self.OPROFILEDIRS and fnmatch(filename, '*.so.*'):
return True
# wine
if dirname in self.WINEDIRS and filename.endswith('.so'):
return True
# sane drivers
if dirname in self.SANEDIRS and filename.startswith('libsane-'):
return True
return False
class DevelMultilibMethod(RuntimeMultilibMethod):
def __init__(self, config):
self.name = 'devel'
self.config = '/etc/multilib.conf'
cp = ConfigParser()
cp.read(self.config)
self.whitelist = cp.get(self.name, 'white')
self.blacklist = cp.get(self.name, 'black')
def select(self, po):
if po.name in self.blacklist:
return False
if po.name in self.whitelist:
return True
if RuntimeMultilibMethod.select(self, po):
return True
if po.name.startswith('ghc-'):
return False
if po.name.startswith('kernel'):
for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:
if p_name == 'kernel-devel':
return False
if p_name.endswith('-devel') or p_name.endswith('-static'):
return True
if po.name.endswith('-devel'):
return True
if po.name.endswith('-static'):
return True
return False

269
multilib/test.py Executable file
View file

@ -0,0 +1,269 @@
#!/usr/bin/python -tt
try:
# RHEL 6 and earlier
import simplejson as json
except ImportError:
# RHEL 7 and later
import json
import bz2
from ConfigParser import ConfigParser
import fakepo
from fnmatch import fnmatch
import multilib
# if you want to test the testing with the original mash code
# import mash.multilib as multilib
class test_methods(object):
@classmethod
def setup_class(cls):
try:
fd = bz2.BZ2File('testdata/RHEL-7.1-Server-x86_64.json.bz2', 'r')
pj = json.load(fd)
except IOError:
print 'Run the tests in the same directory as multilib.py'
print 'There should be a testdata subdirectory there'
raise
cls.packages = pj
fd.close()
@classmethod
def teardown_class(cls):
pass
def print_fpo(self, fpo):
fpod = fpo.convert()
return '%s.%s' % (fpod['name'], fpod['arch'])
def test_no(self):
meth = multilib.NoMultilibMethod(None)
for pinfo in self.packages.values():
if not pinfo['details']:
# None pops up when a 32-bit RPM was seen without a
# corresponding 64-bit one of the same name. This can happen
# because of dependencies I guess?
continue
fpo = fakepo.FakePackageObject(d=pinfo['details'])
assert not meth.select(fpo), 'Should be False: %s' % self.print_fpo(fpo)
def test_all(self):
meth = multilib.AllMultilibMethod(None)
for pinfo in self.packages.values():
if not pinfo['details']:
continue
fpo = fakepo.FakePackageObject(d=pinfo['details'])
assert meth.select(fpo), 'Should be True: %s' % self.print_fpo(fpo)
def test_kernel(self):
meth = multilib.KernelMultilibMethod(None)
for pinfo in self.packages.values():
if not pinfo['details']:
continue
fpo = fakepo.FakePackageObject(d=pinfo['details'])
if fpo.arch.find('64') != -1:
if fpo.name.startswith('kernel'):
provides = False
for (p_name, p_flag, (p_e, p_v, p_r)) in fpo.provides:
if p_name == 'kernel' or p_name == 'kernel-devel':
provides = True
if provides:
assert meth.select(fpo), 'Should be True: %s' % self.print_fpo(fpo)
continue
assert not meth.select(fpo), 'Should be False: %s' % self.print_fpo(fpo)
def test_yaboot(self):
meth = multilib.YabootMultilibMethod(None)
for pinfo in self.packages.values():
if not pinfo['details']:
continue
fpo = fakepo.FakePackageObject(d=pinfo['details'])
if fpo.arch == 'ppc' and fpo.name.startswith('yaboot'):
assert meth.select(fpo), 'Should be True: %s' % self.print_fpo(fpo)
else:
assert not meth.select(fpo), 'Should be False: %s' % self.print_fpo(fpo)
def test_file(self):
sect = 'runtime'
meth = multilib.FileMultilibMethod(sect)
cp = ConfigParser()
cp.read('/etc/multilib.conf')
self.list = cp.get(sect, 'white')
for pinfo in self.packages.values():
if not pinfo['details']:
continue
fpo = fakepo.FakePackageObject(d=pinfo['details'])
for item in meth.list:
if fnmatch(fpo.name, item):
assert meth.select(fpo), 'Should be True: %s' % self.print_fpo(fpo)
continue
assert not meth.select(fpo), 'Should be False: %s' % self.print_fpo(fpo)
def test_runtime(self):
meth = multilib.RuntimeMultilibMethod(None)
sect = 'runtime'
cp = ConfigParser()
cp.read('/etc/multilib.conf')
wl = cp.get(sect, 'white')
bl = cp.get(sect, 'black')
for pinfo in self.packages.values():
if not pinfo['details']:
continue
fpo = fakepo.FakePackageObject(d=pinfo['details'])
if fpo.name in bl:
assert not meth.select(fpo), 'Blacklisted, should be False: %s' % self.print_fpo(fpo)
continue
if fpo.name in wl:
assert meth.select(fpo), 'Whitelisted, should be True: %s' % self.print_fpo(fpo)
continue
if not self.do_runtime(fpo, meth):
assert not meth.select(fpo), 'should be False: %s' % self.print_fpo(fpo)
def do_runtime(self, fpo, meth):
if fpo.arch.find('64') != -1:
if fpo.name in meth.PREFER_64:
assert meth.select(fpo), 'preferred 64-bit, should be True: %s' % self.print_fpo(fpo)
return True
if fpo.name.startswith('kernel'):
provides = False
for (p_name, p_flag, (p_e, p_v, p_r)) in fpo.provides:
if p_name == 'kernel' or p_name == 'kernel-devel':
provides = True
if provides:
assert meth.select(fpo), '64-bit kernel, should be True: %s' % self.print_fpo(fpo)
return True
if fpo.name.startswith('kernel'):
# looks redundant, but we're not 64-bit here
for (p_name, p_flag, (p_e, p_v, p_r)) in fpo.provides:
if p_name == 'kernel':
assert not meth.select(fpo), '32-bit kernel should be False: %s' % self.print_fpo(fpo)
return False
for file in fpo.returnFileEntries():
(dirname, filename) = file.rsplit('/', 1)
# libraries in standard dirs
if dirname in meth.LIBDIRS and fnmatch(filename, '*.so.*'):
assert meth.select(fpo), '.so.x files, should be True: %s' % self.print_fpo(fpo)
return True
if dirname in meth.by_dir:
assert meth.select(fpo), 'std dirs, should be True: %s' % self.print_fpo(fpo)
return True
# mysql, qt, etc.
if dirname == '/etc/ld.so.conf.d' and filename.endswith('.conf'):
assert meth.select(fpo), 'ld config, should be True: %s' % self.print_fpo(fpo)
return True
# nss (Some nss modules end in .so instead of .so.X)
# db (db modules end in .so instead of .so.X)
if dirname in meth.ROOTLIBDIRS and (filename.startswith('libnss_') or filename.startswith('libdb-')):
assert meth.select(fpo), '.so files, should be True: %s' % self.print_fpo(fpo)
return True
# Optimization:
# All tests beyond here are for things in USRLIBDIRS
if not dirname.startswith(tuple(meth.USRLIBDIRS)):
# The dirname does not start with a USRLIBDIR so we can move
# on to the next file
continue
if dirname.startswith(('/usr/lib/gtk-2.0', '/usr/lib64/gtk-2.0')):
# gtk2-engines
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/engines'):
assert meth.select(fpo), 'gtk2 engines should be True: %s' % self.print_fpo(fpo)
return True
# accessibility
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/modules'):
assert meth.select(fpo), 'accessibility should be True: %s' % self.print_fpo(fpo)
return True
# scim-bridge-gtk
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/immodules'):
assert meth.select(fpo), 'scim-bridge-gtk should be True: %s' % self.print_fpo(fpo)
return True
# images
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/loaders'):
assert meth.select(fpo), 'image loaders should be True: %s' % self.print_fpo(fpo)
return True
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/printbackends'):
assert meth.select(fpo), 'image backends should be True: %s' % self.print_fpo(fpo)
return True
if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/filesystems'):
assert meth.select(fpo), 'gtk filesystems should be True: %s' % self.print_fpo(fpo)
return True
# Optimization:
# No tests beyond here for things in /usr/lib*/gtk-2.0
continue
# gstreamer
if dirname.startswith(('/usr/lib/gstreamer-', '/usr/lib64/gstreamer-')):
assert meth.select(fpo), 'gstreamer should be True: %s' % self.print_fpo(fpo)
return True
# qt/kde fun
if fnmatch(dirname, '/usr/lib*/qt*/plugins/*'):
assert meth.select(fpo), 'qt plugins should be True: %s' % self.print_fpo(fpo)
return True
if fnmatch(dirname, '/usr/lib*/kde*/plugins/*'):
assert meth.select(fpo), 'kde plugins should be True: %s' % self.print_fpo(fpo)
return True
# qml
if fnmatch(dirname, '/usr/lib*/qt5/qml/*'):
assert meth.select(fpo), 'qml should be True: %s' % self.print_fpo(fpo)
return True
# images
if fnmatch(dirname, '/usr/lib*/gdk-pixbuf-2.0/*/loaders'):
assert meth.select(fpo), 'gdk-pixbuf should be True: %s' % self.print_fpo(fpo)
return True
# xine-lib
if fnmatch(dirname, '/usr/lib*/xine/plugins/*'):
assert meth.select(fpo), 'xine-lib should be True: %s' % self.print_fpo(fpo)
return True
# oprofile
if dirname in meth.OPROFILEDIRS and fnmatch(filename, '*.so.*'):
assert meth.select(fpo), 'oprofile should be True: %s' % self.print_fpo(fpo)
return True
# wine
if dirname in meth.WINEDIRS and filename.endswith('.so'):
assert meth.select(fpo), 'wine .so should be True: %s' % self.print_fpo(fpo)
return True
# sane drivers
if dirname in meth.SANEDIRS and filename.startswith('libsane-'):
assert meth.select(fpo), 'sane drivers should be True: %s' % self.print_fpo(fpo)
return True
return False
def test_devel(self):
sect = 'devel'
cp = ConfigParser()
cp.read('/etc/multilib.conf')
wl = cp.get(sect, 'white')
bl = cp.get(sect, 'black')
meth = multilib.DevelMultilibMethod(None)
for pinfo in self.packages.values():
if not pinfo['details']:
continue
fpo = fakepo.FakePackageObject(d=pinfo['details'])
if fpo.name in bl:
assert not meth.select(fpo), 'Blacklisted, should be False: %s' % self.print_fpo(fpo)
continue
if fpo.name in wl:
assert meth.select(fpo), 'Whitelisted, should be True: %s' % self.print_fpo(fpo)
continue
if self.do_runtime(fpo, meth):
# returns True if a value was identified and asserted, False otherwise
continue
if fpo.name.startswith('ghc-'):
assert not meth.select(fpo), 'ghc package, should be False: %s' % self.print_fpo(fpo)
continue
if fpo.name.startswith('kernel'):
# looks redundant, but we're not 64-bit here
for (p_name, p_flag, (p_e, p_v, p_r)) in fpo.provides:
if p_name == 'kernel-devel':
assert not meth.select(fpo), 'kernel-devel, should be False: %s' % self.print_fpo(fpo)
continue
if p_name.endswith('-devel') or p_name.endswith('-static'):
assert meth.select(fpo), 'kernel-*-devel, should be True: %s' % self.print_fpo(fpo)
continue
if fpo.name.endswith('-devel'):
assert meth.select(fpo), '-devel package, should be True: %s' % self.print_fpo(fpo)
continue
if fpo.name.endswith('-static'):
assert meth.select(fpo), '-static package, should be True: %s' % self.print_fpo(fpo)
continue
assert not meth.select(fpo), 'should be False: %s' % self.print_fpo(fpo)

Binary file not shown.