Add bin/compose-package-moves
Signed-off-by: Tomas Mlcoch <tmlcoch@redhat.com>
This commit is contained in:
parent
4c1dcc7260
commit
db7abc11f9
3 changed files with 369 additions and 0 deletions
1
AUTHORS
1
AUTHORS
|
|
@ -2,3 +2,4 @@ Authors:
|
|||
|
||||
Daniel Mach <dmach@redhat.com>
|
||||
Lubomír Sedlář <lsedlar@redhat.com>
|
||||
Tomas Mlcoch <tmlcoch@redhat.com>
|
||||
|
|
|
|||
45
bin/compose-report-package-moves
Executable file
45
bin/compose-report-package-moves
Executable file
|
|
@ -0,0 +1,45 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*- encoding: utf-8 -*-
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
import productmd.compose
|
||||
|
||||
here = sys.path[0]
|
||||
if here != '/usr/bin':
|
||||
sys.path[0] = os.path.dirname(here)
|
||||
|
||||
from compose_utils.package_moves import ComposePackageMoves
|
||||
|
||||
|
||||
def run(opts):
|
||||
old_compose = productmd.compose.Compose(opts.compose1)
|
||||
new_compose = productmd.compose.Compose(opts.compose2)
|
||||
|
||||
pkgm = ComposePackageMoves()
|
||||
try:
|
||||
data = pkgm.get_report(old_compose, new_compose)
|
||||
except RuntimeError as exc:
|
||||
print('Error: {}'.format(exc), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
name = opts.name or new_compose.info.compose.id
|
||||
pkgm.write(data, path=opts.path, name=name)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('compose1', metavar='OLD_COMPOSE')
|
||||
parser.add_argument('compose2', metavar='NEW_COMPOSE')
|
||||
parser.add_argument('-p', '--path',
|
||||
help='Target directory where to write the output. '
|
||||
'Defaults to current directory.')
|
||||
parser.add_argument('-n', '--name',
|
||||
help='Identifier to be included in generated filenames. '
|
||||
'Defaults to compose id of new compose.')
|
||||
opts = parser.parse_args()
|
||||
run(opts)
|
||||
323
compose_utils/package_moves.py
Normal file
323
compose_utils/package_moves.py
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
__all__ = (
|
||||
"ComposePackageMoves",
|
||||
)
|
||||
|
||||
import os
|
||||
import json
|
||||
|
||||
from kobo.rpmlib import parse_nvra
|
||||
|
||||
|
||||
class ExtraJsonEncoder(json.JSONEncoder):
|
||||
"""Json encoder that can handle sets and tuples"""
|
||||
def default(self, obj):
|
||||
if isinstance(obj, set):
|
||||
return list(obj)
|
||||
if isinstance(obj, tuple):
|
||||
return list(obj)
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
class ComposePackageMoves(object):
|
||||
"""Identify package moves between related variants (e.g. Optional->Base)"""
|
||||
|
||||
def _load_compose_id(self, compose):
|
||||
"""
|
||||
Get compose ID from RPM manifest. Raises `RuntimeError` if metadata
|
||||
file is not found.
|
||||
"""
|
||||
try:
|
||||
return compose.rpms.compose.id
|
||||
except AttributeError:
|
||||
raise RuntimeError('Failed to load metadata from %s'
|
||||
% compose.compose_path)
|
||||
|
||||
def _variant_parent_map(self, variants):
|
||||
"""
|
||||
Construct dict with "family relations".
|
||||
{ 'variant uid': 'parent uid' }
|
||||
"""
|
||||
variant_parent = {}
|
||||
for variant in variants:
|
||||
parent_uid = variant.parent.uid if variant.parent else None
|
||||
variant_parent[variant.uid] = parent_uid
|
||||
return variant_parent
|
||||
|
||||
def _build_struct(self, compose):
|
||||
"""
|
||||
result = {'variant': {'arch': { ('rpm_name', 'rpm_arch'): [{details}] }}}
|
||||
"""
|
||||
rpms = compose.rpms.rpms
|
||||
|
||||
result = {}
|
||||
for variant, arches in rpms.iteritems():
|
||||
result[variant] = {}
|
||||
for arch, srpms in arches.iteritems():
|
||||
if arch == "src":
|
||||
# Ignore src arch in rpm manifest
|
||||
continue
|
||||
result[variant][arch] = {}
|
||||
for srpm_nevra, rpms in srpms.iteritems():
|
||||
for rpm_nevra, data in rpms.iteritems():
|
||||
# Parse filename
|
||||
nvra_dict = parse_nvra(os.path.basename(data["path"]))
|
||||
rpm_name = nvra_dict["name"]
|
||||
rpm_arch = nvra_dict["arch"]
|
||||
# Prepare key
|
||||
key = "%s.%s" % (rpm_name, rpm_arch)
|
||||
# Prepare details
|
||||
details = {
|
||||
"rpm_name": rpm_name,
|
||||
"rpm_arch": rpm_arch,
|
||||
"rpm_nevra": rpm_nevra,
|
||||
"path": data["path"],
|
||||
"category": data["category"],
|
||||
"srpm_nevra": srpm_nevra,
|
||||
}
|
||||
if key in result[variant][arch]:
|
||||
# Compose should have only latest version of each rpm -> print warning
|
||||
print "Warning: There are multiple packages with name %s (%s)" % (rpm_name, rpm_arch)
|
||||
result[variant][arch][key].append(details)
|
||||
else:
|
||||
result[variant][arch][key] = [details]
|
||||
|
||||
return result
|
||||
|
||||
def _cmp_structures(self, old_struct, new_struct, new_variants_map):
|
||||
"""Compare structures returned by _build_struct() method
|
||||
|
||||
Returns:
|
||||
{
|
||||
"new_items": {
|
||||
"variant_name.arch": {
|
||||
"variant": "variant_name",
|
||||
"arch": "arch",
|
||||
"items": {"rpm_name.rpm_arch": [{details},..], ..}
|
||||
}, ..},
|
||||
"moved_items": [{
|
||||
"from": "variant_name",
|
||||
"to": "variant_name",
|
||||
"arch": "arch",
|
||||
"parent": "variant_name",
|
||||
"items": {"rpm_name.rpm_arch": [{details},..], ..}
|
||||
}, ..]
|
||||
}
|
||||
"""
|
||||
|
||||
result = {}
|
||||
result["new_items"] = {}
|
||||
result["moved_items"] = []
|
||||
|
||||
for variant, parent_variant in new_variants_map.iteritems():
|
||||
|
||||
if not parent_variant:
|
||||
# Ignore variants which doesn't have parent
|
||||
continue
|
||||
|
||||
# Get data for both variants in both composes
|
||||
old_variant_data = old_struct.get(variant, {})
|
||||
old_parent_variant_data = old_struct.get(parent_variant, {})
|
||||
new_variant_data = new_struct.get(variant, {})
|
||||
new_parent_variant_data = new_struct.get(parent_variant, {})
|
||||
|
||||
# Do comparison per architecture
|
||||
available_arches = set(old_variant_data.keys())
|
||||
available_arches |= set(old_parent_variant_data.keys())
|
||||
available_arches |= set(new_variant_data.keys())
|
||||
available_arches |= set(new_parent_variant_data.keys())
|
||||
|
||||
for arch in available_arches:
|
||||
old_variant_pkgs = old_variant_data.get(arch, {})
|
||||
old_parent_variant_pkgs = old_parent_variant_data.get(arch, {})
|
||||
new_variant_pkgs = new_variant_data.get(arch, {})
|
||||
new_parent_variant_pkgs = new_parent_variant_data.get(arch, {})
|
||||
|
||||
# == New items ==
|
||||
# Packages which weren't in the variants in previous compose
|
||||
extra_pkgs_in_variant = set(new_variant_pkgs.keys()) - set(old_variant_pkgs.keys())
|
||||
extra_pkgs_in_parent_variant = set(new_parent_variant_pkgs.keys()) - set(old_parent_variant_pkgs.keys())
|
||||
|
||||
# Add variant into new_items list
|
||||
variant_new_items = {
|
||||
"variant": variant,
|
||||
"arch": arch,
|
||||
"items": dict((key, new_variant_pkgs[key]) for key in extra_pkgs_in_variant)
|
||||
}
|
||||
variant_key = "%s.%s" % (variant, arch)
|
||||
assert variant_key not in result["new_items"]
|
||||
result["new_items"][variant_key] = variant_new_items
|
||||
|
||||
# Add parent variant into new_items list
|
||||
parent_variant_new_items = {
|
||||
"variant": parent_variant,
|
||||
"arch": arch,
|
||||
"items": dict((key, new_parent_variant_pkgs[key]) for key in extra_pkgs_in_parent_variant)
|
||||
}
|
||||
variant_key = "%s.%s" % (parent_variant, arch)
|
||||
if variant_key not in result["new_items"]:
|
||||
result["new_items"][variant_key] = parent_variant_new_items
|
||||
else:
|
||||
assert parent_variant_new_items == result["new_items"][variant_key]
|
||||
|
||||
# == Moved items ==
|
||||
# (packages which were previously available in the other variant)
|
||||
moves_into_variant_from_parent = extra_pkgs_in_variant & set(old_parent_variant_pkgs.keys())
|
||||
moves_into_parent_from_variant = extra_pkgs_in_parent_variant & set(old_variant_pkgs.keys())
|
||||
|
||||
# Moved from parent to child
|
||||
moved_items_dict = {
|
||||
"from": parent_variant,
|
||||
"to": variant,
|
||||
"arch": arch,
|
||||
"parent": parent_variant,
|
||||
"items": dict((key, new_variant_pkgs[key]) for key in moves_into_variant_from_parent)
|
||||
}
|
||||
result["moved_items"].append(moved_items_dict)
|
||||
|
||||
# Moved from child to parent
|
||||
moved_items_dict = {
|
||||
"from": variant,
|
||||
"to": parent_variant,
|
||||
"arch": arch,
|
||||
"parent": parent_variant,
|
||||
"items": dict((key, new_parent_variant_pkgs[key]) for key in moves_into_parent_from_variant)
|
||||
}
|
||||
result["moved_items"].append(moved_items_dict)
|
||||
|
||||
return result
|
||||
|
||||
def get_report(self, old_compose, new_compose):
|
||||
"""Generate report
|
||||
|
||||
Returns a dict with all related data.
|
||||
"""
|
||||
# Prepare header
|
||||
header = {
|
||||
"old_compose": self._load_compose_id(old_compose),
|
||||
"new_compose": self._load_compose_id(new_compose),
|
||||
}
|
||||
|
||||
# Build dicts with family relations between variants
|
||||
new_variants = new_compose.info.variants.get_variants(recursive=True)
|
||||
new_variant_parent = self._variant_parent_map(new_variants)
|
||||
|
||||
# Prepare data structures for comparison
|
||||
old_struct = self._build_struct(old_compose)
|
||||
new_struct = self._build_struct(new_compose)
|
||||
|
||||
# Compare the structures
|
||||
result = self._cmp_structures(old_struct, new_struct, new_variant_parent)
|
||||
result["header"] = header
|
||||
|
||||
return result
|
||||
|
||||
def _gen_log_new_items(self, data):
|
||||
"""Generate report about new packages in each variant.
|
||||
|
||||
Returns: String
|
||||
"""
|
||||
log = []
|
||||
|
||||
tmp = {} # {'variant': {'arch': {items}, ..}, ..}
|
||||
for _, val in data.iteritems():
|
||||
variant = val["variant"]
|
||||
arch = val["arch"]
|
||||
items = val["items"]
|
||||
assert arch not in tmp.get(variant, {})
|
||||
tmp.setdefault(variant, {})[arch] = items
|
||||
|
||||
for variant in sorted(tmp.keys()):
|
||||
for arch in sorted(tmp[variant].keys()):
|
||||
for namearch in sorted(tmp[variant][arch].keys()):
|
||||
details = tmp[variant][arch][namearch]
|
||||
for detail in details:
|
||||
log.append('%s.%s: %s - %s' %
|
||||
(variant, arch, namearch, detail.get("path")))
|
||||
|
||||
return "\n".join(log)
|
||||
|
||||
def _gen_log_moved_items(self, data):
|
||||
"""Generate report about packages that moved between variants
|
||||
|
||||
Returns: String
|
||||
"""
|
||||
log = []
|
||||
|
||||
tmp = {} # {"variant->variant": {"arch": {item}, ..}, ..}
|
||||
for item in data:
|
||||
key = "%s->%s" % (item["from"], item["to"])
|
||||
assert item["arch"] not in tmp.get(key, {})
|
||||
tmp.setdefault(key, {})[item["arch"]] = item
|
||||
|
||||
for direction in sorted(tmp.keys()):
|
||||
for arch in sorted(tmp[direction].keys()):
|
||||
obj = tmp[direction][arch]
|
||||
for namearch in sorted(obj["items"].keys()):
|
||||
details = tmp[direction][arch]["items"][namearch]
|
||||
for detail in details:
|
||||
log.append("%s->%s.%s: %s (%s)" %
|
||||
(obj["from"],
|
||||
obj["to"],
|
||||
obj["arch"],
|
||||
detail.get("path"),
|
||||
detail.get("srpm_nevra")))
|
||||
|
||||
return "\n".join(log)
|
||||
|
||||
def _write_json(self, data, filename):
|
||||
"""Write json file"""
|
||||
json.dump(data,
|
||||
open(filename, "w"),
|
||||
sort_keys=True,
|
||||
indent=2,
|
||||
separators=(',', ': '),
|
||||
cls=ExtraJsonEncoder)
|
||||
|
||||
|
||||
def write(self, data, path=None, name=None):
|
||||
"""Write reports"""
|
||||
if name:
|
||||
name = "package_moves-%s" % name
|
||||
else:
|
||||
name = "package_moves"
|
||||
name = name.replace(" ", "_")
|
||||
name = name.replace("/", "_")
|
||||
name = name.replace("\\", "_")
|
||||
|
||||
path = path or ""
|
||||
|
||||
# JSONs
|
||||
json_complete = os.path.join(path, "%s.json" % name)
|
||||
json_new_items = os.path.join(path, "%s-new_items.json" % name)
|
||||
json_moved_items = os.path.join(path, "%s-moved_items.json" % name)
|
||||
|
||||
# Complete
|
||||
self._write_json(data, json_complete)
|
||||
|
||||
# New items
|
||||
tmp_data = {
|
||||
"header": data.get("header", {}),
|
||||
"new_items": data.get("new_items", {}),
|
||||
}
|
||||
self._write_json(tmp_data, json_new_items)
|
||||
|
||||
# Moved items
|
||||
tmp_data = {
|
||||
"header": data.get("header", {}),
|
||||
"move_items": data.get("moved_items", {}),
|
||||
}
|
||||
self._write_json(tmp_data, json_moved_items)
|
||||
|
||||
# Logs
|
||||
log_new_items = os.path.join(path, "%s-new_items.log" % name)
|
||||
log_moved_items = os.path.join(path, "%s-moved_items.log" % name)
|
||||
|
||||
log = self._gen_log_new_items(data.get("new_items", {}))
|
||||
open(log_new_items, "w").write(log)
|
||||
|
||||
log = self._gen_log_moved_items(data.get("moved_items", {}))
|
||||
open(log_moved_items, "w").write(log)
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue