ccp.py: add an 'all' mode that works like critpath.py

This is probably what we want for Bodhi purposes. It only operates
on Branched and Rawhide...we could work on the container and cloud
composes for stable releases and get a 'compose critical' set
that way, but we can't get the source packages, and it's of
questionable value anyway.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

quality-assurance/critpath/ccp.py

@@ -18,6 +18,29 @@ from collections import defaultdict
 KOJI = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
 RPMD = {}
 SRCCACHE = {}
+BODHI_RELEASEURL = "https://bodhi.fedoraproject.org/releases/?rows_per_page=500"
+# used as a cache by get_bodhi_releases
+BODHIRELEASES = {}
+
+
+# mostly duplicated with critpath.py but I don't feel like setting up a
+# shared lib just for this
+def get_bodhi_releases():
+    global BODHIRELEASES
+    if not BODHIRELEASES:
+        resp = requests.get(BODHI_RELEASEURL)
+        if resp.status_code != 200:
+            sys.exit(f"Could not get Bodhi release data from {BODHI_RELEASEURL}")
+        bodhijson = resp.json()["releases"]
+        devrels = {
+            int(rel['version']) for rel in bodhijson if rel['state'] in ('pending', 'frozen') and
+            rel['id_prefix'] == 'FEDORA' and rel["version"].isdigit()
+        }
+        if devrels:
+            BODHIRELEASES[str(max(devrels))] = "rawhide"
+        if len(devrels) > 1:
+            BODHIRELEASES[str(min(devrels))] = "branched"
+    return BODHIRELEASES
 
 
 def get_source(pkg, arch):
@@ -146,6 +169,9 @@ def parse_args():
             "Operates on latest Rawhide compose if no arguments are passed. "
             "Outputs to [COMPOSE_ID]-ccp.txt and [COMPOSE_ID]-ccp.json, and "
             "optionally [COMPOSE_ID]-ccp-source.txt and [COMPOSE_ID]-ccp-source.json."
+            "If called with magic value 'all', runs on the current compose for all "
+            "release branches discovered from Bodhi, and outputs to files with base "
+            "names 'fNN' and 'rawhide', similar to critpath.py."
         )
     )
     parser.add_argument(
@@ -157,25 +183,24 @@ def parse_args():
     )
     parser.add_argument(
         "curl",
-        help="URL to compose top level",
+        metavar="COMPOSE_URL_OR_'all'",
+        help="URL to compose top level, or magic string 'all'",
         nargs="?",
-        default="https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide",
+        default="all",
     )
     return parser.parse_args()
 
 
-def run(args):
-    cidurl = f"{args.curl}/COMPOSE_ID"
-    cid = requests.get(cidurl).text
-    durl = f"{args.curl}/logs/global/deliverables-v2.json"
+def get_crits(curl, source):
+    durl = f"{curl}/logs/global/deliverables-v2.json"
     resp = requests.get(durl)
     if resp.status_code != 200:
         sys.exit(f"Could not find deliverables at {durl}")
     deliverables = resp.json()
-    if args.source:
+    if source:
         # populate global copy of RPM metadata for later use
         global RPMD
-        RPMD = requests.get(f"{args.curl}/compose/metadata/rpms.json").json()["payload"]["rpms"]["Everything"]
+        RPMD = requests.get(f"{curl}/compose/metadata/rpms.json").json()["payload"]["rpms"]["Everything"]
 
     critpkgs = dict()
     critsrc = dict()
@@ -193,18 +218,21 @@ def run(args):
                  kids = KOJI.getTaskChildren(image["koji_task"])
                  createtasks = [(kid["id"], kid["arch"]) for kid in kids if kid["method"] == kidmethod]
                  for createtask, arch in createtasks:
-                    process_task(createtask, arch, variant, args.source, critpkgs, critsrc, getfunc)
+                    process_task(createtask, arch, variant, source, critpkgs, critsrc, getfunc)
             else:
-                process_task(image["koji_task"], image["arch"], variant, args.source, critpkgs, critsrc, getfunc)
+                process_task(image["koji_task"], image["arch"], variant, source, critpkgs, critsrc, getfunc)
+    return (critpkgs, critsrc)
 
+
+def output(critpkgs, critsrc, prefixes, source):
     allcrit = set()
-    if args.source:
+    if source:
         allscrit = set()
     for variant in critpkgs:
         print(f"Variant: {variant}")
         for arch in critpkgs[variant]:
             allcrit.update(critpkgs[variant][arch]["buildroot"].union(critpkgs[variant][arch]["image"]))
-            if args.source:
+            if source:
                 allscrit.update(critsrc[variant][arch]["buildroot"].union(critsrc[variant][arch]["image"]))
             # convert to lists for JSON output
             critpkgs[variant][arch]["buildroot"] = sorted(critpkgs[variant][arch]["buildroot"])
@@ -214,33 +242,48 @@ def run(args):
             brcount = len(critpkgs[variant][arch]["buildroot"])
             icount = len(critpkgs[variant][arch]["image"])
             print(f"Arch: {arch} - buildroot {brcount}, image {icount}")
-            if args.source:
+            if source:
                 brscount = len(critsrc[variant][arch]["buildroot"])
                 iscount = len(critsrc[variant][arch]["image"])
                 print(f"Arch: {arch} - buildroot source {brscount}, image source {iscount}")
     print(f"Total: {len(allcrit)}")
-    if args.source:
+    if source:
         print(f"Source total: {len(allscrit)}")
-    with open(f"{cid}-ccp.txt", "w", encoding="utf-8") as outfh:
-        outfh.write("\n".join(sorted(allcrit)))
-        print(f"Wrote {cid}-ccp.txt")
-    with open(f"{cid}-ccp.json", "w", encoding="utf-8") as jsonoutfh:
-        json.dump(critpkgs, jsonoutfh)
-        print(f"Wrote {cid}-ccp.json")
-    if args.source:
-        with open(f"{cid}-ccp-source.txt", "w", encoding="utf-8") as soutfh:
-            soutfh.write("\n".join(sorted(allscrit)))
-            print(f"Wrote {cid}-ccp-source.txt")
-        with open(f"{cid}-ccp-source.json", "w", encoding="utf-8") as sjsonoutfh:
-            json.dump(critsrc, sjsonoutfh)
-            print(f"Wrote {cid}-ccp-source.json")
+    for prefix in prefixes:
+        with open(f"{prefix}-ccp.txt", "w", encoding="utf-8") as outfh:
+            outfh.write("\n".join(sorted(allcrit)))
+            print(f"Wrote {prefix}-ccp.txt")
+        with open(f"{prefix}-ccp.json", "w", encoding="utf-8") as jsonoutfh:
+            json.dump(critpkgs, jsonoutfh)
+            print(f"Wrote {prefix}-ccp.json")
+        if source:
+            with open(f"{prefix}-ccp-source.txt", "w", encoding="utf-8") as soutfh:
+                soutfh.write("\n".join(sorted(allscrit)))
+                print(f"Wrote {prefix}-ccp-source.txt")
+            with open(f"{prefix}-ccp-source.json", "w", encoding="utf-8") as sjsonoutfh:
+                json.dump(critsrc, sjsonoutfh)
+                print(f"Wrote {prefix}-ccp-source.json")
 
 
 def main():
     """Main loop."""
     try:
         args = parse_args()
-        run(args)
+        if args.curl == "all":
+            for (rel, typ) in get_bodhi_releases().items():
+                prefixes = [f"f{rel}"]
+                if typ == "rawhide":
+                    curl = "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide"
+                    prefixes.append(typ)
+                elif typ == "branched":
+                    curl = f"https://kojipkgs.fedoraproject.org/compose/branched/latest-Fedora-{rel}"
+                    prefixes.append(typ)
+                (critpkgs, critsrc) = get_crits(curl, args.source)
+                output(critpkgs, critsrc, prefixes, args.source)
+        else:
+            cid = requests.get(f"{args.curl}/COMPOSE_ID").text
+            (critpkgs, critsrc) = get_crits(args.curl, args.source)
+            output(critpkgs, critsrc, [cid], args.source)
     except KeyboardInterrupt:
         sys.stderr.write("Interrupted, exiting...\n")
         sys.exit(1)