scripts_new/packages/distgit/give_package.py

@@ -0,0 +1,49 @@
+#! /usr/bin/python3 -tt
+""" Give a package in pagure-on-dist-git from one user to another.
+
+This can also be used to give the package to the 'orphan' user.
+
+You need a privileged pagure token in /etc/fedrepo_req/config.ini
+
+    [admin]
+    pagure_api_token = something secret
+
+You can generate such a token on pkgs02 with:
+
+    $ PAGURE_CONFIG=/etc/pagure/pagure.cfg pagure-admin admin-token --help
+"""
+# Copyright (c) 2017 Red Hat
+# SPDX-License-Identifier:	GPL-2.0
+#
+# Authors:
+#     Ralph Bean <rbean@redhat.com>
+
+import argparse
+import sys
+
+try:
+    import utilities
+except ImportError:
+    print("Try setting PYTHONPATH to find the utilities.py file.")
+    raise
+
+PAGURE_URL = 'https://src.fedoraproject.org/api/0/'
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("package", help="The package that should be given.")
+    parser.add_argument("custodian", help="The user taking over the package.")
+    args = parser.parse_args()
+    session = utilities.retry_session()
+    try:
+        namespace, package = args.package.split('/')
+    except:
+        print("Package must be like <namespace>/<name>, not %r" % args.package)
+        sys.exit(1)
+
+    utilities.give_package(session, namespace, package, args.custodian)
+
+
+if __name__ == "__main__":
+    main()

scripts_new/packages/distgit/orphan_all_packages.py

@@ -0,0 +1,95 @@
+#! /usr/bin/python3 -tt
+""" Orphan all packages of a given set of users.
+
+If there are other committers on a package, the first one is promoted to be the
+new owner.
+
+If there are no other committers, then the package is given to the `orphan`
+user.
+
+You need a privileged pagure token in /etc/fedrepo_req/config.ini
+
+    [admin]
+    pagure_api_token = something secret
+
+You can generate such a token on pkgs02 with:
+
+    $ PAGURE_CONFIG=/etc/pagure/pagure.cfg pagure-admin admin-token --help
+"""
+# Copyright (c) 2017 Red Hat
+# SPDX-License-Identifier:	GPL-2.0
+#
+# Authors:
+#     Ralph Bean <rbean@redhat.com>
+
+import argparse
+
+try:
+    import utilities
+except ImportError:
+    print("Try setting PYTHONPATH to find the utilities.py file.")
+    raise
+
+
+PAGURE_URL = 'https://src.fedoraproject.org/api/0/'
+
+
+def get_all_packages_for_user(session, user):
+    url = PAGURE_URL + 'projects'
+    params = dict(owner=user, fork=False)
+    response = session.get(url, params=params, timeout=400)
+    if not bool(response):
+        raise IOError("Failed GET %r %r" % (response.request.url, response))
+    for project in response.json()['projects']:
+        yield project
+
+
+def triage_packages(packages, user):
+    for package in packages:
+        for kind in ('admin', 'commit'):
+            others = package['access_users'][kind]
+            try:
+                others.remove(user)
+            except ValueError:
+                # Owner doesn't have commit.  Weird, but ok.
+                pass
+            if others:
+                # Select the first one to become the new owner.
+                yield package, others[0]
+                break
+        else:
+            yield package, 'orphan'
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("users", nargs="*",
+                        help="Users to remove.")
+    args = parser.parse_args()
+
+    session = utilities.retry_session()
+    for user in args.users:
+        print("Investigating packages for user %r" % user)
+        packages = get_all_packages_for_user(session, user)
+        transfers = triage_packages(packages, user)
+
+        # Exhaust the generator
+        transfers = list(transfers)
+
+        for package, custodian in transfers:
+            print("%s/%s will be given to %s" % (
+                package['namespace'], package['name'], custodian))
+
+        response = input("Is this okay? [y/N]")
+        if response.lower() not in ('y', 'yes'):
+            print("!! OK.  Bailing out for %r" % user)
+            continue
+
+        print("Starting transfers")
+        for package, custodian in transfers:
+            namespace, name = package['namespace'], package['name']
+            utilities.give_package(session, namespace, name, custodian)
+
+
+if __name__ == "__main__":
+    main()

scripts_new/packages/distgit/orphan_listed_packages.py

@@ -0,0 +1,111 @@
+"""
+This script is useful to bulk orphan listed packages.
+E.g. when they fail to install or fail to build.
+"""
+
+import logging
+import os
+import sys
+
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+REASON = "Important bug not fixed"
+# REASON = "Fails to build from source"
+# REASON = "Orphaned by releng"
+
+PACKAGES = {
+    # "pkg_name": "https://bugzilla.redhat.com/XXX or a different info",
+}
+
+PAGURE_TOKEN = os.getenv("PAGURE_TOKEN")
+LOG = logging.getLogger(__name__)
+BASE_URL = "https://src.fedoraproject.org"
+
+
+def retry_session():
+    session = requests.Session()
+    retry = Retry(
+        total=5,
+        read=5,
+        connect=5,
+        backoff_factor=0.3,
+        status_forcelist=(500, 502, 504),
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
+
+
+def orphan_package(name, namespace="rpms", reason=REASON, reason_info=None):
+    """Give the specified project on dist_git to the ``orphan`` user."""
+    LOG.debug("Going to orphan: %s/%s", namespace, name)
+    session = retry_session()
+
+    # Orphan the package
+    url = f"{BASE_URL}/_dg/orphan/{namespace}/{name}"
+    headers = {"Authorization": f"token {PAGURE_TOKEN}"}
+    data = {
+        "orphan_reason": reason,
+    }
+    if reason:
+        data["orphan_reason_info"] = reason_info
+
+    req = session.post(url, data=data, headers=headers)
+    if not req.ok:
+        print("**** REQUEST FAILED")
+        print("  - Orphan package")
+        print(req.url)
+        print(data)
+        print(headers)
+        print(req.text)
+    else:
+        print(f"{namespace}/{name} is orphaned")
+
+    session.close()
+
+
+def get_bugzilla_overrides(name, namespace="rpms"):
+    """Returns bugzilla overrides of the specified package.. """
+    LOG.debug("Checking for bugzilla overrides on %s/%s", namespace, name)
+    session = retry_session()
+    req = session.get(f"{BASE_URL}/_dg/bzoverrides/{namespace}/{name}")
+    return req.json()
+
+
+def reset_bugzilla_overrides(name, namespace="rpms"):
+    """ Reset the Fedora bugzilla overrides of the specified package."""
+    overrides = get_bugzilla_overrides(name)
+
+    if overrides["fedora_assignee"] is None:
+        LOG.debug("No bugzilla overrides on %s/%s", namespace, name)
+        return
+
+    LOG.debug("Resetting bugzilla overrides on %s/%s", namespace, name)
+    url = f"{BASE_URL}/_dg/bzoverrides/{namespace}/{name}"
+    headers = {"Authorization": f"token {PAGURE_TOKEN}"}
+
+    username = overrides["fedora_assignee"]
+    overrides["fedora_assignee"] = None
+
+    session = retry_session()
+    req = session.post(url, headers=headers, data=overrides)
+    if not req.ok:
+        print("**** REQUEST FAILED")
+        print("  - Remove bugzilla overrides")
+        print(req.url)
+        print(req.text)
+    else:
+        print(f"  {username} has no longer a bugzilla overrides on {namespace}/{name}")
+    session.close()
+
+
+if __name__ == "__main__":
+    if not PACKAGES:
+        sys.exit("Define PACKAGES first")
+
+    for package in PACKAGES:
+        orphan_package(package, reason_info=PACKAGES[package])
+        reset_bugzilla_overrides(package)

scripts_new/packages/distgit/retire_packagers.py

@@ -0,0 +1,390 @@
+#!/usr/bin/python3
+
+"""
+This script queries dist-git for all the packages a given packager maintains,
+has commit or watches.
+Package that the packager is the main admin are then orphaned. The packager is
+then removed from all packages that they have commit for and their watch status
+is reset on every packages that they are watching.
+
+"""
+
+import argparse
+import collections
+import logging
+import os
+import sys
+
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+_log = logging.getLogger(__name__)
+dist_git_base = "https://src.fedoraproject.org"
+pagure_token = None
+
+
+def retry_session():
+    session = requests.Session()
+    retry = Retry(
+        total=5,
+        read=5,
+        connect=5,
+        backoff_factor=0.3,
+        status_forcelist=(500, 502, 504),
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
+
+
+def setup_logging(log_level: int):
+    handlers = []
+
+    _log.setLevel(log_level)
+    # We want all messages logged at level INFO or lower to be printed to stdout
+    info_handler = logging.StreamHandler(stream=sys.stdout)
+    handlers.append(info_handler)
+
+    if log_level == logging.INFO:
+        # In normal operation, don't decorate messages
+        for handler in handlers:
+            handler.setFormatter(logging.Formatter("%(message)s"))
+
+    logging.basicConfig(level=log_level, handlers=handlers)
+
+
+def get_arguments(args):
+    """ Load and parse the CLI arguments."""
+    parser = argparse.ArgumentParser(
+        description="Looks for the specified list of users what they "
+        "maintain or watch in dist-git.\nIf --retire is specified, all the ACL "
+        "the packager(s) have in dist-git will be removed. If they are main admins "
+        "of some packages, these packages will be orphaned. If they have commit "
+        "access on some packages, they will no longer have these access. If they "
+        "watch a package, their watch status will be reset. Note: the source of "
+        "information is refreshed hourly, so if you run the script twice with "
+        "`--retire` you may not see a difference here."
+    )
+    parser.add_argument(
+        dest="usernames", nargs="*", help="Names of the users to retire.",
+    )
+    parser.add_argument(
+        "--from-file",
+        dest="users_file",
+        help="Path to a file containing the users to check (one per line).",
+    )
+    parser.add_argument(
+        "--retire",
+        action="store_true",
+        default=False,
+        help="Retire the user(s) (ie: orphan, remove from ACL, reset watch)",
+    )
+    parser.add_argument(
+        "--api-token",
+        dest="pagure_token",
+        default=os.environ.get("PAGURE_TOKEN"),
+        help="Pagure token to use to interact with dist-git. It can also be set "
+        "via the PAGURE_TOKEN environment variable. (This script requires the "
+        "`modifyproject` ACL to work)",
+    )
+    report_group = parser.add_mutually_exclusive_group()
+    report_group.add_argument(
+        "--watch",
+        action="store_const",
+        dest="report",
+        const="watch",
+        default="all",
+        help="Only report/act on watched projects",
+    )
+    report_group.add_argument(
+        "--maintain",
+        action="store_const",
+        dest="report",
+        const="maintain",
+        help="Only report/act projects the packagers have commit access to",
+    )
+
+    log_level_group = parser.add_mutually_exclusive_group()
+    log_level_group.add_argument(
+        "--debug",
+        action="store_const",
+        dest="log_level",
+        const=logging.DEBUG,
+        default=logging.INFO,
+        help="Enable debugging output",
+    )
+
+    return parser.parse_args(args)
+
+
+def user_access(session, username, namespace_name):
+    """ Returns whether the specified username is listed in the maintainers
+    list of the specified package and a set of all maintainers therein. """
+    req = session.get(f"{dist_git_base}/api/0/{namespace_name}")
+    project = req.json()
+    maintainers = set()
+    for acl in project["access_users"]:
+        maintainers.update(set(project["access_users"][acl]))
+    if username == project["user"]["name"]:
+        level = "main admin"
+    elif username in maintainers:
+        level = "maintainer"
+    else:
+        level = None
+
+    return level, maintainers
+
+
+def get_bugzilla_overrides(username, namespace, name):
+    """ Returns whether the specified username is set in the bugzilla overrides
+    of the specified package.. """
+    _log.debug(
+        "Checking for bugzilla overrides on %s/%s for %s", namespace, name, username
+    )
+    base_url = dist_git_base.rstrip("/")
+    session = retry_session()
+    req = session.get(f"{dist_git_base}/_dg/bzoverrides/{namespace}/{name}")
+    return req.json()
+
+
+def unwatch_package(namespace, name, username):
+    """ Reset the watch status of the given user on the specified project. """
+    _log.debug("Going to reset watch status of %s on %s/%s", username, namespace, name)
+    base_url = dist_git_base.rstrip("/")
+    session = retry_session()
+
+    # Reset the watching status
+    url = f"{base_url}/api/0/{namespace}/{name}/watchers/update"
+    headers = {"Authorization": f"token {pagure_token}"}
+    data = {"status": -1, "watcher": username}
+
+    req = session.post(url, data=data, headers=headers)
+    if not req.ok:
+        print("**** REQUEST FAILED")
+        print("  - Unwatch package")
+        print(req.url)
+        print(data)
+        print(headers)
+        print(req.text)
+    else:
+        print(f"  {username} is no longer watching {namespace}/{name}")
+    session.close()
+
+
+def orphan_package(session, namespace, name, username):
+    """ Give the specified project on dist_git to the ``orphan`` user.
+    """
+    _log.debug("Going to orphan: %s/%s from %s", namespace, name, username)
+    base_url = dist_git_base.rstrip("/")
+    session = retry_session()
+
+    # Orphan the package
+    url = f"{base_url}/_dg/orphan/{namespace}/{name}"
+    headers = {"Authorization": f"token {pagure_token}"}
+    data = {
+        "orphan_reason": "Orphaned by releng",
+    }
+
+    req = session.post(url, data=data, headers=headers)
+    if not req.ok:
+        print("**** REQUEST FAILED")
+        print("  - Orphan package")
+        print(req.url)
+        print(data)
+        print(headers)
+        print(req.text)
+    else:
+        print(f"  {username} is no longer the main admin of {namespace}/{name}")
+
+    session.close()
+
+
+def remove_access(namespace, name, username, usertype):
+    """ Remove the ACL of the specified user/group on the specified project. """
+    _log.debug("Going to remove %s from %s/%s", username, namespace, name)
+    base_url = dist_git_base.rstrip("/")
+    session = retry_session()
+
+    # Remove ACL on the package
+    url = f"{base_url}/api/0/{namespace}/{name}/git/modifyacls"
+    headers = {"Authorization": f"token {pagure_token}"}
+    data = {
+        "user_type": usertype,
+        "name": username,
+    }
+
+    req = session.post(url, data=data, headers=headers)
+    if not req.ok:
+        print("**** REQUEST FAILED")
+        print("  - Remove ACL")
+        print(req.url)
+        print(data)
+        print(req.text)
+    else:
+        print(f"  {username} is no longer maintaining {namespace}/{name}")
+    session.close()
+
+    if usertype == "user":
+        # Reset the watching status
+        unwatch_package(namespace, name, username)
+
+
+def reset_bugzilla_overrides(username, namespace, name, overrides):
+    """ Reset the the bugzilla overrides of the specified package so that the
+    specified user no longer has one. """
+    _log.debug(
+        "Resetting bugzilla overrides on %s/%s for %s", namespace, name, username
+    )
+    base_url = dist_git_base.rstrip("/")
+    url = f"{base_url}/_dg/bzoverrides/{namespace}/{name}"
+    headers = {"Authorization": f"token {pagure_token}"}
+
+    for key in overrides:
+        if overrides[key] == username:
+            overrides[key] = None
+
+    session = retry_session()
+    req = session.post(url, headers=headers, data=overrides)
+    if not req.ok:
+        print("**** REQUEST FAILED")
+        print("  - Remove bugzilla overrides")
+        print(req.url)
+        print(data)
+        print(req.text)
+    else:
+        print(f"  {username} has no longer a bugzilla overrides on {namespace}/{name}")
+    session.close()
+
+
+def main(args):
+    """ For the specified list of users, retrieve what they are maintaining
+    or watching in dist-git."""
+
+    args = get_arguments(args)
+    setup_logging(log_level=args.log_level)
+    _log.debug("Log level set to: %s", args.log_level)
+
+    if args.pagure_token:
+        global pagure_token
+        pagure_token = args.pagure_token
+
+    if not pagure_token and args.retire:
+        _log.debug(
+            "Trying to retrieve pagure_api_token from the fedscm configuration file"
+        )
+        try:
+            import fedscm_admin.config
+            from fedscm_admin import CONFIG
+
+            api_token = fedscm_admin.config.get_config_item(CONFIG, "pagure_api_token")
+        except:
+            pass
+
+    if not pagure_token and args.retire:
+        print(
+            "No pagure token set in the CLI argument or via the PAGURE_TOKEN "
+            "environment variable or found in the fedscm configuration file. "
+            "Going to ignore --retire"
+        )
+        args.retire = False
+
+    usernames = []
+    if args.users_file:
+        _log.debug("Loading usernames for file: %s", args.users_file)
+        if not os.path.exists(args.users_file):
+            _log.info("No such file found: %s", args.users_file)
+        try:
+            with open(args.users_file) as stream:
+                usernames = [
+                    l.strip() for l in stream.readlines() if l.strip()
+                ]
+        except Exception as err:
+            _log.debug(
+                "Failed to load/read the file: %s, error is: %s", args.users_file, err
+            )
+    else:
+        _log.debug("Loading usernames for the CLI arguments")
+        usernames = args.usernames
+
+    # We load the info from the pagure_bz file which will tell us everything
+    # that would be synced to bugzilla (POC and CC)
+    _log.debug("Loading info from dist-git's pagure_bz.json file")
+    session = retry_session()
+    req = session.get(f"{dist_git_base}/extras/pagure_bz.json")
+    pagure_bz = req.json()
+    session.close()
+
+    packages_per_user = collections.defaultdict(set)
+    for namespace in pagure_bz:
+        for package in pagure_bz[namespace]:
+            _log.debug("Processing %s/%s", namespace, package)
+            for user in pagure_bz[namespace][package]:
+                if user in usernames:
+                    packages_per_user[user].add(f"{namespace}/{package}")
+
+    # On the top of this, we'll also query the list from dist-git directly as
+    # the previous source of info while quicker to query will not include
+    # the packages that the packagers have access to but set their watch status
+    # to "unwatch".
+    # However, we only need to run this if we want to know about packages someone
+    # maintains (ie: we can bypass this section if ``--watch`` is passed to the
+    # CLI).
+    if args.report in ["all", "maintain"]:
+        for username in sorted(usernames):
+            _log.debug("Loading info from dist-git's %s's page", username)
+            url = f"{dist_git_base}/api/0/user/{username}?per_page=50"
+            while url:
+                req = session.get(url)
+                data = req.json()
+                for repo in data.get("repos", []):
+                    maintainers = set(repo["user"]["name"])
+                    for acl in repo["access_users"]:
+                        maintainers.update(set(repo["access_users"][acl]))
+                    if username in maintainers:
+                        namespace = repo["namespace"]
+                        package = repo["name"]
+                        packages_per_user[username].add(f"{namespace}/{package}")
+                url = data.get("repos_pagination", {}).get("next")
+                if not url:
+                    break
+
+    for username in sorted(usernames):
+        _log.debug("Processing user: %s", username)
+        for pkg in sorted(packages_per_user[username]):
+            level, maintainers = user_access(session, username, pkg)
+            namespace, name = pkg.split("/", 1)
+            if level:
+                if args.report in ["all", "maintain"]:
+                    print(f"{username} is {level} of {namespace}/{name}")
+                    if level == "main admin" and len(maintainers) > 1:
+                        maintainers_strs = (f"@{m}" for m in sorted(maintainers - {username}))
+                        maintainers_str = ", ".join(maintainers_strs)
+                        print(f"  {namespace}/{name} co-maintainers: {maintainers_str}")
+                    if args.retire:
+                        if level == "main admin":
+                            orphan_package(session, namespace, name, username)
+                        elif level == "maintainer":
+                            remove_access(namespace, name, username, "user")
+
+            else:
+                if args.report in ["all", "watch"]:
+                    print(f"{username} is watching {namespace}/{name}")
+                    if args.retire:
+                        unwatch_package(namespace, name, username)
+
+            overrides = get_bugzilla_overrides(username, namespace, name)
+            if username in overrides.values():
+                print(f"{username} has a bugzilla override on {namespace}/{name}")
+                if args.retire:
+                    reset_bugzilla_overrides(username, namespace, name, overrides)
+
+        print()
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main(sys.argv[1:]))
+    except KeyboardInterrupt:
+        pass

scripts_new/packages/distgit/utilities.py

@@ -0,0 +1,62 @@
+#! /usr/bin/python -tt
+""" Utilities for manipulating dist-git (pagure). """
+# Copyright (c) 2017 Red Hat
+# SPDX-License-Identifier:	GPL-2.0
+#
+# Authors:
+#     Ralph Bean <rbean@redhat.com>
+
+import json
+import pprint
+import sys
+import traceback
+
+import requests
+
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+try:
+    from fedscm_admin.pagure import get_pagure_auth_header
+    admin_headers = get_pagure_auth_header('admin')
+except:
+    traceback.print_exc()
+    print("Failed to load admin tokens from fedrepo-req-admin")
+    sys.exit(1)
+
+PAGURE_URL = 'https://src.fedoraproject.org/api/0/'
+
+
+def retry_session():
+    session = requests.Session()
+    retry = Retry(
+        total=5,
+        read=5,
+        connect=5,
+        backoff_factor=0.3,
+        status_forcelist=(500, 502, 504),
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session
+
+
+def give_package(session, namespace, package, custodian):
+    print("Giving %s/%s to %s" % (
+        namespace, package, custodian))
+
+    url = PAGURE_URL + namespace + '/' + package
+    payload = json.dumps({'main_admin': custodian})
+    response = session.patch(
+        url,
+        data=payload,
+        headers=admin_headers,
+        timeout=60,
+    )
+    if not bool(response):
+        try:
+            pprint.pprint(response.json())
+        except:
+            pass
+        raise IOError("Failed PATCH %r %r" % (response.request.url, response))

scripts_new/packages/orphaned/orphaned-packages-process/Dockerfile

@@ -0,0 +1,19 @@
+FROM registry.fedoraproject.org/fedora:latest
+
+RUN set -xeuo pipefail ;\
+    dnf install --setopt=install_weak_deps=False -y \
+        python3 \
+        python3-dnf \
+        python3-dogpile-cache \
+        python3-koji \
+        python3-requests \
+        python3-texttable \
+        wget \
+    ;\
+    dnf clean all
+
+COPY find_unblocked_orphans.py /usr/local/bin/find_unblocked_orphans.py
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+STOPSIGNAL SIGINT

scripts_new/packages/orphaned/orphaned-packages-process/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# If UPDATE is passed, this script will download the latest version of
+# find_unblocked_orphans from Pagure
+UPDATE="${UPDATE-}"
+
+script="/usr/local/bin/find_unblocked_orphans.py"
+raw_url="https://www.pagure.io/releng/raw/main/f/scripts/orphaned-packages-process/find_unblocked_orphans.py"
+
+if [ -n "${UPDATE}" ]; then
+    dl_dir="$(mktemp -d)"
+    script="${dl_dir}/find_unblocked_orphans.py"
+    wget "${raw_url}" -O "${script}"
+fi
+
+exec python3 "${script}" "$@"

scripts_new/packages/orphaned/orphaned-packages-process/find_unblocked_orphans.py

@@ -0,0 +1,996 @@
+#! /usr/bin/python3
+#
+# find_unblocked_orphans.py - A utility to find orphaned packages in pagure
+#                             that are unblocked in koji and to show what
+#                             may require those orphans
+#
+# Copyright (c) 2009-2013 Red Hat
+# SPDX-License-Identifier:	GPL-2.0
+#
+# Authors:
+#     Jesse Keating <jkeating@redhat.com>
+#     Till Maas <opensource@till.name>
+
+import argparse
+import datetime
+import email.mime.text
+import hashlib
+import json
+import os
+import smtplib
+import sys
+import textwrap
+import time
+import traceback
+from collections import OrderedDict, defaultdict
+from functools import lru_cache
+from pathlib import Path
+from queue import Queue
+from threading import Thread
+from typing import IO, Any, NotRequired, TypedDict, cast
+
+import dnf
+import dogpile.cache
+import koji
+import requests
+
+try:
+    import texttable
+
+    with_table = True
+except ImportError:
+    with_table = False
+
+
+@lru_cache(maxsize=20480)
+def SRPM(query, package):
+    # This function was stolen from pungi
+    """Given a package object, get a package object for the
+    corresponding source rpm. Requires dnf still configured
+    and a valid package object."""
+
+    srpm, *_ = package.sourcerpm.split(".src.rpm")
+    name, version, release = srpm.rsplit("-", 2)
+    try:
+        srpmpo = query.filter(
+            name=name, version=version, release=release, arch="src"
+        ).run()[0]
+        return srpmpo
+    except IndexError:
+        eprint(f"Error: Cannot find a source rpm for {name}-{version}-{release}")
+        sys.exit(1)
+
+
+cache_dir = os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
+os.makedirs(cache_dir, exist_ok=True)
+cache = dogpile.cache.make_region().configure(
+    "dogpile.cache.dbm",
+    expiration_time=86400,
+    arguments=dict(filename=os.path.join(cache_dir, "dist-git-orphans-cache.dbm")),
+)
+PAGURE_URL = "https://src.fedoraproject.org"
+PAGURE_MAX_ENTRIES_PER_PAGE = 100
+
+
+EPEL7_RELEASE = dict(
+    repo="https://kojipkgs.fedoraproject.org/compose/updates/epel7/"
+    "compose/Everything/x86_64/os/",
+    source_repo="https://kojipkgs.fedoraproject.org/compose/updates/epel7/"
+    "compose/Everything/source/tree/",
+    koji_tag="epel7",
+    koji_hub="https://koji.fedoraproject.org/kojihub",
+    pagure_branch="epel7",
+    mailto="epel-announce@lists.fedoraproject.org",
+    bcc=[],
+)
+
+EPEL8_RELEASE = dict(
+    repo="https://kojipkgs.fedoraproject.org/compose/updates/epel8/"
+    "compose/Everything/x86_64/os/",
+    source_repo="https://kojipkgs.fedoraproject.org/compose/updates/epel8/"
+    "compose/Everything/source/tree/",
+    koji_tag="epel8",
+    koji_hub="https://koji.fedoraproject.org/kojihub",
+    pagure_branch="epel8",
+    mailto="epel-announce@lists.fedoraproject.org",
+    bcc=[],
+)
+
+EPEL9_RELEASE = dict(
+    repo="https://kojipkgs.fedoraproject.org/compose/updates/epel9/"
+    "compose/Everything/x86_64/os/",
+    source_repo="https://kojipkgs.fedoraproject.org/compose/updates/epel9/"
+    "compose/Everything/source/tree/",
+    koji_tag="epel9",
+    koji_hub="https://koji.fedoraproject.org/kojihub",
+    pagure_branch="epel9",
+    mailto="epel-announce@lists.fedoraproject.org",
+    bcc=[],
+)
+
+RAWHIDE_RELEASE = dict(
+    repo="https://kojipkgs.fedoraproject.org/compose/rawhide/"
+    "latest-Fedora-Rawhide/compose/Everything/x86_64/os",
+    source_repo="https://kojipkgs.fedoraproject.org/compose/rawhide/"
+    "latest-Fedora-Rawhide/compose/Everything/source/tree/",
+    koji_tag="f44",
+    koji_hub="https://koji.fedoraproject.org/kojihub",
+    pagure_branch="rawhide",
+    mailto="devel@lists.fedoraproject.org",
+    bcc=[],
+)
+
+BRANCHED_RELEASE = dict(
+    repo="https://kojipkgs.fedoraproject.org/compose/branched/"
+    "latest-Fedora-43/compose/Everything/x86_64/os",
+    source_repo="https://kojipkgs.fedoraproject.org/compose/branched/"
+    "latest-Fedora-43/compose/Everything/source/tree/",
+    koji_tag="f43",
+    pagure_branch="f43",
+    koji_hub="https://koji.fedoraproject.org/kojihub",
+    mailto="devel@lists.fedoraproject.org",
+    bcc=[],
+)
+
+RELEASES = {
+    "rawhide": RAWHIDE_RELEASE,
+    "branched": BRANCHED_RELEASE,
+    "epel9": EPEL9_RELEASE,
+    "epel8": EPEL8_RELEASE,
+    "epel7": EPEL7_RELEASE,
+}
+
+# pagure uid for orphan
+ORPHAN_UID = "orphan"
+
+HEADER = """\
+SPECIAL NOTE: As of https://pagure.io/fesco/issue/3447,
+all packages containing Golang libraries that are not leaves are exempted from
+automatic retirement until the new Golang Packaging Guidelines are approved by
+the Packaging Committee and published.
+These packages are still listed in the report, but there is an additional list
+of packages with exemptions included at the end.
+Applications written in Go that do not include libraries used by other Go packages
+are NOT subject to this exemption and will be retired as usual.
+It is recommended not to unorphan any Golang libraries in the interim.
+Instead, the Go SIG suggests waiting until the new guidelines are published and
+then porting your packages to the new tooling that uses vendored dependencies,
+obsoleting the current approach involving golang-*-devel packages.
+Packagers interested in being early adopters or testers of the new tooling are
+welcome to join #golang:fedoraproject.org on Matrix.
+Once the new guidelines are fully implemented, automatic retirements of
+orphaned Golang packages will resume after a six-week grace period.
+
+The following packages are orphaned and will be retired when they
+are orphaned for six weeks, unless someone adopts them. If you know for sure
+that the package should be retired, please do so now with a proper reason:
+https://fedoraproject.org/wiki/How_to_remove_a_package_at_end_of_life
+
+Note: If you received this mail directly you (co)maintain one of the affected
+packages or a package that depends on one. Please adopt the affected package or
+retire your depending package to avoid broken dependencies, otherwise your
+package will be retired when the affected package gets retired.
+
+Request package ownership via the *Take* button in the left column on
+https://src.fedoraproject.org/rpms/<pkgname>
+
+Full report available at:
+https://a.gtmx.me/orphans/orphans.txt
+grep it for your FAS username and follow the dependency chain.
+
+For human readable dependency chains,
+see https://packager-dashboard.fedoraproject.org/
+For all orphaned packages,
+see https://packager-dashboard.fedoraproject.org/orphan
+"""
+
+FOOTER = """-- \nThe script creating this output is run and developed by Fedora
+Release Engineering. Please report issues at its pagure instance:
+https://pagure.io/releng/
+The sources of this script can be found at:
+https://pagure.io/releng/blob/main/f/scripts/find_unblocked_orphans.py
+"""
+
+
+def eprint(*args, **kwargs):
+    kwargs.setdefault("file", sys.stderr)
+    kwargs.setdefault("flush", True)
+    print(*args, **kwargs)
+
+
+def send_mail(from_, to, subject, text, bcc=None):
+    if bcc is None:
+        bcc = []
+
+    msg = email.mime.text.MIMEText(text)
+    msg["Subject"] = subject
+    msg["From"] = from_
+    msg["To"] = to
+    if isinstance(to, str):
+        to = [to]
+    smtp = smtplib.SMTP("127.0.0.1")
+    errors = smtp.sendmail(from_, to + bcc, msg.as_string())
+    smtp.quit()
+    return errors
+
+
+class PagureInfo:
+    def __init__(self, package, branch=RELEASES["rawhide"]["pagure_branch"], ns="rpms"):
+        self.package = package
+        self.branch = branch
+
+        try:
+            response = requests.get(f"{PAGURE_URL}/api/0/{ns}/{package}")
+            self.pkginfo = response.json()
+            if "error" in self.pkginfo:
+                # This is likely a "project not found" 404 error.
+                raise ValueError(self.pkginfo["error"])
+        except Exception:
+            eprint(f"Error getting pagure info for {ns}/{package} on {branch}")
+            traceback.print_exc(file=sys.stderr)
+            self.pkginfo = None
+            return
+
+    def get_people_and_emails(self) -> tuple[list[str], list[str]]:
+        if self.pkginfo is None:
+            return [], []
+        people = set()
+        emails = set()
+        for kind in ["access_users", "access_groups"]:
+            for persons in self.pkginfo[kind].values():
+                for person in persons:
+                    if kind == "access_groups":
+                        people.add("@" + person)
+                        emails.add(f"{person}-members@fedoraproject.org")
+                    else:
+                        people.add(person)
+                        emails.add(f"{person}@fedoraproject.org")
+        return sorted(people), sorted(emails)
+
+    @property
+    def age(self):
+        then = self.status_change
+        now = datetime.datetime.now(datetime.timezone.utc)
+        return now - then
+
+    @property
+    def status_change(self):
+        if self.pkginfo is None:
+            return datetime.datetime.now(datetime.timezone.utc)
+        # See https://pagure.io/pagure/issue/2412
+        if "date_modified" in self.pkginfo:
+            status_change = float(self.pkginfo["date_modified"])
+        else:
+            status_change = float(self.pkginfo["date_created"])
+        status_change_dt = datetime.datetime.fromtimestamp(
+            status_change, tz=datetime.timezone.utc
+        )
+        return status_change_dt
+
+    def __getitem__(self, *args, **kwargs):
+        return self.pkginfo.__getitem__(*args, **kwargs)
+
+
+def setup_dnf(
+    repo=RELEASES["rawhide"]["repo"],
+    source_repo=RELEASES["rawhide"]["source_repo"],
+):
+    """Setup dnf query with two repos"""
+    base = dnf.Base()
+    # use digest to make repo id unique for each URL
+    for baseurl, name in (repo, "repo"), (source_repo, "repo-source"):
+        r = base.repos.add_new_repo(
+            name + "-" + hashlib.sha256(baseurl.encode()).hexdigest(),
+            base.conf,
+            baseurl=[baseurl],
+            skip_if_unavailable=False,
+        )
+        r.enable()
+        r.load()
+
+    base.fill_sack(load_system_repo=False, load_available_repos=True)
+    return base.sack.query()
+
+
+@cache.cache_on_arguments()
+def orphan_packages(namespace="rpms"):
+    pkgs, pages = get_pagure_orphans(namespace)
+    eprint(f"({pages} pages)", end=" ")
+    for page in range(2, pages + 1):
+        if page % 10:
+            eprint(".", end="")
+        else:
+            eprint(page, end="")
+        new_pkgs, _ = get_pagure_orphans(namespace, page)
+        pkgs.update(new_pkgs)
+    return pkgs
+
+
+@cache.cache_on_arguments()
+def get_pagure_orphans(namespace, page=1):
+    url = PAGURE_URL + "/api/0/projects"
+    params = dict(
+        owner=ORPHAN_UID,
+        namespace=namespace,
+        page=page,
+        per_page=PAGURE_MAX_ENTRIES_PER_PAGE,
+    )
+    tries = 0
+    response = requests.get(url, params=params)
+    while not bool(response):
+        msg = f"{response.request.url!r} gave {response!r}"
+        if tries > 20:
+            raise IOError(msg)
+        print(msg, file=sys.stderr)
+        time.sleep(tries)
+        tries += 1
+        response = requests.get(url, params=params)
+    pkgs = response.json()["projects"]
+    pages = response.json()["pagination"]["pages"]
+    return {p["name"]: p for p in pkgs}, pages
+
+
+def unblocked_packages(
+    packages,
+    tagID=RELEASES["rawhide"]["koji_tag"],
+    kojihub=RELEASES["rawhide"]["koji_hub"],
+):
+    unblocked = []
+    kojisession = koji.ClientSession(kojihub)
+
+    kojisession.multicall = True
+    for p in packages:
+        kojisession.listPackages(tagID=tagID, pkgID=p, inherited=True)
+    listings = kojisession.multiCall()
+
+    # Check the listings for unblocked packages.
+
+    for pkgname, result in zip(packages, listings):
+        if isinstance(result, list):
+            [pkg] = result
+            if pkg:
+                if not pkg[0]["blocked"]:
+                    package_name = pkg[0]["package_name"]
+                    unblocked.append(package_name)
+            else:
+                # TODO - what state does this condition represent?
+                pass
+        else:
+            print(f"ERROR: {pkgname}: {result}")
+    return unblocked
+
+
+class DepChecker:
+    def __init__(self, release, repo=None, source_repo=None, namespace="rpms"):
+        self.release = release
+        repo = repo or RELEASES[release]["repo"]
+        source_repo = source_repo or RELEASES[release]["source_repo"]
+
+        dnfquery = setup_dnf(repo=repo, source_repo=source_repo)
+        self.dnfquery = dnfquery
+        self.pagureinfo_queue = Queue()
+        self.pagure_dict = {}
+        self.not_in_repo = []
+        self.dep_chain = defaultdict(set)
+
+        # create_mapping()
+        src_by_bin = {}  # Dict of source pkg objects by binary package objects
+        bin_by_src = {}  # Dict of binary pkgobjects by srpm name
+        # Populate the dicts
+        for rpm_package in self.dnfquery:
+            if rpm_package.arch == "src":
+                continue
+            srpm = SRPM(self.dnfquery, rpm_package)
+            src_by_bin[rpm_package] = srpm
+            if srpm.name in bin_by_src:
+                bin_by_src[srpm.name].append(rpm_package)
+            else:
+                bin_by_src[srpm.name] = [rpm_package]
+
+        self._src_by_bin = src_by_bin
+        self._bin_by_src = bin_by_src
+
+    @property
+    def by_src(self):
+        return self._bin_by_src
+
+    @property
+    def by_bin(self):
+        return self._src_by_bin
+
+    def find_dependent_packages(self, srpmname, ignore):
+        """Return packages depending on packages built from SRPM ``srpmname``
+        that are built from different SRPMS not specified in ``ignore``.
+
+        :param ignore: list of binary package names that will not be
+            returned as dependent packages or considered as alternate
+            providers
+        :type ignore: list() of str()
+
+        :returns: OrderedDict dependent_package: list of requires only
+            provided by package ``srpmname`` {dep_pkg: [prov, ...]}
+        """
+        # Some of this code was stolen from repoquery
+        dependent_packages = {}
+
+        # Handle packags not found in the repo
+        try:
+            rpms = self.by_src[srpmname]
+        except KeyError:
+            # If we don't have a package in the repo, there is nothing to do
+            eprint(f"Package {srpmname} not found in repo")
+            self.not_in_repo.append(srpmname)
+            rpms = []
+
+        # provides of all packages built from ``srpmname``
+        provides = []
+        for pkg in rpms:
+            # add all the provides from the package as strings
+            string_provides = [str(prov) for prov in pkg.provides]
+            provides.extend(string_provides)
+
+            # add all files as provides
+            # pkg.files is a list of paths
+            # sometimes paths start with "//" instead of "/"
+            # normalise "//" to "/":
+            # os.path.normpath("//") == "//", but
+            # os.path.normpath("///") == "/"
+            file_provides = [os.path.normpath(f"//{fn}") for fn in pkg.files]
+            provides.extend(file_provides)
+
+        # Zip through the provides and find what's needed
+        for prov in provides:
+            # check only base provide, ignore specific versions
+            # "foo = 1.fc20" -> "foo"
+            base_provide, *_ = prov.split()
+
+            # FIXME: Workaround for:
+            # https://bugzilla.redhat.com/show_bug.cgi?id=1191178
+            if base_provide[0] == "/":
+                base_provide = base_provide.replace("[", "?")
+                base_provide = base_provide.replace("]", "?")
+
+            # Elide provide if also provided by another package
+            for pkg in self.dnfquery.filter(provides=base_provide):
+                # FIXME: might miss broken dependencies in case the other
+                # provider depends on a to-be-removed package as well
+                if pkg.name in ignore:
+                    # eprint(f"Ignoring provider package {pkg.name}")
+                    pass
+                elif pkg not in rpms:
+                    break
+            else:
+                for dependent_pkg in self.dnfquery.filter(requires=base_provide):
+                    # skip if the dependent rpm package belongs to the
+                    # to-be-removed Fedora package
+                    if dependent_pkg in self.by_src[srpmname]:
+                        continue
+
+                    # use setdefault to either create an entry for the
+                    # dependent package or add the required prov
+                    dependent_packages.setdefault(dependent_pkg, set()).add(prov)
+        return OrderedDict(sorted(dependent_packages.items()))
+
+    def pagure_worker(self):
+        branch = RELEASES[self.release]["pagure_branch"]
+        while True:
+            package = self.pagureinfo_queue.get()
+            if package not in self.pagure_dict:
+                pkginfo = PagureInfo(package, branch)
+                qsize = self.pagureinfo_queue.qsize()
+                eprint(f"Got info for {package} on {branch}, todo: {qsize}")
+                self.pagure_dict[package] = pkginfo
+            self.pagureinfo_queue.task_done()
+
+    def recursive_deps(self, packages, max_deps=20):
+        incomplete = []
+        # Start threads to get information about (co)maintainers for packages
+        for _ in range(0, 2):
+            people_thread = Thread(target=self.pagure_worker)
+            people_thread.daemon = True
+            people_thread.start()
+        # get a list of all rpm_pkgs that are to be removed
+        rpm_pkg_names = []
+        for name in packages:
+            self.pagureinfo_queue.put(name)
+            # Empty list if pkg is only for a different arch
+            bin_pkgs = self.by_src.get(name, [])
+            rpm_pkg_names.extend([p.name for p in bin_pkgs])
+
+        # dict for all dependent packages for each to-be-removed package
+        dep_map = OrderedDict()
+        for name in sorted(packages):
+            self.dep_chain[name] = (
+                set()
+            )  # explicitly initialize the set for the orphaned
+            eprint(f"Getting packages depending on: {name}")
+            ignore = rpm_pkg_names
+            dep_map[name] = OrderedDict()
+            to_check = [name]
+            allow_more = True
+            seen = []
+            while True:
+                eprint(f"to_check ({len(to_check)}): {to_check}")
+                check_next = to_check.pop(0)
+                seen.append(check_next)
+                dependent_packages = self.find_dependent_packages(check_next, ignore)
+                if dependent_packages:
+                    new_names = []
+                    new_srpm_names = set()
+                    for pkg, dependencies in dependent_packages.items():
+                        if pkg.arch != "src":
+                            srpm_name = self.by_bin[pkg].name
+                        else:
+                            srpm_name = pkg.name
+                        if (
+                            srpm_name not in to_check
+                            and srpm_name not in new_names
+                            and srpm_name not in seen
+                        ):
+                            new_names.append(srpm_name)
+                        new_srpm_names.add(srpm_name)
+
+                        for dep in dependencies:
+                            dep_map[name].setdefault(
+                                srpm_name, OrderedDict()
+                            ).setdefault(pkg, set()).add(dep)
+                    for new_srpm_name in new_srpm_names:
+                        self.dep_chain[new_srpm_name].add(check_next)
+                        self.pagureinfo_queue.put(new_srpm_name)
+
+                    ignore.extend(new_names)
+                    if allow_more:
+                        to_check.extend(new_names)
+                        found_deps = dep_map[name].keys()
+                        dep_count = len(set(found_deps) | set(to_check))
+                        if dep_count > max_deps:
+                            todo_deps = max_deps - len(found_deps)
+                            if todo_deps < 0:
+                                todo_deps = 0
+                            incomplete.append(name)
+                            eprint(f"Dep count is {dep_count}")
+                            eprint(f"incomplete is {incomplete}")
+
+                            allow_more = False
+                            to_check = to_check[0:todo_deps]
+                if not to_check:
+                    break
+            if not allow_more:
+                eprint(
+                    f"More than {max_deps} broken deps for package "
+                    f"'{name}', dependency check not completed"
+                )
+
+        eprint("Waiting for (co)maintainer information...", end=" ")
+        self.pagureinfo_queue.join()
+        eprint("done")
+        return dep_map, incomplete
+
+
+def maintainer_table(
+    packages, pagure_dict
+) -> tuple[Any, dict[str, set[str]], list[str]]:
+    affected_people: dict[str, set[str]] = {}
+    all_addresses: set[str] = set()
+
+    if with_table:
+        table = texttable.Texttable(max_width=80)
+        table.header(["Package", "(co)maintainers", "Status Change"])
+        table.set_cols_align(["l", "l", "l"])
+        table.set_deco(table.HEADER)
+    else:
+        table = ""
+
+    for package_name in packages:
+        pkginfo = pagure_dict[package_name]
+        people, addresses = pkginfo.get_people_and_emails()
+        all_addresses.update(addresses)
+        for p in people:
+            affected_people.setdefault(p, set()).add(package_name)
+        p = ", ".join(people)
+        age = pkginfo.age
+        agestr = f"{age.days // 7} weeks ago"
+
+        if with_table:
+            table.add_row([package_name, p, agestr])
+        else:
+            table += f"{package_name} {p} {agestr}\n"
+    all_addresses.discard(f"{ORPHAN_UID}@fedoraproject.org")
+
+    if with_table:
+        table = table.draw()
+    return table, affected_people, sorted(all_addresses)
+
+
+def dependency_info(dep_map, affected_people, pagure_dict, incomplete):
+    info = ""
+    for package_name, subdict in dep_map.items():
+        if subdict:
+            pkginfo = pagure_dict[package_name]
+            status_change = pkginfo.status_change.strftime("%Y-%m-%d")
+            age = pkginfo.age.days // 7
+            fmt = "Depending on: {} ({}), status change: {} ({} weeks ago)\n"
+            info += fmt.format(package_name, len(subdict.keys()), status_change, age)
+            for fedora_package, dependent_packages in subdict.items():
+                people, _ = pagure_dict[fedora_package].get_people_and_emails()
+                for p in people:
+                    affected_people.setdefault(p, set()).add(package_name)
+                p = ", ".join(people)
+                info += f"\t{fedora_package} (maintained by: {p})\n"
+                for dep in dependent_packages:
+                    provides = ", ".join(sorted(dependent_packages[dep]))
+                    info += f"\t\t{dep} requires {provides}\n"
+                info += "\n"
+        if package_name in incomplete:
+            info += f"\tToo many dependencies for {package_name}, "
+            info += "not all listed here\n\n"
+    return info
+
+
+def maintainer_info(affected_people):
+    info = ""
+    for person in sorted(affected_people):
+        packages = affected_people[person]
+        if person == ORPHAN_UID:
+            continue
+        info += f"{person}: {', '.join(packages)}\n"
+    return info
+
+
+def stream_to_set(stream: IO[str]) -> set[str]:
+    with stream:
+        result: set[str] = set()
+        for line in stream:
+            result.add(line.strip())
+        return result
+
+
+def get_golang_exemptions(package_data_path: Path, packages: list[str]) -> list[str]:
+    result: list[str] = []
+    all_golang = stream_to_set(package_data_path.joinpath("all_packages").open())
+    not_exempt = stream_to_set(
+        package_data_path.joinpath("fesco_3447_not_exempt").open()
+    )
+    for package in packages:
+        if package in all_golang and package not in not_exempt:
+            result.append(package)
+    return result
+
+
+class PackageInfoDict(TypedDict):
+    """
+    Represents a JSON blob containing a info about orphaned packages
+    """
+
+    affected_people: dict[str, list[str]]
+    addresses: list[str]
+    orphans: list[str]
+    orphans_breaking_deps: list[str]
+    orphans_breaking_deps_stale: list[str]
+    orphans_not_breaking_deps: list[str]
+    orphans_not_breaking_deps_stale: list[str]
+    ftbfs_breaking_deps: list[str]
+    ftbfs_not_breaking_deps: list[str]
+    # Superset of affected_people and co-maintainers of dependencies
+    all_affected_people: dict[str, list[str]]
+    # Only included when the script is run with a path to the Go package data
+    golang_exemptions: NotRequired[list[str]]
+
+
+def package_info(
+    unblocked,
+    dep_map,
+    depchecker,
+    orphans=None,
+    failed=None,
+    week_limit=6,
+    release="",
+    incomplete=[],
+    go_package_data: Path | None = None,
+) -> tuple[str, PackageInfoDict]:
+    info = ""
+    info_dict: dict[str, Any] = {}
+    pagure_dict = depchecker.pagure_dict
+
+    table, affected_people, addresses = maintainer_table(unblocked, pagure_dict)
+    info_dict["affected_people"] = {
+        key: list(value) for key, value in affected_people.items()
+    }
+    info_dict["addresses"] = addresses
+
+    info += table
+    info += "\n\nThe following packages require above mentioned packages:\n"
+    info += dependency_info(dep_map, affected_people, pagure_dict, incomplete)
+    info_dict["all_affected_people"] = {
+        key: list(value) for key, value in affected_people.items()
+    }
+
+    info += "Affected (co)maintainers\n"
+    info += maintainer_info(affected_people)
+
+    if release:
+        release_text = f" ({release})"
+        branch = RELEASES[release]["pagure_branch"]
+    else:
+        release_text = ""
+
+    wrapper = textwrap.TextWrapper(
+        break_long_words=False, subsequent_indent="    ", break_on_hyphens=False
+    )
+
+    def wrap_and_format(label, pkgs):
+        count = len(pkgs)
+        text = f"{label} ({count}): {' '.join(pkgs)}"
+        wrappedtext = "\n" + wrapper.fill(text) + "\n\n"
+        return wrappedtext
+
+    if orphans:
+        orphans = [o for o in orphans if o in unblocked]
+        info_dict["orphans"] = orphans
+        info += wrap_and_format("Orphans", orphans)
+
+        orphans_breaking_deps = [o for o in orphans if dep_map.get(o)]
+        info_dict["orphans_breaking_deps"] = orphans_breaking_deps
+        info += wrap_and_format("Orphans (dependend on)", orphans_breaking_deps)
+
+        orphans_breaking_deps_stale = [
+            o
+            for o in orphans_breaking_deps
+            if (pagure_dict[o].age.days // 7) >= week_limit
+        ]
+        info_dict["orphans_breaking_deps_stale"] = orphans_breaking_deps_stale
+
+        info += wrap_and_format(
+            f"Orphans{release_text} for at least {week_limit} " "weeks (dependend on)",
+            orphans_breaking_deps_stale,
+        )
+
+        orphans_not_breaking_deps = [o for o in orphans if not dep_map.get(o)]
+        info_dict["orphans_not_breaking_deps"] = orphans_not_breaking_deps
+
+        info += wrap_and_format(
+            f"Orphans{release_text} (not depended on)", orphans_not_breaking_deps
+        )
+
+        orphans_not_breaking_deps_stale = [
+            o
+            for o in orphans_not_breaking_deps
+            if (pagure_dict[o].age.days // 7) >= week_limit
+        ]
+        info_dict["orphans_not_breaking_deps_stale"] = orphans_not_breaking_deps_stale
+
+        if orphans_not_breaking_deps_stale:
+            eprint(
+                f"fedretire --orphan --branch {branch} -- "
+                + " ".join(orphans_not_breaking_deps_stale)
+            )
+
+        info += wrap_and_format(
+            f"Orphans{release_text} for at least {week_limit} "
+            "weeks (not dependend on)",
+            orphans_not_breaking_deps_stale,
+        )
+
+    breaking: set[str] = set()
+    for package, deps in dep_map.items():
+        breaking = breaking.union(set(deps))
+
+    if breaking:
+        info += wrap_and_format("Depending packages" + release_text, sorted(breaking))
+
+        if orphans:
+            reverse_deps: dict[str, list[str]] = OrderedDict()
+            stale_breaking: set[str] = set()
+            for package in orphans_breaking_deps_stale:
+                for depender in dep_map[package]:
+                    reverse_deps.setdefault(depender, []).append(package)
+                stale_breaking = stale_breaking.union(set(dep_map[package].keys()))
+            for depender, providers in reverse_deps.items():
+                eprint(
+                    f"fedretire --orphan-dependent {' '.join(providers)} "
+                    f"--branch {branch} -- {depender}"
+                )
+                for providingpkg in providers:
+                    eprint("fedretire --orphan --branch " f"{branch} -- {providingpkg}")
+            info += wrap_and_format(
+                f"Packages depending on packages orphaned{release_text} "
+                f"for more than {week_limit} weeks",
+                sorted(stale_breaking),
+            )
+
+    if failed:
+        ftbfs_label = "FTBFS" + release_text
+        info += wrap_and_format(ftbfs_label, failed)
+
+        ftbfs_breaking_deps = [o for o in failed if o in dep_map and dep_map[o]]
+        info_dict["ftbfs_breaking_deps"] = ftbfs_breaking_deps
+
+        info += wrap_and_format(ftbfs_label + " (depended on)", ftbfs_breaking_deps)
+
+        ftbfs_not_breaking_deps = [
+            o for o in failed if o not in dep_map or not dep_map[o]
+        ]
+        info_dict["ftbfs_not_breaking_deps"] = ftbfs_not_breaking_deps
+
+        info += wrap_and_format(
+            ftbfs_label + " (not depended on)", ftbfs_not_breaking_deps
+        )
+
+    if depchecker.not_in_repo:
+        info += wrap_and_format(
+            "Not found in repo" + release_text, sorted(depchecker.not_in_repo)
+        )
+
+    if go_package_data:
+        info_dict["golang_exemptions"] = get_golang_exemptions(
+            go_package_data, unblocked
+        )
+        info += wrap_and_format(
+            f"Golang orphans{release_text} that are exempted from retirement",
+            info_dict["golang_exemptions"],
+        )
+
+    return info, cast(PackageInfoDict, info_dict)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--skip-orphans",
+        dest="skip_orphans",
+        help="Do not look for orphans",
+        default=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "--max_deps",
+        dest="max_deps",
+        type=int,
+        help="set max_deps on recursive find deps",
+        default=20,
+    )
+    parser.add_argument("--release", choices=RELEASES.keys(), default="rawhide")
+    parser.add_argument(
+        "--mailto", default=None, help="Send mail to this address (for testing)"
+    )
+    parser.add_argument(
+        "--send",
+        default=False,
+        action="store_true",
+        help="Actually send mail including Bcc addresses to mailing list",
+    )
+    parser.add_argument(
+        "--source-repo", default=None, help="Source repo URL to use for depcheck"
+    )
+    parser.add_argument("--repo", default=None, help="Repo URL to use for depcheck")
+    parser.add_argument(
+        "--json",
+        default=None,
+        help="Export info about orphaned " "packages to a specified JSON file",
+    )
+    parser.add_argument(
+        "--no-skip-blocked",
+        default=True,
+        dest="skipblocked",
+        action="store_false",
+        help="Do not skip blocked pkgs",
+    )
+    parser.add_argument("--mailfrom", default="nobody@fedoraproject.org")
+    filetype = argparse.FileType("w", encoding="utf-8")
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output report to a text file",
+        type=filetype,
+        default=filetype("-"),
+    )
+    parser.add_argument(
+        "failed", nargs="*", help="Additional packages, e.g. FTBFS packages"
+    )
+    go_package_data_env = os.environ.get("GO_PACKAGE_DATA")
+    parser.add_argument(
+        "--go-package-data",
+        help="Path to https://gitlab.com/fedora/sigs/go/package-data checkout",
+        default=Path(go_package_data_env) if go_package_data_env else None,
+        type=Path,
+    )
+    args = parser.parse_args()
+    failed = args.failed
+
+    if args.source_repo is not None:
+        RELEASES[args.release]["source_repo"] = args.source_repo
+
+    if args.repo is not None:
+        RELEASES[args.release]["repo"] = args.repo
+
+    text = "Report started at %s\n\n" % (
+        datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
+    )
+
+    if args.skip_orphans:
+        orphans = []
+        unblocked = failed
+    else:
+        # list of orphans from pagure
+        eprint("Contacting pagure for list of orphans...", end=" ")
+        orphans = sorted(orphan_packages())
+        eprint("done")
+        allpkgs = sorted(list(set(list(orphans) + failed)))
+
+        if args.skipblocked:
+            eprint("Getting builds from koji...", end=" ")
+            koji_tag = RELEASES[args.release]["koji_tag"]
+            koji_hub = RELEASES[args.release]["koji_hub"]
+            unblocked = unblocked_packages(allpkgs, tagID=koji_tag, kojihub=koji_hub)
+            eprint("done")
+
+    text += HEADER.format(RELEASES[args.release]["koji_tag"].upper())
+    eprint("Setting up dependency checker...", end=" ")
+    depchecker = DepChecker(args.release)
+    eprint("done")
+
+    eprint("Calculating dependencies...", end=" ")
+    # Create dnf object and depsolve out if requested.
+    # TODO: add app args to either depsolve or not
+    dep_map, incomplete = depchecker.recursive_deps(unblocked, args.max_deps)
+    eprint("done")
+    info, package_info_dict = package_info(
+        unblocked,
+        dep_map,
+        depchecker,
+        orphans=orphans,
+        failed=failed,
+        release=args.release,
+        incomplete=incomplete,
+        go_package_data=args.go_package_data,
+    )
+    addresses = package_info_dict["addresses"]
+    text += "\n"
+    text += info
+    text += FOOTER
+    text += "\nReport finished at %s" % (
+        datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
+    )
+    args.output.write(text + "\n")
+
+    if args.json is not None:
+        eprint(f"Saving {args.json} with machine readable info")
+        sc = {
+            pkg: depchecker.pagure_dict[pkg].status_change.isoformat()
+            for pkg in orphans
+            if pkg in depchecker.pagure_dict
+        }
+        ap = {pkg: sorted(reasons) for pkg, reasons in depchecker.dep_chain.items()}
+        json_data = {
+            "status_change": sc,
+            "affected_packages": ap,
+            **dict(package_info_dict),
+        }
+        try:
+            with open(args.json, "w") as f:
+                json.dump(json_data, f, indent=4, sort_keys=True)
+        except OSError as e:
+            eprint(f"Cannot save {args.json}:", end=" ")
+            eprint(f"{type(e).__name__}: e")
+
+    if args.mailto or args.send:
+        now = datetime.datetime.now(datetime.timezone.utc)
+        today = now.strftime("%Y-%m-%d")
+        subject = f"Orphaned Packages in {args.release} ({today})"
+        if args.mailto:
+            mailto = args.mailto
+        else:
+            mailto = RELEASES[args.release]["mailto"]
+        if args.send:
+            bcc = addresses + RELEASES[args.release]["bcc"]
+        else:
+            bcc = None
+        mail_errors = send_mail(args.mailfrom, mailto, subject, text, bcc)
+        if mail_errors:
+            eprint("mail errors: " + repr(mail_errors))
+
+    eprint(f"Addresses ({len(addresses)}):", ", ".join(addresses))
+
+
+if __name__ == "__main__":
+    main()

scripts_new/packages/orphaned/orphaned-packages-process/pyproject.toml

@@ -0,0 +1,14 @@
+[tool.black]
+line-length = 89
+
+[tool.isort]
+profile = "black"
+
+[[tool.mypy.overrides]]
+module = [
+    "bugzilla",
+    "dnf.*",
+    "koji.*",
+    "texttable.*",
+]
+ignore_missing_imports = true

scripts_new/packages/orphaned/orphaned-packages-process/requirements.txt

@@ -0,0 +1,8 @@
+# retire.py
+python-bugzilla
+click
+# find_unblocked_orphans.py
+dogpile.cache
+koji
+requests
+texttable

scripts_new/packages/orphaned/orphaned-packages-process/retire.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2024 Maxwell G <maxwell@gtmx.me>
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+from __future__ import annotations
+
+import dataclasses
+import datetime
+import json
+import os.path
+import subprocess
+from collections.abc import Iterator, Sequence
+from contextlib import AbstractContextManager, nullcontext
+from functools import partial
+from tempfile import TemporaryDirectory
+from typing import IO, TYPE_CHECKING
+from urllib.parse import urljoin
+
+import click
+import requests
+import requests.adapters
+from bugzilla import Bugzilla
+
+if TYPE_CHECKING:
+    from _typeshed import StrPath
+
+    from find_unblocked_orphans import PackageInfoDict
+
+
+JSON_DOWNLOAD = "https://a.gtmx.me/orphans/orphans.json"
+ORPHAN_UID = "orphan"
+PACKAGE_API_URL = "https://src.fedoraproject.org/api/0/rpms/"
+BUGZILLA_API_URL = "https://bugzilla.redhat.com"
+
+DEFAULT_DISTGIT_MESSAGE = "Orphaned for 6+ weeks"
+TEMPLATE = """Automation has figured out the package is retired in Fedora {}.
+
+If you like it to be unretired, please open a ticket at
+https://pagure.io/releng/new_issue?template=package_unretirement
+"""
+
+
+def get_requests_session() -> requests.Session:
+    session = requests.Session()
+    retry = requests.adapters.Retry()
+    for protocol in "http://", "https://":
+        session.mount(protocol, requests.adapters.HTTPAdapter(max_retries=retry))
+    return session
+
+
+bz_session = Bugzilla(BUGZILLA_API_URL)
+session = get_requests_session()
+
+
+def run(
+    cmd: Sequence[StrPath],
+    *,
+    capture_text: bool = False,
+    dry_run=False,
+    log: bool = True,
+    **kwargs,
+) -> subprocess.CompletedProcess | None:
+    kwargs.setdefault("check", True)
+    if capture_text:
+        kwargs["text"] = True
+        kwargs["capture_output"] = True
+    if log:
+        start = "Would run" if dry_run else "Running"
+        click.secho(
+            f"* {start}: {tuple(map(str, cmd))}",
+            err=True,
+            fg="yellow" if dry_run else "blue",
+        )
+    if dry_run:
+        return None
+    return subprocess.run(cmd, **kwargs)  # noqa: PLW1510
+
+
+@dataclasses.dataclass()
+class CLIContext:
+    json_data: PackageInfoDict
+    package_list: Sequence[str] | None = None
+
+    @property
+    def orphans_stale(self) -> list[str]:
+        if self.package_list is not None:
+            return sorted(self.package_list)
+        return sorted(
+            {
+                *self.json_data["orphans_breaking_deps_stale"],
+                *self.json_data["orphans_not_breaking_deps_stale"],
+            }
+        )
+
+    def package_iter(self, check: bool = True) -> Iterator[str]:
+        for package in self.orphans_stale:
+            if check and (owner := ensure_orphaned(package)):
+                click.secho(f"! {package} is owned by {owner}", fg="red", err=True)
+                continue
+            yield package
+
+
+def ensure_orphaned(package: str) -> str | None:
+    req = session.get(urljoin(PACKAGE_API_URL, package))
+    req.raise_for_status()
+    owner = req.json()["user"]["name"]
+    return owner if owner != ORPHAN_UID else None
+
+
+pass_obj = click.make_pass_decorator(CLIContext, True)
+
+
+@click.group(
+    context_settings={"help_option_names": ["-h", "--help"], "show_default": True}
+)
+@click.option("--json", "json_file", default=JSON_DOWNLOAD)
+@click.option("--list", "package_list_file", type=click.File())
+@click.pass_context
+def main(ctx: click.Context, json_file: str, package_list_file: IO[str] | None) -> None:
+    if json_file.startswith(("http://", "https://")):
+        data = session.get(json_file).json()
+    else:
+        with open(json_file, "r", encoding="utf-8") as fp:
+            data = json.load(fp)
+    package_list: list[str] | None = None
+    if package_list_file:
+        package_list = [p.strip() for p in package_list_file]
+        package_list_file.close()
+    ctx.obj = CLIContext(json_data=data, package_list=package_list)
+
+
+CHECK_OPT = partial(
+    click.option,
+    "--check / --no-check",
+    default=False,
+    help="Whether to check that packages are actually orphaned",
+)
+
+
+@main.command(name="list")
+@CHECK_OPT(default=False)
+@pass_obj
+def list_command(args: CLIContext, check: bool) -> None:
+    """
+    List files that have been orphaned for over 6 weeks
+    """
+    for package in args.package_iter(check):
+        click.echo(package)
+
+
+def retire_distgit(
+    package: str,
+    directory: str,
+    dry_run: bool,
+    branches: Sequence[str] = (),
+    message: str | None = None,
+) -> None:
+    dir = os.path.join(directory, package)
+    in_dir = partial(run, dry_run=dry_run, cwd=dir)
+    run(["fedpkg", "clone", package, dir])
+    in_dir(["fedpkg", "retire", message or DEFAULT_DISTGIT_MESSAGE])
+    for branch in branches:
+        run(["git", "switch", branch], cwd=dir)
+        in_dir(["git", "merge", "--gpg-sign", "rawhide"])
+        in_dir(["git", "push"])
+
+
+def retire_bugs(package: str, dry_run: bool, branches: Sequence[str] = ()) -> None:
+    branches = ("rawhide", *branches)
+    for branch in branches:
+        query = bz_session.build_query(
+            product="Fedora",
+            version=branch.lstrip("f"),
+            component=package,
+            status=["__open__"],
+        )
+        bugs = bz_session.query(query)
+        bug_ids: list[str] = []
+        for bug in bugs:
+            begin = "Would close" if dry_run else "Closing"
+            fg = "yellow" if dry_run else "blue"
+            click.secho(f"* {begin}: {bug.id} --- {bug.short_desc}", err=True, fg=fg)
+            bug_ids.append(bug.id)
+        if not dry_run:
+            update = bz_session.build_update(
+                comment=TEMPLATE.format(branch.title()),
+                status="CLOSED",
+                resolution="WONTFIX",
+            )
+            bz_session.update_bugs(bug_ids, update)
+
+
+def retire(
+    package: str,
+    directory: str,
+    dry_run: bool,
+    branches: Sequence[str] = (),
+    message: str | None = None,
+) -> None:
+    click.secho("distgit", bold=True)
+    retire_distgit(package, directory, dry_run, branches, message)
+    click.secho("bugzilla", bold=True)
+    retire_bugs(package, dry_run, branches)
+
+
+@main.command(name="retire")
+@click.option("-n", "--dry-run", default=False, is_flag=True)
+@click.option("--workdir")
+@click.option("-b", "--branch", "branches", multiple=True)
+@click.option(
+    "--lf",
+    "--log-file",
+    "log_file",
+    help="Append a list of retired packages to a file",
+    type=click.File("a"),
+)
+@click.option("--message", help="Message to use for distgit retirement commit")
+@CHECK_OPT(default=True)
+@pass_obj
+def retire_command(
+    args: CLIContext,
+    dry_run: bool,
+    workdir: str,
+    branches: Sequence[str],
+    check: bool,
+    log_file: IO[str] | None,
+    message: str | None,
+) -> None:
+    """
+    Retire packages that have been orphaned for over 6 weeks
+    """
+    if workdir:
+        os.makedirs(workdir)
+    cm: AbstractContextManager[str] = (
+        nullcontext(workdir) if workdir else TemporaryDirectory()  # type: ignore[assignment]
+    )
+    now = format(datetime.datetime.now(datetime.timezone.utc), "%Y-%m-%d")
+    with cm as workdir:
+        for package in args.package_iter(check):
+            click.secho(f"{package}", underline=True, fg="green")
+            retire(package, workdir, dry_run, branches, message)
+            if log_file:
+                log_file.write(f"{package} {now}\n")
+            click.echo()
+
+
+if __name__ == "__main__":
+    main()

scripts_new/packages/orphaned/orphaned-packages-process/tox.ini

@@ -0,0 +1,35 @@
+[tox]
+env_list =
+    formatters
+    lint
+    typing
+
+[testenv:formatters]
+description = Run formatters
+skip_install = true
+deps =
+    isort
+    black
+commands =
+    black {posargs} find_unblocked_orphans.py retire.py
+    isort {posargs} find_unblocked_orphans.py retire.py
+
+[testenv:lint]
+description = Run linters
+skip_install = true
+deps =
+    ruff
+commands =
+    ruff check {posargs} find_unblocked_orphans.py retire.py
+
+[testenv:typing]
+description = Run type checkers
+skip_install = true
+deps =
+    -r requirements.txt
+    mypy
+    types-requests
+commands =
+    mypy {posargs} find_unblocked_orphans.py retire.py
+set_env =
+    PYTHONPATH=${PWD}

scripts_new/release-process/post-release/README.md

@@ -0,0 +1,3 @@
+# Post release tools
+
+Here will go scripts that are used for post release processes, such as cleanup.

scripts_new/release-process/post-release/failed_composes_cleanup.py

@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+#
+# failed_composes_cleanup.py - A utility to clean all the older issues in the releng/failed-compose tracker.
+#
+#
+# Authors:
+#     Samyak Jain <samyak.jn11@gmail.com>
+# Copyright (C) 2024 Red Hat Inc,
+# SPDX-License-Identifier:	GPL-2.0+
+
+import requests
+
+API_TOKEN = 'YOUR_API_TOKEN'
+
+def get_open_issues(repo):
+    base_url = 'https://pagure.io/api/0/'
+    endpoint = f'{repo}/issues'
+    url = base_url + endpoint
+    
+    # Include API token in headers for authentication
+    headers = {'Authorization': f'token {API_TOKEN}'}
+    
+    open_issues = []
+    page = 1
+    while True:
+        # Make the GET request to retrieve open issues for the current page
+        params = {'page': page}
+        response = requests.get(url, headers=headers, params=params)
+        
+        # Check if the request was successful
+        if response.status_code == 200:
+            page_issues = response.json()['issues']
+            open_issues.extend(issue['id'] for issue in page_issues if issue['status'] == 'Open')
+            # Check if there are more pages
+            if not page_issues:
+                break
+            else:
+                page += 1
+        else:
+            print(f"Failed to retrieve open issues for {repo}.")
+            print(f"Response: {response.text}")
+            return []
+
+    return open_issues
+
+def close_pagure_issue(repo, issue_id, close_status=None):
+    base_url = 'https://pagure.io/api/0/'
+    endpoint = f'{repo}/issue/{issue_id}/status'
+    url = base_url + endpoint
+
+    # Include API token in headers for authentication
+    headers = {'Authorization': f'token {API_TOKEN}'}
+    
+    # Set the new status of the issue
+    payload = {'status': 'Closed'}
+    
+    # Make the POST request to change the status of the issue
+    response = requests.post(url, headers=headers, data=payload)
+    
+    # Check if the request was successful
+    if response.status_code == 200:
+        print(f"Issue #{issue_id} closed successfully.")
+    else:
+        print(f"Failed to close issue #{issue_id}.")
+        print(f"Response: {response.text}")
+        
+repo = 'releng/failed-composes'
+# Get list of open issues
+open_issues = get_open_issues(repo)
+
+print("Open Issues:", open_issues)
+# 
+# Close each open issue
+for issue_id in open_issues:
+    close_pagure_issue(repo, issue_id)