release-notes/scripts/gen_release_notes.py

#!/usr/bin/python3
"""
gen_release_notes.py

Retrieve release notes from the wiki, and generate 
release notes adoc file for each release notes category in
modules/release-notes/pages/$category_name.adoc.

The generated category document is based on the matching
template $category_name.adoc.j2 in the templates directory.
If this template does not exists, it will use
default_category.adoc.j2 instead.

Requires:
python3-requests
python3-jinja2

dnf install -y python3-requests python3-jinja2
or
pip install -r requirements.txt
"""

import requests
import sys
import re
import jinja2
import logging
import urllib.parse
import os
import argparse

api_url = "https://fedoraproject.org/w/api.php"


def list_changes(release):
    """
    Retrieve list of changes from ChangeAcceptedFxx wiki category
    returns a list of wiki pages:
        [{
                "pageid": 12345,
                "ns": 0,
                "title": "Changes/MyAwesomeChange"
        },...]
    """
    params = {
        "action": "query",
        "list": "categorymembers",
        "cmtitle": f"Category:ChangeAcceptedF{release}",
        "cmlimit": 100,
        "format": "json",
    }
    s = requests.Session()
    req = s.get(url=api_url, params=params)
    if req.status_code != 200:
        logging.error(req.status_code)
        return []

    data = req.json()
    return data["query"]["categorymembers"]


def get_change_info(pageid):
    """
    Get change release notes and categories from the wiki page
    returns a dict:
        {
            "title": MyAwesomeChange,
            "categories": ["SystemWideChange", "SomeOtherCategory"],
            "url": f"https://fedoraproject.org/wiki/Changes/MyAwesomeChange",
            "release_notes": "You are awesome!",
        }
    """
    info = {}
    s = requests.Session()
    params = {
        "action": "parse",
        "pageid": pageid,
        "prop": "categories|sections",
        "format": "json",
    }
    req = s.get(url=api_url, params=params)
    page_data = req.json()
    rn_sid = None
    info = {
        "title": page["title"].replace("Changes/", ""),
        "categories": [],
        "url": f"https://fedoraproject.org/wiki/{urllib.parse.quote(page['title'])}",
        "release_notes": "",
    }

    # Get categories
    for category in page_data["parse"]["categories"]:
        if category["*"].startswith("ChangeAccepted"):
            continue
        info["categories"].append(category["*"])

    # Find release notes section
    for section in page_data["parse"]["sections"]:
        if section["line"] == "Release Notes":
            rn_sid = section["index"]
            break
    if rn_sid:
        # Fetch release notes text
        params = {
            "action": "parse",
            "pageid": page["pageid"],
            "section": rn_sid,
            "prop": "wikitext",
            "format": "json",
        }
        req = s.get(url=api_url, params=params)
        section_data = req.json()
        # Remove html comments and empty lines
        info["release_notes"] = "\n".join(
            re.sub(
                "<!--.*?-->",
                "",
                section_data["parse"]["wikitext"]["*"],
                flags=re.MULTILINE | re.DOTALL,
            ).splitlines()[1::]
        ).strip()

    if not info["release_notes"]:
        # If no release notes, fallback to summary
        params = {
            "action": "parse",
            "pageid": page["pageid"],
            "section": 2,  # I expect the summary to be always the first section (after the title) of the page
            "prop": "wikitext",
            "format": "json",
        }
        req = s.get(url=api_url, params=params)
        section_data = req.json()
        # Remove html comments and empty lines
        info["release_notes"] = "\n".join(
            re.sub(
                "<!--.*?-->",
                "",
                section_data["parse"]["wikitext"]["*"],
                flags=re.MULTILINE | re.DOTALL,
            ).splitlines()[1::]
        ).strip()

    return info


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    abs_path = os.path.dirname(os.path.abspath(sys.argv[0]))
    rn_per_cats = {}
    parser = argparse.ArgumentParser(
        description="Generate release notes from the ChangeAccepted wiki category"
    )
    parser.add_argument("release", help="Fedora Linux version")
    args = parser.parse_args()
    env = jinja2.Environment(loader=jinja2.FileSystemLoader(f"{abs_path}/templates"))
    default_tpl = env.get_template("default_category.adoc.j2")

    logging.info(f"Fetching changes for Fedora {args.release}...")
    for page in list_changes(args.release):
        change = get_change_info(page["pageid"])
        logging.info(f"Processed change {change['title']}...")
        for cat in change["categories"]:
            if cat not in rn_per_cats:
                rn_per_cats[cat] = []
            rn_per_cats[cat].append(change)

    for cat_name, changes in rn_per_cats.items():
        logging.info(f"Generating {cat_name}.adoc...")
        if os.path.exists(f"{abs_path}/templates/{cat_name}.adoc.j2"):
            tpl = env.get_template(f"{cat_name}.adoc.j2")
        else:
            tpl = default_tpl
        output_from_parsed_template = tpl.render(cat_name=cat_name, changes=changes)

        with open(
            f"{abs_path}/../modules/release-notes/pages/{cat_name}.adoc", "w"
        ) as fh:
            fh.write(output_from_parsed_template)

    logging.info("Completed.")