add gen_release_notes.py script that generate release notes pages

scripts/gen_release_notes.py

@@ -0,0 +1,181 @@
+#!/usr/bin/python3
+"""
+gen_release_notes.py
+
+Retrieve release notes from the wiki, and generate 
+release notes adoc file for each release notes category in
+modules/release-notes/pages/$category_name.adoc.
+
+The generated category document is based on the matching
+template $category_name.adoc.j2 in the templates directory.
+If this template does not exists, it will use
+default_category.adoc.j2 instead.
+
+Requires:
+python3-requests
+python3-jinja2
+
+dnf install -y python3-requests python3-jinja2
+or
+pip install -r requirements.txt
+"""
+
+import requests
+import sys
+import re
+import jinja2
+import logging
+import urllib.parse
+import os
+import argparse
+
+api_url = "https://fedoraproject.org/w/api.php"
+
+
+def list_changes(release):
+    """
+    Retrieve list of changes from ChangeAcceptedFxx wiki category
+    returns a list of wiki pages:
+        [{
+                "pageid": 12345,
+                "ns": 0,
+                "title": "Changes/MyAwesomeChange"
+        },...]
+    """
+    params = {
+        "action": "query",
+        "list": "categorymembers",
+        "cmtitle": f"Category:ChangeAcceptedF{release}",
+        "cmlimit": 100,
+        "format": "json",
+    }
+    s = requests.Session()
+    req = s.get(url=api_url, params=params)
+    if req.status_code != 200:
+        logging.error(req.status_code)
+        return []
+
+    data = req.json()
+    return data["query"]["categorymembers"]
+
+
+def get_change_info(pageid):
+    """
+    Get change release notes and categories from the wiki page
+    returns a dict:
+        {
+            "title": MyAwesomeChange,
+            "categories": ["SystemWideChange", "SomeOtherCategory"],
+            "url": f"https://fedoraproject.org/wiki/Changes/MyAwesomeChange",
+            "release_notes": "You are awesome!",
+        }
+    """
+    info = {}
+    s = requests.Session()
+    params = {
+        "action": "parse",
+        "pageid": pageid,
+        "prop": "categories|sections",
+        "format": "json",
+    }
+    req = s.get(url=api_url, params=params)
+    page_data = req.json()
+    rn_sid = None
+    info = {
+        "title": page["title"].replace("Changes/", ""),
+        "categories": [],
+        "url": f"https://fedoraproject.org/wiki/{urllib.parse.quote(page['title'])}",
+        "release_notes": "",
+    }
+
+    # Get categories
+    for category in page_data["parse"]["categories"]:
+        if category["*"].startswith("ChangeAccepted"):
+            continue
+        info["categories"].append(category["*"])
+
+    # Find release notes section
+    for section in page_data["parse"]["sections"]:
+        if section["line"] == "Release Notes":
+            rn_sid = section["index"]
+            break
+    if rn_sid:
+        # Fetch release notes text
+        params = {
+            "action": "parse",
+            "pageid": page["pageid"],
+            "section": rn_sid,
+            "prop": "wikitext",
+            "format": "json",
+        }
+        req = s.get(url=api_url, params=params)
+        section_data = req.json()
+        # Remove html comments and empty lines
+        info["release_notes"] = "\n".join(
+            re.sub(
+                "<!--.*?-->",
+                "",
+                section_data["parse"]["wikitext"]["*"],
+                flags=re.MULTILINE | re.DOTALL,
+            ).splitlines()[1::]
+        ).strip()
+
+    if not info["release_notes"]:
+        # If no release notes, fallback to summary
+        params = {
+            "action": "parse",
+            "pageid": page["pageid"],
+            "section": 2,  # I expect the summary to be always the first section (after the title) of the page
+            "prop": "wikitext",
+            "format": "json",
+        }
+        req = s.get(url=api_url, params=params)
+        section_data = req.json()
+        # Remove html comments and empty lines
+        info["release_notes"] = "\n".join(
+            re.sub(
+                "<!--.*?-->",
+                "",
+                section_data["parse"]["wikitext"]["*"],
+                flags=re.MULTILINE | re.DOTALL,
+            ).splitlines()[1::]
+        ).strip()
+
+    return info
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    abs_path = os.path.dirname(os.path.abspath(sys.argv[0]))
+    rn_per_cats = {}
+    parser = argparse.ArgumentParser(
+        description="Generate release notes from the ChangeAccepted wiki category"
+    )
+    parser.add_argument("release", help="Fedora Linux version")
+    args = parser.parse_args()
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(f"{abs_path}/templates"))
+    default_tpl = env.get_template("default_category.adoc.j2")
+
+    logging.info(f"Fetching changes for Fedora {args.release}...")
+    for page in list_changes(args.release):
+        change = get_change_info(page["pageid"])
+        logging.info(f"Processed change {change['title']}...")
+        for cat in change["categories"]:
+            if cat not in rn_per_cats:
+                rn_per_cats[cat] = []
+            rn_per_cats[cat].append(change)
+
+    for cat_name, changes in rn_per_cats.items():
+        logging.info(f"Generating {cat_name}.adoc...")
+        if os.path.exists(f"{abs_path}/templates/{cat_name}.adoc.j2"):
+            tpl = env.get_template(f"{cat_name}.adoc.j2")
+        else:
+            tpl = default_tpl
+        output_from_parsed_template = tpl.render(cat_name=cat_name, changes=changes)
+
+        with open(
+            f"{abs_path}/../modules/release-notes/pages/{cat_name}.adoc", "w"
+        ) as fh:
+            fh.write(output_from_parsed_template)
+
+    logging.info("Completed.")

scripts/requirements.txt

@@ -0,0 +1,2 @@
+Jinja2
+requests

scripts/templates/SelfContainedChange.adoc.j2

@@ -0,0 +1,16 @@
+
+include::{partialsdir}/entities.adoc[]
+
+= Self-Contained Changes
+
+A self-contained change is a change to isolated package(s), or a general change with limited scope and impact on the rest of the distribution/project.
+Examples include addition of a group of leaf packages, or a coordinated effort within a Special Interest Group (SIG) with limited impact outside the SIG’s functional area.
+
+{% for change in changes %}
+== {{ change['title'] }}
+
+{{ change['release_notes'] }}
+
+__Read {{ change['url'] }}[more information] about this change.__
+
+{% endfor %}

scripts/templates/default_category.adoc.j2

@@ -0,0 +1,13 @@
+
+include::{partialsdir}/entities.adoc[]
+
+= {{ cat_name }}
+
+{% for change in changes %}
+== {{ change['title'] }}
+
+{{ change['release_notes'] }}
+
+__Read {{ change['url'] }}[more information] about this change.__
+
+{% endfor %}