Merge pull request 'Drop fedora jenkins docs' (#4676) from tkopecek/koji:drop-jenkins into master

Reviewed-on: #4676
Reviewed-by: Mike McLean <mikem@redhat.com>

.bandit.yaml

@@ -0,0 +1,3 @@
+markupsafe_xss:
+    allowed_calls:
+        - _MarkTrustedValue

.coveragerc3

@@ -4,7 +4,9 @@ data_file = .coverage3
 omit =
     setup.py
     /usr/*
+    /tmp/*
     tests/*
+    devtools/*
     .tox/*
 
 [report]

.editorconfig

@@ -0,0 +1,27 @@
+# Topmost editorconfig for project, don't ascend
+# to parent directories when scanning configs
+root = true
+
+# All files: UTF-8 with Unix-style newlines,
+# no trailing whitespace, and a final newline
+[*]
+charset = utf-8
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# JavaScript, CSS, Python, shell: 4-space indents
+[{**/*.{js,css,py,sh},runtests,cli/koji,vm/kojivmd}]
+indent_style = space
+indent_size = 4
+
+# Makefile: tab indents
+[**/Makefile]
+indent_style = tab
+tab_width = 8
+
+# jinja html: 2-space indents
+[**/*.html.j2]
+indent_style = space
+indent_size = 2
+

.forgejo/workflows/tests.yaml

@@ -0,0 +1,37 @@
+on: [push, pull_request]
+jobs:
+  flake8:
+    runs-on: podman
+    container:
+      image: forge.fedoraproject.org/koji/koji-test:fedora-44@sha256:5e1187ee4536674e08961cbe6b1b84dfc9df15a513dee328365909be5a0d5ed8
+    steps:
+      - run: >-
+          git clone ${{ forgejo.event.pull_request.head.repo.clone_url || format('{0}/{1}', forgejo.server_url, forgejo.repository) }} . &&
+          git checkout ${{ forgejo.event.pull_request.head.sha || forgejo.sha }}
+      - run: make flake8
+  bandit:
+    runs-on: podman
+    container:
+      image: forge.fedoraproject.org/koji/koji-test:fedora-44@sha256:5e1187ee4536674e08961cbe6b1b84dfc9df15a513dee328365909be5a0d5ed8
+    steps:
+      - run: >-
+          git clone ${{ forgejo.event.pull_request.head.repo.clone_url || format('{0}/{1}', forgejo.server_url, forgejo.repository) }} . &&
+          git checkout ${{ forgejo.event.pull_request.head.sha || forgejo.sha }}
+      - run: make bandit
+  tests:
+    runs-on: podman
+    strategy:
+      matrix:
+        variant:
+          - fedora-43@sha256:cf1ef55da0d7dd9251b79263c4d1e2350d4291fcd6dc54a1e751d3fe2cf4614d
+          - fedora-44@sha256:5e1187ee4536674e08961cbe6b1b84dfc9df15a513dee328365909be5a0d5ed8
+          - fedora-rawhide@sha256:5e1187ee4536674e08961cbe6b1b84dfc9df15a513dee328365909be5a0d5ed8
+          - centos-9@sha256:c5fe30a62d4b62b46acdf764bf1f7293e9fa4287972ec6a86ccb43209646dc32
+          - centos-10@sha256:1f53b2b040e449b0cfce526f1a221e6dbd6648506270220ea333728a6b52a90b
+    container:
+      image: forge.fedoraproject.org/koji/koji-test:${{matrix.variant}}
+    steps:
+      - run: >-
+          git clone ${{ forgejo.event.pull_request.head.repo.clone_url || format('{0}/{1}', forgejo.server_url, forgejo.repository) }} . &&
+          git checkout ${{ forgejo.event.pull_request.head.sha || forgejo.sha }}
+      - run: make test3

.readthedocs.yaml

@@ -0,0 +1,22 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version, and other tools you might need
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# Optionally, but recommended,
+# declare the Python requirements required to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# python:
+#    install:
+#    - requirements: docs/requirements.txt

Makefile

@@ -71,6 +71,9 @@ clean:
 	rm -rf __pycache__
 	rm -rf devtools/fakehubc devtools/fakewebc devtools/__pycache__
 	rm -rf koji-$(VERSION)
+	rm -rf build dist koji.egg-info
+	find -name '*.pyi' -delete
+	find -name py.typed -delete
 	for d in $(SUBDIRS); do make -s -C $$d clean; done
 	find ./tests -name "__pycache__" -exec rm -rf {} \; ||:
 	find ./tests -name "*.pyc" -exec rm -f {} \;

README.md

@@ -27,32 +27,33 @@ Communicate
 Bugs/RFEs
 ---------
 
-If you have found a bug or would like to request a new feature, please [report an issue in Pagure](https://pagure.io/koji/issues).
+If you have found a bug or would like to request a new feature, please [report an issue in Forgejo](https://forge.fedoraproject.org/koji/koji/issues).
 
 Download
 --------
 
 The koji source code can be downloaded with git via:
 
-    git clone https://pagure.io/koji.git
+    git clone https://forge.fedoraproject.org/koji/koji.git
 
-You may browse code at https://pagure.io/koji
+You may browse code at https://forge.fedoraproject.org/koji/koji
 
-Archived releases can be found at https://pagure.io/koji/releases
+Archived releases can be found at https://forge.fedoraproject.org/koji/koji/releases
 
 Documentation
 -------------
 
-See: https://docs.pagure.org/koji/
+See: https://koji.readthedocs.io/en/latest/index.html
 
 
 Related Software
 ----------------
 
 * [Mock](https://fedoraproject.org/wiki/Projects/Mock): The tool Koji uses to generate buildroots
-* [Yum](http://yum.baseurl.org/)
-* [Pungi](https://pagure.io/pungi): Use Pungi to "compose" Koji builds into highly customizable Yum repositories.
-* [Koji Tools](https://pagure.io/koji-tools): Various utilities for Koji
+* [Yum](http://yum.baseurl.org/) - RPM package manager
+* [DNF](https://github.com/rpm-software-management/dnf) / [DNF5](https://github.com/rpm-software-management/dnf5) - Next-gen RPM package manager
+* [Pungi](https://forge.fedoraproject.org/pungi/pungi/): Use Pungi to "compose" Koji builds into highly customizable Yum/DNF repositories.
+* [Koji Tools](https://forge.fedoraproject.org/koji/koji-tools): Various utilities for Koji
 * [Kojiji](https://github.com/release-engineering/kojiji): Koji Java Interface
 * [txkoji](https://github.com/ktdreyer/txkoji): Async interface to Koji, using Twisted
 * [koji-hs](https://hackage.haskell.org/package/koji): Koji Haskell Interface

builder/Makefile

@@ -2,6 +2,7 @@ PYVER_MAJOR := $(shell $(PYTHON) -c 'import sys; print(".".join(sys.version.spli
 
 BINFILES = kojid
 LIBEXECFILES = mergerepos
+SBINDIR ?= /usr/sbin
 SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir)
 
 _default:
@@ -19,8 +20,8 @@ _install:
 		exit 1; \
 	fi
 
-	mkdir -p $(DESTDIR)/usr/sbin
-	install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin
+	mkdir -p $(DESTDIR)$(SBINDIR)
+	install -p -m 755 $(BINFILES) $(DESTDIR)$(SBINDIR)
 
 	@if [ "$(PYVER_MAJOR)" -lt 3 ] ; then \
 		mkdir -p $(DESTDIR)/usr/libexec/kojid; \

builder/koji.conf

@@ -0,0 +1,3 @@
+#Type Name            ID             GECOS                 Home directory Shell
+u     kojibuilder     -              "Koji builder"        /builddir      /usr/sbin/nologin
+m     kojibuilder     mock

builder/kojid.service

@@ -1,9 +1,9 @@
 [Unit]
 Description=Koji build server
-Documentation=https://docs.pagure.org/koji/server_howto/
+Documentation=https://koji.readthedocs.io/en/latest/server_howto.html
 
 # If the koji-hub is on this host, httpd.service is needed first
-After=network.target httpd.service
+After=network-online.target httpd.service chrony-wait.service
 
 [Service]
 ExecStart=/usr/sbin/kojid \

cli/koji

@@ -100,16 +100,18 @@ def load_plugins(plugin_paths):
                 register_plugin(tracker.load(name, path=path, reload=True))
 
 
-def get_options():
+def get_options(argv=None):
     """process options from command line and config file"""
 
+    if argv is None:
+        argv = sys.argv
     common_commands = ['build', 'help', 'download-build',
                        'latest-build', 'search', 'list-targets']
     usage = "%%prog [global-options] command [command-options-and-arguments]\n\n" \
             "Common commands: %s" % ', '.join(sorted(common_commands))
     parser = OptionParser(usage=usage)
     parser.disable_interspersed_args()
-    progname = os.path.basename(sys.argv[0]) or 'koji'
+    progname = os.path.basename(argv[0]) or 'koji'
     parser.__dict__['origin_format_help'] = parser.format_help
     parser.__dict__['format_help'] = lambda formatter=None: (
         "%(origin_format_help)s%(epilog)s" % ({
@@ -144,7 +146,7 @@ def get_options():
     parser.add_option("--plugin-paths", metavar='PATHS',
                       help="specify additional plugin paths (colon separated)")
     parser.add_option("--help-commands", action="store_true", default=False, help="list commands")
-    (options, args) = parser.parse_args()
+    (options, args) = parser.parse_args(argv[1:])
 
     # load local config
     try:

cli/koji_cli/lib.py

@@ -23,7 +23,6 @@ import koji
 from koji import parse_arches
 from koji import _  # noqa: F401
 from koji.util import md5_constructor, to_list
-from koji.xmlrpcplus import xmlrpc_client
 
 
 # for compatibility with plugins based on older version of lib
@@ -59,7 +58,7 @@ class TimeOption(optparse.Option):
                "parsed by dateutil.parser. e.g. \"2020-12-31 12:35\" or \"December 31st 12:35\""
 
 
-greetings = ('hello', 'hi', 'yo', "what's up", "g'day", 'back to work',
+greetings = ('hello', 'hi', 'yo', "what's up", "g'day", 'back to work', 'howdy',
              'bonjour',
              'hallo',
              'ciao',
@@ -208,7 +207,7 @@ class TaskWatcher(object):
         error = None
         try:
             self.session.getTaskResult(self.id)
-        except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e:
+        except (koji.xmlrpcplus.Fault, koji.GenericError) as e:
             error = e
         if error is None:
             # print("%s: complete" % self.str())
@@ -663,8 +662,8 @@ def download_rpm(build, rpm, topurl, sigkey=None, quiet=False, noprogress=False,
         error("Downloaded rpm %s is not valid rpm file, deleting" % path)
 
     # payload hash
-    sigmd5 = koji.get_header_fields(path, ['sigmd5'])['sigmd5']
-    if rpm['payloadhash'] != koji.hex_string(sigmd5):
+    hdr = koji.get_rpm_header(path)
+    if rpm['payloadhash'] != koji.get_rpm_ident(hdr):
         os.unlink(path)
         error("Downloaded rpm %s doesn't match db, deleting" % path)
 
@@ -922,6 +921,6 @@ def truncate_string(s, length=47):
 
 class DatetimeJSONEncoder(json.JSONEncoder):
     def default(self, o):
-        if isinstance(o, xmlrpc_client.DateTime):
+        if isinstance(o, koji.xmlrpcplus.DateTime):
             return str(o)
         return json.JSONEncoder.default(self, o)

devtools/check-api

@@ -0,0 +1,394 @@
+#!/usr/bin/env python3
+import inspect
+import json
+import os
+import os.path
+import subprocess
+import sys
+import types
+from collections import OrderedDict
+
+# import koji code from our checkout
+if os.path.exists(__file__):
+    kojitop = os.path.normpath(os.path.dirname(__file__) + '/..')
+else:
+    # e.g. <stdin>
+    kojitop = os.getcwd()
+sys.path.insert(0, kojitop)
+
+setup = kojitop + '/setup.py'
+try:
+    proc = subprocess.Popen([sys.executable, setup, '--version'], stdout=subprocess.PIPE)
+    (out, _) = proc.communicate()
+    VERSTR = out.decode().strip()
+    VERSION = tuple([int(x) for x in VERSTR.split('.')])
+    # we use this version to adapt our api scan a bit, but we record
+    # the version from koji lib in the api data
+except Exception:
+    VERSTR = '0.0.0'
+    VERSION = (0, 0, 0)
+
+import koji
+import koji.arch
+import koji.plugin
+import koji.tasks
+import koji.util
+import koji.xmlrpcplus
+
+if VERSION >= (1, 32, 0):
+    from kojihub import kojixmlrpc
+else:
+    from hub import kojixmlrpc
+    kojixmlrpc.load_scripts({'SCRIPT_FILENAME': kojixmlrpc.__file__})
+
+
+def main():
+
+    if len(sys.argv) == 1:
+        # just print the api
+        data = read_api()
+        print(json.dumps(data, indent=4))
+
+    elif len(sys.argv) == 2:
+        # read an api dump and compare to current
+        old = read_fn(sys.argv[1])
+        new = read_api()
+        compare(old, new)
+        report()
+
+    elif len(sys.argv) == 3:
+        # compare two api dumps (use '.' to read current)
+        old = read_fn(sys.argv[1])
+        new = read_fn(sys.argv[2])
+        compare(old, new)
+        report()
+
+    if ERRORS:
+        return 1
+    else:
+        return 0
+
+
+def read_fn(fn):
+    if fn == '.':
+        return read_api()
+    else:
+        with open(fn, 'rt') as fo:
+            return json.load(fo)
+
+
+def read_api():
+    data = OrderedDict()
+    # we use OrderedDicts so that our json output diffs nicely
+
+    data['version'] = list(getattr(koji, '__version_info__', (0, 0, 0)))
+
+    lib_modules = (koji, koji.arch, koji.util, koji.tasks, koji.xmlrpcplus, koji.plugin)
+    data['lib'] = OrderedDict()
+    for mod in lib_modules:
+        data['lib'][mod.__name__] = dump_module(mod)
+
+    # hub rpc calls (no plugins)
+    registry = kojixmlrpc.get_registry(opts={}, plugins=None)
+    data['rpc'] = OrderedDict()
+    for name in sorted(registry.funcs):
+        func = registry.funcs[name]
+        data['rpc'][name] = dump_func(func)
+
+    return data
+
+
+def dump_module(mod):
+    info = OrderedDict()
+    file = inspect.getsourcefile(mod)
+    names = [n for n in vars(mod) if not n.startswith('_')]
+    # TODO defer filtering _ to check code
+    names.sort()
+    for name in names:
+        value = getattr(mod, name)
+        vinfo = OrderedDict()
+        _type = str(type(value))
+        if '__future__' in _type:
+            continue
+        if type(value) in (types.BuiltinFunctionType, types.BuiltinMethodType):
+            vinfo['is_external'] = True
+            vinfo['is_builtin'] = True
+        vinfo['type'] = str(type(value))
+        info[name] = vinfo
+        if inspect.ismodule(value):
+            vinfo['is_module'] = True
+            continue
+        if inspect.isclass(value):
+            if 'builtins' in value.__module__:
+                vinfo['is_external'] = True
+                vinfo['is_builtin'] = True
+                continue
+            elif inspect.getsourcefile(value) != file:
+                vinfo['is_external'] = True
+                continue
+            vinfo['is_class'] = True
+            vinfo.update(dump_class(value))
+        elif inspect.isfunction(value):
+            if inspect.getsourcefile(value) != file:
+                vinfo['is_external'] = True
+                continue
+            vinfo['is_function'] = True
+            vinfo.update(dump_func(value))
+    return info
+
+
+def dump_func(func):
+    info = OrderedDict()
+    name = func.__name__
+    if inspect.isbuiltin(func):
+        info['is_builtin'] = True
+        # no need to dig deeper
+        return info
+    if inspect.isgeneratorfunction(func):
+        info['is_generator_function'] = True
+    sig = inspect.signature(func)
+    info['desc'] = '(%s)' % ', '.join([str(x) for x in sig.parameters.values()])
+    args = []
+    for pname, param in sig.parameters.items():
+        if param.default == param.empty:
+            arg = OrderedDict(name=pname)
+        else:
+            # default could be all sorts of things, repr is better than json here
+            default = repr(param.default)
+            arg = OrderedDict(name=pname, default=default)
+        args.append(arg)
+    info['args'] = args
+    spec = inspect.getfullargspec(func)
+    # FullArgSpec(args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations)
+    if spec.kwonlyargs or spec.kwonlydefaults:
+        # we don't use these
+        error(f'Found keyword-only args in rpc call {name}')
+    info['varargs'] = spec.varargs
+    info['varkw'] = spec.varkw
+    return info
+
+
+def dump_class(cls):
+    members = OrderedDict()
+    names = [n for n in vars(cls) if not n.startswith('_')]
+    names.sort()
+    for name in names:
+        value = getattr(cls, name)
+        vinfo = OrderedDict()
+        vinfo['type'] = str(type(value))
+        if inspect.isfunction(value):
+            vinfo['is_function'] = True
+            vinfo.update(dump_func(value))
+        members[name] = vinfo
+    return {'members': members}
+
+
+def compare(old, new):
+    top_keys = {'version', 'lib', 'rpc'}
+    if set(old) != top_keys:
+        error(f'Invalid keys in old data {list(old)}')
+    if set(new) != top_keys:
+        error(f'Invalid keys in new data {list(new)}')
+    compare_version(old['version'], new['version'])
+    compare_lib(old['lib'], new['lib'])
+    compare_rpc(old['rpc'], new['rpc'])
+
+
+NEW_MAJOR = False
+
+
+def compare_version(old, new):
+    global NEW_MAJOR
+    if old < new:
+        if old[:2] < new[:2]:
+            print(f'Major version increase {old} -> {new}')
+            NEW_MAJOR = True
+        else:
+            print(f'Minor version increase {old} -> {new}')
+    elif old > new:
+        error(f'Version DECREASED: {old} -> {new}')
+
+
+def compare_lib(old, new):
+    names1 = set(old)
+    names2 = set(new)
+    added = names2 - names1
+    dropped = names1 - names2
+    both = names1.intersection(names2)
+    for name in sorted(added):
+        warn(f'Added module: {name}')
+    for name in sorted(dropped):
+        error(f'Dropped module: {name}')
+    for name in sorted(both):
+        compare_mod(name, old[name], new[name])
+
+
+def compare_mod(mod, old, new):
+    names1 = set(old)
+    names2 = set(new)
+    added = names2 - names1
+    dropped = names1 - names2
+    both = names1.intersection(names2)
+
+    for name in sorted(added):
+        warn(f'Added module global: {mod}.{name}')
+    for name in sorted(dropped):
+        if old[name].get('is_module'):
+            error(f'Dropped module import {mod}.{name}')
+        elif old[name].get('is_external'):
+            error(f'Dropped imported global {mod}.{name}')
+        else:
+            # TODO figure out a way to distinguish deprecations
+            error(f'Dropped module global: {mod}.{name}')
+    for name in sorted(both):
+        compare_mod_global(mod, name, old[name], new[name])
+
+
+def compare_mod_global(mod, name, old, new):
+    if old['type'] != new['type']:
+        if old.get('is_external'):
+            # not from our code, so just warn
+            warn(f'{mod}.{name} (external) changed type: {old["type"]} -> {new["type"]}')
+        else:
+            error(f'{mod}.{name} changed type: {old["type"]} -> {new["type"]}')
+        # this prevents further comparison
+        return
+    desc = f'{mod}.{name}'
+    if old.get('is_function'):
+        compare_function(desc, old, new)
+    elif old.get('is_class'):
+        compare_class(desc, old, new)
+
+
+def compare_class(cls, old, new):
+    if not new.get('is_class'):
+        error(f'No longer a class: {cls}')
+        return
+    names1 = set(old['members'])
+    names2 = set(new['members'])
+    added = names2 - names1
+    dropped = names1 - names2
+    both = names1.intersection(names2)
+
+    for name in sorted(added):
+        warn(f'Added class variable: {cls}.{name}')
+    for name in sorted(dropped):
+        # TODO figure out a way to distinguish deprecations
+        error(f'Dropped class variable: {cls}.{name}')
+    for name in sorted(both):
+        compare_class_var(cls, name, old['members'][name], new['members'][name])
+
+
+def compare_class_var(cls, name, old, new):
+    if old['type'] != new['type']:
+        error(f'{cls}.{name} changed type: {old["type"]} -> {new["type"]}')
+        # this prevents further comparison
+        return
+    desc = f'{cls}.{name}'
+    if old.get('is_function'):
+        compare_function(desc, old, new)
+
+
+def compare_rpc(old, new):
+    names1 = set(old)
+    names2 = set(new)
+    added = names2 - names1
+    dropped = names1 - names2
+    both = names1.intersection(names2)
+
+    for name in sorted(added):
+        warn(f'Added RPC call: {name}')
+    for name in sorted(dropped):
+        # TODO figure out a way to distinguish deprecations
+        error(f'Dropped RPC call: {name}')
+    for name in sorted(both):
+        compare_rpc_call(name, old[name], new[name])
+
+
+def compare_rpc_call(name, old, new):
+    desc = f'rpc call {name}'
+    compare_function(desc, old, new)
+
+
+def compare_function(desc, old, new):
+    if old['desc'] != new['desc']:
+        warn(f'Signature changed for {desc}:\n  old: {old["desc"]}\n  new: {new["desc"]}')
+        # this is mainly for human readability, the code below will note errors
+
+    if not old['varargs']:
+        if new['varargs']:
+            warn(f'Added varargs for {desc}: {new["varargs"]}')
+            # not really a problem, but worth noting
+    elif not new['varargs']:
+        error(f'Dropped varargs for {desc}')
+    elif old['varargs'] != new['varargs']:
+        # not really a problem
+        warn(f'Renamed varargs for {desc}: {old["varargs"]} -> {new["varargs"]}')
+
+    if not old['varkw']:
+        if new['varkw']:
+            warn(f'Added varkw for {desc}: {new["varkw"]}')
+            # not really a problem, but worth noting
+    elif not new['varkw']:
+        error(f'Dropped varkw for {desc}')
+    elif old['varkw'] != new['varkw']:
+        # not really a problem
+        warn(f'Renamed varkw for {desc}: {old["varkw"]} -> {new["varkw"]}')
+
+    oargs = old['args']
+    nargs = new['args']
+
+    # arg counts
+    if len(nargs) < len(oargs):
+        error(f'Arg count reduced for {desc}')
+        # this will break rest of code, so we stop checking here
+        return
+
+    # note extended args
+    added = [a['name'] for a in nargs[len(oargs):]]
+    if added:
+        warn(f'Args extended for {desc}: {added!r}')
+
+    # details for overlap
+    for i, (oarg, narg) in enumerate(zip(oargs, nargs)):
+        if oarg['name'] != narg['name']:
+            error(f'Overlapping args do not match for {desc} arg {i}')
+            # this breaks rest of checks
+            return
+        if 'default' in oarg:
+            if 'default' not in narg:
+                error(f'Dropped default value for {desc} arg {oarg["name"]}')
+            elif narg['default'] != oarg['default']:
+                error(f'Changed default value for {desc} arg {oarg["name"]}: '
+                      f'{oarg["default"]} -> {narg["default"]}')
+        elif 'default' in narg:
+            warn(f'Added default value for {desc} arg {oarg["name"]}')
+
+
+WARNINGS = []
+ERRORS = []
+
+
+def warn(msg):
+    WARNINGS.append(msg)
+    sys.stderr.write(msg)
+    sys.stderr.write('\n')
+
+
+def error(msg):
+    ERRORS.append(msg)
+    sys.stderr.write('ERROR: ')
+    sys.stderr.write(msg)
+    sys.stderr.write('\n')
+
+
+def report():
+    print(f'Got {len(WARNINGS)} warnings and {len(ERRORS)} errors')
+
+
+if __name__ == '__main__':
+    rv = main()
+    sys.exit(rv)
+
+
+# the end

devtools/containers/Dockerfile.centos10

@@ -0,0 +1,17 @@
+FROM quay.io/centos/centos:stream10
+
+RUN \
+  dnf -y update --nodocs --setopt=install_weak_deps=False && \
+  dnf install -y --nodocs --setopt=install_weak_deps=False epel-release && \
+  dnf install -y --nodocs --setopt=install_weak_deps=False --enablerepo crb \
+  git \
+  gcc \
+  glibc-langpack-en \
+  krb5-devel \
+  python3-defusedxml \
+  python3-devel \
+  python3-librepo \
+  python3-setuptools \
+  python3-tox \
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.centos7

@@ -5,29 +5,17 @@ RUN \
     dnf \
     dnf-plugins-core \
     gcc \
-    git \
-    make \
     krb5-devel \
     libffi-devel \
-    openssl-devel \
-    pyOpenSSL \
-    python-cheetah \
     python-devel \
     python-librepo \
     python-requests \
-    redhat-rpm-config \
-    rpm-build \
-    rpm-python \
-    yum-utils && \
+    rpm-build  && \
   yum install -y epel-release && \
   yum install -y \
-    python-flake8 \
-    python-multilib \
+    python-defusedxml \
     python-pip \
-    python-psycopg2 \
-    python-qpid-proton \
-    python-requests-kerberos \
-    yumdownloader && \
+    python-psycopg2 && \
   yum clean all && \
   pip install -U 'pip==9.0.1' && \
   pip install -U tox

devtools/containers/Dockerfile.centos8

@@ -4,25 +4,13 @@ RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False epel-release && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
   python3-librepo \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
+  rpm-build && \
   dnf clean all

devtools/containers/Dockerfile.centos9

@@ -4,25 +4,14 @@ RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False epel-release && \
   dnf install -y --nodocs --setopt=install_weak_deps=False --enablerepo crb \
-  'dnf-command(download)' \
-  gcc \
   git \
-  glib2-devel \
+  gcc \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
   python3-librepo \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
+  rpm-build && \
   dnf clean all

devtools/containers/Dockerfile.f34

@@ -2,25 +2,13 @@ FROM registry.fedoraproject.org/fedora:34
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-librepo \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.f35

@@ -2,25 +2,13 @@ FROM registry.fedoraproject.org/fedora:35
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-librepo \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.f36

@@ -2,25 +2,13 @@ FROM registry.fedoraproject.org/fedora:36
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-librepo \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.f37

@@ -2,25 +2,13 @@ FROM registry.fedoraproject.org/fedora:37
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-librepo \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.f38

@@ -2,25 +2,13 @@ FROM registry.fedoraproject.org/fedora:38
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-librepo \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.f39

@@ -2,25 +2,13 @@ FROM registry.fedoraproject.org/fedora:39
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
-  git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-librepo \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
-  dnf clean all\
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.f43

@@ -0,0 +1,27 @@
+FROM registry.fedoraproject.org/fedora:42
+RUN \
+  dnf -y update --nodocs --setopt=install_weak_deps=False && \
+  dnf install -y --nodocs --setopt=install_weak_deps=False \
+  gcc \
+  git \
+  glibc-langpack-en \
+  krb5-devel \
+  python3-cffi \
+  python3-cheetah \
+  python3-dateutil \
+  python3-defusedxml \
+  python3-devel \
+  python3-dnf \
+  python3-jinja2 \
+  python3-librepo \
+  python3-multilib \
+  python3-pytest \
+  python3-requests \
+  python3-requests-gssapi \
+  python3-requests-mock \
+  python3-rpm \
+  python3-setuptools \
+  python3-six \
+  python3-tox \
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.f44

@@ -0,0 +1,28 @@
+FROM registry.fedoraproject.org/fedora:44
+RUN \
+  dnf -y update --nodocs --setopt=install_weak_deps=False && \
+  dnf install -y --nodocs --setopt=install_weak_deps=False \
+  flake8 \
+  gcc \
+  git \
+  glibc-langpack-en \
+  krb5-devel \
+  python3-cffi \
+  python3-cheetah \
+  python3-dateutil \
+  python3-defusedxml \
+  python3-devel \
+  python3-dnf \
+  python3-jinja2 \
+  python3-librepo \
+  python3-multilib \
+  python3-pytest \
+  python3-requests \
+  python3-requests-gssapi \
+  python3-requests-mock \
+  python3-rpm \
+  python3-setuptools \
+  python3-six \
+  python3-tox \
+  rpm-build && \
+  dnf clean all

devtools/containers/Dockerfile.rawhide

@@ -2,25 +2,14 @@ FROM registry.fedoraproject.org/fedora:rawhide
 RUN \
   dnf -y update --nodocs --setopt=install_weak_deps=False && \
   dnf install -y --nodocs --setopt=install_weak_deps=False \
-  'dnf-command(download)' \
   gcc \
   git \
-  glib2-devel \
   glibc-langpack-en \
   krb5-devel \
-  libffi-devel \
-  python3-librepo \
-  libxml2-devel \
-  make \
-  openssl-devel \
+  python3-defusedxml \
   python3-devel \
-  python3-pip \
-  python3-pytest \
-  python3-rpm \
+  python3-librepo \
+  python3-setuptools \
   python3-tox \
-  redhat-rpm-config \
-  rpm-build \
-  rpm-devel \
-  sqlite-devel \
-  yum-utils && \
+  rpm-build && \
   dnf clean all

devtools/containers/README.md

@@ -6,21 +6,31 @@ provided:
 
 * [`./Dockerfile.centos7`](./Dockerfile.centos7) CentOS 7, for testing with python2.7
 * [`./Dockerfile.centos8`](./Dockerfile.centos8) CentOS 8, for testing with python3.6
-* [`./Dockerfile.f32`](./Dockerfile.f32) Fedora 32, for testing with python3.8
-* [`./Dockerfile.f33`](./Dockerfile.f33) Fedora 33, for testing with python3.9
+* [`./Dockerfile.centos9`](./Dockerfile.centos8) CentOS 9, for testing with python3.9
+* [`./Dockerfile.f34`](...) Fedora 34, for testing with python3.9
+* [`./Dockerfile.f35`](./Dockerfile.f32) Fedora 35, for testing with python3.10
+* [`./Dockerfile.f36`](./Dockerfile.f32) Fedora 36, for testing with python3.10
+* [`./Dockerfile.f37`](./Dockerfile.f32) Fedora 37, for testing with python3.11
+* [`./Dockerfile.f38`](./Dockerfile.f32) Fedora 38, for testing with python3.11
+* [`./Dockerfile.f39`](./Dockerfile.f32) Fedora 39, for testing with python3.12
 * [`./Dockerfile.rawhide`](./Dockerfile.rawhie) Fedora Rawhide, for testing with python3.?
 
 To use them, taking fedora as an example:
 
-    docker build -t koji_test_fedora:32 --no-cache -f Dockerfile.f32
-    docker run --rm -v $PWD:/koji --name koji_test koji_test_fedora:32 bash -c "cd /koji && tox -e flake8,py3"
+    docker build -t koji_test_fedora:39 --no-cache -f Dockerfile.f39
+    docker run --rm -v $PWD:/koji --name koji_test koji_test_fedora:39 bash -c "cd /koji && tox -e flake8,py3,bandit"
 
-Or CentOS as an example:
+Or CentOS with py3 as an example:
 
     docker build -t koji_test_centos:8 --no-cache -f Dockerfile.centos8
-    docker run --rm -v $PWD:/koji --name koji_test koji_test_centos:8 bash -c "cd /koji && tox -e py2"
+    docker run --rm -v $PWD:/koji --name koji_test koji_test_centos:8 bash -c "cd /koji && tox -e flake8,py3,bandit"
+
+Or CentOS with py2 as an example:
+
+    docker build -t koji_test_centos:7 --no-cache -f Dockerfile.centos8
+    docker run --rm -v $PWD:/koji --name koji_test koji_test_centos:7 bash -c "cd /koji && tox -e py2"
 
 When running with Podman and SELinux enabled, use the "--security-opt
 label=disable" option:
 
-    podman run --rm -v $PWD:/koji --security-opt label=disable --name koji_test koji_test_fedora:32 bash -c "cd /koji && ls -l /koji && tox -e flake8,py3"
+    podman run --rm -v $PWD:/koji --security-opt label=disable --name koji_test koji_test_fedora:39 bash -c "cd /koji && ls -l /koji && tox -e flake8,py3,bandit"

devtools/convert-cheetah

@@ -0,0 +1,190 @@
+#!/usr/bin/python3
+
+import os.path
+import re
+import sys
+import tempfile
+from optparse import OptionParser
+
+
+"""
+Poorly convert some cheetah template code to jinja
+
+This is NOT a full or accurate conversion, it is a set of simple
+tranformations that reduce some of the manual work.
+
+Always review the changes.
+Always review the changes.
+"""
+
+
+def main():
+    global subs
+    global options
+    parser = OptionParser(usage="%prog <filename>")
+    parser.add_option('-w', '--write', action='store_true', help='write changes to file')
+    options, args = parser.parse_args()
+    options.args = args
+
+    if len(args) != 1:
+        error('Please specify one template')
+    fn = args[0]
+    handle_file(fn)
+
+
+def handle_file(fn):
+    outp = None
+    if options.write:
+        dirname = os.path.dirname(fn)
+        basename = os.path.basename(fn)
+        outfile = tempfile.NamedTemporaryFile(dir=dirname, mode='wt', prefix=f'_{basename}', delete=False)
+        with outfile as outp:
+            _handle_file(fn, outp)
+        os.replace(outfile.name, fn)
+        print(f'Wrote {fn}')
+    else:
+        _handle_file(fn, outp)
+
+
+def _handle_file(fn, outp):
+    with open(fn, 'rt') as fp:
+        for lineno, line in enumerate(fp):
+            line = handle_line(lineno, line, outp)
+            if line is not None and outp is not None:
+                outp.write(line)
+
+
+def handle_line(lineno, line, outp):
+    orig = line
+    matches = 0
+    skip = False
+    rules = list(SUBS)
+    while rules:
+        prog, repl = rules.pop(0)
+        last = line
+        if repl == SKIP:
+            m = prog.search(line)
+            if m:
+                print(f'{lineno}: Matched skip pattern {prog.pattern!r}')
+                print(line, end='')
+                skip = True
+                line = orig
+                break
+            continue
+        elif repl == DROP:
+            m = prog.search(line)
+            if m:
+                print(f'{lineno}: Matched DROP pattern {prog.pattern!r}')
+                print(line, end='')
+                return None
+        elif repl == BREAK:
+            m = prog.search(line)
+            if m:
+                print(f'{lineno}: Matched BREAK pattern {prog.pattern!r}')
+                print(line, end='')
+                break
+        elif isinstance(repl, Jump):
+            # forget remaing rules and use target rules from here
+            m = prog.search(line)
+            if m:
+                rules = list(repl.target)
+        else:
+            line, n = prog.subn(repl, line)
+            if n:
+                matches += n
+                print(f'{lineno}: Matched {prog.pattern!r} (count: {n})')
+    if matches:
+        print(f'Made {matches} substitutions for line {lineno}')
+        print(f'ORIG: {orig}', end='')
+        print(f' NEW: {line}')
+    return collapse(line)
+
+
+def rules(subs):
+    # compile subs
+    return [(re.compile(pat, flags), repl) for pat, repl, flags in subs]
+
+
+class Jump:
+    # jump to new set of substitutions
+    def __init__(self, target):
+        self.target = target
+
+
+SKIP = ('skip subs for this line',)
+DROP = ('drop line',)
+BREAK = ('stop subs checks for this line',)
+STATE = rules([
+    # subrules for some line statements
+    [r'[$]', '', 0],
+    [r'len\(([\w.$]+)\)', r'(\1 |length)', 0 ],
+])
+SUBS = rules([
+    # [pattern, replacement, flags]
+    [r'util.(toggleOrder|rowToggle|sortImage|passthrough_except|passthrough|authToken)\b', r'util.\g<1>2', 0],
+    [r'(#include .*)header.chtml', r'\1header2.chtml', 0],
+    [r'(#include .*)footer.chtml', r'\1footer2.chtml', 0],
+    [r'^#import', DROP, 0],
+    [r'^#from .* import', DROP, 0],
+    [r'^\s*#(if|for|elif|set)', Jump(STATE), 0],
+    [r'#end if', r'#endif', 0],
+    [r'#end for', r'#endfor', 0],
+    [r'[(][$]self, ', r'(', 0 ],
+    [r'\([$]self\)', r'()', 0 ],
+    [r'len\(([\w.$]+)\)', r'(\1 |length)', 0 ],
+    [r'[$](([\w.]+)[(][^()]*[)])', r'{{ \1 }}', 0 ],
+    [r'${\s*([^{}]+)\s*}', r'{{ \1 }}', 0 ],
+    [r'#echo ([^#]+)#', r'{{ \1 }}', 0 ],
+    [r'#if ([^#]+) then ([^#]+) else ([^#]+)\s*#', r'{{ \2 if \1 else \3 }}', 0 ],
+    [r'''[$]([\w.]+)\['(\w+)'\]''', r'{{ \1.\2 }}', 0],
+    [r'''[$]([\w.]+)\["(\w+)"\]''', r'{{ \1.\2 }}', 0],
+    [r'[$]([\w.]+)[([]', SKIP, 0],
+    [r'^(\s*)#attr ', r'\1#set ', 0],
+    [r'^\s*#', BREAK, 0],
+    [r'[$]([\w.]+)', r'{{ \1 }}', 0],
+])
+
+
+def error(msg):
+    print(msg)
+    sys.exit(1)
+
+
+BRACES = re.compile(r'({{ | }})')
+
+def collapse(line):
+    """Collapse nested double braces"""
+    tokens = BRACES.split(line)
+
+    depth = 0
+    tokens2 = []
+    for tok in tokens:
+        if tok == '{{ ':
+            # only keep braces at the outer layer
+            if depth == 0:
+                tokens2.append(tok)
+            depth += 1
+        elif tok == ' }}':
+            depth -= 1
+            if depth < 0:
+                warning("Brace  mismatch. Can't collapse")
+                break
+            elif depth == 0:
+                # only keep braces at the outer layer
+                tokens2.append(tok)
+        else:
+            # keep everything else
+            tokens2.append(tok)
+
+    if depth < 0:
+        warning('Unexpected }}')
+        return line
+    elif depth > 0:
+        warning('Missing }}')
+        return line
+    else:
+        return ''.join(tokens2)
+
+
+if __name__ == '__main__':
+    main()

devtools/fakehub

@@ -14,7 +14,7 @@ from urllib.parse import quote
 
 sys.path.insert(0, os.getcwd())
 import koji
-from kojihub import auth, kojixmlrpc, kojihub
+from kojihub import auth, kojixmlrpc, kojihub, db
 from koji.context import context
 import koji.xmlrpcplus
 
@@ -35,6 +35,17 @@ def handle_rpc(self, environ):
         raise
 
 
+def override_load_config(opts):
+    original_load_config = kojixmlrpc.load_config
+
+    def my_load_config(environ):
+        oldopts = original_load_config(environ)
+        oldopts.update(opts)
+        return oldopts
+
+    kojixmlrpc.load_config = my_load_config
+
+
 # Fake session for authenticated calls
 class FakeSession(auth.Session):
 
@@ -104,6 +115,10 @@ def get_options():
     parser.add_option('--user', '-u', help='execute as user')
     parser.add_option('--exclusive', '-x', action='store_true',
                       help='emulate an exclusive session')
+    parser.add_option('-n', '--no-commit', action='store_true',
+                      help='skip commit')
+    parser.add_option('-o', '--config-option', help='override config option',
+                      action='append', metavar='NAME=VALUE')
     opts, args = parser.parse_args()
 
     # parse request from args
@@ -113,12 +128,21 @@ def get_options():
     for s in args[1:]:
         if '=' in s:
             k, v = s.split('=', 1)
-            v = nice_literal(v)
-            kwargs[k] = v
-        else:
-            callargs.append(nice_literal(s))
+            if k.isidentifier():
+                v = nice_literal(v)
+                kwargs[k] = v
+                continue
+        # else
+        callargs.append(nice_literal(s))
     callargs = koji.encode_args(*callargs, **kwargs)
     opts.request = koji.xmlrpcplus.dumps(callargs, method, allow_none=1)
+    if opts.config_option:
+        overrides = {}
+        for s in opts.config_option:
+            k, v = s.split('=', 1)
+            v = nice_literal(v)
+            overrides[k] = v
+        opts.config_option = overrides
 
     return opts
 
@@ -147,10 +171,18 @@ def set_config(environ):
         environ['koji.hub.ConfigDir'] = lconfigd
 
 
+def skip_commit(cnx):
+    print('Skipping commit')
+
+
 def main():
     options = get_options()
     if options.pdb:
         kojixmlrpc.ModXMLRPCRequestHandler.handle_rpc = handle_rpc
+    if options.no_commit:
+        db.DBWrapper.commit = skip_commit
+    if options.config_option:
+        override_load_config(options.config_option)
 
     environ = {}
     environ['SCRIPT_FILENAME'] = kojixmlrpc.__file__

devtools/fakepolicy

@@ -0,0 +1,632 @@
+#!/usr/bin/python3
+
+from __future__ import absolute_import, print_function
+
+import ast
+import argparse
+import os
+import os.path
+import pprint
+import sys
+
+from pprint import pformat
+from unittest import mock
+
+sys.path.insert(0, os.getcwd())
+import koji
+from kojihub import auth, kojixmlrpc, kojihub, db
+from koji.context import context
+from koji.util import dslice, extract_build_task
+from koji.tasks import parse_task_params
+from koji.daemon import SCM
+import koji.xmlrpcplus
+
+
+"""
+This is a tool for developers to test out policy code.
+
+It works by mocking hub code to make calls to a remote hub rather than using
+direct db access as hub code normally does. This allows simulating a policy
+on the client side, but using the data on the remote hub.
+
+This code is based partially on fakehub
+"""
+
+
+# Fake session for simulating different auth
+class FakeSession(auth.Session):
+
+    def __init__(self, user=None, exclusive=False, session=None):
+        if user:
+            user = session.getUser(user, strict=True)
+            self.logged_in = True
+            self.user_id = user['id']
+            self.authtype = koji.AUTHTYPES['GSSAPI']
+            self.user_data = user
+        else:
+            self.logged_in = False
+            self.user_id = None
+            self.user_data = {}
+        self.exclusive = exclusive
+        self.id = 1
+        self.hostip = '127.0.0.1'
+        self.master = None
+        self.callnum = 1
+        self.message = 'THIS IS A FAKE SESSION'
+        self.session_data = {'msg': 'this is a fake session'}
+        self._perms = None
+        self._groups = None
+        self._host_id = ''
+
+
+def nice_literal(value):
+    try:
+        return ast.literal_eval(value)
+    except (ValueError, SyntaxError):
+        return value
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Simulate hub policy tests')
+    parser.add_argument('--pdb', action='store_true',
+                        help='drop into pdb on error')
+    parser.add_argument('-p', '--profile', default='koji', help='pick a profile')
+    parser.add_argument('--user', '-u', help='execute as user')
+    parser.add_argument('--exclusive', '-x', action='store_true',
+                        help='emulate an exclusive session')
+    parser.add_argument('-o', '--config-option', help='override config option',
+                        action='append', metavar='NAME=VALUE')
+    parser.add_argument('--config', '-c', help='config file')
+    parser.add_argument('--configdir', '-d', help='config dir')
+    parser.add_argument('--policyfile', '-f', help='policy file')
+    parser.add_argument('--eval', dest='mode', action='store_const', const='eval', default='eval')
+    parser.add_argument('--assert', dest='mode', action='store_const', const='assert')
+    parser.add_argument('--check', dest='mode', action='store_const', const='check')
+    subparsers = parser.add_subparsers(required=True, dest='command',
+                                       help='Available subcommands')
+
+    ## subcommands ##
+    subp = subparsers.add_parser('manual', help='manually specify policy and data')
+    subp.add_argument('policy', help='select policy')
+    subp.add_argument('data', help='provide policy data')
+    subp.set_defaults(func=manual_params)
+
+    subp = subparsers.add_parser('make-task', help='simulate policies for make_task')
+    subp.add_argument('--policy', help='select policy', default='channel')
+    subp.add_argument('--req-channel', help='simulate a requested channel', metavar='CHANNEL')
+    subp.add_argument('task', type=int, help='select task')
+    subp.set_defaults(func=make_task_params)
+
+    subp = subparsers.add_parser('promote-build',
+                                 help='simulate draft_promotion policy for promote_build')
+    subp.add_argument('build', help='select build')
+    subp.set_defaults(func=promote_build_params)
+
+    subp = subparsers.add_parser('apply-volume', help='simulate volume policy for apply_volume')
+    subp.add_argument('build', help='select build')
+    subp.set_defaults(func=apply_volume_params)
+
+    subp = subparsers.add_parser('tag-build', help='simulate policy for tagging a build')
+    subp.add_argument('tag', help='select tag')
+    subp.add_argument('build', help='select build')
+    subp.add_argument('--fromtag', help='select from tag (i.e. move)')
+    subp.add_argument('--by-tag-subtask', action='store_true', help='mark operation as by_tag_subtask')
+    subp.set_defaults(func=tag_build_params)
+
+    subp = subparsers.add_parser('move-build', help='simulate policy for moving a build')
+    subp.add_argument('fromtag', help='select from tag')
+    subp.add_argument('tag', help='select destination tag')
+    subp.add_argument('build', help='select build')
+    subp.set_defaults(func=tag_build_params)  # this handler does double duty
+
+    subp = subparsers.add_parser('untag-build', help='simulate policy for untagging a build')
+    subp.add_argument('tag', help='select tag')
+    subp.add_argument('build', help='select build')
+    subp.set_defaults(func=untag_build_params)
+
+    subp = subparsers.add_parser('add-pkg', help='simulate policy for adding a package')
+    subp.add_argument('tag', help='select tag')
+    subp.add_argument('package', help='select package')
+    subp.add_argument('--action', help='select action', default='add')
+    subp.add_argument('--force', help='select force setting', action='store_true', default=False)
+    subp.set_defaults(func=add_pkg_params)
+
+    subp = subparsers.add_parser('cg-import', help='simulate policy for cg import')
+    subp.add_argument('-b', '--build', help='fetch data from existing build')
+    subp.add_argument('-m', '--metadata', help='fetch data from json file')
+    subp.add_argument('--metadata-only', help='', action='store_true', default=False)
+    subp.set_defaults(func=cg_import_params)
+
+    subp = subparsers.add_parser('build-from-scm', help='simulate build_from_scm policy check')
+    subp.add_argument('-t', '--task', type=int, help='fetch data from existing task')
+    subp.add_argument('-u', '--url', help='specify scm url')
+    subp.add_argument('-o', '--owner', help='specify task owner')
+    subp.add_argument('-c', '--channel', help='specify task channel')
+    subp.add_argument('--scratch', action='store_true', help='specify scratch=True')
+    subp.add_argument('--no-scratch', action='store_false', dest='scratch', help='specify scratch=False')
+    subp.set_defaults(func=build_from_scm_params)
+    ## end subcommands ##
+
+    args = parser.parse_args()
+
+    if args.config_option:
+        overrides = {}
+        for s in args.config_option:
+            k, v = s.split('=', 1)
+            v = nice_literal(v)
+            overrides[k] = v
+        args.config_option = overrides
+    else:
+        args.config_option = {}
+
+    return args
+
+
+def set_config(args, environ):
+    lconfig = "%s/devtools/fakehub.conf" % os.getcwd()
+    lconfigd = "%s/devtools/fakehub.conf.d" % os.getcwd()
+    if os.path.exists(lconfig) or os.path.exists(lconfigd):
+        environ['koji.hub.ConfigFile'] = lconfig
+        environ['koji.hub.ConfigDir'] = lconfigd
+
+
+def update_policy(pfile, opts):
+    """Load policy (only) from a config file, updating existing"""
+    policy = opts.setdefault('policy', {})
+    config = koji.read_config_files([(pfile, True)], raw=True)
+    if not config.has_section('policy'):
+        raise Exception('No policy found in %s' % pfile)
+    updates = dict(config.items('policy'))
+    policy.update(updates)
+
+
+def skip_commit(cnx):
+    print('Skipping commit')
+
+
+def main():
+    global koji  # we replace it with a profile module
+    args = get_args()
+    db.DBWrapper.commit = skip_commit
+
+    # set up session for querying remote
+    koji = koji.get_profile_module(args.profile)
+    session_opts = koji.grab_session_options(koji.config)
+    session = koji.ClientSession(koji.config.server, session_opts)
+    # no auth - we're only reading
+
+    ## Simulate key parts of server_setup ##
+    # 1. load config
+    environ = {}
+    set_config(args, environ)  # XXX?
+    if args.configdir:
+        environ['koji.hub.ConfigDir'] = args.configdir
+    if args.config:
+        environ['koji.hub.ConfigFile'] = args.config
+    opts = kojixmlrpc.load_config(environ)
+    opts.update(args.config_option)
+    if args.policyfile:
+        update_policy(args.policyfile, opts)
+
+    # 2. logging
+    kojixmlrpc.setup_logging1()
+    kojixmlrpc.setup_logging2(opts)
+    # TODO
+
+    # 3. plugins
+    plugins = kojixmlrpc.load_plugins(opts)
+
+    # 4. load policies
+    policy_index = kojixmlrpc.get_policy(opts, plugins)
+    ## end server_setup ##
+
+    # set up context as if for a call handler
+    context._threadclear()
+    context.commit_pending = False
+    context.opts = opts
+    context.policy = policy_index
+    cnx = mock.MagicMock()
+    cnx.cursor.side_effect = Exception('db access disabled')
+    cnx.set_session.side_effect = Exception('db access disabled')
+    context.cnx = cnx
+
+    if args.user:
+        context.session = FakeSession(args.user, args.exclusive, session)
+    else:
+        context.session = FakeSession()
+
+    # mock key points for data access by policy code
+    do_mocks(session, plugins)
+
+    policy, policy_data = args.func(args, session)
+    # ^ parts of this rely on the mocks
+
+    # run the policy
+    print(f'Testing policy {policy} with data:\n{pformat(policy_data)}')
+    handler = None
+    params = (policy, policy_data)
+    kwargs = {}
+    if args.mode == 'eval':
+        handler = kojihub.eval_policy
+    elif args.mode == 'assert':
+        handler = kojihub.assert_policy
+    elif args.mode == 'check':
+        handler = kojihub.check_policy
+    else:
+        raise Exception('Invalid mode: %s' % args.mode)
+    try:
+        result = handler(*params, **kwargs)
+    except Exception:
+        if not args.pdb:
+            raise
+        import pdb
+        import traceback
+        etype, e, tb = sys.exc_info()
+        traceback.print_exc()
+        pdb.post_mortem(tb)
+
+    print(f'RESULT: {result}')
+
+
+def manual_params(args, session):
+    data = nice_literal(args.data)
+    return args.policy, data
+
+
+def make_task_params(args, session):
+    # simulate make_task policy data
+    if args.policy not in ('channel', 'priority'):
+        print('The make_task subcommand is meant for use with the channel or priority policies')
+    task_id = args.task
+    tinfo = session.getTaskInfo(task_id, strict=True)
+    taskargs = session.getTaskRequest(task_id)
+    data = dslice(tinfo, ['method', 'arch', 'parent', 'label', 'owner'])
+    data['user_id'] = tinfo['owner']
+    if args.req_channel:
+        data['req_channel'] = args.req_channel
+    if tinfo['parent']:
+        data.update(kojihub.policy_data_from_task(tinfo['parent'], recurse=True))
+    data.update(kojihub.policy_data_from_task_args(tinfo['method'], taskargs))
+    # ^ this relies on the mocks to work
+    return args.policy, data
+
+
+def promote_build_params(args, session):
+    # simulate promote_build policy data
+    policy = 'draft_promotion'
+    bld = args.build
+    if bld.isdigit:
+        bld = int(bld)
+    binfo = session.getBuild(bld, strict=True)
+    if binfo['draft']:
+        target_release = koji.parse_target_release(binfo['release'])
+    else:
+        target_release = binfo['release']
+    data = {
+        'build': binfo['id'],
+        'target_release': target_release
+    }
+    return policy, data
+
+
+def apply_volume_params(args, session):
+    # simulate apply_volume_policy policy data
+    policy = 'volume'
+    bld = args.build
+    if bld.isdigit():
+        bld = int(bld)
+    binfo = session.getBuild(bld, strict=True)
+    data = {'build': binfo}
+    # the hub's apply_volume_policy() adds in task data if present
+    task_id = extract_build_task(binfo)
+    if task_id:
+        tinfo = session.getTaskInfo(task_id)
+        args = session.getTaskRequest(task_id)
+        data.update(kojihub.policy_data_from_task_args(tinfo['method'], args))
+    return policy, data
+
+
+def tag_build_params(args, session):
+    # simulate policy for tagging a build
+    policy = 'tag'
+    operation = 'tag'
+
+    tag = args.tag
+    if tag.isdigit():
+        tag = int(tag)
+    tinfo = session.getTag(tag, strict=True)
+
+    bld = args.build
+    if bld.isdigit():
+        bld = int(bld)
+    binfo = session.getBuild(bld, strict=True)
+
+    fromtag = args.fromtag
+    if fromtag:
+        if fromtag.isdigit():
+            fromtag = int(fromtag)
+        fromtag = session.getTag(fromtag, strict=True)['id']
+        operation = 'move'
+
+    # kojihub is a bit inconsistent about the data passed (e.g. id vs name vs dict)
+    # we mimic the main tagBuild handler
+    data = {'tag': tinfo['id'], 'build': binfo['id'], 'fromtag': fromtag, 'operation': operation}
+
+    if args.by_tag_subtask:
+        data['by_tag_subtask'] = True
+
+    return policy, data
+
+
+def untag_build_params(args, session):
+    # simulate policy for untagging a build
+    policy = 'tag'
+
+    tag = args.tag
+    if tag.isdigit():
+        tag = int(tag)
+    tinfo = session.getTag(tag, strict=True)
+
+    bld = args.build
+    if bld.isdigit():
+        bld = int(bld)
+    binfo = session.getBuild(bld, strict=True)
+
+    data = {'tag': None, 'build': binfo['id'], 'fromtag': tinfo['id'], 'operation': 'untag'}
+
+    return policy, data
+
+
+def add_pkg_params(args, session):
+    # simulate policy for adding a package list entry
+    policy = 'package_list'
+
+    tag = args.tag
+    if tag.isdigit():
+        tag = int(tag)
+    tinfo = session.getTag(tag, strict=True)
+
+    package = args.package
+    force = args.force
+    action = args.action
+
+    # kojihub is a bit inconsistent about the package field format
+    # we opt for the string passed in, as happens with the add-pkg command
+    data = {'tag': tinfo['id'], 'action': action, 'package': package, 'force': force}
+
+    return policy, data
+
+
+def cg_import_params(args, session):
+    # simulate policy for cg imports
+    policy = 'cg_import'
+    data = {}
+
+    if args.build:
+        # pull metadata from existing build
+        if args.build.isdigit():
+            binfo = session.getBuild(int(args.build), strict=True)
+        else:
+            binfo = session.getBuild(args.build, strict=True)
+        for key in ('version', 'release', 'source', 'draft'):
+            data[key] = binfo[key]
+        data['package'] = binfo['name']
+        data['btypes'] = list(session.getBuildType(binfo['id']))
+        data['cg_list'] = [binfo['cg_id']]
+        data['metadata_only'] = False
+
+    elif args.metadata:
+        metadata = koji.load_json(args.metadata)
+        build = metadata['build']
+        for key in ('version', 'release'):
+            data[key] = build[key]
+        for key in ('source', 'draft'):
+            data[key] = build.get(key)
+        data['package'] = build['name']
+
+        # get btypes
+        typeinfo = build.get('extra', {}).get('typeinfo', {})
+        for btype in ('maven', 'win', 'image'):
+            # legacy types can also be at top level of extra
+            if btype in build['extra']:
+                typeinfo.setdefault(btype, build['extra'][btype])
+        data['btypes'] = list(typeinfo)
+
+        # get cg id
+        cg_id = None
+        for brdata in metadata['buildroots']:
+            cginfo = brdata['content_generator']
+            cg = kojihub.lookup_name('content_generator', cginfo['name'], strict=True)
+            # ^ this relies on our mocks
+            # just take the first one. we're not validating
+            cg_id = cg['id']
+            break
+        data['cg_list'] = [cg_id]
+
+        # check for metadata-only flag
+        data['metadata_only'] = False
+        for fileinfo in metadata['output']:
+            if fileinfo.get('metadata_only', False):
+                data['metadata_only'] = True
+                break
+
+    if args.metadata_only:
+        data['metadata_only'] = True
+
+    return policy, data
+
+
+def build_from_scm_params(args, session):
+    # simulate policy for adding a package list entry
+    policy = 'build_from_scm'
+
+    task_params = None
+    if args.task:
+        task = args.task  # should be int
+        tinfo = session.getTaskInfo(task, request=True, strict=False)
+        if tinfo['method'] != 'buildSRPMFromSCM':
+            # TODO extend
+            raise NotImplementedError('Only buildSRPMFrom SCM is supported yet')
+        task_params = parse_task_params(tinfo['method'], tinfo['request'])
+
+    if args.url:
+        url = args.url
+    elif task_params:
+        url = task_params['url']
+    else:
+        raise Exception('No url or task specified')
+
+    extra = {}
+    if task_params:
+        extra['user_id'] = tinfo['owner']
+        extra['channel'] = session.getChannel(tinfo['channel_id'], strict=True)
+        extra['scratch'] = task_params.get('opts', {}).get('scratch')
+    if args.scratch is not None:
+        extra['scratch'] = args.scratch
+    if args.owner:
+        extra['user_id'] = session.getUser(args.owner, strict=True)['id']
+    if args.channel:
+        extra['channel'] = session.getChannel(args.channel, strict=True)
+
+    # generate policy data similar to assert_allowed_by_policy
+    scm = SCM(url)
+    scminfo = scm.get_info()
+    data = {}
+    for k in scminfo:
+        v = scminfo[k]
+        if not k.startswith('scm_'):
+            k = 'scm_' + k
+        data[k] = v
+    # and add the normal extra data
+    data.update(extra)
+
+    return policy, data
+
+
+def do_mocks(session, plugins):
+    # a number of internal functions are directly exported
+    simple_exports = [
+        # [name, export_name]
+        ['get_user', 'getUser'],
+        ['get_build', 'getBuild'],
+        ['list_rpms', 'listRPMs'],
+        ['list_archives', 'listArchives'],
+        ['get_buildroot', 'getBuildroot'],
+        ['get_tag', 'getTag'],
+        ['list_tags', 'listTags'],
+        ['get_build_target', 'getBuildTarget'],
+        ['get_build_type', 'getBuildType'],
+        ['readFullInheritance', 'getFullInheritance'],  # ??
+        ['lookup_package', 'getPackage'],
+    ]
+
+    # generator function for wrappers
+    def wrap(export):
+        def wrapper(*a, **kw):
+            return session._callMethod(export, a, kw)
+        return wrapper
+
+    def patchall(name, func):
+        """Apply patch in kojihub and any plugins"""
+        mock.patch('kojihub.kojihub.%s' % name, new=func).start()
+        # also patch in any plugins that import
+        for pname in plugins.plugins:
+            pmod = plugins.plugins[pname]
+            if hasattr(pmod, name):
+                # e.g. the plugin has done from kojihub import foo
+                mock.patch.object(pmod, name, new=func).start()
+
+    # mock the simple exports
+    for name, export in simple_exports:
+        func = wrap(export)
+        patchall(name, func)
+
+    def get_user_perms(user_id, with_groups=True, inheritance_data=False):
+        if inheritance_data:
+            raise NotImplementedError('inheritance_data')
+        return session.getUserPerms(user_id, with_groups)
+    patchall('get_user_perms', get_user_perms)
+
+    def get_user_groups(user_id):
+        # internal returns a dictionary group_id:name
+        # export returns a list of id/name dicts
+        groups = session.getUserGroups(user_id)
+        return {g['id']: g['name'] for g in groups}
+    patchall('get_user_groups', get_user_groups)
+
+    def lookup_name(table, info, strict=False, create=False):
+        # NOT exported, but we can fake some cases
+        if create:
+            raise NotImplementedError('lookup_name with create')
+        if table == 'volume':
+            return session.getVolume(info, strict)
+        elif table == 'content_generator':
+            cgs = session.listCGs()
+            # cgs is a dictionary indexed by cg name
+            for cg in cgs:
+                cginfo = cgs[cg]
+                if info in (cg, cginfo['id']):
+                    return {'id': cginfo['id'], 'name': cg}
+        elif table == 'package':
+            return session.getPackage(info, strict)
+        elif table == 'tag':
+            taginfo = session.getTag(info, strict, event="auto")
+            return dslice(taginfo, ['id', 'name'])
+        elif table == 'build_target':
+            tgt = session.getBuildTarget(info, event=None, strict=False)
+            if tgt:
+                return dslice(tgt, ['id', 'name'])
+                return tgt
+            # the above will not find deleted targets, fall back to history query
+            hist = session.queryHistory(tables=['build_target_config'], build_target=info,
+                                        queryOpts={'limit': 1, 'order': '-create_event'})
+            hist = hist['build_target_config']
+            if not hist:
+                if strict:
+                    raise koji.GenericError('No such target: %s' % info)
+                else:
+                    return None
+            else:
+                # pull out the name and id
+                return {'id': hist[0]['build_target_id'], 'name': hist[0]['build_target.name']}
+        else:
+            raise NotImplementedError(f'lookup_name for {table}')
+    patchall('lookup_name', lookup_name)
+
+    # handle task.getInfo
+    class FakeTask:
+
+        def __init__(self, id):
+            self.id = int(id)
+
+        def getInfo(self, strict=True, request=False):
+            return session.getTaskInfo(self.id, request, strict)
+
+    patchall('Task', FakeTask)
+
+    # handle BuildRoot.getList
+    class FakeBuildRoot:
+
+        def __init__(self, id):
+            # we require the id value
+            self.load(id)
+
+        def load(self, id):
+            data = session.getBuildroot(id)
+            self.id = id
+            self.data = data
+
+        def getList(self):
+            data = session.listRPMs(componentBuildrootID=self.id)
+            for row in data:
+                row['rpm_id'] = row['id']
+            return data
+
+    patchall('BuildRoot', FakeBuildRoot)
+
+
+if __name__ == '__main__':
+    main()
+
+
+# the end

devtools/fakeweb

@@ -2,9 +2,11 @@
 
 from __future__ import absolute_import, print_function
 
+import ast
 import mimetypes
 import os
 import os.path
+import optparse
 import pprint
 import sys
 from urllib.parse import quote
@@ -17,6 +19,7 @@ sys.path.insert(0, CWD)
 sys.path.insert(1, os.path.join(CWD, 'www/lib'))
 sys.path.insert(1, os.path.join(CWD, 'www/kojiweb'))
 import wsgi_publisher
+import index as kojiweb_handlers
 
 
 def get_url(environ):
@@ -111,6 +114,7 @@ def application(environ, start_response):
     # provide some needed info
     environ['SCRIPT_FILENAME'] = wsgi_publisher.__file__
     environ['REQUEST_URI'] = get_url(environ)
+    environ['REQUEST_SCHEME'] = environ['wsgi.url_scheme']
     set_config(environ)
     if FIRST:
         pprint.pprint(environ)
@@ -121,11 +125,87 @@ def application(environ, start_response):
     return wsgi_publisher.application(environ, start_response)
 
 
+def nice_literal(value):
+    try:
+        return ast.literal_eval(value)
+    except (ValueError, SyntaxError):
+        return value
+
+
+def get_options():
+    parser = optparse.OptionParser(usage='%prog [options]')
+    # parser.add_option('--pdb', action='store_true',
+    #                   help='drop into pdb on error')
+    parser.add_option('--user', '-u', help='fake login as user')
+    parser.add_option('--time', '-t', type="float", help='mock time as value')
+    parser.add_option('-o', '--config-option', help='override config option',
+                      action='append', metavar='NAME=VALUE')
+    opts, args = parser.parse_args()
+
+    if args:
+        parser.error('This command takes no args, just options')
+
+    if opts.config_option:
+        overrides = {}
+        for s in opts.config_option:
+            k, v = s.split('=', 1)
+            v = nice_literal(v)
+            overrides[k] = v
+        opts.config_option = overrides
+
+    return opts
+
+
+def override_load_config(opts):
+    original_load_config = wsgi_publisher.Dispatcher.load_config
+
+    def my_load_config(_self, environ):
+        oldopts = original_load_config(_self, environ)
+        oldopts.update(opts)
+        _self.options = oldopts
+        return oldopts
+
+    wsgi_publisher.Dispatcher.load_config = my_load_config
+
+
+def fake_login(user):
+    original_assertLogin = kojiweb_handlers._assertLogin
+    original_getServer = kojiweb_handlers._getServer
+
+    def my_assertLogin(environ):
+        pass
+
+    def my_getServer(environ):
+        session = original_getServer(environ)
+        environ['koji.currentUser'] = session.getUser(user)
+        return session
+
+    kojiweb_handlers._assertLogin = my_assertLogin
+    kojiweb_handlers._getServer = my_getServer
+
+
 def main():
+    options = get_options()
+    if options.config_option:
+        override_load_config(options.config_option)
+    if options.user:
+        fake_login(options.user)
+    if options.time:
+        from unittest import mock
+        import datetime
+        dt = datetime.datetime.fromtimestamp(options.time)
+        mock.patch('time.time', return_value=options.time).start()
+        # mocking datetime is tricky, can't mock the c object directly
+        # and can't mock the datetime class globally without breaking other code
+        # so we just mock the handlers import of it
+        my_datetime = mock.patch.object(kojiweb_handlers.kojiweb.util, 'datetime', wraps=datetime).start()
+        my_datetime.datetime.now.return_value = dt
+
     # koji.add_file_logger('koji', 'fakeweb.log')
     httpd = make_server('', 8000, application)
     print("Serving kojiweb on http://localhost:8000 ...")
     httpd.serve_forever()
 
+
 if __name__ == '__main__':
     main()

docs/source/CVEs/CVE-2024-9427.rst

@@ -0,0 +1,42 @@
+=============
+CVE-2024-9427
+=============
+
+New XSS attack on kojiweb
+
+Summary
+-------
+
+An unsanitized input allows for an XSS attack. Javascript code from a malicious
+link could be reflected in the resulting web page. At present, we do not
+believe that this can be used to submit an action or make a change in Koji due
+to existing XSS protections in the code. Even so, this is a serious issue and
+we recommend applying this update promptly.
+
+Bug fix
+-------
+
+We are releasing updates for affected versions of Koji from within the
+past year.
+The following releases all contain the fix:
+
+- 1.35.1
+- 1.34.3
+- 1.33.2
+
+Anyone using a Koji version older than a year should update to a more
+current version as soon as possible.
+
+For users who have customized their Koji code, we recommend rebasing your work
+onto the appropriate update release. Please see Koji
+`issue #4204 <https://pagure.io/koji/issue/4204>`_ for the code details.
+
+As with all changes to web code, you must restart httpd for the changes to
+take effect.
+
+Links
+-----
+
+Fixed versions can be found at our releases page:
+
+    https://pagure.io/koji/releases

docs/source/CVEs/CVEs.rst

@@ -5,6 +5,7 @@ Koji CVEs
 .. toctree::
     :titlesonly:
 
+    CVE-2024-9427
     CVE-2020-15856
     CVE-2019-17109
     CVE-2018-1002161

docs/source/HOWTO.rst

@@ -199,7 +199,7 @@ Koji is comprised of several components:
    koji-hub via XML-RPC.
 
 -  **koji-web** is a set of scripts that run in mod\_wsgi and use the
-   Cheetah templating engine to provide an web interface to Koji.
+   Jinja templating engine to provide an web interface to Koji.
    koji-web exposes a lot of information and also provides a means for
    certain operations, such as cancelling builds.
 -  **koji** is a CLI written in Python that provides many hooks into Koji.

docs/source/configuring_jenkins.rst

@@ -1,134 +0,0 @@
-Unit tests in Fedora's Jenkins
-==============================
-
-We're using CentOS's `Jenkins
-<https://jenkins-fedora-infra.apps.ocp.ci.centos.org/job/koji>`_ infrastructure
-for automatically running unit tests for new commits in master branch.
-
-Pagure triggers tests for any new pull request. It can be also triggered
-manually by pressing "Rerun CI" button in PR.
-
-Currently we run tests on Fedora 33/34/rawhide and CentOS 7/8 platforms.
-
-Usage
------
-
-If you need any change in jenkins setup, please file a pagure issue. As part
-of solving issue, this documentation must be updated to reflect current state
-in jenkins.
-
-Configuration
--------------
-
-- Public access is for everyone with no need to log in to jenkins.
-- Admin access is via same interface and currently tkopecek have
-   access there. If you need access for yourself (you're probably part of
-   brew/koji team) create ticket in https://pagure.io/centos-infra requesting this.
-   Prerequisite for this is CentOS account.
-
-- Setup - Following items are set-up via
-   https://jenkins-fedora-infra.apps.ocp.ci.centos.org/job/koji/configure
-
-- *Job notifications* - when jenkins job finishes, it will send some info to
-   pagure. Such call will add a comment to PR. For this REST hook needs to
-   be configured:
-
-   * Format: JSON
-   * Protocol: HTTP
-   * Event: Job Finalized
-   * URL: <url taken from koji pagure settings which will look similar to https://pagure.io/api/0/ci/jenkins//koji/<hash>/build-finished
-   * Timeout: 30000
-   * Log: 0
-
-- *This build is parametrized* (checkbox set to true) - it allows jenkins
-   to use other branches than master (especially for PRs). Three parameters
-   are defined there:
-
-   * REPO - The repository for the pull request.
-   * BRANCH - The branch for the pull request.
-   * BRANCH_TO - A branch into which the pull request should be merged.
-
-- *Discard Old Builds*
-
-   * Strategy: Rotation
-   * Days to keep build: 20
-   * Max # of builds to keep: empty
-
-- *Disable build* - it could be used if a lot of failing build happens with
-    no vision of early recovery - temporarily suspend jenkins jobs
-- *Restrict where this project can be run* - F33 is the only used agent now.
-    Basically we need an agent which supports podman as all the tests are run in
-    docker images managed by podman.
-
-- *Source Code Management*
-
-  * Git
-  * Repositories
-
-    * Repository URL: https://pagure.io/koji.git
-    * Credentials: none
-    * Branches to build: origin/master
-
-- *Build triggers*
-
-  * *Trigger builds remotely*: true
-
-    * Authentication tokens: <token from koji pagure settings>
-
-  * *Poll SCM*:
-
-    * Schedule: H/5 * * * *
-
-- *Build* - most important part - script which runs tests itself. Basically we
-  run tests in containers, last one also run flake8 and is used for coverage
-  report.
-
-
-.. code-block:: shell
-
-    # merge PR into main repository  
-    if [ -n "$REPO" -a -n "$BRANCH" ]; then  
-        git config --global user.email "test@example.com"  
-        git config --global user.name "Tester"
-        git remote rm proposed || true  
-        git remote add proposed "$REPO"  
-        git fetch proposed   
-        git checkout "origin/${BRANCH_TO:-master}"  
-        git merge --no-ff "proposed/$BRANCH" -m "Merge PR"  
-    fi  
-
-    # centos:7
-    rm -rf .tox
-    podman run --rm --pull=always -v $PWD:/koji --security-opt label=disable --name koji-centos-test-7 quay.io/tkopecek/koji-centos-test:7 bash -c "cd /koji && tox -e py2"
-    podman rmi quay.io/tkopecek/koji-centos-test:7
-
-    # centos:8
-    rm -rf .tox
-    podman run --rm --pull=always -v $PWD:/koji --security-opt label=disable --name koji-centos-test-8 quay.io/tkopecek/koji-centos-test:8 bash -c "cd /koji && tox -e py3"
-    podman rmi quay.io/tkopecek/koji-centos-test:8
-
-    # fedora:32
-    rm -rf .tox
-    podman run --rm --pull=always -v $PWD:/koji --security-opt label=disable --name koji-fedora-test-32 quay.io/tkopecek/koji-fedora-test:32 bash -c "cd /koji && tox -e py3"
-    podman rmi quay.io/tkopecek/koji-fedora-test:32
-
-    # fedora:33
-    rm -rf .tox
-    podman run --rm --pull=always -v $PWD:/koji --security-opt label=disable --name koji-fedora-test-33 quay.io/tkopecek/koji-fedora-test:33 bash -c "cd /koji && tox -e py3"
-    podman rmi quay.io/tkopecek/koji-fedora-test:33
-
-    # fedora:rawhide
-    rm -rf .tox
-    podman run --rm --pull=always -v $PWD:/koji --security-opt label=disable --name koji-fedora-test-rawhide quay.io/tkopecek/koji-fedora-test:rawhide bash -c "cd /koji && tox -e flake8,py3"
-    podman rmi quay.io/tkopecek/koji-fedora-test:rawhide
-
-- *Post-build actions*
-
-  * *Publish Cobertura Coverage report*: coverage.xml - this will create coverage report accessible via jenkins web ui
-  * *Report Violations* - *pep8*: flake8_report.txt
-  * *E-mail notification*:
-
-    * Recipients: tkopecek@redhat.com exd-sp-rhel-build-alerts@redhat.com
-    * Send separate e-mails to individuals who broke the build
-
-- *Send messages to fedmsg*

docs/source/content_generator_metadata.rst

@@ -46,6 +46,7 @@ The build map contains the following entries:
    is optional.
 -  build_id: Reserved build ID. This field is optional.
 -  task_id: In case there is a corresponding task in koji (int/None)
+-  draft: True if the build should be imported as a draft. Optional.
 -  extra: A map of extra metadata associated with the build, which
    must include at least one of:
 
@@ -90,18 +91,37 @@ Each map in the buildroots list contains the following entries:
    -  version: Version of the tool used.
 
 -  components: List of maps containing information about content
-   installed in the build environment (if any). Each map is guaranteed
+   installed in the build environment (if any). Each map is required
    to contain a **type** field, which determines what other fields are
-   present in the map. For maps where **type = rpm**, the following
-   fields will be present:
+   present in the map. The currently available types are detailed below.
+
+-  For maps where **type = rpm**, the following fields must be present:
 
    -  name: The rpm name.
    -  version: The rpm version.
    -  release: The rpm release.
    -  epoch: The rpm epoch.
    -  arch: The rpm arch.
-   -  sigmd5: The SIGMD5 tag from the rpm header.
-   -  signature: The signature used to sign the rpm (if any).
+
+   Additionally, the following checksums from the rpm header
+   must be included if they exist in the header.
+   If given, these values must be lowercase hex strings.
+
+   -  sigmd5: The SIGMD5 tag value
+   -  sha1header: The SHA1HEADER tag value
+   -  sha256header: The SHA256HEADER tag value
+   -  sha3_256header: The SHA3_256HEADER tag value
+
+   For backwards compatibility, if is sufficient to specify only the sigmd5
+   checksum for rpm files having v4 and earlier formats.
+
+   The rpm id may be specified. If given, this must be the id of the rpm in Koji.
+
+   -  id: The Koji rpm id
+
+   Lastly for rpms, the following legacy field is accepted but ignored:
+
+   -  signature: ignored
 
 -  For maps where **type = file**, the following fields will be present:
 
@@ -138,6 +158,9 @@ Each map in the output list contains the following entries:
 -  buildroot\_id: The id of the buildroot used to create this file. Must
    match an entry in the buildroots list.
 -  filename: The name of the file.
+-  relpath: relative path for the uploaded file. I.e. the file was uploaded to
+   $upload_dir/$relpath/$filename
+-  subdir: subdir for final location. Only valid for logs and non-legacy btypes
 -  filesize: The size of the file.
 -  arch: The architecture of the file (if applicable).
 -  checksum: The checksum of the file.
@@ -194,31 +217,31 @@ The below JSON is based loosely on the output of a docker image build.
                                      "epoch": null,
                                      "arch": "x86_64",
                                      "sigmd5": "a1b2c3...",
-                                     "signature": "fd431d51"},
+                                     "sha1header": "2a1c91..."},
                                     {"type": "rpm",
                                      "name": "openssl",
                                      "version": "1.0.1e",
                                      "release": "42.el7",
                                      "epoch": null,
                                      "arch": "x86_64",
-                                     "sigmd5": "d4e5f6...",
-                                     "signature": "fd431d51"},
+                                     "sigmd5": "f8e857...",
+                                     "sha1header": "ea4c84..."},
                                     {"type": "rpm",
                                      "name": "bind-libs",
                                      "version": "9.9.4",
                                      "release": "18.el7",
                                      "epoch": 32,
                                      "arch": "x86_64",
-                                     "sigmd5": "987abc...",
-                                     "signature": null},
+                                     "sigmd5": "5b2ca0...",
+                                     "sha1header": "e73c78..."},
                                     {"type": "rpm",
                                      "name": "python-urllib3",
                                      "version": "1.5",
                                      "release": "8.el7",
                                      "epoch": null,
                                      "arch": "noarch",
-                                     "sigmd5": "123hgf...",
-                                     "signature": null},
+                                     "sigmd5": "5f38e3...",
+                                     "sha1header": "25a47b..."},
                                     {"type": "file",
                                      "filename": "jboss-eap-6.3.3-full-build.zip",
                                      "filesize": 12345678,

docs/source/content_generators.rst

@@ -155,7 +155,7 @@ Following koji plugins uses CGs as part of their workflow.
 
 - `OpenShift Build Service <https://github.com/containerbuildsystem/>`__
 - `OSBuild <https://github.com/osbuild/>`__
-- `MBS <https://pagure.io/fm-orchestrator/blob/master/f/module_build_service/builder/KojiContentGenerator.py>`__
+- `MBS <https://forge.fedoraproject.org/koji/mbs/src/branch/master/module_build_service/builder/KojiContentGenerator.py>`__
 
 Standalone CG utilites are here:
 - `CoreOS <https://github.com/coreos/coreos-assembler/blob/master/src/cmd-koji-upload>`__

docs/source/defining_hub_policies.rst

@@ -237,6 +237,13 @@ Available tests
 ``none``
     * an alias of false
 
+``bool``
+    * return boolean value of argument. E.g. ``bool scratch`` will return true
+      for scratch builds
+
+``has``
+    * test if policy contains argument field
+
 ``operation``
     * for tag operations, the operation is one of: tag, untag, move. This test
       checks its arguments against the name of the operation and returns true if
@@ -291,6 +298,9 @@ Available tests
 ``buildtype``
     * checks the build type(s) against the arguments
 
+``is_draft``
+    * checks whether the build is a draft. See :doc:`draft_builds`
+
 ``skip_tag``
     * checks to see if the --skip-tag option was used
     * only applicable to the build_from_* policies
@@ -346,5 +356,20 @@ Available tests
     * the user matched is the user performing the action
 
 ``match``
-    * matches a field in the data against glob patterns 
+    * matches a field in the data against glob patterns
     * true if any pattern matches
+
+``match_any``
+    * Matches any item of a list/tuple/set value in the data against glob patterns
+    * True if any of the expressions matches any item in the list/tuple/set, else False.
+      If the field doesn't exist or isn't a list/tuple/set, the test returns False
+
+``match_all``
+    * Matches all items of a list/tuple/set value in the data against glob patterns
+    * True if any of the expressions matches all items in the list/tuple/set, else False.
+      If the field doesn't exist or isn't a list/tuple/set, the test returns False
+
+``compare``
+    * Simple numeric field comparison
+    * Supports basic numeric comparisons. The right operand must be a valid
+      number (<,>,<=,>=,=,!=)

docs/source/draft_builds.rst

@@ -0,0 +1,169 @@
+
+Draft Builds
+============
+
+Draft builds allow for a workflow of building the name rpm NVR multiple times and then choosing
+one of those builds to promote.
+
+The draft build feature was added in :doc:`version 1.34.0 <release_notes/release_notes_1.34>`
+
+What is a draft build?
+----------------------
+
+In the simplest terms, draft builds are builds with the ``draft`` flag set to True.
+This flag has multiple implications:
+
+1. Koji adds a *draft suffix* to the release value for draft builds. The format of this suffix
+   is ``,draft{build_id}``. This ensures that the NVR of draft builds is unique, even if the
+   base NVR (without the suffix) is not.
+2. The NVRA values for rpms included in a draft build are not required to be unique.
+3. System policy (notably the tag policy) may set different rules for draft builds.
+
+Draft builds should not be confused with scratch builds. Scratch builds are
+simply stored as files and cannot be tagged in Koji. Draft builds, on the other
+hand, are actual builds. They can be tagged (subject to policy) and hence potentially
+used in buildroots. They can be promoted to become normal builds.
+
+
+Building a draft build
+----------------------
+
+A draft build is triggered simply by passing the ``--draft`` option to the ``build`` command.
+The resulting build will be a imported as a draft build.
+Otherwise the build process is identical.
+
+At the moment, this feature is only relevant for rpm builds, but it may be extended in the future.
+
+
+Promoting a draft build
+-----------------------
+
+Draft builds can be "promoted" to non-draft using the ``promote-build`` cli
+command.
+
+When a draft build is promoted, it is renamed to remove the draft suffix.
+The build directory is moved to its new location, and the original
+path (the one that is includes the draft suffix) is replaced with a symlink to the new location.
+
+Build promotion is a one-time transition, i.e. builds cannot be "unpromoted".
+Only one draft build for a given NVR can be promoted, and once Koji has a
+non-draft build for a given NVR, further draft builds for that NVR are blocked.
+
+
+An Example
+----------
+
+You could trigger a draft build with a command like the following:
+
+::
+
+    $ koji build --draft f39-candidate mypackage-1.1-35.src.rpm
+
+Once the build completes, you would have a build named something like:
+
+::
+
+    mypackage-1.1-35,draft_51515
+
+The number in the draft suffix will be the same as the build id.
+This makes draft builds stand out.
+However, note that the component rpms *do not* have the draft suffix added.
+
+::
+
+    $ koji buildinfo 51515
+    BUILD: mypackage-1.1-35,draft_51515 [51515]
+    Draft: YES
+    State: COMPLETE
+    Built by: kojiuser
+    Source: mypackage-1.1-35.src.rpm
+    Volume: DEFAULT
+    Task: 12082 build (f39-candidate, mypackage-1.1-35.src.rpm)
+    Finished: Thu, 04 Jan 2024 20:02:18 EST
+    Tags: f39-candidate
+    Extra: {'source': {'original_url': 'mypackage-1.1-35.src.rpm'}}
+    RPMs:
+    /mnt/koji/packages/mypackage/1.1/35,draft_51515/noarch/mypackage-1.1-35.noarch.rpm
+    /mnt/koji/packages/mypackage/1.1/35,draft_51515/noarch/mypackage-devel-1.1-35.noarch.rpm
+    /mnt/koji/packages/mypackage/1.1/35,draft_51515/noarch/mypackage-1.1-35.src.rpm
+
+You can build the same NVR as a draft multiple times.
+
+::
+
+    $ koji build --draft f39-candidate mypackage-1.1-35.src.rpm
+
+::
+
+    $ koji buildinfo 51516
+    BUILD: mypackage-1.1-35,draft_51516 [51516]
+    Draft: YES
+    State: COMPLETE
+    ...
+
+Once you have a build that is satisfactory, you can promote it.
+
+::
+
+    $ koji promote-build mypackage-1.1-35,draft_51516
+    mypackage-1.1-35,draft_51516 has been promoted to mypackage-1.1-35
+
+::
+
+    $ koji buildinfo mypackage-1.1-35
+    BUILD: mypackage-1.1-35 [51516]
+    State: COMPLETE
+    Built by: kojiuser
+    Source: mypackage-1.1-35.src.rpm
+    Volume: DEFAULT
+    Task: 12082 build (f39-candidate, mypackage-1.1-35.src.rpm)
+    Finished: Thu, 04 Jan 2024 20:02:18 EST
+    Promoted by: kojiuser
+    Promoted at: Tue, 09 Jan 2024 14:13:48 EST
+    Tags: f39-candidate
+    Extra: {'source': {'original_url': 'mypackage-1.1-35.src.rpm'}}
+    RPMs:
+    /mnt/koji/packages/mypackage/1.1/35,draft_51515/noarch/mypackage-1.1-35.noarch.rpm
+    /mnt/koji/packages/mypackage/1.1/35,draft_51515/noarch/mypackage-devel-1.1-35.noarch.rpm
+    /mnt/koji/packages/mypackage/1.1/35,draft_51515/noarch/mypackage-1.1-35.src.rpm
+
+
+
+Dealing with duplicate NVRAs
+----------------------------
+
+The NVRAs for all *non-draft* rpms in Koji are required to be unique.
+This is enforced by a database constraint.
+However, this constraint does not apply to draft builds.
+So, it is possible to have rpms of the same name in Koji if draft builds are used.
+
+When Koji is asked to look up an rpm by name, it will report the non-draft rpm if it exists.
+Otherwise, it will chose the matching draft rpm with the highest id.
+
+Because of this, the *safest* way to specify an rpm is to use its rpm id.
+
+
+Policies for draft builds
+-------------------------
+
+The ``is_draft`` policy test returns True if the build in question is a draft and False otherwise.
+This is primarily useful in the ``tag`` policy to limit where draft builds can go.
+
+Note that Koji's default policies do not include special rules for draft builds, so by default
+draft builds can be used anywhere a regular build can.
+Koji administrators can adjust their policies to match their own workflows.
+
+For example, you could use the tag policy to restrict draft builds to only certain tags.
+E.g.
+
+::
+
+    [policy]
+    tag =
+        is_draft && operation move tag :: {
+            tag *-draft *-draft-* !! deny draft builds are only allowed in draft tags
+        }
+        ...
+
+For further information on Koji policies,
+see the :doc:`policy documentation <defining_hub_policies>`

docs/source/exporting_repositories.rst

@@ -6,7 +6,7 @@ Koji provides some *limited* features for exporting repositories of RPMs.
 Please note that Koji is a build system, not a repository manager, and these
 features are secondary.
 If you need more robust repository generation than Koji provides, then you may
-want to look into using `pungi <https://pagure.io/pungi/>`_.
+want to look into using `pungi <https://forge.fedoraproject.org/pungi/pungi/>`_.
 
 
 Koji's internal repositories
@@ -90,14 +90,13 @@ For a full list of options, see ``koji dist-repo --help``.
 
 **Koji Hub plugin**
 
-Fedora release engineering uses a hub plugin `tag2distrepo
-<https://pagure.io/releng/tag2distrepo>`_ to automatically export dist-repos
-for certain tags.
+Fedora release engineering uses the tag2distrepo hub plugin (now integrated
+into Koji) to automatically export dist-repos for certain tags.
 
 Beyond Koji
 ===========
 
 If you're aiming to have more control about repositories, varieties of
-distribution flavours, etc. use `pungi <https://pagure.io/pungi/>`_ which can
+distribution flavours, etc. use `pungi <https://forge.fedoraproject.org/pungi/pungi/>`_ which can
 create whole composes and which uses Koji for some of the subtasks.
 Pungi + koji is what Fedora currently uses for composes.

docs/source/hub_conf.rst

@@ -233,6 +233,8 @@ Notification options
 
 For more on notifications in Koji, see :ref:`notification-basics`
 
+.. _hub_resource_limits:
+
 Resource limits
 ^^^^^^^^^^^^^^^
 If configured, the following resource limits are applied by the hub at startup.

docs/source/image_build.rst

@@ -101,7 +101,7 @@ so:
 ::
 
     $ koji spin-livemedia --release 20200922.n.0  \
-                          --ksurl 'git+https://pagure.io/fedora-kickstarts.git?#b4956c05028a088c641d0ce6e1a31b6d8b20176f' \
+                          --ksurl 'git+https://forge.fedoraproject.org/releng/spin-kickstarts.git?#b4956c05028a088c641d0ce6e1a31b6d8b20176f' \
                           --install-tree-url 'https://kojipkgs.fedoraproject.org/compose/branched/Fedora-33-20200922.n.0/compose/Everything/$basearch/os' \
                           --repo 'https://kojipkgs.fedoraproject.org/compose/branched/Fedora-33-20200922.n.0/compose/Everything/$basearch/os' \
                           --can-fail ppc64le,aarch64 \

docs/source/index.rst

@@ -11,7 +11,7 @@ Koji
 ====
 
 Koji is the software that builds `RPM packages for the Fedora project`_. It
-uses `Mock`_ to create chroot environments to perform builds. To download the
+uses :doc:`Mock <mock_config>` to create chroot environments to perform builds. To download the
 source code, report bugs, join the mailing list etc., see the `Koji project
 website`_.
 
@@ -29,8 +29,11 @@ Contents
     external_repo_server_bootstrap
     image_build
     winbuild
+    draft_builds
+    repo_generation
     exporting_repositories
     tag_inheritance
+    managing_comps_data
     misc
     release_notes/release_notes
     migrations/migrations
@@ -42,6 +45,7 @@ Contents
     database_howto
     kojid_conf
     hub_conf
+    mock_config
     using_the_koji_build_system
     setting_rpm_macros
     profiles
@@ -50,7 +54,6 @@ Contents
     writing_koji_code
     content_generators
     content_generator_metadata
-    configuring_jenkins
     utils
     supported_platforms
 
@@ -120,7 +123,7 @@ and communicates with koji-hub via XML-RPC.
 
 Koji-Web
 --------
-koji-web is a set of scripts that run in mod_wsgi and use the Cheetah
+koji-web is a set of scripts that run in mod_wsgi and use the Jinja
 templating engine to provide a web interface to Koji. It acts as a client to
 koji-hub providing a visual interface to perform a limited amount of
 administration. koji-web exposes a lot of information and also provides a means
@@ -301,4 +304,4 @@ Indices and tables
 
 .. _Mock: https://github.com/rpm-software-management/mock/wiki
 .. _RPM packages for the Fedora project: https://koji.fedoraproject.org/koji/
-.. _Koji project website: https://pagure.io/koji
+.. _Koji project website: https://forge.fedoraproject.org/koji/koji

docs/source/kerberos_gssapi_debug.rst

@@ -24,9 +24,9 @@ koji server connected to:
 
     $ koji hello
     2021-04-23 09:45:22,732 [ERROR] koji: (gssapi auth failed: requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://koji.fedoraproject.org/kojihub/ssllogin)
-    Use following documentation to debug kerberos/gssapi auth issues. https://docs.pagure.org/koji/kerberos_gssapi_debug/
+    Use following documentation to debug kerberos/gssapi auth issues. https://koji.readthedocs.io/en/latest/kerberos_gssapi_debug.html
     2021-04-23 09:45:22,734 [ERROR] koji: GSSAPIAuthError: unable to obtain a session (gssapi auth failed: requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://koji.fedoraproject.org/kojihub/ssllogin)
-    Use following documentation to debug kerberos/gssapi auth issues. https://docs.pagure.org/koji/kerberos_gssapi_debug/
+    Use following documentation to debug kerberos/gssapi auth issues. https://koji.readthedocs.io/en/latest/kerberos_gssapi_debug.html
 
 In this case we see not one but two errors while trying to authenticate. Let's
 see what more info we can get:
@@ -38,7 +38,7 @@ see what more info we can get:
     2021-04-23 09:47:58,731 [DEBUG] koji: Opening new requests session
     2021-04-23 09:47:59,446 [DEBUG] koji: Opening new requests session
     2021-04-23 09:47:59,446 [ERROR] koji: (gssapi auth failed: requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://koji.fedoraproject.org/kojihub/ssllogin)
-    Use following documentation to debug kerberos/gssapi auth issues. https://docs.pagure.org/koji/kerberos_gssapi_debug/
+    Use following documentation to debug kerberos/gssapi auth issues. https://koji.readthedocs.io/en/latest/kerberos_gssapi_debug.html
     Traceback (most recent call last):
       File "/usr/bin/koji", line 335, in <module>
         rv = locals()[command].__call__(options, session, args)
@@ -49,7 +49,7 @@ see what more info we can get:
       File "/home/tkopecek/cvs/koji/koji/__init__.py", line 2513, in gssapi_login
         raise GSSAPIAuthError(err)
     koji.GSSAPIAuthError: unable to obtain a session (gssapi auth failed: requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://koji.fedoraproject.org/kojihub/ssllogin)
-    Use following documentation to debug kerberos/gssapi auth issues. https://docs.pagure.org/koji/kerberos_gssapi_debug/
+    Use following documentation to debug kerberos/gssapi auth issues. https://koji.readthedocs.io/en/latest/kerberos_gssapi_debug.html
 
 We see no output from krbV library. It indicates that there was no significant
 interaction with the library. Main reason would be missing ``kinit``. Fixing

docs/source/kojid_conf.rst

@@ -1,11 +1,15 @@
 kojid.conf Options
 ------------------
-kojid.conf is standard .ini-like configuration file. Its main section is
-called ``[kojid]`` and contains following options. They are split by function
-to section but can occur anywhere in ``[kojid]`` section in config file.
 
-General
-^^^^^^^
+``/etc/kojid/kojid.conf`` is a standard .ini-like configuration file. Its main
+section is named ``[kojid]`` and can contain the following options.
+For readability, we have broken them into categories in this document,
+but they should all go under the ``[kojid]`` block of the config file.
+
+General Options
+^^^^^^^^^^^^^^^
+These options affect overall kojid behavior:
+
 .. glossary::
    chroot_tmpdir=/chroot_tmpdir
       The temporary directory in buildroot. It is advised to not use ``/tmp``
@@ -31,7 +35,7 @@ General
       buildroot_final_cleanup_delay
 
    buildroot_final_cleanup_delay=86400
-      Time after which buildroot (pre-cleand after
+      Time after which buildroot (pre-cleaned after
       ``buildroot_basic_cleanup_delay``) is deleted completely. (1 day in
       seconds)
 
@@ -96,8 +100,10 @@ General
    workdir=/tmp/koji
       The directory root for temporary storage on builder.
 
-Building
-^^^^^^^^
+Build Options
+^^^^^^^^^^^^^
+The following options affect various build processes in kojid:
+
 .. glossary::
    allowed_scms=scm.example.com:/cvs/example git.example.org:/example svn.example.org:/users/\*:no
       Controls which source control systems the builder will accept. It is a
@@ -192,8 +198,10 @@ Building
       the image. Default value is compromise between speed and resource usage.
       Only one option (not space-separated) is allowed here for now.
 
-RPM Builds
-^^^^^^^^^^
+RPM Build Options
+^^^^^^^^^^^^^^^^^
+The following options affect rpms builds:
+
 .. glossary::
    distribution=Koji
       The distribution to use in rpm headers. Value is propagated via macros
@@ -214,8 +222,27 @@ RPM Builds
       The vendor to use in rpm headers. Value is propagated via macros to
       rpmbuild.
 
-Mock
-^^^^
+.. _rlimits_kojid:
+
+General RLIMIT options
+^^^^^^^^^^^^^^^^^^^^^^
+Kojid accepts the same rlimit options as the hub.
+
+.. glossary::
+   RLIMIT_*
+      Same resource limit options as in the :ref:`hub config
+      <hub_resource_limits>`. Note, that when ``mock.new_chroot`` is set, those
+      are not propagated to the mock chroot. To apply rlimits to mock in
+      such cases, you can set them separately in
+      ``/etc/mock/site-defaults.cfg``. See :doc:`mock_config` for more information.
+
+
+.. _mock_kojid_conf:
+
+Mock Options
+^^^^^^^^^^^^
+The following options affect how Koji uses Mock:
+
 .. glossary::
    mockdir=/var/lib/mock
       The directory root for mock.
@@ -234,20 +261,26 @@ Mock
    yum_proxy=None
       Address of proxy server which will be passed via mock to yum.
 
-Notifications
-^^^^^^^^^^^^^
+   rlimits
+      The ``RLIMIT_*`` options can affect mock's behaviour and are described :ref:`here
+      <rlimits_kojid>`.
+
+Notification Options
+^^^^^^^^^^^^^^^^^^^^
 .. glossary::
    admin_emails=''
       Space-separated list of addresses for sending logs.
 
-   from_addr=Koji Build System <buildsys@example.com>
+   from_addr=Koji Build System <buildsys\@example.com>
       The From address used when sending email notifications.
 
    smtphost=example.com
       The mail host to use for sending email notifications.
 
-Kerberos Authentication
-^^^^^^^^^^^^^^^^^^^^^^^
+Kerberos Options
+^^^^^^^^^^^^^^^^
+The following options control Kerberos authentication:
+
 .. glossary::
    ccache=/var/tmp/kojid.ccache
       Credentials cache used for krbV login.
@@ -260,8 +293,10 @@ Kerberos Authentication
       Location of the keytab.
 
 
-SSL Authentication
-^^^^^^^^^^^^^^^^^^
+SSL Options
+^^^^^^^^^^^
+The following options control SSL authentication:
+
 .. glossary::
    ca=''
       noop, obsoleted, will be removed soon.

docs/source/managing_comps_data.rst

@@ -0,0 +1,157 @@
+Managing comps data for tags
+============================
+
+Koji can record comps data for each tag and uses that data to write a comps
+file whenever it generates a yum repo for that tag. This document describes
+how this data is stored, how to read it, and how to change it.
+
+For background on comps files, see
+
+* https://dnf5.readthedocs.io/en/latest/misc/comps.7.html
+* https://fedoraproject.org/wiki/How_to_use_and_edit_comps.xml_for_package_groups
+
+The comps data *affects how rpm builds are performed*.
+Various buildroots generated by Koji use these groups for their base install.
+E.g.
+
+* rpm builds use the ``build`` group
+* source rpm builds use the ``srpm-build`` group
+* maven builds use the ``maven-build`` group
+* wrapper rpm builds use the ``wrapper-rpm-build`` group
+* appliance builds use the ``appliance-build`` group
+* livecd builds use the ``livecd-build`` group
+* livemedia builds use the ``livemedia-build`` group
+
+
+Reading the data
+----------------
+
+The ``koji list-groups`` cli command will output the comps data for a given tag
+
+.. code-block:: text
+
+    Usage: koji list-groups [options] <tag> [<group>]
+    (Specify the --help global option for a list of other help options)
+
+    Options:
+      -h, --help      show this help message and exit
+      --event=EVENT#  query at event
+      --ts=TIMESTAMP  query at last event before timestamp
+      --repo=REPO#    query at event for a repo
+      --show-blocked  Show blocked packages and groups
+
+The command will optionally accept a group name to limit the output to a single group.
+
+Like many other types of data in Koji, the comps data is versioned and Koji remembers its history.
+The ``--event/--ts/--repo`` options can be used to query this data from an earlier point in time.
+
+The ``show-blocked`` option will include blocked entries in the output.
+
+Alternately ``show-groups`` command can be used to present the groups data in different formats.
+This command is a historical oddity, but still useful to export the data in comps format.
+
+Example
+-------
+
+The following command prints the srpm-build group data for the f41-build tag.
+
+.. code-block:: text
+
+    $ koji list-groups f41-build srpm-build
+    srpm-build  [f41]
+      bash: None, mandatory  [f41]
+      fedora-release: None, mandatory  [f41]
+      fedpkg-minimal: None, mandatory  [f41]
+      glibc-minimal-langpack: None, mandatory  [f41]
+      gnupg2: None, mandatory  [f41]
+      redhat-rpm-config: None, mandatory  [f41]
+      rpm-build: None, mandatory  [f41]
+      shadow-utils: None, mandatory  [f41]
+
+Because our command included a specific group name, the command only outputs that one group.
+Under this group, we see the list of packages in the group and their parameters.
+The bracketed value at the end shows the tag that the entry is inherited from.
+
+
+Groups data and inheritance
+---------------------------
+
+Like most other data associated with tags, groups data is inheritable.
+Because the groups data is complex, so is the inheritance.
+The groups data is inherited piecemeal, with individual groups, group package
+entries, and group dependencies separately inheritable.
+
+You can prevent an entry from being inherited by blocking it.
+This can be done via the ``block-group``, ``block-group-pkg``, and ``block-group-req``
+commands respectively (and undone with the corresponding unblock command).
+
+As noted above, you can see which tags an entry comes from via the ``list-groups`` command.
+
+So, a given tag will have a list of groups that includes inherited entries.
+Each of those groups will have a list of packages and a list of group dependencies
+which include inherited entries.
+
+For admin sanity, we recommend defining a comps in a base tag in your inheritance
+and keeping adjustments in child tags to a minimum.
+A common case is to add additional packages to the build group for a specialized
+build tag.
+
+
+Changing comps data
+-------------------
+
+You can add, remove, block, and unblock individual entries with the relevant command
+
+.. code-block:: text
+
+    add-group                 Add a group to a tag
+    remove-group              Remove group from tag
+    block-group               Block group in tag
+    unblock-group             Unblock a group from tag
+    add-group-pkg             Add a package to a group's package listing
+    remove-group-pkg          Remove packages from a group's package listing
+    block-group-pkg           Block a package from a group's package listing
+    unblock-group-pkg         Unblock a package from a group's package listing
+    add-group-req             Add a group to a group's required list
+    block-group-req           Block a group's requirement listing
+    remove-group-req          Remove entries from a group's requirement listing
+    unblock-group-req         Unblock a group's requirement listing
+
+Additionally, you can also import groups data directly from a comps file with the ``import-comps`` command.
+This is useful when setting up a new tag from scratch.
+
+.. code-block:: text
+
+    Usage: koji import-comps [options] <file> <tag>
+    (Specify the --help global option for a list of other help options)
+
+    Options:
+      -h, --help  show this help message and exit
+      --force     force import
+
+
+Blocking vs removing
+--------------------
+
+The various remove commands remove a specific entry from a specific tag.
+They will fail if the entry is not actually in the specified tag.
+This commonly happens when an entry is inherited.
+
+When your tag has an inherited comps entry that you want to remove, you have two
+options:
+
+* block the entry in the tag
+* remove the entry from the ancestor tag it is inherited from
+
+It is generally safest to block, as this only affects the tag of concern.
+Removing the entry in an ancestor could affect other tags negatively.
+However, there are certainly cases where it makes sense to remove the entry
+from the ancestor.
+
+
+Group dependencies
+------------------
+
+For historical reasons, Koji's comps data allows specifying group dependecies/requirements.
+This is antiquated feature of comps that is longer used by dnf,
+so in practice there is no need to specify such entries (i.e. the various ``*-req`` commands).

docs/source/migrations/migrating_to_1.34.rst

@@ -12,7 +12,7 @@ As in previous releases, we provide a migration script that updates the database
 
 ::
 
-    # psql koji koji < /usr/share/doc/koji/docs/schema-upgrade-1.33-1.34.sql
+    # psql koji koji < /usr/share/koji/schema-upgrade-1.33-1.34.sql
 
 
 Other changes

docs/source/migrations/migrating_to_1.35.rst

@@ -0,0 +1,62 @@
+Migrating to Koji 1.35
+======================
+
+You should consider the following changes when migrating to 1.35:
+
+DB Updates
+----------
+
+Changes for new repo handling introduced a few new tables.
+
+As in previous releases, we provide a migration script that updates the database.
+
+::
+
+    # psql koji koji < /usr/share/koji/schema-upgrade-1.34-1.35.sql
+
+
+Repo Generation
+---------------
+
+The way that Koji handles repo generation has changed significantly.
+For full details, please see the release notes and :doc:`../repo_generation`.
+
+The following kojira configuration options are no longer used and have no analog
+in the new system:
+
+* ``queue_file``
+* ``ignore_tags``
+* ``no_repo_effective_age``
+* ``repo_tasks_limit``
+
+We recommend you remove these settings from ``kojira.conf``.
+Kojira will warn at startup if they are present.
+
+Additionally, these kojira configuration options have moved into the hub config:
+
+* ``max_repo_tasks`` -> ``MaxRepoTasks``
+* ``max_repo_tasks_maven`` -> ``MaxRepoTasksMaven``
+* ``debuginfo_tags`` -> ``DebuginfoTags``
+* ``source_tags`` -> ``SourceTags``
+* ``separate_source_tags`` -> ``SeparateSourceTags``
+
+Admins that relied on these kojira options should now set the corresponding hub options and remove
+the old kojira ones.
+For the latter three, note that there are now other ways to control these repo options.
+
+Finally, if you have systems outside of Koji that rely on Koji keeping particular tag repos up to
+date, you may need to take extra steps to ensure regeneration.
+If the tag sees regular build use within Koji, then that might trigger regens often enough.
+If not, there are two options:
+
+* set ``repo.auto=True`` for the tag in question
+* have the external system create a repo request via the api
+
+Again, for full details, please see the release notes and :doc:`../repo_generation`.
+
+
+Other changes
+-------------
+
+There are numerous other changes in 1.35 that should not have a direct impact
+on migration. For details see: :doc:`../release_notes/release_notes_1.35`

docs/source/migrations/migrating_to_1.36.rst

@@ -0,0 +1,65 @@
+Migrating to Koji 1.36
+======================
+
+You should consider the following changes when migrating to 1.36:
+
+DB Updates
+----------
+
+The changes to support rpm v6 include new fields in the ``rpminfo`` table.
+Additionally, we have added new archive types and new indexes.
+
+As in previous releases, we provide a migration script that updates the database.
+
+::
+
+    # psql koji koji < /usr/share/koji/schema-upgrade-1.35-1.36.sql
+
+
+Volume toplinks
+---------------
+
+.. warning:: Action required for non-default volumes
+
+In order to address a previous issue with promoting builds, the system now
+requires that each volume directory contain a ``toplink`` symlink that points
+to the main volume.
+This is simply a symlink that points to the main volume. E.g.
+
+::
+
+     $ file /mnt/koji/vol/vol3/toplink
+     /mnt/koji/vol/vol3/toplink: symbolic link to /mnt/koji
+
+You **must** create this symlink for any existing volumes used in your system.
+Failure to do so can result in build promotion failures.
+
+We recommend using an absolute symlink, as in the example above.
+
+When adding a new volume, admins should create this symlink when they create
+the volume directory. If not, the hub will automatically create it using the
+configured topdir path and log a warning.
+
+For more information, see
+
+* :ref:`Fix draft symlink handling <draft-symlink-fix-1-36>`
+* :ref:`adding-new-volume`
+
+
+Notable configuration changes
+-----------------------------
+
+The ``RepoLagWindow`` hub option is no longer used.
+For details see https://pagure.io/koji/issue/4499
+
+The ``DisableURLSessions`` hub option now defaults to True.
+This will prevent clients older than Koji 1.31 from authenticating.
+For details see https://pagure.io/koji/issue/4420
+
+
+
+Other changes
+-------------
+
+There are numerous other changes in 1.36 that should not have a direct impact
+on migration. For details see: :doc:`../release_notes/release_notes_1.36`

docs/source/migrations/migrations.rst

@@ -5,6 +5,8 @@ Migrations
 .. toctree::
     :maxdepth: 1
 
+    migrating_to_1.36
+    migrating_to_1.35
     migrating_to_1.34
     migrating_to_1.33
     migrating_to_1.32

docs/source/mock_config.rst

@@ -0,0 +1,159 @@
+Interaction with Mock
+---------------------
+
+`Mock`_ is crucial component for building in Koji. Most build types use mock to
+prepare a fresh, isolated environment for building content.
+
+For further reading on the mock tool itself,
+see the `mock documentation <https://rpm-software-management.github.io/mock/>`_.
+
+
+
+Mock config files
+=================
+
+For each buildroot Koji prepares a configuration file for mock to use.
+The content of the file depends in part on some administrator's decisions
+and can be tweaked by options in the build tag.
+These files are managed by kojid and stored under ``/etc/mock/koji`` on
+the build host.
+
+Since Koji 1.34.1, builds will save the mock configuration file as a log named
+``mock_config.log`` and store it alongside other logs from the build.
+
+The contents of this file are generated by Koji and are affected by:
+
+* :ref:`kojid configuration <mock_kojid_conf>`
+* the build tag
+* the build arch
+* the repo used for the build
+* build tag options, see :ref:`tuning-mock-per-tag`
+* the controlling build task (e.g. buildArch for rpms)
+
+You can generate mock configs locally in the same way that kojid does.
+See :ref:`cli_mock_config`
+
+
+Site defaults
+=============
+
+In addition to the mock configuration file for the buildroot, the mock command
+also consults ``/etc/mock/site-defaults.cfg`` on the build host.
+You can use this file to set global defaults on your builders.
+
+The following settings are recommended for use with Koji:
+
+   - ``use_bootstrap = False`` - Bootstrap is a) not needed in most cases b)
+     slows down the build c) e.g. in combination with next option could lead to
+     unreproducible build. While this feature is pretty useful in some cases,
+     it should be turned off as default and could be selectively turned on for
+     specific buildroots per tag configuration.
+
+   - ``use_bootstrap_image`` - As noted before, this is generally unsafe
+     option, especially if used with image tags which are expected to be moved
+     (e.g. "latest"). Note, that to use this option (even via tag's
+     configuration), you need to explicitly allow it in :doc:`kojid.conf
+     <kojid_conf>` via ``mock_bootstrap_image = True``.
+
+   - ``nspawn_args`` - if you want to apply rlimit settings for mock with nspawn
+     isolation, you will need to use this option. See :ref:`rlimits_mock` below.
+
+   - ``macros`` - some rpm macros could be defined every time for every build
+     based on builder itself. Typical usecase is to limit usage of CPUs on
+     given builder by setting ``config_opts['macros']['%_smp_ncpus_max'] = 2``.
+
+   - ``plugin_conf`` - Various plugins could be enabled and configured to be
+     available in build environment. Example is `rpmautospec`_ which allows
+     various ``%auto*`` macros in spec files.
+
+
+.. _rlimits_mock:
+
+RLIMIT settings
+^^^^^^^^^^^^^^^
+
+When mock uses simple isolation (aka old-chroot), the
+:ref:`RLIMIT_* settings <rlimits_kojid>` from ``kojid.conf`` will propagate
+to mock.
+
+However, when mock uses nspawn isolation (aka new-chroot, controlled by the
+``mock.new_chroot`` tag extra option), these limits are not propagated automatically.
+To apply such limits to the mock nspawn chroots, you can add them to the
+``nspawn_args`` option in ``site-defaults.cfg``.
+For example, you might set:
+
+::
+
+    config_opts['nspawn_args'] = ['--rlimit=RLIMIT_NOFILE=16384:16384']
+
+If your system uses a mix of mock isolation settings, then you will likely want
+to apply rlimit settings in *both* ``kojid.conf`` and ``site-defaults.cfg`` for
+consistency.
+
+
+.. _mock_gpg_verification:
+
+GPG signature verification
+==========================
+
+Koji generates mock configs with ``gpgcheck=0`` in the yum/dnf configuration,
+because packages in Koji's buildroot repos are unsigned.
+
+Starting with Fedora 45, RPM defaults ``%_pkgverify_level`` to ``all``,
+meaning RPM itself refuses to install unsigned packages regardless of the
+dnf-level ``gpgcheck`` setting. This causes buildroot setup to fail with
+``GPG check FAILED`` when installing packages from the Koji repo.
+
+DNF5 (>= 5.2.14.0, available in Fedora 42+) has built-in support for
+disabling per-package signature verification for repos with ``gpgcheck=0``.
+For Fedora 45+ build tags, set ``mock.package_manager=dnf5`` to use this
+mechanism:
+
+::
+
+    koji edit-tag f45-build -x mock.package_manager=dnf5
+
+Older Fedora versions and RHEL are not affected, as they default
+``%_pkgverify_level`` to ``digest`` (or do not have the macro at all).
+
+See :ref:`tuning-mock-per-tag` for other mock-related tag options.
+
+
+.. _cli_mock_config:
+
+Local mock configs
+==================
+
+The ``koji mock-config`` command can be used to generate a mock config file locally
+in the same way that the builders do. These can be useful for local testing or debugging.
+
+::
+
+    $ koji mock-config --help
+    Usage: koji mock-config [options]
+    (Specify the --help global option for a list of other help options)
+
+    Options:
+      -h, --help            show this help message and exit
+      -a ARCH, --arch=ARCH  Specify the arch
+      -n NAME, --name=NAME  Specify the name for the buildroot
+      --tag=TAG             Create a mock config for a tag
+      --target=TARGET       Create a mock config for a build target
+      --task=TASK           Duplicate the mock config of a previous task
+      --latest              use the latest redirect url
+      --buildroot=BUILDROOT
+                            Duplicate the mock config for the specified buildroot
+                            id
+      --mockdir=DIR         Specify mockdir
+      --topdir=DIR          Specify topdir, topdir tops the topurl
+      --topurl=URL          URL under which Koji files are accessible, when topdir
+                            is specified, topdir tops the topurl
+      --distribution=DISTRIBUTION
+                            Change the distribution macro
+      --yum-proxy=YUM_PROXY
+                            Specify a yum proxy
+      -o FILE               Output to a file
+
+
+.. _Mock: https://rpm-software-management.github.io/mock/
+.. _rpmautospec: https://fedora-infra.github.io/rpmautospec-docs/

docs/source/plugins.rst

@@ -100,6 +100,13 @@ koji due to easier integration with rest of the code.
 It is used for managing `sidetags` which are light-weight short-lived build tags
 for developer's use. Sidetag creation is governed by hub's policy.
 
+Sidetag creation involves creating one tag based on some existing buildtag (it
+is the only parameter needed for sidetag creation). It automatically creates
+inheritance chain from it. Second part is creating target which has same name,
+source and destination. So, any build created for this target will appear in
+the tag itself. Developers can reuse new builds as part of the sidetag
+buildroot and build whole dependency chain if they need.
+
 Hub
 ---
 
@@ -173,6 +180,31 @@ like:
    f30-build-side-123456
    Successfully waited 1:36 for a new f30-build-side-123456 repo
 
+   $ koji taginfo fedora-30-build-side-123456
+   Tag: fedora-30-build-side-123456 [123456]
+   Arches: x86_64
+   Groups: appliance-build, build, kiwi-build, livecd-build, livemedia-build, srpm-build
+   Maven support?: no
+   Include all Maven archives?: no
+   Tag options:
+      mock.use_bootstrap : 1       [fedora-30-build]
+      sidetag : True
+      sidetag_user : 'username'
+     sidetag_user_id : 1
+   Targets that build into this tag:
+     fedora-30-build-side-123456 (fedora-30-build-side-123456, no active repo)
+     This tag is a buildroot for one or more targets
+     Current repo: no active repo
+   Targets that build from this tag:
+       fedora-30-build-side-123456
+   Inheritance:
+     0    .... fedora-30-build [84]
+
+   $ koji list-targets --name=fedora-30-build-side-123456
+   Name                           Buildroot                      Destination
+   ---------------------------------------------------------------------------------------------
+   fedora-30-build-side-123456    fedora-30-build-side-123456    fedora-30-build-side-123456
+
    $ koji remove-sidetag f30-build-side-123456
 
 API