996 lines
43 KiB
Python
996 lines
43 KiB
Python
# -*- coding:utf-8 -*-
|
|
#
|
|
# MIT License
|
|
#
|
|
# Copyright (c) 2022 Mattia Verga <mattia.verga@protonmail.com>
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
#
|
|
|
|
"""Detect inactive user in the packager group.
|
|
|
|
The script will search for any activity in the last year in:
|
|
|
|
- koji builds
|
|
- src.fedoraproject.org
|
|
- pagure.io
|
|
- bodhi (through datagrepper)
|
|
- Fedora mailing lists
|
|
- Fedora Discussion
|
|
- Red Hat bugzilla
|
|
|
|
The step-one subcommand will output the 'inactive_packagers.csv' file with a list of all users
|
|
in the packager group and their email which are found to be inactive. This can be used to contact
|
|
the inactive users, asking them to show some activity in pagure, datagrepper or mailing lists.
|
|
Optionally, it can open tickets against inactive packagers in a Pagure repository.
|
|
|
|
The check-impact subcommand will iterate on the list of tickets opened and provide information
|
|
about how many packages will possibly be orphaned based on the packagers detected as inactive.
|
|
It can be used with the `--post-comment` to post a comment on each inactive_packager ticket
|
|
with a list of packages / users / groups affected by user removal.
|
|
|
|
The step-two subcommand can either check a list of inactive users from the csv file generated
|
|
in step-one, or check a list of users based on open Pagure tickets. In this last case, the tickets
|
|
can be closed appropriately. Another csv file is generated as output to list users which are still
|
|
inactive.
|
|
|
|
Use the '--privacy' flag to not print emails in the log.
|
|
|
|
To run the script you'll need:
|
|
|
|
- click, fasjson_client, koji, and python-bugzilla to be installed
|
|
- an active kerberos ticket to login to fasjson.fedoraproject.org
|
|
- a bugzilla API key for bugzilla.redhat.com in your ~/.bugzillarc config file
|
|
|
|
Optionally, providing a Pagure repository URL and a token the script can automatically
|
|
open and manage an issue ticket for each detected inactive packager.
|
|
|
|
"""
|
|
|
|
import click
|
|
import logging
|
|
import requests
|
|
import sys
|
|
from copy import copy
|
|
from datetime import datetime, timedelta, timezone
|
|
from os import getenv
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.retry import Retry
|
|
|
|
from bugzilla import Bugzilla
|
|
from fasjson_client import Client
|
|
from koji import ClientSession
|
|
|
|
PAGURE_API_KEY = getenv('PAGURE_API_KEY', None)
|
|
PAGURE_API_BASE_URL = getenv('PAGURE_API_BASE_URL', 'https://pagure.io/api/0/find-inactive-packagers')
|
|
# A tag for marking "inactive packager" ticket or empty string
|
|
PAGURE_NEW_TICKET_TAG = getenv('PAGURE_NEW_TICKET_TAG', 'inactive_packager')
|
|
# A tag for marking tickets where user cannot be tagged because is not registered in pagure.io
|
|
PAGURE_NOUSER_TAG = getenv('PAGURE_NOUSER_TAG', 'user_not_taggable')
|
|
# A tag for marking tickets where user cannot be identified in bugzilla
|
|
PAGURE_NOBUGZILLAUSER_TAG = getenv('PAGURE_NOBUGZILLAUSER_TAG', 'bugzilla_user_not_found')
|
|
# The tag used when a user reply to a ticket asking to be removed from packager
|
|
PAGURE_ASK_REMOVAL_TAG = getenv('PAGURE_ASK_REMOVAL_TAG', 'agreed_removal')
|
|
# A username to whom assign the ticket by default or None
|
|
PAGURE_NEW_TICKET_ASSIGNEE = getenv('PAGURE_NEW_TICKET_ASSIGNEE', None)
|
|
|
|
FASCLIENT_URL = 'https://fasjson.fedoraproject.org/'
|
|
|
|
KOJI_HUB = 'https://koji.fedoraproject.org/kojihub'
|
|
KOJI_BUILD_CUTOFF = (datetime.now(timezone.utc) - timedelta(weeks=52)).strftime("%Y-%m-%d 00:00:00")
|
|
|
|
# A list of system users that should never be removed from the packager group
|
|
EXCLUDE_USERS = ['orphan', 'packagerbot', 'releng', 'rhcontainerbot']
|
|
|
|
PING_INACTIVE_TEXT = '''Hello @{username},
|
|
|
|
during our periodic check as per the "[Inactive packagers policy](https://docs.fedoraproject.org/en-US/fesco/Policy_for_inactive_packagers/)"
|
|
we detected no activity from you as a packager, nor in
|
|
other Fedora community places, like Bodhi or mailing lists.
|
|
|
|
|
|
In order to reduce security risks from possible accounts hijacking
|
|
we\'re trying to contact you to know if you\'re still reachable and
|
|
if your email set in Fedora Account System ({email}) is still valid.
|
|
|
|
|
|
Please, let us know if you\'re still intersted in participating
|
|
in Fedora and if you still need your account to be listed in the `packager` group.
|
|
|
|
Without any reply from you, in two months the `packager` group membership will be
|
|
removed from your account and any package for which you are the main admin will
|
|
be orphaned, so that co-maintainers can pick them up.
|
|
'''
|
|
|
|
CHECK_IMPACT_OUTPUT = '''### Found {total_packages} packages which will possibly be orphaned: ###
|
|
|
|
{package_list}
|
|
|
|
Packages marked with an asterisk means that the user replied to the ticket asking to be removed
|
|
from the packager group, therefeore you can try to contact them and ask for the package to be
|
|
assigned to you, if you're interested in continue maintaining it.
|
|
|
|
Have a wonderful day and see you (maybe?) at the next run!
|
|
|
|
'''
|
|
|
|
CHECK_IMPACT_COMMENT = '''The following is a list of packages that may become orphan if the user
|
|
@{username} is removed from the `packager` group. Affected co-maintainers users and groups are
|
|
provided as well:
|
|
|
|
{package_list}
|
|
|
|
Note that this is just an early warning for package co-maintainers/collaborators: the maintainer
|
|
can still show up before the orphaning happen.
|
|
Anyone who wants to become the maintainer of a package before it is orphaned can try to contact
|
|
the current maintainer themselves or, if you **really** can't wait after the automatic orphaning,
|
|
open a ticket to Fedora Releng.
|
|
'''
|
|
|
|
logging.basicConfig(level=logging.INFO,
|
|
format='%(message)s')
|
|
log = logging.getLogger('find-inactive-packagers')
|
|
|
|
|
|
def _retry_session(retries, session=None, backoff_factor=0.3):
|
|
session = session or requests.Session()
|
|
retry = Retry(
|
|
total=retries,
|
|
read=retries,
|
|
connect=retries,
|
|
backoff_factor=backoff_factor
|
|
)
|
|
adapter = HTTPAdapter(max_retries=retry)
|
|
session.mount('http://', adapter)
|
|
session.mount('https://', adapter)
|
|
return session
|
|
|
|
session = _retry_session(retries=5)
|
|
|
|
|
|
def _fetch_tickets():
|
|
"""Get open tickets from pagure.
|
|
|
|
Retrieve the list of open "inactive packager" tickets and return two dicts of
|
|
usernames with their associated ticket id: one dict is about users which didn't yet
|
|
reply, the second dict is about users which replied asking to be removed.
|
|
"""
|
|
page = 1
|
|
user_ticket_waiting_reply = dict()
|
|
user_ticket_pending_removal = dict()
|
|
log.info(f'### Fetching open tickets. ###')
|
|
while True:
|
|
try:
|
|
data = session.get(f'{PAGURE_API_BASE_URL}/issues?status=Open&tags={PAGURE_NEW_TICKET_TAG}'
|
|
f'&per_page=100&page={page}').json()
|
|
issues = data.get('issues', [])
|
|
total_pages = data.get('pagination', dict()).get('pages', 1)
|
|
for issue in issues:
|
|
ticket_id = issue.get('id')
|
|
if not issue.get('title').startswith('Inactive packager detected for user '):
|
|
log.error(f"ERROR: Ticket {ticket_id} does not appear to be a valid inactive_packager ticket.")
|
|
continue
|
|
username = issue.get('title').split()[-1]
|
|
if PAGURE_ASK_REMOVAL_TAG in issue.get('tags'):
|
|
user_ticket_pending_removal[username] = ticket_id
|
|
else:
|
|
user_ticket_waiting_reply[username] = ticket_id
|
|
log.info(f"Done {page} pages out of {total_pages}.")
|
|
if page >= total_pages:
|
|
break
|
|
page += 1
|
|
except Exception:
|
|
log.warning('Error while retrieving ticket list.')
|
|
break
|
|
return user_ticket_waiting_reply, user_ticket_pending_removal
|
|
|
|
|
|
def _comment_pagure_ticket(ticket_id, comment):
|
|
"""Post a comment on pagure ticket."""
|
|
headers = {'Authorization': f'token {PAGURE_API_KEY}'}
|
|
data = {'comment': comment}
|
|
try:
|
|
log.debug(f'Posting comment on ticket {ticket_id}')
|
|
resp = session.post(f'{PAGURE_API_BASE_URL}/issue/{ticket_id}/comment', data=data, headers=headers)
|
|
if resp.status_code == 401:
|
|
log.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
if resp.status_code != 200:
|
|
log.error(f'ERROR: Error posting comment on ticket {ticket_id}')
|
|
except Exception:
|
|
log.error(f'ERROR: Error posting comment on ticket {ticket_id}')
|
|
|
|
|
|
def _close_pagure_ticket(ticket_id, close_status, comment=''):
|
|
"""Close pagure ticket with comment."""
|
|
headers = {'Authorization': f'token {PAGURE_API_KEY}'}
|
|
if comment:
|
|
data = {'comment': comment}
|
|
try:
|
|
log.debug(f'Posting comment on ticket {ticket_id}')
|
|
resp = session.post(f'{PAGURE_API_BASE_URL}/issue/{ticket_id}/comment', data=data, headers=headers)
|
|
if resp.status_code == 401:
|
|
log.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
if resp.status_code != 200:
|
|
log.error(f'ERROR: Error posting comment on ticket {ticket_id}')
|
|
except Exception:
|
|
log.error(f'ERROR: Error posting comment on ticket {ticket_id}')
|
|
|
|
log.debug(f'Closing ticket {ticket_id}')
|
|
data = {'status': 'Closed',
|
|
'close_status': close_status}
|
|
try:
|
|
resp = session.post(f'{PAGURE_API_BASE_URL}/issue/{ticket_id}/status', data=data, headers=headers)
|
|
if resp.status_code == 401:
|
|
log.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
if resp.status_code != 200:
|
|
log.error(f'ERROR: Error on changing status of ticket {ticket_id}')
|
|
except Exception:
|
|
log.error(f'ERROR: Error on changing status of ticket {ticket_id}')
|
|
|
|
|
|
def _check_koji_activity(kojisession, username):
|
|
"""Look for user latest builds in a Koji."""
|
|
if not username:
|
|
log.error('ERROR: Cannot check an empty username!')
|
|
return False
|
|
try:
|
|
packagerUser = kojisession.getUser(username)
|
|
except Exception:
|
|
raise AttributeError(f'Error retrieving user {username} information in koji')
|
|
if not packagerUser:
|
|
raise AttributeError(f'User {username} not found in koji')
|
|
packagerUid = packagerUser['id']
|
|
if kojisession.listBuilds(userID=packagerUid, createdAfter=KOJI_BUILD_CUTOFF):
|
|
log.info(f'Found recent builds in Koji for user {username}')
|
|
return True
|
|
return False
|
|
|
|
|
|
def _check_pagure_activity(user, base_url='https://src.fedoraproject.org'):
|
|
"""Look for user activity in a Pagure deployment."""
|
|
if not user:
|
|
log.error('ERROR: Cannot check an empty username!')
|
|
return False
|
|
resp_src = session.get(f'{base_url}/api/0/user/{user}/activity/stats')
|
|
try:
|
|
resp_json = resp_src.json()
|
|
except Exception:
|
|
raise ValueError(f'Received bad JSON data for user {user}')
|
|
if resp_json.get('error', None) is not None:
|
|
if resp_json.get('error_code', None) == 'ENOUSER':
|
|
raise AttributeError(f'User {user} not found in {base_url}')
|
|
log.info(f'Error checking user {user} in {base_url}: {error}')
|
|
return False
|
|
if not bool(resp_json):
|
|
log.info(f'No activity detected for user {user} in {base_url}')
|
|
return False
|
|
log.info(f'User {user} was active in {base_url}')
|
|
return True
|
|
|
|
|
|
def _check_bodhi_activity(user):
|
|
"""Look for user activity in Fedora Bodhi through datagrepper messages."""
|
|
if not user:
|
|
log.error('ERROR: Cannot check an empty username!')
|
|
return False
|
|
try:
|
|
page = 1
|
|
while True:
|
|
r = session.get(f'https://apps.fedoraproject.org/datagrepper/raw/?user={user}'
|
|
f'&category=bodhi&order=desc&delta=31536000&rows_per_page=50'
|
|
f'&page={page}').json()
|
|
for message in r.get('raw_messages', []):
|
|
agent = message.get('msg', dict()).get('agent', None)
|
|
if agent == user:
|
|
log.info(f'Found recent activity in Bodhi for user {user}')
|
|
return True
|
|
if page >= r.get('pages', 1):
|
|
return False
|
|
else:
|
|
page += 1
|
|
except Exception:
|
|
# May happen if there are a lot of entries in datagrepper
|
|
log.error(f'ERROR: Error while retrieving data for user {user}')
|
|
return False
|
|
|
|
|
|
def _check_maillists_activity(fasclient, user, delta=365):
|
|
"""Look for user recent activity in Fedora mailing lists.
|
|
|
|
Returns a tuple where the first item is a bool and the second item is the list
|
|
of email addresses checked.
|
|
"""
|
|
if not user:
|
|
log.error('ERROR: Cannot check an empty username!')
|
|
return (False, [])
|
|
try:
|
|
packager = fasclient.get_user(username=user).result
|
|
emails = packager['emails']
|
|
if packager['rhbzemail']:
|
|
emails.append(packager['rhbzemail'])
|
|
if not emails:
|
|
log.error(f'ERROR: User {user} has no emails in FAS')
|
|
return (False, [])
|
|
for email in emails:
|
|
r = session.get(f'https://lists.fedoraproject.org/archives/api/sender/{email}/emails/?ordering=-date').json()
|
|
if posts := r.get('results'):
|
|
last_email_date = datetime.fromisoformat(posts[0]['date'].replace('Z', '+00:00'))
|
|
time_delta = datetime.now(timezone.utc) - last_email_date
|
|
if time_delta.days <= delta:
|
|
log.info(f'Found recent email from user {user}: {last_email_date}.')
|
|
return (True, emails)
|
|
except Exception:
|
|
log.error(f'ERROR: Error while retrieving last mailing lists activity for user {user}.')
|
|
return (False, [])
|
|
return (False, emails)
|
|
|
|
|
|
def _check_discourse_activity(user, delta=365):
|
|
"""Look for user recent activity in Fedora Discussion."""
|
|
if not user:
|
|
log.error('ERROR: Cannot check an empty username!')
|
|
return False
|
|
try:
|
|
# I cannot find any documentation, but action types 4 and 5 should mean user posting to
|
|
# a new thread or replying to existing thread
|
|
r = session.get(f'https://discussion.fedoraproject.org/user_actions.json?username={user}&filter=4,5&limit=1').json()
|
|
if r.get('errors', None) or not r.get('user_actions', []):
|
|
return False
|
|
last_user_action_date = datetime.fromisoformat(r.get('user_actions')[0]['created_at'].replace('Z', '+00:00'))
|
|
time_delta = datetime.now(timezone.utc) - last_user_action_date
|
|
if time_delta.days <= delta:
|
|
log.info(f'Found recent post from user {user}: {last_user_action_date}.')
|
|
return True
|
|
except Exception:
|
|
log.error(f'ERROR: Error while retrieving Fedora Discussion activity for user {user}.')
|
|
return False
|
|
return False
|
|
|
|
|
|
def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privacy=False, delta=365):
|
|
"""Look for user recent activity in Bugzilla.
|
|
|
|
Arg packager is a tuple of username and a list of emails.
|
|
"""
|
|
username = packager[0]
|
|
if not username:
|
|
log.error('ERROR: Cannot check an empty username!')
|
|
return False
|
|
emails = packager[1]
|
|
bzuser = None
|
|
if check_fedora_alias and f'{username}@fedoraproject.org' not in emails:
|
|
# Some users have their @fedoraproject.org alias set as BZ email
|
|
emails.append(f'{username}@fedoraproject.org')
|
|
for bzemail in emails:
|
|
try:
|
|
bzuser = bzclient.getuser(bzemail)
|
|
log.debug(f'Found user {username} by email {mask_email(bzemail, privacy=privacy)}.')
|
|
break
|
|
except Exception:
|
|
log.warning(f'Unable to find user {username} by email {mask_email(bzemail, privacy=privacy)} in Bugzilla.')
|
|
if not bzuser:
|
|
raise AttributeError(f'ERROR: Unable to check user {username} activity in Bugzilla because I cannot find them.')
|
|
try:
|
|
bugs = bzclient.query({
|
|
"email1" : bzuser.email,
|
|
"emaillongdesc1" : "1",
|
|
"emailtype1" : "substring",
|
|
"query_format" : "advanced",
|
|
"order": "changeddate DESC",
|
|
"chfieldfrom" : "-1y",
|
|
"chfieldto" : "Now",
|
|
"limit": 1000})
|
|
except Exception:
|
|
log.error(f'ERROR: Error while retrieving Bugzilla data for user {username}')
|
|
return False
|
|
ids = [bug.bug_id for bug in bugs]
|
|
user_comments = []
|
|
try:
|
|
bzbugs = bzclient.getbugs(ids)
|
|
except Exception:
|
|
log.error(f'ERROR: Error while retrieving Bugzilla comments for user {username}')
|
|
return False
|
|
for bug in bzbugs:
|
|
user_comments.extend([com for com in bug.longdescs if com['creator_id'] == bzuser.userid])
|
|
if not user_comments:
|
|
return False
|
|
last_com_date = None
|
|
for comment in user_comments:
|
|
if not last_com_date or last_com_date < comment.get('time'):
|
|
last_com_date = comment.get('time')
|
|
# convert DateTime object to datetime.datetime
|
|
last_com_date = datetime.fromisoformat(f'{last_com_date.value}Z')
|
|
time_delta = datetime.now(timezone.utc) - last_com_date
|
|
if time_delta.days <= delta:
|
|
log.info(f'User {username} made a comment in Bugzilla on {last_com_date.strftime("%Y_%m_%d")}.')
|
|
return True
|
|
return False
|
|
|
|
|
|
def _check_user_activity(user, privacy=False, fasclient=None, bzclient=None):
|
|
"""Check user activity."""
|
|
if not fasclient:
|
|
try:
|
|
fasclient = Client(FASCLIENT_URL)
|
|
except Exception:
|
|
log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain '
|
|
'a Kerberos ticket.')
|
|
raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain '
|
|
'a Kerberos ticket.')
|
|
|
|
kojiSession = ClientSession(KOJI_HUB)
|
|
|
|
log.info(f'Checking {user} activity in Koji...')
|
|
try:
|
|
if _check_koji_activity(kojiSession, user):
|
|
return True
|
|
except AttributeError as ex:
|
|
log.info(f'{ex}')
|
|
log.info(f'Checking {user} activity in Pagure...')
|
|
try:
|
|
src_fpo = _check_pagure_activity(user)
|
|
except AttributeError as ex:
|
|
log.info(f'{ex}')
|
|
src_fpo = False
|
|
except ValueError as ex:
|
|
log.info(f'{ex}')
|
|
src_fpo = False
|
|
try:
|
|
pagure = _check_pagure_activity(user, base_url='https://pagure.io')
|
|
except AttributeError as ex:
|
|
log.info(f'{ex}')
|
|
pagure = False
|
|
except ValueError as ex:
|
|
log.info(f'{ex}')
|
|
pagure = False
|
|
if src_fpo or pagure:
|
|
return True
|
|
log.info(f'Checking {user} activity in Fedora Discussion...')
|
|
if _check_discourse_activity(user):
|
|
return True
|
|
log.info(f'Checking {user} activity in Bodhi...')
|
|
if _check_bodhi_activity(user):
|
|
return True
|
|
log.info(f'Checking {user} activity in mailing lists...')
|
|
maillist_result = _check_maillists_activity(fasclient, user)
|
|
if maillist_result[0]:
|
|
return True
|
|
if bzclient:
|
|
log.info(f'Checking {user} activity in bugzilla...')
|
|
packager = (user, maillist_result[1])
|
|
try:
|
|
bugzilla = _check_bugzilla_activity(bzclient, packager, privacy=privacy)
|
|
except AttributeError as ex:
|
|
log.error(f'{ex}')
|
|
bugzilla = False
|
|
if bugzilla:
|
|
return True
|
|
return False
|
|
|
|
|
|
def mask_email(email, preserve_chars=3, mask_char='*', privacy=True):
|
|
"""Obfuscate the end of the local part of an email address.
|
|
|
|
By default the three starting characters are preserved, while the ending of the local
|
|
part is masked.
|
|
The function always tries to mask at least 2 characters, in case of short addresses.
|
|
|
|
Args:
|
|
email: a valid email address to mask.
|
|
preserve_chars: number of characters to preserve at the beginning of the local part.
|
|
mask_char: the character used as a replacement.
|
|
"""
|
|
if not privacy:
|
|
return email
|
|
valid_email = email.split('@')
|
|
if not len(valid_email) == 2:
|
|
log.debug(f'Invalid email: {email}')
|
|
return email
|
|
local_length = len(valid_email[0])
|
|
if local_length - preserve_chars < 2:
|
|
preserve_chars = local_length - 2
|
|
if preserve_chars < 1:
|
|
local_part = mask_char * 2
|
|
else:
|
|
local_part = (valid_email[0][:preserve_chars]
|
|
+ mask_char * (local_length - preserve_chars))
|
|
return f'{local_part}@{valid_email[1]}'
|
|
|
|
|
|
def _get_affected_packages_users(session, username, mark=''):
|
|
"""Get the list of packages orphaned when a user is removed from packager group.
|
|
|
|
Also get a list of affected co-maintainers impacted by the orphaning process.
|
|
|
|
Args:
|
|
username: the username which will be removed from packagers.
|
|
mark: used in 'check-impact'
|
|
Returns:
|
|
A list of tuples containing:
|
|
(package name, username, mark, set(co-maintainers, set(affected groups))
|
|
"""
|
|
page = 1
|
|
list_affected = []
|
|
while True:
|
|
try:
|
|
data = session.get(f'https://src.fedoraproject.org/api/0/user/{username}'
|
|
f'?per_page=100&repopage={page}')
|
|
try:
|
|
assert data != ''
|
|
data = data.json()
|
|
count = 0
|
|
except Exception:
|
|
log.warning(f'Received empty response data for user {username} on page {page}.')
|
|
count += 1
|
|
if count >= 5:
|
|
raise Exception
|
|
else:
|
|
continue
|
|
repos = data.get('repos', [])
|
|
total_pages = data.get('repos_pagination', dict()).get('pages', 1)
|
|
for repo in repos:
|
|
if username in repo.get('access_users', dict()).get('owner'):
|
|
package = repo.get('name')
|
|
if repo.get('namespace', 'rpms') != 'rpms':
|
|
package += f'[{repo.get("namespace")}]'
|
|
co_maintainers = set(repo.get('access_users').get('admin', []) +
|
|
repo.get('access_users').get('collaborator', []) +
|
|
repo.get('access_users').get('commit', []))
|
|
groups = set(repo.get('access_groups').get('admin', []) +
|
|
repo.get('access_groups').get('collaborator', []) +
|
|
repo.get('access_groups').get('commit', []))
|
|
list_affected.append((package, username, mark, co_maintainers, groups))
|
|
log.info(f"Done {page} pages out of {total_pages}.")
|
|
if page >= total_pages:
|
|
break
|
|
page += 1
|
|
except Exception as e:
|
|
log.warning('Error while retrieving user info.')
|
|
print(e)
|
|
break
|
|
return list_affected
|
|
|
|
|
|
@click.group()
|
|
@click.option('--privacy', is_flag=True, default=False, help='Hide users email in log.')
|
|
@click.option('-D', '--debug', is_flag=True, default=False, help='Enable logging of debug messages.')
|
|
@click.pass_context
|
|
def cli(ctx, debug, privacy):
|
|
"""Detect inactive users in the packager group."""
|
|
ctx.ensure_object(dict)
|
|
ctx.obj['privacy'] = privacy
|
|
ctx.obj['debug'] = debug
|
|
|
|
|
|
@cli.command()
|
|
@click.option('--open-tickets', is_flag=True, default=False, help='File tickets in Pagure.')
|
|
@click.option('--with-bz-check/--without-bz-check', default=True,
|
|
help='Run (or not) activity checks on RedHat Bugzilla.')
|
|
@click.pass_context
|
|
def step_one(ctx, with_bz_check, open_tickets):
|
|
"""Find inactive packagers.
|
|
|
|
If open-tickets flag is activated, the script will file tickets in Pagure, otherwise
|
|
it will only output a csv file.
|
|
"""
|
|
privacy = ctx.obj['privacy']
|
|
fh = logging.FileHandler(f'step_one_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+')
|
|
if ctx.obj['debug']:
|
|
log.setLevel(logging.DEBUG)
|
|
log.addHandler(fh)
|
|
|
|
if open_tickets:
|
|
if not PAGURE_API_KEY or not PAGURE_API_BASE_URL:
|
|
log.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, '
|
|
'queue processing will stop immediately.')
|
|
raise SystemExit()
|
|
|
|
if with_bz_check:
|
|
bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi')
|
|
try:
|
|
assert bzclient.logged_in
|
|
except Exception:
|
|
log.error('ERROR: An authenticated session to bugzilla.redhat.com is required, '
|
|
'maybe you forgot to provide an API key for logging in? '
|
|
'Queue processing will stop immediately.')
|
|
raise SystemExit()
|
|
|
|
kojiSession = ClientSession(KOJI_HUB)
|
|
|
|
try:
|
|
fasclient = Client(FASCLIENT_URL)
|
|
except Exception:
|
|
log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain '
|
|
'a Kerberos ticket.')
|
|
raise SystemExit()
|
|
packagers = fasclient.list_group_members(groupname='packager').result
|
|
log.info(f'### Found {len(packagers)} users in the packager group. ###')
|
|
inactive_packagers = []
|
|
|
|
# Check for activity in Koji
|
|
log.info('Checking users activity in Koji...')
|
|
for i, p in enumerate(packagers):
|
|
if i > 0 and i % 100 == 0:
|
|
log.info(f'Done {i}.')
|
|
user = p.get('username')
|
|
if user in EXCLUDE_USERS:
|
|
continue
|
|
try:
|
|
koji_active = _check_koji_activity(kojiSession, user)
|
|
except AttributeError as ex:
|
|
log.info(f'{ex}')
|
|
koji_active = False
|
|
if not koji_active:
|
|
inactive_packagers.append(user)
|
|
|
|
log.info(f'### Found {len(inactive_packagers)} users with no builds in Koji over the last year. ###')
|
|
|
|
# Check for activity in Pagure
|
|
log.info('Checking users activity in Pagure...')
|
|
nouser_in_pagureio = []
|
|
nouser_in_bugzilla = []
|
|
for i, user in enumerate(copy(inactive_packagers)):
|
|
if i > 0 and i % 100 == 0:
|
|
log.info(f'Done {i}.')
|
|
try:
|
|
src_fpo = _check_pagure_activity(user)
|
|
except AttributeError as ex:
|
|
log.info(f'{ex}')
|
|
src_fpo = False
|
|
except ValueError as ex:
|
|
log.info(f'{ex}')
|
|
src_fpo = False
|
|
pagure = False
|
|
if not src_fpo:
|
|
try:
|
|
pagure = _check_pagure_activity(user, base_url='https://pagure.io')
|
|
except AttributeError as ex:
|
|
log.info(f'{ex}')
|
|
pagure = False
|
|
nouser_in_pagureio.append(user)
|
|
except ValueError as ex:
|
|
log.info(f'{ex}')
|
|
pagure = False
|
|
nouser_in_pagureio.append(user)
|
|
if src_fpo or pagure:
|
|
inactive_packagers.remove(user)
|
|
|
|
log.info(f'### Found {len(inactive_packagers)} users with no activity in pagure.io or src.fp.org over the last year. ###')
|
|
|
|
# Check for recent posts in Fedora Discussion
|
|
log.info('Checking users activity in Fedora Discussion...')
|
|
for i, p in enumerate(copy(inactive_packagers)):
|
|
if i > 0 and i % 100 == 0:
|
|
log.info(f'Done {i}.')
|
|
if _check_discourse_activity(p):
|
|
inactive_packagers.remove(p)
|
|
|
|
log.info(f'### Found {len(inactive_packagers)} users which didn\'t post any message in Fedora Discussion over the last year. ###')
|
|
|
|
# Check for activity in Bodhi through datagrepper messages
|
|
log.info('Checking users activity in Bodhi...')
|
|
for i, p in enumerate(copy(inactive_packagers)):
|
|
if i > 0 and i % 100 == 0:
|
|
log.info(f'Done {i}.')
|
|
if _check_bodhi_activity(p):
|
|
inactive_packagers.remove(p)
|
|
|
|
log.info(f'### Found {len(inactive_packagers)} users which also show no activity in Bodhi over the last year. ###')
|
|
|
|
# Check for any activity in mailing lists
|
|
log.info('Checking users activity in mailing lists...')
|
|
packager_email_map = {}
|
|
for i, p in enumerate(inactive_packagers):
|
|
if i > 0 and i % 100 == 0:
|
|
log.info(f'Done {i}.')
|
|
maillist_results = _check_maillists_activity(fasclient, p)
|
|
if not maillist_results[0]:
|
|
packager_email_map[p] = maillist_results[1]
|
|
|
|
log.info(f'### Found {len(packager_email_map)} users which also show no activity in mailing lists over the last year. ###')
|
|
|
|
# Check for activity in Bugzilla
|
|
if with_bz_check:
|
|
log.info('Checking users activity in bugzilla...')
|
|
for i, p in enumerate(copy(packager_email_map).items()):
|
|
if i > 0 and i % 100 == 0:
|
|
log.info(f'Done {i}.')
|
|
try:
|
|
bugzilla = _check_bugzilla_activity(bzclient, p, privacy=privacy)
|
|
except AttributeError as ex:
|
|
log.error(f'{ex}')
|
|
bugzilla = False
|
|
nouser_in_bugzilla.append(p[0])
|
|
if bugzilla:
|
|
packager_email_map.pop(p[0])
|
|
log.info(f'### Found {len(packager_email_map)} users which also show no activity in Bugzilla over the last year. ###')
|
|
|
|
if packager_email_map:
|
|
with open('inactive_packagers.csv', 'w') as fout:
|
|
for user, emails in sorted(packager_email_map.items()):
|
|
# Open Pagure tickets
|
|
ticket_id = 'NONE'
|
|
tktags = [PAGURE_NEW_TICKET_TAG]
|
|
if user in nouser_in_pagureio:
|
|
nosuchuser = 'not registered in pagure.io'
|
|
tktags.append(PAGURE_NOUSER_TAG)
|
|
else:
|
|
nosuchuser = ''
|
|
if user in nouser_in_bugzilla:
|
|
nobugzilla = 'not found in bugzilla'
|
|
tktags.append(PAGURE_NOBUGZILLAUSER_TAG)
|
|
else:
|
|
nobugzilla = ''
|
|
if open_tickets:
|
|
log.debug(f'Opening ticket for user {user}')
|
|
headers = {'Authorization': f'token {PAGURE_API_KEY}'}
|
|
ping_email = mask_email(emails[0]) if len(emails) > 0 else '**ENOEMAIL**'
|
|
data = {'title': f'Inactive packager detected for user {user}',
|
|
'issue_content': PING_INACTIVE_TEXT.format(username = user, email = ping_email),
|
|
'tag': ','.join(tktags),
|
|
'assignee': PAGURE_NEW_TICKET_ASSIGNEE}
|
|
try:
|
|
resp = session.post(f'{PAGURE_API_BASE_URL}/new_issue', data=data, headers=headers)
|
|
if resp.status_code == 401:
|
|
log.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.')
|
|
if resp.status_code != 200:
|
|
log.error(f'ERROR: Error opening Pagure ticket for user {user}')
|
|
ticket_id = 'ERROR'
|
|
else:
|
|
ticket_id = resp.json().get('issue', dict()).get('id', 'ERROR')
|
|
except Exception:
|
|
log.error(f'ERROR: Error opening Pagure ticket for user {user}')
|
|
ticket_id = 'ERROR'
|
|
# Write results to file
|
|
emailstring = '|'.join([mask_email(em, privacy=privacy) for em in emails])
|
|
log.info(f'{user} - {ticket_id} - {emailstring} - {nosuchuser} - {nobugzilla}')
|
|
fout.write(f'{user},{ticket_id},{emailstring},{nosuchuser},{nobugzilla}\n')
|
|
else:
|
|
log.info('### No inactive packagers detected, YHAY! Nothing to do. ###')
|
|
|
|
|
|
@cli.command()
|
|
@click.option('--from-file', type=click.File('r'), help='Use local csv files instead of Pagure.')
|
|
@click.option('--close-tickets', is_flag=True, default=False, help='Close tickets in Pagure.')
|
|
@click.option('--with-bz-check/--without-bz-check', default=True,
|
|
help='Run (or not) activity checks on RedHat Bugzilla.')
|
|
@click.pass_context
|
|
def step_two(ctx, with_bz_check, close_tickets, from_file):
|
|
"""Check any user still inactive.
|
|
|
|
If a csv file is provided, the script will use that as input, rather than fetching tickets from
|
|
Pagure. Users listed in the csv file will be checked for activity and another csv file will
|
|
be provided to list users that are still detected inactive.
|
|
|
|
Otherwise, the list of inactive packagers is obtained by open Pagure tickets. These will be
|
|
checked for activity and tickets will be closed appropriately at the same time a csv file is
|
|
provided in output. Special flags applied to open tickets (for example 'asked_removal') will
|
|
be considered appropriately.
|
|
"""
|
|
privacy = ctx.obj['privacy']
|
|
fh = logging.FileHandler(f'step_two_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+')
|
|
if ctx.obj['debug']:
|
|
log.setLevel(logging.DEBUG)
|
|
log.addHandler(fh)
|
|
|
|
if close_tickets:
|
|
if not PAGURE_API_KEY or not PAGURE_API_BASE_URL:
|
|
log.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, '
|
|
'queue processing will stop immediately.')
|
|
raise SystemExit()
|
|
|
|
if with_bz_check:
|
|
bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi')
|
|
try:
|
|
assert bzclient.logged_in
|
|
except Exception:
|
|
log.error('ERROR: An authenticated session to bugzilla.redhat.com is required, '
|
|
'maybe you forgot to provide an API key for logging in? '
|
|
'Queue processing will stop immediately.')
|
|
raise SystemExit()
|
|
else:
|
|
bzclient = None
|
|
|
|
try:
|
|
fasclient = Client(FASCLIENT_URL)
|
|
except Exception:
|
|
log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain '
|
|
'a Kerberos ticket.')
|
|
raise SystemExit()
|
|
|
|
if from_file:
|
|
log.debug('Using CSV file')
|
|
pending_removal_users = []
|
|
for line in from_file:
|
|
user, ticket, emails, pagureio = line.split(',')
|
|
pending_removal_users.append(user)
|
|
log.info(f'### Checking {len(pending_removal_users)} users from csv file ###')
|
|
for i, p in enumerate(copy(pending_removal_users)):
|
|
if i > 0 and i % 100 == 0:
|
|
log.info(f'Done {i}.')
|
|
if _check_user_activity(p, privacy=privacy, fasclient=fasclient, bzclient=bzclient):
|
|
pending_removal_users.remove(p)
|
|
if pending_removal_users:
|
|
pending_removal_user.sort()
|
|
log.info(f'### These {len(pending_removal_users)} users didn\'t react to previous inquiry: ###')
|
|
with open('still_inactive.csv', 'w') as finactive:
|
|
for user in pending_removal_users:
|
|
log.info(f'{user}')
|
|
finactive.write(f'{user}\n')
|
|
else:
|
|
log.info('### All users resumed activity, YHAY! Nothing to do. ###')
|
|
else:
|
|
log.debug('Using Pagure tickets')
|
|
confirmed_removal = []
|
|
resumed_activity = []
|
|
waiting, pending = _fetch_tickets()
|
|
log.info(f'### Pagure returned {len(waiting)} users to check a second time and {len(pending)} users ready for removal ###')
|
|
if waiting:
|
|
log.info('Starting second check on users...')
|
|
for user, ticket_id in waiting.items():
|
|
if not _check_user_activity(user, privacy=privacy, fasclient=fasclient, bzclient=bzclient):
|
|
confirmed_removal.append((user, ticket_id))
|
|
else:
|
|
resumed_activity.append((user, ticket_id))
|
|
|
|
if confirmed_removal:
|
|
confirmed_removal.sort()
|
|
log.info(f'### These {len(confirmed_removal)} users didn\'t reply and are going to be removed from packagers: ###')
|
|
for user, ticket in confirmed_removal:
|
|
log.info(f'- {user}')
|
|
else:
|
|
log.info('### All users resumed activity, HURRAY! ###')
|
|
|
|
if pending:
|
|
log.info(f'### These users agreed to be removed from packagers: ###')
|
|
for user, ticket_id in sorted(pending.items()):
|
|
log.info(f'- {user}')
|
|
confirmed_removal.append((user, ticket_id))
|
|
|
|
if resumed_activity:
|
|
resumed_activity.sort()
|
|
log.info(f'### These {len(resumed_activity)} users resumed activity: ###')
|
|
for user, ticket_id in resumed_activity:
|
|
log.info(f'- {user}')
|
|
|
|
if confirmed_removal:
|
|
confirmed_removal.sort()
|
|
if close_tickets:
|
|
log.info(f'### Closing tickets and generating output file ###')
|
|
for user, ticket_id in resumed_activity:
|
|
log.info(f'Closing ticket {ticket_id} for user {user}...')
|
|
_close_pagure_ticket(
|
|
ticket_id,
|
|
'Keep packager status',
|
|
comment=(f'Recent activity has been detected for user {user} therefore '
|
|
'the ticket will be closed without further actions.')
|
|
)
|
|
with open('still_inactive.csv', 'w') as finactive:
|
|
for user, ticket_id in confirmed_removal:
|
|
if close_tickets:
|
|
log.info(f'Closing ticket {ticket_id} for user {user}...')
|
|
_close_pagure_ticket(
|
|
ticket_id,
|
|
'Removed from packagers',
|
|
comment=f'User {user} will be removed from packager group.'
|
|
)
|
|
# Write users list to file
|
|
finactive.write(f'{user}\n')
|
|
|
|
|
|
@cli.command()
|
|
@click.option('--on-the-fly', 'otf', is_flag=True, default=False, help='Check user activity on the fly.')
|
|
@click.option('--post-comment', 'comment', is_flag=True, default=False, help='Post comments on pagure tickets.')
|
|
@click.pass_context
|
|
def check_impact(ctx, comment, otf):
|
|
"""Report packages expected to be orphaned.
|
|
|
|
This report will show how many packages are going to be orphaned based on the
|
|
"inactive packager" tickets currently opened.
|
|
|
|
Running with the '--on-the-fly' flag active will cause the script to perform a check
|
|
about user activity for those users which haven't replied to the ticket yet, so it mimics
|
|
the final output of 'step-two'.
|
|
"""
|
|
privacy = ctx.obj['privacy']
|
|
fh = logging.FileHandler(f'check_impact_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+')
|
|
if ctx.obj['debug']:
|
|
log.setLevel(logging.DEBUG)
|
|
log.addHandler(fh)
|
|
|
|
if otf:
|
|
bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi')
|
|
try:
|
|
assert bzclient.logged_in
|
|
except Exception:
|
|
log.error('ERROR: An authenticated session to bugzilla.redhat.com is required, '
|
|
'maybe you forgot to provide an API key for logging in? '
|
|
'Queue processing will stop immediately.')
|
|
raise SystemExit()
|
|
|
|
try:
|
|
fasclient = Client(FASCLIENT_URL)
|
|
except Exception:
|
|
log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain '
|
|
'a Kerberos ticket.')
|
|
raise SystemExit()
|
|
|
|
waiting, pending = _fetch_tickets()
|
|
affected_packages = []
|
|
log.info(f'### Fetching affected users info ###')
|
|
for u, t in waiting.items():
|
|
log.info(f'Getting user {u} information')
|
|
if otf:
|
|
if _check_user_activity(u, privacy=privacy, fasclient=fasclient, bzclient=bzclient):
|
|
log.info(f"Activity detected for user {u}.")
|
|
continue
|
|
affected_packages.extend(_get_affected_packages_users(session, u))
|
|
for u, t in pending.items():
|
|
log.info(f'Getting user {u} information')
|
|
affected_packages.extend(_get_affected_packages_users(session, u, '*'))
|
|
package_list = "\n".join([f'- {mark}{package} (owned by {user})' for package, user, mark, co_maintainers, groups in sorted(affected_packages)])
|
|
log.info(CHECK_IMPACT_OUTPUT.format(total_packages=len(affected_packages), package_list=package_list))
|
|
|
|
if comment:
|
|
tickets = waiting | pending
|
|
for u, t in tickets.items():
|
|
user_package_map = []
|
|
for package, user, mark, co_maintainers, groups in sorted(affected_packages):
|
|
if user == u:
|
|
affected_users = ", ".join([f'@{affected_user}' for affected_user in co_maintainers])
|
|
affected_groups = ", ".join([f'@{affected_group}' for affected_group in groups])
|
|
user_package_map.append((package, affected_users, affected_groups))
|
|
package_list_comment = "\n".join([f'- {package}{" (affected users: "+affected_users+")" if affected_users else ""}{" (affected groups: "+affected_groups+")" if affected_groups else ""}' for package, affected_users, affected_groups in user_package_map])
|
|
if package_list_comment:
|
|
_comment_pagure_ticket(t, CHECK_IMPACT_COMMENT.format(package_list=package_list_comment, username=u))
|
|
|
|
|
|
@cli.command()
|
|
@click.argument('username')
|
|
@click.option('--with-bz-check/--without-bz-check', default=True,
|
|
help='Run (or not) activity checks on RedHat Bugzilla.')
|
|
@click.pass_context
|
|
def check_user(ctx, with_bz_check, username):
|
|
"""Check if any activity is detected for a specific username."""
|
|
privacy = ctx.obj['privacy']
|
|
if ctx.obj['debug']:
|
|
log.setLevel(logging.DEBUG)
|
|
|
|
if with_bz_check:
|
|
bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi')
|
|
try:
|
|
assert bzclient.logged_in
|
|
except Exception:
|
|
log.error('ERROR: An authenticated session to bugzilla.redhat.com is required, '
|
|
'maybe you forgot to provide an API key for logging in? '
|
|
'Queue processing will stop immediately.')
|
|
raise SystemExit()
|
|
|
|
if _check_user_activity(username, privacy=privacy, bzclient=bzclient):
|
|
click.echo(f"Activity detected for user {username}.")
|
|
else:
|
|
click.echo(f"User {username} appear to be inactive.")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
cli()
|