From 2f3adaa1ad12993d47ed757a04264eee20aa4a66 Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 04 2022 11:02:13 +0000 Subject: [PATCH 1/9] Add check-impact subcommand Signed-off-by: Mattia Verga --- diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index 04dd487..8242e2e 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -38,6 +38,10 @@ group and their email which are found to be inactive. This can be used to contac the inactive users, asking them to show some activity in pagure, datagrepper or mailing lists. The step-one subcommand will open tickets against inactive packagers in a Pagure repository. + +The check-impact subcommand will iterate on the list of tickets opened and provide information +about how many packages will possibly be orphaned based on the packagers detected as inactive. + The step-two is used to confront the results of a previous run (provided by the csv file) and a fresh run, to identify which users still don't show any activity. Therefore we should consider these users to have abandoned Fedora. @@ -60,15 +64,19 @@ from copy import copy from datetime import datetime, timezone from dateutil import parser as dateparser from os import getenv +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from bugzilla import Bugzilla from fasjson_client import Client BZ_API_KEY = getenv('BZ_API_KEY', None) PAGURE_API_KEY = getenv('PAGURE_API_KEY', None) -PAGURE_NEW_TICKET_URL = getenv('PAGURE_NEW_TICKET_URL', 'https://pagure.io/api/0/find-inactive-packagers/new_issue') -# A list of comma separated values to assign tags to the created ticket or empty string -PAGURE_NEW_TICKET_TAGS = getenv('PAGURE_NEW_TICKET_TAGS', 'inactive_packager') +PAGURE_API_BASE_URL = getenv('PAGURE_API_BASE_URL', 'https://pagure.io/api/0/find-inactive-packagers') +# A tag for marking "inactive packager" ticket or empty string +PAGURE_NEW_TICKET_TAG = getenv('PAGURE_NEW_TICKET_TAG', 'inactive_packager') +# The tag used when a user reply to a ticket asking to be removed from packager +PAGURE_ASK_REMOVAL_TAG = getenv('PAGURE_ASK_REMOVAL_TAG', 'asked_removal') # A username to whom assign the ticket by default or None PAGURE_NEW_TICKET_ASSIGNEE = getenv('PAGURE_NEW_TICKET_ASSIGNEE', None) @@ -93,6 +101,19 @@ in Fedora and if you still need your account to be listed in the `packager` grou Without any reply from you, in two months we will proceed to remove your account from the `packager` group. Your account will still be active. ''' + +CHECK_IMPACT_OUTPUT = '''### Found {total_packages} packages which will possibly be orphaned: ### + +{package_list} + +Packages marked with an asterisk means that the user replied to the ticket asking to be removed +from the packager group, therefeore you can try to contact them and ask for the package to be +assigned to you, if you're interested in continue maintaining it. + +Have a wonderful day and see you (maybe?) at the next run! + +''' + logging.basicConfig(level=logging.INFO, format='%(message)s', handlers=[ @@ -101,16 +122,63 @@ logging.basicConfig(level=logging.INFO, ]) -@click.group(invoke_without_command=True) -@click.option('--privacy', default=False, help='Hide users email in log.') -@click.pass_context -def cli(ctx, privacy): - """Detect inactive user in the packager group. +def _retry_session(retries, session=None, backoff_factor=0.3): + session = session or requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + +session = _retry_session(retries=5) + + +def _fetch_tickets(): + """Get open tickets from pagure. - Running the script without step-one or step-two subcommands only outputs results in csv files. + Retrieve the list of open "inactive packager" tickets and return two dicts of + usernames with their associated ticket id: one dict is about users which didn't yet + reply, the second dict is about users which replied asking to be removed. """ - ctx.ensure_object(dict) + page = 1 + user_ticket_waiting_reply = dict() + user_ticket_pending_removal = dict() + logging.info(f'### Fetching open tickets. ###') + while True: + try: + data = session.get(f'{PAGURE_API_BASE_URL}/issues?status=Open&tags={PAGURE_NEW_TICKET_TAG}' + f'&per_page=100&page={page}').json() + issues = data.get('issues', []) + total_pages = data.get('pagination', dict()).get('pages', 1) + for issue in issues: + ticket_id = issue.get('id') + username = issue.get('title').split()[-1] + if PAGURE_ASK_REMOVAL_TAG in issue.get('tags'): + user_ticket_pending_removal[username] = ticket_id + else: + user_ticket_waiting_reply[username] = ticket_id + logging.info(f"Done {page} pages out of {total_pages}.") + if page >= total_pages: + break + page += 1 + except Exception: + logging.warning('Error while retrieving ticket list.') + break + return user_ticket_waiting_reply, user_ticket_pending_removal + +def _check_activity(privacy=False): + """Run activity checks. + + Check for user activity on several Fedora related services and return a dictionary + listing users detected as inactive with their associated email. + Also output a csv file with the results. + """ fasclient = Client('https://fasjson.fedoraproject.org/') packagers = fasclient.list_group_members(groupname='packager').result logging.info(f'### Found {len(packagers)} users in the packager group. ###') @@ -126,10 +194,10 @@ def cli(ctx, privacy): if user in EXCLUDE_USERS: continue try: - resp_src = requests.get(f'https://src.fedoraproject.org/api/0/user/{user}/activity/stats').json() + resp_src = session.get(f'https://src.fedoraproject.org/api/0/user/{user}/activity/stats').json() if not resp_src: logging.info(f'No packaging activity detected for user {user}') - resp_pag = requests.get(f'https://pagure.io/api/0/user/{user}/activity/stats').json() + resp_pag = session.get(f'https://pagure.io/api/0/user/{user}/activity/stats').json() if not resp_pag: logging.info(f'No pagure activity detected for user {user}') inactive_packagers.append(user) @@ -147,7 +215,7 @@ def cli(ctx, privacy): page = 1 found_activity = False while True: - r = requests.get(f'https://apps.fedoraproject.org/datagrepper/raw/?user={p}' + r = session.get(f'https://apps.fedoraproject.org/datagrepper/raw/?user={p}' f'&category=bodhi&order=desc&delta=31536000&rows_per_page=50' f'&page={page}').json() for message in r.get('raw_messages', []): @@ -177,7 +245,7 @@ def cli(ctx, privacy): packager = fasclient.get_user(username=p).result recent_found = False for email in packager['emails']: - r = requests.get(f'https://lists.fedoraproject.org/archives/api/sender/{email}/emails/?ordering=-date').json() + r = session.get(f'https://lists.fedoraproject.org/archives/api/sender/{email}/emails/?ordering=-date').json() if r.get('count', 0) != 0: last_email_date = datetime.fromisoformat(r.get('results')[0]['date'].replace('Z', '+00:00')) time_delta = datetime.now(timezone.utc) - last_email_date @@ -251,7 +319,6 @@ def cli(ctx, privacy): packager_email_map.pop(username) logging.info(f'### Found {len(packager_email_map)} users which also show no activity in Bugzilla over the last year. ###') - ctx.obj['packager_email_map'] = packager_email_map if packager_email_map: with open('inactive_packagers.csv', 'w') as fout: @@ -259,25 +326,35 @@ def cli(ctx, privacy): emailstring = '|'.join(emails) logging.info(f'{user} - {emailstring if not privacy else "***"}') fout.write(f'{user},{emailstring}\n') + return packager_email_map + + +@click.group() +@click.option('--privacy', default=False, help='Hide users email in log.') +@click.pass_context +def cli(ctx, privacy): + """Detect inactive users in the packager group.""" + ctx.ensure_object(dict) + ctx.obj['privacy'] = privacy @cli.command() @click.pass_context def step_one(ctx): """Open Pagure tickets against inactive packagers.""" - packager_email_map = ctx.obj['packager_email_map'] + packager_email_map = _check_activity(privacy=ctx.obj['privacy']) if packager_email_map: # Open Pagure tickets - if PAGURE_API_KEY and PAGURE_NEW_TICKET_URL: + if PAGURE_API_KEY and PAGURE_API_BASE_URL: headers = {'Authorization': f'token {PAGURE_API_KEY}'} for user, emails in packager_email_map.items(): data = {'title': f'Inactive packager detected for user {user}', 'issue_content': PING_INACTIVE_TEXT.format(username = user, email = emails[0]), - 'tag': PAGURE_NEW_TICKET_TAGS.split(','), + 'tag': PAGURE_NEW_TICKET_TAG, 'assignee': PAGURE_NEW_TICKET_ASSIGNEE} try: - resp = requests.post(PAGURE_NEW_TICKET_URL, data=data, headers=headers) + resp = requests.post(f'{PAGURE_API_BASE_URL}/new_issue', data=data, headers=headers) if resp.status_code == 401: logging.error(f'Invalid or expired Pagure token, queue processing will stop immediately.') break @@ -292,13 +369,13 @@ def step_one(ctx): @click.argument('csvfile', type=click.File('r')) @click.pass_context def step_two(ctx, csvfile): - """Open Pagure tickets against inactive packagers. + """Check any user still inactive. Confront the list of inactive packagers detected in a previous run provided by CSVFILE with the list of inactive users from a fresh run and output the list of users which haven't replied to the Pagure ticket or that still don't show any activity. """ - packager_email_map = ctx.obj['packager_email_map'] + packager_email_map = _check_activity(privacy=ctx.obj['privacy']) prev_results = {} for line in csvfile: @@ -317,5 +394,55 @@ def step_two(ctx, csvfile): logging.info(f'### The provided csv file listed no users. ###') +@cli.command() +def check_impact(): + """Report packages expected to be orphaned. + + This report will show how many packages are going to be orphaned based on the + "inactive packager" tickets currently opened. + """ + waiting, pending = _fetch_tickets() + affected_packages = [] + logging.info(f'### Fetching affected users info ###') + for u, t in waiting.items(): + page = 1 + logging.info(f'Getting user {u} information') + while True: + try: + data = session.get(f'https://src.fedoraproject.org/api/0/user/{u}' + f'?per_page=100&repopage={page}').json() + repos = data.get('repos', []) + total_pages = data.get('repos_pagination', dict()).get('pages', 1) + for repo in repos: + if u in repo.get('access_users', dict()).get('owner'): + affected_packages.append((repo.get('name'), u, '')) + logging.info(f"Done {page} pages out of {total_pages}.") + if page >= total_pages: + break + page += 1 + except Exception: + logging.warning('Error while retrieving user info.') + for u, t in pending.items(): + page = 1 + logging.info(f'Getting user {u} information') + while True: + try: + data = session.get(f'https://src.fedoraproject.org/api/0/user/{u}' + f'?per_page=100&repopage={page}').json() + repos = data.get('repos', []) + total_pages = data.get('repos_pagination', dict()).get('pages', 1) + for repo in repos: + if u in repo.get('access_users', dict()).get('owner'): + affected_packages.append((repo.get('name'), u, '*')) + logging.info(f"Done {page} pages out of {total_pages}.") + if page >= total_pages: + break + page += 1 + except Exception: + logging.warning('Error while retrieving ticket list.') + package_list = "\n".join([f'- {mark}{package} (owned by {user})' for package, user, mark in sorted(affected_packages)]) + logging.info(CHECK_IMPACT_OUTPUT.format(total_packages=len(affected_packages), package_list=package_list)) + + if __name__ == '__main__': cli() From 76bc38fb3ebacec6e25423f739fb38ccf4bd40c1 Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 08 2022 08:04:49 +0000 Subject: [PATCH 2/9] Move csv file generation under step-one --- diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index 8242e2e..0ff924f 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -117,7 +117,7 @@ Have a wonderful day and see you (maybe?) at the next run! logging.basicConfig(level=logging.INFO, format='%(message)s', handlers=[ - logging.FileHandler(f'find_inactive_packagers_{datetime.now(timezone.utc).strftime("%Y_%m_%d")}.log', 'w+'), + logging.FileHandler(f'find_inactive_packagers_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+'), logging.StreamHandler(sys.stdout) ]) @@ -319,13 +319,6 @@ def _check_activity(privacy=False): packager_email_map.pop(username) logging.info(f'### Found {len(packager_email_map)} users which also show no activity in Bugzilla over the last year. ###') - - if packager_email_map: - with open('inactive_packagers.csv', 'w') as fout: - for user, emails in packager_email_map.items(): - emailstring = '|'.join(emails) - logging.info(f'{user} - {emailstring if not privacy else "***"}') - fout.write(f'{user},{emailstring}\n') return packager_email_map @@ -339,30 +332,52 @@ def cli(ctx, privacy): @cli.command() +@click.option('--open-tickets', is_flag=True, default=False, help='File tickets in Pagure.') @click.pass_context -def step_one(ctx): - """Open Pagure tickets against inactive packagers.""" - packager_email_map = _check_activity(privacy=ctx.obj['privacy']) +def step_one(ctx, open_tickets): + """Find inactive packagers. + + If open-tickets flag is activated, the script will file tickets in Pagure, otherwise + it will only output a csv file. + """ + privacy = ctx.obj['privacy'] + + if open_tickets: + if not PAGURE_API_KEY or not PAGURE_NEW_TICKET_URL: + logging.error('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' + 'queue processing will stop immediately.') + raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' + 'queue processing will stop immediately.') + packager_email_map = _check_activity(privacy=privacy) if packager_email_map: - # Open Pagure tickets - if PAGURE_API_KEY and PAGURE_API_BASE_URL: - headers = {'Authorization': f'token {PAGURE_API_KEY}'} + with open('inactive_packagers.csv', 'w') as fout: for user, emails in packager_email_map.items(): - data = {'title': f'Inactive packager detected for user {user}', - 'issue_content': PING_INACTIVE_TEXT.format(username = user, email = emails[0]), - 'tag': PAGURE_NEW_TICKET_TAG, - 'assignee': PAGURE_NEW_TICKET_ASSIGNEE} - try: - resp = requests.post(f'{PAGURE_API_BASE_URL}/new_issue', data=data, headers=headers) - if resp.status_code == 401: - logging.error(f'Invalid or expired Pagure token, queue processing will stop immediately.') - break - if resp.status_code != 200: + # Open Pagure tickets + ticket_id = 'NONE' + if open_tickets: + headers = {'Authorization': f'token {PAGURE_API_KEY}'} + data = {'title': f'Inactive packager detected for user {user}', + 'issue_content': PING_INACTIVE_TEXT.format(username = user, email = emails[0]), + 'tag': PAGURE_NEW_TICKET_TAGS.split(','), + 'assignee': PAGURE_NEW_TICKET_ASSIGNEE} + try: + resp = requests.post(PAGURE_NEW_TICKET_URL, data=data, headers=headers) + if resp.status_code == 401: + logging.error('Invalid or expired Pagure token, queue processing will stop immediately.') + sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.') + if resp.status_code != 200: + logging.error(f'Error opening Pagure ticket for user {user}') + ticket_id = 'ERROR' + else: + ticket_id = resp.json().get('issue', dict()).get('id', 'ERROR') + except Exception: logging.error(f'Error opening Pagure ticket for user {user}') - except Exception: - logging.error(f'Error opening Pagure ticket for user {user}') - continue + ticket_id = 'ERROR' + # Write results to file + emailstring = '|'.join(emails) + logging.info(f'{user} - {ticket_id} - {emailstring if not privacy else "***"}') + fout.write(f'{user},{ticket_id},{emailstring}\n') @cli.command() From 91128950d8abb510496f9830bf04c2f5ed50ffcd Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 10 2022 08:40:29 +0000 Subject: [PATCH 3/9] Make checks reusable --- diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index 0ff924f..0abd6b3 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -33,11 +33,10 @@ The script will search for any activity in the last year in: - Fedora mailing lists - Red Hat bugzilla -It will output the 'inactive_packagers.csv' file with a list of all users in the packager -group and their email which are found to be inactive. This can be used to contact +The step-one subcommand will output the 'inactive_packagers.csv' file with a list of all users +in the packager group and their email which are found to be inactive. This can be used to contact the inactive users, asking them to show some activity in pagure, datagrepper or mailing lists. - -The step-one subcommand will open tickets against inactive packagers in a Pagure repository. +Optionally, it can open tickets against inactive packagers in a Pagure repository. The check-impact subcommand will iterate on the list of tickets opened and provide information about how many packages will possibly be orphaned based on the packagers detected as inactive. @@ -172,38 +171,186 @@ def _fetch_tickets(): return user_ticket_waiting_reply, user_ticket_pending_removal -def _check_activity(privacy=False): - """Run activity checks. +def _check_pagure_activity(user, base_url='https://src.fedoraproject.org'): + """Look for user activity in a Pagure deployment.""" + if not user: + logging.error('Cannot check an empty username!') + return False + try: + resp_src = session.get(f'{base_url}/api/0/user/{user}/activity/stats').json() + if not resp_src: + logging.info(f'No activity detected for user {user} in {base_url}') + return False + except Exception: + # May happen when the user never interacted with pagure + logging.warning(f'Error while retrieving data for user {user}.') + return False + logging.info(f'User {user} was active in {base_url}') + return True + + +def _check_bodhi_activity(user): + """Look for user activity in Fedora Bodhi through datagrepper messages.""" + if not user: + logging.error('Cannot check an empty username!') + return False + try: + page = 1 + while True: + r = session.get(f'https://apps.fedoraproject.org/datagrepper/raw/?user={user}' + f'&category=bodhi&order=desc&delta=31536000&rows_per_page=50' + f'&page={page}').json() + for message in r.get('raw_messages', []): + agent = message.get('msg', dict()).get('agent', None) + if agent == user: + logging.info(f'Found recent activity in Bodhi for user {user}') + return True + if page >= r.get('pages', 1): + return False + else: + page += 1 + except Exception: + # May happen if there are a lot of entries in datagrepper + logging.error(f'Error while retrieving data for user {user}') + return False + + +def _check_maillists_activity(fasclient, user, delta=365): + """Look for user recent activity in Fedora mailing lists. - Check for user activity on several Fedora related services and return a dictionary - listing users detected as inactive with their associated email. - Also output a csv file with the results. + Returns a tuple where the first item is a bool and the second item is the list + of email addresses checked. """ - fasclient = Client('https://fasjson.fedoraproject.org/') + if not user: + logging.error('Cannot check an empty username!') + return (False, []) + try: + packager = fasclient.get_user(username=user).result + for email in packager['emails']: + r = session.get(f'https://lists.fedoraproject.org/archives/api/sender/{email}/emails/?ordering=-date').json() + if r.get('count', 0) != 0: + last_email_date = datetime.fromisoformat(r.get('results')[0]['date'].replace('Z', '+00:00')) + time_delta = datetime.now(timezone.utc) - last_email_date + if time_delta.days <= delta: + logging.info(f'Found recent email from user {user}: {last_email_date}.') + return (True, packager['emails']) + except Exception: + logging.error(f'Error while retrieving last mailing lists activity for user {user}.') + return (False, []) + return (False, packager['emails']) + + +def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privacy=False, delta=365): + """Look for user recent activity in Bugzilla. + + Arg packager is a tuple of username and a list of emails. + """ + username = packager[0] + if not username: + logging.error('Cannot check an empty username!') + return False + emails = packager[1] + bzuser = None + if check_fedora_alias: + # Some users have their @fedoraproject.org alias set as BZ email + emails.append(f'{username}@fedoraproject.org') + for bzemail in emails: + try: + bzuser = bzclient.getuser(bzemail) + logging.debug(f'Found user {username} by email {bzemail if not privacy else "***"}.') + break + except Exception: + logging.warning(f'Unable to find user {username} by email {bzemail if not privacy else "***"} in Bugzilla.') + if not bzuser: + logging.error(f'Unable to check user {username} activity in Bugzilla because I cannot find them.') + return False + try: + bugs = bzclient.query({ + "email1" : bzuser.email, + "emaillongdesc1" : "1", + "emailtype1" : "substring", + "query_format" : "advanced", + "order": "changeddate DESC", + "chfieldfrom" : "-1y", + "chfieldto" : "Now", + "limit": 1000}) + except Exception: + logging.error(f'Error while retrieving Bugzilla data for user {username}') + return False + ids = [bug.bug_id for bug in bugs] + user_comments = [] + try: + bzbugs = bzclient.getbugs(ids) + except Exception: + logging.error(f'Error while retrieving Bugzilla comments for user {username}') + return False + for bug in bzbugs: + user_comments.extend([com for com in bug.longdescs if com['creator_id'] == bzuser.userid]) + if not user_comments: + return False + last_com_date = None + for comment in user_comments: + if not last_com_date or last_com_date < comment.get('time'): + last_com_date = comment.get('time') + # convert DateTime object to datetime.datetime + last_com_date = dateparser.parse(f'{last_com_date.value} UTC') + time_delta = datetime.now(timezone.utc) - last_com_date + if time_delta.days <= delta: + logging.info(f'User {username} made a comment in Bugzilla on {last_com_date.strftime("%Y_%m_%d")}.') + return True + return False + + +@click.group() +@click.option('--privacy', default=False, help='Hide users email in log.') +@click.pass_context +def cli(ctx, privacy): + """Detect inactive users in the packager group.""" + ctx.ensure_object(dict) + ctx.obj['privacy'] = privacy + + +@cli.command() +@click.option('--open-tickets', is_flag=True, default=False, help='File tickets in Pagure.') +@click.pass_context +def step_one(ctx, open_tickets): + """Find inactive packagers. + + If open-tickets flag is activated, the script will file tickets in Pagure, otherwise + it will only output a csv file. + """ + privacy = ctx.obj['privacy'] + + if open_tickets: + if not PAGURE_API_KEY or not PAGURE_NEW_TICKET_URL: + logging.error('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' + 'queue processing will stop immediately.') + raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' + 'queue processing will stop immediately.') + + try: + fasclient = Client('https://fasjson.fedoraproject.org/') + except Exception: + logging.error('Unable to connect to fasclient, you probably forgot to obtain ' + 'a Kerberos ticket.') + raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' + 'a Kerberos ticket.') packagers = fasclient.list_group_members(groupname='packager').result logging.info(f'### Found {len(packagers)} users in the packager group. ###') inactive_packagers = [] # Check for activity in Pagure - # may take a while... - logging.info(f'Checking users activity in src.fedoraproject.org...') + logging.info(f'Checking users activity in Pagure...') for i, p in enumerate(packagers): if i > 0 and i % 100 == 0: logging.info(f'Done {i}.') user = p.get('username') if user in EXCLUDE_USERS: continue - try: - resp_src = session.get(f'https://src.fedoraproject.org/api/0/user/{user}/activity/stats').json() - if not resp_src: - logging.info(f'No packaging activity detected for user {user}') - resp_pag = session.get(f'https://pagure.io/api/0/user/{user}/activity/stats').json() - if not resp_pag: - logging.info(f'No pagure activity detected for user {user}') - inactive_packagers.append(user) - except Exception: - # May happen when the user never interacted with pagure - logging.warning(f'Error while retrieving data for user {user}.') + if ( + not _check_pagure_activity(user) + and not _check_pagure_activity(user, base_url='https://pagure.io') + ): inactive_packagers.append(user) logging.info(f'### Found {len(inactive_packagers)} users with no activity in pagure/src.fp.org over the last year. ###') @@ -211,29 +358,10 @@ def _check_activity(privacy=False): # Check for activity in Bodhi through datagrepper messages logging.info(f'Checking users activity in Bodhi...') for i, p in enumerate(copy(inactive_packagers)): - try: - page = 1 - found_activity = False - while True: - r = session.get(f'https://apps.fedoraproject.org/datagrepper/raw/?user={p}' - f'&category=bodhi&order=desc&delta=31536000&rows_per_page=50' - f'&page={page}').json() - for message in r.get('raw_messages', []): - agent = message.get('msg', dict()).get('agent', None) - if agent == p: - logging.info(f'Found recent activity in Bodhi for user {p}') - inactive_packagers.remove(p) - found_activity = True - break - if found_activity or page >= r.get('pages', 1): - break - else: - page += 1 - except Exception: - # May happen if there are a lot of entries in datagrepper - logging.error(f'Error while retrieving data for user {p}') - if i % 100 == 0: + if i > 0 and i % 100 == 0: logging.info(f'Done {i}.') + if _check_bodhi_activity(p): + inactive_packagers.remove(p) logging.info(f'### Found {len(inactive_packagers)} users which also show no activity in Bodhi over the last year. ###') @@ -241,115 +369,26 @@ def _check_activity(privacy=False): logging.info(f'Checking users activity in mailing lists...') packager_email_map = {} for i, p in enumerate(inactive_packagers): - try: - packager = fasclient.get_user(username=p).result - recent_found = False - for email in packager['emails']: - r = session.get(f'https://lists.fedoraproject.org/archives/api/sender/{email}/emails/?ordering=-date').json() - if r.get('count', 0) != 0: - last_email_date = datetime.fromisoformat(r.get('results')[0]['date'].replace('Z', '+00:00')) - time_delta = datetime.now(timezone.utc) - last_email_date - if time_delta.days <= 365: - logging.info(f'Found recent email from user {p}: {last_email_date}.') - recent_found = True - break - if not recent_found: - packager_email_map[p] = packager['emails'] - except Exception: - logging.error(f'Error while retrieving last mailing lists activity for user {p}.') - packager_email_map[p] = packager['emails'] - if i % 100 == 0: + if i > 0 and i % 100 == 0: logging.info(f'Done {i}.') + maillist_results = _check_maillists_activity(fasclient, p) + if not maillist_results[0]: + packager_email_map[p] = maillist_results[1] logging.info(f'### Found {len(packager_email_map)} users which show also no activity in mailing lists over the last year. ###') # Check for activity in Bugzilla - # Let's run last since it's the slowest if BZ_API_KEY: logging.info(f'Checking users activity in bugzilla...') bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) for i, p in enumerate(copy(packager_email_map).items()): if i > 0 and i % 100 == 0: logging.info(f'Done {i}.') - username = p[0] - bzuser = None - # Some users have their @fedoraproject.org alias set as BZ email - for bzemail in p[1] + [f'{username}@fedoraproject.org']: - try: - bzuser = bzclient.getuser(bzemail) - break - except Exception: - logging.warning(f'Unable to find user {username} by email {bzemail if not privacy else "***"} in Bugzilla.') - if not bzuser: - logging.error(f'Unable to check user {username} activity in Bugzilla because I cannot find them.') - continue - try: - bugs = bzclient.query({ - "email1" : bzuser.email, - "emaillongdesc1" : "1", - "emailtype1" : "substring", - "query_format" : "advanced", - "order": "changeddate DESC", - "chfieldfrom" : "-1y", - "chfieldto" : "Now", - "limit": 1000}) - except Exception: - logging.error(f'Error while retrieving Bugzilla data for user {username}') - continue - ids = [bug.bug_id for bug in bugs] - user_comments = [] - try: - bzbugs = bzclient.getbugs(ids) - except Exception: - logging.error(f'Error while retrieving Bugzilla comments for user {username}') - continue - for bug in bzbugs: - user_comments.extend([com for com in bug.longdescs if com['creator_id'] == bzuser.userid]) - if not user_comments: - continue - last_com_date = None - for comment in user_comments: - if not last_com_date or last_com_date < comment.get('time'): - last_com_date = comment.get('time') - # convert DateTime object to datetime.datetime - last_com_date = dateparser.parse(f'{last_com_date.value} UTC') - time_delta = datetime.now(timezone.utc) - last_com_date - if time_delta.days <= 365: - logging.info(f'User {username} made a comment in Bugzilla on {last_com_date.strftime("%Y_%m_%d")}.') - packager_email_map.pop(username) + if _check_bugzilla_activity(bzclient, p, privacy=privacy): + packager_email_map.pop(p[0]) logging.info(f'### Found {len(packager_email_map)} users which also show no activity in Bugzilla over the last year. ###') - return packager_email_map - - -@click.group() -@click.option('--privacy', default=False, help='Hide users email in log.') -@click.pass_context -def cli(ctx, privacy): - """Detect inactive users in the packager group.""" - ctx.ensure_object(dict) - ctx.obj['privacy'] = privacy - - -@cli.command() -@click.option('--open-tickets', is_flag=True, default=False, help='File tickets in Pagure.') -@click.pass_context -def step_one(ctx, open_tickets): - """Find inactive packagers. - - If open-tickets flag is activated, the script will file tickets in Pagure, otherwise - it will only output a csv file. - """ - privacy = ctx.obj['privacy'] - - if open_tickets: - if not PAGURE_API_KEY or not PAGURE_NEW_TICKET_URL: - logging.error('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' - 'queue processing will stop immediately.') - raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' - 'queue processing will stop immediately.') - packager_email_map = _check_activity(privacy=privacy) if packager_email_map: with open('inactive_packagers.csv', 'w') as fout: for user, emails in packager_email_map.items(): @@ -378,6 +417,8 @@ def step_one(ctx, open_tickets): emailstring = '|'.join(emails) logging.info(f'{user} - {ticket_id} - {emailstring if not privacy else "***"}') fout.write(f'{user},{ticket_id},{emailstring}\n') + else: + logging.info('No inactive packagers detected, YHAY! Nothing to do.') @cli.command() From af98f804ae9fa394e5dd26e02e81bb4f39950cef Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 11 2022 15:34:04 +0000 Subject: [PATCH 4/9] Manage Pagure tickets automatically in step-two --- diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index 0abd6b3..76e84f6 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -41,9 +41,10 @@ Optionally, it can open tickets against inactive packagers in a Pagure repositor The check-impact subcommand will iterate on the list of tickets opened and provide information about how many packages will possibly be orphaned based on the packagers detected as inactive. -The step-two is used to confront the results of a previous run (provided by the csv file) -and a fresh run, to identify which users still don't show any activity. Therefore we should -consider these users to have abandoned Fedora. +The step-two subcommand can either check a list of inactive users from the csv file generated +in step-one, or check a list of users based on open Pagure tickets. In this last case, the tickets +can be closed appropriately. Another csv file is generated as output to list users which are still +inactive. Use the '--privacy' flag to not print emails in the log. @@ -79,6 +80,8 @@ PAGURE_ASK_REMOVAL_TAG = getenv('PAGURE_ASK_REMOVAL_TAG', 'asked_removal') # A username to whom assign the ticket by default or None PAGURE_NEW_TICKET_ASSIGNEE = getenv('PAGURE_NEW_TICKET_ASSIGNEE', None) +FASCLIENT_URL = 'https://fasjson.fedoraproject.org/' + # A list of system users that should never be removed from the packager group EXCLUDE_USERS = ['releng'] @@ -156,6 +159,9 @@ def _fetch_tickets(): total_pages = data.get('pagination', dict()).get('pages', 1) for issue in issues: ticket_id = issue.get('id') + if not issue.get('title').startswith('Inactive packager detected for user '): + logging.error(f"ERROR: Ticket {ticket_id} does not appear to be a valid inactive_packager ticket.") + continue username = issue.get('title').split()[-1] if PAGURE_ASK_REMOVAL_TAG in issue.get('tags'): user_ticket_pending_removal[username] = ticket_id @@ -171,16 +177,50 @@ def _fetch_tickets(): return user_ticket_waiting_reply, user_ticket_pending_removal +def _close_pagure_ticket(ticket_id, close_status, comment=''): + """Close pagure ticket with comment.""" + headers = {'Authorization': f'token {PAGURE_API_KEY}'} + if comment: + data = {'comment': comment} + try: + logging.debug(f'Posting comment on ticket {ticket_id}') + resp = session.post(f'{PAGURE_API_BASE_URL}/issue/{ticket_id}/comment', data=data, headers=headers) + if resp.status_code == 401: + logging.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') + sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.') + if resp.status_code != 200: + logging.error(f'ERROR: Error posting comment on ticket {ticket_id}') + except Exception: + logging.error(f'ERROR: Error posting comment on ticket {ticket_id}') + + logging.debug(f'Closing ticket {ticket_id}') + data = {'status': 'Closed', + 'close_status': close_status} + try: + resp = session.post(f'{PAGURE_API_BASE_URL}/issue/{ticket_id}/status', data=data, headers=headers) + if resp.status_code == 401: + logging.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') + sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.') + if resp.status_code != 200: + logging.error(f'ERROR: Error on changing status of ticket {ticket_id}') + except Exception: + logging.error(f'ERROR: Error on changing status of ticket {ticket_id}') + + def _check_pagure_activity(user, base_url='https://src.fedoraproject.org'): """Look for user activity in a Pagure deployment.""" if not user: - logging.error('Cannot check an empty username!') + logging.error('ERROR: Cannot check an empty username!') return False try: resp_src = session.get(f'{base_url}/api/0/user/{user}/activity/stats').json() if not resp_src: logging.info(f'No activity detected for user {user} in {base_url}') return False + error = resp_src.get('error', None) + if error is not None: + logging.info(f'Error checking user {user} in {base_url}: {error}') + return False except Exception: # May happen when the user never interacted with pagure logging.warning(f'Error while retrieving data for user {user}.') @@ -192,7 +232,7 @@ def _check_pagure_activity(user, base_url='https://src.fedoraproject.org'): def _check_bodhi_activity(user): """Look for user activity in Fedora Bodhi through datagrepper messages.""" if not user: - logging.error('Cannot check an empty username!') + logging.error('ERROR: Cannot check an empty username!') return False try: page = 1 @@ -211,7 +251,7 @@ def _check_bodhi_activity(user): page += 1 except Exception: # May happen if there are a lot of entries in datagrepper - logging.error(f'Error while retrieving data for user {user}') + logging.error(f'ERROR: Error while retrieving data for user {user}') return False @@ -222,7 +262,7 @@ def _check_maillists_activity(fasclient, user, delta=365): of email addresses checked. """ if not user: - logging.error('Cannot check an empty username!') + logging.error('ERROR: Cannot check an empty username!') return (False, []) try: packager = fasclient.get_user(username=user).result @@ -235,7 +275,7 @@ def _check_maillists_activity(fasclient, user, delta=365): logging.info(f'Found recent email from user {user}: {last_email_date}.') return (True, packager['emails']) except Exception: - logging.error(f'Error while retrieving last mailing lists activity for user {user}.') + logging.error(f'ERROR: Error while retrieving last mailing lists activity for user {user}.') return (False, []) return (False, packager['emails']) @@ -247,11 +287,11 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac """ username = packager[0] if not username: - logging.error('Cannot check an empty username!') + logging.error('ERROR: Cannot check an empty username!') return False emails = packager[1] bzuser = None - if check_fedora_alias: + if check_fedora_alias and f'{username}@fedoraproject.org' not in emails: # Some users have their @fedoraproject.org alias set as BZ email emails.append(f'{username}@fedoraproject.org') for bzemail in emails: @@ -262,7 +302,7 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac except Exception: logging.warning(f'Unable to find user {username} by email {bzemail if not privacy else "***"} in Bugzilla.') if not bzuser: - logging.error(f'Unable to check user {username} activity in Bugzilla because I cannot find them.') + logging.error(f'ERROR: Unable to check user {username} activity in Bugzilla because I cannot find them.') return False try: bugs = bzclient.query({ @@ -275,14 +315,14 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac "chfieldto" : "Now", "limit": 1000}) except Exception: - logging.error(f'Error while retrieving Bugzilla data for user {username}') + logging.error(f'ERROR: Error while retrieving Bugzilla data for user {username}') return False ids = [bug.bug_id for bug in bugs] user_comments = [] try: bzbugs = bzclient.getbugs(ids) except Exception: - logging.error(f'Error while retrieving Bugzilla comments for user {username}') + logging.error(f'ERROR: Error while retrieving Bugzilla comments for user {username}') return False for bug in bzbugs: user_comments.extend([com for com in bug.longdescs if com['creator_id'] == bzuser.userid]) @@ -301,6 +341,37 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac return False +def _check_user_activity(user, privacy=False): + """Check user activity.""" + try: + fasclient = Client(FASCLIENT_URL) + except Exception: + logging.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' + 'a Kerberos ticket.') + raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' + 'a Kerberos ticket.') + logging.info(f'Checking {user} activity in Pagure...') + if ( + _check_pagure_activity(user) + or _check_pagure_activity(user, base_url='https://pagure.io') + ): + return True + logging.info(f'Checking {user} activity in Bodhi...') + if _check_bodhi_activity(user): + return True + logging.info(f'Checking {user} activity in mailing lists...') + maillist_result = _check_maillists_activity(fasclient, user) + if maillist_result[0]: + return True + if BZ_API_KEY: + logging.info(f'Checking {user} activity in bugzilla...') + bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) + packager = (user, maillist_result[1]) + if _check_bugzilla_activity(bzclient, packager, privacy=privacy): + return True + return False + + @click.group() @click.option('--privacy', default=False, help='Hide users email in log.') @click.pass_context @@ -322,16 +393,16 @@ def step_one(ctx, open_tickets): privacy = ctx.obj['privacy'] if open_tickets: - if not PAGURE_API_KEY or not PAGURE_NEW_TICKET_URL: - logging.error('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' + if not PAGURE_API_KEY or not PAGURE_API_BASE_URL: + logging.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' 'queue processing will stop immediately.') - raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_NEW_TICKET_URL, ' + raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' 'queue processing will stop immediately.') try: - fasclient = Client('https://fasjson.fedoraproject.org/') + fasclient = Client(FASCLIENT_URL) except Exception: - logging.error('Unable to connect to fasclient, you probably forgot to obtain ' + logging.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' 'a Kerberos ticket.') raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' 'a Kerberos ticket.') @@ -340,7 +411,7 @@ def step_one(ctx, open_tickets): inactive_packagers = [] # Check for activity in Pagure - logging.info(f'Checking users activity in Pagure...') + logging.info('Checking users activity in Pagure...') for i, p in enumerate(packagers): if i > 0 and i % 100 == 0: logging.info(f'Done {i}.') @@ -356,7 +427,7 @@ def step_one(ctx, open_tickets): logging.info(f'### Found {len(inactive_packagers)} users with no activity in pagure/src.fp.org over the last year. ###') # Check for activity in Bodhi through datagrepper messages - logging.info(f'Checking users activity in Bodhi...') + logging.info('Checking users activity in Bodhi...') for i, p in enumerate(copy(inactive_packagers)): if i > 0 and i % 100 == 0: logging.info(f'Done {i}.') @@ -366,7 +437,7 @@ def step_one(ctx, open_tickets): logging.info(f'### Found {len(inactive_packagers)} users which also show no activity in Bodhi over the last year. ###') # Check for any activity in mailing lists - logging.info(f'Checking users activity in mailing lists...') + logging.info('Checking users activity in mailing lists...') packager_email_map = {} for i, p in enumerate(inactive_packagers): if i > 0 and i % 100 == 0: @@ -379,7 +450,7 @@ def step_one(ctx, open_tickets): # Check for activity in Bugzilla if BZ_API_KEY: - logging.info(f'Checking users activity in bugzilla...') + logging.info('Checking users activity in bugzilla...') bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) for i, p in enumerate(copy(packager_email_map).items()): if i > 0 and i % 100 == 0: @@ -395,59 +466,132 @@ def step_one(ctx, open_tickets): # Open Pagure tickets ticket_id = 'NONE' if open_tickets: + logging.debug(f'Opening ticket for user {user}') headers = {'Authorization': f'token {PAGURE_API_KEY}'} data = {'title': f'Inactive packager detected for user {user}', 'issue_content': PING_INACTIVE_TEXT.format(username = user, email = emails[0]), - 'tag': PAGURE_NEW_TICKET_TAGS.split(','), + 'tag': PAGURE_NEW_TICKET_TAG, 'assignee': PAGURE_NEW_TICKET_ASSIGNEE} try: - resp = requests.post(PAGURE_NEW_TICKET_URL, data=data, headers=headers) + resp = session.post(f'{PAGURE_API_BASE_URL}/new_issue', data=data, headers=headers) if resp.status_code == 401: - logging.error('Invalid or expired Pagure token, queue processing will stop immediately.') + logging.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.') if resp.status_code != 200: - logging.error(f'Error opening Pagure ticket for user {user}') + logging.error(f'ERROR: Error opening Pagure ticket for user {user}') ticket_id = 'ERROR' else: ticket_id = resp.json().get('issue', dict()).get('id', 'ERROR') except Exception: - logging.error(f'Error opening Pagure ticket for user {user}') + logging.error(f'ERROR: Error opening Pagure ticket for user {user}') ticket_id = 'ERROR' # Write results to file emailstring = '|'.join(emails) logging.info(f'{user} - {ticket_id} - {emailstring if not privacy else "***"}') fout.write(f'{user},{ticket_id},{emailstring}\n') else: - logging.info('No inactive packagers detected, YHAY! Nothing to do.') + logging.info('### No inactive packagers detected, YHAY! Nothing to do. ###') @cli.command() -@click.argument('csvfile', type=click.File('r')) +@click.option('--from-file', type=click.File('r'), help='Use local csv files instead of Pagure.') +@click.option('--close-tickets', is_flag=True, default=False, help='Close tickets in Pagure.') @click.pass_context -def step_two(ctx, csvfile): +def step_two(ctx, close_tickets, from_file): """Check any user still inactive. - Confront the list of inactive packagers detected in a previous run provided by CSVFILE - with the list of inactive users from a fresh run and output the list of users which - haven't replied to the Pagure ticket or that still don't show any activity. + If a csv file is provided, the script will use that as input, rather than fetching tickets from + Pagure. Users listed in the csv file will be checked for activity and another csv file will + be provided to list users that are still detected inactive. + + Otherwise, the list of inactive packagers is obtained by open Pagure tickets. These will be + checked for activity and tickets will be closed appropriately at the same time a csv file is + provided in output. Special flags applied to open tickets (for example 'asked_removal') will + be considered appropriately. """ - packager_email_map = _check_activity(privacy=ctx.obj['privacy']) + privacy = ctx.obj['privacy'] - prev_results = {} - for line in csvfile: - user, emails = line.split(',') - prev_results[user] = emails.strip() + if close_tickets: + if not PAGURE_API_KEY or not PAGURE_API_BASE_URL: + logging.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' + 'queue processing will stop immediately.') + raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' + 'queue processing will stop immediately.') - if packager_email_map: - if prev_results: - still_inactive = set(prev_results.keys()).intersection(packager_email_map.keys()) - logging.info(f'### These {len(still_inactive)} users didn\'t react to previous inquiry: ###') + if from_file: + logging.debug('Using CSV file') + pending_removal_users = [] + for line in from_file: + user, ticket, emails = line.split(',') + pending_removal_users.append(user) + logging.info(f'### Checking {len(pending_removal_users)} users from csv file ###') + for i, p in enumerate(copy(pending_removal_users)): + if i > 0 and i % 100 == 0: + logging.info(f'Done {i}.') + if _check_user_activity(p, privacy=privacy): + pending_removal_users.remove(p) + if pending_removal_users: + logging.info(f'### These {len(pending_removal_users)} users didn\'t react to previous inquiry: ###') with open('still_inactive.csv', 'w') as finactive: - for user in still_inactive: + for user in pending_removal_users: logging.info(f'{user}') finactive.write(f'{user}\n') else: - logging.info(f'### The provided csv file listed no users. ###') + logging.info('### All users resumed activity, YHAY! Nothing to do. ###') + else: + logging.debug('Using Pagure tickets') + confirmed_removal = [] + resumed_activity = [] + waiting, pending = _fetch_tickets() + logging.info(f'### Pagure returned {len(waiting)} users to check a second time and {len(pending)} users ready for removal ###') + if waiting: + logging.info('Starting second check on users...') + for user, ticket_id in waiting.items(): + if not _check_user_activity(user, privacy=privacy): + confirmed_removal.append((user, ticket_id)) + else: + resumed_activity.append((user, ticket_id)) + + if confirmed_removal: + logging.info(f'### These {len(confirmed_removal)} users didn\'t reply and are going to be removed from packagers: ###') + for user, ticket in confirmed_removal: + logging.info(f'- {user}') + else: + logging.info('### All users resumed activity, HURRAY! ###') + + if pending: + logging.info(f'### These users agreed to be removed from packagers: ###') + for user, ticket_id in pending.items(): + logging.info(f'- {user}') + confirmed_removal.append((user, ticket_id)) + + if resumed_activity: + logging.info(f'### These {len(resumed_activity)} users resumed activity: ###') + for user, ticket_id in resumed_activity: + logging.info(f'- {user}') + + if confirmed_removal: + if close_tickets: + logging.info(f'### Closing tickets and generating output file ###') + for user, ticket_id in resumed_activity: + logging.info(f'Closing ticket {ticket_id} for user {user}...') + _close_pagure_ticket( + ticket_id, + 'Keep packager status', + comment=(f'Recent activity has been detected for user {user} therefore ' + 'the ticket will be closed without further actions.') + ) + with open('still_inactive.csv', 'w') as finactive: + for user, ticket_id in confirmed_removal: + if close_tickets: + logging.info(f'Closing ticket {ticket_id} for user {user}...') + _close_pagure_ticket( + ticket_id, + 'Removed from packagers', + comment=f'User {user} will be removed from packager group.' + ) + # Write users list to file + finactive.write(f'{user}\n') @cli.command() @@ -500,5 +644,17 @@ def check_impact(): logging.info(CHECK_IMPACT_OUTPUT.format(total_packages=len(affected_packages), package_list=package_list)) +@cli.command() +@click.argument('username') +@click.pass_context +def check_user(ctx, username): + """Check if any activity is detected for a specific username.""" + privacy = ctx.obj['privacy'] + if _check_user_activity(username, privacy=privacy): + click.echo(f"Activity detected for user {username}.") + else: + click.echo(f"User {username} appear to be inactive.") + + if __name__ == '__main__': cli() From b7b31693c2c47e3c1f9519e3d1a3afa5f2f9a527 Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 12 2022 07:12:08 +0000 Subject: [PATCH 5/9] Optimize logging --- diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index 76e84f6..b2800cc 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -117,11 +117,8 @@ Have a wonderful day and see you (maybe?) at the next run! ''' logging.basicConfig(level=logging.INFO, - format='%(message)s', - handlers=[ - logging.FileHandler(f'find_inactive_packagers_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+'), - logging.StreamHandler(sys.stdout) - ]) + format='%(message)s') +log = logging.getLogger('find-inactive-packagers') def _retry_session(retries, session=None, backoff_factor=0.3): @@ -150,7 +147,7 @@ def _fetch_tickets(): page = 1 user_ticket_waiting_reply = dict() user_ticket_pending_removal = dict() - logging.info(f'### Fetching open tickets. ###') + log.info(f'### Fetching open tickets. ###') while True: try: data = session.get(f'{PAGURE_API_BASE_URL}/issues?status=Open&tags={PAGURE_NEW_TICKET_TAG}' @@ -160,19 +157,19 @@ def _fetch_tickets(): for issue in issues: ticket_id = issue.get('id') if not issue.get('title').startswith('Inactive packager detected for user '): - logging.error(f"ERROR: Ticket {ticket_id} does not appear to be a valid inactive_packager ticket.") + log.error(f"ERROR: Ticket {ticket_id} does not appear to be a valid inactive_packager ticket.") continue username = issue.get('title').split()[-1] if PAGURE_ASK_REMOVAL_TAG in issue.get('tags'): user_ticket_pending_removal[username] = ticket_id else: user_ticket_waiting_reply[username] = ticket_id - logging.info(f"Done {page} pages out of {total_pages}.") + log.info(f"Done {page} pages out of {total_pages}.") if page >= total_pages: break page += 1 except Exception: - logging.warning('Error while retrieving ticket list.') + log.warning('Error while retrieving ticket list.') break return user_ticket_waiting_reply, user_ticket_pending_removal @@ -183,56 +180,56 @@ def _close_pagure_ticket(ticket_id, close_status, comment=''): if comment: data = {'comment': comment} try: - logging.debug(f'Posting comment on ticket {ticket_id}') + log.debug(f'Posting comment on ticket {ticket_id}') resp = session.post(f'{PAGURE_API_BASE_URL}/issue/{ticket_id}/comment', data=data, headers=headers) if resp.status_code == 401: - logging.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') + log.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.') if resp.status_code != 200: - logging.error(f'ERROR: Error posting comment on ticket {ticket_id}') + log.error(f'ERROR: Error posting comment on ticket {ticket_id}') except Exception: - logging.error(f'ERROR: Error posting comment on ticket {ticket_id}') + log.error(f'ERROR: Error posting comment on ticket {ticket_id}') - logging.debug(f'Closing ticket {ticket_id}') + log.debug(f'Closing ticket {ticket_id}') data = {'status': 'Closed', 'close_status': close_status} try: resp = session.post(f'{PAGURE_API_BASE_URL}/issue/{ticket_id}/status', data=data, headers=headers) if resp.status_code == 401: - logging.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') + log.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.') if resp.status_code != 200: - logging.error(f'ERROR: Error on changing status of ticket {ticket_id}') + log.error(f'ERROR: Error on changing status of ticket {ticket_id}') except Exception: - logging.error(f'ERROR: Error on changing status of ticket {ticket_id}') + log.error(f'ERROR: Error on changing status of ticket {ticket_id}') def _check_pagure_activity(user, base_url='https://src.fedoraproject.org'): """Look for user activity in a Pagure deployment.""" if not user: - logging.error('ERROR: Cannot check an empty username!') + log.error('ERROR: Cannot check an empty username!') return False try: resp_src = session.get(f'{base_url}/api/0/user/{user}/activity/stats').json() if not resp_src: - logging.info(f'No activity detected for user {user} in {base_url}') + log.info(f'No activity detected for user {user} in {base_url}') return False error = resp_src.get('error', None) if error is not None: - logging.info(f'Error checking user {user} in {base_url}: {error}') + log.info(f'Error checking user {user} in {base_url}: {error}') return False except Exception: # May happen when the user never interacted with pagure - logging.warning(f'Error while retrieving data for user {user}.') + log.warning(f'Error while retrieving data for user {user}.') return False - logging.info(f'User {user} was active in {base_url}') + log.info(f'User {user} was active in {base_url}') return True def _check_bodhi_activity(user): """Look for user activity in Fedora Bodhi through datagrepper messages.""" if not user: - logging.error('ERROR: Cannot check an empty username!') + log.error('ERROR: Cannot check an empty username!') return False try: page = 1 @@ -243,7 +240,7 @@ def _check_bodhi_activity(user): for message in r.get('raw_messages', []): agent = message.get('msg', dict()).get('agent', None) if agent == user: - logging.info(f'Found recent activity in Bodhi for user {user}') + log.info(f'Found recent activity in Bodhi for user {user}') return True if page >= r.get('pages', 1): return False @@ -251,7 +248,7 @@ def _check_bodhi_activity(user): page += 1 except Exception: # May happen if there are a lot of entries in datagrepper - logging.error(f'ERROR: Error while retrieving data for user {user}') + log.error(f'ERROR: Error while retrieving data for user {user}') return False @@ -262,7 +259,7 @@ def _check_maillists_activity(fasclient, user, delta=365): of email addresses checked. """ if not user: - logging.error('ERROR: Cannot check an empty username!') + log.error('ERROR: Cannot check an empty username!') return (False, []) try: packager = fasclient.get_user(username=user).result @@ -272,10 +269,10 @@ def _check_maillists_activity(fasclient, user, delta=365): last_email_date = datetime.fromisoformat(r.get('results')[0]['date'].replace('Z', '+00:00')) time_delta = datetime.now(timezone.utc) - last_email_date if time_delta.days <= delta: - logging.info(f'Found recent email from user {user}: {last_email_date}.') + log.info(f'Found recent email from user {user}: {last_email_date}.') return (True, packager['emails']) except Exception: - logging.error(f'ERROR: Error while retrieving last mailing lists activity for user {user}.') + log.error(f'ERROR: Error while retrieving last mailing lists activity for user {user}.') return (False, []) return (False, packager['emails']) @@ -287,7 +284,7 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac """ username = packager[0] if not username: - logging.error('ERROR: Cannot check an empty username!') + log.error('ERROR: Cannot check an empty username!') return False emails = packager[1] bzuser = None @@ -297,12 +294,12 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac for bzemail in emails: try: bzuser = bzclient.getuser(bzemail) - logging.debug(f'Found user {username} by email {bzemail if not privacy else "***"}.') + log.debug(f'Found user {username} by email {bzemail if not privacy else "***"}.') break except Exception: - logging.warning(f'Unable to find user {username} by email {bzemail if not privacy else "***"} in Bugzilla.') + log.warning(f'Unable to find user {username} by email {bzemail if not privacy else "***"} in Bugzilla.') if not bzuser: - logging.error(f'ERROR: Unable to check user {username} activity in Bugzilla because I cannot find them.') + log.error(f'ERROR: Unable to check user {username} activity in Bugzilla because I cannot find them.') return False try: bugs = bzclient.query({ @@ -315,14 +312,14 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac "chfieldto" : "Now", "limit": 1000}) except Exception: - logging.error(f'ERROR: Error while retrieving Bugzilla data for user {username}') + log.error(f'ERROR: Error while retrieving Bugzilla data for user {username}') return False ids = [bug.bug_id for bug in bugs] user_comments = [] try: bzbugs = bzclient.getbugs(ids) except Exception: - logging.error(f'ERROR: Error while retrieving Bugzilla comments for user {username}') + log.error(f'ERROR: Error while retrieving Bugzilla comments for user {username}') return False for bug in bzbugs: user_comments.extend([com for com in bug.longdescs if com['creator_id'] == bzuser.userid]) @@ -336,7 +333,7 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac last_com_date = dateparser.parse(f'{last_com_date.value} UTC') time_delta = datetime.now(timezone.utc) - last_com_date if time_delta.days <= delta: - logging.info(f'User {username} made a comment in Bugzilla on {last_com_date.strftime("%Y_%m_%d")}.') + log.info(f'User {username} made a comment in Bugzilla on {last_com_date.strftime("%Y_%m_%d")}.') return True return False @@ -346,25 +343,25 @@ def _check_user_activity(user, privacy=False): try: fasclient = Client(FASCLIENT_URL) except Exception: - logging.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' + log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' 'a Kerberos ticket.') raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' 'a Kerberos ticket.') - logging.info(f'Checking {user} activity in Pagure...') + log.info(f'Checking {user} activity in Pagure...') if ( _check_pagure_activity(user) or _check_pagure_activity(user, base_url='https://pagure.io') ): return True - logging.info(f'Checking {user} activity in Bodhi...') + log.info(f'Checking {user} activity in Bodhi...') if _check_bodhi_activity(user): return True - logging.info(f'Checking {user} activity in mailing lists...') + log.info(f'Checking {user} activity in mailing lists...') maillist_result = _check_maillists_activity(fasclient, user) if maillist_result[0]: return True if BZ_API_KEY: - logging.info(f'Checking {user} activity in bugzilla...') + log.info(f'Checking {user} activity in bugzilla...') bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) packager = (user, maillist_result[1]) if _check_bugzilla_activity(bzclient, packager, privacy=privacy): @@ -374,11 +371,13 @@ def _check_user_activity(user, privacy=False): @click.group() @click.option('--privacy', default=False, help='Hide users email in log.') +@click.option('-D', '--debug', is_flag=True, default=False, help='Enable logging of debug messages.') @click.pass_context -def cli(ctx, privacy): +def cli(ctx, debug, privacy): """Detect inactive users in the packager group.""" ctx.ensure_object(dict) ctx.obj['privacy'] = privacy + ctx.obj['debug'] = debug @cli.command() @@ -391,10 +390,14 @@ def step_one(ctx, open_tickets): it will only output a csv file. """ privacy = ctx.obj['privacy'] + fh = logging.FileHandler(f'step_one_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+') + if ctx.obj['debug']: + log.setLevel(logging.DEBUG) + log.addHandler(fh) if open_tickets: if not PAGURE_API_KEY or not PAGURE_API_BASE_URL: - logging.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' + log.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' 'queue processing will stop immediately.') raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' 'queue processing will stop immediately.') @@ -402,19 +405,19 @@ def step_one(ctx, open_tickets): try: fasclient = Client(FASCLIENT_URL) except Exception: - logging.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' + log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' 'a Kerberos ticket.') raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' 'a Kerberos ticket.') packagers = fasclient.list_group_members(groupname='packager').result - logging.info(f'### Found {len(packagers)} users in the packager group. ###') + log.info(f'### Found {len(packagers)} users in the packager group. ###') inactive_packagers = [] # Check for activity in Pagure - logging.info('Checking users activity in Pagure...') + log.info('Checking users activity in Pagure...') for i, p in enumerate(packagers): if i > 0 and i % 100 == 0: - logging.info(f'Done {i}.') + log.info(f'Done {i}.') user = p.get('username') if user in EXCLUDE_USERS: continue @@ -424,41 +427,41 @@ def step_one(ctx, open_tickets): ): inactive_packagers.append(user) - logging.info(f'### Found {len(inactive_packagers)} users with no activity in pagure/src.fp.org over the last year. ###') + log.info(f'### Found {len(inactive_packagers)} users with no activity in pagure/src.fp.org over the last year. ###') # Check for activity in Bodhi through datagrepper messages - logging.info('Checking users activity in Bodhi...') + log.info('Checking users activity in Bodhi...') for i, p in enumerate(copy(inactive_packagers)): if i > 0 and i % 100 == 0: - logging.info(f'Done {i}.') + log.info(f'Done {i}.') if _check_bodhi_activity(p): inactive_packagers.remove(p) - logging.info(f'### Found {len(inactive_packagers)} users which also show no activity in Bodhi over the last year. ###') + log.info(f'### Found {len(inactive_packagers)} users which also show no activity in Bodhi over the last year. ###') # Check for any activity in mailing lists - logging.info('Checking users activity in mailing lists...') + log.info('Checking users activity in mailing lists...') packager_email_map = {} for i, p in enumerate(inactive_packagers): if i > 0 and i % 100 == 0: - logging.info(f'Done {i}.') + log.info(f'Done {i}.') maillist_results = _check_maillists_activity(fasclient, p) if not maillist_results[0]: packager_email_map[p] = maillist_results[1] - logging.info(f'### Found {len(packager_email_map)} users which show also no activity in mailing lists over the last year. ###') + log.info(f'### Found {len(packager_email_map)} users which show also no activity in mailing lists over the last year. ###') # Check for activity in Bugzilla if BZ_API_KEY: - logging.info('Checking users activity in bugzilla...') + log.info('Checking users activity in bugzilla...') bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) for i, p in enumerate(copy(packager_email_map).items()): if i > 0 and i % 100 == 0: - logging.info(f'Done {i}.') + log.info(f'Done {i}.') if _check_bugzilla_activity(bzclient, p, privacy=privacy): packager_email_map.pop(p[0]) - logging.info(f'### Found {len(packager_email_map)} users which also show no activity in Bugzilla over the last year. ###') + log.info(f'### Found {len(packager_email_map)} users which also show no activity in Bugzilla over the last year. ###') if packager_email_map: with open('inactive_packagers.csv', 'w') as fout: @@ -466,7 +469,7 @@ def step_one(ctx, open_tickets): # Open Pagure tickets ticket_id = 'NONE' if open_tickets: - logging.debug(f'Opening ticket for user {user}') + log.debug(f'Opening ticket for user {user}') headers = {'Authorization': f'token {PAGURE_API_KEY}'} data = {'title': f'Inactive packager detected for user {user}', 'issue_content': PING_INACTIVE_TEXT.format(username = user, email = emails[0]), @@ -475,22 +478,22 @@ def step_one(ctx, open_tickets): try: resp = session.post(f'{PAGURE_API_BASE_URL}/new_issue', data=data, headers=headers) if resp.status_code == 401: - logging.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') + log.error('ERROR: Invalid or expired Pagure token, queue processing will stop immediately.') sys.exit('Invalid or expired Pagure token, queue processing will stop immediately.') if resp.status_code != 200: - logging.error(f'ERROR: Error opening Pagure ticket for user {user}') + log.error(f'ERROR: Error opening Pagure ticket for user {user}') ticket_id = 'ERROR' else: ticket_id = resp.json().get('issue', dict()).get('id', 'ERROR') except Exception: - logging.error(f'ERROR: Error opening Pagure ticket for user {user}') + log.error(f'ERROR: Error opening Pagure ticket for user {user}') ticket_id = 'ERROR' # Write results to file emailstring = '|'.join(emails) - logging.info(f'{user} - {ticket_id} - {emailstring if not privacy else "***"}') + log.info(f'{user} - {ticket_id} - {emailstring if not privacy else "***"}') fout.write(f'{user},{ticket_id},{emailstring}\n') else: - logging.info('### No inactive packagers detected, YHAY! Nothing to do. ###') + log.info('### No inactive packagers detected, YHAY! Nothing to do. ###') @cli.command() @@ -510,42 +513,46 @@ def step_two(ctx, close_tickets, from_file): be considered appropriately. """ privacy = ctx.obj['privacy'] + fh = logging.FileHandler(f'step_two_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+') + if ctx.obj['debug']: + log.setLevel(logging.DEBUG) + log.addHandler(fh) if close_tickets: if not PAGURE_API_KEY or not PAGURE_API_BASE_URL: - logging.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' + log.error('ERROR: You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' 'queue processing will stop immediately.') raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' 'queue processing will stop immediately.') if from_file: - logging.debug('Using CSV file') + log.debug('Using CSV file') pending_removal_users = [] for line in from_file: user, ticket, emails = line.split(',') pending_removal_users.append(user) - logging.info(f'### Checking {len(pending_removal_users)} users from csv file ###') + log.info(f'### Checking {len(pending_removal_users)} users from csv file ###') for i, p in enumerate(copy(pending_removal_users)): if i > 0 and i % 100 == 0: - logging.info(f'Done {i}.') + log.info(f'Done {i}.') if _check_user_activity(p, privacy=privacy): pending_removal_users.remove(p) if pending_removal_users: - logging.info(f'### These {len(pending_removal_users)} users didn\'t react to previous inquiry: ###') + log.info(f'### These {len(pending_removal_users)} users didn\'t react to previous inquiry: ###') with open('still_inactive.csv', 'w') as finactive: for user in pending_removal_users: - logging.info(f'{user}') + log.info(f'{user}') finactive.write(f'{user}\n') else: - logging.info('### All users resumed activity, YHAY! Nothing to do. ###') + log.info('### All users resumed activity, YHAY! Nothing to do. ###') else: - logging.debug('Using Pagure tickets') + log.debug('Using Pagure tickets') confirmed_removal = [] resumed_activity = [] waiting, pending = _fetch_tickets() - logging.info(f'### Pagure returned {len(waiting)} users to check a second time and {len(pending)} users ready for removal ###') + log.info(f'### Pagure returned {len(waiting)} users to check a second time and {len(pending)} users ready for removal ###') if waiting: - logging.info('Starting second check on users...') + log.info('Starting second check on users...') for user, ticket_id in waiting.items(): if not _check_user_activity(user, privacy=privacy): confirmed_removal.append((user, ticket_id)) @@ -553,28 +560,28 @@ def step_two(ctx, close_tickets, from_file): resumed_activity.append((user, ticket_id)) if confirmed_removal: - logging.info(f'### These {len(confirmed_removal)} users didn\'t reply and are going to be removed from packagers: ###') + log.info(f'### These {len(confirmed_removal)} users didn\'t reply and are going to be removed from packagers: ###') for user, ticket in confirmed_removal: - logging.info(f'- {user}') + log.info(f'- {user}') else: - logging.info('### All users resumed activity, HURRAY! ###') + log.info('### All users resumed activity, HURRAY! ###') if pending: - logging.info(f'### These users agreed to be removed from packagers: ###') + log.info(f'### These users agreed to be removed from packagers: ###') for user, ticket_id in pending.items(): - logging.info(f'- {user}') + log.info(f'- {user}') confirmed_removal.append((user, ticket_id)) if resumed_activity: - logging.info(f'### These {len(resumed_activity)} users resumed activity: ###') + log.info(f'### These {len(resumed_activity)} users resumed activity: ###') for user, ticket_id in resumed_activity: - logging.info(f'- {user}') + log.info(f'- {user}') if confirmed_removal: if close_tickets: - logging.info(f'### Closing tickets and generating output file ###') + log.info(f'### Closing tickets and generating output file ###') for user, ticket_id in resumed_activity: - logging.info(f'Closing ticket {ticket_id} for user {user}...') + log.info(f'Closing ticket {ticket_id} for user {user}...') _close_pagure_ticket( ticket_id, 'Keep packager status', @@ -584,7 +591,7 @@ def step_two(ctx, close_tickets, from_file): with open('still_inactive.csv', 'w') as finactive: for user, ticket_id in confirmed_removal: if close_tickets: - logging.info(f'Closing ticket {ticket_id} for user {user}...') + log.info(f'Closing ticket {ticket_id} for user {user}...') _close_pagure_ticket( ticket_id, 'Removed from packagers', @@ -595,18 +602,24 @@ def step_two(ctx, close_tickets, from_file): @cli.command() -def check_impact(): +@click.pass_context +def check_impact(ctx): """Report packages expected to be orphaned. This report will show how many packages are going to be orphaned based on the "inactive packager" tickets currently opened. """ + fh = logging.FileHandler(f'check_impact_{datetime.now(timezone.utc).strftime("%Y_%m_%d_%H:%M:%S")}.log', 'w+') + if ctx.obj['debug']: + log.setLevel(logging.DEBUG) + log.addHandler(fh) + waiting, pending = _fetch_tickets() affected_packages = [] - logging.info(f'### Fetching affected users info ###') + log.info(f'### Fetching affected users info ###') for u, t in waiting.items(): page = 1 - logging.info(f'Getting user {u} information') + log.info(f'Getting user {u} information') while True: try: data = session.get(f'https://src.fedoraproject.org/api/0/user/{u}' @@ -616,15 +629,15 @@ def check_impact(): for repo in repos: if u in repo.get('access_users', dict()).get('owner'): affected_packages.append((repo.get('name'), u, '')) - logging.info(f"Done {page} pages out of {total_pages}.") + log.info(f"Done {page} pages out of {total_pages}.") if page >= total_pages: break page += 1 except Exception: - logging.warning('Error while retrieving user info.') + log.warning('Error while retrieving user info.') for u, t in pending.items(): page = 1 - logging.info(f'Getting user {u} information') + log.info(f'Getting user {u} information') while True: try: data = session.get(f'https://src.fedoraproject.org/api/0/user/{u}' @@ -634,14 +647,14 @@ def check_impact(): for repo in repos: if u in repo.get('access_users', dict()).get('owner'): affected_packages.append((repo.get('name'), u, '*')) - logging.info(f"Done {page} pages out of {total_pages}.") + log.info(f"Done {page} pages out of {total_pages}.") if page >= total_pages: break page += 1 except Exception: - logging.warning('Error while retrieving ticket list.') + log.warning('Error while retrieving ticket list.') package_list = "\n".join([f'- {mark}{package} (owned by {user})' for package, user, mark in sorted(affected_packages)]) - logging.info(CHECK_IMPACT_OUTPUT.format(total_packages=len(affected_packages), package_list=package_list)) + log.info(CHECK_IMPACT_OUTPUT.format(total_packages=len(affected_packages), package_list=package_list)) @cli.command() @@ -650,6 +663,9 @@ def check_impact(): def check_user(ctx, username): """Check if any activity is detected for a specific username.""" privacy = ctx.obj['privacy'] + if ctx.obj['debug']: + log.setLevel(logging.DEBUG) + if _check_user_activity(username, privacy=privacy): click.echo(f"Activity detected for user {username}.") else: From 153945a54da84d653f25f8618ff0cc1242d9876c Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 12 2022 07:15:42 +0000 Subject: [PATCH 6/9] Update README --- diff --git a/README.md b/README.md index f903a48..d31470c 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,20 @@ The script will search for any activity in the last year in: - pagure.io - bodhi (through datagrepper) - Fedora mailing lists -- Red Hat Bugzilla +- Red Hat bugzilla -It will output the 'inactive_packagers.csv' file with a list of all users in the packager -group and their email which are found to be inactive. This can be used to contact +The step-one subcommand will output the 'inactive_packagers.csv' file with a list of all users +in the packager group and their email which are found to be inactive. This can be used to contact the inactive users, asking them to show some activity in pagure, datagrepper or mailing lists. +Optionally, it can open tickets against inactive packagers in a Pagure repository. -The step-one subcommand will open tickets against inactive packagers in a Pagure repository. -The step-two is used to confront the results of a previous run (provided by the csv file) -and a fresh run, to identify which users still don't show any activity. Therefore we should -consider these users to have abandoned Fedora. +The check-impact subcommand will iterate on the list of tickets opened and provide information +about how many packages will possibly be orphaned based on the packagers detected as inactive. + +The step-two subcommand can either check a list of inactive users from the csv file generated +in step-one, or check a list of users based on open Pagure tickets. In this last case, the tickets +can be closed appropriately. Another csv file is generated as output to list users which are still +inactive. Use the '--privacy' flag to not print emails in the log. @@ -26,7 +30,7 @@ To run the script you'll need: - a bugzilla API key stored as OS env variable 'BZ_API_KEY' Optionally, providing a Pagure repository URL and a token the script can automatically -open an issue ticket for each detected inactive packager. +open and manage an issue ticket for each detected inactive packager. ## Example usage diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index b2800cc..decb33b 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -54,6 +54,9 @@ To run the script you'll need: - an active kerberos ticket to login to fasjson.fedoraproject.org - a bugzilla API key stored as OS env variable 'BZ_API_KEY' +Optionally, providing a Pagure repository URL and a token the script can automatically +open and manage an issue ticket for each detected inactive packager. + """ import click From 01fa3cb17367ea4383c50f2173d55d58770b5171 Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 12 2022 07:43:16 +0000 Subject: [PATCH 7/9] Always mask email in Pagure tickets --- diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index decb33b..c67ec13 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -297,10 +297,10 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac for bzemail in emails: try: bzuser = bzclient.getuser(bzemail) - log.debug(f'Found user {username} by email {bzemail if not privacy else "***"}.') + log.debug(f'Found user {username} by email {mask_email(bzemail, privacy=privacy)}.') break except Exception: - log.warning(f'Unable to find user {username} by email {bzemail if not privacy else "***"} in Bugzilla.') + log.warning(f'Unable to find user {username} by email {mask_email(bzemail, privacy=privacy)} in Bugzilla.') if not bzuser: log.error(f'ERROR: Unable to check user {username} activity in Bugzilla because I cannot find them.') return False @@ -372,8 +372,37 @@ def _check_user_activity(user, privacy=False): return False +def mask_email(email, preserve_chars=3, mask_char='*', privacy=True): + """Obfuscate the end of the local part of an email address. + + By default the three starting characters are preserved, while the ending of the local + part is masked. + The function always tries to mask at least 2 characters, in case of short addresses. + + Args: + email: a valid email address to mask. + preserve_chars: number of characters to preserve at the beginning of the local part. + mask_char: the character used as a replacement. + """ + if not privacy: + return email + valid_email = email.split('@') + if not len(valid_email) == 2: + log.debug(f'Invalid email: {email}') + return email + local_length = len(valid_email[0]) + if local_length - preserve_chars < 2: + preserve_chars = local_length - 2 + if preserve_chars < 1: + local_part = mask_char * 2 + else: + local_part = (valid_email[0][:preserve_chars] + + mask_char * (local_length - preserve_chars)) + return f'{local_part}@{valid_email[1]}' + + @click.group() -@click.option('--privacy', default=False, help='Hide users email in log.') +@click.option('--privacy', is_flag=True, default=False, help='Hide users email in log.') @click.option('-D', '--debug', is_flag=True, default=False, help='Enable logging of debug messages.') @click.pass_context def cli(ctx, debug, privacy): @@ -475,7 +504,7 @@ def step_one(ctx, open_tickets): log.debug(f'Opening ticket for user {user}') headers = {'Authorization': f'token {PAGURE_API_KEY}'} data = {'title': f'Inactive packager detected for user {user}', - 'issue_content': PING_INACTIVE_TEXT.format(username = user, email = emails[0]), + 'issue_content': PING_INACTIVE_TEXT.format(username = user, email = mask_email(emails[0])), 'tag': PAGURE_NEW_TICKET_TAG, 'assignee': PAGURE_NEW_TICKET_ASSIGNEE} try: @@ -492,8 +521,8 @@ def step_one(ctx, open_tickets): log.error(f'ERROR: Error opening Pagure ticket for user {user}') ticket_id = 'ERROR' # Write results to file - emailstring = '|'.join(emails) - log.info(f'{user} - {ticket_id} - {emailstring if not privacy else "***"}') + emailstring = '|'.join([mask_email(em, privacy=privacy) for em in emails]) + log.info(f'{user} - {ticket_id} - {emailstring}') fout.write(f'{user},{ticket_id},{emailstring}\n') else: log.info('### No inactive packagers detected, YHAY! Nothing to do. ###') From 8007f5c6dc6e4cd6b7476dc8552bf1f8e56b95d2 Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Sep 12 2022 08:30:53 +0000 Subject: [PATCH 8/9] Do not recreate fasclient and bzclient for every user --- diff --git a/find_inactive_packagers.py b/find_inactive_packagers.py index c67ec13..31908d5 100644 --- a/find_inactive_packagers.py +++ b/find_inactive_packagers.py @@ -341,15 +341,16 @@ def _check_bugzilla_activity(bzclient, packager, check_fedora_alias=True, privac return False -def _check_user_activity(user, privacy=False): +def _check_user_activity(user, privacy=False, fasclient=None, bzclient=None): """Check user activity.""" - try: - fasclient = Client(FASCLIENT_URL) - except Exception: - log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' + if not fasclient: + try: + fasclient = Client(FASCLIENT_URL) + except Exception: + log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' 'a Kerberos ticket.') - raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' - 'a Kerberos ticket.') + raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' + 'a Kerberos ticket.') log.info(f'Checking {user} activity in Pagure...') if ( _check_pagure_activity(user) @@ -365,7 +366,8 @@ def _check_user_activity(user, privacy=False): return True if BZ_API_KEY: log.info(f'Checking {user} activity in bugzilla...') - bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) + if not bzclient: + bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) packager = (user, maillist_result[1]) if _check_bugzilla_activity(bzclient, packager, privacy=privacy): return True @@ -556,6 +558,16 @@ def step_two(ctx, close_tickets, from_file): 'queue processing will stop immediately.') raise SystemExit('You need to provide PAGURE_API_KEY and PAGURE_API_BASE_URL, ' 'queue processing will stop immediately.') + try: + fasclient = Client(FASCLIENT_URL) + except Exception: + log.error('ERROR: Unable to connect to fasclient, you probably forgot to obtain ' + 'a Kerberos ticket.') + raise SystemExit('Unable to connect to fasclient, you probably forgot to obtain ' + 'a Kerberos ticket.') + + if BZ_API_KEY: + bzclient = Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi', api_key=BZ_API_KEY) if from_file: log.debug('Using CSV file') @@ -567,7 +579,7 @@ def step_two(ctx, close_tickets, from_file): for i, p in enumerate(copy(pending_removal_users)): if i > 0 and i % 100 == 0: log.info(f'Done {i}.') - if _check_user_activity(p, privacy=privacy): + if _check_user_activity(p, privacy=privacy, fasclient=fasclient, bzclient=bzclient): pending_removal_users.remove(p) if pending_removal_users: log.info(f'### These {len(pending_removal_users)} users didn\'t react to previous inquiry: ###') @@ -586,7 +598,7 @@ def step_two(ctx, close_tickets, from_file): if waiting: log.info('Starting second check on users...') for user, ticket_id in waiting.items(): - if not _check_user_activity(user, privacy=privacy): + if not _check_user_activity(user, privacy=privacy, fasclient=fasclient, bzclient=bzclient): confirmed_removal.append((user, ticket_id)) else: resumed_activity.append((user, ticket_id)) From 4ee74a8d448905f6673908e3f3b13206f6783c7b Mon Sep 17 00:00:00 2001 From: Mattia Verga Date: Oct 23 2022 09:18:53 +0000 Subject: [PATCH 9/9] Add SOP file Signed-off-by: Mattia Verga --- diff --git a/SOP.md b/SOP.md new file mode 100644 index 0000000..7cf3ee3 --- /dev/null +++ b/SOP.md @@ -0,0 +1,80 @@ +# SOP for periodic inactive packagers removal + +The [inactive packagers removal policy](https://docs.fedoraproject.org/en-US/fesco/Policy_for_inactive_packagers/) +is performed through the use of the [find_inactive_packagers.py](https://pagure.io/find-inactive-packagers) script. + +## Setting up the script + +Download the git repo and make sure your system (or your virtualenv) has `fasjson_client` and `bugzilla` +installed. You will also need to acquire a kerberos ticket to the Fedora infrastructure - the esiest +way is to use `fkinit` command provided by `fedora-packager-kerberos` package. + +Then, set up some settings provided at the start of the script file. You will need to provide a Bugzilla API +key (mandatory) and a Pagure API key (optional, only needed to manage tickets, but the script can also be +used "read-only" for testing purposes). Other settings should be fine with their defaults. + +## Step one: identify inactive packagers + +The first step of the policy is to identify packagers that show no activity in the Fedora ecosystem in the +last year. This is done by running: + +```python find_inactive_packagers.py step-one``` + +or: + +```python find_inactive_packagers.py step-one --open-tickets``` + +Both command will fetch the packagers list and look in several Fedora related systems for activity of each +user. It may take a lot of time, depending on the number of packagers and their activity. + +The ``--open-tickets`` option **MUST** only be used by the Fedora PGM when running the policy. If you want to test +the opening tickets functionality, please change `PAGURE_API_BASE_URL` settings to a testing repository of +your own and **remove** the `@` from `PING_INACTIVE_TEXT`, otherwise users will be mentioned and notified +about these testing tickets! + +The command will provide a `inactive_packagers.csv` file, listing detected inactive packagers usernames, +the opened ticket id (if `--open-tickets`) and the list of emails associated to the packager. An extensive +log is also provided, if you plan to share the log for debugging purpose you may want to run the commands +with the `--privacy` flag, so that emails are masked in the logs. + +## Managing tickets + +The script will open one ticket for each packager detected as inactive. Those tickets are tagged +`inactive_packager` to enable automatic ticket managing. + +At this point a user can reply to a ticket in two ways: + +- if the user replies that they want to maintain the packager status, the ticket must be closed + as `Keep packager status`. +- if the user agrees to be removed from packagers, **do not** manually close the ticket, but add the + `asked_removal` tag. The ticket will be closed automatically on step two. + +## Optional intermediate step: check packages going to be orphaned + +Once tickets against inactive packagers are opened, you may want to check how many and what packages +will be orphaned when the affected packagers will be removed from the packagers group. To make that +run: + +```python find_inactive_packagers.py check-impact``` + +This command will fetch the list of **open** tickets and provide a list of packages which are possibly +going to be orphaned or which will surely be orphaned (for those tickets which were tagged `asked_removal`). +Please note that this is based only on the opened tickets, there's no check about recent activity +from the user. + +## Step two: provide the final list of inactive packagers + +The step two of the policy will look for packagers which didn't reply to the inquiry ticket and, like the +step one, can be run "read-only": + +```python find_inactive_packagers.py step-two``` + +or not (**reserved to the Fedora PGM**): + +```python find_inactive_packagers.py step-two --close-tickets``` + +The output file `still_inactive.csv` will be the final list of usernames to be removed from the packagers +group. If the command is run with ``--close-tickets``, it will automatically close tickets in the right way. +That means that if a user didn't reply to the ticket, but the script detects recent activity of the same user, +the ticket will be closed as `Keep packager status` and will not add the username to the list of users +going to be removed.