From 75398708bf577654a18beb3508ffc3df770613d6 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Feb 12 2017 11:05:54 +0000
Subject: [PATCH 1/4] Beginning of refactor to remove TracImporter object


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index 5e22a64..0f3004d 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -2,7 +2,6 @@ import click
 import pagure_importer
 from pagure_importer.app import app, REPO_PATH
 from pagure_importer.utils import importer_trac, get_pagure_namespace
-from pagure_importer.utils.fas import FASclient
 
 import pagure_importer.utils.git as gitutils
 
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index ffaee9d..004a7bd 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -1,15 +1,20 @@
+import os
 import sys
 import re
 import time
 import click
 import requests
-import shutil
-import os
+from random import randint
 from base64 import b64decode
 from datetime import datetime
 from pagure_importer.utils import (
     get_close_status, is_image, issue_to_json, get_secure_filename)
 from pagure_importer.utils.models import User, Issue, IssueComment
+from pagure_importer.utils.fas import FASclient
+from pagure_importer.app import REPO_PATH
+
+SOMEBODY = User(name='somebody', fullname='somebody',
+                emails=['some@body.com'])
 
 
 def to_timestamp(tm):
@@ -21,298 +26,276 @@ def to_timestamp(tm):
     return ts
 
 
-class TracImporter(object):
-    ''' Pagure importer for trac instance '''
-
-    def __init__(self, project_url, username, password, offset, repo_name,
-                 repo_folder, nopush, fasclient=None, tags=False, private=False):
-        ''' Instantiate a TracImporter object '''
-        self.username = username
-        self.password = password
-        self.repo_name = repo_name
-        self.repo_folder = repo_folder
-        self.clone_repo_location = os.path.join(
-            repo_folder, 'clone-' + repo_name)
-        self.nopush = nopush
-        self.url = project_url
-        self.fas = fasclient
-        self.tags = tags
-        self.private = private
-        self.offset = offset
-        self.somebody = User(name='somebody', fullname='somebody',
-                             emails=['some@body.com'])
-        self.reqid = 0
-        self.custom_fields = []
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        ''' Delete the cloned repo where the commits were going '''
-        if os.path.exists(self.clone_repo_location):
-            if not self.nopush:
-                shutil.rmtree(self.clone_repo_location)
-
-    def request(self, method, *args):
-        ''' Common method for querying trac '''
-
-        self.reqid += 1
-        req = {'params': args,
-               'method': method,
-               'id': self.reqid}
-        resp = requests.post(self.url, json=req,
-                             auth=(self.username, self.password))
-        resp = resp.json()
-        if resp['id'] != self.reqid:
-            click.echo('ERROR: Invalid response for request! ' +
-                       'ID does not match')
-            sys.exit(1)
-        if resp['error'] is not None:
-            # Ignore missing attachment errors
-            if 'Attachment ' not in resp['error']['message'] and \
-               ' not found' not in resp['error']['message']:
-                click.echo("ERROR: Error in response: %s" % resp['error'])
-                sys.exit(1)
-
-        return resp['result']
-
-    def get_custom_fields(self):
-        ''' Queries the fedorahosted api to get all ticket fields
-        and filters all the custom fields, returns
-        a list of dicts - dict with keys 'name' and 'key_type' '''
-
-        all_ticket_fields = self.request('ticket.getTicketFields')
-        custom_fields = []
-        for field in all_ticket_fields:
-            if field.get('custom') is True:
-                current_field = {}
-                current_field['name'] = field['name']
-                key_type = 'text'
-                if field['type'] == 'checkbox':
-                    key_type = 'boolean'
-                current_field['key_type'] = key_type
-                custom_fields.append(current_field)
-        return custom_fields
-
-    def import_issues(self, repo_name, trac_query='max=0&order=id'):
-        ''' Queries the trac instance via its jsonrpc API and convert the
-        tickets into JSON blob to be imported into pagure's ticket git repo.
-
-        :arg repo_name: the name of the repository
-        :arg repo_folder: the folder in which is the repository
-        :kwarg trac_query: the query to call trac with in order to retrieve
-            all the tickets.
-            Defaults to ``max=0&order=id``
-
-        '''
-
-        tickets_id = self.request('ticket.query', trac_query)
-        self.custom_fields = self.get_custom_fields()
-
-        for ticket_id in tickets_id:
-            pagure_issue = self.create_issue(ticket_id)
-            pagure_issue.comments = []
-            pagure_issue_comments = self.request('ticket.changeLog', ticket_id)
-            comments = self.create_comments(pagure_issue_comments)
-            # add all the comments to the issue object
-            for key in comments:
-                if comments[key].attachment is not None and \
-                   any(attachment in comments[key].attachment for attachment in
-                       pagure_issue.attachment):
-
-                    for attach_name in comments[key].attachment:
-                        filename = get_secure_filename(
-                            pagure_issue.attachment[attach_name], attach_name)
-                        url = '/%s/issue/raw/files/%s' % (repo_name, filename)
-                        if is_image(attach_name):
-                            comments[key].comment += ('\n[![%s](%s)](%s)' %
-                                                      (attach_name, url, url))
-                        else:
-                            comments[key].comment += ('\n[%s](%s)' %
-                                                      (attach_name, url))
-                pagure_issue.comments.append(comments[key].to_json())
-            click.echo('Updated ' + repo_name + ' with issue :' +
-                       str(ticket_id) + '/' + str(tickets_id[-1]))
-            issue_to_json(pagure_issue, self.clone_repo_location)
-
-    def get_custom_fields_of_ticket(self, trac_ticket):
-        ''' Given the trac ticket, it will return all the
-        custom fields of the ticket, in a form that it can
-        be used for pagure Issue '''
-
-        pagure_fields = []
-        for field in self.custom_fields:
-            if field['name'] in trac_ticket:
-                pagure_field = {}
-                pagure_field['name'] = field.get('name')
-                pagure_field['key_type'] = field.get('key_type')
-                pagure_field['value'] = trac_ticket.get(
-                                    pagure_field['name'], "").strip()
-                if pagure_field['value']:
-                    pagure_fields.append(pagure_field)
-        return pagure_fields
-
-    def create_issue(self, ticket_id):
-        ''' Create Issue object from track ticket '''
-
-        trac_ticket_info = self.request('ticket.get', ticket_id)
-        trac_ticket = trac_ticket_info[3]
-        trac_attachments = self.request('ticket.listAttachments', ticket_id)
-
-        pagure_attachment = {}
-        for attachment in trac_attachments:
-            filename = attachment[0]
-            attachment_resp = self.request(
-                'ticket.getAttachment',
-                ticket_id, filename)
-            if attachment_resp:
-                if is_image(filename):
-                    content = b64decode(attachment_resp['__jsonclass__'][1])
-                    pagure_attachment[filename] = content
-                else:
-                    content = b64decode(
-                        attachment_resp['__jsonclass__'][1].replace('\n', ''))
-                    pagure_attachment[filename] = content
-
-        pagure_custom_fields = self.get_custom_fields_of_ticket(trac_ticket)
-        pagure_issue_title = trac_ticket['summary']
-
-        pagure_issue_content = trac_ticket['description']
-        if pagure_issue_content == '':
-            pagure_issue_content = '#No Description Provided'
-
-        issue_status, close_status = self.get_ticket_status(trac_ticket)
-
-        pagure_issue_created_at = to_timestamp(
-            trac_ticket_info[1]['__jsonclass__'][1])
-
-        if self.fas:
-            pagure_issue_assignee = self.fas.find_fas_user(
-                trac_ticket['owner'])
-            pagure_issue_user = self.fas.find_fas_user(trac_ticket['reporter'])
-            if not pagure_issue_user.name:
-                pagure_issue_user = User(
-                    name=trac_ticket['reporter'],
-                    fullname=trac_ticket['reporter'],
-                    emails=[trac_ticket['reporter'] + '@fedoraproject.org'])
-        else:
-            pagure_issue_assignee = User(name='', fullname='', emails=[])
-            pagure_issue_user = User(
-                name=trac_ticket['reporter'],
-                fullname=trac_ticket['reporter'],
-                emails=[trac_ticket['reporter'] + '@fedoraproject.org'])
-
-        # The milestone of the issue
-        pagure_milestone = None
-        if 'milestone' in trac_ticket and trac_ticket['milestone'] != '':
-            pagure_milestone = trac_ticket['milestone']
-
-        # Issue tags
-        pagure_issue_tags = []
-        if self.tags:
-            pagure_issue_tags = filter(
-                lambda x: x != '', trac_ticket['keywords'].split(' '))
-
-        pagure_issue_tags = self.pre_process_tags(pagure_issue_tags)
-
-        pagure_issue_depends = []
-        pagure_issue_blocks = []
-        if self.private:
-            pagure_issue_is_private = True
-        else:
-            pagure_issue_is_private = False
-
-        pagure_issue = Issue(
-            id=ticket_id + self.offset,
-            title=pagure_issue_title,
-            content=pagure_issue_content,
-            status=issue_status,
-            close_status=close_status,
-            date_created=pagure_issue_created_at,
-            user=pagure_issue_user.to_json(),
-            private=pagure_issue_is_private,
-            attachment=pagure_attachment,
-            tags=pagure_issue_tags,
-            milestone=pagure_milestone,
-            depends=pagure_issue_depends,
-            blocks=pagure_issue_blocks,
-            assignee=pagure_issue_assignee.to_json(),
-            custom_fields=pagure_custom_fields,)
-        return pagure_issue
-
-    def pre_process_tags(self, tags):
-        ''' Pre process the tags before sending it to pagure '''
-
-        delims_str = ',|/|;|:|-|\+|\*|\'|\"'
-
-        # Remove the delims between the tags and convert all tags to lower case
-        pagure_issue_tags = list(set([j for k in [
-            re.split(delims_str, i.lower()) for i in tags] for j in k]))
-
-        return pagure_issue_tags
-
-    def get_ticket_status(self, trac_ticket):
-        ''' Returns the corresponding status of ticket on pagure '''
-        close_status = get_close_status()
-        if close_status is not None:
-            if trac_ticket['status'] != 'closed':
-                return ('Open', '')
-            else:
-                for status in close_status:
-                    if trac_ticket['resolution'] in close_status[status]:
-                        return ('Closed', status)
-                return ('Closed', 'Fixed')
-        else:
-            click.echo('ERROR: Close Status not read from config file')
-            sys.exit(1)
+def pre_process_tags(tags):
+    ''' Pre process the tags before sending it to pagure '''
 
-    def get_comment_user(self, comment):
-        ''' Returns the user who commented on the ticket '''
+    delims_str = ',|/|;|:|-|\+|\*|\'|\"'
 
-        # The User who commented
-        if self.fas and comment[1]:
-            pagure_issue_comment_user = self.fas.find_fas_user(comment[1])
-            if not pagure_issue_comment_user.name:
-                pagure_issue_comment_user = self.somebody
-        else:
-            pagure_issue_comment_user = self.somebody
-        return pagure_issue_comment_user
+    # Remove the delims between the tags and convert all tags to lower case
+    pagure_issue_tags = list(set([j for k in [
+        re.split(delims_str, i.lower()) for i in tags] for j in k]))
 
-    def create_comments(self, trac_comments):
-        ''' Create IssueComment objects from the trac comments '''
+    return pagure_issue_tags
 
-        comments = {}
-        for comment in trac_comments:
-            ts = to_timestamp(comment[0]['__jsonclass__'][1])
 
-            if comment[2] == 'comment' and comment[4] != '':
-                if ts in comments:
-                    attachment = comments[ts].attachment
-                else:
-                    attachment = []
+def request(method, rest, *args):
+    ''' Common method for querying trac '''
 
-                pagure_issue_comment_body = comment[4]
-                pagure_issue_comment_created_at = ts
+    reqid = randint(0, 100)
+    req = {'params': args,
+           'method': method,
+           'id': reqid}
+    resp = requests.post(rest.url, json=req,
+                         auth=(rest.username, rest.password))
+    resp = resp.json()
+    if resp['id'] != reqid:
+        click.echo('ERROR: Invalid response for request! ' +
+                   'ID does not match')
+        sys.exit(1)
+    if resp['error'] is not None:
+        # Ignore missing attachment errors
+        if 'Attachment ' not in resp['error']['message'] and \
+                        ' not found' not in resp['error']['message']:
+            click.echo("ERROR: Error in response: %s" % resp['error'])
+            sys.exit(1)
 
+    return resp['result']
+
+
+def import_issues(repo_name, rest, trac_query='max=0&order=id'):
+    ''' Queries the trac instance via its jsonrpc API and convert the
+    tickets into JSON blob to be imported into pagure's ticket git repo.
+
+    :arg repo_name: the name of the repository
+    :arg repo_folder: the folder in which is the repository
+    :kwarg trac_query: the query to call trac with in order to retrieve
+        all the tickets.
+        Defaults to ``max=0&order=id``
+
+    '''
+
+    tickets_id = request('ticket.query', rest, trac_query)
+    ticket_fields = request('ticket.getTicketFields', rest)
+    custom_fields = get_custom_fields(ticket_fields)
+    clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name)
+
+    for ticket_id in tickets_id:
+        pagure_issue = create_issue(ticket_id=ticket_id,
+                                    custom_fields=custom_fields,
+                                    rest=rest)
+        pagure_issue.comments = []
+        pagure_issue_comments = request('ticket.changeLog', rest, ticket_id)
+        comments = create_comments(pagure_issue_comments)
+        # add all the comments to the issue object
+        pagure_issue = add_comment_to_issue(comments, pagure_issue)
+
+        click.echo('Updated ' + repo_name + ' with issue :' +
+                   str(ticket_id) + '/' + str(tickets_id[-1]))
+        issue_to_json(pagure_issue, clone_repo_location)
+
+
+def get_custom_fields(ticket_fields):
+    ''' Queries the fedorahosted api to get all ticket fields
+    and filters all the custom fields, returns
+    a list of dicts - dict with keys 'name' and 'key_type' '''
+
+    custom_fields = []
+    for field in ticket_fields:
+        if field.get('custom') is True:
+            current_field = {}
+            current_field['name'] = field['name']
+            key_type = 'text'
+            if field['type'] == 'checkbox':
+                key_type = 'boolean'
+            current_field['key_type'] = key_type
+            custom_fields.append(current_field)
+    return custom_fields
+
+
+def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags):
+    ''' Create Issue object from track ticket '''
+
+    trac_ticket_info = request('ticket.get', rest, ticket_id)
+    trac_ticket = trac_ticket_info[3]
+    trac_attachments = request('ticket.listAttachments', rest, ticket_id)
+
+    pagure_attachment = {}
+    for attachment in trac_attachments:
+        filename = attachment[0]
+        attachment_resp = request('ticket.getAttachment',
+                                  rest, ticket_id, filename)
+        if attachment_resp:
+            if is_image(filename):
+                content = b64decode(attachment_resp['__jsonclass__'][1])
+                pagure_attachment[filename] = content
+            else:
+                content = b64decode(
+                    attachment_resp['__jsonclass__'][1].replace('\n', ''))
+                pagure_attachment[filename] = content
+
+    pagure_custom_fields = get_custom_fields_of_ticket(trac_ticket=trac_ticket,
+                                                       custom_fields=custom_fields)
+    pagure_issue_title = trac_ticket['summary']
+
+    pagure_issue_content = trac_ticket['description']
+    if pagure_issue_content == '':
+        pagure_issue_content = '#No Description Provided'
+
+    issue_status, close_status = get_ticket_status(trac_ticket)
+
+    pagure_issue_created_at = to_timestamp(
+        trac_ticket_info[1]['__jsonclass__'][1])
+
+    fas = FASclient(rest.username, rest.password,
+                    'https://admin.fedoraproject.org/accounts')
+    pagure_issue_assignee = fas.find_fas_user(trac_ticket['owner'])
+    pagure_issue_user = fas.find_fas_user(trac_ticket['reporter'])
+    if not pagure_issue_user.name:
+        pagure_issue_user = User(
+            name=trac_ticket['reporter'],
+            fullname=trac_ticket['reporter'],
+            emails=[trac_ticket['reporter'] + '@fedoraproject.org'])
+
+    # The milestone of the issue
+    pagure_milestone = None
+    if 'milestone' in trac_ticket and trac_ticket['milestone'] != '':
+        pagure_milestone = trac_ticket['milestone']
+
+    # Issue tags
+    pagure_issue_tags = []
+    if tags:
+        pagure_issue_tags = filter(
+            lambda x: x != '', trac_ticket['keywords'].split(' '))
+
+    pagure_issue_tags = pre_process_tags(pagure_issue_tags)
+
+    pagure_issue_depends = []
+    pagure_issue_blocks = []
+    if private:
+        pagure_issue_is_private = True
+    else:
+        pagure_issue_is_private = False
+
+    pagure_issue = Issue(
+        id=ticket_id + offset,
+        title=pagure_issue_title,
+        content=pagure_issue_content,
+        status=issue_status,
+        close_status=close_status,
+        date_created=pagure_issue_created_at,
+        user=pagure_issue_user.to_json(),
+        private=pagure_issue_is_private,
+        attachment=pagure_attachment,
+        tags=pagure_issue_tags,
+        milestone=pagure_milestone,
+        depends=pagure_issue_depends,
+        blocks=pagure_issue_blocks,
+        assignee=pagure_issue_assignee.to_json(),
+        custom_fields=pagure_custom_fields,)
+    return pagure_issue
+
+
+def get_custom_fields_of_ticket(trac_ticket, custom_fields):
+    ''' Given the trac ticket, it will return all the
+    custom fields of the ticket, in a form that it can
+    be used for pagure Issue '''
+
+    pagure_fields = []
+    for field in custom_fields:
+        if field['name'] in trac_ticket:
+            pagure_field = {}
+            pagure_field['name'] = field.get('name')
+            pagure_field['key_type'] = field.get('key_type')
+            pagure_field['value'] = trac_ticket.get(pagure_field['name'])
+            pagure_fields.append(pagure_field)
+    return pagure_fields
+
+
+def get_ticket_status(trac_ticket):
+    ''' Returns the corresponding status of ticket on pagure '''
+    close_status = get_close_status()
+    if close_status is not None:
+        if trac_ticket['status'] != 'closed':
+            return ('Open', '')
+        else:
+            for status in close_status:
+                if trac_ticket['resolution'] in close_status[status]:
+                    return ('Closed', status)
+            return ('Closed', 'Fixed')
+    else:
+        click.echo('ERROR: Close Status not read from config file')
+        sys.exit(1)
+
+
+def create_comments(trac_comments):
+    ''' Create IssueComment objects from the trac comments '''
+
+    comments = {}
+    for comment in trac_comments:
+        ts = to_timestamp(comment[0]['__jsonclass__'][1])
+
+        if comment[2] == 'comment' and comment[4] != '':
+            if ts in comments:
+                attachment = comments[ts].attachment
+            else:
+                attachment = []
+
+            pagure_issue_comment_body = comment[4]
+            pagure_issue_comment_created_at = ts
+
+            pagure_issue_comment_user = get_comment_user(comment)
+            # Object to represent comment on an issue
+            comments[ts] = IssueComment(
+                id=None,
+                comment=pagure_issue_comment_body,
+                date_created=pagure_issue_comment_created_at,
+                attachment=attachment,
+                user=pagure_issue_comment_user.to_json())
+
+        elif comment[2] == 'attachment':
+            if ts in comments:
+                comments[ts].attachment.append(comment[4])
+            else:
                 pagure_issue_comment_user = self.get_comment_user(comment)
-                # Object to represent comment on an issue
                 comments[ts] = IssueComment(
                     id=None,
-                    comment=pagure_issue_comment_body,
-                    date_created=pagure_issue_comment_created_at,
-                    attachment=attachment,
+                    comment='attachment',
+                    date_created=ts,
+                    attachment=[comment[4]],
                     user=pagure_issue_comment_user.to_json())
 
-            elif comment[2] == 'attachment':
-                if ts in comments:
-                    comments[ts].attachment.append(comment[4])
+    return comments
+
+
+def get_comment_user(comment, fas):
+    ''' Returns the user who commented on the ticket '''
+
+    # The User who commented
+    if fas and comment[1]:
+        pagure_issue_comment_user = fas.find_fas_user(comment[1])
+        if not pagure_issue_comment_user.name:
+            pagure_issue_comment_user = SOMEBODY
+    else:
+        pagure_issue_comment_user = SOMEBODY
+    return pagure_issue_comment_user
+
+
+def add_comment_to_issue(comments, pagure_issue, repo_name):
+    '''Adding comment to an issue'''
+    for key in comments:
+        if comments[key].attachment is not None and \
+                any(attachment in comments[key].attachment for attachment in
+                    pagure_issue.attachment):
+
+            for attach_name in comments[key].attachment:
+                filename = get_secure_filename(
+                    pagure_issue.attachment[attach_name], attach_name)
+                url = '/%s/issue/raw/files/%s' % (repo_name, filename)
+                if is_image(attach_name):
+                    comments[key].comment += ('\n[![%s](%s)](%s)' %
+                                              (attach_name, url, url))
                 else:
-                    pagure_issue_comment_user = self.get_comment_user(comment)
-                    comments[ts] = IssueComment(
-                        id=None,
-                        comment='attachment',
-                        date_created=ts,
-                        attachment=[comment[4]],
-                        user=pagure_issue_comment_user.to_json())
-
-        return comments
+                    comments[key].comment += ('\n[%s](%s)' %
+                                              (attach_name, url))
+        pagure_issue.comments.append(comments[key].to_json())
+    return pagure_issue

From cb8e1f576cee7efd168f0c24a65b9962258727f7 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Feb 12 2017 11:05:54 +0000
Subject: [PATCH 2/4] Move logic to import issues into fedorahosted command code


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index 0f3004d..cc4d822 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -1,9 +1,13 @@
+import os
+from collections import namedtuple
 import click
 import pagure_importer
 from pagure_importer.app import app, REPO_PATH
-from pagure_importer.utils import importer_trac, get_pagure_namespace
+from pagure_importer.utils import get_pagure_namespace, issue_to_json
+from pagure_importer.utils.fas import FASclient
 
 import pagure_importer.utils.git as gitutils
+import pagure_importer.utils.importer_trac as libtrac
 
 
 @app.command()
@@ -19,36 +23,60 @@ import pagure_importer.utils.git as gitutils
               help='Number of issue in pagure before import')
 @click.option('--nopush', is_flag=True,
               help="Do not push the result of pagure-importer back")
-def fedorahosted(
-        project_url, tags, private, username, password, offset, nopush):
-    fasclient = FASclient(username, password,
-                          'https://admin.fedoraproject.org/accounts')
+def fedorahosted(project_url, tags, private, username, password,
+                 offset, nopush):
+    """Import issues from fedorahosted"""
+
     if project_url.endswith('.git'):
         project_url = project_url.replace('.git', '')
     project_url += '/login/jsonrpc'
     repos = pagure_importer.utils.display_repo()
+
     if repos:
         repo_index = click.prompt('Choose the import destination repo ',
                                   default=1)
         repo_name = repos[int(repo_index)-1]
-        newpath, new_repo = gitutils.clone_repo(repo_name, REPO_PATH)
+        _, new_repo = gitutils.clone_repo(repo_name, REPO_PATH)
         project = get_pagure_namespace(REPO_PATH, repo_name)
-        with importer_trac.TracImporter(project_url=project_url,
-                                        username=username,
-                                        password=password,
-                                        offset=offset,
-                                        repo_name=repo_name,
-                                        repo_folder=REPO_PATH,
-                                        fasclient=fasclient,
-                                        tags=tags,
-                                        private=private,
-                                        nopush=nopush) as trac_importer:
-
-            trac_importer.import_issues(project)
+        rest_param = namedtuple('rest_param', 'url username password')
+        rest = rest_param(project_url, username, password)
+        fas = FASclient(rest.username, rest.password,
+                        'https://admin.fedoraproject.org/accounts')
+
+        # import issues
+        tickets_id = libtrac.get_project_tickets(rest=rest)
+        ticket_fields = libtrac.get_project_tickets_fields(rest=rest)
+        custom_fields = libtrac.get_custom_fields(ticket_fields=ticket_fields)
+        clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name)
+
+        for ticket_id in tickets_id:
+
+            pagure_issue = libtrac.create_issue(ticket_id=ticket_id,
+                                                custom_fields=custom_fields,
+                                                rest=rest,
+                                                fas=fas,
+                                                offset=offset,
+                                                private=private,
+                                                tags=tags)
+            pagure_issue.comments = []
+            pagure_issue_comments = libtrac.get_ticket_comments(rest=rest,
+                                                                ticket_id=ticket_id)
+            comments = libtrac.create_comments(trac_comments=pagure_issue_comments,
+                                               fas=fas)
+            # add all the comments to the issue object
+            pagure_issue = libtrac.add_comment_to_issue(comments=comments,
+                                                        pagure_issue=pagure_issue,
+                                                        repo_name=project)
+
+            click.echo('Updated ' + repo_name + ' with issue :' +
+                       str(ticket_id) + '/' + str(tickets_id[-1]))
+            issue_to_json(issue=pagure_issue, folder=clone_repo_location)
+
             # update the local git repo
             new_repo = gitutils.update_git(
                 new_repo,
                 commit_message='Imported issues from fedorahosted project: %s' % repo_name)
+
             if not nopush:
                 gitutils.push_repo(new_repo)
     else:

From ce3531f9fb16f50f88160a397766770d46da5bc4 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Feb 12 2017 11:05:54 +0000
Subject: [PATCH 3/4] Remove TracImporter object and use function instead


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 004a7bd..0457883 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -1,17 +1,14 @@
-import os
 import sys
 import re
 import time
-import click
-import requests
 from random import randint
 from base64 import b64decode
 from datetime import datetime
+import click
+import requests
 from pagure_importer.utils import (
-    get_close_status, is_image, issue_to_json, get_secure_filename)
+    get_close_status, is_image, get_secure_filename)
 from pagure_importer.utils.models import User, Issue, IssueComment
-from pagure_importer.utils.fas import FASclient
-from pagure_importer.app import REPO_PATH
 
 SOMEBODY = User(name='somebody', fullname='somebody',
                 emails=['some@body.com'])
@@ -29,7 +26,7 @@ def to_timestamp(tm):
 def pre_process_tags(tags):
     ''' Pre process the tags before sending it to pagure '''
 
-    delims_str = ',|/|;|:|-|\+|\*|\'|\"'
+    delims_str = r',|/|;|:|-|\+|\*|\'|\"'
 
     # Remove the delims between the tags and convert all tags to lower case
     pagure_issue_tags = list(set([j for k in [
@@ -62,36 +59,33 @@ def request(method, rest, *args):
     return resp['result']
 
 
-def import_issues(repo_name, rest, trac_query='max=0&order=id'):
-    ''' Queries the trac instance via its jsonrpc API and convert the
-    tickets into JSON blob to be imported into pagure's ticket git repo.
-
-    :arg repo_name: the name of the repository
-    :arg repo_folder: the folder in which is the repository
-    :kwarg trac_query: the query to call trac with in order to retrieve
-        all the tickets.
-        Defaults to ``max=0&order=id``
+def get_project_tickets(rest, trac_query='max=0&order=id'):
+    """ Queries the trac instance via the jsonrpc API to get a list
+    tickets. By default returns all the project tickets
+     :arg rest: the rest connection parameters (url, username, password)
+     :arg trac_query: query use by trac to return tickets
+    """
+    ticket_id = request('ticket.query', rest, trac_query)
+    return ticket_id
 
-    '''
 
-    tickets_id = request('ticket.query', rest, trac_query)
+def get_project_tickets_fields(rest):
+    """ Queries the trac instance via the jsonrpc API to get a list
+    of all the field used in the trac tickets
+    :arg rest: the rest connection parameters (url, username, password)
+    """
     ticket_fields = request('ticket.getTicketFields', rest)
-    custom_fields = get_custom_fields(ticket_fields)
-    clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name)
+    return ticket_fields
 
-    for ticket_id in tickets_id:
-        pagure_issue = create_issue(ticket_id=ticket_id,
-                                    custom_fields=custom_fields,
-                                    rest=rest)
-        pagure_issue.comments = []
-        pagure_issue_comments = request('ticket.changeLog', rest, ticket_id)
-        comments = create_comments(pagure_issue_comments)
-        # add all the comments to the issue object
-        pagure_issue = add_comment_to_issue(comments, pagure_issue)
 
-        click.echo('Updated ' + repo_name + ' with issue :' +
-                   str(ticket_id) + '/' + str(tickets_id[-1]))
-        issue_to_json(pagure_issue, clone_repo_location)
+def get_ticket_comments(rest, ticket_id):
+    """ Queries the trac instance via the jsonrpc API to get a list
+    of all the ticket's comments
+    :arg rest: the rest connection parameters (url, username, password)
+    :arg ticket_id: id of the ticket to query
+    """
+    issue_comments = request('ticket.changeLog', rest, ticket_id)
+    return issue_comments
 
 
 def get_custom_fields(ticket_fields):
@@ -112,18 +106,51 @@ def get_custom_fields(ticket_fields):
     return custom_fields
 
 
+def get_ticket_info(rest, ticket_id):
+    """ Queries the trac instance via the jsonrpc API to get the
+    details of a ticket.
+    :arg rest: the rest connection parameters (url, username, password)
+    :arg ticket_id: id of the ticket to query
+    """
+    ticket_info = request('ticket.get', rest, ticket_id)
+    return ticket_info
+
+
+def get_ticket_attachments(rest, ticket_id):
+    """ Queries the trac instance via the jsonrpc API to get all
+    of a ticket's attachments.
+    :arg rest: the rest connection parameters (url, username, password)
+    :arg ticket_id: id of the ticket to query
+    """
+    attachments = request('ticket.listAttachments', rest, ticket_id)
+    return attachments
+
+
+def get_attachment(rest, ticket_id, attachment):
+    """ Queries the trac instance via the jsonrpc API to get a
+    ticket's attachment.
+    :arg rest: the rest connection parameters (url, username, password)
+    :arg ticket_id: id of the ticket to query
+    :arg attachment: name of the attachment
+    """
+    attachment = request('ticket.getAttachment',
+                         rest, ticket_id, attachment)
+    return attachment
+
+
 def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags):
     ''' Create Issue object from track ticket '''
 
-    trac_ticket_info = request('ticket.get', rest, ticket_id)
+    trac_ticket_info = get_ticket_info(rest=rest, ticket_id=ticket_id)
     trac_ticket = trac_ticket_info[3]
-    trac_attachments = request('ticket.listAttachments', rest, ticket_id)
+    trac_attachments = get_ticket_attachments(rest=rest, ticket_id=ticket_id)
 
     pagure_attachment = {}
     for attachment in trac_attachments:
         filename = attachment[0]
-        attachment_resp = request('ticket.getAttachment',
-                                  rest, ticket_id, filename)
+        attachment_resp = get_attachment(rest=rest,
+                                         ticket_id=ticket_id,
+                                         attachment=filename)
         if attachment_resp:
             if is_image(filename):
                 content = b64decode(attachment_resp['__jsonclass__'][1])
@@ -146,8 +173,6 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags):
     pagure_issue_created_at = to_timestamp(
         trac_ticket_info[1]['__jsonclass__'][1])
 
-    fas = FASclient(rest.username, rest.password,
-                    'https://admin.fedoraproject.org/accounts')
     pagure_issue_assignee = fas.find_fas_user(trac_ticket['owner'])
     pagure_issue_user = fas.find_fas_user(trac_ticket['reporter'])
     if not pagure_issue_user.name:
@@ -171,10 +196,6 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags):
 
     pagure_issue_depends = []
     pagure_issue_blocks = []
-    if private:
-        pagure_issue_is_private = True
-    else:
-        pagure_issue_is_private = False
 
     pagure_issue = Issue(
         id=ticket_id + offset,
@@ -184,7 +205,7 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags):
         close_status=close_status,
         date_created=pagure_issue_created_at,
         user=pagure_issue_user.to_json(),
-        private=pagure_issue_is_private,
+        private=private,
         attachment=pagure_attachment,
         tags=pagure_issue_tags,
         milestone=pagure_milestone,
@@ -227,7 +248,7 @@ def get_ticket_status(trac_ticket):
         sys.exit(1)
 
 
-def create_comments(trac_comments):
+def create_comments(trac_comments, fas):
     ''' Create IssueComment objects from the trac comments '''
 
     comments = {}
@@ -243,7 +264,7 @@ def create_comments(trac_comments):
             pagure_issue_comment_body = comment[4]
             pagure_issue_comment_created_at = ts
 
-            pagure_issue_comment_user = get_comment_user(comment)
+            pagure_issue_comment_user = get_comment_user(comment, fas)
             # Object to represent comment on an issue
             comments[ts] = IssueComment(
                 id=None,
@@ -256,7 +277,7 @@ def create_comments(trac_comments):
             if ts in comments:
                 comments[ts].attachment.append(comment[4])
             else:
-                pagure_issue_comment_user = self.get_comment_user(comment)
+                pagure_issue_comment_user = get_comment_user(comment, fas)
                 comments[ts] = IssueComment(
                     id=None,
                     comment='attachment',

From be87612049548e9395e98691bbcc81a7c24db619 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Feb 12 2017 11:05:54 +0000
Subject: [PATCH 4/4] Remove TracImporter object from test suite


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/tests/test_pgimport_importer_trac.py b/tests/test_pgimport_importer_trac.py
index fe3886b..a2c0539 100644
--- a/tests/test_pgimport_importer_trac.py
+++ b/tests/test_pgimport_importer_trac.py
@@ -1,28 +1,23 @@
 import unittest
 from unittest.mock import MagicMock
-from pagure_importer.utils.importer_trac import (TracImporter, to_timestamp)
+from pagure_importer.utils.fas import FASclient
+from pagure_importer.utils.models import User
+import pagure_importer.utils.importer_trac as libtrac
 
 
 class PgimportImporterTrac (unittest.TestCase):
 
     def setUp(self):
-        self.trac = TracImporter(project_url="https://foobar.bar",
-                                 username="foo",
-                                 password="bar",
-                                 offset=0,
-                                 repo_folder="myfolder",
-                                 repo_name="myname",
-                                 nopush=False)
+        self.fas = FASclient('foo', 'bar', 'http://example.com')
+        self.fas_mock = MagicMock()
+        self.fas.find_fas_user = self.fas_mock
+        self.fas.find_fas_user.return_value = User(name='',
+                                                   fullname='',
+                                                   emails=[''])
+
         self.somebody = {'name': 'somebody', 'fullname': 'somebody',
                          'emails': ['some@body.com']}
 
-        self.mock_request = MagicMock()
-        self.old_request = TracImporter.request
-        TracImporter.request = self.mock_request
-
-    def tearDown(self):
-        TracImporter.request = self.old_request
-
     def test_import_2_comments(self):
 
         # Case 1 - Import 2 comments
@@ -32,9 +27,9 @@ class PgimportImporterTrac (unittest.TestCase):
             [{u'__jsonclass__': [u'datetime', u'2016-11-08T20:32:20']}, u'foobar',
              u'comment', u'2', u'Well this will improve the code base :).', 1]]
 
-        results = TracImporter.create_comments(self.trac, trac_comments)
+        results = libtrac.create_comments(trac_comments, self.fas)
 
-        ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
+        ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
 
         self.assertEqual("Yeah, let\'s have some test.",
                          results[ts_1].comment)
@@ -42,7 +37,7 @@ class PgimportImporterTrac (unittest.TestCase):
         self.assertEqual([], results[ts_1].attachment)
         self.assertEqual(self.somebody, results[ts_1].user)
 
-        ts_2 = to_timestamp(trac_comments[1][0]['__jsonclass__'][1])
+        ts_2 = libtrac.to_timestamp(trac_comments[1][0]['__jsonclass__'][1])
 
         self.assertEqual("Well this will improve the code base :).",
                          results[ts_2].comment)
@@ -56,7 +51,7 @@ class PgimportImporterTrac (unittest.TestCase):
             [{u'__jsonclass__': [u'datetime', u'2016-11-08T20:30:13']}, u'foobar',
              u'comment', u'1', u'', 1]]
 
-        results = TracImporter.create_comments(self.trac, trac_comments)
+        results = libtrac.create_comments(trac_comments, self.fas)
         self.assertEqual({}, results)
 
     def test_import_not_a_comments(self):
@@ -65,7 +60,7 @@ class PgimportImporterTrac (unittest.TestCase):
             [{u'__jsonclass__': [u'datetime', u'2014-04-09T16:02:20']}, u'foobar',
              u'resolution', u'', u'wontfix', 1]]
 
-        results = TracImporter.create_comments(self.trac, trac_comments)
+        results = libtrac.create_comments(trac_comments, self.fas)
         self.assertEqual({}, results)
 
     def test_import_attachment(self):
@@ -74,8 +69,8 @@ class PgimportImporterTrac (unittest.TestCase):
             [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar',
              u'attachment', u'', u'mytest.png', 0]]
 
-        results = TracImporter.create_comments(self.trac, trac_comments)
-        ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
+        results = libtrac.create_comments(trac_comments, self.fas)
+        ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
 
         self.assertEqual("attachment",
                          results[ts_1].comment)
@@ -91,8 +86,8 @@ class PgimportImporterTrac (unittest.TestCase):
             [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar',
              u'comment', u'', u'test attachment', 0]]
 
-        results = TracImporter.create_comments(self.trac, trac_comments)
-        ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
+        results = libtrac.create_comments(trac_comments, self.fas)
+        ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
 
         self.assertEqual("test attachment",
                          results[ts_1].comment)
@@ -108,8 +103,8 @@ class PgimportImporterTrac (unittest.TestCase):
             [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar',
              u'attachment', u'', u'mytest.png', 0]]
 
-        results = TracImporter.create_comments(self.trac, trac_comments)
-        ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
+        results = libtrac.create_comments(trac_comments, self.fas)
+        ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
 
         self.assertEqual("test attachment",
                          results[ts_1].comment)
@@ -127,8 +122,8 @@ class PgimportImporterTrac (unittest.TestCase):
             [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar',
              u'comment', u'', u'test 2 attachments', 0]]
 
-        results = TracImporter.create_comments(self.trac, trac_comments)
-        ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
+        results = libtrac.create_comments(trac_comments, self.fas)
+        ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1])
 
         self.assertEqual("test 2 attachments",
                          results[ts_1].comment)
@@ -140,60 +135,67 @@ class PgimportImporterTrac (unittest.TestCase):
 
         trac_ticket = {'status': 'open'}
 
-        results = TracImporter.get_ticket_status(self.trac, trac_ticket)
+        results = libtrac.get_ticket_status(trac_ticket)
         self.assertEqual(('Open', ''), results)
 
     def test_get_ticket_status_closedInvalid(self):
 
         trac_ticket = {'status': 'closed', 'resolution': 'invalid'}
 
-        results = TracImporter.get_ticket_status(self.trac, trac_ticket)
+        results = libtrac.get_ticket_status(trac_ticket)
         self.assertEqual(('Closed', 'Invalid'), results)
 
     def test_get_ticket_status_closedWontFix(self):
         trac_ticket = {'status': 'closed', 'resolution': 'wontfix'}
 
-        results = TracImporter.get_ticket_status(self.trac, trac_ticket)
+        results = libtrac.get_ticket_status(trac_ticket)
         self.assertEqual(('Closed', 'Invalid'), results)
 
     def test_get_ticket_status_closedWorksforme(self):
         trac_ticket = {'status': 'closed', 'resolution': 'worksforme'}
 
-        results = TracImporter.get_ticket_status(self.trac, trac_ticket)
+        results = libtrac.get_ticket_status(trac_ticket)
         self.assertEqual(('Closed', 'Invalid'), results)
 
     def test_get_ticket_status_closedDuplicate(self):
         trac_ticket = {'status': 'closed', 'resolution': 'duplicate'}
 
-        results = TracImporter.get_ticket_status(self.trac, trac_ticket)
+        results = libtrac.get_ticket_status(trac_ticket)
         self.assertEqual(('Closed', 'Duplicate'), results)
 
     def test_get_ticket_status_closedInsufficient(self):
         trac_ticket = {'status': 'closed', 'resolution': 'insufficient_info'}
 
-        results = TracImporter.get_ticket_status(self.trac, trac_ticket)
+        results = libtrac.get_ticket_status(trac_ticket)
         self.assertEqual(('Closed', 'Insufficient data'), results)
 
     def test_get_ticket_status_closedFixed(self):
         trac_ticket = {'status': 'closed', 'resolution': 'fixed'}
 
-        results = TracImporter.get_ticket_status(self.trac, trac_ticket)
+        results = libtrac.get_ticket_status(trac_ticket)
         self.assertEqual(('Closed', 'Fixed'), results)
 
     def test_get_custom_fieldsText(self):
-        self.mock_request.return_value = [{'custom': True, 'name': 'foo', 'type': 'textarea'}]
-        results = TracImporter.get_custom_fields(self.trac)
+        ticket_fields = [{'custom': True, 'name': 'foo', 'type': 'textarea'}]
+        results = libtrac.get_custom_fields(ticket_fields)
         self.assertEqual([{'key_type': 'text', 'name': 'foo'}], results)
 
     def test_get_custom_fieldsBoolean(self):
-        self.mock_request.return_value = [{'custom': True, 'name': 'bar', 'type': 'checkbox'}]
-        results = TracImporter.get_custom_fields(self.trac)
+        ticket_fields = [{'custom': True, 'name': 'bar', 'type': 'checkbox'}]
+        results = libtrac.get_custom_fields(ticket_fields)
         self.assertEqual([{'key_type': 'boolean', 'name': 'bar'}], results)
 
     def test_get_tickets_custom_fiedls(self):
-        self.trac.custom_fields = [{'key_type': 'text', 'name': 'bar'},
-                                   {'key_type': 'boolean', 'name': 'foo'}]
+        custom_fields = [{'key_type': 'text', 'name': 'bar'},
+                         {'key_type': 'boolean', 'name': 'foo'}]
         trac_ticket = {'bar': 'hello', 'foo': 'world', 'foobar': 'hello world'}
-        results = TracImporter.get_custom_fields_of_ticket(self.trac, trac_ticket)
-        self.assertDictEqual({'key_type': 'text', 'value': 'hello', 'name': 'bar'}, results[0])
-        self.assertDictEqual({'key_type': 'boolean', 'value': 'world', 'name': 'foo'}, results[1])
+        results = libtrac.get_custom_fields_of_ticket(trac_ticket, custom_fields)
+        self.assertDictEqual({'key_type': 'text', 'value': 'hello', 'name': 'bar'},
+                             results[0])
+        self.assertDictEqual({'key_type': 'boolean', 'value': 'world', 'name': 'foo'},
+                             results[1])
+
+
+if __name__ == '__main__':
+    SUITE = unittest.TestLoader().loadTestsFromTestCase(PgimportImporterTrac)
+    unittest.TextTestRunner(verbosity=2).run(SUITE)

