From c63e119e6de2d46612ebaa5c147d69ad41aecd71 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Aug 21 2016 10:11:18 +0000
Subject: [PATCH 1/7] Replaced xml-rpc login url to use json rpc requests


---

diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index e11d794..f241c2c 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -10,20 +10,17 @@ from pagure_importer.utils.fas import FASclient
 @click.argument('project_url')
 @click.option('--tags', help="Import pagure tags:", is_flag=True)
 def fedorahosted(project_url, tags):
-    fas_username = raw_input('Enter you FAS Username: ')
-    fas_password = getpass.getpass('Enter your FAS password: ')
-    fasclient = FASclient(fas_username, fas_password,
+    username = raw_input('Enter you FAS Username: ')
+    password = getpass.getpass('Enter your FAS password: ')
+    fasclient = FASclient(username, password,
                           'https://admin.fedoraproject.org/accounts')
-
-    rpc_login = fas_username + ':' + fas_password + '@'
-    url_index = project_url.find('://')
-    rpc_url = project_url[:url_index+3] + rpc_login +\
-        project_url[url_index+3:] + '/login/xmlrpc'
+    project_url = project_url + '/login/jsonrpc'
     repos = pagure_importer.utils.display_repo()
     if repos:
         repo_index = raw_input('Choose the import destination repo (default 1) : ') or 1
         repo_name = repos[int(repo_index)-1]
-        trac_importer = importer_trac.TracImporter(rpc_url, fasclient, tags)
+        trac_importer = importer_trac.TracImporter(project_url, username,
+                                                   password, fasclient, tags)
         trac_importer.import_issues(repo_name=repo_name, repo_folder=REPO_PATH)
     else:
         click.echo('No ticket repository found. Use pgimport clone command')

From a174c25bec1ed4d6eb0bab74cc9ce3130be09dae Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Aug 21 2016 10:41:41 +0000
Subject: [PATCH 2/7] Started using http requests for json rpc instead of xml rpc


---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index a298bb3..d2c3cf3 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -1,4 +1,4 @@
-from xmlrpclib import ServerProxy
+import requests
 from datetime import datetime
 from pagure_importer.utils.git import update_git, get_secure_filename
 from pagure_importer.utils.models import User, Issue, IssueComment
@@ -7,16 +7,35 @@ from pagure_importer.utils.models import User, Issue, IssueComment
 class TracImporter():
     '''Pagure importer for trac instance'''
 
-    def __init__(self, trac_project_url, fasclient=None, tags=False):
-        self.trac = ServerProxy(trac_project_url)
+    def __init__(self, project_url, username, password, fasclient=None, tags=False):
+        self.url = project_url
+        self.username = username
+        self.password = password
         self.fas = fasclient
         self.tags = tags
         self.somebody = User(name='somebody', fullname='somebody', emails=['some@body.com'])
+        self.reqid = 0
+
+    def request(self, method, *args):
+        self.reqid += 1
+        req = {'params': args,
+               'method': method,
+               'id': self.reqid}
+        resp = requests.post(self.url, json=req, auth=(self.username, self.password))
+        resp = resp.json()
+        if resp['id'] != self.reqid:
+            print('ERROR: Invalid response for request! ID does not match')
+            sys.exit(1)
+        if resp['error'] != None:
+            print("ERROR: Error in response: %s" % resp['error'])
+            sys.exit(1)
+
+        return resp['result']
 
     def import_issues(self, repo_name, repo_folder,
                       trac_query='max=0&order=id'):
         '''Import issues from trac instance using xmlrpc API'''
-        tickets_id = self.trac.ticket.query(trac_query)
+        tickets_id = self.request('ticket.query', trac_query)
 
         for ticket_id in tickets_id:
             pagure_issue = self.create_issue(ticket_id)
@@ -39,13 +58,13 @@ class TracImporter():
 
     def create_issue(self, ticket_id):
 
-        trac_ticket = self.trac.ticket.get(ticket_id)[3]
-        trac_attachments = self.trac.ticket.listAttachments(ticket_id)
+        trac_ticket = self.request('ticket.get', ticket_id)[3]
+        trac_attachments = self.request('ticket.listAttachments', ticket_id)
 
         pagure_attachment = {}
         for attachment in trac_attachments:
             filename = attachment[0]
-            content = self.trac.ticket.getAttachment(ticket_id, filename)
+            content = self.request('ticket.getAttachment', ticket_id, filename)
             pagure_attachment[filename] = content
 
         pagure_issue_title = trac_ticket['summary']

From fab2e17e7de3bfd6eb91af8bfaae21f872df4c12 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Aug 21 2016 14:45:00 +0000
Subject: [PATCH 3/7] Trac importer using jsonrpc


---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index d2c3cf3..439ba35 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -1,4 +1,5 @@
 import requests
+import time
 from datetime import datetime
 from pagure_importer.utils.git import update_git, get_secure_filename
 from pagure_importer.utils.models import User, Issue, IssueComment
@@ -32,6 +33,12 @@ class TracImporter():
 
         return resp['result']
 
+    def to_timestamp(self, tm):
+        tm = tm.replace('+00:00', '')
+        date = datetime.strptime(tm, '%Y-%m-%dT%H:%M:%S')
+        ts = str(time.mktime(date.timetuple()))[:-2]  # Strip the .0
+        return ts
+
     def import_issues(self, repo_name, repo_folder,
                       trac_query='max=0&order=id'):
         '''Import issues from trac instance using xmlrpc API'''
@@ -40,7 +47,7 @@ class TracImporter():
         for ticket_id in tickets_id:
             pagure_issue = self.create_issue(ticket_id)
             pagure_issue.comments = []
-            pagure_issue_comments = self.trac.ticket.changeLog(ticket_id)
+            pagure_issue_comments = self.request('ticket.changeLog', ticket_id)
             comments = self.create_comments(pagure_issue_comments)
             # add all the comments to the issue object
             for key in comments:
@@ -75,8 +82,7 @@ class TracImporter():
 
         pagure_issue_status = self.get_ticket_status(trac_ticket)
 
-        pagure_issue_created_at = datetime.strptime(
-            self.trac.ticket.get(ticket_id)[1].value, "%Y%m%dT%H:%M:%S")
+        pagure_issue_created_at = self.to_timestamp(self.request('ticket.get', ticket_id)[1]['__jsonclass__'][1])
 
         if self.fas:
             pagure_issue_assignee = self.fas.find_fas_user(trac_ticket['owner'])
@@ -132,7 +138,7 @@ class TracImporter():
     def create_comments(self, trac_comments):
         comments = {}
         for comment in trac_comments:
-            ts = datetime.strptime(comment[0].value, "%Y%m%dT%H:%M:%S")
+            ts = self.to_timestamp(comment[0]['__jsonclass__'][1])
 
             if comment[2] == 'comment' and comment[4] != '':
                 if ts in comments:
diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py
index 3c129d6..1e18710 100644
--- a/pagure_importer/utils/models.py
+++ b/pagure_importer/utils/models.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-import datetime
 import json
 import uuid
 
@@ -36,7 +35,7 @@ class Issue():
             'title': self.title,
             'content': self.content,
             'status': self.status,
-            'date_created': self.date_created.strftime('%s'),
+            'date_created': self.date_created,
             'user': self.user,
             'private': self.private,
             'tags': self.tags,
@@ -79,9 +78,9 @@ class IssueComment():
             'id': self.id,
             'comment': self.comment,
             'parent': self.parent,
-            'date_created': self.date_created.strftime('%s'),
+            'date_created': self.date_created,
             'user': self.user,
-            'edited_on': self.edited_on.strftime('%s') if self.edited_on else None,
+            'edited_on': self.edited_on if self.edited_on else None,
             'editor': self.editor or None
         }
 

From ac3cd227ef495b0da79808f526e29d8ed46bd18d Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Aug 23 2016 20:19:36 +0000
Subject: [PATCH 4/7] Fix attachment and reduce number of request to optimise runtime


---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 439ba35..e6055e2 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -1,5 +1,6 @@
 import requests
 import time
+import base64
 from datetime import datetime
 from pagure_importer.utils.git import update_git, get_secure_filename
 from pagure_importer.utils.models import User, Issue, IssueComment
@@ -65,14 +66,15 @@ class TracImporter():
 
     def create_issue(self, ticket_id):
 
-        trac_ticket = self.request('ticket.get', ticket_id)[3]
+        trac_ticket_info =  self.request('ticket.get', ticket_id)
+        trac_ticket = trac_ticket_info[3]        
         trac_attachments = self.request('ticket.listAttachments', ticket_id)
 
         pagure_attachment = {}
         for attachment in trac_attachments:
             filename = attachment[0]
-            content = self.request('ticket.getAttachment', ticket_id, filename)
-            pagure_attachment[filename] = content
+            content = self.request('ticket.getAttachment', ticket_id, filename)['__jsonclass__'][1].replace('\n', '')
+            pagure_attachment[filename] = base64.b64decode(content)
 
         pagure_issue_title = trac_ticket['summary']
 
@@ -82,7 +84,7 @@ class TracImporter():
 
         pagure_issue_status = self.get_ticket_status(trac_ticket)
 
-        pagure_issue_created_at = self.to_timestamp(self.request('ticket.get', ticket_id)[1]['__jsonclass__'][1])
+        pagure_issue_created_at = self.to_timestamp(trac_ticket_info[1]['__jsonclass__'][1])
 
         if self.fas:
             pagure_issue_assignee = self.fas.find_fas_user(trac_ticket['owner'])

From 14ce7742c7b6002536659deea11b622a457aae40 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Aug 29 2016 06:52:20 +0000
Subject: [PATCH 5/7] Remove install from git as this is not working


---

diff --git a/README.md b/README.md
index 0ee4dd6..a04cda4 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,10 @@
 # pagure-importer
 CLI tool for importing issues etc. from different sources like github to pagure
 
-## Installation
+## Installation in a Virtual Environment
 ---
 *  Install it using ```pip``` . ```pip install pagure_importer```
 
-
 ## How to run
 ---
 0. Clone the issue tracker for issues from pagure. Use: ```pgimport clone  ssh://git@pagure.io/tickets/foobar.git```
@@ -15,18 +14,6 @@ CLI tool for importing issues etc. from different sources like github to pagure
 4. The script will make commits in your cloned repo: push the changes back to pagure. Use : ```pgimport push foobar.git```
 
 
-## Run from git
----
-
-* Install the dependencies:
-
-    sudo dnf install python-github python-pygit2 python-fedora python-click python-werkzeug
-
-* Run the application
-
-    PYTHONPATH=. python pagure_importer/app.py
-
-
 ## Usage
 ---
 

From 0273adabd2657b4b1b8add0f3345d6897e29874e Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Aug 29 2016 06:52:45 +0000
Subject: [PATCH 6/7] Add requests library to requirements.txt


---

diff --git a/requirements.txt b/requirements.txt
index 780699f..9723285 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ click
 python-fedora
 pygit2 >= 0.20.1
 werkzeug
+requests

From e7a2c65e9027de4b8817d2f6d1267e5968bdd515 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Aug 29 2016 09:59:57 +0000
Subject: [PATCH 7/7] Protect call to fas api is user is None


---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index e6055e2..9c9fd7e 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -129,7 +129,7 @@ class TracImporter():
 
     def get_comment_user(self, comment):
         # The User who commented
-        if self.fas:
+        if self.fas and comment[1]:
             pagure_issue_comment_user = self.fas.find_fas_user(comment[1])
             if not pagure_issue_comment_user.name:
                 pagure_issue_comment_user = self.somebody

