From aea65c7ec717003e67cf6469b1b39852b1378174 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:16:50 +0000
Subject: [PATCH 1/8] Let Click manage the prompt


---

diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 5aae121..77ed4c6 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -1,5 +1,4 @@
 import click
-import getpass
 import pagure_importer
 from pagure_importer.app import app, REPO_PATH
 from pagure_importer.utils.importer_github import GithubImporter
@@ -10,23 +9,19 @@ from pagure_importer.utils import (
 )
 
 
-def form_github_issues():
-    github_username = raw_input('Enter you Github Username: ')
-    github_password = getpass.getpass('Enter your github password: ')
-    github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ')
-    return (github_username, github_password, github_project_name)
-
-
 @app.command()
-def github():
-    github_username, github_password, github_project_name = form_github_issues()
+@click.option('--username', prompt="Enter your Github Username: ")
+@click.option('--password', prompt=True, hide_input=True)
+@click.option('--project',
+              prompt='Enter github project name like: pypingou/pagure ')
+def github(username, password, project):
     gen_json = raw_input(
         'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ')
     if gen_json == 'n':
         github_importer = GithubImporter(
-            github_username=github_username,
-            github_password=github_password,
-            github_project_name=github_project_name)
+            github_username=username,
+            github_password=password,
+            github_project_name=project)
 
         repos = pagure_importer.utils.display_repo()
         if repos:

From 2bd56f9c6ec0776407bda658dc9da8894df164cd Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:16:50 +0000
Subject: [PATCH 2/8] Fixed contributors generation


---

diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 77ed4c6..117d434 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -13,15 +13,15 @@ from pagure_importer.utils import (
 @click.option('--username', prompt="Enter your Github Username: ")
 @click.option('--password', prompt=True, hide_input=True)
 @click.option('--project',
-              prompt='Enter github project name like: pypingou/pagure ')
+              prompt='Enter github project name like pypingou/pagure: ')
 def github(username, password, project):
     gen_json = raw_input(
-        'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ')
+        "Do you want to generate jsons for project's contributers and issue commentors? (y/n): ")
     if gen_json == 'n':
         github_importer = GithubImporter(
-            github_username=username,
-            github_password=password,
-            github_project_name=project)
+            username=username,
+            password=password,
+            project=project)
 
         repos = pagure_importer.utils.display_repo()
         if repos:
@@ -33,12 +33,12 @@ def github(username, password, project):
 
     else:
         generate_json_for_github_contributors(
-            github_username,
-            github_password,
-            github_project_name)
+            username,
+            password,
+            project)
         generate_json_for_github_issue_commentors(
-            github_username,
-            github_password,
-            github_project_name)
+            username,
+            password,
+            project)
         assemble_github_contributors_commentors()
     return
diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index a0a37c0..a57ce54 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -1,7 +1,7 @@
 from github import Github
+from github.GithubException import TwoFactorException
 
-from pagure_importer.utils import models
-from pagure_importer.utils import github_get_commentor_email
+from pagure_importer.utils import models, github_get_commentor_email
 from pagure_importer.utils.git import (
     clone_repo, push_delete_repo, update_git)
 from pagure_importer.utils.exceptions import (
@@ -12,26 +12,32 @@ from pagure_importer.utils.exceptions import (
 
 class GithubImporter():
     ''' Imports from Github using PyGithub and libpagure '''
-    def __init__(
-            self,
-            github_username,
-            github_password,
-            github_project_name):
-        self.github_username = github_username
-        self.github_password = github_password
-        self.github_project_name = github_project_name
-        self.github = Github(github_username, github_password)
+    def __init__(self, username, password, project):
+        self.github_username = username
+        self.github_password = password
+        self.github_project_name = project
+        self.github = Github(username, password)
 
-    def import_issues(self, repo_path, repo_folder, status='all'):
-        ''' Imports the issues on github for
-        the given project
-        '''
-        github_user = None
+        user = None
         try:
-            github_user = self.github.get_user(self.github_username)
+            user = self.github.get_user()
+            try:
+                otp_auth = user.create_authorization(scopes=['user'],
+                                                     note='otp_auth')
+            except TwoFactorException:
+                otp_key = raw_input("Enter github Two-Factor Auth key: ")
+                otp_auth = user.create_authorization(scopes=['user'],
+                                                     note='otp_auth',
+                                                     onetime_password=otp_key)
+            self.github = Github(otp_auth.token)
         except:
             raise GithubBadCredentials(
                     'Given github credentials are not correct')
+
+    def import_issues(self, repo_path, repo_folder, status='all'):
+        ''' Imports the issues on github for
+        the given project
+        '''
         repo = self.github.get_repo(self.github_project_name)
         try:
             repo_name = repo.name

From 890cd3fead150d6fbabcb1abc53d8cd7f4b27150 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:16:50 +0000
Subject: [PATCH 3/8] Even more click prompting


---

diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 117d434..026cac9 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -10,28 +10,17 @@ from pagure_importer.utils import (
 
 
 @app.command()
-@click.option('--username', prompt="Enter your Github Username: ")
-@click.option('--password', prompt=True, hide_input=True)
+@click.option('--username', prompt="Enter your Github Username: ",
+              help="Github username")
+@click.option('--password', prompt=True, hide_input=True,
+              help="Github password")
 @click.option('--project',
-              prompt='Enter github project name like pypingou/pagure: ')
+              prompt='Enter github project name like pypingou/pagure: ',
+              help="Github project like pypingou/pagure")
 def github(username, password, project):
-    gen_json = raw_input(
-        "Do you want to generate jsons for project's contributers and issue commentors? (y/n): ")
-    if gen_json == 'n':
-        github_importer = GithubImporter(
-            username=username,
-            password=password,
-            project=project)
-
-        repos = pagure_importer.utils.display_repo()
-        if repos:
-            repo_index = raw_input('Choose the import destination repo (default 1) : ') or 1
-            repo_name = repos[int(repo_index)-1]
-            github_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH)
-        else:
-            click.echo('No ticket repository found. Use pgimport clone command')
-
-    else:
+    gen_json = click.confirm(
+        "Do you want to generate jsons for project's contributers and issue commentors?")
+    if gen_json:
         generate_json_for_github_contributors(
             username,
             password,
@@ -41,4 +30,20 @@ def github(username, password, project):
             password,
             project)
         assemble_github_contributors_commentors()
+    else:
+        github_importer = GithubImporter(
+            username=username,
+            password=password,
+            project=project)
+
+        repos = pagure_importer.utils.display_repo()
+        if repos:
+            repo_index = click.prompt(
+                'Choose the import destination repo (default 1) : ', default=1)
+            repo_name = repos[int(repo_index)-1]
+            github_importer.import_issues(
+                repo_path=repo_name, repo_folder=REPO_PATH)
+        else:
+            click.echo(
+                'No ticket repository found. Use pgimport clone command')
     return

From 29c05293b6745b6f01af15413f8f8ae6ae99e8c6 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:16:50 +0000
Subject: [PATCH 4/8] Do not attempt to connect to github

if not repo to populate in temp folder

---

diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index 026cac9..e94917b 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -31,19 +31,22 @@ def github(username, password, project):
             project)
         assemble_github_contributors_commentors()
     else:
-        github_importer = GithubImporter(
-            username=username,
-            password=password,
-            project=project)
-
         repos = pagure_importer.utils.display_repo()
         if repos:
             repo_index = click.prompt(
-                'Choose the import destination repo (default 1) : ', default=1)
+                'Choose the import destination repo', default=1)
             repo_name = repos[int(repo_index)-1]
+
+            github_importer = GithubImporter(
+                username=username,
+                password=password,
+                project=project)
+
             github_importer.import_issues(
                 repo_path=repo_name, repo_folder=REPO_PATH)
         else:
             click.echo(
                 'No ticket repository found. Use pgimport clone command')
+            return
+
     return

From 4f2250324cdc6df3e3ddf44b8fc7a17837d8c6c3 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:16:50 +0000
Subject: [PATCH 5/8] Add two-factor authentication to github importer

It will be use to create a Personal access token and save it in a config file

---

diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index a57ce54..8963b93 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -1,3 +1,6 @@
+import click
+import os
+import ConfigParser
 from github import Github
 from github.GithubException import TwoFactorException
 
@@ -18,21 +21,33 @@ class GithubImporter():
         self.github_project_name = project
         self.github = Github(username, password)
 
-        user = None
+        otp_auth = self.get_auth_token()
+        self.github = Github(otp_auth)
+
+    def get_auth_token(self):
+        cfg_path = os.path.join(os.environ.get('HOME'), '.pgimport')
+        if os.path.exists(cfg_path):
+            parser = ConfigParser.RawConfigParser()
+            parser.read(cfg_path)
+            otp_auth = parser.get('github', 'auth_token')
+        else:
+            otp_auth = self.create_auth_token()
+            otp_auth = otp_auth.token
+            with click.open_file(cfg_path, 'w+') as fp:
+                fp.write('[github] \nauth_token : %s' % otp_auth)
+        return otp_auth
+
+    def create_auth_token(self):
+        user = self.github.get_user()
         try:
-            user = self.github.get_user()
-            try:
-                otp_auth = user.create_authorization(scopes=['user'],
-                                                     note='otp_auth')
-            except TwoFactorException:
-                otp_key = raw_input("Enter github Two-Factor Auth key: ")
-                otp_auth = user.create_authorization(scopes=['user'],
-                                                     note='otp_auth',
-                                                     onetime_password=otp_key)
-            self.github = Github(otp_auth.token)
-        except:
-            raise GithubBadCredentials(
-                    'Given github credentials are not correct')
+            otp_auth = user.create_authorization(scopes=['user'],
+                                                 note='pgimport')
+        except TwoFactorException:
+            otp_key = click.prompt("Enter github Two-Factor Auth key: ")
+            otp_auth = user.create_authorization(scopes=['user'],
+                                                 note='pgimport',
+                                                 onetime_password=otp_key)
+        return otp_auth
 
     def import_issues(self, repo_path, repo_folder, status='all'):
         ''' Imports the issues on github for

From 7d8b2756fd2bd6f21fed29161fa99aaa9008f1e8 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:16:50 +0000
Subject: [PATCH 6/8] Updated github import to match changes to models.py

done for fedorahosted json-rpc protocol

---

diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index 8963b93..05d954c 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -8,7 +8,6 @@ from pagure_importer.utils import models, github_get_commentor_email
 from pagure_importer.utils.git import (
     clone_repo, push_delete_repo, update_git)
 from pagure_importer.utils.exceptions import (
-    GithubBadCredentials,
     GithubRepoNotFound
 )
 
@@ -77,8 +76,7 @@ class GithubImporter():
             else:
                 pagure_issue_status = 'Fixed'
 
-            pagure_issue_created_at = github_issue.created_at
-
+            pagure_issue_created_at = github_issue.created_at.strftime('%s')
             # Not sure how to deal with this atm
             pagure_issue_assignee = None
 
@@ -106,6 +104,7 @@ class GithubImporter():
                     date_created=pagure_issue_created_at,
                     user=pagure_issue_user.to_json(),
                     private=pagure_issue_is_private,
+                    attachment=None,
                     tags=pagure_issue_tags,
                     depends=pagure_issue_depends,
                     blocks=pagure_issue_blocks,
@@ -118,8 +117,8 @@ class GithubImporter():
                 comment_user = comment.user
                 pagure_issue_comment_user_email = comment_user.email
                 pagure_issue_comment_body = comment.body
-                pagure_issue_comment_created_at = comment.created_at
-                pagure_issue_comment_updated_at = comment.updated_at
+                pagure_issue_comment_created_at = comment.created_at.strftime('%s')
+                pagure_issue_comment_updated_at = comment.updated_at.strftime('%s')
 
                 # No idea what to do with this right now
                 # editor: not supported by github api

From 08c5399a8bfa0a1e783a0c6d7b4a86e817d35644 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:16:50 +0000
Subject: [PATCH 7/8] Move 2 FA auth to utils module, so that contributors generator can use it


---

diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py
index e94917b..c986260 100644
--- a/pagure_importer/commands/github.py
+++ b/pagure_importer/commands/github.py
@@ -10,12 +10,12 @@ from pagure_importer.utils import (
 
 
 @app.command()
-@click.option('--username', prompt="Enter your Github Username: ",
+@click.option('--username', prompt='Enter your Github Username',
               help="Github username")
 @click.option('--password', prompt=True, hide_input=True,
               help="Github password")
 @click.option('--project',
-              prompt='Enter github project name like pypingou/pagure: ',
+              prompt='Enter github project name like pypingou/pagure',
               help="Github project like pypingou/pagure")
 def github(username, password, project):
     gen_json = click.confirm(
diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py
index 94750fd..1fd1718 100644
--- a/pagure_importer/utils/__init__.py
+++ b/pagure_importer/utils/__init__.py
@@ -1,39 +1,68 @@
-import git
-import models
-
 import csv
 import os
-import getpass
 import requests
 import json
+import click
+import ConfigParser
 from github import Github
+from github.GithubException import TwoFactorException
+
 from requests.auth import HTTPBasicAuth
 
 from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound
 from pagure_importer.app import REPO_PATH
 
 
+def create_auth_token(github):
+    user = github.get_user()
+    try:
+        otp_auth = user.create_authorization(scopes=['user'],
+                                             note='pgimport')
+    except TwoFactorException:
+        otp_key = click.prompt("Enter github Two-Factor Auth key: ")
+        otp_auth = user.create_authorization(scopes=['user'],
+                                             note='pgimport',
+                                             onetime_password=otp_key)
+    return otp_auth
+
+
+def get_auth_token(github):
+    cfg_path = os.path.join(os.environ.get('HOME'), '.pgimport')
+    if os.path.exists(cfg_path):
+        parser = ConfigParser.RawConfigParser()
+        parser.read(cfg_path)
+        otp_auth = parser.get('github', 'auth_token')
+    else:
+        otp_auth = create_auth_token(github)
+        otp_auth = otp_auth.token
+        with click.open_file(cfg_path, 'w+') as fp:
+            fp.write('[github] \nauth_token : %s' % otp_auth)
+    return otp_auth
+
+
 def display_repo():
     repo = []
     index = 0
-    print '#### Repo available ####'
+    click.secho('#### Repo available ####', fg='blue')
     for file in os.listdir(REPO_PATH):
         if file.endswith('.git'):
             index += 1
-            print str(index) + ' - ' + file
+            click.echo(str(index) + ' - ' + file)
             repo.append(file)
     print
     return repo
 
 
 def generate_json_for_github_contributors(github_username,
-                                                                               github_password,
-                                                                               github_project_name):
+                                          github_password,
+                                          github_project_name):
     ''' Creates a file containing a list of dicts containing the username and
     emails of the contributors in the given github project
     '''
 
     github_obj = Github(github_username, github_password)
+    otp_auth = get_auth_token(github_obj)
+    github_obj = Github(otp_auth)
     project = github_obj.get_repo(github_project_name)
     commits_url = project.commits_url.replace('{/sha}', '')
 
@@ -55,7 +84,7 @@ def generate_json_for_github_contributors(github_username,
                 contributor_fullname = contributor['name']
                 contributor_name = data['committer']['login']
             except TypeError:
-                print 'Maybe one of the contributors is dropped because of lack of details'
+                click.echo('Maybe one of the contributors is dropped because of lack of details')
                 continue
 
             json_data = {
@@ -71,7 +100,7 @@ def generate_json_for_github_contributors(github_username,
                     break
 
             if not present:
-                print 'contributor added: ', contributor_name
+                click.echo('contributor added: ', contributor_name)
                 contributors.append(json_data)
 
     with open('contributors.json', 'w') as f:
@@ -81,13 +110,15 @@ def generate_json_for_github_contributors(github_username,
 
 
 def generate_json_for_github_issue_commentors(github_username,
-                                                                                          github_password,
-                                                                                          github_project_name):
+                                              github_password,
+                                              github_project_name):
     ''' Will create a json file containing details of all the user
     who have commented on any issue in the given project
     '''
 
     github_obj = Github(github_username, github_password)
+    otp_auth = get_auth_token(github_obj)
+    github_obj = Github(otp_auth)
     project = github_obj.get_repo(github_project_name)
     issue_comment_url = project.issue_comment_url.replace('{/number}', '')
 
@@ -106,7 +137,7 @@ def generate_json_for_github_issue_commentors(github_username,
             try:
                 commentor = data['user']['login']
             except TypeError:
-                print 'Maybe one of the issue commentors have been dropped because of lack of details'
+                click.echo('Maybe one of the issue commentors have been dropped because of lack of details')
                 continue
 
             present = False
@@ -116,7 +147,7 @@ def generate_json_for_github_issue_commentors(github_username,
                     break
 
             if not present:
-                print 'commentor added: ', commentor
+                click.echo('commentor added: ', commentor)
                 issue_commentors.append(commentor)
 
     with open('issue_commentors.json', 'w') as f:
diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index 05d954c..80e7758 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -1,10 +1,7 @@
-import click
-import os
-import ConfigParser
 from github import Github
-from github.GithubException import TwoFactorException
 
-from pagure_importer.utils import models, github_get_commentor_email
+from pagure_importer.utils import (
+    models, github_get_commentor_email, get_auth_token)
 from pagure_importer.utils.git import (
     clone_repo, push_delete_repo, update_git)
 from pagure_importer.utils.exceptions import (
@@ -20,34 +17,9 @@ class GithubImporter():
         self.github_project_name = project
         self.github = Github(username, password)
 
-        otp_auth = self.get_auth_token()
+        otp_auth = get_auth_token(self.github)
         self.github = Github(otp_auth)
 
-    def get_auth_token(self):
-        cfg_path = os.path.join(os.environ.get('HOME'), '.pgimport')
-        if os.path.exists(cfg_path):
-            parser = ConfigParser.RawConfigParser()
-            parser.read(cfg_path)
-            otp_auth = parser.get('github', 'auth_token')
-        else:
-            otp_auth = self.create_auth_token()
-            otp_auth = otp_auth.token
-            with click.open_file(cfg_path, 'w+') as fp:
-                fp.write('[github] \nauth_token : %s' % otp_auth)
-        return otp_auth
-
-    def create_auth_token(self):
-        user = self.github.get_user()
-        try:
-            otp_auth = user.create_authorization(scopes=['user'],
-                                                 note='pgimport')
-        except TwoFactorException:
-            otp_key = click.prompt("Enter github Two-Factor Auth key: ")
-            otp_auth = user.create_authorization(scopes=['user'],
-                                                 note='pgimport',
-                                                 onetime_password=otp_key)
-        return otp_auth
-
     def import_issues(self, repo_path, repo_folder, status='all'):
         ''' Imports the issues on github for
         the given project

From 12903423a247ba40061e89bf9472c7f91cce528e Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Sep 26 2016 17:31:11 +0000
Subject: [PATCH 8/8] Fixed review comments


---

diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py
index 1fd1718..585895b 100644
--- a/pagure_importer/utils/__init__.py
+++ b/pagure_importer/utils/__init__.py
@@ -84,7 +84,8 @@ def generate_json_for_github_contributors(github_username,
                 contributor_fullname = contributor['name']
                 contributor_name = data['committer']['login']
             except TypeError:
-                click.echo('Maybe one of the contributors is dropped because of lack of details')
+                click.echo('Maybe one of the contributors is dropped because\
+                            of lack of details')
                 continue
 
             json_data = {
@@ -100,7 +101,7 @@ def generate_json_for_github_contributors(github_username,
                     break
 
             if not present:
-                click.echo('contributor added: ', contributor_name)
+                click.echo('contributor added: ' + contributor_name)
                 contributors.append(json_data)
 
     with open('contributors.json', 'w') as f:
@@ -137,7 +138,8 @@ def generate_json_for_github_issue_commentors(github_username,
             try:
                 commentor = data['user']['login']
             except TypeError:
-                click.echo('Maybe one of the issue commentors have been dropped because of lack of details')
+                click.echo('Maybe one of the issue commentors have been\
+                            dropped because of lack of details')
                 continue
 
             present = False
@@ -147,7 +149,7 @@ def generate_json_for_github_issue_commentors(github_username,
                     break
 
             if not present:
-                click.echo('commentor added: ', commentor)
+                click.echo('commentor added: ' + commentor)
                 issue_commentors.append(commentor)
 
     with open('issue_commentors.json', 'w') as f:
diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py
index 80e7758..d699166 100644
--- a/pagure_importer/utils/importer_github.py
+++ b/pagure_importer/utils/importer_github.py
@@ -98,7 +98,7 @@ class GithubImporter():
                 pagure_issue_comment_editor = None
 
                 # comment updated at
-                pagure_issue_comment_edited_on = comment.updated_at
+                pagure_issue_comment_edited_on = comment.updated_at.strftime('%s')
 
                 # The User who commented
                 pagure_issue_comment_user = models.User(
@@ -115,7 +115,8 @@ class GithubImporter():
                         date_created=pagure_issue_comment_created_at,
                         user=pagure_issue_comment_user.to_json(),
                         edited_on=pagure_issue_comment_edited_on,
-                        editor=pagure_issue_comment_editor)
+                        editor=pagure_issue_comment_editor,
+                        attachment=None)
 
                 comments.append(pagure_issue_comment.to_json())