From 9cf4e4a39b8cbcaf44831fa88cb9e7c1233f139d Mon Sep 17 00:00:00 2001
From: Mark Reynolds <mreynolds@redhat.com>
Date: Feb 24 2017 19:48:39 +0000
Subject: [PATCH 1/3] Convert wiki/RST format to Markdown for Trac Ticket Migration


---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index ffaee9d..47bcf2b 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -11,6 +11,19 @@ from pagure_importer.utils import (
     get_close_status, is_image, issue_to_json, get_secure_filename)
 from pagure_importer.utils.models import User, Issue, IssueComment
 
+wikilink_pattern = re.compile('\[http(.*)\]')
+wikilink_extract = re.compile('\[(.*)\]')
+wikiheading1_pattern = re.compile('^= (.*) =$')
+wikiheading2_pattern = re.compile('^== (.*) ==$')
+wikiheading3_pattern = re.compile('^=== (.*) ===$')
+strikethrough_pattern = re.compile('~~(.*)~~')
+# Trac priorities
+priority_map = {'blocker': 1,
+                'critical': 2,
+                'major': 3,
+                'minor': 4,
+                'trivial': 5}
+
 
 def to_timestamp(tm):
     ''' Convert to timestamp which can be jsonified '''
@@ -21,6 +34,28 @@ def to_timestamp(tm):
     return ts
 
 
+def strip_wikilink(content):
+    ''' Need to remove wiki link format from custom fields.  They come in a
+    variety of forms that can be comma or whitespace separated.  They can also
+    include link names which must also be removed.
+
+        [https://bugzilla.redhat.com/show_bug.cgi?id=772777]
+        [https://bugzilla.com/123456789], [http://bugzilla.com/7777777 7777777]
+        [https://bugzilla.com/6666666 6666666]
+    '''
+
+    links = []
+    if wikilink_pattern.search(content):
+        # Looks like we have a link in here
+        links = []
+        mylist = re.findall(r'\[([^]]*)\]', content)
+        for i in mylist:
+            links.append(i.split(' ', 1)[0])
+        return ', '.join(links)
+    else:
+        return content
+
+
 class TracImporter(object):
     ''' Pagure importer for trac instance '''
 
@@ -84,7 +119,12 @@ class TracImporter(object):
         all_ticket_fields = self.request('ticket.getTicketFields')
         custom_fields = []
         for field in all_ticket_fields:
-            if field.get('custom') is True:
+            if (field.get('custom') is True or
+                field.get('label').lower() == "component" or
+                field.get('label').lower() == "keywords" or
+                field.get('label').lower() == "version" or
+                field.get('label').lower() == "type" or
+                field.get('label').lower() == "cc"):
                 current_field = {}
                 current_field['name'] = field['name']
                 key_type = 'text'
@@ -92,8 +132,133 @@ class TracImporter(object):
                     key_type = 'boolean'
                 current_field['key_type'] = key_type
                 custom_fields.append(current_field)
+
         return custom_fields
 
+    def WikiToMD(self, content):
+        ''' Convert wiki/RST format to Markdown.  Code blocks, bold/italics,
+        wiki links, lists, striked text, and headers. '''
+
+        code_block = False
+        in_list = False
+        nested_level = 0
+        prev_indent = 0
+        new_content = ""
+
+        for line in content.split('\n'):
+            line = line.replace("\r", "")
+            if line == "{{{":
+                code_block = True
+                new_content = "%s\n" % (new_content)
+            elif line == "}}}":
+                code_block = False
+                new_content = "%s\n" % (new_content)
+            else:
+                if code_block:
+                    # Prefix the Markdown 4 space code block gap.  Do not
+                    # process any other wiki formats until we are out of the
+                    # code block.
+                    line = "    %s" % (line)
+                else:
+                    #
+                    # Convert bullet lists.  The start and end of a list needs
+                    # an empty line.  wikiformat uses both '*' and '-' for its
+                    # lists.  However, markdown only supports '-'.
+                    #
+                    if line.startswith('* '):
+                        if not in_list:
+                            new_content = "%s\n" % (new_content)
+                        in_list = True
+                        line = line[1:]
+                        line = '-%s' % (line)
+                    elif line.startswith('- '):
+                        # No need to modify the line, just add the new line
+                        if not in_list:
+                            new_content = "%s\n" % (new_content)
+                        in_list = True
+                    elif line.startswith(' '):
+                        # Check for nested lists
+                        nested_line = line.lstrip(' ')
+                        if nested_line.startswith('* ') or \
+                           nested_line.startswith('- '):
+                            # Adjust the nested list level as needed
+                            indent = len(line) - len(nested_line)
+                            if indent > prev_indent:
+                                nested_level += 1
+                            elif indent < prev_indent:
+                                nested_level -= 1
+                            prev_indent = indent
+
+                            # Set the proper indentation for markdown
+                            line = ('%s-%s' % ('    ' * nested_level,
+                                               nested_line[1:]))
+                    else:
+                        if in_list:
+                            # Add the closing empty line
+                            new_content = "%s\n" % (new_content)
+                        in_list = False
+                        nested_level = 0
+                        prev_indent = 0
+
+                    #
+                    # Convert headers
+                    #
+                    level_count = 1
+                    for header in [wikiheading1_pattern,
+                                   wikiheading2_pattern,
+                                   wikiheading3_pattern]:
+                        try:
+                            level = header.search(line).group(1)
+                            if level:
+                                line = "%s %s" % ('#' * level_count, level)
+                                break  # No need to check other heading levels
+                        except:
+                            # Try the next heading level
+                            pass
+                        level_count += 1
+
+                    #
+                    # Convert wiki links
+                    #
+                    if wikilink_pattern.search(line):
+                        try:
+                            result = wikilink_extract.search(line).group(1)
+                            if result:
+                                parts = result.split(' ', 1)
+                                if len(parts) == 1:
+                                    mdlink = '[%s](%s)' % (parts[0], parts[0])
+                                elif len(parts) == 2:
+                                    mdlink = '[%s](%s)' % (parts[1], parts[0])
+                                line = line.replace('[' + result + ']', mdlink)
+                        except:
+                            # Not a link, not a problem
+                            pass
+
+                    #
+                    # Convert striked through text
+                    #
+                    striked_result = strikethrough_pattern.search(line)
+                    if striked_result:
+                        try:
+                            striked_text = striked_result.group(1)
+                            if striked_text:
+                                orig_text = '~~%s~~' % striked_text
+                                new_text = '<s>%s</s>' % striked_text
+                                line = line.replace(orig_text, new_text)
+                        except:
+                            # Not striked
+                            pass
+
+                    #
+                    # Convert bold and italic text (do this last)
+                    #
+                    line = line.replace("'''", "**")  # Convert bold text
+                    line = line.replace("''", "*")  # Convert italic text
+
+                new_content = "%s%s\n" % (new_content, line)
+
+        return new_content
+
     def import_issues(self, repo_name, trac_query='max=0&order=id'):
         ''' Queries the trac instance via its jsonrpc API and convert the
         tickets into JSON blob to be imported into pagure's ticket git repo.
@@ -103,7 +268,6 @@ class TracImporter(object):
         :kwarg trac_query: the query to call trac with in order to retrieve
             all the tickets.
             Defaults to ``max=0&order=id``
-
         '''
 
         tickets_id = self.request('ticket.query', trac_query)
@@ -130,9 +294,13 @@ class TracImporter(object):
                         else:
                             comments[key].comment += ('\n[%s](%s)' %
                                                       (attach_name, url))
+                else:
+                    # Convert any RST formatting to Markdown
+                    comments[key].comment = self.WikiToMD(comments[key].comment)
                 pagure_issue.comments.append(comments[key].to_json())
             click.echo('Updated ' + repo_name + ' with issue :' +
                        str(ticket_id) + '/' + str(tickets_id[-1]))
+
             issue_to_json(pagure_issue, self.clone_repo_location)
 
     def get_custom_fields_of_ticket(self, trac_ticket):
@@ -149,6 +317,8 @@ class TracImporter(object):
                 pagure_field['value'] = trac_ticket.get(
                                     pagure_field['name'], "").strip()
                 if pagure_field['value']:
+                    pagure_field['value'] = \
+                        strip_wikilink(pagure_field['value'])
                     pagure_fields.append(pagure_field)
         return pagure_fields
 
@@ -180,6 +350,8 @@ class TracImporter(object):
         pagure_issue_content = trac_ticket['description']
         if pagure_issue_content == '':
             pagure_issue_content = '#No Description Provided'
+        else:
+            pagure_issue_content = self.WikiToMD(pagure_issue_content)
 
         issue_status, close_status = self.get_ticket_status(trac_ticket)
 
@@ -207,6 +379,11 @@ class TracImporter(object):
         if 'milestone' in trac_ticket and trac_ticket['milestone'] != '':
             pagure_milestone = trac_ticket['milestone']
 
+        # The priority of the issue
+        priority = None
+        if 'priority' in trac_ticket and trac_ticket['priority'] != '':
+            priority = priority_map[trac_ticket['priority']]
+
         # Issue tags
         pagure_issue_tags = []
         if self.tags:
@@ -227,6 +404,7 @@ class TracImporter(object):
             title=pagure_issue_title,
             content=pagure_issue_content,
             status=issue_status,
+            priority=priority,
             close_status=close_status,
             date_created=pagure_issue_created_at,
             user=pagure_issue_user.to_json(),
diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py
index 7a17ea6..7ce0fb2 100644
--- a/pagure_importer/utils/models.py
+++ b/pagure_importer/utils/models.py
@@ -9,12 +9,13 @@ class Issue():
             self, id, title, content,
             status, date_created, user, private, attachment, tags,
             depends, blocks, assignee, close_status, comments=None,
-            milestone=None, custom_fields=None):
+            milestone=None, custom_fields=None, priority=None):
 
         self.id = id
         self.title = title
         self.content = content
         self.status = status
+        self.priority = priority
         self.close_status = close_status
         self.date_created = date_created
         self.user = user
@@ -42,6 +43,7 @@ class Issue():
             'date_created': self.date_created,
             'user': self.user,
             'private': self.private,
+            'priority': self.priority,
             'tags': self.tags,
             'depends': self.depends,
             'blocks': self.blocks,

From 7fc0acf8245f6abfab4ce2da5ed7be50791717c9 Mon Sep 17 00:00:00 2001
From: Mark Reynolds <mreynolds@redhat.com>
Date: Feb 24 2017 19:48:39 +0000
Subject: [PATCH 2/3] Fix pep8 errors


---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 47bcf2b..505d523 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -60,7 +60,8 @@ class TracImporter(object):
     ''' Pagure importer for trac instance '''
 
     def __init__(self, project_url, username, password, offset, repo_name,
-                 repo_folder, nopush, fasclient=None, tags=False, private=False):
+                 repo_folder, nopush, fasclient=None, tags=False,
+                 private=False):
         ''' Instantiate a TracImporter object '''
         self.username = username
         self.password = password
@@ -120,11 +121,11 @@ class TracImporter(object):
         custom_fields = []
         for field in all_ticket_fields:
             if (field.get('custom') is True or
-                field.get('label').lower() == "component" or
-                field.get('label').lower() == "keywords" or
-                field.get('label').lower() == "version" or
-                field.get('label').lower() == "type" or
-                field.get('label').lower() == "cc"):
+                    field.get('label').lower() == "component" or
+                    field.get('label').lower() == "keywords" or
+                    field.get('label').lower() == "version" or
+                    field.get('label').lower() == "type" or
+                    field.get('label').lower() == "cc"):
                 current_field = {}
                 current_field['name'] = field['name']
                 key_type = 'text'
@@ -296,7 +297,8 @@ class TracImporter(object):
                                                       (attach_name, url))
                 else:
                     # Convert any RST formatting to Markdown
-                    comments[key].comment = self.WikiToMD(comments[key].comment)
+                    comments[key].comment = \
+                        self.WikiToMD(comments[key].comment)
                 pagure_issue.comments.append(comments[key].to_json())
             click.echo('Updated ' + repo_name + ' with issue :' +
                        str(ticket_id) + '/' + str(tickets_id[-1]))
@@ -381,7 +383,8 @@ class TracImporter(object):
 
         # The priority of the issue
         priority = None
-        if 'priority' in trac_ticket and trac_ticket['priority'] != '':
+        if 'priority' in trac_ticket and trac_ticket['priority'] != '' and \
+           trac_ticket['priority'] in priority_map:
             priority = priority_map[trac_ticket['priority']]
 
         # Issue tags

From a41618e625307eec0280ce09e9740013422a2230 Mon Sep 17 00:00:00 2001
From: Mark Reynolds <mreynolds@redhat.com>
Date: Feb 24 2017 19:48:39 +0000
Subject: [PATCH 3/3] Break main conversion function into small functions.  Also improved code block conversion.


---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index 505d523..46dd70b 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -56,6 +56,63 @@ def strip_wikilink(content):
         return content
 
 
+def convert_headers(line):
+    ''' Convert wikiformat headers
+    '''
+    level_count = 1
+    for header in [wikiheading1_pattern,
+                   wikiheading2_pattern,
+                   wikiheading3_pattern]:
+        try:
+            level = header.search(line).group(1)
+            if level:
+                line = "%s %s" % ('#' * level_count, level)
+                break  # No need to check other heading levels
+        except:
+            # Try the next heading level
+            pass
+        level_count += 1
+
+    return line
+
+
+def convert_wikilinks(line):
+    ''' Convert wikiformat links
+    '''
+    if wikilink_pattern.search(line):
+        try:
+            result = wikilink_extract.search(line).group(1)
+            if result:
+                parts = result.split(' ', 1)
+                if len(parts) == 1:
+                    mdlink = '[%s](%s)' % (parts[0], parts[0])
+                elif len(parts) == 2:
+                    mdlink = '[%s](%s)' % (parts[1], parts[0])
+                line = line.replace('[' + result + ']', mdlink)
+        except:
+            # Not a link, not a problem
+            pass
+
+    return line
+
+
+def convert_strike(line):
+    ''' Convert wikiformat striked text
+    '''
+    striked_result = strikethrough_pattern.search(line)
+    if striked_result:
+        try:
+            striked_text = striked_result.group(1)
+            if striked_text:
+                orig_text = '~~%s~~' % striked_text
+                new_text = '<s>%s</s>' % striked_text
+                line = line.replace(orig_text, new_text)
+        except:
+            # Not striked
+            pass
+    return line
+
+
 class TracImporter(object):
     ''' Pagure importer for trac instance '''
 
@@ -148,115 +205,67 @@ class TracImporter(object):
 
         for line in content.split('\n'):
             line = line.replace("\r", "")
-            if line == "{{{":
+            if "{{{" in line:
                 code_block = True
-                new_content = "%s\n" % (new_content)
-            elif line == "}}}":
+                line = line.replace("{{{", "```")
+            if "}}}" in line:
                 code_block = False
-                new_content = "%s\n" % (new_content)
-            else:
-                if code_block:
-                    # Prefix the Markdown 4 space code block gap.  Do not
-                    # process any other wiki formats until we are out of the
-                    # code block.
-                    line = "    %s" % (line)
+                line = line.replace("}}}", "```")
+            if not code_block:
+                #
+                # Convert bullet lists.  The start and end of a list needs
+                # an empty line.  wikiformat uses both '*' and '-' for its
+                # lists.  However, markdown only supports '-'.
+                #
+                if line.startswith('* '):
+                    if not in_list:
+                        new_content = "%s\n" % (new_content)
+                    in_list = True
+                    line = line[1:]
+                    line = '-%s' % (line)
+                elif line.startswith('- '):
+                    # No need to modify the line, just add the new line
+                    if not in_list:
+                        new_content = "%s\n" % (new_content)
+                    in_list = True
+                elif line.startswith(' '):
+                    # Check for nested lists
+                    nested_line = line.lstrip(' ')
+                    if nested_line.startswith('* ') or \
+                       nested_line.startswith('- '):
+                        # Adjust the nested list level as needed
+                        indent = len(line) - len(nested_line)
+                        if indent > prev_indent:
+                            nested_level += 1
+                        elif indent < prev_indent:
+                            nested_level -= 1
+                        prev_indent = indent
+
+                        # Set the proper indentation for markdown
+                        line = ('%s-%s' % ('    ' * nested_level,
+                                           nested_line[1:]))
                 else:
-                    #
-                    # Convert bullet lists.  The start and end of a list needs
-                    # an empty line.  wikiformat uses both '*' and '-' for its
-                    # lists.  However, markdown only supports '-'.
-                    #
-                    if line.startswith('* '):
-                        if not in_list:
-                            new_content = "%s\n" % (new_content)
-                        in_list = True
-                        line = line[1:]
-                        line = '-%s' % (line)
-                    elif line.startswith('- '):
-                        # No need to modify the line, just add the new line
-                        if not in_list:
-                            new_content = "%s\n" % (new_content)
-                        in_list = True
-                    elif line.startswith(' '):
-                        # Check for nested lists
-                        nested_line = line.lstrip(' ')
-                        if nested_line.startswith('* ') or \
-                           nested_line.startswith('- '):
-                            # Adjust the nested list level as needed
-                            indent = len(line) - len(nested_line)
-                            if indent > prev_indent:
-                                nested_level += 1
-                            elif indent < prev_indent:
-                                nested_level -= 1
-                            prev_indent = indent
-
-                            # Set the proper indentation for markdown
-                            line = ('%s-%s' % ('    ' * nested_level,
-                                               nested_line[1:]))
-                    else:
-                        if in_list:
-                            # Add the closing empty line
-                            new_content = "%s\n" % (new_content)
-                        in_list = False
-                        nested_level = 0
-                        prev_indent = 0
-
-                    #
-                    # Convert headers
-                    #
-                    level_count = 1
-                    for header in [wikiheading1_pattern,
-                                   wikiheading2_pattern,
-                                   wikiheading3_pattern]:
-                        try:
-                            level = header.search(line).group(1)
-                            if level:
-                                line = "%s %s" % ('#' * level_count, level)
-                                break  # No need to check other heading levels
-                        except:
-                            # Try the next heading level
-                            pass
-                        level_count += 1
-
-                    #
-                    # Convert wiki links
-                    #
-                    if wikilink_pattern.search(line):
-                        try:
-                            result = wikilink_extract.search(line).group(1)
-                            if result:
-                                parts = result.split(' ', 1)
-                                if len(parts) == 1:
-                                    mdlink = '[%s](%s)' % (parts[0], parts[0])
-                                elif len(parts) == 2:
-                                    mdlink = '[%s](%s)' % (parts[1], parts[0])
-                                line = line.replace('[' + result + ']', mdlink)
-                        except:
-                            # Not a link, not a problem
-                            pass
-
-                    #
-                    # Convert striked through text
-                    #
-                    striked_result = strikethrough_pattern.search(line)
-                    if striked_result:
-                        try:
-                            striked_text = striked_result.group(1)
-                            if striked_text:
-                                orig_text = '~~%s~~' % striked_text
-                                new_text = '<s>%s</s>' % striked_text
-                                line = line.replace(orig_text, new_text)
-                        except:
-                            # Not striked
-                            pass
-
-                    #
-                    # Convert bold and italic text (do this last)
-                    #
-                    line = line.replace("'''", "**")  # Convert bold text
-                    line = line.replace("''", "*")  # Convert italic text
-
-                new_content = "%s%s\n" % (new_content, line)
+                    if in_list:
+                        # Add the closing empty line
+                        new_content = "%s\n" % (new_content)
+                    in_list = False
+                    nested_level = 0
+                    prev_indent = 0
+
+                # Convert headers
+                line = convert_headers(line)
+
+                # Convert wiki links
+                line = convert_wikilinks(line)
+
+                # Convert striked through text
+                line = convert_strike(line)
+
+                # Convert bold and italic text (do this last)
+                line = line.replace("'''", "**")  # Convert bold text
+                line = line.replace("''", "*")  # Convert italic text
+
+            new_content = "%s%s\n" % (new_content, line)
 
         return new_content
 

