From c6f38e1cb86c3abf3692e06b990efc9bdf20dcbc Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Feb 23 2016 08:22:42 +0000
Subject: [PATCH 1/11] Model to represent an issue as json


---

diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py
new file mode 100644
index 0000000..7ec0231
--- /dev/null
+++ b/pagure_importer/lib/models.py
@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+
+import datetime
+import json
+import uuid
+
+class Issue():
+    ''' Represents an Issue '''
+
+    def __init__(
+            self, id, title, content,
+            status, date_created, user, private, tags,
+            depends, blocks, assignee, comments=None):
+
+        self.id = id
+        self.title = title
+        self.content = content
+        self.status = status
+        self.date_created = date_created
+        self.user = user
+        self.private = private
+        self.tags = tags
+        self.depends = depends
+        self.blocks = blocks
+        self.assignee = assignee
+        self.comments = comments
+        self.uid = uuid.uuid4().hex
+
+    def to_json(self):
+        ''' Returns a dictionary representation of the issue.
+
+        '''
+        output = {
+            'id': self.id,
+            'title': self.title,
+            'content': self.content,
+            'status': self.status,
+            'date_created': self.date_created.strftime('%s'),
+            'user': self.user,
+            'private': self.private,
+            'tags': self.tags,
+            'depends': self.depends,
+            'blocks': self.blocks,
+            'assignee': self.assignee,
+            'comments': self.comments
+        }
+
+        return output
+
+    @property
+    def isa(self):
+        return 'issue'
+
+
+class IssueComment():
+    ''' Represent a comment for an issue '''
+
+    def __init__(
+            self, id, comment, date_created,
+            user, parent=None, edited_on=None, editor=None):
+
+        self.id = id
+        self.comment = comment
+        self.parent = parent
+        self.date_created = date_created
+        self.user = user
+        self.edited_on = edited_on
+        self.editor = editor
+
+    def to_json(self):
+        ''' Returns a dictionary representation of the issue. '''
+
+        output = {
+            'id': self.id,
+            'comment': self.comment,
+            'parent': self.parent,
+            'date_created': self.date_created.strftime('%s'),
+            'user': self.user,
+            'edited_on': self.edited_on.strftime('%s') if self.edited_on else None,
+            'editor': self.editor or None
+        }
+        return output
+
+
+class User():
+    ''' Represents a User '''
+
+    def __init__(
+            self, name, emails,
+            fullname=None):
+        self.name = name
+        self.fullname = fullname
+        self.emails = emails
+
+    def to_json(self):
+        ''' Return a representation of the User in a dictionary. '''
+
+        output = {
+            'name': self.name,
+            'fullname': self.fullname,
+            'emails': self.emails
+        }
+
+        return output

From 07b97f010463d3043701a35894668d0d19043fa8 Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Feb 23 2016 08:36:47 +0000
Subject: [PATCH 2/11] added repo.py from pagure


---

diff --git a/pagure_importer/lib/repo.py b/pagure_importer/lib/repo.py
new file mode 100644
index 0000000..f46b31a
--- /dev/null
+++ b/pagure_importer/lib/repo.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+
+''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/repo.py
+    by pingou@pingoured.fr
+'''
+
+
+import pygit2
+import sys
+
+
+def get_pygit2_version():
+    ''' Return pygit2 version as a tuple of integers.
+    This is needed for correct version comparison.
+    '''
+    return tuple([int(i) for i in pygit2.__version__.split('.')])
+
+
+class PagureRepo(pygit2.Repository):
+    """ An utility class allowing to go around pygit2's inability to be
+    stable.
+
+    """
+
+    @staticmethod
+    def push(remote, refname):
+        """ Push the given reference to the specified remote. """
+        pygit2_version = get_pygit2_version()
+        if pygit2_version >= (0, 22):
+            remote.push([refname])
+        else:
+            remote.push(refname)
+
+    def pull(self, remote_name='origin', branch='master', force=False):
+        ''' pull changes for the specified remote (defaults to origin).
+
+        Code from MichaelBoselowitz at:
+        https://github.com/MichaelBoselowitz/pygit2-examples/blob/
+            68e889e50a592d30ab4105a2e7b9f28fac7324c8/examples.py#L58
+        licensed under the MIT license.
+        '''
+
+        for remote in self.remotes:
+            if remote.name == remote_name:
+                remote.fetch()
+                remote_master_id = self.lookup_reference(
+                    'refs/remotes/origin/%s' % branch).target
+
+                if force:
+                    repo_branch = self.lookup_reference(
+                        'refs/heads/%s' % branch)
+                    repo_branch.set_target(remote_master_id)
+
+                merge_result, _ = self.merge_analysis(remote_master_id)
+                # Up to date, do nothing
+                if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+                    return
+                # We can just fastforward
+                elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
+                    self.checkout_tree(self.get(remote_master_id))
+                    master_ref = self.lookup_reference(
+                        'refs/heads/%s' % branch)
+                    master_ref.set_target(remote_master_id)
+                    self.head.set_target(remote_master_id)
+                elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
+                    sys.exit('Pulling remote changes leads to a conflict')
+                else:
+                    print 'Unexpected merge result: %s' % (
+                            pygit2.GIT_MERGE_ANALYSIS_NORMAL)
+                    raise AssertionError('Unknown merge analysis result')

From 4830732621f4053fca4d4fbb3f12e60da34f8206 Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Feb 23 2016 08:40:04 +0000
Subject: [PATCH 3/11] added git.py from pagure code


---

diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py
new file mode 100644
index 0000000..eff314f
--- /dev/null
+++ b/pagure_importer/lib/git.py
@@ -0,0 +1,98 @@
+''' Code taken from https://pagure.io/pagure/blob/master/f/pagure/lib/git.py
+    by pingou@pingoured.fr
+'''
+
+import shutil
+import os
+import pygit2
+import tempfile
+import json
+
+from repo import *
+
+def update_git(obj, repo_path, repofolder):
+    """ Update the given issue in its git.
+    This method forks the provided repo, add/edit the issue whose file name
+    is defined by the uid field of the issue and if there are additions/
+    changes commit them and push them back to the original repo.
+    """
+
+    if not repofolder:
+        return
+
+    # Get the fork
+    repopath = os.path.join(repofolder, repo_path)
+
+    # Clone the repo into a temp folder
+    newpath = tempfile.mkdtemp(prefix='pagure-')
+    new_repo = pygit2.clone_repository(repopath, newpath)
+
+    file_path = os.path.join(newpath, obj.uid)
+
+    # Get the current index
+    index = new_repo.index
+
+    # Are we adding files
+    added = False
+    if not os.path.exists(file_path):
+        added = True
+
+    # Write down what changed
+    with open(file_path, 'w') as stream:
+        stream.write(json.dumps(
+            obj.to_json(), sort_keys=True, indent=4,
+            separators=(',', ': ')))
+
+    # Retrieve the list of files that changed
+    diff = new_repo.diff()
+    files = []
+    for p in diff:
+        if hasattr(p, 'new_file_path'):
+            files.append(p.new_file_path)
+        elif hasattr(p, 'delta'):
+            files.append(p.delta.new_file.path)
+
+    # Add the changes to the index
+    if added:
+        index.add(obj.uid)
+    for filename in files:
+        index.add(filename)
+
+    # If not change, return
+    if not files and not added:
+        shutil.rmtree(newpath)
+        return
+
+    # See if there is a parent to this commit
+    parent = None
+    try:
+        parent = new_repo.head.get_object().oid
+    except pygit2.GitError:
+        pass
+
+    parents = []
+    if parent:
+        parents.append(parent)
+
+    # Author/commiter will always be this one
+    author = pygit2.Signature(name='pagure', email='pagure')
+
+    # Actually commit
+    new_repo.create_commit(
+        'refs/heads/master',
+        author,
+        author,
+        'Updated %s %s: %s' % (obj.isa, obj.uid, obj.title),
+        new_repo.index.write_tree(),
+        parents)
+    index.write()
+
+    # Push to origin
+    ori_remote = new_repo.remotes[0]
+    master_ref = new_repo.lookup_reference('HEAD').resolve()
+    refname = '%s:%s' % (master_ref.name, master_ref.name)
+
+    PagureRepo.push(ori_remote, refname)
+
+    # Remove the clone
+    shutil.rmtree(newpath)

From 4f74da44a6ca18c76fe74ac85fc7f3104421775f Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 10 2016 10:54:24 +0000
Subject: [PATCH 4/11] Basics of pygit2 approach for github issues completed


---

diff --git a/pagure_importer/__init__.py b/pagure_importer/__init__.py
index de741ae..fbab797 100644
--- a/pagure_importer/__init__.py
+++ b/pagure_importer/__init__.py
@@ -1 +1,2 @@
-from sources import *
+import lib
+import settings
diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py
index 1aa883a..ddc24f0 100644
--- a/pagure_importer/forms.py
+++ b/pagure_importer/forms.py
@@ -1,35 +1,10 @@
-from sources.importer_github import GithubImporter
 import getpass
 
+from lib.sources.importer_github_new import GithubImporter
+from settings import REPO_PATH, REPO_NAME
+
 def form_github_issues():
     github_username = raw_input('Enter you Github Username: ')
     github_password = getpass.getpass('Enter your github password: ')
-    github_project_name = raw_input('Enter github project name: ')
-    pagure_api_key = raw_input('Enter your pagure api key: ')
-    pagure_project_name = raw_input('Enter pagure project name: ')
-
-    is_forked = raw_input('Is the pagure project a forked repo ? (y/n): ') or 'n'
-    if is_forked.lower() == 'y':
-        pagure_username = raw_input('Enter your pagure username: ')
-    else:
-        pagure_username = None
-
-    is_pagure_io = raw_input(
-                'Is the pagure instance url - https://pagure.io ?: (y/n) ') or 'y'
-    if is_pagure_io.lower() == 'n':
-        pagure_instance = raw_input('Enter the pagure instance url: ') or 'https://pagure.io'
-    else:
-        pagure_instance = 'https://pagure.io'
-
-    status = raw_input(
-            'Enter status of the issues to be imported (all/open/closed): ') or 'all'
-
-    github_importer = GithubImporter(
-                        github_username=github_username,
-                        github_password=github_password,
-                        github_project_name=github_project_name,
-                        pagure_api_key=pagure_api_key,
-                        pagure_project_name=pagure_project_name,
-                        pagure_username=pagure_username,
-                        instance_url=pagure_instance)
-    github_importer.import_issues(status)
+    github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ')
+    return (github_username, github_password, github_project_name)
diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py
new file mode 100644
index 0000000..c1724e7
--- /dev/null
+++ b/pagure_importer/lib/__init__.py
@@ -0,0 +1,153 @@
+import git
+import models
+
+import os
+import getpass
+import requests
+import json
+from github import Github
+from requests.auth import HTTPBasicAuth
+import pagure_importer
+import pagure_importer.lib
+from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound
+
+def generate_json_for_github_contributors(github_username, github_password, \
+                                                github_project_name):
+    ''' Creates a file containing a list of dicts containing the username and emails
+    of the contributors in the given github project
+    '''
+
+    github_obj = Github(github_username, github_password)
+    project = github_obj.get_repo(github_project_name)
+    commits_url = project.commits_url.replace('{/sha}', '')
+
+    page = 0
+    contributors = []
+    while True:
+        page += 1
+        payload = {'page': page }
+        data_ = json.loads(requests.get(commits_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text)
+
+        if not data_:
+            break
+
+        for data in data_:
+            try:
+                contributor = data['commit']['committer']
+                contributor_email = contributor['email']
+                contributor_fullname = contributor['name']
+                contributor_name = data['committer']['login']
+            except TypeError:
+                print 'Maybe one of the contributors is dropped because of lack of details'
+                continue
+
+            json_data = {
+                'name': contributor_name,
+                'fullname': contributor_fullname,
+                'emails': [contributor_email]
+            }
+
+            present = False
+            for i in contributors:
+                if i == json_data:
+                    present = True
+                    break
+
+            if not present:
+                print 'contributor added: ', len(contributors) + 1
+                contributors.append(json_data)
+
+    with open('contributors.json', 'w') as f:
+        f.write(json.dumps(contributors))
+
+    return
+
+
+def generate_json_for_github_issue_commentors(github_username, github_password, \
+                                                github_project_name):
+    ''' Will create a json file containing details of all the user
+    who have commented on any issue in the given project
+    '''
+
+    github_obj = Github(github_username, github_password)
+    project = github_obj.get_repo(github_project_name)
+    issue_comment_url = project.issue_comment_url.replace('{/number}', '')
+
+    page = 0
+    issue_commentors = []
+    while True:
+        page += 1
+        payload = {'page': page }
+        data_ = json.loads(requests.get(issue_comment_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text)
+
+        if not data_:
+            break
+
+        for data in data_:
+            try:
+                commentor = data['user']['login']
+            except TypeError:
+                print 'Maybe one of the issue commentors have been dropped because of lack of details'
+                continue
+
+            present = False
+            for i in issue_commentors:
+                if i == commentor:
+                    present = True
+                    break
+
+            if not present:
+                print 'commentor added: ', len(issue_commentors) + 1
+                issue_commentors.append(commentor)
+
+    with open('issue_commentors.json', 'w') as f:
+        f.write(json.dumps(issue_commentors))
+    return
+
+
+def assemble_github_contributors_commentors():
+    ''' It uses the files: issue_commentors.json and contributors.json
+    Assembles and creates a file: assembled_commentors.json
+    To use: just fill the None and [] in the final file '''
+
+    with open('issue_commentors.json', 'r') as ic:
+        issue_names = json.load(ic)
+
+    with open('contributors.json', 'r') as c:
+        contributors = json.load(c)
+
+    names = []
+    for i in issue_names:
+        found = False
+        for j in contributors:
+            if j.get('name', None) == i:
+                names.append(j)
+                found = True
+
+        if not found:
+            d = {'name': i, 'fullname': None, 'emails': []}
+            names.append(d)
+
+    with open('assembled_commentors.json', 'w') as ac:
+        json.dump(names, ac)
+
+
+def github_get_commentor_email(name):
+    ''' Will return the issue commentor email as given in the
+    assembled_commentors.json file
+    '''
+
+    if not os.path.exists('assembled_commentors.json'):
+        raise FileNotFound('The assembled_commentors.json file must be present \
+                Rerun the program and choose to generate the json files')
+
+    with open('assembled_commentors.json') as ac:
+        data = json.load(ac)
+
+    for i in data:
+        if i.get('name', None) == name:
+            if i['emails']:
+                return i['emails']
+            else:
+                raise EmailNotFound('You need to fill out all the emails of the \
+                        issue commentors')
diff --git a/pagure_importer/lib/exceptions.py b/pagure_importer/lib/exceptions.py
new file mode 100644
index 0000000..a5ab5fe
--- /dev/null
+++ b/pagure_importer/lib/exceptions.py
@@ -0,0 +1,22 @@
+class GithubBadCredentials(Exception):
+    ''' Raised when username/password for github is wrong '''
+    def __init__(self, msg):
+        self.msg = msg
+
+
+class GithubRepoNotFound(Exception):
+    ''' Raised when the repo is not found for the user '''
+    def __init__(self, msg):
+        self.msg = msg
+
+
+class FileNotFound(Exception):
+    ''' Raised when a certain file is not found '''
+    def __init__(self, msg):
+        self.msg = msg
+
+
+class EmailNotFound(Exception):
+    ''' Raised when email is not found '''
+    def __init__(self, msg):
+        self.msg = msg
diff --git a/pagure_importer/lib/git.py b/pagure_importer/lib/git.py
index eff314f..33eba48 100644
--- a/pagure_importer/lib/git.py
+++ b/pagure_importer/lib/git.py
@@ -10,18 +10,18 @@ import json
 
 from repo import *
 
-def update_git(obj, repo_path, repofolder):
+def update_git(obj, repo_path, repo_folder):
     """ Update the given issue in its git.
     This method forks the provided repo, add/edit the issue whose file name
     is defined by the uid field of the issue and if there are additions/
     changes commit them and push them back to the original repo.
     """
 
-    if not repofolder:
+    if not repo_folder:
         return
 
     # Get the fork
-    repopath = os.path.join(repofolder, repo_path)
+    repopath = os.path.join(repo_folder, repo_path)
 
     # Clone the repo into a temp folder
     newpath = tempfile.mkdtemp(prefix='pagure-')
diff --git a/pagure_importer/lib/models.py b/pagure_importer/lib/models.py
index 7ec0231..3aa3d83 100644
--- a/pagure_importer/lib/models.py
+++ b/pagure_importer/lib/models.py
@@ -79,6 +79,7 @@ class IssueComment():
             'edited_on': self.edited_on.strftime('%s') if self.edited_on else None,
             'editor': self.editor or None
         }
+
         return output
 
 
diff --git a/pagure_importer/lib/sources/__init__.py b/pagure_importer/lib/sources/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pagure_importer/lib/sources/__init__.py
diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py
new file mode 100644
index 0000000..2e635b6
--- /dev/null
+++ b/pagure_importer/lib/sources/importer_github.py
@@ -0,0 +1,133 @@
+from github import Github
+import pagure_importer
+import pagure_importer.lib
+from pagure_importer.lib import models
+from pagure_importer.lib import github_get_commentor_email
+from pagure_importer.lib.exceptions import GithubBadCredentials, GithubRepoNotFound
+
+class GithubImporter():
+    ''' Imports from Github using PyGithub and libpagure '''
+    def __init__(
+            self,
+            github_username,
+            github_password,
+            github_project_name):
+        self.github_username = github_username
+        self.github_password = github_password
+        self.github_project_name = github_project_name
+        self.github = Github(github_username, github_password)
+
+    def import_issues(self, repo_path, repo_folder, status='all'):
+        ''' Imports the issues on github for
+        the given project
+        '''
+        github_user = None
+        try:
+            github_user = self.github.get_user(self.github_username)
+        except:
+            raise GithubBadCredentials(
+                    'Given github credentials are not correct')
+        repo = self.github.get_repo(self.github_project_name)
+        try:
+            repo_name = repo.name
+        except:
+            raise GithubRepoNotFound(
+                    'Repo not found, project name wrong')
+
+        for github_issue in repo.get_issues(state=status):
+
+            #title of the issue
+            pagure_issue_title = github_issue.title
+
+            #body of the issue
+            if github_issue.body:
+                pagure_issue_content = github_issue.body
+            else:
+                pagure_issue_content = '#No Description Provided'
+
+            #Some details of a issue
+            if github_issue.state != 'closed':
+                pagure_issue_status = 'Open'
+            else:
+                pagure_issue_status = 'Fixed'
+
+            pagure_issue_created_at = github_issue.created_at
+
+            #Not sure how to deal with this atm
+            pagure_issue_assignee = None
+
+            if github_issue.labels:
+                pagure_issue_tags = [i.name for i in github_issue.labels]
+            else:
+                pagure_issue_tags = []
+
+
+            #few things not supported by github
+            pagure_issue_depends = []
+            pagure_issue_blocks = []
+            pagure_issue_is_private = False
+
+
+            #User who created the issue
+            pagure_issue_user = models.User(
+                    name=github_issue.user.login,
+                    fullname=github_issue.user.name,
+                    emails=[github_issue.user.email])
+
+
+            pagure_issue = models.Issue(
+                    id=None,
+                    title = pagure_issue_title,
+                    content = pagure_issue_content,
+                    status = pagure_issue_status,
+                    date_created = pagure_issue_created_at,
+                    user = pagure_issue_user.to_json(),
+                    private = pagure_issue_is_private,
+                    tags = pagure_issue_tags,
+                    depends = pagure_issue_depends,
+                    blocks = pagure_issue_blocks,
+                    assignee = pagure_issue_assignee)
+
+
+            #comments on the issue
+            comments = []
+            for comment in github_issue.get_comments():
+
+                comment_user = comment.user
+                pagure_issue_comment_user_email = comment_user.email
+                pagure_issue_comment_body = comment.body
+                pagure_issue_comment_created_at = comment.created_at
+                pagure_issue_comment_updated_at = comment.updated_at
+
+
+                #No idea what to do with this right now
+                #editor: not supported by github api
+                pagure_issue_comment_parent = None
+                pagure_issue_comment_editor = None
+
+                #comment updated at
+                pagure_issue_comment_edited_on = comment.updated_at
+
+                #The User who commented
+                pagure_issue_comment_user = models.User(
+                        name=comment_user.login,
+                        fullname=comment_user.name,
+                        emails=[comment_user.email] if comment_user.email else github_get_commentor_email(comment_user.login))
+
+                #Object to represent comment on an issue
+                pagure_issue_comment = models.IssueComment(
+                        id=None,
+                        comment=pagure_issue_comment_body,
+                        parent=pagure_issue_comment_parent,
+                        date_created=pagure_issue_comment_created_at,
+                        user=pagure_issue_comment_user.to_json(),
+                        edited_on=pagure_issue_comment_edited_on,
+                        editor=pagure_issue_comment_editor)
+
+                comments.append(pagure_issue_comment.to_json())
+
+            #add all the comments to the issue object
+            pagure_issue.comments = comments
+
+            #update the local git repo
+            pagure_importer.lib.git.update_git(pagure_issue, repo_path, repo_folder)
diff --git a/pagure_importer/run.py b/pagure_importer/run.py
index a485493..2753298 100644
--- a/pagure_importer/run.py
+++ b/pagure_importer/run.py
@@ -1,11 +1,35 @@
 #!/usr/bin/env python
-
+import getpass
 from forms import form_github_issues
-from settings import IMPORT_SOURCES, IMPORT_OPTIONS
+from settings import IMPORT_SOURCES, IMPORT_OPTIONS, REPO_NAME, REPO_PATH
+import pagure_importer
+import pagure_importer.lib
+import pagure_importer.lib.sources
+from pagure_importer.lib.sources.importer_github import GithubImporter
+from pagure_importer.lib import generate_json_for_github_contributors, \
+            generate_json_for_github_issue_commentors, \
+            assemble_github_contributors_commentors
 
 def github_handler(item):
     if item.lower() == 'issues':
-        form_github_issues()
+        github_username, github_password, github_project_name = form_github_issues()
+        gen_json = raw_input('Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ')
+        if gen_json == 'n':
+            github_importer = GithubImporter(
+                            github_username=github_username,
+                            github_password=github_password,
+                            github_project_name=github_project_name)
+            github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH)
+        else:
+            generate_json_for_github_contributors(
+                                            github_username,
+                                            github_password,
+                                            github_project_name)
+            generate_json_for_github_issue_commentors(
+                                            github_username,
+                                            github_password,
+                                            github_project_name)
+            assemble_github_contributors_commentors()
     return
 
 def main():
@@ -21,7 +45,6 @@ def main():
 
     if source.lower() == 'github':
         github_handler(item)
-
     return
 
 if __name__ == '__main__':
diff --git a/pagure_importer/settings.py b/pagure_importer/settings.py
index 13f8332..9761aed 100644
--- a/pagure_importer/settings.py
+++ b/pagure_importer/settings.py
@@ -1,2 +1,7 @@
+import os
+
 IMPORT_SOURCES = ['github']
 IMPORT_OPTIONS = {'github': ['issues']}
+
+REPO_NAME = os.environ.get('REPO_NAME', None) #this has to be a bare repo
+REPO_PATH = os.environ.get('REPO_PATH', None) #the parent of the git directory
diff --git a/pagure_importer/sources/__init__.py b/pagure_importer/sources/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/pagure_importer/sources/__init__.py
+++ /dev/null
diff --git a/pagure_importer/sources/exceptions.py b/pagure_importer/sources/exceptions.py
deleted file mode 100644
index 8393fb8..0000000
--- a/pagure_importer/sources/exceptions.py
+++ /dev/null
@@ -1,9 +0,0 @@
-class GithubBadCredentials(Exception):
-    ''' Raised when username/password for github is wrong '''
-    def __init__(self, msg):
-        self.msg = msg
-
-class GithubRepoNotFound(Exception):
-    ''' Raised when the repo is not found for the user '''
-    def __init__(self, msg):
-        self.msg = msg
diff --git a/pagure_importer/sources/importer_github.py b/pagure_importer/sources/importer_github.py
deleted file mode 100644
index cc85f7b..0000000
--- a/pagure_importer/sources/importer_github.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import libpagure
-from libpagure.libpagure import Pagure
-from github import Github
-
-from exceptions import GithubBadCredentials, GithubRepoNotFound
-
-
-class GithubImporter():
-    ''' Imports from Github using PyGithub and libpagure '''
-
-
-    def __init__(
-            self,
-            github_username,
-            github_password,
-            github_project_name,
-            pagure_api_key,
-            pagure_project_name,
-            pagure_username=None,
-            instance_url='https://pagure.io'):
-
-        self.github_username = github_username
-        self.github_password = github_password
-        self.github_project_name = github_project_name
-        self.pagure_project_name = pagure_project_name
-        self.github = Github(github_username, github_password)
-        self.pagure = Pagure(pagure_api_key, pagure_project_name,
-                                pagure_username, instance_url)
-
-
-    def _get_available_issue_id(self):
-        ''' Private method which checks the id
-        which would be available for the new issue
-        '''
-        issues = self.pagure.list_issues()
-        pull_requests = self.pagure.list_requests()
-        max_issues = None
-        max_pull_requests = None
-        try:
-            max_issues = max([int(issue['id']) for issue in issues])
-        except ValueError:
-            max_issues = 0
-
-        try:
-            max_pull_requests = max([int(pr['id']) for pr in pull_requests])
-        except ValueError:
-            max_pull_requests = 0
-
-        return max(max_issues, max_pull_requests) + 1
-
-
-    def _get_repo(self, github_user):
-        ''' Private method to get the repo object
-        using the given github project name
-        '''
-        repos = github_user.get_repos()
-        for repo in repos:
-            if repo.name == self.github_project_name:
-                return repo
-        raise GithubRepoNotFound(
-                'No user repository with given github project name found')
-
-
-    def import_issues(self, status='all'):
-        ''' Imports the issues on github for
-        the given project
-        '''
-        github_user = None
-        try:
-            github_user = self.github.get_user(self.github_username)
-        except:
-            raise GithubBadCredentials(
-                    'Given github credentials are not correct')
-
-        repo = self._get_repo(github_user)
-        for github_issue in repo.get_issues(state=status):
-            pagure_issue_title = github_issue.title
-            if github_issue.body:
-                pagure_issue_content = github_issue.body
-            else:
-                pagure_issue_content = '#No Description Provided'
-
-            issue_id = self._get_available_issue_id()
-            self.pagure.create_issue(
-                    pagure_issue_title, pagure_issue_content)
-
-            #comments on the issue
-            for comment in github_issue.get_comments():
-                self.pagure.comment_issue(issue_id, str(comment.body))
-
-            #change status of the issue if closed
-            if github_issue.state.lower() == 'closed':
-                self.pagure.change_issue_status(issue_id, 'Fixed')
-

From 80300ad87bb0de42075b2a8ab33d9154cb099805 Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 10 2016 10:54:46 +0000
Subject: [PATCH 5/11] Readme for the new approach


---

diff --git a/README.md b/README.md
index b0c883d..46a707d 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,53 @@
 # pagure-importer
 CLI tool for importing issues etc. from different sources like github to pagure
 
-## How to run
+## Installation
 1. Install it using ```pip``` . ```pip install pagure_importer```
+
+## How to run
+0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare```
+and set the env variables: 'REPO_NAME' and 'REPO_PATH'
+ex: REPO_NAME='abc.git'; REPO_PATH='/home/vivek/'
+1. Activate the pagure tickets hook from project settings.
 2. Execute ```pgimport```
-3. Just answer what is asked, one by one.
+3. Just answer what is asked. Check below instructions for particular source
+4. The script will make commits in your cloned bare repo: push the changes back to pagure.
 
 ### Present options for sources: github
 ### Present options for items: issues
 
 ### Tools used:
-1. [libpagure](https://pagure.io/libpagure) - a python library for [pagure](https://pagure.io) api.
-2. [PyGithub](https://github.com/PyGithub/PyGithub) - a python library for [github](https://github.com/) api.
+1. [PyGithub](https://github.com/PyGithub/PyGithub) - a python library for [github](https://github.com/) api.
+
+
+## How it works: Github Issues
+0. For github issues, there is a bit of pre-processing so, the process is
+not very user friendly. The reason behind the pre-processing is that: github
+doesn't give away the email ids of issue commentors unless the commentor
+is you (if you are logged in) or if the commentor is the issue reporter
+himself. So, to overcome this problem, we will be taking email ids from their
+commits, if they have contributed to the project but if they haven't, : start
+panicking and read below.
+
+1. We will have to run the script two times. The first time, it will
+generate a json file containing all the issue commentors with their details,
+if the emails are found, no edit for that particular commentor is required.
+Otherwise, you will have to manually fill the emails. Fullnames not required.
+
+2. After running the program and answering the 'source' and 'items', you
+will be asked a question on whether you want to generate a json file for
+contributors and issue commentors. If you are running the script for github
+for the first time, the answer is 'y'.
+
+3. The above step will create 3 different json files: ```contributors.json```
+```issue_commentors.json``` and ```assembled_commentors.json```. The last file
+is where all the edit has to go. All the missing entries in the assembled
+commentors file has to be filled for the running of the script.
+
+4. Run the script again, filling the same details but answer 'n' when asked for
+whether you want to create the json files. In this step, your local issues git
+repo gets updated with all the issues from github issue tracker.
+
+5. Now push the local git repo changes to the remote repo on pagure. It will
+update the db and if the user is not found, it will create them from the
+details given.

From a705a035b4352c7030864fe1bfb593ed998aa08d Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 10 2016 16:19:44 +0000
Subject: [PATCH 6/11] Removed libpagure form requirements


---

diff --git a/requirements.txt b/requirements.txt
index eeb84d8..52fc450 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,2 @@
-libpagure
 PyGithub
 requests

From aa46f6b9d0622f2d3bed1f0c7a783555c11de639 Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 12 2016 22:17:24 +0000
Subject: [PATCH 7/11] Change the output file to csv type


---

diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py
index c1724e7..702786c 100644
--- a/pagure_importer/lib/__init__.py
+++ b/pagure_importer/lib/__init__.py
@@ -1,6 +1,7 @@
 import git
 import models
 
+import csv
 import os
 import getpass
 import requests
@@ -13,8 +14,8 @@ from pagure_importer.lib.exceptions import FileNotFound, EmailNotFound
 
 def generate_json_for_github_contributors(github_username, github_password, \
                                                 github_project_name):
-    ''' Creates a file containing a list of dicts containing the username and emails
-    of the contributors in the given github project
+    ''' Creates a file containing a list of dicts containing the username and
+    emails of the contributors in the given github project
     '''
 
     github_obj = Github(github_username, github_password)
@@ -26,7 +27,8 @@ def generate_json_for_github_contributors(github_username, github_password, \
     while True:
         page += 1
         payload = {'page': page }
-        data_ = json.loads(requests.get(commits_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text)
+        data_ = json.loads(requests.get(commits_url, params=payload,
+                    auth=HTTPBasicAuth(github_username, github_password)).text)
 
         if not data_:
             break
@@ -54,7 +56,7 @@ def generate_json_for_github_contributors(github_username, github_password, \
                     break
 
             if not present:
-                print 'contributor added: ', len(contributors) + 1
+                print 'contributor added: ', contributor_name
                 contributors.append(json_data)
 
     with open('contributors.json', 'w') as f:
@@ -78,7 +80,8 @@ def generate_json_for_github_issue_commentors(github_username, github_password, 
     while True:
         page += 1
         payload = {'page': page }
-        data_ = json.loads(requests.get(issue_comment_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text)
+        data_ = json.loads(requests.get(issue_comment_url, params=payload,
+                    auth=HTTPBasicAuth(github_username, github_password)).text)
 
         if not data_:
             break
@@ -97,7 +100,7 @@ def generate_json_for_github_issue_commentors(github_username, github_password, 
                     break
 
             if not present:
-                print 'commentor added: ', len(issue_commentors) + 1
+                print 'commentor added: ', commentor
                 issue_commentors.append(commentor)
 
     with open('issue_commentors.json', 'w') as f:
@@ -128,8 +131,13 @@ def assemble_github_contributors_commentors():
             d = {'name': i, 'fullname': None, 'emails': []}
             names.append(d)
 
-    with open('assembled_commentors.json', 'w') as ac:
-        json.dump(names, ac)
+    with open('assembled_commentors.csv', 'w') as ac:
+        field_names = ['name', 'fullname', 'emails']
+        writer = csv.DictWriter(ac, fieldnames=field_names)
+
+        writer.writeheader()
+        for name in names:
+            writer.writerow(name)
 
 
 def github_get_commentor_email(name):
@@ -137,12 +145,19 @@ def github_get_commentor_email(name):
     assembled_commentors.json file
     '''
 
-    if not os.path.exists('assembled_commentors.json'):
+    if not os.path.exists('assembled_commentors.csv'):
         raise FileNotFound('The assembled_commentors.json file must be present \
                 Rerun the program and choose to generate the json files')
 
-    with open('assembled_commentors.json') as ac:
-        data = json.load(ac)
+    data = []
+    with open('assembled_commentors.csv') as ac:
+        reader = csv.DictReader(ac)
+        for row in reader:
+            data.append(dict( \
+                (('name', row['name']), \
+                ('fullname', row['fullname']), \
+                ('emails', row['emails']))))
+
 
     for i in data:
         if i.get('name', None) == name:

From a7ec751d7eec0ca88d6b943e81f2d23ca37cb7b1 Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 12 2016 22:17:44 +0000
Subject: [PATCH 8/11] Corrected import


---

diff --git a/pagure_importer/forms.py b/pagure_importer/forms.py
index ddc24f0..a111114 100644
--- a/pagure_importer/forms.py
+++ b/pagure_importer/forms.py
@@ -1,6 +1,6 @@
 import getpass
 
-from lib.sources.importer_github_new import GithubImporter
+from lib.sources.importer_github import GithubImporter
 from settings import REPO_PATH, REPO_NAME
 
 def form_github_issues():

From ebd70ed42fd9b68b0da45468dd4ac9cdee7d07ec Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 14 2016 07:50:21 +0000
Subject: [PATCH 9/11] Email as a string in csv


---

diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py
index 702786c..91e0dd4 100644
--- a/pagure_importer/lib/__init__.py
+++ b/pagure_importer/lib/__init__.py
@@ -124,11 +124,12 @@ def assemble_github_contributors_commentors():
         found = False
         for j in contributors:
             if j.get('name', None) == i:
+                j['emails'] = j.get('emails')[0]
                 names.append(j)
                 found = True
 
         if not found:
-            d = {'name': i, 'fullname': None, 'emails': []}
+            d = {'name': i, 'fullname': None, 'emails': None}
             names.append(d)
 
     with open('assembled_commentors.csv', 'w') as ac:
@@ -142,7 +143,7 @@ def assemble_github_contributors_commentors():
 
 def github_get_commentor_email(name):
     ''' Will return the issue commentor email as given in the
-    assembled_commentors.json file
+    assembled_commentors.csv file
     '''
 
     if not os.path.exists('assembled_commentors.csv'):
@@ -162,7 +163,8 @@ def github_get_commentor_email(name):
     for i in data:
         if i.get('name', None) == name:
             if i['emails']:
-                return i['emails']
+                return str(i['emails'])
             else:
                 raise EmailNotFound('You need to fill out all the emails of the \
                         issue commentors')
+
diff --git a/pagure_importer/lib/sources/importer_github.py b/pagure_importer/lib/sources/importer_github.py
index 2e635b6..2859e99 100644
--- a/pagure_importer/lib/sources/importer_github.py
+++ b/pagure_importer/lib/sources/importer_github.py
@@ -112,7 +112,8 @@ class GithubImporter():
                 pagure_issue_comment_user = models.User(
                         name=comment_user.login,
                         fullname=comment_user.name,
-                        emails=[comment_user.email] if comment_user.email else github_get_commentor_email(comment_user.login))
+                        emails=[comment_user.email] if comment_user.email \
+                                else [github_get_commentor_email(comment_user.login)])
 
                 #Object to represent comment on an issue
                 pagure_issue_comment = models.IssueComment(

From 52ab30a6b7659a93e8e80e52e417735d173f1318 Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 14 2016 08:04:53 +0000
Subject: [PATCH 10/11] reflect change in output file name in Readme


---

diff --git a/README.md b/README.md
index 46a707d..b2075e2 100644
--- a/README.md
+++ b/README.md
@@ -6,12 +6,12 @@ CLI tool for importing issues etc. from different sources like github to pagure
 
 ## How to run
 0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare```
-and set the env variables: 'REPO_NAME' and 'REPO_PATH'
-ex: REPO_NAME='abc.git'; REPO_PATH='/home/vivek/'
-1. Activate the pagure tickets hook from project settings.
-2. Execute ```pgimport```
-3. Just answer what is asked. Check below instructions for particular source
-4. The script will make commits in your cloned bare repo: push the changes back to pagure.
+1. set the env variables: ```REPO_NAME``` and ```REPO_PATH```
+ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/
+2. Activate the pagure tickets hook from project settings.
+3. Execute ```pgimport```
+4. Just answer what is asked. Check below instructions for particular source
+5. The script will make commits in your cloned bare repo: push the changes back to pagure.
 
 ### Present options for sources: github
 ### Present options for items: issues
@@ -39,8 +39,8 @@ will be asked a question on whether you want to generate a json file for
 contributors and issue commentors. If you are running the script for github
 for the first time, the answer is 'y'.
 
-3. The above step will create 3 different json files: ```contributors.json```
-```issue_commentors.json``` and ```assembled_commentors.json```. The last file
+3. The above step will create 3 different files: ```contributors.json```
+```issue_commentors.json``` and ```assembled_commentors.csv```. The last file
 is where all the edit has to go. All the missing entries in the assembled
 commentors file has to be filled for the running of the script.
 

From 4a6bb8177ace90afbe44980c5b08725e6011a543 Mon Sep 17 00:00:00 2001
From: Vivek Anand <vivekanand1101@gmail.com>
Date: Apr 14 2016 08:22:23 +0000
Subject: [PATCH 11/11] Doc string corrected for a func


---

diff --git a/pagure_importer/lib/__init__.py b/pagure_importer/lib/__init__.py
index 91e0dd4..9a64e3b 100644
--- a/pagure_importer/lib/__init__.py
+++ b/pagure_importer/lib/__init__.py
@@ -110,8 +110,8 @@ def generate_json_for_github_issue_commentors(github_username, github_password, 
 
 def assemble_github_contributors_commentors():
     ''' It uses the files: issue_commentors.json and contributors.json
-    Assembles and creates a file: assembled_commentors.json
-    To use: just fill the None and [] in the final file '''
+    Assembles and creates a file: assembled_commentors.csv
+    To use: just fill the empty blocks under emails column'''
 
     with open('issue_commentors.json', 'r') as ic:
         issue_names = json.load(ic)