From a07755a2cd0b94ac63f3cca667bcff71cbba1497 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 29 2016 20:45:14 +0000 Subject: [PATCH 1/6] Added clone command to pgimport Command enables user to clone a ticket repo using the url provided by pagure --- diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 7c92a53..9523ffe 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -17,6 +17,7 @@ __all__ = [ from .commands import fedorahosted from .commands import github +from .commands import clone if __name__ == '__main__': app() diff --git a/pagure_importer/commands/clone.py b/pagure_importer/commands/clone.py new file mode 100644 index 0000000..86ac1b8 --- /dev/null +++ b/pagure_importer/commands/clone.py @@ -0,0 +1,14 @@ +import click +import os +import subprocess as sp +from pagure_importer.app import app + +@app.command() +@click.argument('repo_url') +def clone (repo_url): + repo = os.path.join('/tmp/',repo_url.split('/')[-1]) + cmd = ['git', 'clone', '--bare',repo_url, repo] + proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.STDOUT) + output, _ = proc.communicate() + output = output.decode('utf-8') + click.echo(output) From 8503f08cd7d555c2296698f2cedbfbdb957bd616 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 29 2016 20:47:12 +0000 Subject: [PATCH 2/6] Modify fedorahosted import to diplay the ticket repo found in the tmp folder --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 403d269..253aa08 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,9 +1,12 @@ import click +import os import getpass -from pagure_importer.app import app, REPO_NAME, REPO_PATH -from pagure_importer.utils import importer_trac +from pagure_importer.app import app, REPO_NAME +from pagure_importer.utils import importer_trac, display_repo from pagure_importer.utils.fas import FASclient +REPO_PATH='/tmp/' + @app.command() @click.argument('project_url') @click.option('--tags', help="Import pagure tags:", is_flag=True) @@ -17,6 +20,8 @@ def fedorahosted(project_url, tags): url_index = project_url.find('://') rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ + '/login/xmlrpc' + display_repo() + repo_name = raw_input('Choose the import destination repo : ') trac_importer = importer_trac.TracImporter(rpc_url, fasclient) - trac_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH, + trac_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH, tags=tags) diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 05ccdb8..5afed11 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -11,6 +11,13 @@ from requests.auth import HTTPBasicAuth from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound +def display_repo(): + print '#### Repo available ####' + for file in os.listdir('/tmp/'): + if file.endswith('.git'): + print ' * ' + file + print + def generate_json_for_github_contributors(github_username, github_password, \ github_project_name): ''' Creates a file containing a list of dicts containing the username and From bf38ad9a888e8cb71abcfc94dc6d87391bf93757 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 29 2016 21:11:06 +0000 Subject: [PATCH 3/6] Removed dependency to /tmp folder and used tempfile lib Fixed pep8 code syntax --- diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 9523ffe..303088f 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -1,10 +1,9 @@ #!/usr/bin/env python - import click -import os +import tempfile -REPO_NAME = os.environ.get('REPO_NAME', None) # this has to be a bare repo -REPO_PATH = os.environ.get('REPO_PATH', None) # the parent of the git directory +REPO_NAME = '' +REPO_PATH = tempfile.gettempdir() @click.group() diff --git a/pagure_importer/commands/clone.py b/pagure_importer/commands/clone.py index 86ac1b8..b6eb03a 100644 --- a/pagure_importer/commands/clone.py +++ b/pagure_importer/commands/clone.py @@ -1,13 +1,14 @@ import click import os import subprocess as sp -from pagure_importer.app import app +from pagure_importer.app import app, REPO_PATH + @app.command() @click.argument('repo_url') -def clone (repo_url): - repo = os.path.join('/tmp/',repo_url.split('/')[-1]) - cmd = ['git', 'clone', '--bare',repo_url, repo] +def clone(repo_url): + repo = os.path.join(REPO_PATH, repo_url.split('/')[-1]) + cmd = ['git', 'clone', '--bare', repo_url, repo] proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.STDOUT) output, _ = proc.communicate() output = output.decode('utf-8') diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 253aa08..8955160 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,11 +1,10 @@ import click import os import getpass -from pagure_importer.app import app, REPO_NAME -from pagure_importer.utils import importer_trac, display_repo +from pagure_importer.app import app, REPO_PATH +from pagure_importer.utils import importer_trac from pagure_importer.utils.fas import FASclient -REPO_PATH='/tmp/' @app.command() @click.argument('project_url') @@ -14,13 +13,13 @@ def fedorahosted(project_url, tags): fas_username = raw_input('Enter you FAS Username: ') fas_password = getpass.getpass('Enter your FAS password: ') fasclient = FASclient(fas_username, fas_password, - 'https://admin.fedoraproject.org/accounts') + 'https://admin.fedoraproject.org/accounts') rpc_login = fas_username + ':' + fas_password + '@' url_index = project_url.find('://') - rpc_url = project_url[:url_index+3] + rpc_login + project_url[url_index+3:]\ - + '/login/xmlrpc' - display_repo() + rpc_url = project_url[:url_index+3] + rpc_login +\ + project_url[url_index+3:] + '/login/xmlrpc' + pagure_importer.utils.display_repo() repo_name = raw_input('Choose the import destination repo : ') trac_importer = importer_trac.TracImporter(rpc_url, fasclient) trac_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH, diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 5afed11..5f3b2ae 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -10,16 +10,20 @@ from github import Github from requests.auth import HTTPBasicAuth from pagure_importer.utils.exceptions import FileNotFound, EmailNotFound +from pagure_importer.app import REPO_PATH + def display_repo(): - print '#### Repo available ####' - for file in os.listdir('/tmp/'): + print '#### Repo available ####' + for file in os.listdir(REPO_PATH): if file.endswith('.git'): print ' * ' + file print -def generate_json_for_github_contributors(github_username, github_password, \ - github_project_name): + +def generate_json_for_github_contributors(github_username, + github_password, + github_project_name): ''' Creates a file containing a list of dicts containing the username and emails of the contributors in the given github project ''' @@ -32,7 +36,7 @@ def generate_json_for_github_contributors(github_username, github_password, \ contributors = [] while True: page += 1 - payload = {'page': page } + payload = {'page': page} data_ = json.loads(requests.get(commits_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) @@ -71,8 +75,9 @@ def generate_json_for_github_contributors(github_username, github_password, \ return -def generate_json_for_github_issue_commentors(github_username, github_password, \ - github_project_name): +def generate_json_for_github_issue_commentors(github_username, + github_password, + github_project_name): ''' Will create a json file containing details of all the user who have commented on any issue in the given project ''' @@ -85,7 +90,7 @@ def generate_json_for_github_issue_commentors(github_username, github_password, issue_commentors = [] while True: page += 1 - payload = {'page': page } + payload = {'page': page} data_ = json.loads(requests.get(issue_comment_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) @@ -160,12 +165,11 @@ def github_get_commentor_email(name): with open('assembled_commentors.csv') as ac: reader = csv.DictReader(ac) for row in reader: - data.append(dict( \ - (('name', row['name']), \ - ('fullname', row['fullname']), \ + data.append(dict( + (('name', row['name']), + ('fullname', row['fullname']), ('emails', row['emails'])))) - for i in data: if i.get('name', None) == name: if i['emails']: From 7dc2b1c8092146c3004b2cec25375f713ec7d8af Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 09:54:55 +0000 Subject: [PATCH 4/6] Removed REPO_NAME env variable from github import --- diff --git a/pagure_importer/app.py b/pagure_importer/app.py index 303088f..680de1c 100644 --- a/pagure_importer/app.py +++ b/pagure_importer/app.py @@ -2,7 +2,6 @@ import click import tempfile -REPO_NAME = '' REPO_PATH = tempfile.gettempdir() diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index 26ae4fa..13f80e7 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -1,7 +1,7 @@ import click import getpass -from pagure_importer.app import app, REPO_NAME, REPO_PATH +from pagure_importer.app import app, REPO_PATH from pagure_importer.utils.importer_github import GithubImporter from pagure_importer.utils import ( generate_json_for_github_contributors, @@ -16,17 +16,21 @@ def form_github_issues(): github_project_name = raw_input('Enter github project name like: "pypingou/pagure" without quotes: ') return (github_username, github_password, github_project_name) + @app.command() def github(): github_username, github_password, github_project_name = form_github_issues() gen_json = raw_input( - 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') + 'Do you want to generate jsons for project\'s contributers and issue commentors? (y/n): ') if gen_json == 'n': github_importer = GithubImporter( github_username=github_username, github_password=github_password, github_project_name=github_project_name) - github_importer.import_issues(repo_path=REPO_NAME, repo_folder=REPO_PATH) + + pagure_importer.utils.display_repo() + repo_name = raw_input('Choose the import destination repo : ') + github_importer.import_issues(repo_path=repo_name, repo_folder=REPO_PATH) else: generate_json_for_github_contributors( github_username, From aa5af06cc390c096651bd09bbd127339c5e02d64 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 10:53:45 +0000 Subject: [PATCH 5/6] Updated README with pgimport clone usage --- diff --git a/README.md b/README.md index 4e8c209..acd5132 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,11 @@ CLI tool for importing issues etc. from different sources like github to pagure ## How to run --- -0. Clone the issue tracker for issues from pagure. Use: ```git clone --bare``` -1. set the env variables: ```REPO_NAME``` and ```REPO_PATH``` -ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ -2. Activate the pagure tickets hook from project settings. -3. Execute ```pgimport```. See Usage section -4. Just answer what is asked. Check below instructions for particular source -5. The script will make commits in your cloned bare repo: push the changes back to pagure. +0. Clone the issue tracker for issues from pagure. Use: ```pgimport clone ssh://git@pagure.io/tickets/foobar.git``` +1. Activate the pagure tickets hook from project settings. +2. Execute ```pgimport```. See Usage section +3. Just answer what is asked. Check below instructions for particular source +4. The script will make commits in your cloned bare repo: push the changes back to pagure. ## Usage @@ -27,13 +25,26 @@ ex: REPO_NAME=abc.git; REPO_PATH=/home/vivek/ --help Show this message and exit. Commands: + clone fedorahosted github +The clone command can be used to clone the pagure ticket repository: + + $ pgimport clone ssh://git@pagure.io/tickets/foobar.git + The fedorahosted command can be used to import issues from a fedorahosted project to pagure + + $ pgimport fedorahosted --help + Usage: pgimport fedorahosted [OPTIONS] PROJECT_URL + + Options: + --tags Import pagure tags: + --help Show this message and exit. + - $ pgimport fedorahosted https://fedorahosted.org/fedocal + $ pgimport fedorahosted https://fedorahosted.org/foobar The github command can be used to import issues from a github project to pagure From 70a4f8df8fd8604a32654953849c0cc4dcc91acf Mon Sep 17 00:00:00 2001 From: Clement Verna Date: May 30 2016 11:48:24 +0000 Subject: [PATCH 6/6] Fixed missing import for pagure_importer.utils.display_repo() --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 8955160..9c50992 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,6 +1,7 @@ import click import os import getpass +import pagure_importer from pagure_importer.app import app, REPO_PATH from pagure_importer.utils import importer_trac from pagure_importer.utils.fas import FASclient