From d8fe8881cc35bc0a97aca2a25958c7beabde5a14 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Oct 03 2016 19:17:52 +0000 Subject: some beautification in our code --- diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index c986260..37c4fb6 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -19,7 +19,8 @@ from pagure_importer.utils import ( help="Github project like pypingou/pagure") def github(username, password, project): gen_json = click.confirm( - "Do you want to generate jsons for project's contributers and issue commentors?") + "Do you want to generate jsons for project's contributers" + " and issue commentors?") if gen_json: generate_json_for_github_contributors( username, diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 585895b..5debda2 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -14,6 +14,9 @@ from pagure_importer.app import REPO_PATH def create_auth_token(github): + ''' Creates github authentication token. If Two Factor Authentication + is enabled, the user will be asked to enter the key ''' + user = github.get_user() try: otp_auth = user.create_authorization(scopes=['user'], @@ -27,6 +30,9 @@ def create_auth_token(github): def get_auth_token(github): + ''' Checks the .pgimport file for github authentication key, + if it is not present, creates it. ''' + cfg_path = os.path.join(os.environ.get('HOME'), '.pgimport') if os.path.exists(cfg_path): parser = ConfigParser.RawConfigParser() @@ -41,6 +47,8 @@ def get_auth_token(github): def display_repo(): + ''' Displays the list of repos elegantly ''' + repo = [] index = 0 click.secho('#### Repo available ####', fg='blue') @@ -49,7 +57,7 @@ def display_repo(): index += 1 click.echo(str(index) + ' - ' + file) repo.append(file) - print + click.echo() return repo @@ -71,7 +79,8 @@ def generate_json_for_github_contributors(github_username, while True: page += 1 payload = {'page': page} - data_ = json.loads(requests.get(commits_url, params=payload, + data_ = json.loads(requests.get( + commits_url, params=payload, auth=HTTPBasicAuth(github_username, github_password)).text) if not data_: @@ -128,8 +137,9 @@ def generate_json_for_github_issue_commentors(github_username, while True: page += 1 payload = {'page': page} - data_ = json.loads(requests.get(issue_comment_url, params=payload, - auth=HTTPBasicAuth(github_username, github_password)).text) + data_ = json.loads(requests.get( + issue_comment_url, params=payload, + auth=HTTPBasicAuth(github_username, github_password)).text) if not data_: break @@ -196,8 +206,9 @@ def github_get_commentor_email(name): ''' if not os.path.exists('assembled_commentors.csv'): - raise FileNotFound('The assembled_commentors.json file must be present \ - Rerun the program and choose to generate the json files') + raise FileNotFound('The assembled_commentors.json file must be present' + ' Rerun the program and choose to generate the json' + ' files') data = [] with open('assembled_commentors.csv') as ac: @@ -205,13 +216,13 @@ def github_get_commentor_email(name): for row in reader: data.append(dict( (('name', row['name']), - ('fullname', row['fullname']), - ('emails', row['emails'])))) + ('fullname', row['fullname']), + ('emails', row['emails'])))) for i in data: if i.get('name', None) == name: if i['emails']: return str(i['emails']) else: - raise EmailNotFound('You need to fill out all the emails of the \ - issue commentors') + raise EmailNotFound('You need to fill out all the emails of' + ' the issue commentors') diff --git a/pagure_importer/utils/fas.py b/pagure_importer/utils/fas.py index f62b5f5..24b04ae 100644 --- a/pagure_importer/utils/fas.py +++ b/pagure_importer/utils/fas.py @@ -3,7 +3,11 @@ from pagure_importer.utils.models import User class FASclient (): + ''' Creates a FAS User object based on the credentials given ''' + def __init__(self, fas_username, fas_password, fas_url): + ''' Instantiate a FASclient object ''' + self.fasclient = AccountSystem(fas_url, username=fas_username, password=fas_password) @@ -11,6 +15,7 @@ class FASclient (): self.fasuser = {'': anonymous} def find_fas_user(self, user): + ''' Queries FAS and returns the FAS user ''' if user not in self.fasuser.keys(): person = self.fasclient.person_by_username(user) diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py index c1cfb4a..fafbf06 100644 --- a/pagure_importer/utils/git.py +++ b/pagure_importer/utils/git.py @@ -3,6 +3,7 @@ Modified by Clement Verna to add attachment support ''' + import shutil import os import pygit2 @@ -10,16 +11,20 @@ import json import hashlib import werkzeug -from repo import * +from repo import PagureRepo def get_secure_filename(attachment, filename): + ''' Hashes the file name, same as pagure ''' + filename = '%s-%s' % (hashlib.sha256(str(attachment)).hexdigest(), werkzeug.secure_filename(str(filename))) return filename def clone_repo(repo_name, repo_folder): + ''' Clone the original repo where the commits will be made + before pushing back ''' if not repo_folder: return @@ -34,6 +39,8 @@ def clone_repo(repo_name, repo_folder): def push_delete_repo(newpath, new_repo): + ''' Push the changes to the originally cloned repo from pagure and delete + the cloned repo where the commits were going ''' # Push to origin ori_remote = new_repo.remotes[0] @@ -63,7 +70,7 @@ def update_git(obj, newpath, new_repo): added = True # If we have attachments - attachments = obj.get_attachment() + attachments = obj.attachment if attachments: if not os.path.exists(os.path.join(newpath, 'files')): os.mkdir(os.path.join(newpath, 'files')) diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index e4fe435..8fefaaa 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -11,7 +11,10 @@ from pagure_importer.utils.exceptions import ( class GithubImporter(): ''' Imports from Github using PyGithub and libpagure ''' + def __init__(self, username, password, project): + ''' Instantiate GithubImporter object ''' + self.github_username = username self.github_password = password self.github_project_name = project @@ -58,7 +61,7 @@ class GithubImporter(): pagure_issue_tags = [] pagure_issue_milestone = github_issue.milestone.title \ - if github_issue.milestone else None + if github_issue.milestone else None # few things not supported by github pagure_issue_depends = [] @@ -91,10 +94,8 @@ class GithubImporter(): for comment in github_issue.get_comments(): comment_user = comment.user - pagure_issue_comment_user_email = comment_user.email pagure_issue_comment_body = comment.body pagure_issue_comment_created_at = comment.created_at.strftime('%s') - pagure_issue_comment_updated_at = comment.updated_at.strftime('%s') # No idea what to do with this right now # editor: not supported by github api @@ -108,8 +109,8 @@ class GithubImporter(): pagure_issue_comment_user = models.User( name=comment_user.login, fullname=comment_user.name, - emails=[comment_user.email] if comment_user.email \ - else [github_get_commentor_email(comment_user.login)]) + emails=[comment_user.email] if comment_user.email + else [github_get_commentor_email(comment_user.login)]) # Object to represent comment on an issue pagure_issue_comment = models.IssueComment( diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index a4fc751..1a42004 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,8 +1,9 @@ -import requests +import sys import time import base64 -import sys import click +import requests + from datetime import datetime from pagure_importer.utils.git import ( clone_repo, get_secure_filename, push_delete_repo, update_git) @@ -10,10 +11,12 @@ from pagure_importer.utils.models import User, Issue, IssueComment class TracImporter(): - '''Pagure importer for trac instance''' + ''' Pagure importer for trac instance ''' def __init__(self, project_url, username, password, fasclient=None, tags=False, private=False): + ''' Instantiate a TracImporter object ''' + self.url = project_url self.username = username self.password = password @@ -25,6 +28,8 @@ class TracImporter(): self.reqid = 0 def request(self, method, *args): + ''' Common method for querying trac ''' + self.reqid += 1 req = {'params': args, 'method': method, @@ -35,13 +40,15 @@ class TracImporter(): if resp['id'] != self.reqid: click.echo('ERROR: Invalid response for request! ID does not match') sys.exit(1) - if resp['error'] != None: + if resp['error'] is not None: click.echo("ERROR: Error in response: %s" % resp['error']) sys.exit(1) return resp['result'] def to_timestamp(self, tm): + ''' Convert to timestamp which can be jsonified ''' + tm = tm.replace('+00:00', '') date = datetime.strptime(tm, '%Y-%m-%dT%H:%M:%S') ts = str(time.mktime(date.timetuple()))[:-2] # Strip the .0 @@ -49,7 +56,8 @@ class TracImporter(): def import_issues(self, repo_name, repo_folder, trac_query='max=0&order=id'): - '''Import issues from trac instance using xmlrpc API''' + ''' Import issues from trac instance using jsonrpc API ''' + newpath, new_repo = clone_repo(repo_name, repo_folder) tickets_id = self.request('ticket.query', trac_query) @@ -75,6 +83,7 @@ class TracImporter(): push_delete_repo(newpath, new_repo) def create_issue(self, ticket_id): + ''' Create Issue object from track ticket ''' trac_ticket_info = self.request('ticket.get', ticket_id) trac_ticket = trac_ticket_info[3] @@ -83,7 +92,9 @@ class TracImporter(): pagure_attachment = {} for attachment in trac_attachments: filename = attachment[0] - content = self.request('ticket.getAttachment', ticket_id, filename)['__jsonclass__'][1].replace('\n', '') + content = self.request( + 'ticket.getAttachment', + ticket_id, filename)['__jsonclass__'][1].replace('\n', '') pagure_attachment[filename] = base64.b64decode(content) pagure_issue_title = trac_ticket['summary'] @@ -94,7 +105,8 @@ class TracImporter(): pagure_issue_status = self.get_ticket_status(trac_ticket) - pagure_issue_created_at = self.to_timestamp(trac_ticket_info[1]['__jsonclass__'][1]) + pagure_issue_created_at = self.to_timestamp( + trac_ticket_info[1]['__jsonclass__'][1]) if self.fas: pagure_issue_assignee = self.fas.find_fas_user( @@ -147,6 +159,7 @@ class TracImporter(): return pagure_issue def get_ticket_status(self, trac_ticket): + ''' Returns the corresponding status of ticket on pagure ''' if trac_ticket['status'] != 'closed': ticket_status = 'Open' @@ -155,6 +168,8 @@ class TracImporter(): return ticket_status def get_comment_user(self, comment): + ''' Returns the user who commented on the ticket ''' + # The User who commented if self.fas and comment[1]: pagure_issue_comment_user = self.fas.find_fas_user(comment[1]) @@ -165,6 +180,8 @@ class TracImporter(): return pagure_issue_comment_user def create_comments(self, trac_comments): + ''' Create IssueComment objects from the trac comments ''' + comments = {} for comment in trac_comments: ts = self.to_timestamp(comment[0]['__jsonclass__'][1]) diff --git a/pagure_importer/utils/models.py b/pagure_importer/utils/models.py index 77f9add..c4c4e4e 100644 --- a/pagure_importer/utils/models.py +++ b/pagure_importer/utils/models.py @@ -48,9 +48,6 @@ class Issue(): return output - def get_attachment(self): - return self.attachment - @property def isa(self): return 'issue'