From 96c928ba79d0d0ca851a79475f2cc95be1ca7054 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: Use attachments folder as local cache asn taskify adding attachments to git Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/issue.py b/pagure/api/issue.py index 140dc88..c8eccd7 100644 --- a/pagure/api/issue.py +++ b/pagure/api/issue.py @@ -227,10 +227,10 @@ def api_new_issue(repo, username=None, namespace=None): # If there is a file attached, attach it. filestream = flask.request.files.get('filestream') if filestream and '' in issue.content: - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 8f4d0d9..df3a339 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -22,6 +22,7 @@ except ImportError: # pragma: no cover import json import datetime +import hashlib import logging import markdown import os @@ -2493,6 +2494,39 @@ def reset_status_pull_request(session, project): session.commit() +def add_attachment(repo, issue, attachmentfolder, user, filename, filestream): + ''' Add a file to the attachments folder of repo and update git. ''' + _log.info( + 'Addinf file: %s to the git repo: %s', + repo.path, werkzeug.secure_filename(filename)) + + # Prefix the filename with a timestamp: + filename = '%s-%s' % ( + hashlib.sha256(filestream.read()).hexdigest(), + werkzeug.secure_filename(filename) + ) + filedir = os.path.join(attachmentfolder, repo.fullname) + filepath = os.path.join(filedir, filename) + + if os.path.exists(filepath): + return filename + + if not os.path.exists(filedir): + os.makedirs(filedir) + + # Write file + filestream.seek(0) + with open(filepath, 'w') as stream: + stream.write(filestream.read()) + + tasks.add_file_to_git.delay( + repo.name, repo.namespace, + repo.user.username if repo.is_fork else None, + user, issue.uid, filename) + + return filename + + def get_issue_statuses(session): ''' Return the complete list of status an issue can have. ''' @@ -2670,7 +2704,6 @@ def avatar_url_from_email(email, size=64, default='retro', dns=False): ) else: import urllib - import hashlib query = urllib.urlencode({'s': size, 'd': default}) hashhex = hashlib.sha256(email).hexdigest() return "https://seccdn.libravatar.org/avatar/%s?%s" % ( diff --git a/pagure/lib/git.py b/pagure/lib/git.py index a695ce1..18a8522 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -826,30 +826,19 @@ def update_request_from_git( session.commit() -def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): +def _add_file_to_git(repo, issue, attachmentfolder, ticketfolder, user, + filename): ''' Add a given file to the specified ticket git repository. :arg repo: the Project object from the database + :arg attachmentfolder: the folder on the filesystem where the attachments + are stored :arg ticketfolder: the folder on the filesystem where the git repo for tickets are stored :arg user: the user object with its username and email :arg filename: the name of the file to save - :arg filestream: the actual content of the file ''' - _log.info( - 'Addinf file: %s to the git repo: %s', - repo.path, werkzeug.secure_filename(filename)) - - if not ticketfolder: - return - - # Prefix the filename with a timestamp: - filename = '%s-%s' % ( - hashlib.sha256(filestream.read()).hexdigest(), - werkzeug.secure_filename(filename) - ) - # Get the fork repopath = os.path.join(ticketfolder, repo.path) @@ -864,10 +853,7 @@ def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): index = new_repo.index # Are we adding files - added = False - if not os.path.exists(file_path): - added = True - else: + if os.path.exists(file_path): # File exists, remove the clone and return shutil.rmtree(newpath) return os.path.join('files', filename) @@ -875,26 +861,19 @@ def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): if not os.path.exists(folder_path): os.mkdir(folder_path) - # Write down what changed - filestream.seek(0) - with open(file_path, 'w') as stream: - stream.write(filestream.read()) + # Copy from attachments directory + src = os.path.join(attachmentfolder, repo.fullname, filename) + shutil.copyfile(src, file_path) # Retrieve the list of files that changed diff = new_repo.diff() files = [patch.new_file_path for patch in diff] # Add the changes to the index - if added: - index.add(os.path.join('files', filename)) + index.add(os.path.join('files', filename)) for filename in files: index.add(filename) - # If not change, return - if not files and not added: - shutil.rmtree(newpath) - return - # See if there is a parent to this commit parent = None try: diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 7e449d0..e035ad1 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -19,11 +19,15 @@ import pygit2 import tempfile import six +import logging + import pagure from pagure import APP import pagure.lib import pagure.lib.git +_log = logging.getLogger(__name__) + conn = Celery('tasks', broker='redis://%s' % APP.config['REDIS_HOST'], @@ -335,3 +339,18 @@ def merge_pull_request(name, namespace, user, requestid, user_merger): refresh_pr_cache.delay(name, namespace, user) session.remove() return ret('view_repo', repo=name, username=user, namespace=namespace) + + +@conn.task +def add_file_to_git(name, namespace, user, user_attacher, issueuid, filename): + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, + name=name, user=user) + issue = pagure.lib.get_issue_by_uid(session, issueuid) + + pagure.lib.git._add_file_to_git( + project, issue, APP.config['ATTACHMENTS_FOLDER'], + APP.config['TICKETS_FOLDER'], user_attacher, filename) + + session.remove() diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 7d1687c..9c897b4 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -904,10 +904,10 @@ def new_issue(repo, username=None, namespace=None): # If there is a file attached, attach it. filestream = flask.request.files.get('filestream') if filestream and '' in issue.content: - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, @@ -1164,10 +1164,10 @@ def edit_issue(repo, issueid, username=None, namespace=None): # If there is a file attached, attach it. filestream = flask.request.files.get('filestream') if filestream and '' in issue.content: - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, @@ -1250,10 +1250,10 @@ def upload_issue(repo, issueid, username=None, namespace=None): if form.validate_on_submit(): filestream = flask.request.files['filestream'] - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, @@ -1291,33 +1291,48 @@ def view_issue_raw_file( if not repo.settings.get('issue_tracker', True): flask.abort(404, 'No issue tracker found for this project') - reponame = os.path.join(APP.config['TICKETS_FOLDER'], repo.path) + mimetype, encoding = mimetypes.guess_type(filename) - repo_obj = pygit2.Repository(reponame) + attachdir = os.path.join(APP.config['ATTACHMENTS_FOLDER'], repo.fullname) + attachpath = os.path.join(attachdir, filename) + if not os.path.exists(attachpath): + if not os.path.exists(attachdir): + os.makedirs(attachdir) - if repo_obj.is_empty: - flask.abort(404, 'Empty repo cannot have a file') + # Try to copy from git repo to attachments folder + reponame = os.path.join(APP.config['TICKETS_FOLDER'], repo.path) + repo_obj = pygit2.Repository(reponame) - branch = repo_obj.lookup_branch('master') - commit = branch.get_object() + if repo_obj.is_empty: + flask.abort(404, 'Empty repo cannot have a file') - mimetype = None - encoding = None + branch = repo_obj.lookup_branch('master') + commit = branch.get_object() - content = __get_file_in_tree( - repo_obj, commit.tree, filename.split('/'), bail_on_tree=True) - if not content or isinstance(content, pygit2.Tree): - flask.abort(404, 'File not found') + content = __get_file_in_tree( + repo_obj, commit.tree, ['files', filename], bail_on_tree=True) + if not content or isinstance(content, pygit2.Tree): + flask.abort(404, 'File not found') - mimetype, encoding = mimetypes.guess_type(filename) - data = repo_obj[content.oid].data + data = repo_obj[content.oid].data + + if not data: + flask.abort(404, 'No content found') + + _log.info("Migrating file %s for project %s to attachments", + filename, repo.fullname) + + with open(attachpath, 'w') as stream: + stream.write(data) + data = None - if not data: - flask.abort(404, 'No content found') + # At this moment, attachpath exists and points to the file + with open(attachpath, 'r') as f: + data = f.read() if not raw \ and (filename.endswith('.patch') or filename.endswith('.diff')) \ - and not is_binary_string(content.data): + and not is_binary_string(data): # We have a patch file attached to this issue, render the diff in html orig_filename = filename.partition('-')[2] return flask.render_template(