From ded1fa294a6a9813e7926cfaa49d636e4f775e87 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 07 2018 15:20:01 +0000 Subject: [PATCH 1/4] Add the ability to generate archive from a commit or tag This commit adds a new UI endpoint that is able to generate an archive for a specified commit or tag in a repository. Fixes https://pagure.io/pagure/issue/861 Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index d734925..99eac95 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -23,6 +23,8 @@ import shutil import subprocess import requests import tempfile +import tarfile +import zipfile import arrow import pygit2 @@ -867,13 +869,12 @@ class TemporaryClone(object): _project = None _action = None _repotype = None - _origpath = None - + _origrepopath = None repopath = None repo = None - def __init__(self, project, repotype, action, path=None): + def __init__(self, project, repotype, action, path=None, parent=None): """ Initializes a TempoaryClone instance. Args: @@ -882,6 +883,12 @@ class TemporaryClone(object): main, docs, requests, tickets action (string): Type of action performing, used in the temporary directory name + path (string or None): the path to clone, allows cloning, for + example remote git repo for remote PRs instead of the + default one + parent (string or None): Adds this directory to the path in + which the project is cloned + """ if repotype not in pagure.lib.query.get_repotypes(): raise NotImplementedError("Repotype %s not known" % repotype) @@ -890,10 +897,15 @@ class TemporaryClone(object): self._repotype = repotype self._action = action self._path = path + self._parent = parent def __enter__(self): """ Enter the context manager, creating the clone. """ self.repopath = tempfile.mkdtemp(prefix="pagure-%s-" % self._action) + self._origrepopath = self.repopath + if self._parent: + self.repopath = os.path.join(self.repopath, self._parent) + os.makedirs(self.repopath) if not self._project.is_on_repospanner: # This is the simple case. Just do a local clone # use either the specified path or the use the path of the @@ -2632,3 +2644,77 @@ def get_stats_patch(patch): ) return output + + +def generate_archive(project, commit, tag, name, archive_fmt): + """ Generate the desired archive of the specified project for the + specified commit with the given name and archive format. + + Args: + project (pagure.lib.model.Project): the project's repository from + which to generate the archive + commit (str): the commit hash to generate the archive of + name (str): the name to give to the archive + archive_fmt (str): the format of the archive to generate, can be + either gzip or tag or tar.gz + Returns: None + Raises (pagure.exceptions.PagureException): if an un-supported archive + format is specified + + """ + + def _exclude_git(filename): + return ".git" in filename + + with TemporaryClone(project, "main", "archive", parent=name) as tempclone: + repo_obj = tempclone.repo + commit_obj = repo_obj[commit] + repo_obj.checkout_tree(commit_obj.tree) + archive_folder = pagure_config.get("ARCHIVE_FOLDER") + + tag_path = "" + if tag: + tag_path = os.path.join("tags", tag) + target_path = os.path.join( + archive_folder, project.fullname, tag_path, commit + ) + if not os.path.exists(target_path): + os.makedirs(target_path) + fullpath = os.path.join(target_path, name) + + if archive_fmt == "tar": + with tarfile.open(name=fullpath + ".tar", mode="w") as tar: + tar.add( + name=tempclone.repopath, exclude=_exclude_git, arcname=name + ) + elif archive_fmt == "tar.gz": + with tarfile.open(name=fullpath + ".tar.gz", mode="w:gz") as tar: + tar.add( + name=tempclone.repopath, exclude=_exclude_git, arcname=name + ) + elif archive_fmt == "zip": + # Code from /usr/lib64/python2.7/zipfile.py adjusted for our + # needs + def addToZip(zf, path, zippath): + if _exclude_git(path): + return + if os.path.isfile(path): + zf.write(path, zippath, zipfile.ZIP_DEFLATED) + elif os.path.isdir(path): + if zippath: + zf.write(path, zippath) + for nm in os.listdir(path): + if _exclude_git(path): + continue + addToZip( + zf, + os.path.join(path, nm), + os.path.join(zippath, nm), + ) + + with zipfile.ZipFile(fullpath + ".zip", "w") as zipstream: + addToZip(zipstream, tempclone.repopath, name) + else: + raise pagure.exceptions.PagureException( + "Un-support archive format requested: %s", archive_fmt + ) diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 051d785..7a155a8 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -1172,3 +1172,43 @@ def git_garbage_collect(self, session, repopath): # https://github.com/libgit2/libgit2/issues/3247 _log.info("Running 'git gc --auto' for repo %s", repopath) subprocess.check_output(["git", "gc", "--auto", "-q"], cwd=repopath) + + +@conn.task(queue=pagure_config.get("FAST_CELERY_QUEUE", None), bind=True) +@pagure_task +def generate_archive( + self, session, project, namespace, username, commit, tag, name, archive_fmt +): + """ Generate the archive of the specified project on the specified + commit with the given name and archive format. + Currently only support the following format: gzip and tar.gz + + """ + project = pagure.lib.query._get_project( + session, namespace=namespace, name=project, user=username + ) + + _log.debug( + "Generating archive for %s, commit: %s as: %s.%s", + project.fullname, + commit, + name, + archive_fmt, + ) + + pagure.lib.git.generate_archive(project, commit, tag, name, archive_fmt) + + if archive_fmt == "gzip": + endpoint = "ui_ns.get_project_archive_gzip" + elif archive_fmt == "tar": + endpoint = "ui_ns.get_project_archive_tar" + else: + endpoint = "ui_ns.get_project_archive_tar_gz" + return ret( + endpoint, + repo=project.name, + ref=commit, + name=name, + namespace=project.namespace, + username=project.user.user if project.is_fork else None, + ) diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 537d84a..e068eeb 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -3354,3 +3354,160 @@ def edit_tag(repo, tag, username=None, namespace=None): return flask.render_template( "edit_tag.html", username=username, repo=repo, form=form, tagname=tag ) + + +@UI_NS.route("//archive//.tar") +@UI_NS.route("///archive//.tar") +@UI_NS.route("/fork///archive//.tar") +@UI_NS.route("/fork////archive//.tar") +def get_project_archive_tar(repo, ref, name, namespace=None, username=None): + """ Generate an archive or redirect the user to where it already exists + """ + + return generate_project_archive( + repo, + ref, + name, + extension="tar", + namespace=namespace, + username=username, + ) + + +@UI_NS.route("//archive//.tar.gz") +@UI_NS.route("///archive//.tar.gz") +@UI_NS.route("/fork///archive//.tar.gz") +@UI_NS.route("/fork////archive//.tar.gz") +def get_project_archive_tar_gz(repo, ref, name, namespace=None, username=None): + """ Generate an archive or redirect the user to where it already exists + """ + return generate_project_archive( + repo, + ref, + name, + extension="tar.gz", + namespace=namespace, + username=username, + ) + + +@UI_NS.route("//archive//.zip") +@UI_NS.route("///archive//.zip") +@UI_NS.route("/fork///archive//.zip") +@UI_NS.route("/fork////archive//.zip") +def get_project_archive_zip(repo, ref, name, namespace=None, username=None): + """ Generate an archive or redirect the user to where it already exists + """ + return generate_project_archive( + repo, + ref, + name, + extension="zip", + namespace=namespace, + username=username, + ) + + +def generate_project_archive( + repo, ref, name, extension, namespace=None, username=None +): + """ Generate an archive or redirect the user to where it already + exists. + """ + + archive_folder = pagure_config.get("ARCHIVE_FOLDER") + + if not archive_folder: + _log.debug("No ARCHIVE_FOLDER specified in the configuration") + flask.abort( + 404, + "This pagure instance isn't configured to support this feature") + if not os.path.exists(archive_folder): + _log.debug("No ARCHIVE_FOLDER could not be found on disk") + flask.abort( + 500, + "Incorrect configuration, please contact your admin") + + extensions = ["tar.gz", "tar", "zip"] + if extension not in extensions: + _log.debug("%s no in %s", extension, extensions) + flask.abort(400, "Invalid archive format specified") + + name = werkzeug.secure_filename(name) + + repo_obj = flask.g.repo_obj + + ref_string = "refs/tags/%s" % ref + + commit = None + tag = None + if ref_string in repo_obj.listall_references(): + reference = repo_obj.lookup_reference(ref_string) + tag = repo_obj[reference.target] + if not isinstance(tag, pygit2.Tag): + flask.abort(400, "Invalid reference provided") + commit = tag.get_object() + else: + try: + commit = repo_obj.get(ref) + except ValueError: + flask.abort(404, "Invalid commit provided") + + if not isinstance(commit, pygit2.Commit): + flask.abort(400, "Invalid reference specified") + + tag_path = "" + tag_filename = None + if tag: + tag_filename = werkzeug.secure_filename(ref) + tag_path = os.path.join("tags", tag_filename) + + path = os.path.join( + archive_folder, + flask.g.repo.fullname, + tag_path, + commit.oid.hex, + "%s.%s" % (name, extension), + ) + headers = { + str("Content-Disposition"): "attachment", + str("Content-Type"): "application/x-gzip", + } + if os.path.exists(path): + + def _send_data(): + with open(path, "rb") as stream: + yield stream.read() + + _log.info("Sending the existing archive") + return flask.Response( + flask.stream_with_context(_send_data()), headers=headers + ) + + _log.info("Re-generating the archive") + task = pagure.lib.tasks.generate_archive.delay( + repo, + namespace=namespace, + username=username, + commit=commit.oid.hex, + tag=tag_filename, + name=name, + archive_fmt=extension, + ) + + def _wait_for_task_and_send_data(): + while not task.ready(): + import time + + _log.info("waiting") + time.sleep(0.5) + with open(path, "rb") as stream: + yield stream.read() + + _log.info("Sending the existing archive") + return flask.Response( + flask.stream_with_context(_wait_for_task_and_send_data()), + headers=headers, + ) + + return pagure.utils.wait_for_task(task) diff --git a/tests/test_pagure_flask_ui_archives.py b/tests/test_pagure_flask_ui_archives.py new file mode 100644 index 0000000..3755421 --- /dev/null +++ b/tests/test_pagure_flask_ui_archives.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2018 - Copyright Red Hat Inc + + Authors: + Clement Verna + +""" + +from __future__ import unicode_literals + +import unittest +import sys +import os +import time + +import mock +import pygit2 + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) + +import pagure.lib.git +import pagure.lib.query +import tests +from tests.test_pagure_lib_git_get_tags_objects import add_repo_tag + + +class PagureFlaskUiArchivesTest(tests.Modeltests): + """ Tests checking the archiving mechanism. """ + + + def setUp(self): + """ Set up the environnment, ran before every tests. """ + super(PagureFlaskUiArchivesTest, self).setUp() + tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) + project = pagure.lib.query._get_project(self.session, 'test') + + # test has both commits and tags + repopath = os.path.join(self.path, 'repos', 'test.git') + tests.add_readme_git_repo(repopath) + repo = pygit2.Repository(repopath) + add_repo_tag(self.path, repo, ['v1.0', 'v1.1'], 'test.git') + + # test2 has only commits + tests.add_readme_git_repo(os.path.join( + self.path, 'repos', 'test2.git')) + + # somenamespace/test3 has neither commits nor tags + + # Create the archive folder: + self.archive_path = os.path.join(self.path, 'archives') + os.mkdir(self.archive_path) + + def test_project_no_conf(self): + """ Test getting the archive when pagure isn't configured. """ + output = self.app.get( + '/somenamespace/test3/archive/tag1/test3-tag1.zip', + follow_redirects=True) + + self.assertEqual(output.status_code, 404) + self.assertIn( + "This pagure instance isn't configured to support " + "this feature", output.get_data(as_text=True)) + + self.assertEqual(os.listdir(self.archive_path), []) + + def test_project_invalid_conf(self): + """ Test getting the archive when pagure is wrongly configured. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'invalid')}): + output = self.app.get( + '/somenamespace/test3/archive/tag1/test3-tag1.zip', + follow_redirects=True) + + self.assertEqual(output.status_code, 500) + self.assertIn( + "Incorrect configuration, please contact your admin", + output.get_data(as_text=True)) + + self.assertEqual(os.listdir(self.archive_path), []) + + def test_project_invalid_format(self): + """ Test getting the archive when the format provided is invalid. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/somenamespace/test3/archive/tag1/test3-tag1.unzip', + follow_redirects=True) + + self.assertEqual(output.status_code, 404) + + self.assertEqual(os.listdir(self.archive_path), []) + + def test_project_no_commit(self): + """ Test getting the archive of an empty project. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/somenamespace/test3/archive/tag1/test3-tag1.zip', + follow_redirects=True) + + self.assertEqual(output.status_code, 404) + self.assertIn( + "

Invalid commit provided

", + output.get_data(as_text=True)) + + self.assertEqual(os.listdir(self.archive_path), []) + + def test_project_no_tag(self): + """ Test getting the archive of a non-empty project but without + tags. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/test2/archive/tag1/test2-tag1.zip', + follow_redirects=True) + + self.assertEqual(output.status_code, 404) + self.assertIn( + "

Invalid commit provided

", + output.get_data(as_text=True)) + + self.assertEqual(os.listdir(self.archive_path), []) + + def test_project_no_tag(self): + """ Test getting the archive of an empty project. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/test2/archive/tag1/test2-tag1.zip', + follow_redirects=True) + + self.assertEqual(output.status_code, 404) + self.assertIn( + "

Invalid commit provided

", + output.get_data(as_text=True)) + + self.assertEqual(os.listdir(self.archive_path), []) + + def test_project_w_tag_zip(self): + """ Test getting the archive from a tag. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/test/archive/v1.0/test-v1.0.zip', + follow_redirects=True) + + self.assertEqual(output.status_code, 200) + + self.assertEqual( + os.listdir(self.archive_path), ['test']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test')), + ['tags']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test', 'tags')), + ['v1.0']) + + self.assertEqual( + len(os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0'))), + 1) + + files = os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0')) + self.assertEqual( + os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0', files[0])), + ['test-v1.0.zip']) + + def test_project_w_tag_tar(self): + """ Test getting the archive from a tag. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/test/archive/v1.0/test-v1.0.tar', + follow_redirects=True) + + self.assertEqual(output.status_code, 200) + + self.assertEqual( + os.listdir(self.archive_path), ['test']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test')), + ['tags']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test', 'tags')), + ['v1.0']) + + self.assertEqual( + len(os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0'))), + 1) + + files = os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0')) + self.assertEqual( + os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0', files[0])), + ['test-v1.0.tar']) + + def test_project_w_tag_tar_gz(self): + """ Test getting the archive from a tag. """ + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/test/archive/v1.0/test-v1.0.tar.gz', + follow_redirects=True) + + self.assertEqual(output.status_code, 200) + + self.assertEqual( + os.listdir(self.archive_path), ['test']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test')), + ['tags']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test', 'tags')), + ['v1.0']) + + self.assertEqual( + len(os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0'))), + 1) + + files = os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0')) + self.assertEqual( + os.listdir(os.path.join( + self.archive_path, 'test', 'tags', 'v1.0', files[0])), + ['test-v1.0.tar.gz']) + + def test_project_w_commit_tar_gz(self): + """ Test getting the archive from a commit. """ + repopath = os.path.join(self.path, 'repos', 'test.git') + repo = pygit2.Repository(repopath) + commit = repo.head.target.hex + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/test/archive/%s/test-v1.0.tar.gz' % commit, + follow_redirects=True) + + self.assertEqual(output.status_code, 200) + + self.assertEqual( + os.listdir(self.archive_path), ['test']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test')), + [commit]) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test', commit)), + ['test-v1.0.tar.gz']) + + def test_project_w_commit_tar_gz_twice(self): + """ Test getting the archive from a commit twice, so we hit the + disk cache. """ + repopath = os.path.join(self.path, 'repos', 'test.git') + repo = pygit2.Repository(repopath) + commit = repo.head.target.hex + with mock.patch.dict( + 'pagure.config.config', + {'ARCHIVE_FOLDER': os.path.join(self.path, 'archives')}): + output = self.app.get( + '/test/archive/%s/test-v1.0.tar.gz' % commit, + follow_redirects=True) + + self.assertEqual(output.status_code, 200) + + output = self.app.get( + '/test/archive/%s/test-v1.0.tar.gz' % commit, + follow_redirects=True) + + self.assertEqual(output.status_code, 200) + + self.assertEqual( + os.listdir(self.archive_path), ['test']) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test')), + [commit]) + self.assertEqual( + os.listdir(os.path.join(self.archive_path, 'test', commit)), + ['test-v1.0.tar.gz']) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 551326f1a4bce18818d323124b8038897eaccd2f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 07 2018 15:20:01 +0000 Subject: [PATCH 2/4] Add download options on the release page Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/releases.html b/pagure/templates/releases.html index 4baec5d..1fb40bd 100644 --- a/pagure/templates/releases.html +++ b/pagure/templates/releases.html @@ -73,25 +73,64 @@ {% endif %}
- - {{ tag['object'].oid | short }} + + {{ tag['object'].oid | short }} + +
+ + + +
+ +
+ + -
- +
+
  • +
      + - - + ref=tag['tagname'], + name='%s-%s' % (repo.name, tag['tagname'])) + }}" target="_blank" rel="noopener noreferrer"> + {{repo.name}}-{{ tag['tagname'] }}.zip + +
    + +
    • +
    +
    +
    {% if tag['body_msg'] %} @@ -110,4 +149,4 @@ -{% endblock %} \ No newline at end of file +{% endblock %} From 0d3c396b810021670d74e97690ba2415ce953669 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 07 2018 15:20:01 +0000 Subject: [PATCH 3/4] If the tag leads to a tree, skip it This fixes showing the linux releases/tags. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 99eac95..cd8ead6 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -2237,7 +2237,10 @@ def get_git_tags_objects(project): theobject = None objecttype = "" if isinstance(theobject, pygit2.Tag): - commit_time = theobject.get_object().commit_time + underlying_obj = theobject.get_object() + if isinstance(underlying_obj, pygit2.Tree): + continue + commit_time = underlying_obj.commit_time objecttype = "tag" elif isinstance(theobject, pygit2.Commit): commit_time = theobject.commit_time From 3c614c4ea9498b9e3360e98a86d0b39061c5790f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Dec 07 2018 15:20:01 +0000 Subject: [PATCH 4/4] Small black fix Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/query.py b/pagure/lib/query.py index 59b6c20..fb01c1d 100644 --- a/pagure/lib/query.py +++ b/pagure/lib/query.py @@ -3013,9 +3013,7 @@ def search_issues( ) if search_content is not None: - query = query.outerjoin( - model.IssueComment - ).filter( + query = query.outerjoin(model.IssueComment).filter( sqlalchemy.or_( model.Issue.content.ilike("%%%s%%" % search_content), sqlalchemy.and_(