From f613608bd96120c24681ec047113b8c51fc98b47 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 30 2020 07:49:02 +0000 Subject: [PATCH 1/2] Import and call pygit2 Instead of importing some specific function from pygit2 which are then called in the code, use the . approach instead. This will allow us to use more of the pygit2 module content without having to change the import line everytime. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/api/project.py b/pagure/api/project.py index 0fc7d28..9855898 100644 --- a/pagure/api/project.py +++ b/pagure/api/project.py @@ -13,9 +13,10 @@ from __future__ import unicode_literals, absolute_import import flask import logging +import pygit2 + from sqlalchemy.exc import SQLAlchemyError from six import string_types -from pygit2 import GitError, Repository try: from pygit2 import AlreadyExistsError @@ -456,7 +457,7 @@ def api_new_git_tags(repo, username=None, namespace=None): created = True except AlreadyExistsError: created = False - except GitError as err: + except pygit2.GitError as err: _log.exception(err) raise pagure.exceptions.APIError( 400, error_code=APIERROR.EGITERROR, error=str(err) @@ -1626,7 +1627,7 @@ def api_new_branch(repo, username=None, namespace=None): from_branch=from_branch, from_commit=from_commit, ) - except GitError: # pragma: no cover + except pygit2.GitError: # pragma: no cover raise pagure.exceptions.APIError(400, error_code=APIERROR.EGITERROR) except pagure.exceptions.PagureException as error: raise pagure.exceptions.APIError( @@ -1700,7 +1701,7 @@ def api_commit_flags(repo, commit_hash, username=None, namespace=None): repo = _get_repo(repo, username, namespace) reponame = pagure.utils.get_repo_path(repo) - repo_obj = Repository(reponame) + repo_obj = pygit2.Repository(reponame) try: repo_obj.get(commit_hash) except ValueError: @@ -1834,7 +1835,7 @@ def api_commit_add_flag(repo, commit_hash, username=None, namespace=None): output = {} reponame = pagure.utils.get_repo_path(repo) - repo_obj = Repository(reponame) + repo_obj = pygit2.Repository(reponame) try: repo_obj.get(commit_hash) except ValueError: From dd207ee3bf8be12c72e51afc11e33ee9d316c2be Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 03 2020 07:36:27 +0000 Subject: [PATCH 2/2] Add an API endpoint to view the content of a git repo This will return a JSON blob with the content of a git repo or one of its sub-folder or file. It will include a link where one can find the actual file or content of the folder. Fixes https://pagure.io/pagure/issue/4808 Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/api/__init__.py b/pagure/api/__init__.py index f7277f4..9db448e 100644 --- a/pagure/api/__init__.py +++ b/pagure/api/__init__.py @@ -127,6 +127,9 @@ class APIERROR(enum.Enum): EINVALIDPERPAGEVALUE = "The per_page value must be between 1 and 100" EGITERROR = "An error occurred during a git operation" ENOCOMMIT = "No such commit found in this repository" + EEMPTYGIT = "This git repository is empty" + EBRANCHNOTFOUND = "Branch not found in this git repository" + EFILENOTFOUND = "File not found in this git repository" ENOTHIGHENOUGH = ( "You do not have sufficient permissions to perform this action" ) diff --git a/pagure/api/project.py b/pagure/api/project.py index 9855898..49c5a22 100644 --- a/pagure/api/project.py +++ b/pagure/api/project.py @@ -686,6 +686,220 @@ def api_git_branches(repo, username=None, namespace=None): ) +@API.route("//tree") +@API.route("//tree/") +@API.route("//tree//f/") +@API.route("///tree") +@API.route("///tree/") +@API.route("///tree//f/") +@API.route("/fork///tree") +@API.route("/fork///tree/") +@API.route("/fork///tree//f/") +@API.route("/fork////tree") +@API.route("/fork////tree//") +@API.route( + "/fork////tree//" + "f/" +) +@api_method +def api_view_file( + repo, username=None, namespace=None, identifier=None, filename=None +): + """ + List files in a project + ----------------------- + Lists the files present in a project or one of its subfolder. + + :: + + GET /api/0/tree + GET /api/0/tree/master + GET /api/0/tree/master/f/ + GET /api/0/tree/master/f// + GET /api/0/tree/master/f/// + + + :: + + GET /api/0/fork//tree + GET /api/0/fork//tree/master + GET /api/0/fork//tree/master/f/ + GET /api/0/fork//tree/master/f// + GET /api/0/fork//tree/master/f/// + + + Sample response + ^^^^^^^^^^^^^^^ + + :: + + { + "content": [ + { + "content_url": "https://pagure.io/api/0/pagure/tree/master/f/alembic", + "name": "alembic", + "path": "alembic", + "type": "folder" + }, + { + "content_url": "https://pagure.io/api/0/pagure/tree/master/f/fedmsg.d", + "name": "fedmsg.d", + "path": "fedmsg.d", + "type": "folder" + }, + { + "content_url": "https://pagure.io/pagure/raw/master/f/tox.ini", + "name": "tox.ini", + "path": "tox.ini", + "type": "file" + } + ], + "name": null, + "type": "folder" + } + + { + "content": [ + { + "content_url": "https://pagure.io/pagure/raw/master/f/fedmsg.d/pagure.py", + "name": "pagure.py", + "path": "fedmsg.d/pagure.py", + "type": "file" + }, + { + "content_url": "https://pagure.io/pagure/raw/master/f/fedmsg.d/pagure_ci.py", + "name": "pagure_ci.py", + "path": "fedmsg.d/pagure_ci.py", + "type": "file" + } + ], + "name": "fedmsg.d", + "type": "folder" + } + + """ # noqa + repo = _get_repo(repo, username, namespace) + repopath = pagure.utils.get_repo_path(repo) + repo_obj = pygit2.Repository(repopath) + + if repo_obj.is_empty: + raise pagure.exceptions.APIError(404, error_code=APIERROR.EEMPTYGIT) + + if identifier in repo_obj.listall_branches(): + branchname = identifier + branch = repo_obj.lookup_branch(identifier) + commit = branch.peel(pygit2.Commit) + else: + try: + commit = repo_obj.get(identifier) + branchname = identifier + except (ValueError, TypeError): + # If an identifier was provided, bail, the provided info is wrong + if identifier: + raise pagure.exceptions.APIError( + 404, error_code=APIERROR.EFILENOTFOUND + ) + # If it's not a commit id then it's part of the filename + if not repo_obj.head_is_unborn: + branchname = repo_obj.head.shorthand + commit = repo_obj[repo_obj.head.target] + + if isinstance(commit, pygit2.Tag): + commit = commit.peel(pygit2.Commit) + + tree = None + if isinstance(commit, pygit2.Tree): + tree = commit + elif isinstance(commit, pygit2.Commit): + tree = commit.tree + + if tree and not filename: + content = sorted(tree, key=lambda x: x.filemode) + elif tree and commit and not isinstance(commit, pygit2.Blob): + content = pagure.utils.__get_file_in_tree( + repo_obj, tree, filename.split("/"), bail_on_tree=True + ) + if not content: + raise pagure.exceptions.APIError( + 404, error_code=APIERROR.EFILENOTFOUND + ) + content = repo_obj[content.oid] + else: + content = commit + + if not content: + raise pagure.exceptions.APIError( + 404, error_code=APIERROR.EFILENOTFOUND + ) + + output_type = "tree" + if isinstance(content, pygit2.Blob): + output_type = "file" + elif isinstance(content, pygit2.Commit): + raise pagure.exceptions.APIError( + 404, error_code=APIERROR.EFILENOTFOUND + ) + + if output_type == "file": + output = { + "type": "file", + "name": filename, + "content_url": flask.url_for( + "ui_ns.view_raw_file", + repo=repo.name, + username=username, + namespace=repo.namespace, + identifier=branchname, + filename=filename, + _external=True, + ), + } + else: + content_list = [] + for entry in content: + path = filename + "/" + entry.name if filename else entry.name + url_content = flask.url_for( + "api_ns.api_view_file", + repo=repo.name, + username=username, + namespace=repo.namespace, + identifier=branchname, + filename=path, + _external=True, + ) + if entry.filemode == 16384: + file_type = "folder" + elif entry.filemode == 40960: + file_type = "link" + elif entry.filemode == 57344: + file_type = "submodule" + else: + file_type = "file" + url_content = flask.url_for( + "ui_ns.view_raw_file", + repo=repo.name, + username=username, + namespace=repo.namespace, + identifier=branchname, + filename=path, + _external=True, + ) + tmp = { + "type": file_type, + "name": entry.name, + "path": path, + "content_url": url_content, + } + content_list.append(tmp) + output = { + "type": "folder", + "name": filename, + "content": content_list, + } + + return flask.jsonify(output) + + @API.route("/projects") @api_method def api_projects(): diff --git a/tests/test_pagure_flask_api.py b/tests/test_pagure_flask_api.py index a6ef4f5..08c5f30 100644 --- a/tests/test_pagure_flask_api.py +++ b/tests/test_pagure_flask_api.py @@ -176,13 +176,16 @@ class PagureFlaskApitests(tests.SimplePagureTest): output = self.app.get("/api/0/-/error_codes") self.assertEqual(output.status_code, 200) data = json.loads(output.get_data(as_text=True)) - self.assertEqual(len(data), 42) + self.assertEqual(len(data), 45) self.assertEqual( sorted(data.keys()), sorted( [ + "EBRANCHNOTFOUND", "EDATETIME", "EDBERROR", + "EEMPTYGIT", + "EFILENOTFOUND", "EGITERROR", "EINVALIDISSUEFIELD", "EINVALIDISSUEFIELD_LINK", diff --git a/tests/test_pagure_flask_api_project_view_file.py b/tests/test_pagure_flask_api_project_view_file.py new file mode 100644 index 0000000..708a968 --- /dev/null +++ b/tests/test_pagure_flask_api_project_view_file.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2020 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals, absolute_import + +import datetime +import json +import unittest +import shutil +import sys +import tempfile +import os + +import pygit2 +from celery.result import EagerResult +from mock import patch, Mock + +sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +) + +import pagure.flask_app +import pagure.lib.query +import tests +from pagure.lib.repo import PagureRepo + + +class PagureFlaskApiProjectViewFiletests(tests.Modeltests): + """ Tests for the flask API of pagure for issue """ + + maxDiff = None + + def setUp(self): + super(PagureFlaskApiProjectViewFiletests, self).setUp() + tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, "repos"), bare=True) + tests.add_readme_git_repo(os.path.join(self.path, "repos", "test.git")) + + def test_view_file_invalid_project(self): + output = self.app.get("/api/0/invalid/tree") + self.assertEqual(output.status_code, 404) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, {"error": "Project not found", "error_code": "ENOPROJECT"} + ) + + def test_view_file_invalid_ref_and_path(self): + output = self.app.get("/api/0/test/tree/branchname/f/foldername") + self.assertEqual(output.status_code, 404) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "error": "File not found in this git repository", + "error_code": "EFILENOTFOUND", + }, + ) + + def test_view_file_empty_project(self): + output = self.app.get("/api/0/test2/tree") + self.assertEqual(output.status_code, 404) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "error": "This git repository is empty", + "error_code": "EEMPTYGIT", + }, + ) + + def test_view_file_basic(self): + output = self.app.get("/api/0/test/tree") + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "content": [ + { + "content_url": "http://localhost/test/raw/master/" + "f/README.rst", + "name": "README.rst", + "path": "README.rst", + "type": "file", + } + ], + "name": None, + "type": "folder", + }, + ) + + def test_view_file_with_folder(self): + tests.add_content_git_repo( + os.path.join(self.path, "repos", "test.git") + ) + output = self.app.get("/api/0/test/tree") + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "content": [ + { + "content_url": "http://localhost/api/0/test/tree/" + "master/f/folder1", + "name": "folder1", + "path": "folder1", + "type": "folder", + }, + { + "content_url": "http://localhost/test/raw/master/f/" + "README.rst", + "name": "README.rst", + "path": "README.rst", + "type": "file", + }, + { + "content_url": "http://localhost/test/raw/master/f/" + "sources", + "name": "sources", + "path": "sources", + "type": "file", + }, + ], + "name": None, + "type": "folder", + }, + ) + + def test_view_file_specific_file(self): + tests.add_content_git_repo( + os.path.join(self.path, "repos", "test.git") + ) + output = self.app.get("/api/0/test/tree/master/f/README.rst") + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "content_url": "http://localhost/test/raw/master/f/README.rst", + "name": "README.rst", + "type": "file", + }, + ) + + def test_view_file_invalid_ref(self): + tests.add_content_git_repo( + os.path.join(self.path, "repos", "test.git") + ) + output = self.app.get("/api/0/test/tree/invalid/f/folder1") + print(output.data) + self.assertEqual(output.status_code, 404) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "error": "File not found in this git repository", + "error_code": "EFILENOTFOUND", + }, + ) + + def test_view_file_invalid_folder(self): + tests.add_content_git_repo( + os.path.join(self.path, "repos", "test.git") + ) + output = self.app.get("/api/0/test/tree/master/f/inv/invalid") + self.assertEqual(output.status_code, 404) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "error": "File not found in this git repository", + "error_code": "EFILENOTFOUND", + }, + ) + + def test_view_file_valid_branch(self): + tests.add_content_git_repo( + os.path.join(self.path, "repos", "test.git") + ) + output = self.app.get("/api/0/test/tree/master/f/folder1") + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "content": [ + { + "content_url": "http://localhost/api/0/test/tree/" + "master/f/folder1/folder2", + "name": "folder2", + "path": "folder1/folder2", + "type": "folder", + } + ], + "name": "folder1", + "type": "folder", + }, + ) + + def test_view_file_non_ascii_name(self): + # View file with a non-ascii name + tests.add_commit_git_repo( + os.path.join(self.path, "repos", "test.git"), + ncommits=1, + filename="Šource", + ) + output = self.app.get("/api/0/test/tree") + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True).encode("utf-8")) + self.assertDictEqual( + data, + { + "content": [ + { + "content_url": "http://localhost/test/raw/master/f/" + "README.rst", + "name": "README.rst", + "path": "README.rst", + "type": "file", + }, + { + "content_url": "http://localhost/test/raw/master/f/%C5%A0ource", + "name": "Šource", + "path": "Šource", + "type": "file", + }, + ], + "name": None, + "type": "folder", + }, + ) + + def test_view_file_from_commit(self): + repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git")) + commit = repo.revparse_single("HEAD") + + output = self.app.get("/api/0/test/tree/%s" % commit.oid.hex) + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "content": [ + { + "content_url": "http://localhost/test/raw/" + "%s/f/README.rst" % commit.oid.hex, + "name": "README.rst", + "path": "README.rst", + "type": "file", + } + ], + "name": None, + "type": "folder", + }, + ) + + def test_view_file_from_tree(self): + tests.add_content_git_repo( + os.path.join(self.path, "repos", "test.git") + ) + repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git")) + commit = repo.revparse_single("HEAD") + + output = self.app.get( + "/api/0/test/tree/%s/f/folder1" % commit.tree.oid.hex + ) + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "content": [ + { + "content_url": "http://localhost/api/0/test/tree/" + "%s/f/folder1/folder2" % commit.tree.oid.hex, + "name": "folder2", + "path": "folder1/folder2", + "type": "folder", + } + ], + "name": "folder1", + "type": "folder", + }, + ) + + def test_view_file_from_tag_hex(self): + repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git")) + commit = repo.revparse_single("HEAD") + tagger = pygit2.Signature("Alice Doe", "adoe@example.com", 12347, 0) + tag = repo.create_tag( + "v1.0_tag", + commit.oid.hex, + pygit2.GIT_OBJ_COMMIT, + tagger, + "Release v1.0", + ) + + output = self.app.get("/api/0/test/tree/%s" % tag.hex) + self.assertEqual(output.status_code, 200) + data = json.loads(output.get_data(as_text=True)) + self.assertDictEqual( + data, + { + "content": [ + { + "content_url": "http://localhost/test/raw/" + "%s/f/README.rst" % tag.hex, + "name": "README.rst", + "path": "README.rst", + "type": "file", + } + ], + "name": None, + "type": "folder", + }, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2)