#4912 Add an API endpoint to view the content of a git repo
Merged 3 years ago by pingou. Opened 3 years ago by pingou.

file modified
+3
@@ -127,6 +127,9 @@ 

      EINVALIDPERPAGEVALUE = "The per_page value must be between 1 and 100"

      EGITERROR = "An error occurred during a git operation"

      ENOCOMMIT = "No such commit found in this repository"

+     EEMPTYGIT = "This git repository is empty"

+     EBRANCHNOTFOUND = "Branch not found in this git repository"

+     EFILENOTFOUND = "File not found in this git repository"

      ENOTHIGHENOUGH = (

          "You do not have sufficient permissions to perform this action"

      )

file modified
+220 -5
@@ -13,9 +13,10 @@ 

  import flask

  import logging

  

+ import pygit2

+ 

  from sqlalchemy.exc import SQLAlchemyError

  from six import string_types

- from pygit2 import GitError, Repository

  

  try:

      from pygit2 import AlreadyExistsError
@@ -456,7 +457,7 @@ 

              created = True

          except AlreadyExistsError:

              created = False

-         except GitError as err:

+         except pygit2.GitError as err:

              _log.exception(err)

              raise pagure.exceptions.APIError(

                  400, error_code=APIERROR.EGITERROR, error=str(err)
@@ -685,6 +686,220 @@ 

      )

  

  

+ @API.route("/<repo>/tree")

+ @API.route("/<repo>/tree/<path:identifier>")

+ @API.route("/<repo>/tree/<path:identifier>/f/<path:filename>")

+ @API.route("/<namespace>/<repo>/tree")

+ @API.route("/<namespace>/<repo>/tree/<path:identifier>")

+ @API.route("/<namespace>/<repo>/tree/<path:identifier>/f/<path:filename>")

+ @API.route("/fork/<username>/<repo>/tree")

+ @API.route("/fork/<username>/<repo>/tree/<path:identifier>")

+ @API.route("/fork/<username>/<repo>/tree/<path:identifier>/f/<path:filename>")

+ @API.route("/fork/<username>/<namespace>/<repo>/tree")

+ @API.route("/fork/<username>/<namespace>/<repo>/tree/<path:identifier>/")

+ @API.route(

+     "/fork/<username>/<namespace>/<repo>/tree/<path:identifier>/"

+     "f/<path:filename>"

+ )

+ @api_method

+ def api_view_file(

+     repo, username=None, namespace=None, identifier=None, filename=None

+ ):

+     """

+     List files in a project

+     -----------------------

+     Lists the files present in a project or one of its subfolder.

+ 

+     ::

+ 

+         GET /api/0/<repo>tree

+         GET /api/0/<repo>tree/master

+         GET /api/0/<repo>tree/master/f/<filename>

+         GET /api/0/<repo>tree/master/f/<folder>/

+         GET /api/0/<repo>tree/master/f/<folder1>/<folder2>/<filename>

+ 

+ 

+     ::

+ 

+         GET /api/0/fork/<username>/<repo>tree

+         GET /api/0/fork/<username>/<repo>tree/master

+         GET /api/0/fork/<username>/<repo>tree/master/f/<filename>

+         GET /api/0/fork/<username>/<repo>tree/master/f/<folder>/

+         GET /api/0/fork/<username>/<repo>tree/master/f/<folder1>/<folder2>/<filename>

+ 

+ 

+     Sample response

+     ^^^^^^^^^^^^^^^

+ 

+     ::

+ 

+         {

+           "content": [

+             {

+               "content_url": "https://pagure.io/api/0/pagure/tree/master/f/alembic",

+               "name": "alembic",

+               "path": "alembic",

+               "type": "folder"

+             },

+             {

+               "content_url": "https://pagure.io/api/0/pagure/tree/master/f/fedmsg.d",

+               "name": "fedmsg.d",

+               "path": "fedmsg.d",

+               "type": "folder"

+             },

+             {

+               "content_url": "https://pagure.io/pagure/raw/master/f/tox.ini",

+               "name": "tox.ini",

+               "path": "tox.ini",

+               "type": "file"

+             }

+           ],

+           "name": null,

+           "type": "folder"

+         }

+ 

+         {

+           "content": [

+             {

+               "content_url": "https://pagure.io/pagure/raw/master/f/fedmsg.d/pagure.py",

+               "name": "pagure.py",

+               "path": "fedmsg.d/pagure.py",

+               "type": "file"

+             },

+             {

+               "content_url": "https://pagure.io/pagure/raw/master/f/fedmsg.d/pagure_ci.py",

+               "name": "pagure_ci.py",

+               "path": "fedmsg.d/pagure_ci.py",

+               "type": "file"

+             }

+           ],

+           "name": "fedmsg.d",

+           "type": "folder"

+         }

+ 

+     """  # noqa

+     repo = _get_repo(repo, username, namespace)

+     repopath = pagure.utils.get_repo_path(repo)

+     repo_obj = pygit2.Repository(repopath)

+ 

+     if repo_obj.is_empty:

+         raise pagure.exceptions.APIError(404, error_code=APIERROR.EEMPTYGIT)

+ 

+     if identifier in repo_obj.listall_branches():

+         branchname = identifier

+         branch = repo_obj.lookup_branch(identifier)

+         commit = branch.peel(pygit2.Commit)

+     else:

+         try:

+             commit = repo_obj.get(identifier)

+             branchname = identifier

+         except (ValueError, TypeError):

+             # If an identifier was provided, bail, the provided info is wrong

+             if identifier:

+                 raise pagure.exceptions.APIError(

+                     404, error_code=APIERROR.EFILENOTFOUND

+                 )

+             # If it's not a commit id then it's part of the filename

+             if not repo_obj.head_is_unborn:

+                 branchname = repo_obj.head.shorthand

+                 commit = repo_obj[repo_obj.head.target]

+ 

+     if isinstance(commit, pygit2.Tag):

+         commit = commit.peel(pygit2.Commit)

+ 

+     tree = None

+     if isinstance(commit, pygit2.Tree):

+         tree = commit

+     elif isinstance(commit, pygit2.Commit):

+         tree = commit.tree

+ 

+     if tree and not filename:

+         content = sorted(tree, key=lambda x: x.filemode)

+     elif tree and commit and not isinstance(commit, pygit2.Blob):

+         content = pagure.utils.__get_file_in_tree(

+             repo_obj, tree, filename.split("/"), bail_on_tree=True

+         )

+         if not content:

+             raise pagure.exceptions.APIError(

+                 404, error_code=APIERROR.EFILENOTFOUND

+             )

+         content = repo_obj[content.oid]

+     else:

+         content = commit

+ 

+     if not content:

+         raise pagure.exceptions.APIError(

+             404, error_code=APIERROR.EFILENOTFOUND

+         )

+ 

+     output_type = "tree"

+     if isinstance(content, pygit2.Blob):

+         output_type = "file"

+     elif isinstance(content, pygit2.Commit):

+         raise pagure.exceptions.APIError(

+             404, error_code=APIERROR.EFILENOTFOUND

+         )

+ 

+     if output_type == "file":

+         output = {

+             "type": "file",

+             "name": filename,

+             "content_url": flask.url_for(

+                 "ui_ns.view_raw_file",

+                 repo=repo.name,

+                 username=username,

+                 namespace=repo.namespace,

+                 identifier=branchname,

+                 filename=filename,

+                 _external=True,

+             ),

+         }

+     else:

+         content_list = []

+         for entry in content:

+             path = filename + "/" + entry.name if filename else entry.name

+             url_content = flask.url_for(

+                 "api_ns.api_view_file",

+                 repo=repo.name,

+                 username=username,

+                 namespace=repo.namespace,

+                 identifier=branchname,

+                 filename=path,

+                 _external=True,

+             )

+             if entry.filemode == 16384:

+                 file_type = "folder"

+             elif entry.filemode == 40960:

+                 file_type = "link"

+             elif entry.filemode == 57344:

+                 file_type = "submodule"

+             else:

+                 file_type = "file"

+                 url_content = flask.url_for(

+                     "ui_ns.view_raw_file",

+                     repo=repo.name,

+                     username=username,

+                     namespace=repo.namespace,

+                     identifier=branchname,

+                     filename=path,

+                     _external=True,

+                 )

+             tmp = {

+                 "type": file_type,

+                 "name": entry.name,

+                 "path": path,

+                 "content_url": url_content,

+             }

+             content_list.append(tmp)

+         output = {

+             "type": "folder",

+             "name": filename,

+             "content": content_list,

+         }

+ 

+     return flask.jsonify(output)

+ 

+ 

  @API.route("/projects")

  @api_method

  def api_projects():
@@ -1626,7 +1841,7 @@ 

              from_branch=from_branch,

              from_commit=from_commit,

          )

-     except GitError:  # pragma: no cover

+     except pygit2.GitError:  # pragma: no cover

          raise pagure.exceptions.APIError(400, error_code=APIERROR.EGITERROR)

      except pagure.exceptions.PagureException as error:

          raise pagure.exceptions.APIError(
@@ -1700,7 +1915,7 @@ 

      repo = _get_repo(repo, username, namespace)

  

      reponame = pagure.utils.get_repo_path(repo)

-     repo_obj = Repository(reponame)

+     repo_obj = pygit2.Repository(reponame)

      try:

          repo_obj.get(commit_hash)

      except ValueError:
@@ -1834,7 +2049,7 @@ 

      output = {}

  

      reponame = pagure.utils.get_repo_path(repo)

-     repo_obj = Repository(reponame)

+     repo_obj = pygit2.Repository(reponame)

      try:

          repo_obj.get(commit_hash)

      except ValueError:

@@ -176,13 +176,16 @@ 

          output = self.app.get("/api/0/-/error_codes")

          self.assertEqual(output.status_code, 200)

          data = json.loads(output.get_data(as_text=True))

-         self.assertEqual(len(data), 42)

+         self.assertEqual(len(data), 45)

          self.assertEqual(

              sorted(data.keys()),

              sorted(

                  [

+                     "EBRANCHNOTFOUND",

                      "EDATETIME",

                      "EDBERROR",

+                     "EEMPTYGIT",

+                     "EFILENOTFOUND",

                      "EGITERROR",

                      "EINVALIDISSUEFIELD",

                      "EINVALIDISSUEFIELD_LINK",

@@ -0,0 +1,328 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ """

+  (c) 2020 - Copyright Red Hat Inc

+ 

+  Authors:

+    Pierre-Yves Chibon <pingou@pingoured.fr>

+ 

+ """

+ 

+ from __future__ import unicode_literals, absolute_import

+ 

+ import datetime

+ import json

+ import unittest

+ import shutil

+ import sys

+ import tempfile

+ import os

+ 

+ import pygit2

+ from celery.result import EagerResult

+ from mock import patch, Mock

+ 

+ sys.path.insert(

+     0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")

+ )

+ 

+ import pagure.flask_app

+ import pagure.lib.query

+ import tests

+ from pagure.lib.repo import PagureRepo

+ 

+ 

+ class PagureFlaskApiProjectViewFiletests(tests.Modeltests):

+     """ Tests for the flask API of pagure for issue """

+ 

+     maxDiff = None

+ 

+     def setUp(self):

+         super(PagureFlaskApiProjectViewFiletests, self).setUp()

+         tests.create_projects(self.session)

+         tests.create_projects_git(os.path.join(self.path, "repos"), bare=True)

+         tests.add_readme_git_repo(os.path.join(self.path, "repos", "test.git"))

+ 

+     def test_view_file_invalid_project(self):

+         output = self.app.get("/api/0/invalid/tree")

+         self.assertEqual(output.status_code, 404)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data, {"error": "Project not found", "error_code": "ENOPROJECT"}

+         )

+ 

+     def test_view_file_invalid_ref_and_path(self):

+         output = self.app.get("/api/0/test/tree/branchname/f/foldername")

+         self.assertEqual(output.status_code, 404)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "error": "File not found in this git repository",

+                 "error_code": "EFILENOTFOUND",

+             },

+         )

+ 

+     def test_view_file_empty_project(self):

+         output = self.app.get("/api/0/test2/tree")

+         self.assertEqual(output.status_code, 404)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "error": "This git repository is empty",

+                 "error_code": "EEMPTYGIT",

+             },

+         )

+ 

+     def test_view_file_basic(self):

+         output = self.app.get("/api/0/test/tree")

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "content": [

+                     {

+                         "content_url": "http://localhost/test/raw/master/"

+                         "f/README.rst",

+                         "name": "README.rst",

+                         "path": "README.rst",

+                         "type": "file",

+                     }

+                 ],

+                 "name": None,

+                 "type": "folder",

+             },

+         )

+ 

+     def test_view_file_with_folder(self):

+         tests.add_content_git_repo(

+             os.path.join(self.path, "repos", "test.git")

+         )

+         output = self.app.get("/api/0/test/tree")

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "content": [

+                     {

+                         "content_url": "http://localhost/api/0/test/tree/"

+                         "master/f/folder1",

+                         "name": "folder1",

+                         "path": "folder1",

+                         "type": "folder",

+                     },

+                     {

+                         "content_url": "http://localhost/test/raw/master/f/"

+                         "README.rst",

+                         "name": "README.rst",

+                         "path": "README.rst",

+                         "type": "file",

+                     },

+                     {

+                         "content_url": "http://localhost/test/raw/master/f/"

+                         "sources",

+                         "name": "sources",

+                         "path": "sources",

+                         "type": "file",

+                     },

+                 ],

+                 "name": None,

+                 "type": "folder",

+             },

+         )

+ 

+     def test_view_file_specific_file(self):

+         tests.add_content_git_repo(

+             os.path.join(self.path, "repos", "test.git")

+         )

+         output = self.app.get("/api/0/test/tree/master/f/README.rst")

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "content_url": "http://localhost/test/raw/master/f/README.rst",

+                 "name": "README.rst",

+                 "type": "file",

+             },

+         )

+ 

+     def test_view_file_invalid_ref(self):

+         tests.add_content_git_repo(

+             os.path.join(self.path, "repos", "test.git")

+         )

+         output = self.app.get("/api/0/test/tree/invalid/f/folder1")

+         print(output.data)

+         self.assertEqual(output.status_code, 404)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "error": "File not found in this git repository",

+                 "error_code": "EFILENOTFOUND",

+             },

+         )

+ 

+     def test_view_file_invalid_folder(self):

+         tests.add_content_git_repo(

+             os.path.join(self.path, "repos", "test.git")

+         )

+         output = self.app.get("/api/0/test/tree/master/f/inv/invalid")

+         self.assertEqual(output.status_code, 404)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "error": "File not found in this git repository",

+                 "error_code": "EFILENOTFOUND",

+             },

+         )

+ 

+     def test_view_file_valid_branch(self):

+         tests.add_content_git_repo(

+             os.path.join(self.path, "repos", "test.git")

+         )

+         output = self.app.get("/api/0/test/tree/master/f/folder1")

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "content": [

+                     {

+                         "content_url": "http://localhost/api/0/test/tree/"

+                         "master/f/folder1/folder2",

+                         "name": "folder2",

+                         "path": "folder1/folder2",

+                         "type": "folder",

+                     }

+                 ],

+                 "name": "folder1",

+                 "type": "folder",

+             },

+         )

+ 

+     def test_view_file_non_ascii_name(self):

+         # View file with a non-ascii name

+         tests.add_commit_git_repo(

+             os.path.join(self.path, "repos", "test.git"),

+             ncommits=1,

+             filename="Å ource",

+         )

+         output = self.app.get("/api/0/test/tree")

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True).encode("utf-8"))

+         self.assertDictEqual(

+             data,

+             {

+                 "content": [

+                     {

+                         "content_url": "http://localhost/test/raw/master/f/"

+                         "README.rst",

+                         "name": "README.rst",

+                         "path": "README.rst",

+                         "type": "file",

+                     },

+                     {

+                         "content_url": "http://localhost/test/raw/master/f/%C5%A0ource",

+                         "name": "Å ource",

+                         "path": "Å ource",

+                         "type": "file",

+                     },

+                 ],

+                 "name": None,

+                 "type": "folder",

+             },

+         )

+ 

+     def test_view_file_from_commit(self):

+         repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git"))

+         commit = repo.revparse_single("HEAD")

+ 

+         output = self.app.get("/api/0/test/tree/%s" % commit.oid.hex)

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "content": [

+                     {

+                         "content_url": "http://localhost/test/raw/"

+                         "%s/f/README.rst" % commit.oid.hex,

+                         "name": "README.rst",

+                         "path": "README.rst",

+                         "type": "file",

+                     }

+                 ],

+                 "name": None,

+                 "type": "folder",

+             },

+         )

+ 

+     def test_view_file_from_tree(self):

+         tests.add_content_git_repo(

+             os.path.join(self.path, "repos", "test.git")

+         )

+         repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git"))

+         commit = repo.revparse_single("HEAD")

+ 

+         output = self.app.get(

+             "/api/0/test/tree/%s/f/folder1" % commit.tree.oid.hex

+         )

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "content": [

+                     {

+                         "content_url": "http://localhost/api/0/test/tree/"

+                         "%s/f/folder1/folder2" % commit.tree.oid.hex,

+                         "name": "folder2",

+                         "path": "folder1/folder2",

+                         "type": "folder",

+                     }

+                 ],

+                 "name": "folder1",

+                 "type": "folder",

+             },

+         )

+ 

+     def test_view_file_from_tag_hex(self):

+         repo = pygit2.Repository(os.path.join(self.path, "repos", "test.git"))

+         commit = repo.revparse_single("HEAD")

+         tagger = pygit2.Signature("Alice Doe", "adoe@example.com", 12347, 0)

+         tag = repo.create_tag(

+             "v1.0_tag",

+             commit.oid.hex,

+             pygit2.GIT_OBJ_COMMIT,

+             tagger,

+             "Release v1.0",

+         )

+ 

+         output = self.app.get("/api/0/test/tree/%s" % tag.hex)

+         self.assertEqual(output.status_code, 200)

+         data = json.loads(output.get_data(as_text=True))

+         self.assertDictEqual(

+             data,

+             {

+                 "content": [

+                     {

+                         "content_url": "http://localhost/test/raw/"

+                         "%s/f/README.rst" % tag.hex,

+                         "name": "README.rst",

+                         "path": "README.rst",

+                         "type": "file",

+                     }

+                 ],

+                 "name": None,

+                 "type": "folder",

+             },

+         )

+ 

+ 

+ if __name__ == "__main__":

+     unittest.main(verbosity=2)

no initial comment

2 new commits added

  • Add an API endpoint to view the content of a git repo
  • Import and call pygit2
3 years ago

2 new commits added

  • Add an API endpoint to view the content of a git repo
  • Import and call pygit2
3 years ago

Pull-Request has been merged by pingou

3 years ago