#3054 Add linking between tickets and pull-requests
Merged 6 years ago by pingou. Opened 6 years ago by pingou.

@@ -0,0 +1,42 @@ 

+ """Add the pr_to_issue table

+ 

+ Revision ID: 369deb8c8b63

+ Revises: eab41ce5f92a

+ Create Date: 2018-03-12 11:38:00.955252

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = '369deb8c8b63'

+ down_revision = 'eab41ce5f92a'

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     ''' Create the pr_to_issue table. '''

+ 

+     op.create_table(

+         'pr_to_issue',

+         sa.Column(

+             'pull_request_uid',

+             sa.String(32),

+             sa.ForeignKey(

+                 'pull_requests.uid', ondelete='CASCADE', onupdate='CASCADE',

+             ),

+             primary_key=True),

+         sa.Column(

+             'issue_uid',

+             sa.String(32),

+             sa.ForeignKey(

+                 'issues.uid', ondelete='CASCADE', onupdate='CASCADE',

+             ),

+             primary_key=True)

+     )

+ 

+ 

+ def downgrade():

+     ''' Drop the pr_to_issue table. '''

+ 

+     op.drop_table('pr_to_issue')

@@ -149,6 +149,10 @@ 

          # Link to existing PRs if there are any

          seen = len(prs) != 0

          for pr in prs:

+             # Link tickets with pull-requests if the commit mentions it

+             pagure.lib.tasks.link_pr_to_ticket.delay(pr.uid)

+ 

+             # Inform the user about the PR

              print('View pull-request for %s' % refname)

              print('   %s/%s/pull-request/%s' % (

                  _config['APP_URL'].rstrip('/'),

file modified
+25
@@ -1773,6 +1773,8 @@ 

      pagure.lib.git.update_git(

          request, repo=request.project, repofolder=requestfolder)

  

+     pagure.lib.tasks.link_pr_to_ticket.delay(request.uid)

+ 

      log_action(session, 'created', request, user_obj)

  

      if notify:
@@ -5055,3 +5057,26 @@ 

      )

  

      return [parent] + query.all()

+ 

+ 

+ def link_pr_issue(session, issue, request):

+     ''' Associate the specified issue with the specified pull-requets.

+ 

+     :arg session: The SQLAlchemy session to use

+     :type session: sqlalchemy.orm.session.Session

+     :arg issue: The issue mentioned in the commits of the pull-requests to

+         be associated with

+     :type issue: pagure.lib.model.Issue

+     :arg request: A pull-request to associate the specified issue with

+     :type request: pagure.lib.model.PullRequest

+ 

+     '''

+ 

+     associated_issues = [iss.uid for iss in request.related_issues]

+     if issue.uid not in associated_issues:

+         obj = model.PrToIssue(

+             pull_request_uid=request.uid,

+             issue_uid=issue.uid

+         )

+         session.add(obj)

+         session.flush()

file modified
+2 -1
@@ -1395,7 +1395,8 @@ 

      branch = None

      if branch_to:

          branch = orig_repo.lookup_branch(branch_to)

-         if not branch and not orig_repo.is_empty:

+         local_branches = orig_repo.listall_branches(pygit2.GIT_BRANCH_LOCAL)

+         if not branch and local_branches:

              raise pagure.exceptions.BranchNotFoundException(

                  'Branch %s could not be found in the target repo' % branch_to

              )

file modified
+30
@@ -1312,6 +1312,28 @@ 

          primary_key=True)

  

  

+ class PrToIssue(BASE):

+     """ Stores the associations between issues and pull-requests.

+ 

+     Table -- pr_to_issue

+     """

+ 

+     __tablename__ = 'pr_to_issue'

+ 

+     pull_request_uid = sa.Column(

+         sa.String(32),

+         sa.ForeignKey(

+             'pull_requests.uid', ondelete='CASCADE', onupdate='CASCADE',

+         ),

+         primary_key=True)

+     issue_uid = sa.Column(

+         sa.String(32),

+         sa.ForeignKey(

+             'issues.uid', ondelete='CASCADE', onupdate='CASCADE',

+         ),

+         primary_key=True)

+ 

+ 

  class IssueComment(BASE):

      """ Stores the comments made on a commit/file.

  
@@ -1765,6 +1787,14 @@ 

          viewonly=True

      )

  

+     related_issues = relation(

+         "Issue",

+         secondary="pr_to_issue",

+         primaryjoin="pull_requests.c.uid==pr_to_issue.c.pull_request_uid",

+         secondaryjoin="pr_to_issue.c.issue_uid==issues.c.uid",

+         backref=backref("related_prs", order_by="pull_requests.c.id.desc()")

+     )

+ 

      def __repr__(self):

          return 'PullRequest(%s, project:%s, user:%s, title:%s)' % (

              self.id, self.project.name, self.user.user, self.title

file modified
+77 -3
@@ -32,9 +32,11 @@ 

  import pagure.lib

  import pagure.lib.git

  import pagure.lib.git_auth

+ import pagure.lib.link

  import pagure.lib.repo

  import pagure.utils

  from pagure.config import config as pagure_config

+ from pagure.utils import get_parent_repo_path

  

  # logging.config.dictConfig(pagure_config.get('LOGGING') or {'version': 1})

  _log = logging.getLogger(__name__)
@@ -293,6 +295,7 @@ 

              with open(http_clone_file, 'w') as stream:

                  pass

  

+         docrepo = None

          if pagure_config.get('DOCS_FOLDER'):

              docrepo = os.path.join(

                  pagure_config['DOCS_FOLDER'], project.path)
@@ -307,13 +310,15 @@ 

                  _log.debug('Create git repo at: %s', docrepo)

                  pygit2.init_repository(docrepo, bare=True)

  

+         ticketrepo = None

          if pagure_config.get('TICKETS_FOLDER'):

              ticketrepo = os.path.join(

                  pagure_config['TICKETS_FOLDER'], project.path)

              if os.path.exists(ticketrepo):

                  if not ignore_existing_repo:

                      shutil.rmtree(gitrepo)

-                     shutil.rmtree(docrepo)

+                     if docrepo:

+                         shutil.rmtree(docrepo)

                      session.remove()

                      raise pagure.exceptions.RepoExistsException(

                          'The tickets repo "%s" already exists' %
@@ -330,8 +335,10 @@ 

          if os.path.exists(requestrepo):

              if not ignore_existing_repo:

                  shutil.rmtree(gitrepo)

-                 shutil.rmtree(docrepo)

-                 shutil.rmtree(ticketrepo)

+                 if docrepo:

+                     shutil.rmtree(docrepo)

+                 if ticketrepo:

+                     shutil.rmtree(ticketrepo)

                  session.remove()

                  raise pagure.exceptions.RepoExistsException(

                      'The requests repo "%s" already exists' %
@@ -850,3 +857,70 @@ 

          dates[arrow.get(commit.commit_time).date().isoformat()] += 1

  

      return [(key, dates[key]) for key in sorted(dates)]

+ 

+ 

+ @conn.task(bind=True)

+ @pagure_task

+ def link_pr_to_ticket(self, session, pr_uid):

+     """ Link the specified pull-request against the ticket(s) mentioned in

+     the commits of the pull-request

+ 

+     """

+     _log.info(

+         'LINK_PR_TO_TICKET: Linking ticket(s) to PR for: %s' % pr_uid)

+ 

+     request = pagure.lib.get_request_by_uid(session, pr_uid)

+     if not request:

+         _log.info('LINK_PR_TO_TICKET: Not PR found for: %s' % pr_uid)

+         return

+ 

+     if request.remote:

+         repopath = pagure.utils.get_remote_repo_path(

+             request.remote_git, request.branch_from)

+         parentpath = pagure.utils.get_repo_path(request.project)

+     else:

+         repo_from = request.project_from

+         repopath = pagure.utils.get_repo_path(repo_from)

+         parentpath = get_parent_repo_path(repo_from)

+ 

+     repo_obj = pygit2.Repository(repopath)

+     orig_repo = pygit2.Repository(parentpath)

+ 

+     diff_commits = pagure.lib.git.diff_pull_request(

+         session, request, repo_obj, orig_repo,

+         requestfolder=pagure_config['REQUESTS_FOLDER'], with_diff=False)

+ 

+     _log.info(

+         'LINK_PR_TO_TICKET: Found %s commits in that PR' % len(diff_commits))

+ 

+     name = request.project.name

+     namespace = request.project.namespace

+     user = request.project.user.user \

+         if request.project.is_fork else None

+ 

+     for line in pagure.lib.git.read_git_lines(

+             ['log', '--no-walk']

+             + [c.oid.hex for c in diff_commits]

+             + ['--'], repopath):

+ 

+         line = line.strip()

+         for issue in pagure.lib.link.get_relation(

+                 session, name, user, namespace, line, 'fixes',

+                 include_prs=False):

+             _log.info(

+                 'LINK_PR_TO_TICKET: Link ticket %s to PRs %s' % (

+                     issue, request))

+             pagure.lib.link_pr_issue(session, issue, request)

+ 

+         for issue in pagure.lib.link.get_relation(

+                 session, name, user, namespace, line, 'relates'):

+             _log.info(

+                 'LINK_PR_TO_TICKET: Link ticket %s to PRs %s' % (

+                     issue, request))

+             pagure.lib.link_pr_issue(session, issue, request)

+ 

+     try:

+         session.commit()

+     except SQLAlchemyError:

+         _log.exception('Could not link ticket to PR :(')

+         session.rollback()

@@ -365,6 +365,32 @@ 

      </div>

    {% endif %}

  

+   {% if issue.related_prs %}

+   <div class="col-md-4 card-block">

+     <div class="card">

+       <div class="card-block">

+         <label><strong>Related PR(s)</strong></label>

+         <div id="pr_list">

+           <ul class="list-unstyled">

+             {% for pr in issue.related_prs %}

+             <li>

+               <a class="label label-default" href="{{ url_for(

+                   'ui_ns.request_pull',

+                   repo=pr.project.name,

+                   username=pr.project.user.user if pr.project.is_fork else none,

+                   namespace=pr.project.namespace,

+                   requestid=pr.id) }}">#{{pr.id}}</a>

+               {{ pr.status if pr.status != 'Open' else 'Last updated'

+                 }} {{ pr.last_updated | humanize }}

+             </li>

+             {% endfor %}

+           </ul>

+         </div>

+       </div>

+     </div>

+   </div>

+   {% endif %}

+ 

    {% if repo.issue_keys %}

    <div class="col-md-4 right">

      <div class="card">

file modified
+4 -16
@@ -33,25 +33,13 @@ 

  import pagure.forms

  from pagure.config import config as pagure_config

  from pagure.ui import UI_NS

- from pagure.utils import login_required, __get_file_in_tree

+ from pagure.utils import (

+     login_required, __get_file_in_tree, get_parent_repo_path)

  

  

  _log = logging.getLogger(__name__)

  

  

- def _get_parent_repo_path(repo):

-     """ Return the path of the parent git repository corresponding to the

-     provided Repository object from the DB.

-     """

-     if repo.parent:

-         parentpath = os.path.join(

-             pagure_config['GIT_FOLDER'], repo.parent.path)

-     else:

-         parentpath = os.path.join(pagure_config['GIT_FOLDER'], repo.path)

- 

-     return parentpath

- 

- 

  def _get_parent_request_repo_path(repo):

      """ Return the path of the parent git repository corresponding to the

      provided Repository object from the DB.
@@ -326,7 +314,7 @@ 

      else:

          repo_from = request.project_from

          repopath = pagure.utils.get_repo_path(repo_from)

-         parentpath = _get_parent_repo_path(repo_from)

+         parentpath = get_parent_repo_path(repo_from)

  

      repo_obj = pygit2.Repository(repopath)

      orig_repo = pygit2.Repository(parentpath)
@@ -1105,7 +1093,7 @@ 

      repo_obj = flask.g.repo_obj

  

      if not project_to:

-         parentpath = _get_parent_repo_path(repo)

+         parentpath = get_parent_repo_path(repo)

          orig_repo = pygit2.Repository(parentpath)

      else:

          p_namespace = None

file modified
+13
@@ -376,3 +376,16 @@ 

              _, user, namespace, project_name = project_items

  

      return (user, namespace, project_name)

+ 

+ 

+ def get_parent_repo_path(repo):

+     """ Return the path of the parent git repository corresponding to the

+     provided Repository object from the DB.

+     """

+     if repo.parent:

+         parentpath = os.path.join(

+             pagure_config['GIT_FOLDER'], repo.parent.path)

+     else:

+         parentpath = os.path.join(pagure_config['GIT_FOLDER'], repo.path)

+ 

+     return parentpath

file modified
+13 -7
@@ -476,31 +476,37 @@ 

          return self.dic[key]

  

  

- def create_projects(session):

+ def create_projects(session, is_fork=False, user_id=1, hook_token_suffix=''):

      """ Create some projects in the database. """

      item = pagure.lib.model.Project(

-         user_id=1,  # pingou

+         user_id=user_id,  # pingou

          name='test',

+         is_fork=is_fork,

+         parent_id=1 if is_fork else None,

          description='test project #1',

-         hook_token='aaabbbccc',

+         hook_token='aaabbbccc' + hook_token_suffix,

      )

      item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate']

      session.add(item)

  

      item = pagure.lib.model.Project(

-         user_id=1,  # pingou

+         user_id=user_id,  # pingou

          name='test2',

+         is_fork=is_fork,

+         parent_id=2 if is_fork else None,

          description='test project #2',

-         hook_token='aaabbbddd',

+         hook_token='aaabbbddd' + hook_token_suffix,

      )

      item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate']

      session.add(item)

  

      item = pagure.lib.model.Project(

-         user_id=1,  # pingou

+         user_id=user_id,  # pingou

          name='test3',

+         is_fork=is_fork,

+         parent_id=3 if is_fork else None,

          description='namespaced test project',

-         hook_token='aaabbbeee',

+         hook_token='aaabbbeee' + hook_token_suffix,

          namespace='somenamespace',

      )

      item.close_status = ['Invalid', 'Insufficient data', 'Fixed', 'Duplicate']

@@ -0,0 +1,160 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ """

+  (c) 2018 - Copyright Red Hat Inc

+ 

+  Authors:

+    Pierre-Yves Chibon <pingou@pingoured.fr>

+ 

+ """

+ 

+ __requires__ = ['SQLAlchemy >= 0.8']

+ import pkg_resources

+ 

+ import datetime

+ import json

+ import unittest

+ import re

+ import shutil

+ import sys

+ import tempfile

+ import time

+ import os

+ 

+ import pygit2

+ from mock import ANY, patch, MagicMock

+ 

+ sys.path.insert(0, os.path.join(os.path.dirname(

+     os.path.abspath(__file__)), '..'))

+ 

+ import pagure

+ import pagure.lib

+ import tests

+ from pagure.lib.repo import PagureRepo

+ 

+ 

+ class PagureFlaskPrIssueLinkTest(tests.Modeltests):

+     """ Tests pagure when linking PRs to tickets """

+ 

+     maxDiff = None

+ 

+     def setUp(self):

+         """ Set up the environnment, ran before every tests. """

+         super(PagureFlaskPrIssueLinkTest, self).setUp()

+ 

+         tests.create_projects(self.session)

+         tests.create_projects(

+             self.session, is_fork=True, user_id=2, hook_token_suffix='bar')

+         tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True)

+         tests.create_projects_git(os.path.join(

+             self.path, 'repos', 'forks', 'foo'), bare=True)

+ 

+         repo = pagure.lib.get_authorized_project(self.session, 'test')

+ 

+         # Create issues to play with

+         msg = pagure.lib.new_issue(

+             session=self.session,

+             repo=repo,

+             title=u'tést íssüé',

+             content='We should work on this',

+             user='pingou',

+             ticketfolder=None

+         )

+         self.session.commit()

+         self.assertEqual(msg.title, u'tést íssüé')

+ 

+         msg = pagure.lib.new_issue(

+             session=self.session,

+             repo=repo,

+             title=u'tést íssüé #2',

+             content='We should still work on this',

+             user='foo',

+             ticketfolder=None

+         )

+         self.session.commit()

+         self.assertEqual(msg.title, u'tést íssüé #2')

+ 

+         # Add a commit to the fork

+ 

+         newpath = tempfile.mkdtemp(prefix='pagure-fork-test')

+         repopath = os.path.join(newpath, 'test')

+         clone_repo = pygit2.clone_repository(os.path.join(

+             self.path, 'repos', 'forks', 'foo', 'test.git'), repopath)

+ 

+         # Create a file in that git repo

+         with open(os.path.join(repopath, 'sources'), 'w') as stream:

+             stream.write('foo\n bar')

+         clone_repo.index.add('sources')

+         clone_repo.index.write()

+ 

+         try:

+             com = repo.revparse_single('HEAD')

+             prev_commit = [com.oid.hex]

+         except:

+             prev_commit = []

+ 

+         # Commits the files added

+         tree = clone_repo.index.write_tree()

+         author = pygit2.Signature(

+             'Alice Author', 'alice@authors.tld')

+         committer = pygit2.Signature(

+             'Cecil Committer', 'cecil@committers.tld')

+         clone_repo.create_commit(

+             'refs/heads/master',  # the name of the reference to update

+             author,

+             committer,

+             'Add sources file for testing\n\n Relates to #2',

+             # binary string representing the tree object ID

+             tree,

+             # list of binary strings representing parents of the new commit

+             prev_commit

+         )

+         refname = 'refs/heads/master:refs/heads/master'

+         ori_remote = clone_repo.remotes[0]

+         PagureRepo.push(ori_remote, refname)

+ 

+         # Create the corresponding PR

+ 

+         repo = pagure.lib.get_authorized_project(self.session, 'test')

+         fork_repo = pagure.lib.get_authorized_project(

+             self.session, 'test', user='foo')

+ 

+         request = pagure.lib.new_pull_request(

+             self.session,

+             branch_from='master',

+             repo_to=repo,

+             branch_to='master',

+             title='test PR',

+             user='foo',

+             requestfolder=None,

+             initial_comment=None,

+             repo_from=fork_repo,

+         )

+         self.session.commit()

+ 

+         pagure.lib.tasks.link_pr_to_ticket(request.uid)

+         self.assertEqual(request.id, 3)

+ 

+     def test_ticket_no_link(self):

+         """ Test that no Related PR(s) block is showing in the issue page.

+         """

+         output = self.app.get('/test/issue/1')

+         self.assertEqual(output.status_code, 200)

+         self.assertNotIn(

+             u'<strong>Related PR(s)</strong>',

+             output.data.decode('utf-8'))

+ 

+     def test_ticket_link(self):

+         """ Test that no Related PR(s) block is showing in the issue page.

+         """

+         time.sleep(1)

+         output = self.app.get('/test/issue/2')

+         print output.data.decode('utf-8')

+         self.assertEqual(output.status_code, 200)

+         self.assertIn(

+             u'<strong>Related PR(s)</strong>',

+             output.data.decode('utf-8'))

+ 

+ 

+ if __name__ == '__main__':

+     unittest.main(verbosity=2)

This still requires tests though :)

2 new commits added

  • Add the possibility to link issues to pull-requests
  • Move _get_parent_repo_path to pagure.utils as get_parent_repo_path
6 years ago

Is there a missing migration for the new model? I'm trying to run it locally via docker-compose and getting ProgrammingError: (psycopg2.ProgrammingError) relation "pr_to_issue" does not exist.

Yes indeed, I added the table locally by running the createdb script but it does need a migration, will add it :)

1 new commit added

  • Add an alembic migration to create the pr_to_issue table
6 years ago

rebased onto 56e7d0cd7580969c28a25633814a23febcf27bc1

6 years ago

rebased onto 8c6f32d2644c899ef08be0fd8aa0891989d2f878

6 years ago

1 new commit added

  • Ensure the order is always consistent
6 years ago

1 new commit added

  • Remove all bare except
6 years ago

rebased onto 81aeff3280bbf9da5ecbceece0820ce1bbec78e2

6 years ago

rebased onto ae078d5a8a71ed3733bd2f34a0155c0605ff1bcc

6 years ago

rebased onto d270fd6e3596329df9c10fe9fc4fc36ea4216da4

6 years ago

rebased onto 888d25c03946bff8c9402f6ce513c5d282ddaf6c

6 years ago

5 new commits added

  • Flake8 fixes
  • Ensure variables are instantiated before trying to use them
  • Add an alembic migration to create the pr_to_issue table
  • Add the possibility to link issues to pull-requests
  • Move _get_parent_repo_path to pagure.utils as get_parent_repo_path
6 years ago

It might be good to pluralize this variable name since it is a list.

Do we want this commit() to be indented, or do you really want it to happen no matter what?

This flush() isn't needed because the commit() below will run a flush() for you.

Well, it's not really no matter what.

But maybe I could flush here and do the commit closer to the end of the work with the appropriate try/except/rollback

rebased onto b58d7a3ee12647c494338980c7483d46eeb05d65

6 years ago

@bowlofeggs I addressed your first concerns

7 new commits added

  • Rely on the list of branches rather than the .empty attribute
  • Fix typo, variable name and restructure when we commit to the DB
  • Flake8 fixes
  • Ensure variables are instantiated before trying to use them
  • Add an alembic migration to create the pr_to_issue table
  • Add the possibility to link issues to pull-requests
  • Move _get_parent_repo_path to pagure.utils as get_parent_repo_path
6 years ago

rebased onto 78ba11d

6 years ago

Pull-Request has been merged by pingou

6 years ago