From 5b06f78b74a616cc0e1fc2804092fb281076e9fd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 06 2017 06:04:57 +0000 Subject: Move deleting a project into its own task This way we can more easily update the gitolite.conf file, remove the git repos from the disk and finally the project from the DB. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/git_auth.py b/pagure/lib/git_auth.py index 14384f2..a5470fd 100644 --- a/pagure/lib/git_auth.py +++ b/pagure/lib/git_auth.py @@ -78,6 +78,22 @@ class GitAuthHelper(object): """ pass + @classmethod + @abc.abstractmethod + def remove_acls(self, session, project): + """ This is the method that is called by pagure to remove a project + from the configuration file. + + :arg cls: the current class + :type: GitAuthHelper + :arg session: the session with which to connect to the database + :arg project: the project to remove from the gitolite configuration + file. + :type project: pagure.lib.model.Project + + """ + pass + def _read_file(filename): """ Reads the specified file and return its content. @@ -420,6 +436,97 @@ class Gitolite2Auth(GitAuthHelper): if postconfig: stream.write(postconfig + '\n') + @classmethod + def remove_acls(cls, session, project): + """ Remove a project from the configuration file for gitolite. + + :arg cls: the current class + :type: Gitolite2Auth + :arg session: the session with which to connect to the database + :arg project: the project to remove from the gitolite configuration + file. + :type project: pagure.lib.model.Project + + """ + _log.info('Remove project from the gitolite configuration file') + + if not project: + raise RuntimeError('Project undefined') + + configfile = pagure.APP.config['GITOLITE_CONFIG'] + preconf = pagure.APP.config.get('GITOLITE_PRE_CONFIG') or None + postconf = pagure.APP.config.get('GITOLITE_POST_CONFIG') or None + + if not os.path.exists(configfile): + _log.info( + 'Not configuration file found at: %s... bailing' % configfile) + return + + preconfig = None + if preconf: + _log.info( + 'Loading the file to include at the top of the generated one') + preconfig = _read_file(preconf) + + postconfig = None + if postconf: + _log.info( + 'Loading the file to include at the end of the generated one') + postconfig = _read_file(postconf) + + config = [] + groups = cls._generate_groups_config(session) + + _log.info('Removing the project from the configuration') + + current_config = cls._get_current_config( + configfile, preconfig, postconfig) + + current_config = cls._clean_current_config( + current_config, project) + + config = current_config + config + + if config: + _log.info('Cleaning the groups from the loaded config') + config = cls._clean_groups(config) + + else: + current_config = cls._get_current_config( + configfile, preconfig, postconfig) + + _log.info( + 'Cleaning the groups from the config on disk') + config = cls._clean_groups(config) + + if not config: + return + + _log.info('Writing the configuration to: %s', configfile) + with open(configfile, 'w') as stream: + if preconfig: + stream.write(preconfig + '\n') + stream.write('# end of header\n') + + if groups: + for key, users in groups.iteritems(): + stream.write('@%s = %s\n' % (key, ' '.join(users))) + stream.write('# end of groups\n\n') + + prev = None + for row in config: + if prev is None: + prev = row + if prev == row == '': + continue + stream.write(row + '\n') + prev = row + + stream.write('# end of body\n') + + if postconfig: + stream.write(postconfig + '\n') + @staticmethod def _get_gitolite_command(): """ Return the gitolite command to run based on the info in the @@ -517,3 +624,22 @@ class GitAuthTestHelper(GitAuthHelper): 'with args: project=%s, group=%s' % (project, group) print(out) return out + + @classmethod + def remove_acls(cls, session, project): + """ Print a statement about which a project would be removed from + the configuration file for gitolite. + + :arg cls: the current class + :type: GitAuthHelper + :arg session: the session with which to connect to the database + :arg project: the project to remove from the gitolite configuration + file. + :type project: pagure.lib.model.Project + + """ + + out = 'Called GitAuthTestHelper.remove_acls() ' \ + 'with args: project=%s' % (project.fullname) + print(out) + return out diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 7fadeca..d9bf248 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -105,19 +105,93 @@ def generate_gitolite_acls(namespace=None, name=None, user=None, group=None): 'Calling helper: %s with arg: project=%s, group=%s', helper, project, group_obj) helper.generate_acls(project=project, group=group_obj) - pagure.lib.update_read_only_mode( - session, project, read_only=False) + + pagure.lib.update_read_only_mode(session, project, read_only=False) try: session.commit() - _log.debug('Project %s is in Read Only Mode', project) + _log.debug('Project %s is no longer in Read Only Mode', project) except SQLAlchemyError: session.rollback() - _log.error( + _log.exception( 'Failed to unmark read_only for: %s project', project) session.remove() gc_clean() +@conn.task(queue=APP.config.get('GITOLITE_CELERY_QUEUE', None)) +def delete_project(namespace=None, name=None, user=None): + """ Delete a project in pagure. + + This is achieved in three steps: + - Remove the project from gitolite.conf + - Remove the git repositories on disk + - Remove the project from the DB + + :kwarg namespace: the namespace of the project + :type namespace: None or str + :kwarg name: the name of the project + :type name: None or str + :kwarg user: the user of the project, only set if the project is a fork + :type user: None or str + + """ + session = pagure.lib.create_session() + project = pagure.lib._get_project( + session, namespace=namespace, name=name, user=user, + case=APP.config.get('CASE_SENSITIVE', False)) + + if not project: + raise RuntimeError( + 'Project: %s/%s from user: %s not found in the DB' % ( + namespace, name, user)) + + # Remove the project from gitolite.conf + helper = pagure.lib.git_auth.get_git_auth_helper( + APP.config['GITOLITE_BACKEND']) + _log.debug('Got helper: %s', helper) + + _log.debug( + 'Calling helper: %s with arg: project=%s', helper, project.fullname) + helper.remove_acls(session=session, project=project) + + # Remove the git repositories on disk + paths = [] + for key in [ + 'GIT_FOLDER', 'DOCS_FOLDER', + 'TICKETS_FOLDER', 'REQUESTS_FOLDER']: + if APP.config[key]: + path = os.path.join(APP.config[key], project.path) + if os.path.exists(path): + paths.append(path) + + try: + for path in paths: + _log.info('Deleting: %s' % path) + shutil.rmtree(path) + except (OSError, IOError) as err: + _log.exception(err) + raise RuntimeError( + 'Could not delete all the repos from the system') + + for path in paths: + _log.info('Path: %s - exists: %s' % (path, os.path.exists(path))) + + # Remove the project from the DB + username = project.user.user + try: + session.delete(project) + session.commit() + except SQLAlchemyError: + session.rollback() + _log.exception( + 'Failed to delete project: %s from the DB', project.fullname) + session.remove() + + gc_clean() + + return ret('view_user', username=username) + + @conn.task def create_project(username, namespace, name, add_readme, ignore_existing_repo): diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 77a3b0c..55bb5ba 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -20,7 +20,6 @@ import datetime import json import logging -import shutil import os from cStringIO import StringIO from math import ceil @@ -47,6 +46,7 @@ import pagure.exceptions import pagure.lib import pagure.lib.git import pagure.lib.plugins +import pagure.lib.tasks import pagure.forms import pagure import pagure.ui.plugins @@ -1434,37 +1434,9 @@ def delete_repo(repo, username=None, namespace=None): 'view_settings', repo=repo.name, username=username, namespace=namespace)) - try: - SESSION.delete(repo) - SESSION.commit() - except SQLAlchemyError as err: # pragma: no cover - SESSION.rollback() - _log.exception(err) - flask.flash('Could not delete the project', 'error') - - paths = [] - for key in [ - 'GIT_FOLDER', 'DOCS_FOLDER', - 'TICKETS_FOLDER', 'REQUESTS_FOLDER']: - if APP.config[key]: - path = os.path.join(APP.config[key], repo.path) - if os.path.exists(path): - paths.append(path) - - try: - for path in paths: - _log.info('Deleting: %s' % path) - shutil.rmtree(path) - except (OSError, IOError) as err: - _log.exception(err) - flask.flash( - 'Could not delete all the repos from the system', 'error') - - for path in paths: - _log.info('Path: %s - exists: %s' % (path, os.path.exists(path))) - - return flask.redirect( - flask.url_for('view_user', username=flask.g.fas_user.username)) + task = pagure.lib.tasks.delete_project.delay( + repo.namespace, repo.name, repo.user.user if repo.is_fork else None) + return pagure.wait_for_task(task.id) @APP.route('//hook_token', methods=['POST']) diff --git a/tests/test_pagure_lib_gitolite_config.py b/tests/test_pagure_lib_gitolite_config.py index d74881f..f6421c6 100644 --- a/tests/test_pagure_lib_gitolite_config.py +++ b/tests/test_pagure_lib_gitolite_config.py @@ -807,6 +807,127 @@ repo requests/somenamespace/test3 #print data self.assertEqual(data, exp) + def test_remove_acls(self): + """ Test the remove_acls function of pagure.lib.git when deleting + a project """ + + with open(self.outputconf, 'w') as stream: + pass + + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls( + self.session, + self.outputconf, + project=-1, + ) + self.assertTrue(os.path.exists(self.outputconf)) + + with open(self.outputconf) as stream: + data = stream.read().decode('utf-8') + + exp = u"""@grp2 = foo +@grp = pingou +# end of groups + +%s + +# end of body +""" % CORE_CONFIG + + #print data + self.assertEqual(data, exp) + + # Test removing a project from the existing config + project = pagure.get_authorized_project( + self.session, project_name='test') + + helper.remove_acls(self.session, project=project) + + with open(self.outputconf) as stream: + data = stream.read().decode('utf-8') + + exp = u"""@grp2 = foo +@grp = pingou +# end of groups + +repo test2 + R = @all + RW+ = pingou + +repo docs/test2 + R = @all + RW+ = pingou + +repo tickets/test2 + RW+ = pingou + +repo requests/test2 + RW+ = pingou + +repo somenamespace/test3 + R = @all + RW+ = pingou + +repo docs/somenamespace/test3 + R = @all + RW+ = pingou + +repo tickets/somenamespace/test3 + RW+ = pingou + +repo requests/somenamespace/test3 + RW+ = pingou + +# end of body +""" + + #print data + self.assertEqual(data, exp) + + def test_remove_acls_no_project(self): + """ Test the remove_acls function of pagure.lib.git when no project + is specified """ + + with open(self.outputconf, 'w') as stream: + pass + + helper = pagure.lib.git_auth.get_git_auth_helper('gitolite3') + helper.write_gitolite_acls( + self.session, + self.outputconf, + project=-1, + ) + self.assertTrue(os.path.exists(self.outputconf)) + + with open(self.outputconf) as stream: + data = stream.read().decode('utf-8') + + exp = u"""@grp2 = foo +@grp = pingou +# end of groups + +%s + +# end of body +""" % CORE_CONFIG + + #print data + self.assertEqual(data, exp) + + # Test nothing changes if no project is specified + + self.assertRaises( + RuntimeError, + helper.remove_acls, + self.session, + project=None + ) + + with open(self.outputconf) as stream: + data = stream.read().decode('utf-8') + + self.assertEqual(data, exp) + if __name__ == '__main__': unittest.main(verbosity=2)