PR#3296: Add a mirroring hook to mirror git repo to other locations - pagure

		`@@ -161,6 +161,16 @@`
		`in the future pull-requests) git repo.`


		`+ %package mirror`
		`+ Summary: The mirroring service for pagure`
		`+ BuildArch: noarch`
		`+ Requires: %{name} = %{version}-%{release}`
		`+ %{?systemd_requires}`
		`+ %description mirror`
		`+ pagure-mirror is the service mirroring projects that asked for it outside`
		`+ of this pagure instance.`
		`+`
		`+`
		`%prep`
		`%autosetup -p1`

		`@@ -226,6 +236,10 @@`
		`install -p -m 644 files/pagure_loadjson.service \`
		`$RPM_BUILD_ROOT/%{_unitdir}/pagure_loadjson.service`

		`+ # Install the systemd file for the mirror service`
		`+ install -p -m 644 files/pagure_mirror.service \`
		`+ $RPM_BUILD_ROOT/%{_unitdir}/pagure_mirror.service`
		`+`
		`# Install the systemd file for the script sending reminder about API key`
		`# expiration`
		`install -p -m 644 files/pagure_api_key_expire_mail.service \`
		`@@ -267,6 +281,8 @@`
		`%systemd_post pagure_logcom.service`
		`%post loadjson`
		`%systemd_post pagure_loadjson.service`
		`+ %post mirror`
		`+ %systemd_post pagure_mirror.service`

		`%preun`
		`%systemd_preun pagure_worker.service`
		`@@ -284,6 +300,8 @@`
		`%systemd_preun pagure_logcom.service`
		`%preun loadjson`
		`%systemd_preun pagure_loadjson.service`
		`+ %preun mirror`
		`+ %systemd_preun pagure_mirror.service`

		`%postun`
		`%systemd_postun_with_restart pagure_worker.service`
		`@@ -301,6 +319,8 @@`
		`%systemd_postun_with_restart pagure_logcom.service`
		`%postun loadjson`
		`%systemd_postun_with_restart pagure_loadjson.service`
		`+ %postun mirror`
		`+ %systemd_postun_with_restart pagure_mirror.service`


		`%files`
		`@@ -363,6 +383,11 @@`
		`%{_unitdir}/pagure_loadjson.service`


		`+ %files mirror`
		`+ %license LICENSE`
		`+ %{_unitdir}/pagure_mirror.service`
		`+`
		`+`
		`%changelog`
		`* Thu Apr 26 2018 Pierre-Yves Chibon <pingou@pingoured.fr> - 4.0.1-1`
		`- Update to 4.0.1`

files/pagure_mirror.service

file added

+19

		`@@ -0,0 +1,19 @@`
		`+ # This is a systemd's service file for the mirroring service, if you change`
		`+ # the default value of the CI_CELERY_QUEUE configuration key, do not`
		`+ # forget to edit it in the ExecStart line below`
		`+`
		`+ [Unit]`
		`+ Description=Pagure service mirroring projects outside of pagure that asked for it`
		`+ After=redis.target`
		`+ Documentation=https://pagure.io/pagure`
		`+`
		`+ [Service]`
		`+ ExecStart=/usr/bin/celery worker -A pagure.lib.tasks_mirror --loglevel=info -Q pagure_mirror`
		`+ Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg"`
		`+ Type=simple`
		`+ User=mirror`
		`+ Group=mirror`
		`+ Restart=on-failure`
		`+`
		`+ [Install]`
		`+ WantedBy=multi-user.target`

pagure/default_config.py

file modified

		`@@ -105,6 +105,7 @@`
		`LOGCOM_CELERY_QUEUE = 'pagure_logcom'`
		`LOADJSON_CELERY_QUEUE = 'pagure_loadjson'`
		`CI_CELERY_QUEUE = 'pagure_ci'`
		`+ MIRRORING_QUEUE = 'pagure_mirror'`

		`# Number of items displayed per page`
		`ITEM_PER_PAGE = 48`
		`@@ -126,6 +127,9 @@`
		`REDIS_DB = 0`
		`EVENTSOURCE_PORT = 8080`

		`+ # Folder where to place the ssh keys for the mirroring feature`
		`+ MIRROR_SSHKEYS_FOLDER = '/var/lib/pagure/sshkeys/'`
		`+`
		`# Folder containing to the git repos`
		`# Note that this must be exactly the same as GL_REPO_BASE in gitolite.rc`
		`GIT_FOLDER = os.path.join(`

pagure/hooks/files/mirror.py

file added

+60

		`@@ -0,0 +1,60 @@`
		`+ #! /usr/bin/env python`
		`+`
		`+`
		`+ """Pagure specific hook to mirror a repo to another location.`
		`+ """`
		`+ from __future__ import unicode_literals, print_function`
		`+`
		`+`
		`+ import logging`
		`+ import os`
		`+ import sys`
		`+`
		`+`
		`+ if 'PAGURE_CONFIG' not in os.environ \`
		`+ and os.path.exists('/etc/pagure/pagure.cfg'):`
		`+ os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'`
		`+`
		`+`
		`+ import pagure.config # noqa: E402`
		`+ import pagure.exceptions # noqa: E402`
		`+ import pagure.lib # noqa: E402`
		`+ import pagure.lib.tasks_mirror # noqa: E402`
		`+ import pagure.ui.plugins # noqa: E402`
		`+`
		`+`
		`+ _log = logging.getLogger(__name__)`
		`+ _config = pagure.config.config`
		`+ abspath = os.path.abspath(os.environ['GIT_DIR'])`
		`+`
		`+`
		`+ def main(args):`
		`+`
		`+ repo = pagure.lib.git.get_repo_name(abspath)`
		`+ username = pagure.lib.git.get_username(abspath)`
		`+ namespace = pagure.lib.git.get_repo_namespace(abspath)`
		`+ if _config.get('HOOK_DEBUG', False):`
		`+ print('repo:', repo)`
		`+ print('user:', username)`
		`+ print('namespace:', namespace)`
		`+`
		`+ session = pagure.lib.create_session(_config['DB_URL'])`
		`+ project = pagure.lib._get_project(`
		`+ session, repo, user=username, namespace=namespace)`
		`+`
		`+ if not project:`
		`+ print('Could not find a project corresponding to this git repo')`
		`+ session.close()`
		`+ return 1`
		`+`
		`+ pagure.lib.tasks_mirror.mirror_project.delay(`
		`+ username=project.user.user if project.is_fork else None,`
		`+ namespace=project.namespace,`
		`+ name=project.name)`
		`+`
		`+ session.close()`
		`+ return 0`
		`+`
		`+`
		`+ if __name__ == '__main__':`
		`+ main(sys.argv[1:])`

pagure/hooks/files/pagure_hook.py

file modified

+1 -1

		`@@ -20,7 +20,7 @@`
		`os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'`


		`- import pagure # noqa: E402`
		`+ import pagure.config # noqa: E402`
		`import pagure.exceptions # noqa: E402`
		`import pagure.lib.link # noqa: E402`

pagure/hooks/mirror_hook.py

file added

+158

		`@@ -0,0 +1,158 @@`
		`+ # -- coding: utf-8 --`
		`+`
		`+ """`
		`+ (c) 2016-2018 - Copyright Red Hat Inc`
		`+`
		`+ Authors:`
		`+ Pierre-Yves Chibon <pingou@pingoured.fr>`
		`+`
		`+ """`
		`+`
		`+`
		`+ import sqlalchemy as sa`
		`+ import wtforms`
		`+`
		`+ try:`
		`+ from flask_wtf import FlaskForm`
		`+ except ImportError:`
		`+ from flask_wtf import Form as FlaskForm`
		`+ from sqlalchemy.orm import relation`
		`+ from sqlalchemy.orm import backref`
		`+`
		`+ import pagure.lib.tasks_mirror`
		`+ from pagure.hooks import BaseHook, RequiredIf`
		`+ from pagure.lib.model import BASE, Project`
		`+ from pagure.utils import get_repo_path, ssh_urlpattern`
		`+`
		`+`
		`+ class MirrorTable(BASE):`
		`+ """ Stores information about the mirroring hook deployed on a project.`
		`+`
		`+ Table -- mirror_pagure`
		`+ """`
		`+`
		`+ __tablename__ = 'hook_mirror'`
		`+`
		`+ id = sa.Column(sa.Integer, primary_key=True)`
		`+ project_id = sa.Column(`
		`+ sa.Integer,`
		`+ sa.ForeignKey(`
		`+ 'projects.id', onupdate='CASCADE', ondelete='CASCADE'),`
		`+ nullable=False,`
		`+ unique=True,`
		`+ index=True)`
		`+`
		`+ active = sa.Column(sa.Boolean, nullable=False, default=False)`
		`+`
		`+ public_key = sa.Column(sa.Text, nullable=True)`
		`+ target = sa.Column(sa.Text, nullable=True)`
		`+ last_log = sa.Column(sa.Text, nullable=True)`
		`+`
		`+ project = relation(`
		`+ 'Project', remote_side=[Project.id],`
		`+ backref=backref(`
		`+ 'mirror_hook', cascade="delete, delete-orphan",`
		`+ single_parent=True, uselist=False)`
		`+ )`
		`+`
		`+`
		`+ class CustomRegexp(wtforms.validators.Regexp):`
		`+`
		`+ def __init__(self, args, *kwargs):`
		`+ self.optional = kwargs.get('optional') or False`
		`+ if self.optional:`
		`+ kwargs.pop('optional')`
		`+ super(CustomRegexp, self).__init__(args, *kwargs)`
		`+`
		`+ def __call__(self, form, field):`
		`+ if self.optional:`
		`+ if field.data:`
		`+ return super(CustomRegexp, self).__call__(form, field)`
		`+ else:`
		`+ return super(CustomRegexp, self).__call__(form, field)`
		`+`
		`+`
		`+ class MirrorForm(FlaskForm):`
		`+ ''' Form to configure the mirror hook. '''`
		`+ active = wtforms.BooleanField(`
		`+ 'Active',`
		`+ [wtforms.validators.Optional()]`
		`+ )`
		`+`
		`+ target = wtforms.TextField(`
		`+ 'Git repo to mirror to',`
		`+ [`
		`+ RequiredIf('active'),`
		`+ CustomRegexp(ssh_urlpattern, optional=True),`
		`+ ]`
		`+ )`
		`+`
		`+ public_key = wtforms.TextAreaField(`
		`+ 'Public SSH key',`
		`+ [wtforms.validators.Optional()]`
		`+ )`
		`+ last_log = wtforms.TextAreaField(`
		`+ 'Log of the last sync:',`
		`+ [wtforms.validators.Optional()]`
		`+ )`
		`+`
		`+`
		`+ DESCRIPTION = '''`
		`+ Pagure specific hook to mirror a repo hosted on pagure to another location.`
		`+`
		`+ The first field below should contain the URL to be set in the git configuration`
		`+ as the URL of the git repository to mirror to.`
		`+ It's format is going to be something like:`
		`+`
		`+ <user>@<host>:<path>`
		`+`
		`+ The public SSH key is being generated by pagure and will be available in this`
		`+ page shortly after the activation of this hook. Just refresh the page until`
		`+ it shows up.`
		`+`
		`+ Finally the log of the last sync at the bottom is meant.`
		`+ '''`
		`+`
		`+`
		`+ class MirrorHook(BaseHook):`
		`+ ''' Mirror hook. '''`
		`+`
		`+ name = 'Mirroring'`
		`+ description = DESCRIPTION`
		`+ form = MirrorForm`
		`+ db_object = MirrorTable`
		`+ backref = 'mirror_hook'`
		`+ form_fields = ['active', 'target', 'public_key', 'last_log']`
		`+ form_fields_readonly = ['public_key', 'last_log']`
		`+`
		`+ @classmethod`
		`+ def install(cls, project, dbobj):`
		`+ ''' Method called to install the hook for a project.`
		`+`
		+ :arg project: a ``pagure.model.Project`` object to which the hook
		`+ should be installed`
		`+`
		`+ '''`
		`+ pagure.lib.tasks_mirror.setup_mirroring.delay(`
		`+ username=project.user.user if project.is_fork else None,`
		`+ namespace=project.namespace,`
		`+ name=project.name)`
		`+`
		`+ repopaths = [get_repo_path(project)]`
		`+ cls.base_install(repopaths, dbobj, 'mirror', 'mirror.py')`
		`+`
		`+ @classmethod`
		`+ def remove(cls, project):`
		`+ ''' Method called to remove the hook of a project.`
		`+`
		+ :arg project: a ``pagure.model.Project`` object to which the hook
		`+ should be installed`
		`+`
		`+ '''`
		`+ pagure.lib.tasks_mirror.teardown_mirroring.delay(`
		`+ username=project.user.user if project.is_fork else None,`
		`+ namespace=project.namespace,`
		`+ name=project.name)`
		`+`
		`+ repopaths = [get_repo_path(project)]`
		`+ cls.base_remove(repopaths, 'mirror')`

pagure/lib/__init__.py

file modified

+4 -3

		`@@ -62,6 +62,7 @@`
		`from pagure.config import config as pagure_config`
		`from pagure.lib import model`
		`from pagure.lib import tasks`
		`+ from pagure.lib import tasks_services`


		`REDIS = None`
		`@@ -1314,7 +1315,7 @@`
		`and request.project.ci_hook \`
		`and request.project.ci_hook.active_pr \`
		`and not request.project.private:`
		`- pagure.lib.tasks_services.trigger_ci_build.delay(`
		`+ tasks_services.trigger_ci_build.delay(`
		`project_name=request.project_from.fullname,`
		`cause=request.id,`
		`branch=request.branch_from,`
		`@@ -1336,7 +1337,7 @@`
		`and pagure_config.get('PAGURE_CI_SERVICES') \`
		`and request.project.ci_hook \`
		`and request.project.ci_hook.active_pr:`
		`- pagure.lib.tasks_services.trigger_ci_build.delay(`
		`+ tasks_services.trigger_ci_build.delay(`
		`project_name=request.project_from.fullname,`
		`cause=request.id,`
		`branch=request.branch_from,`
		`@@ -1818,7 +1819,7 @@`
		`and request.project.ci_hook \`
		`and request.project.ci_hook.active_pr \`
		`and not request.project.private:`
		`- pagure.lib.tasks_services.trigger_ci_build.delay(`
		`+ tasks_services.trigger_ci_build.delay(`
		`project_name=request.project_from.fullname,`
		`cause=request.id,`
		`branch=request.branch_from,`

pagure/lib/git.py

file modified

+41 -9

		`@@ -1002,8 +1002,29 @@`
		`return os.path.join('files', filename)`


		`- def read_output(cmd, abspath, input=None, keepends=False, **kw):`
		`- """ Read the output from the given command to run """`
		`+ def read_output(cmd, abspath, input=None, keepends=False, error=False, **kw):`
		`+ """ Read the output from the given command to run.`
		`+`
		`+ cmd:`
		`+ The command to run, this is a list with each space separated into an`
		`+ element of the list.`
		`+ abspath:`
		`+ The absolute path where the command should be ran.`
		`+ input:`
		`+ Whether the command should take input from stdin or not.`
		`+ (Defaults to False)`
		`+ keepends:`
		`+ Whether to strip the newline characters at the end of the standard`
		`+ output or not.`
		`+ error:`
		`+ Whether to return both the standard output and the standard error,`
		`+ or just the standard output.`
		`+ (Defaults to False).`
		`+ kw*:`
		`+ Any other arguments to be passed onto the subprocess.Popen command,`
		`+ such as env, shell, executable...`
		`+`
		`+ """`
		`if input:`
		`stdin = subprocess.PIPE`
		`else:`
		`@@ -1025,24 +1046,35 @@`
		`print(err)`
		`if not keepends:`
		`out = out.rstrip('\n\r')`
		`- return out`
		`+`
		`+ if error:`
		`+ return (out, err)`
		`+ else:`
		`+ return out`


		`- def read_git_output(args, abspath, input=None, keepends=False, **kw):`
		`+ def read_git_output(`
		`+ args, abspath, input=None, keepends=False, error=False, **kw):`
		`"""Read the output of a Git command."""`

		`return read_output(`
		`- ['git'] + args, abspath, input=input, keepends=keepends, **kw)`
		`+ ['git'] + args, abspath, input=input,`
		`+ keepends=keepends, error=error, **kw)`


		`- def read_git_lines(args, abspath, keepends=False, **kw):`
		`+ def read_git_lines(args, abspath, keepends=False, error=False, **kw):`
		`"""Return the lines output by Git command.`

		`Return as single lines, with newlines stripped off."""`

		`- return read_git_output(`
		`- args, abspath, keepends=keepends, **kw`
		`- ).splitlines(keepends)`
		`+ if error:`
		`+ return read_git_output(`
		`+ args, abspath, keepends=keepends, error=error, **kw`
		`+ )`
		`+ else:`
		`+ return read_git_output(`
		`+ args, abspath, keepends=keepends, **kw`
		`+ ).splitlines(keepends)`


		`def get_revs_between(oldrev, newrev, abspath, refname, forced=False):`

pagure/lib/tasks_mirror.py

file added

+256

		`@@ -0,0 +1,256 @@`
		`+ # -- coding: utf-8 --`
		`+`
		`+ """`
		`+ (c) 2018 - Copyright Red Hat Inc`
		`+`
		`+ Authors:`
		`+ Pierre-Yves Chibon <pingou@pingoured.fr>`
		`+`
		`+ """`
		`+`
		`+ from __future__ import unicode_literals`
		`+`
		`+ import base64`
		`+ import logging`
		`+ import os`
		`+ import shutil`
		`+ import stat`
		`+ import struct`
		`+ import tempfile`
		`+`
		`+ import pygit2`
		`+ import six`
		`+ import werkzeug`
		`+`
		`+ from celery import Celery`
		`+ from cryptography import utils`
		`+ from cryptography.hazmat.backends import default_backend`
		`+ from cryptography.hazmat.primitives.asymmetric import rsa`
		`+ from cryptography.hazmat.primitives import serialization`
		`+`
		`+ import pagure.lib`
		`+ from pagure.config import config as pagure_config`
		`+ from pagure.lib.tasks import pagure_task`
		`+ from pagure.utils import ssh_urlpattern`
		`+`
		`+ # logging.config.dictConfig(pagure_config.get('LOGGING') or {'version': 1})`
		`+ _log = logging.getLogger(__name__)`
		`+`
		`+`
		`+ if os.environ.get('PAGURE_BROKER_URL'): # pragma: no-cover`
		`+ broker_url = os.environ['PAGURE_BROKER_URL']`
		`+ elif pagure_config.get('BROKER_URL'):`
		`+ broker_url = pagure_config['BROKER_URL']`
		`+ else:`
		`+ broker_url = 'redis://%s' % pagure_config['REDIS_HOST']`
		`+`
		`+ conn = Celery('tasks_mirror', broker=broker_url, backend=broker_url)`
		`+ conn.conf.update(pagure_config['CELERY_CONFIG'])`
		`+`
		`+`
		`+ # Code from:`
		`+ # https://github.com/pyca/cryptography/blob/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/primitives/serialization.py#L161`
		`+ # Taken from upstream cryptography since the version we have is too old`
		`+ # and doesn't have this code (yet)`
		`+ def _ssh_write_string(data):`
		`+ return struct.pack(">I", len(data)) + data`
		`+`
		`+`
		`+ def _ssh_write_mpint(value):`
		`+ data = utils.int_to_bytes(value)`
		`+ if six.indexbytes(data, 0) & 0x80:`
		`+ data = b"\x00" + data`
		`+ return _ssh_write_string(data)`
		`+`
		`+`
		`+ # Code from _openssh_public_key_bytes at:`
		`+ # https://github.com/pyca/cryptography/tree/6b08aba7f1eb296461528328a3c9871fa7594fc4/src/cryptography/hazmat/backends/openssl#L1616`
		`+ # Taken from upstream cryptography since the version we have is too old`
		`+ # and doesn't have this code (yet)`
		`+ def _serialize_public_ssh_key(key):`
		`+ if isinstance(key, rsa.RSAPublicKey):`
		`+ public_numbers = key.public_numbers()`
		`+ return b"ssh-rsa " + base64.b64encode(`
		`+ _ssh_write_string(b"ssh-rsa") +`
		`+ _ssh_write_mpint(public_numbers.e) +`
		`+ _ssh_write_mpint(public_numbers.n)`
		`+ )`
		`+ else:`
		`+ # Since we only write RSA keys, drop the other serializations`
		`+ return`
		`+`
		`+`
		`+ def _create_ssh_key(keyfile):`
		`+ ''' Create the public and private ssh keys.`
		`+`
		`+ The specified file name will be the private key and the public one will`
		`+ be in a similar file name ending with a '.pub'.`
		`+`
		`+ '''`
		`+ private_key = rsa.generate_private_key(`
		`+ public_exponent=65537,`
		`+ key_size=4096,`
		`+ backend=default_backend()`
		`+ )`
		`+`
		`+ private_pem = private_key.private_bytes(`
		`+ encoding=serialization.Encoding.PEM,`
		`+ format=serialization.PrivateFormat.TraditionalOpenSSL,`
		`+ encryption_algorithm=serialization.NoEncryption()`
		`+ )`
		`+ with os.fdopen(os.open(`
		`+ keyfile, os.O_WRONLY \| os.O_CREAT \| os.O_EXCL, 0o600), 'wb')\`
		`+ as stream:`
		`+ stream.write(private_pem)`
		`+`
		`+ public_key = private_key.public_key()`
		`+ public_pem = _serialize_public_ssh_key(public_key)`
		`+ if public_pem:`
		`+ with open(keyfile + '.pub', 'wb') as stream:`
		`+ stream.write(public_pem)`
		`+`
		`+`
		`+ @conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True)`
		`+ @pagure_task`
		`+ def setup_mirroring(self, session, username, namespace, name):`
		`+ ''' Setup the specified project for mirroring.`
		`+ '''`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ plugin.db_object()`
		`+`
		`+ project = pagure.lib._get_project(`
		`+ session, namespace=namespace, name=name, user=username)`
		`+`
		`+ public_key_name = werkzeug.secure_filename(project.fullname)`
		`+ ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER']`
		`+`
		`+ if not os.path.exists(ssh_folder):`
		`+ os.makedirs(ssh_folder, mode=0o700)`
		`+ else:`
		`+ if os.path.islink(ssh_folder):`
		`+ raise pagure.exceptions.PagureException(`
		`+ 'SSH folder is a link')`
		`+ folder_stat = os.stat(ssh_folder)`
		`+ filemode = stat.S_IMODE(folder_stat.st_mode)`
		`+ if filemode != int('0700', 8):`
		`+ raise pagure.exceptions.PagureException(`
		`+ 'SSH folder had invalid permissions')`
		`+ if folder_stat.st_uid != os.getuid() \`
		`+ or folder_stat.st_gid != os.getgid():`
		`+ raise pagure.exceptions.PagureException(`
		`+ 'SSH folder does not belong to the user or group running '`
		`+ 'this task')`
		`+`
		`+ public_key_file = os.path.join(ssh_folder, '%s.pub' % public_key_name)`
		`+ _log.info('Public key of interest: %s', public_key_file)`
		`+`
		`+ if os.path.exists(public_key_file):`
		`+ raise pagure.exceptions.PagureException('SSH key already exists')`
		`+`
		`+ _log.info('Creating public key')`
		`+ _create_ssh_key(os.path.join(ssh_folder, public_key_name))`
		`+`
		`+ with open(public_key_file) as stream:`
		`+ public_key = stream.read()`
		`+`
		`+ if project.mirror_hook.public_key != public_key:`
		`+ _log.info('Updating information in the DB')`
		`+ project.mirror_hook.public_key = public_key`
		`+ session.add(project.mirror_hook)`
		`+ session.commit()`
		`+`
		`+`
		`+ @conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True)`
		`+ @pagure_task`
		`+ def teardown_mirroring(self, session, username, namespace, name):`
		`+ ''' Stop the mirroring of the specified project.`
		`+ '''`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ plugin.db_object()`
		`+`
		`+ project = pagure.lib._get_project(`
		`+ session, namespace=namespace, name=name, user=username)`
		`+`
		`+ ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER']`
		`+`
		`+ public_key_name = werkzeug.secure_filename(project.fullname)`
		`+ private_key_file = os.path.join(ssh_folder, public_key_name)`
		`+ public_key_file = os.path.join(`
		`+ ssh_folder, '%s.pub' % public_key_name)`
		`+`
		`+ if os.path.exists(private_key_file):`
		`+ os.unlink(private_key_file)`
		`+`
		`+ if os.path.exists(public_key_file):`
		`+ os.unlink(public_key_file)`
		`+`
		`+ project.mirror_hook.public_key = None`
		`+ session.add(project.mirror_hook)`
		`+ session.commit()`
		`+`
		`+`
		`+ @conn.task(queue=pagure_config['MIRRORING_QUEUE'], bind=True)`
		`+ @pagure_task`
		`+ def mirror_project(self, session, username, namespace, name):`
		`+ ''' Does the actual mirroring of the specified project.`
		`+ '''`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ plugin.db_object()`
		`+`
		`+ project = pagure.lib._get_project(`
		`+ session, namespace=namespace, name=name, user=username)`
		`+`
		`+ repofolder = pagure_config['GIT_FOLDER']`
		`+ repopath = os.path.join(repofolder, project.path)`
		`+ if not os.path.exists(repopath):`
		`+ _log.info('Git folder not found at: %s, bailing', repopath)`
		`+ return`
		`+`
		`+ newpath = tempfile.mkdtemp(prefix='pagure-mirror-')`
		`+ pygit2.clone_repository(repopath, newpath)`
		`+`
		`+ ssh_folder = pagure_config['MIRROR_SSHKEYS_FOLDER']`
		`+ public_key_name = werkzeug.secure_filename(project.fullname)`
		`+ private_key_file = os.path.join(ssh_folder, public_key_name)`
		`+`
		`+ # Get the list of remotes`
		`+ remotes = [`
		`+ remote.strip()`
		`+ for remote in project.mirror_hook.target.split('\n')`
		`+ if project.mirror_hook and remote.strip()`
		`+ and ssh_urlpattern.match(remote.strip())`
		`+ ]`
		`+`
		`+ # Add the remotes`
		`+ for idx, remote in enumerate(remotes):`
		`+ remote_name = '%s_%s' % (public_key_name, idx)`
		`+ _log.info('Adding remote %s as %s', remote, remote_name)`
		`+ (stdout, stderr) = pagure.lib.git.read_git_lines(`
		`+ ['remote', 'add', remote_name, remote, '--mirror=push'],`
		`+ abspath=newpath, error=True)`
		`+ _log.info(`
		`+ "Output from git remote add:\n stdout: %s\n stderr: %s",`
		`+ stdout, stderr)`
		`+`
		`+ # Push`
		`+ logs = []`
		`+ for idx, remote in enumerate(remotes):`
		`+ remote_name = '%s_%s' % (public_key_name, idx)`
		`+ _log.info(`
		`+ 'Pushing to remote %s using key: %s', remote_name,`
		`+ private_key_file)`
		`+ (stdout, stderr) = pagure.lib.git.read_git_lines(`
		`+ ['push', remote_name],`
		`+ abspath=newpath, error=True,`
		`+ env={'GIT_SSH_COMMAND': 'ssh -i %s' % private_key_file})`
		`+ log = "Output from the push:\n stdout: %s\n stderr: %s" % (`
		`+ stdout, stderr)`
		`+ logs.append(log)`
		`+ if logs:`
		`+ project.mirror_hook.last_log = '\n'.join(logs)`
		`+ session.add(project.mirror_hook)`
		`+ session.commit()`
		`+ _log.info('\n'.join(logs))`
		`+`
		`+ # Remove the clone`
		`+ shutil.rmtree(newpath)`

pagure/templates/_formhelper.html

file modified

+2 -2

		`@@ -71,10 +71,10 @@`
		`</div>`
		`{% endmacro %}`

		`- {% macro render_field_in_row(field, after="") %}`
		`+ {% macro render_field_in_row(field, after="", readonly=False) %}`
		`<tr>`
		`<td style="padding-right:2em">{{ field.label }}</td>`
		`- <td>{{ field(class="form-control")\|safe }}</td>`
		`+ <td>{{ field(class="form-control", readonly=readonly)\|safe }}</td>`
		`{% if after %} <td>{{ after }}</td>{% endif %}`
		`{% if field.errors %}`
		`{% for error in field.errors %}`

pagure/templates/plugin.html

file modified

		`@@ -22,7 +22,11 @@`

		`<table>`
		`{% for field in fields %}`
		`+ {% if field.id in form_fields_readonly %}`
		`+ {{ render_field_in_row(field, readonly=True) }}`
		`+ {% else %}`
		`{{ render_field_in_row(field) }}`
		`+ {% endif %}`
		`{% endfor %}`
		`</table>`

pagure/ui/plugins.py

file modified

+7 -1

		`@@ -108,6 +108,10 @@`
		`for field in plugin.form_fields:`
		`fields.append(getattr(form, field))`

		`+ form_fields_readonly = []`
		`+ if hasattr(plugin, 'form_fields_readonly'):`
		`+ form_fields_readonly = plugin.form_fields_readonly`
		`+`
		`if form.validate_on_submit():`
		`form.populate_obj(obj=dbobj)`

		`@@ -174,4 +178,6 @@`
		`username=username,`
		`plugin=plugin,`
		`form=form,`
		`- fields=fields)`
		`+ fields=fields,`
		`+ form_fields_readonly=form_fields_readonly,`
		`+ )`

pagure/utils.py

file modified

+41

		`@@ -344,6 +344,47 @@`
		`urlpattern = re.compile(urlregex)`


		`+ ssh_urlregex = re.compile(`
		`+ u"^"`
		`+ # protocol identifier`
		`+ u"(?:(?:(git\+)?ssh)://)"`
		`+ # user:pass authentication`
		`+ u"(?:\S+(?::\S*)?@)?"`
		`+ u"(?:"`
		`+ u"(?P<private_ip>"`
		`+ # IP address exclusion`
		`+ # private & local networks`
		`+ u"(?:(?:10\|127)" + ip_middle_octet + u"{2}" + ip_last_octet + u")\|"`
		`+ u"(?:(?:169\.254\|192\.168)" + ip_middle_octet + ip_last_octet + u")\|"`
		`+ u"(?:172\.(?:1[6-9]\|2\d\|3[0-1])" + ip_middle_octet + ip_last_octet + u"))"`
		`+ u"\|"`
		`+ # IP address dotted notation octets`
		`+ # excludes loopback network 0.0.0.0`
		`+ # excludes reserved space >= 224.0.0.0`
		`+ # excludes network & broadcast addresses`
		`+ # (first & last IP address of each class)`
		`+ u"(?P<public_ip>"`
		`+ u"(?:[1-9]\d?\|1\d\d\|2[01]\d\|22[0-3])"`
		`+ u"" + ip_middle_octet + u"{2}"`
		`+ u"" + ip_last_octet + u")"`
		`+ u"\|"`
		`+ # host name`
		`+ u"(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)"`
		`+ # domain name`
		`+ u"(?:\.(?:[a-z\u00a1-\uffff0-9]-?)[a-z\u00a1-\uffff0-9]+)"`
		`+ # TLD identifier`
		`+ u"(?:\.(?:[a-z\u00a1-\uffff]{2,}))"`
		`+ u")"`
		`+ # port number`
		`+ u"(?::\d{2,5})?"`
		`+ # resource path`
		`+ u"(?:/\S*)?"`
		`+ u"$",`
		`+ re.UNICODE \| re.IGNORECASE`
		`+ )`
		`+ ssh_urlpattern = re.compile(ssh_urlregex)`
		`+`
		`+`
		`def get_repo_path(repo):`
		`""" Return the path of the git repository corresponding to the provided`
		`Repository object from the DB.`

tests/__init__.py

file modified

		`@@ -362,6 +362,7 @@`
		`pagure_config.update(reload_config())`

		`imp.reload(pagure.lib.tasks)`
		`+ imp.reload(pagure.lib.tasks_mirror)`
		`imp.reload(pagure.lib.tasks_services)`

		`self._app = pagure.flask_app.create_app({'DB_URL': self.dbpath})`

tests/test_pagure_flask_ui_plugins.py

file modified

		`@@ -50,6 +50,7 @@`
		`'Fedmsg',`
		`'IRC',`
		`'Mail',`
		`+ 'Mirroring',`
		`'Pagure',`
		`'Pagure CI',`
		`'Pagure requests',`

tests/test_pagure_flask_ui_plugins_mirror.py

file added

+232

		`@@ -0,0 +1,232 @@`
		`+ # -- coding: utf-8 --`
		`+`
		`+ """`
		`+ (c) 2018 - Copyright Red Hat Inc`
		`+`
		`+ Authors:`
		`+ Pierre-Yves Chibon <pingou@pingoured.fr>`
		`+`
		`+ """`
		`+`
		`+ from __future__ import unicode_literals`
		`+`
		`+ __requires__ = ['SQLAlchemy >= 0.8']`
		`+`
		`+ import unittest`
		`+ import sys`
		`+ import os`
		`+`
		`+ from mock import patch`
		`+`
		`+ sys.path.insert(0, os.path.join(os.path.dirname(`
		`+ os.path.abspath(__file__)), '..'))`
		`+`
		`+ import pagure.lib`
		`+ import tests`
		`+`
		`+`
		`+ class PagureFlaskPluginMirrortests(tests.Modeltests):`
		`+ """ Tests for mirror plugin of pagure """`
		`+`
		`+ def setUp(self):`
		`+ """ Set up the environnment, ran before every tests. """`
		`+ super(PagureFlaskPluginMirrortests, self).setUp()`
		`+`
		`+ tests.create_projects(self.session)`
		`+ tests.create_projects_git(os.path.join(self.path, 'repos'))`
		`+`
		`+ def test_plugin_mirror_no_csrf(self):`
		`+ """ Test setting up the mirror plugin with no csrf. """`
		`+`
		`+ user = tests.FakeUser(username='pingou')`
		`+ with tests.user_set(self.app.application, user):`
		`+ output = self.app.get('/test/settings/Mirroring')`
		`+ self.assertEqual(output.status_code, 200)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings Mirroring - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '<input class="form-control" id="active" name="active" '`
		`+ 'type="checkbox" value="y">', output_text)`
		`+`
		`+ data = {}`
		`+`
		`+ output = self.app.post('/test/settings/Mirroring', data=data)`
		`+ self.assertEqual(output.status_code, 200)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings Mirroring - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '<input class="form-control" id="active" name="active" '`
		`+ 'type="checkbox" value="y">', output_text)`
		`+`
		`+ self.assertFalse(os.path.exists(os.path.join(`
		`+ self.path, 'repos', 'test.git', 'hooks',`
		`+ 'post-receive.mirror')))`
		`+`
		`+ def test_plugin_mirror_no_data(self):`
		`+ """ Test the setting up the mirror plugin when there are no data`
		`+ provided in the request.`
		`+ """`
		`+`
		`+ user = tests.FakeUser(username='pingou')`
		`+ with tests.user_set(self.app.application, user):`
		`+ csrf_token = self.get_csrf()`
		`+`
		`+ data = {'csrf_token': csrf_token}`
		`+`
		`+ # With the git repo`
		`+ output = self.app.post(`
		`+ '/test/settings/Mirroring', data=data, follow_redirects=True)`
		`+ self.assertEqual(output.status_code, 200)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings - test - Pagure</title>', output_text)`
		`+ self.assertIn(`
		`+ '</i> Hook Mirroring deactivated</div>',`
		`+ output_text)`
		`+`
		`+ output = self.app.get('/test/settings/Mirroring', data=data)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings Mirroring - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '<input class="form-control" id="active" name="active" '`
		`+ 'type="checkbox" value="y">', output_text)`
		`+`
		`+ self.assertFalse(os.path.exists(os.path.join(`
		`+ self.path, 'repos', 'test.git', 'hooks',`
		`+ 'post-receive.mirror')))`
		`+`
		`+ def test_plugin_mirror_invalid_target(self):`
		`+ """ Test the setting up the mirror plugin when there are the target`
		`+ provided is invalid.`
		`+ """`
		`+`
		`+ user = tests.FakeUser(username='pingou')`
		`+ with tests.user_set(self.app.application, user):`
		`+ csrf_token = self.get_csrf()`
		`+`
		`+ data = {`
		`+ 'csrf_token': csrf_token,`
		`+ 'active': True,`
		`+ 'target': 'https://host.org/target',`
		`+ }`
		`+`
		`+ # With the git repo`
		`+ output = self.app.post(`
		`+ '/test/settings/Mirroring', data=data, follow_redirects=True)`
		`+ self.assertEqual(output.status_code, 200)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings Mirroring - test - Pagure</title>',`
		`+ output_text)`
		`+ if self.get_wtforms_version() >= (2, 2):`
		`+ self.assertIn(`
		`+ '<td><input class="form-control" id="target" name="target" '`
		`+ 'required type="text" value="https://host.org/target">'`
		`+ '</td>\n<td class="errors">Invalid input.</td>',`
		`+ output_text)`
		`+ else:`
		`+ self.assertIn(`
		`+ '<td><input class="form-control" id="target" name="target" '`
		`+ 'type="text" value="https://host.org/target"></td>\n'`
		`+ '<td class="errors">Invalid input.</td>', output_text)`
		`+`
		`+ output = self.app.get('/test/settings/Mirroring', data=data)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings Mirroring - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '<input class="form-control" id="active" name="active" '`
		`+ 'type="checkbox" value="y">', output_text)`
		`+`
		`+ self.assertFalse(os.path.exists(os.path.join(`
		`+ self.path, 'repos', 'test.git', 'hooks',`
		`+ 'post-receive.mirror')))`
		`+`
		`+ def test_setting_up_mirror(self):`
		`+ """ Test the setting up the mirror plugin.`
		`+ """`
		`+`
		`+ user = tests.FakeUser(username='pingou')`
		`+ with tests.user_set(self.app.application, user):`
		`+ csrf_token = self.get_csrf()`
		`+`
		`+ data = {`
		`+ 'csrf_token': csrf_token,`
		`+ 'active': True,`
		`+ 'target': 'ssh://user@host.org/target',`
		`+ }`
		`+`
		`+ # With the git repo`
		`+ output = self.app.post(`
		`+ '/test/settings/Mirroring', data=data, follow_redirects=True)`
		`+ self.assertEqual(output.status_code, 200)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '</i> Hook Mirroring activated</div>',`
		`+ output_text)`
		`+`
		`+ output = self.app.get('/test/settings/Mirroring', data=data)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings Mirroring - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '<input checked class="form-control" id="active" name="active" '`
		`+ 'type="checkbox" value="y">', output_text)`
		`+`
		`+ self.assertTrue(os.path.exists(os.path.join(`
		`+ self.path, 'repos', 'test.git', 'hooks',`
		`+ 'post-receive.mirror')))`
		`+ self.assertTrue(os.path.exists(os.path.join(`
		`+ self.path, 'repos', 'test.git', 'hooks',`
		`+ 'post-receive')))`
		`+`
		`+ def test_plugin_mirror_deactivate(self):`
		`+ """ Test the deactivating the mirror plugin.`
		`+ """`
		`+ self.test_setting_up_mirror()`
		`+`
		`+ user = tests.FakeUser(username='pingou')`
		`+ with tests.user_set(self.app.application, user):`
		`+ csrf_token = self.get_csrf()`
		`+`
		`+ # De-Activate hook`
		`+ data = {'csrf_token': csrf_token}`
		`+ output = self.app.post(`
		`+ '/test/settings/Mirroring', data=data, follow_redirects=True)`
		`+ self.assertEqual(output.status_code, 200)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '</i> Hook Mirroring deactivated</div>',`
		`+ output_text)`
		`+`
		`+ output = self.app.get('/test/settings/Mirroring', data=data)`
		`+ self.assertEqual(output.status_code, 200)`
		`+ output_text = output.get_data(as_text=True)`
		`+ self.assertIn(`
		`+ '<title>Settings Mirroring - test - Pagure</title>',`
		`+ output_text)`
		`+ self.assertIn(`
		`+ '<input class="form-control" id="active" name="active" '`
		`+ 'type="checkbox" value="y">', output.get_data(as_text=True))`
		`+`
		`+ self.assertFalse(os.path.exists(os.path.join(`
		`+ self.path, 'repos', 'test.git', 'hooks',`
		`+ 'post-receive.mirror')))`
		`+`
		`+`
		`+ if __name__ == '__main__':`
		`+ unittest.main(verbosity=2)`

tests/test_pagure_flask_util.py

file added

+65

		`@@ -0,0 +1,65 @@`
		`+ # -- coding: utf-8 --`
		`+`
		`+ """`
		`+ (c) 2018 - Copyright Red Hat Inc`
		`+`
		`+ Authors:`
		`+ Pierre-Yves Chibon <pingou@pingoured.fr>`
		`+`
		`+ """`
		`+`
		`+ from __future__ import unicode_literals`
		`+`
		`+ __requires__ = ['SQLAlchemy >= 0.8']`
		`+ import pkg_resources`
		`+`
		`+ import unittest`
		`+ import sys`
		`+ import os`
		`+`
		`+ from mock import patch, MagicMock`
		`+`
		`+ sys.path.insert(0, os.path.join(os.path.dirname(`
		`+ os.path.abspath(__file__)), '..'))`
		`+`
		`+ from pagure.utils import ssh_urlpattern`
		`+ import tests`
		`+`
		`+`
		`+ class PagureUtilSSHPatterntests(tests.Modeltests):`
		`+ """ Tests for the ssh_urlpattern in pagure.util """`
		`+`
		`+ def test_ssh_pattern_valid(self):`
		`+ """ Test the ssh_urlpattern with valid patterns. """`
		`+ patterns = [`
		`+ 'ssh://user@host.com/repo.git',`
		`+ 'git+ssh://user@host.com/repo.git',`
		`+ 'ssh://host.com/repo.git'`
		`+ 'git+ssh://host.com/repo.git',`
		`+ 'ssh://127.0.0.1/repo.git',`
		`+ 'git+ssh://127.0.0.1/repo.git',`
		`+ ]`
		`+ for pattern in patterns:`
		`+ print(pattern)`
		`+ self.assertTrue(ssh_urlpattern.match(pattern))`
		`+`
		`+`
		`+ def test_ssh_pattern_invalid(self):`
		`+ """ Test the ssh_urlpattern with invalid patterns. """`
		`+ patterns = [`
		`+ 'http://user@host.com/repo.git',`
		`+ 'git+http://user@host.com/repo.git',`
		`+ 'https://user@host.com/repo.git',`
		`+ 'git+https://user@host.com/repo.git',`
		`+ 'ssh://localhost/repo.git',`
		`+ 'git+ssh://localhost/repo.git',`
		`+ 'ssh://0.0.0.0/repo.git',`
		`+ 'git+ssh://0.0.0.0/repo.git',`
		`+ ]`
		`+ for pattern in patterns:`
		`+ print(pattern)`
		`+ self.assertFalse(ssh_urlpattern.match(pattern))`
		`+`
		`+`
		`+ if __name__ == '__main__':`
		`+ unittest.main(verbosity=2)`

tests/test_pagure_lib_task_mirror.py

file added

+338

		`@@ -0,0 +1,338 @@`
		`+ # -- coding: utf-8 --`
		`+`
		`+ """`
		`+ (c) 2018 - Copyright Red Hat Inc`
		`+`
		`+ Authors:`
		`+ Pierre-Yves Chibon <pingou@pingoured.fr>`
		`+`
		`+ """`
		`+`
		`+ from __future__ import unicode_literals`
		`+`
		`+ import datetime`
		`+ import os`
		`+ import shutil`
		`+ import sys`
		`+ import tempfile`
		`+ import time`
		`+ import unittest`
		`+`
		`+ import pygit2`
		`+ import six`
		`+ from mock import patch, MagicMock, call`
		`+`
		`+ sys.path.insert(0, os.path.join(os.path.dirname(`
		`+ os.path.abspath(__file__)), '..'))`
		`+`
		`+ import pagure`
		`+ import pagure.lib.git`
		`+ import tests`
		`+`
		`+ import pagure.lib.tasks_mirror`
		`+`
		`+`
		`+ class PagureLibTaskMirrortests(tests.Modeltests):`
		`+ """ Tests for pagure.lib.task_mirror """`
		`+`
		`+ maxDiff = None`
		`+`
		`+ def setUp(self):`
		`+ """ Set up the environnment, ran before every tests. """`
		`+ super(PagureLibTaskMirrortests, self).setUp()`
		`+`
		`+ pagure.config.config['REQUESTS_FOLDER'] = None`
		`+ self.sshkeydir = os.path.join(self.path, 'sshkeys')`
		`+ pagure.config.config['MIRROR_SSHKEYS_FOLDER'] = self.sshkeydir`
		`+`
		`+ tests.create_projects(self.session)`
		`+`
		`+ def test_create_ssh_key(self):`
		`+ """ Test the _create_ssh_key method. """`
		`+ # before`
		`+ self.assertFalse(os.path.exists(self.sshkeydir))`
		`+ os.mkdir(self.sshkeydir)`
		`+ self.assertEqual(sorted(os.listdir(self.sshkeydir)), [])`
		`+`
		`+ keyfile = os.path.join(self.sshkeydir, 'testkey')`
		`+ pagure.lib.tasks_mirror._create_ssh_key(keyfile)`
		`+`
		`+ # after`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)),`
		`+ [u'testkey', u'testkey.pub']`
		`+ )`
		`+`
		`+ def test_setup_mirroring(self):`
		`+ """ Test the setup_mirroring method. """`
		`+`
		`+ # before`
		`+ self.assertFalse(os.path.exists(self.sshkeydir))`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook)`
		`+`
		`+ # Install the plugin at the DB level`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ dbobj = plugin.db_object()`
		`+ dbobj.project_id = project.id`
		`+ self.session.add(dbobj)`
		`+ self.session.commit()`
		`+`
		`+ pagure.lib.tasks_mirror.setup_mirroring(`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ # after`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)),`
		`+ [u'test', u'test.pub']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNotNone(project.mirror_hook.public_key)`
		`+ self.assertTrue(`
		`+ project.mirror_hook.public_key.startswith('ssh-rsa '))`
		`+`
		`+ def test_setup_mirroring_ssh_folder_exists_wrong_permissions(self):`
		`+ """ Test the setup_mirroring method. """`
		`+`
		`+ os.makedirs(self.sshkeydir)`
		`+`
		`+ # before`
		`+ self.assertEqual(sorted(os.listdir(self.sshkeydir)), [])`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook)`
		`+`
		`+ # Install the plugin at the DB level`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ dbobj = plugin.db_object()`
		`+ dbobj.project_id = project.id`
		`+ self.session.add(dbobj)`
		`+ self.session.commit()`
		`+`
		`+ self.assertRaises(`
		`+ pagure.exceptions.PagureException,`
		`+ pagure.lib.tasks_mirror.setup_mirroring,`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ # after`
		`+ self.assertEqual(sorted(os.listdir(self.sshkeydir)), [])`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook.public_key)`
		`+`
		`+ def test_setup_mirroring_ssh_folder_symlink(self):`
		`+ """ Test the setup_mirroring method. """`
		`+`
		`+ os.symlink(`
		`+ self.path,`
		`+ self.sshkeydir`
		`+ )`
		`+`
		`+ # before`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)),`
		`+ [u'attachments', u'config', u'forks', u'releases',`
		`+ u'remotes', u'repos', u'sshkeys']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook)`
		`+`
		`+ # Install the plugin at the DB level`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ dbobj = plugin.db_object()`
		`+ dbobj.project_id = project.id`
		`+ self.session.add(dbobj)`
		`+ self.session.commit()`
		`+`
		`+ self.assertRaises(`
		`+ pagure.exceptions.PagureException,`
		`+ pagure.lib.tasks_mirror.setup_mirroring,`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ # after`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)),`
		`+ [u'attachments', u'config', u'forks', u'releases',`
		`+ u'remotes', u'repos', u'sshkeys']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook.public_key)`
		`+`
		`+ @patch('os.getuid', MagicMock(return_value=450))`
		`+ def test_setup_mirroring_ssh_folder_owner(self):`
		`+ """ Test the setup_mirroring method. """`
		`+ os.makedirs(self.sshkeydir, mode=0o700)`
		`+`
		`+ # before`
		`+ self.assertEqual(sorted(os.listdir(self.sshkeydir)), [])`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook)`
		`+`
		`+ # Install the plugin at the DB level`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ dbobj = plugin.db_object()`
		`+ dbobj.project_id = project.id`
		`+ self.session.add(dbobj)`
		`+ self.session.commit()`
		`+`
		`+ self.assertRaises(`
		`+ pagure.exceptions.PagureException,`
		`+ pagure.lib.tasks_mirror.setup_mirroring,`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ # after`
		`+ self.assertEqual(sorted(os.listdir(self.sshkeydir)), [])`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook.public_key)`
		`+`
		`+`
		`+ class PagureLibTaskMirrorSetuptests(tests.Modeltests):`
		`+ """ Tests for pagure.lib.task_mirror """`
		`+`
		`+ maxDiff = None`
		`+`
		`+ def setUp(self):`
		`+ """ Set up the environnment, ran before every tests. """`
		`+ super(PagureLibTaskMirrorSetuptests, self).setUp()`
		`+`
		`+ pagure.config.config['REQUESTS_FOLDER'] = None`
		`+ self.sshkeydir = os.path.join(self.path, 'sshkeys')`
		`+ pagure.config.config['MIRROR_SSHKEYS_FOLDER'] = self.sshkeydir`
		`+`
		`+ tests.create_projects(self.session)`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+`
		`+ # Install the plugin at the DB level`
		`+ plugin = pagure.lib.plugins.get_plugin('Mirroring')`
		`+ dbobj = plugin.db_object()`
		`+ dbobj.target = 'ssh://user@localhost.localdomain/foobar.git'`
		`+ dbobj.project_id = project.id`
		`+ self.session.add(dbobj)`
		`+ self.session.commit()`
		`+`
		`+ pagure.lib.tasks_mirror.setup_mirroring(`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ def test_setup_mirroring_twice(self):`
		`+ """ Test the setup_mirroring method. """`
		`+`
		`+ # before`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNotNone(project.mirror_hook.public_key)`
		`+ before_key = project.mirror_hook.public_key`
		`+ self.assertTrue(`
		`+ project.mirror_hook.public_key.startswith('ssh-rsa '))`
		`+`
		`+ self.assertRaises(`
		`+ pagure.exceptions.PagureException,`
		`+ pagure.lib.tasks_mirror.setup_mirroring,`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ # after`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)),`
		`+ [u'test', u'test.pub']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNotNone(project.mirror_hook.public_key)`
		`+ self.assertEqual(project.mirror_hook.public_key, before_key)`
		`+`
		`+ def test_teardown_mirroring(self):`
		`+ """ Test the teardown_mirroring method. """`
		`+`
		`+ # before`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNotNone(project.mirror_hook.public_key)`
		`+ self.assertTrue(`
		`+ project.mirror_hook.public_key.startswith('ssh-rsa '))`
		`+`
		`+ pagure.lib.tasks_mirror.teardown_mirroring(`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ # after`
		`+ self.session = pagure.lib.create_session(self.dbpath)`
		`+ self.assertEqual(sorted(os.listdir(self.sshkeydir)), [])`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNone(project.mirror_hook.public_key)`
		`+`
		`+ @patch(`
		`+ 'tempfile.mkdtemp',`
		`+ MagicMock(return_value='/tmp/pagure-mirror-fdgqcF'))`
		`+ @patch('pagure.lib.git.read_git_lines')`
		`+ def test_mirror_project(self,rgl):`
		`+ """ Test the mirror_project method. """`
		`+ rgl.return_value = ('stdout', 'stderr')`
		`+ tests.create_projects_git(`
		`+ os.path.join(self.path, 'repos'), bare=True)`
		`+`
		`+ # before`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)), [u'test', u'test.pub']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNotNone(project.mirror_hook.public_key)`
		`+ self.assertTrue(`
		`+ project.mirror_hook.public_key.startswith('ssh-rsa '))`
		`+`
		`+ pagure.lib.tasks_mirror.mirror_project(`
		`+ username=None,`
		`+ namespace=None,`
		`+ name='test')`
		`+`
		`+ # after`
		`+ self.assertEqual(`
		`+ sorted(os.listdir(self.sshkeydir)),`
		`+ [u'test', u'test.pub']`
		`+ )`
		`+ project = pagure.lib.get_authorized_project(self.session, 'test')`
		`+ self.assertIsNotNone(project.mirror_hook.public_key)`
		`+ self.assertTrue(`
		`+ project.mirror_hook.public_key.startswith('ssh-rsa '))`
		`+`
		`+ calls = [`
		`+ call(`
		`+ [`
		`+ u'remote', u'add', u'test_0',`
		`+ u'ssh://user@localhost.localdomain/foobar.git',`
		`+ u'--mirror=push'`
		`+ ],`
		`+ abspath=u'/tmp/pagure-mirror-fdgqcF',`
		`+ error=True`
		`+ ),`
		`+ call(`
		`+ [u'push', u'test_0'],`
		`+ abspath=u'/tmp/pagure-mirror-fdgqcF',`
		`+ env={`
		`+ u'GIT_SSH_COMMAND': u'ssh -i %s/sshkeys/test' % self.path`
		`+ },`
		`+ error=True`
		`+ )`
		`+ ]`
		`+`
		`+ self.assertEqual(rgl.call_count, 2)`
		`+ self.assertEqual(`
		`+ calls,`
		`+ rgl.mock_calls`
		`+ )`
		`+`
		`+`
		`+ if __name__ == '__main__':`
		`+ unittest.main(verbosity=2)`

pingou commented 7 years ago

Fixes https://pagure.io/pagure/issue/537

ngompa commented 7 years ago

@pingou If I'm reading this correctly, it generates an SSH key pair for each mirror hook activated, right?

ngompa commented 7 years ago

By the way, if this is the case, then it shouldn't be that bad to extend this to support pull (inbound) mirroring, too (#1987). That deals with the main problem of using the same key (Gitolite would consider each user "unique" with separate keys).

That said, public repos with anonymous access allowed do not require any of this for inbound mirroring, just the ability to be pulled via git://, http://, or https:// protocols.

Edited 7 years ago by ngompa

pingou commented 7 years ago

@pingou If I'm reading this correctly, it generates an SSH key pair for each mirror hook activated, right?

Yes

That deals with the main problem of using the same key (Gitolite would consider each user "unique" with separate keys).

For this I think I prefer to expand the support of the deploy keys, keep concerns separate.

rebased onto 64ef3c7f397e4eed3ec8864744f2f2967fa80f59

7 years ago

pingou commented 7 years ago

Pretty please pagure-ci rebuild

rebased onto b2a01c078710aad056abdf11b49728f05a26d3e9

7 years ago

ngompa commented 7 years ago

This code looks good to me. The test failures look like redis isn't set up correctly in the test environment, so I'm going to assume the tests pass normally.

So... :thumbsup:

rebased onto 7f62f337e043cfde6459da59836e8b3a6139246c

7 years ago

pingou commented 7 years ago

The test failures look like redis isn't set up correctly in the test environment, so I'm going to assume the tests pass normally.

Thanks, just the pointer I needed to find the error.

Pushing a fix, let's see if it works :)

rebased onto 981a369c75347bcabec234530b4d09216bcc3826

7 years ago

rebased onto 4c95e3b831f6933a92094cb5ad8810461bcef8e6

6 years ago

4 new commits added

Add a mirroring hook to mirror git repo to other locations
Allow plugins to have readonly fields in their form
Add a way to return both stdout and stderr when calling out sub-command
Fix import in the pagure_hook file

6 years ago

5 new commits added

Properly import and call the tasks_services' tasks
Add a mirroring hook to mirror git repo to other locations
Allow plugins to have readonly fields in their form
Add a way to return both stdout and stderr when calling out sub-command
Fix import in the pagure_hook file

6 years ago

2 new commits added

Add a -mirror subpackage to pagure
Include a pagure_mirror systemd file

6 years ago

pingou commented 6 years ago

Pretty please pagure-ci rebuild

rebased onto 7b568a7add61de90709f6c61428fae1872d2d9e6

6 years ago

ngompa commented on line 13 of files/pagure.spec 6 years ago

Drop trollius-redis stuff.

ngompa commented on line 17 of files/pagure.spec 6 years ago

Drop trollius-redis stuff.

pingou commented on line 13 of files/pagure.spec 6 years ago

Actually, the trollius stuff can also go away since the service is using celery (which would be good to list as dependency)

rebased onto f52bf8395441c08ea1d426712fce84452c34353c

6 years ago

ngompa commented on line 74 of files/pagure.spec 6 years ago

Why don't we just require pagure? That's already technically a real requirement, since most of the code is there...

ngompa commented 6 years ago

@pingou This needs to be rebased for current master.

rebased onto 73e7705bb5a3e3f19e46fb9e452ae4ae3c99a8b6

6 years ago

ngompa commented on line 2 of files/pagure_mirror.service 6 years ago

This is the mirror service, not the logcom service. :)

puiterwijk commented on line 47 of pagure/lib/tasks_mirror.py 6 years ago

https://pagure.io/pagure/pull-request/1105#_3,63

puiterwijk commented on line 15 of files/pagure_mirror.service 6 years ago

I think we'd want to run this as a different user.

puiterwijk commented on line 55 of pagure/hooks/files/mirror.py 6 years ago

Note that this assumes that this GIT_DIR will exist when the runner gets to running it. This might not be the case.

puiterwijk commented on line 116 of pagure/lib/tasks_mirror.py 6 years ago

I do not think that ~/.ssh is the best place for this.
This is service data, how about /var/lib/pagure/sshkeys/?

puiterwijk commented on line 93 of pagure/lib/tasks_mirror.py 6 years ago

This writes the private key as chmod 0644.
SSH won't even accept these keys, they'll need to be 0600.

puiterwijk commented on line 119 of pagure/lib/tasks_mirror.py 6 years ago

Note that the directory containing private keys must be 0700.

puiterwijk commented on line 125 of pagure/lib/tasks_mirror.py 6 years ago

I would say that if a key exists at the moment mirroring is enabled, there's definitely something wrong (previous key wasn't cleared up), and it should probably error out?

puiterwijk commented on line 161 of pagure/lib/tasks_mirror.py 6 years ago

I would actually say to not catch this. When you get here, the key really ought to exist, if it doesn't, we need to error out probably, so an admin looks at it.

puiterwijk commented on line 199 of pagure/lib/tasks_mirror.py 6 years ago

Given that, if I read the code correctly, this is done in the main project repo, this would mean you add a remote for each target persistently, and on future pushes, this will error out.
But also, if you edit e.g. the first line in the remote, it will never update that one, etc.
Perhaps it's better to just do this in a new, clean clone of the repository?

puiterwijk commented on line 191 of pagure/lib/tasks_mirror.py 6 years ago

You have no checking at all on the remote strings as far as I can see, this would mean that I could fill in a target of "/srv/git/repositories/pagure.git" in my mirror from attack.git, and since this runs as a user that has R/W access to all repos, would (probably) overwrite the target repository.
Next to checking of the target URLs, I really think this should run as a limited user.

puiterwijk commented 6 years ago

I think that in this PR, you still need:

docs
tests (you currently have UI tests, tests on that the pushing itself works would probably be great)

Edited 6 years ago by puiterwijk

pingou commented on line 15 of files/pagure_mirror.service 6 years ago

Indeed that is the idea, I'll make it more explicit by changing the default here, thanks

pingou commented on line 55 of pagure/hooks/files/mirror.py 6 years ago

How could it not? This is the hook so it's installed in a subfolder in the GIT_DIR

pingou commented on line 116 of pagure/lib/tasks_mirror.py 6 years ago

I guess it should be a configuration variable then, defaulting to this path indeed.

pingou commented on line 93 of pagure/lib/tasks_mirror.py 6 years ago

:thumbsup:

pingou commented on line 119 of pagure/lib/tasks_mirror.py 6 years ago

:thumbsup:

pingou commented on line 191 of pagure/lib/tasks_mirror.py 6 years ago

I'll add a check to ensure the remote is an (ssh) url.

pingou commented on line 199 of pagure/lib/tasks_mirror.py 6 years ago

sounds good

pingou commented on line 47 of pagure/lib/tasks_mirror.py 6 years ago

Oups, thanks

pingou commented on line 125 of pagure/lib/tasks_mirror.py 6 years ago

Let's adjust it :)

pingou commented on line 161 of pagure/lib/tasks_mirror.py 6 years ago

In this case since we're removing the key I'm tempted to let it as is.

rebased onto df8e658f6ba88c27997ea8d9e8d5131810f48a40

6 years ago

7 new commits added

Add a -mirror subpackage to pagure
Include a pagure_mirror systemd file
Add a mirroring hook to mirror git repo to other locations
Properly import and call the tasks_services' tasks
Allow plugins to have readonly fields in their form
Add a way to return both stdout and stderr when calling out sub-command
Fix import in the pagure_hook file

6 years ago

rebased onto bbf8100ead851cc3ddf973415bc3786d6ec1d11c

6 years ago

rebased onto bbf8100ead851cc3ddf973415bc3786d6ec1d11c

6 years ago

puiterwijk commented on line 126 of pagure/lib/tasks_mirror.py 6 years ago

We should probably make sure that if it already exists, it's not a symlink or owned by someone else.
Otherwise, if the containing directory is widely writable, someone with access to the system might be able to instead put in a symlink to a place they control.

pingou commented on line 126 of pagure/lib/tasks_mirror.py 6 years ago

Done :)

rebased onto 4cf012ace80b023218c33de1576855e4d2f6004f

6 years ago

rebased onto 5ca22c7b298562d9840e4228c9742e45c65c901f

6 years ago

7 new commits added

Add a -mirror subpackage to pagure
Include a pagure_mirror systemd file
Add a mirroring hook to mirror git repo to other locations
Properly import and call the tasks_services' tasks
Allow plugins to have readonly fields in their form
Add a way to return both stdout and stderr when calling out sub-command
Fix import in the pagure_hook file

6 years ago

7 new commits added

Add a -mirror subpackage to pagure
Include a pagure_mirror systemd file
Add a mirroring hook to mirror git repo to other locations
Properly import and call the tasks_services' tasks
Allow plugins to have readonly fields in their form
Add a way to return both stdout and stderr when calling out sub-command
Fix import in the pagure_hook file

6 years ago

rebased onto a7226fde7e6df8528d5b6e5e9fbfcea372f0fccd

6 years ago

rebased onto f9db971861e6119bcb71ec7813df3fdf0265e807

6 years ago

rebased onto ebc1afc290ee0cc93c65f9388d87e66f8b3a6f8c

6 years ago

ryanlerch commented on line 5 of files/pagure.spec 6 years ago

test

rebased onto 871024d606ec05634e8726cef25e1c685c9580b7

6 years ago

rebased onto 05f79517747ef0f3a9135ff18c037dad7a9529a9

6 years ago

rebased onto a21c560

6 years ago

puiterwijk commented on line 55 of pagure/hooks/files/mirror.py 6 years ago

User A creates repo X
Repo X is setup with outbound mirroring
Repo X is pushed to
Worker is very busy
Repo X is deleted by User A
Worker gets to running queued mirror job

GIT_DIR now does not exist.

puiterwijk commented on line 102 of pagure/lib/tasks_mirror.py 6 years ago

Maybe add os.O_EXCL as well, so that we make very sure that we create this file, rather than overwrite a file an attacker might've put in place?
That will make sure that if the file already existed, it errors out.

Edited 6 years ago by puiterwijk

puiterwijk commented on line 243 of pagure/lib/tasks_mirror.py 6 years ago

Note that this overwrites the log attribute with just the log of the very last push.
Maybe an idea to accumulate this variable over the for idx, remote in enumerate(remotes): run, so you can actually return the log for all of the mirrors?

puiterwijk commented 6 years ago

+1 to the patch as is, but might fix the two last comments.
They should be small enough and not impact the security constraints, so no need to re-review if just the os.O_EXCL and the log fix.

pingou commented on line 55 of pagure/hooks/files/mirror.py 6 years ago

Hm, good point, I'll adjust the task to check if the absolute path still exists before continuing.

Thanks :)

pingou commented on line 102 of pagure/lib/tasks_mirror.py 6 years ago

will do

pingou commented on line 243 of pagure/lib/tasks_mirror.py 6 years ago

Good catch, fixed via:

+    logs = []
     for idx, remote in enumerate(remotes):
         remote_name = '%s_%s' % (public_key_name, idx)
         _log.info(
@@ -239,10 +241,12 @@ def mirror_project(self, session, username, namespace, name):
             env={'GIT_SSH_COMMAND': 'ssh -i %s' % private_key_file})
         log = "Output from the push:\n  stdout: %s\n  stderr: %s" % (
             stdout, stderr)
-        project.mirror_hook.last_log = log
+        logs.append(log)
+    if logs:
+        project.mirror_hook.last_log = '\n'.join(logs)
         session.add(project.mirror_hook)
         session.commit()
-        _log.info(log)
+        _log.info('\n'.join(logs))

7 new commits added