From 026caf315d7a03b5d64c2162a4820dfbe7c0f4c4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2017 09:07:41 +0000 Subject: Rename the pagure-mirror server to pagure_mirror_server as it should be --- diff --git a/pagure-mirror/pagure_ci_server.py b/pagure-mirror/pagure_ci_server.py deleted file mode 100644 index 215112e..0000000 --- a/pagure-mirror/pagure_ci_server.py +++ /dev/null @@ -1,414 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - (c) 2016 - Copyright Red Hat Inc - - Authors: - Pierre-Yves Chibon - - -This server listens to message sent via redis and set-up/remove mirroring -for the corresponding project. - -""" - -import base64 -import json -import logging -import os -import struct - -import requests -import six -import trollius -import trollius_redis -import werkzeug - -from cryptography import utils -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives import serialization - - - -logging.basicConfig(level=logging.DEBUG) -LOG = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure -import pagure.lib - - -CONFIG_TPL = '''host %(name)s - HostName %(host)s - User %(user)s - IdentityFile ~/.ssh/%(keyname)s - -''' - - -# -# Utility methods used to setup/teardown the mirroring -# - - -# Code from: -# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/primitives/serialization.py#L153 -def _ssh_write_string(data): - return struct.pack(">I", len(data)) + data - - -def _ssh_write_mpint(value): - data = utils.int_to_bytes(value) - if six.indexbytes(data, 0) & 0x80: - data = b"\x00" + data - return _ssh_write_string(data) - - -# Code from _openssh_public_key_bytes at: -# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 -@trollius.coroutine -def serialize_public_ssh_key( key): - if isinstance(key, rsa.RSAPublicKey): - public_numbers = key.public_numbers() - return b"ssh-rsa " + base64.b64encode( - _ssh_write_string(b"ssh-rsa") + - _ssh_write_mpint(public_numbers.e) + - _ssh_write_mpint(public_numbers.n) - ) - else: - # Since we only write RSA keys, drop the other serializations - return - - -@trollius.coroutine -def create_ssh_key(keyfile): - ''' Create the public and private ssh keys. - - The specified file name will be the private key and the public one will - be in a similar file name ending with a '.pub'. - - ''' - private_key = rsa.generate_private_key( - public_exponent=65537, - key_size=4096, - backend=default_backend() - ) - - private_pem = private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption() - ) - with open(keyfile, 'w') as stream: - stream.write(private_pem) - - public_key = private_key.public_key() - public_pem = yield trollius.From(serialize_public_ssh_key(public_key)) - if public_pem: - with open(keyfile + '.pub', 'w') as stream: - stream.write(public_pem) - - -def split_target(target): - ''' Check if the given target follows the expected model. ''' - LOG.info('Checking target: %s', target) - if target.startswith('http'): - raise PagureException( - 'Invalid target %s, we only support mirroring via ssh' % target) - - if target.startswith('ssh://'): - target = target.replace('ssh://', '', 1) - target = target.replace('/', ':', 1) - - if not '@' in target: - raise PagureException( - 'No user specified in %s, we were expecting it before a `@`' - % target) - if not ':' in target: - raise PagureException( - 'No path specified in %s, we were expecting it after a `:`' - % target) - user, host_path = target.split('@', 1) - host, path = host_path.split(':', 1) - return user, host, path - - -@trollius.coroutine -def check_or_create_ssh_config(ssh_folder, key_name, target): - ''' Check or adjust the ~/.ssh/config file ''' - ssh_config_file = os.path.join(ssh_folder, 'config') - - - for idx, remote in enumerate(target.split('\n')): - remote = remote.strip() - if not remote: - continue - - user, host, path = split_target(remote) - - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path), - 'name': '%s_%s' % (key_name, idx), - 'keyname': key_name, - } - - update = True - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - update = False - - if update: - with open(ssh_config_file, 'a') as stream: - stream.write(ssh_config) - - -@trollius.coroutine -def clean_ssh_config(ssh_folder, key_name, target): - ''' Check or adjust the ~/.ssh/config file ''' - ssh_config_file = os.path.join(ssh_folder, 'config') - - for idx, remote in enumerate(target.split('\n')): - remote = remote.strip() - if not remote: - continue - - user, host, path = split_target(remote) - - ssh_config = CONFIG_TPL % { - 'user': user, - 'host': '%s:%s' % (host, path), - 'name': '%s_%s' % (key_name, idx), - 'keyname': key_name, - } - - data = None - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as stream: - data = stream.read() - if ssh_config in data: - data = data.replace(ssh_config, '', 1) - - with open(ssh_config_file, 'w') as stream: - stream.write(data) - - -# -# Actual logic of the service -# - -@trollius.coroutine -def setup_mirroring(project, session, dbobj): - ''' Setup the specified repo for mirroring. - ''' - public_key_name = werkzeug.secure_filename(project.fullname) - ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) - - if not os.path.exists(ssh_folder): - os.makedirs(ssh_folder) - - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - LOG.info('Public key of interest: %s', public_key_file) - - if not os.path.exists(public_key_file): - LOG.info('Creating public key') - yield trollius.From( - create_ssh_key(os.path.join(ssh_folder, public_key_name)) - ) - - LOG.info('Updating ssh configuration') - yield trollius.From( - check_or_create_ssh_config( - ssh_folder, public_key_name, dbobj.target) - ) - - with open(public_key_file) as stream: - public_key = stream.read() - - if dbobj.public_key != public_key: - LOG.info('Updating information in the DB') - dbobj.public_key = public_key - session.add(dbobj) - session.commit() - - -@trollius.coroutine -def mirror_project(repo, session, dbobj, abspath): - ''' Does the actual mirroring of the specified project/repo. - ''' - plugin = pagure.lib.plugins.get_plugin('Mirroring') - dbobj = plugin.db_object() - - # Get the list of remotes - remotes = [ - remote.strip() - for remote in repo.mirror_hook.target.split('\n') - if repo.mirror_hook and remote.strip() - ] - - public_key_name = werkzeug.secure_filename(repo.fullname) - - # Add the remotes - for idx, remote in enumerate(remotes): - lines = pagure.lib.git.read_git_lines( - ['remote', 'add', '%s_%s' % (public_key_name, idx), remote, - '--mirror=push'], abspath) - if pagure.APP.config.get('HOOK_DEBUG', False): - print '\n'.join(lines) - - # Push - for idx, remote in enumerate(remotes): - lines = pagure.lib.git.read_git_lines( - ['push', '%s_%s' % (public_key_name, idx)], abspath) - dbobj.last_log = '\n'.join(lines) - session.add(dbobj) - session.commit() - if pagure.APP.config.get('HOOK_DEBUG', False): - print '\n'.join(lines) - - -@trollius.coroutine -def teardown_mirroring(project, session, dbobj): - ''' Stop the mirroring of the specified repo. - ''' - public_key_name = werkzeug.secure_filename(project.fullname) - ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) - - if not os.path.exists(ssh_folder): - os.makedirs(ssh_folder) - - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - - public_key_name = werkzeug.secure_filename(project.fullname) - private_key_file = os.path.join(ssh_folder, public_key_name) - public_key_file = os.path.join( - ssh_folder, '%s.pub' % public_key_name) - - if os.path.exists(private_key_file): - os.unlink(private_key_file) - - if os.path.exists(public_key_file): - os.unlink(public_key_file) - - yield trollius.From( - clean_ssh_config( - ssh_folder, public_key_name, dbobj.target) - ) - - project.mirror_hook.public_key = None - session.add(project) - session.commit() - - -@trollius.coroutine -def handle_messages(): - ''' Handles connecting to redis and acting upon messages received. - In this case, it means triggering a build on jenkins based on the - information provided. - ''' - - host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') - port = pagure.APP.config.get('REDIS_PORT', 6379) - dbname = pagure.APP.config.get('REDIS_DB', 0) - connection = yield trollius.From(trollius_redis.Connection.create( - host=host, port=port, db=dbname)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.mirror'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - LOG.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - - reponame = data['name'] - username = data['user']['name'] if data['parent'] else None - namespace = data['namespace'] - abspath = data['abspath'] - LOG.info( - 'Looking for project: %s/%s/%s', namespace, username, reponame) - - session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - repo = pagure.lib.get_project( - session, reponame, - user=username, - namespace=namespace) - if not repo: - print 'Unknown repo %s of username: %s in ns: %s' % ( - reponame, username, namespace) - session.close() - sys.exit(1) - - plugin = pagure.lib.plugins.get_plugin('Mirroring') - dbobj = plugin.db_object() - dbobj = getattr(repo, plugin.backref) - - topic = data.get('topic') - if topic == 'pagure.mirror.postcommit': - yield trollius.From(mirror_project(repo, session, dbobj, abspath)) - elif topic == 'pagure.mirror.setup': - yield trollius.From(setup_mirroring(repo, session, dbobj)) - elif topic == 'pagure.mirror.teardown': - yield trollius.From(teardown_mirroring(repo, session, dbobj)) - else: - LOG.error('Unknown topic found: %s', topic) - - session.close() - LOG.info('Ready for another') - - -def main(): - ''' Start the main async loop. ''' - - try: - loop = trollius.get_event_loop() - tasks = [ - trollius.async(handle_messages()), - ] - loop.run_until_complete(trollius.wait(tasks)) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - LOG.info("End Connection") - loop.close() - LOG.info("End") - - -if __name__ == '__main__': - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - LOG.setLevel(logging.DEBUG) - shellhandler = logging.StreamHandler() - shellhandler.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - aslog = logging.getLogger("trollius") - aslog.setLevel(logging.DEBUG) - - shellhandler.setFormatter(formatter) - LOG.addHandler(shellhandler) - main() diff --git a/pagure-mirror/pagure_mirror_server.py b/pagure-mirror/pagure_mirror_server.py new file mode 100644 index 0000000..215112e --- /dev/null +++ b/pagure-mirror/pagure_mirror_server.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + (c) 2016 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis and set-up/remove mirroring +for the corresponding project. + +""" + +import base64 +import json +import logging +import os +import struct + +import requests +import six +import trollius +import trollius_redis +import werkzeug + +from cryptography import utils +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + + + +logging.basicConfig(level=logging.DEBUG) +LOG = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib + + +CONFIG_TPL = '''host %(name)s + HostName %(host)s + User %(user)s + IdentityFile ~/.ssh/%(keyname)s + +''' + + +# +# Utility methods used to setup/teardown the mirroring +# + + +# Code from: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/primitives/serialization.py#L153 +def _ssh_write_string(data): + return struct.pack(">I", len(data)) + data + + +def _ssh_write_mpint(value): + data = utils.int_to_bytes(value) + if six.indexbytes(data, 0) & 0x80: + data = b"\x00" + data + return _ssh_write_string(data) + + +# Code from _openssh_public_key_bytes at: +# https://github.com/pyca/cryptography/blob/master/src/cryptography/hazmat/backends/openssl/backend.py#L1660 +@trollius.coroutine +def serialize_public_ssh_key( key): + if isinstance(key, rsa.RSAPublicKey): + public_numbers = key.public_numbers() + return b"ssh-rsa " + base64.b64encode( + _ssh_write_string(b"ssh-rsa") + + _ssh_write_mpint(public_numbers.e) + + _ssh_write_mpint(public_numbers.n) + ) + else: + # Since we only write RSA keys, drop the other serializations + return + + +@trollius.coroutine +def create_ssh_key(keyfile): + ''' Create the public and private ssh keys. + + The specified file name will be the private key and the public one will + be in a similar file name ending with a '.pub'. + + ''' + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend() + ) + + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + with open(keyfile, 'w') as stream: + stream.write(private_pem) + + public_key = private_key.public_key() + public_pem = yield trollius.From(serialize_public_ssh_key(public_key)) + if public_pem: + with open(keyfile + '.pub', 'w') as stream: + stream.write(public_pem) + + +def split_target(target): + ''' Check if the given target follows the expected model. ''' + LOG.info('Checking target: %s', target) + if target.startswith('http'): + raise PagureException( + 'Invalid target %s, we only support mirroring via ssh' % target) + + if target.startswith('ssh://'): + target = target.replace('ssh://', '', 1) + target = target.replace('/', ':', 1) + + if not '@' in target: + raise PagureException( + 'No user specified in %s, we were expecting it before a `@`' + % target) + if not ':' in target: + raise PagureException( + 'No path specified in %s, we were expecting it after a `:`' + % target) + user, host_path = target.split('@', 1) + host, path = host_path.split(':', 1) + return user, host, path + + +@trollius.coroutine +def check_or_create_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + + + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue + + user, host, path = split_target(remote) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } + + update = True + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + update = False + + if update: + with open(ssh_config_file, 'a') as stream: + stream.write(ssh_config) + + +@trollius.coroutine +def clean_ssh_config(ssh_folder, key_name, target): + ''' Check or adjust the ~/.ssh/config file ''' + ssh_config_file = os.path.join(ssh_folder, 'config') + + for idx, remote in enumerate(target.split('\n')): + remote = remote.strip() + if not remote: + continue + + user, host, path = split_target(remote) + + ssh_config = CONFIG_TPL % { + 'user': user, + 'host': '%s:%s' % (host, path), + 'name': '%s_%s' % (key_name, idx), + 'keyname': key_name, + } + + data = None + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as stream: + data = stream.read() + if ssh_config in data: + data = data.replace(ssh_config, '', 1) + + with open(ssh_config_file, 'w') as stream: + stream.write(data) + + +# +# Actual logic of the service +# + +@trollius.coroutine +def setup_mirroring(project, session, dbobj): + ''' Setup the specified repo for mirroring. + ''' + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + LOG.info('Public key of interest: %s', public_key_file) + + if not os.path.exists(public_key_file): + LOG.info('Creating public key') + yield trollius.From( + create_ssh_key(os.path.join(ssh_folder, public_key_name)) + ) + + LOG.info('Updating ssh configuration') + yield trollius.From( + check_or_create_ssh_config( + ssh_folder, public_key_name, dbobj.target) + ) + + with open(public_key_file) as stream: + public_key = stream.read() + + if dbobj.public_key != public_key: + LOG.info('Updating information in the DB') + dbobj.public_key = public_key + session.add(dbobj) + session.commit() + + +@trollius.coroutine +def mirror_project(repo, session, dbobj, abspath): + ''' Does the actual mirroring of the specified project/repo. + ''' + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + + # Get the list of remotes + remotes = [ + remote.strip() + for remote in repo.mirror_hook.target.split('\n') + if repo.mirror_hook and remote.strip() + ] + + public_key_name = werkzeug.secure_filename(repo.fullname) + + # Add the remotes + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['remote', 'add', '%s_%s' % (public_key_name, idx), remote, + '--mirror=push'], abspath) + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + # Push + for idx, remote in enumerate(remotes): + lines = pagure.lib.git.read_git_lines( + ['push', '%s_%s' % (public_key_name, idx)], abspath) + dbobj.last_log = '\n'.join(lines) + session.add(dbobj) + session.commit() + if pagure.APP.config.get('HOOK_DEBUG', False): + print '\n'.join(lines) + + +@trollius.coroutine +def teardown_mirroring(project, session, dbobj): + ''' Stop the mirroring of the specified repo. + ''' + public_key_name = werkzeug.secure_filename(project.fullname) + ssh_folder = os.path.expanduser(os.path.join('~', '.ssh')) + + if not os.path.exists(ssh_folder): + os.makedirs(ssh_folder) + + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + public_key_name = werkzeug.secure_filename(project.fullname) + private_key_file = os.path.join(ssh_folder, public_key_name) + public_key_file = os.path.join( + ssh_folder, '%s.pub' % public_key_name) + + if os.path.exists(private_key_file): + os.unlink(private_key_file) + + if os.path.exists(public_key_file): + os.unlink(public_key_file) + + yield trollius.From( + clean_ssh_config( + ssh_folder, public_key_name, dbobj.target) + ) + + project.mirror_hook.public_key = None + session.add(project) + session.commit() + + +@trollius.coroutine +def handle_messages(): + ''' Handles connecting to redis and acting upon messages received. + In this case, it means triggering a build on jenkins based on the + information provided. + ''' + + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.mirror'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + LOG.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + + reponame = data['name'] + username = data['user']['name'] if data['parent'] else None + namespace = data['namespace'] + abspath = data['abspath'] + LOG.info( + 'Looking for project: %s/%s/%s', namespace, username, reponame) + + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + repo = pagure.lib.get_project( + session, reponame, + user=username, + namespace=namespace) + if not repo: + print 'Unknown repo %s of username: %s in ns: %s' % ( + reponame, username, namespace) + session.close() + sys.exit(1) + + plugin = pagure.lib.plugins.get_plugin('Mirroring') + dbobj = plugin.db_object() + dbobj = getattr(repo, plugin.backref) + + topic = data.get('topic') + if topic == 'pagure.mirror.postcommit': + yield trollius.From(mirror_project(repo, session, dbobj, abspath)) + elif topic == 'pagure.mirror.setup': + yield trollius.From(setup_mirroring(repo, session, dbobj)) + elif topic == 'pagure.mirror.teardown': + yield trollius.From(teardown_mirroring(repo, session, dbobj)) + else: + LOG.error('Unknown topic found: %s', topic) + + session.close() + LOG.info('Ready for another') + + +def main(): + ''' Start the main async loop. ''' + + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + LOG.info("End Connection") + loop.close() + LOG.info("End") + + +if __name__ == '__main__': + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + LOG.setLevel(logging.DEBUG) + shellhandler = logging.StreamHandler() + shellhandler.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + aslog = logging.getLogger("trollius") + aslog.setLevel(logging.DEBUG) + + shellhandler.setFormatter(formatter) + LOG.addHandler(shellhandler) + main()