From 0f077b9b81c9147216a34e835ffa6d452cff1e52 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: WIP ALL THE THINGS Signed-off-by: Patrick Uiterwijk --- diff --git a/docker-compose.yml b/docker-compose.yml index de3bbb9..aa5ee9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,14 +18,14 @@ services: target: /repos read_only: true - .:/code - backend_git: + worker: build: context: ./docker - dockerfile: backend-git + dockerfile: worker depends_on: - redis - postgresql - image: pagure-backend-git:latest + image: pagure-worker:latest volumes: - type: volume source: repos diff --git a/docker/backend-git b/docker/backend-git deleted file mode 100644 index ee533f5..0000000 --- a/docker/backend-git +++ /dev/null @@ -1,23 +0,0 @@ -FROM registry.fedoraproject.org/fedora:25 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-flask-multistatic python-markdown python-psutil \ - python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 - -RUN dnf install -y python-trollius python-trollius-redis - -WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/pagure_git/pagure_git_server.py"] - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code diff --git a/docker/web b/docker/web index ac0eb62..0d434e1 100644 --- a/docker/web +++ b/docker/web @@ -13,8 +13,11 @@ RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python- python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ python-fedora-flask python2-pillow python2-psycopg2 +RUN dnf install -y python2-celery + WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/runserver.py", "--host", "0.0.0.0", "--config", "/code/openshift.cfg", "--no-debug"] +# Openshift: --no-debug +ENTRYPOINT ["/usr/bin/python", "/code/runserver.py", "--host", "0.0.0.0", "--config", "/code/openshift.cfg"] EXPOSE 5000 # Code injection is last to make optimal use of caches diff --git a/docker/worker b/docker/worker new file mode 100644 index 0000000..5e592f7 --- /dev/null +++ b/docker/worker @@ -0,0 +1,28 @@ +FROM registry.fedoraproject.org/fedora:25 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-flask-multistatic python-markdown python-psutil \ + python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 + +RUN dnf install -y python2-celery + +RUN useradd docker && \ + chown docker:docker /repos + +WORKDIR /code +ENTRYPOINT ["/usr/bin/celery", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info", "--autoreload"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code + +USER docker diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index e978f06..e85f184 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -53,6 +53,7 @@ import pagure.lib.notify import pagure.lib.plugins import pagure.pfmarkdown from pagure.lib import model +from pagure.lib import tasks REDIS = None @@ -1270,6 +1271,8 @@ def new_project(session, user, name, blacklist, allowed_prefix, prevent_40_chars=False, namespace=None, user_ns=False, ignore_existing_repo=False, private=False): ''' Create a new project based on the information provided. + + Is an async operation, and returns task ID. ''' if (not namespace and name in blacklist) \ or (namespace and '%s/%s' % (namespace, name) in blacklist): @@ -1342,6 +1345,21 @@ def new_project(session, user, name, blacklist, allowed_prefix, # Make sure we won't have SQLAlchemy error before we create the repo session.flush() + # Register creation et al + log_action(session, 'created', project, user_obj) + + pagure.lib.notify.log( + project, + topic='project.new', + msg=dict( + project=project.to_json(public=True), + agent=user_obj.username, + ), + ) + + return tasks.create_project.delay(namespace, name, add_readme, + ignore_existing_repo).id + # Add the readme file if it was asked if not add_readme: pygit2.init_repository(gitrepo, bare=True) @@ -1419,24 +1437,6 @@ def new_project(session, user, name, blacklist, allowed_prefix, plugin.set_up(project) plugin.install(project, dbobj) - # create the project in the db - session.commit() - - log_action(session, 'created', project, user_obj) - - pagure.lib.notify.log( - project, - topic='project.new', - msg=dict( - project=project.to_json(public=True), - agent=user_obj.username, - ), - ) - - return 'Project "%s" created' % ( - '%s/%s' % (project.namespace, project.name) if project.namespace - else project.name) - def new_issue(session, repo, title, content, user, ticketfolder, issue_id=None, issue_uid=None, private=False, status=None, close_status=None, diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py new file mode 100644 index 0000000..eb7b2b6 --- /dev/null +++ b/pagure/lib/tasks.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2017 - Copyright Red Hat Inc + + Authors: + Patrick Uiterwijk + +""" + +from celery import Celery +from celery.result import AsyncResult + +import pagure +import pagure.lib + + +conn = Celery('tasks', + broker='redis://%s' % pagure.APP.config['REDIS_HOST'], + backend='redis://%s' % pagure.APP.config['REDIS_HOST']) + + +def get_result(uuid): + return AsyncResult(uuid, conn.backend) + + +def ret(endpoint, **kwargs): + toret = {'endpoint': endpoint} + toret.update(kwargs) + return toret + + +@conn.task +def generate_gitolite_acls(): + # TODO: Implement gitolite acl stuff + return 'TODO' + + +@conn.task +def create_project(namespace, name, add_readme, ignore_existing_repo): + # TODO: Implement creation (see pagure.lib.new_project after return) + generate_gitolite_acls.delay() + return ret('view_repo', repo=name, namespace=namespace) diff --git a/pagure/templates/waiting.html b/pagure/templates/waiting.html new file mode 100644 index 0000000..6e4ff18 --- /dev/null +++ b/pagure/templates/waiting.html @@ -0,0 +1,20 @@ +{% extends "master.html" %} + +{% block title %}Waiting{% endblock %} + +{% block header %} + +{% endblock %} + +{% block content %} + +

+ Waiting +

+ +

+ We are waiting for your task to finish. + This page should be refreshed automatically, but if not click Here +

+ +{% endblock %} diff --git a/pagure/ui/app.py b/pagure/ui/app.py index ed8e78c..19f7d66 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -452,7 +452,7 @@ def new_project(): namespace = namespace.strip() try: - pagure.lib.new_project( + taskid = pagure.lib.new_project( SESSION, name=name, private=private, @@ -474,9 +474,8 @@ def new_project(): user_ns=APP.config.get('USER_NAMESPACE', False), ) SESSION.commit() - pagure.lib.git.generate_gitolite_acls() return flask.redirect(flask.url_for( - 'view_repo', repo=name, namespace=namespace)) + 'wait_task', taskid=taskid)) except pagure.exceptions.PagureException as err: flask.flash(str(err), 'error') except SQLAlchemyError as err: # pragma: no cover @@ -489,6 +488,20 @@ def new_project(): ) +@APP.route('/wait/') +def wait_task(taskid): + result = pagure.lib.tasks.get_result(taskid) + if result.ready: + result = result.get(timeout=0) + print result + endpoint = result.pop('endpoint') + return flask.redirect( + flask.url_for(endpoint, **result)) + else: + return flask.render_template( + 'waiting.html', + taskid=taskid) + @APP.route('/settings/', methods=('GET', 'POST')) @APP.route('/settings', methods=('GET', 'POST')) @login_required diff --git a/pagure_git/pagure_git.service b/pagure_git/pagure_git.service deleted file mode 100644 index 66a6563..0000000 --- a/pagure_git/pagure_git.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Pagure GIT Server -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -ExecStart=/usr/libexec/pagure-git/pagure_git_server.py -Type=simple -User=git -Group=git -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/pagure_git/pagure_git_server.py b/pagure_git/pagure_git_server.py deleted file mode 100644 index 1334c69..0000000 --- a/pagure_git/pagure_git_server.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -""" - (c) 2017 - Copyright Red Hat Inc - - Authors: - Patrick Uiterwijk - - -GIT server for Pagure's editing features. -This server takes messages sent to the redis queue and handles git change -operations. - -This service is required for the core Pagure functionality, since it implements -the creation of new repos, forking of repos and creating and merging pull -requests. -""" - -import logging -import os -import urlparse - -import trollius -import trollius_redis - -log = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure # noqa: E402 -import pagure.lib # noqa: E402 -from pagure.exceptions import PagureEvException # noqa: E402 - -SERVER = None - - -@trollius.coroutine -def handle_messages(): - host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') - port = pagure.APP.config.get('REDIS_PORT', 6379) - dbname = pagure.APP.config.get('REDIS_DB', 0) - connection = yield trollius.From(trollius_redis.Connection.create( - host=host, port=port, db=dbname)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.git'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - log.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - - -def main(): - server = None - try: - loop = trollius.get_event_loop() - tasks = [ - trollius.async(handle_messages()), - ] - loop.run_until_complete(trollius.wait(tasks)) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - log.info("End Connection") - loop.close() - log.info("End") - - -if __name__ == '__main__': - log = logging.getLogger("") - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - - ch.setFormatter(formatter) - log.addHandler(ch) - main()