From 15db1d23609a7fcb9287051ba21362a67cc06f7a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:53 +0000 Subject: [PATCH 1/26] Remove the code handling calling the web-hooks --- diff --git a/pagure/lib/notify.py b/pagure/lib/notify.py index b73412f..c6a959d 100644 --- a/pagure/lib/notify.py +++ b/pagure/lib/notify.py @@ -11,25 +11,20 @@ pagure notifications. import datetime import hashlib -import hmac import json import urlparse import smtplib -import six import time -import uuid import warnings import flask import requests +import six import pagure from email.mime.text import MIMEText -from kitchen.text.converters import to_bytes -_i = 0 - REPLY_MSG = 'To reply, visit the link below' if pagure.APP.config['EVENTSOURCE_SOURCE']: REPLY_MSG += 'or just reply to this email' @@ -49,54 +44,13 @@ def fedmsg_publish(*args, **kwargs): # pragma: no cover warnings.warn(str(err)) -def log(project, topic, msg): +def log(project, topic, msg, redis=None): ''' This is the place where we send notifications to user about actions occuring in pagure. ''' # Send fedmsg notification (if fedmsg is there and set-up) fedmsg_publish(topic, msg) - # Send web-hooks notification - if not isinstance(project, basestring) \ - and project.settings.get('Web-hooks'): # pragma: no cover - global _i - _i += 1 - year = datetime.datetime.now().year - if isinstance(topic, six.text_type): - topic = to_bytes(topic, encoding='utf8', nonstring="passthru") - msg = dict( - topic=topic.decode('utf-8'), - msg=msg, - timestamp=int(time.time()), - msg_id=str(year) + '-' + str(uuid.uuid4()), - i=_i, - ) - - content = json.dumps(msg) - hashhex = hmac.new( - str(project.hook_token), content, hashlib.sha1).hexdigest() - headers = { - 'X-Pagure-Topic': topic, - 'X-Pagure-Signature': hashhex - } - msg = flask.json.dumps(msg) - for url in project.settings.get('Web-hooks').split('\n'): - url = url.strip() - try: - req = requests.post( - url, - headers=headers, - data={'payload': msg} - ) - if not req: - raise pagure.exceptions.PagureException( - 'An error occured while querying: %s - ' - 'Error code: %s' % (url, req.status_code)) - except (requests.exceptions.RequestException, Exception) as err: - raise pagure.exceptions.PagureException( - 'An error occured while querying: %s - Error: %s' % ( - url, err)) - def _clean_emails(emails, user): ''' Remove the email of the user doing the action if it is in the list. From 482d84f1b70c08e7a6383a740201b3b57916e2b0 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:53 +0000 Subject: [PATCH 2/26] Provide the redis server when logging via fedmsg Specifying the redis server, allows sending web-hook calls to notify of an action as we do via fedmsg. --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 7c7c09c..763f9f2 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -215,7 +215,8 @@ def add_issue_comment(session, issue, comment, user, ticketfolder, issue=issue.to_json(public=True), project=issue.project.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) if redis: @@ -290,9 +291,11 @@ def add_tag_obj(session, obj, tags, user, ticketfolder, redis=None): project=obj.project.to_json(public=True), tags=added_tags, agent=user_obj.username, - ) + ), + redis=redis, ) + # Send notification for the event-source server if redis: redis.publish(obj.uid, json.dumps({'added_tags': added_tags})) @@ -323,9 +326,11 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder, issue=issue.to_json(public=True), project=issue.project.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) + # Send notification for the event-source server if redis: redis.publish(issue.uid, json.dumps({'unassigned': '-'})) @@ -354,9 +359,11 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder, issue=issue.to_json(public=True), project=issue.project.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) + # Send notification for the event-source server if redis: redis.publish(issue.uid, json.dumps( {'assigned': assignee_obj.to_json(public=True)})) @@ -365,7 +372,7 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder, def add_pull_request_assignee( - session, request, assignee, user, requestfolder): + session, request, assignee, user, requestfolder, redis=None): ''' Add an assignee to a request, in other words, assigned an issue. ''' __get_user(session, assignee) user_obj = __get_user(session, user) @@ -385,7 +392,8 @@ def add_pull_request_assignee( request=request.to_json(public=True), project=request.project.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) return 'Request reset' @@ -412,7 +420,8 @@ def add_pull_request_assignee( request=request.to_json(public=True), project=request.project.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis ) return 'Request assigned' @@ -457,9 +466,11 @@ def add_issue_dependency( project=issue.project.to_json(public=True), added_dependency=issue_blocked.id, agent=user_obj.username, - ) + ), + redis=redis ) + # Send notification for the event-source server if redis: redis.publish(issue.uid, json.dumps({ 'added_dependency': issue_blocked.id, @@ -515,9 +526,11 @@ def remove_issue_dependency( project=issue.project.to_json(public=True), removed_dependency=child_del, agent=user_obj.username, - ) + ), + redis=redis ) + # Send notification for the event-source server if redis: redis.publish(issue.uid, json.dumps({ 'removed_dependency': child_del, @@ -533,7 +546,7 @@ def remove_issue_dependency( return 'Dependency removed' -def remove_tags(session, project, tags, ticketfolder, user): +def remove_tags(session, project, tags, ticketfolder, user, redis=None): ''' Removes the specified tag of a project. ''' user_obj = __get_user(session, user) @@ -566,7 +579,8 @@ def remove_tags(session, project, tags, ticketfolder, user): project=project.to_json(public=True), tags=removed_tags, agent=user_obj.username, - ) + ), + redis=redis ) return msgs @@ -599,9 +613,11 @@ def remove_tags_obj( project=obj.project.to_json(public=True), tags=removed_tags, agent=user_obj.username, - ) + ), + redis=redis ) + # Send notification for the event-source server if redis: redis.publish(obj.uid, json.dumps( {'removed_tags': removed_tags})) @@ -609,7 +625,8 @@ def remove_tags_obj( return 'Removed tag: %s' % ', '.join(removed_tags) -def edit_issue_tags(session, project, old_tag, new_tag, ticketfolder, user): +def edit_issue_tags(session, project, old_tag, new_tag, ticketfolder, user, + redis=None): ''' Removes the specified tag of a project. ''' user_obj = __get_user(session, user) @@ -668,13 +685,14 @@ def edit_issue_tags(session, project, old_tag, new_tag, ticketfolder, user): old_tag=old_tag, new_tag=new_tag, agent=user_obj.username, - ) + ), + redis=redis ) return msgs -def add_user_to_project(session, project, new_user, user): +def add_user_to_project(session, project, new_user, user, redis=None): ''' Add a specified user to a specified project. ''' new_user_obj = __get_user(session, new_user) user_obj = __get_user(session, user) @@ -701,13 +719,14 @@ def add_user_to_project(session, project, new_user, user): project=project.to_json(public=True), new_user=new_user_obj.username, agent=user_obj.username, - ) + ), + redis=redis, ) return 'User added' -def add_group_to_project(session, project, new_group, user): +def add_group_to_project(session, project, new_group, user, redis=None): ''' Add a specified group to a specified project. ''' group_obj = search_groups(session, group_name=new_group) @@ -748,7 +767,8 @@ def add_group_to_project(session, project, new_group, user): project=project.to_json(public=True), new_group=group_obj.group_name, agent=user, - ) + ), + redis=redis, ) return 'Group added' @@ -778,6 +798,7 @@ def add_pull_request_comment(session, request, commit, filename, row, if notify: pagure.lib.notify.notify_pull_request_comment(pr_comment, user_obj) + # Send notification for the event-source server if redis: redis.publish(request.uid, json.dumps({ 'request_id': len(request.comments), @@ -796,14 +817,15 @@ def add_pull_request_comment(session, request, commit, filename, row, msg=dict( pullrequest=request.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) return 'Comment added' def add_pull_request_flag(session, request, username, percent, comment, url, - uid, user, requestfolder): + uid, user, requestfolder, redis=None): ''' Add a flag to a pull-request. ''' user_obj = __get_user(session, user) @@ -838,7 +860,8 @@ def add_pull_request_flag(session, request, username, percent, comment, url, pullrequest=request.to_json(public=True), flag=pr_flag.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) return 'Flag %s' % action @@ -916,7 +939,7 @@ def new_project(session, user, name, blacklist, msg=dict( project=project.to_json(public=True), agent=user_obj.username, - ) + ), ) return 'Project "%s" created' % name @@ -924,7 +947,7 @@ def new_project(session, user, name, blacklist, def new_issue(session, repo, title, content, user, ticketfolder, issue_id=None, issue_uid=None, private=False, status=None, - notify=True): + notify=True, redis=None): ''' Create a new issue for the specified repo. ''' user_obj = __get_user(session, user) @@ -959,13 +982,14 @@ def new_issue(session, repo, title, content, user, ticketfolder, issue=issue.to_json(public=True), project=issue.project.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis ) return issue -def drop_issue(session, issue, user, ticketfolder): +def drop_issue(session, issue, user, ticketfolder, redis): ''' Delete a specified issue. ''' user_obj = __get_user(session, user) @@ -986,7 +1010,8 @@ def drop_issue(session, issue, user, ticketfolder): issue=issue.to_json(public=True), project=issue.project.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) return issue @@ -996,7 +1021,7 @@ def new_pull_request(session, branch_from, repo_to, branch_to, title, user, requestfolder, repo_from=None, remote_git=None, requestuid=None, requestid=None, - status='Open', notify=True): + status='Open', notify=True, redis=None): ''' Create a new pull request on the specified repo. ''' if not repo_from and not remote_git: raise pagure.exceptions.PagureException( @@ -1033,7 +1058,8 @@ def new_pull_request(session, branch_from, msg=dict( pullrequest=request.to_json(public=True), agent=user_obj.username, - ) + ), + redis=redis, ) return request @@ -1079,7 +1105,8 @@ def edit_issue(session, issue, ticketfolder, user, project=issue.project.to_json(public=True), fields=edit, agent=user_obj.username, - ) + ), + redis=redis, ) if redis and edit: @@ -1100,7 +1127,7 @@ def edit_issue(session, issue, ticketfolder, user, return 'Successfully edited issue #%s' % issue.id -def update_project_settings(session, repo, settings, user): +def update_project_settings(session, repo, settings, user, redis=None): ''' Update the settings of a project. ''' user_obj = __get_user(session, user) @@ -1137,7 +1164,8 @@ def update_project_settings(session, repo, settings, user): project=repo.to_json(public=True), fields=update, agent=user_obj.username, - ) + ), + redis=redis, ) return 'Edited successfully settings of repo: %s' % repo.fullname @@ -1221,7 +1249,7 @@ def fork_project(session, user, repo, gitfolder, msg=dict( project=project.to_json(public=True), agent=user_obj.username, - ) + ), ) return 'Repo "%s" cloned to "%s/%s"' % (repo.name, user, repo.name) @@ -1671,7 +1699,8 @@ def search_pull_requests( return output -def close_pull_request(session, request, user, requestfolder, merged=True): +def close_pull_request(session, request, user, requestfolder, merged=True, + redis=None): ''' Close the provided pull-request. ''' user_obj = __get_user(session, user) @@ -1700,7 +1729,8 @@ def close_pull_request(session, request, user, requestfolder, merged=True): pullrequest=request.to_json(public=True), merged=merged, agent=user_obj.username, - ) + ), + redis=redis, ) From 12f5e64ca855197d6fe77130e60a5edbc2155820 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:53 +0000 Subject: [PATCH 3/26] Add the web-hook server This server is a dedicated service used only for web-hook notifications --- diff --git a/webhook-server/pagure-webhook-server.py b/webhook-server/pagure-webhook-server.py new file mode 100644 index 0000000..50f7858 --- /dev/null +++ b/webhook-server/pagure-webhook-server.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python + +""" + (c) 2015 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + + +This server listens to message sent via redis and send the corresponding +web-hook request. + +Using this mechanism, we no longer block the main application if the +receiving end is offline or so. + +""" + +import datetime +import hashlib +import hmac +import json +import logging +import os +import requests +import time +import uuid + +import six +import trollius +import trollius_redis + +from kitchen.text.converters import to_bytes + + +log = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure +import pagure.lib +from pagure.exceptions import PagureEvException + +_i = 0 + + +def call_web_hooks(project, topic, msg): + ''' Sends the web-hook notification. ''' + log.info("Processing project %s - sending: %s" % ( + project.fullname, topic) + ) + log.debug('msg: %s' % msg) + + # Send web-hooks notification + global _i + _i += 1 + year = datetime.datetime.now().year + if isinstance(topic, six.text_type): + topic = to_bytes(topic, encoding='utf8', nonstring="passthru") + msg = dict( + topic=topic.decode('utf-8'), + msg=msg, + timestamp=int(time.time()), + msg_id=str(year) + '-' + str(uuid.uuid4()), + i=_i, + ) + + content = json.dumps(msg) + hashhex = hmac.new( + str(project.hook_token), content, hashlib.sha1).hexdigest() + headers = { + 'X-Pagure-Topic': topic, + 'X-Pagure-Signature': hashhex + } + msg = json.dumps(msg) + for url in project.settings.get('Web-hooks').split('\n'): + url = url.strip() + log.info('Calling url %s' % url) + try: + req = requests.post( + url, + headers=headers, + data={'payload': msg} + ) + if not req: + raise pagure.exceptions.PagureException( + 'An error occured while querying: %s - ' + 'Error code: %s' % (url, req.status_code)) + except (requests.exceptions.RequestException, Exception) as err: + raise pagure.exceptions.PagureException( + 'An error occured while querying: %s - Error: %s' % ( + url, err)) + + +@trollius.coroutine +def handle_client(): + connection = yield trollius.From(trollius_redis.Connection.create( + host='0.0.0.0', port=6379, db=0)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['hook'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + print(u'Received: ', repr(reply.value), u'on channel', reply.channel) + data = json.loads(reply.value) + username = None + if '/' in data['project']: + username, projectname = data['project'].split('/', 1) + else: + projectname = data['project'] + project = pagure.lib.get_project( + session=pagure.SESSION, name=projectname, user=username) + call_web_hooks(project, data['topic'], data['msg']) + + +def main(): + server = None + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_client()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + log.info("End Connection") + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() From 19bdd7d21ba5efb24fa3fec52db137c69736ecae Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:53 +0000 Subject: [PATCH 4/26] Add the systemd unit file to start the web-hook server --- diff --git a/webhook-server/pagure_webhook.service b/webhook-server/pagure_webhook.service new file mode 100644 index 0000000..e9d0512 --- /dev/null +++ b/webhook-server/pagure_webhook.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure WebHook server (Allowing web-hook notifications) +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-webhook/pagure-webhook-server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 6f94a22fd77f9102f01c15918adc41c1429ca38f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:53 +0000 Subject: [PATCH 5/26] Connect to a redis server if either the web-hooks or the EV service are activated --- diff --git a/pagure/__init__.py b/pagure/__init__.py index 9852dd9..9a6d2d2 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -58,7 +58,7 @@ import pagure.login_forms FAS = FAS(APP) SESSION = pagure.lib.create_session(APP.config['DB_URL']) REDIS = None -if APP.config['EVENTSOURCE_SOURCE']: +if APP.config['EVENTSOURCE_SOURCE'] or APP.config['WEBHOOK']: POOL = redis.ConnectionPool( host=APP.config['REDIS_HOST'], port=APP.config['REDIS_PORT'], From 5b2543aff78e4826f02df9cdeac0873e0d39343e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:53 +0000 Subject: [PATCH 6/26] Adjust the example configuration file for the global WEBHOOK setting --- diff --git a/files/pagure.cfg.sample b/files/pagure.cfg.sample index 9c1c7da..9461ec5 100644 --- a/files/pagure.cfg.sample +++ b/files/pagure.cfg.sample @@ -124,14 +124,16 @@ SHORT_LENGTH = 6 ### List of blacklisted project names that can conflicts for pagure's URLs ### or other -BLACKLISTED_PROJECTS = ['static', 'pv'] +BLACKLISTED_PROJECTS = [ + 'static', 'pv', 'releases', 'new', 'api', 'settings', + 'logout', 'login', 'users', 'groups'] ### IP addresses allowed to access the internal endpoints ### These endpoints are used by the milter and are security sensitive, thus ### the IP filter IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] -### EventSource/Redis configuration +### EventSource/Web-Hook/Redis configuration # The eventsource integration is what allows pagure to refresh the content # on your page when someone else comments on the ticket (and this without # asking you to reload the page. @@ -140,9 +142,6 @@ IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # https://ev.pagure.io or https://pagure.io:8080 or whatever you are using # (Note: the urls sent to it start with a '/' so no need to add one yourself) EVENTSOURCE_SOURCE = None -REDIS_HOST = '0.0.0.0' -REDIS_PORT = 6379 -REDIS_DB = 0 # Port where the event source server is running (maybe be the same port # as the one specified in EVENTSOURCE_SOURCE or a different one if you # have something running in front of the server such as apache or stunnel). @@ -151,6 +150,16 @@ EVENTSOURCE_PORT = 8080 # at this port and will provide information about the number of active # connections running on the first (main) event source server #EV_STATS_PORT = 8888 +# Web-hook can be turned on or off allowing using them for notifications, or +# not. +WEBHOOK = False + +### Redis configuration +# A redis server is required for both the Event-Source server or the web-hook +# server. +REDIS_HOST = '0.0.0.0' +REDIS_PORT = 6379 +REDIS_DB = 0 # Authentication related configuration option From 426110a8460131adafd9648e9f4a71498e9bff3d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 7/26] Turn off the web-hook notifications by default --- diff --git a/pagure/default_config.py b/pagure/default_config.py index bc86d7b..f26737c 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -69,6 +69,7 @@ IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] # Redis configuration EVENTSOURCE_SOURCE = None +WEBHOOK = False REDIS_HOST = '0.0.0.0' REDIS_PORT = 6379 REDIS_DB = 0 From 67a460e7417f713147df29b5abf0469ff22878a3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 8/26] Disable some of the project's settings if the web-hooks are not configured --- diff --git a/pagure/templates/settings.html b/pagure/templates/settings.html index fa76eb0..007ba42 100644 --- a/pagure/templates/settings.html +++ b/pagure/templates/settings.html @@ -68,6 +68,8 @@ + + {% if config.get('WEBHOOK') %}

Private web-hook key

@@ -95,6 +97,7 @@ {{ form.csrf_token }}

+ {% endif %}

API key

@@ -169,6 +172,7 @@ {% for key in repo.settings | sort %} {% if not config.get('ENABLE_TICKETS', True) and key in ['issue_tracker'] %} {% elif not config.get('DOC_APP_URL') and key in ['project_documentation'] %} + {% elif not config.get('WEBHOOK') and key in ['Web-hooks'] %} {% else %} From ec8a64596f5cf67b6eea059062ea3030ccb68175 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 9/26] Raise a 404 upon requesting a new project token if web-hooks are not configured --- diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index ca0cd0b..8a0762b 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -986,6 +986,9 @@ def delete_repo(repo, username=None): def new_repo_hook_token(repo, username=None): """ Re-generate a hook token for the present project. """ + if not pagure.APP.config.get('WEBHOOK', False): + flask.abort(404) + if admin_session_timedout(): flask.flash('Action canceled, try it again', 'error') url = flask.url_for( From 68903e090df54b09f322b74a0ef893131ea604cc Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 10/26] Adjust the documentation about the configuration of pagure for the web-hook server --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 6dc11ed..60d1878 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -239,6 +239,28 @@ This configuration key indicates the port at which the EventSource server is running. This allows adjusting the port via the configuration file instead of hard-coding it in the code. +.. note:: The EventSource server requires a redis server (see ``Redis options`` + below) + + +Web-hooks notifications +----------------------- + +WEBHOOK +~~~~~~~ + +This configuration key allows turning on or off web-hooks notifications for +this pagure instance. + +Defaults to: ``False``. + +.. note:: The Web-hooks server requires a redis server (see ``Redis options`` + below) + + +Redis options +------------- + REDIS_HOST ~~~~~~~~~~ From 9c66ac162a8775c25cf7647a2f4e8d7187f39b84 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 11/26] Expand the documentation about web-hook on the project's settings doc --- diff --git a/doc/project_settings.rst b/doc/project_settings.rst index a84ba7b..b50081e 100644 --- a/doc/project_settings.rst +++ b/doc/project_settings.rst @@ -85,8 +85,10 @@ To vote against a pull-request, use either: Web-hooks --------- -Pagure supports sending notification about event happening on a project -via [web-hooks|]. +Pagure offers the option of sending notification about event happening on a +project via [web-hooks|https://en.wikipedia.org/wiki/Webhook]. This option +is off by default and can be turned on for a pagure instance in its +configuration file. The URL of the web-hooks can be entered in this field. From 4cbf477ea4b40cb71400e289a9ed2b20ab567937 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 12/26] Add a documentation on how to install the web-hook server --- diff --git a/doc/install_webhooks.rst b/doc/install_webhooks.rst new file mode 100644 index 0000000..e493e99 --- /dev/null +++ b/doc/install_webhooks.rst @@ -0,0 +1,49 @@ +Installing pagure's web-hooks notification system +================================================= + +Web-hooks are a notification system upon which a system makes a http POST +request with some data upon doing an action. This allows notifying a system +that an action has occured. + +If you want more information feel free to check out the corresponding page +on wikipedia: `https://en.wikipedia.org/wiki/Webhook +`_. + +Configure your system +--------------------- + +* Install the required dependencies + +:: + + python-redis + python-trollius + python-trollius-redis + +..note:: We ship a systemd unit file for pagure_webhook but we welcome patches + for scripts for other init systems. + + +* Install the files of the web-hook server as follow: + ++----------------------------------------------+----------------------------------------------------------+ +| Source | Destination | ++==============================================+==========================================================+ +| ``webhook-server/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | ++----------------------------------------+----------------------------------------------------------------+ +| ``webhook-server/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | ++--------------------------------------+------------------------------------------------------------------+ + +The first file is the script of the web-hook server itself. + +The secondg file is the systemd service file. + + +* Activate the service and ensure it's started upon boot: + +:: + + systemctl enable redis + systemctl start redis + systemctl enable pagure_webhook + systemctl start pagure_webhook From 655e9011f60f0e2c9c5f462b3309f59fae1dbdc8 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 13/26] Link to the documentation about installing the web-hook server from the front page --- diff --git a/doc/index.rst b/doc/index.rst index 56b2298..b838116 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -33,6 +33,7 @@ Contents: install install_milter install_evs + install_webhooks configuration development contributing From 674992c254037bb6007542cf681f7ca5f4d1bbd7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 14/26] Mentions the web-hook package on the install doc --- diff --git a/doc/install.rst b/doc/install.rst index e239605..d778f1c 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -20,12 +20,12 @@ its derivative via the `EPEL repository `. So installing it is as easy as: :: - dnf install pagure pagure-milters pagure-ev + dnf install pagure pagure-milters pagure-ev pagure-webhook or :: - yum install pagure pagure-milters pagure-ev + yum install pagure pagure-milters pagure-ev pagure-webhook The ``pagure`` package contains the core of the application and the doc server. (See the ``Overview`` page for a global overview of the structure of the @@ -36,7 +36,10 @@ mail filter to hook into a MTA). The ``pagure-ev`` package contains the eventsource server. -..note:: The last two packages are optional, pagure would work fine without +The ``pagure-webhook`` package contains the web-hook server. + + +..note:: The last three packages are optional, pagure would work fine without them. * From the sources From b6a8b247da93ff86af1751400d6005f8f26a4bc8 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 15/26] Fix formatting/syntax --- diff --git a/doc/install.rst b/doc/install.rst index d778f1c..1765740 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -25,6 +25,7 @@ So installing it is as easy as: or :: + yum install pagure pagure-milters pagure-ev pagure-webhook The ``pagure`` package contains the core of the application and the doc server. @@ -39,7 +40,7 @@ The ``pagure-ev`` package contains the eventsource server. The ``pagure-webhook`` package contains the web-hook server. -..note:: The last three packages are optional, pagure would work fine without +.. note:: The last three packages are optional, pagure would work fine without them. * From the sources @@ -94,7 +95,7 @@ To install pagure via this mechanism simply follow these steps: python setup.py build sudo python setup.py install -..note:: To install the eventsource server or the milter, refer to their +.. note:: To install the eventsource server or the milter, refer to their respective documentations. # Install the additional files as follow: diff --git a/doc/install_evs.rst b/doc/install_evs.rst index 2819e3e..51f4965 100644 --- a/doc/install_evs.rst +++ b/doc/install_evs.rst @@ -19,7 +19,7 @@ The eventsource server is easy to set-up. python-trollius python-trollius-redis -..note:: We ship a systemd unit file for pagure_milter but we welcome patches +.. note:: We ship a systemd unit file for pagure_milter but we welcome patches for scripts for other init systems. diff --git a/doc/install_milter.rst b/doc/install_milter.rst index fb029d6..dc1524f 100644 --- a/doc/install_milter.rst +++ b/doc/install_milter.rst @@ -18,10 +18,10 @@ Configure your system python-pymilter -..note:: We ship a systemd unit file for pagure_milter but we welcome patches +.. note:: We ship a systemd unit file for pagure_milter but we welcome patches for scripts for other init systems. -..note:: It also requires a MTA, we used postfix. +.. note:: It also requires a MTA, we used postfix. * Create an alias ``reply`` @@ -57,9 +57,9 @@ In postfix this is done via: | Source | Destination | +======================================+===================================================+ | ``milters/comment_email_milter.py`` | ``/usr/share//pagure/comment_email_milter.py`` | -+----------------------------------------+-------------------------------------------------+ ++--------------------------------------+---------------------------------------------------+ | ``milters/milter_tempfile.conf`` | ``/usr/lib/tmpfiles.d/pagure-milter.conf`` | -+----------------------------------------+-------------------------------------------------+ ++--------------------------------------+---------------------------------------------------+ | ``milters/pagure_milter.service`` | ``/etc/systemd/system/pagure_milter.service`` | +--------------------------------------+---------------------------------------------------+ diff --git a/doc/install_webhooks.rst b/doc/install_webhooks.rst index e493e99..a532941 100644 --- a/doc/install_webhooks.rst +++ b/doc/install_webhooks.rst @@ -20,7 +20,7 @@ Configure your system python-trollius python-trollius-redis -..note:: We ship a systemd unit file for pagure_webhook but we welcome patches +.. note:: We ship a systemd unit file for pagure_webhook but we welcome patches for scripts for other init systems. @@ -30,9 +30,9 @@ Configure your system | Source | Destination | +==============================================+==========================================================+ | ``webhook-server/pagure-webhook-server.py`` | ``/usr/libexec/pagure-webhook/pagure-webhook-server.py`` | -+----------------------------------------+----------------------------------------------------------------+ ++----------------------------------------------+----------------------------------------------------------+ | ``webhook-server/pagure_webhook.service`` | ``/etc/systemd/system/pagure_webhook.service`` | -+--------------------------------------+------------------------------------------------------------------+ ++----------------------------------------------+----------------------------------------------------------+ The first file is the script of the web-hook server itself. From fb7f0c26333508150b177933a09a1586adbcf266 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 16/26] Set the webhook settings to True to test the hook_token re-generation --- diff --git a/tests/test_progit_flask_ui_repo.py b/tests/test_progit_flask_ui_repo.py index 200ee10..78fe9a2 100644 --- a/tests/test_progit_flask_ui_repo.py +++ b/tests/test_progit_flask_ui_repo.py @@ -1520,6 +1520,7 @@ index 0000000..fb7093d user = tests.FakeUser() with tests.user_set(pagure.APP, user): + pagure.APP.config['WEBHOOK'] = True output = self.app.get('/new/') self.assertEqual(output.status_code, 200) self.assertTrue('

New project

' in output.data) @@ -1538,11 +1539,14 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 302) ast.return_value = False + pagure.APP.config['WEBHOOK'] = False + repo = pagure.lib.get_project(self.session, 'test') self.assertEqual(repo.hook_token, 'aaabbbccc') user.username = 'pingou' with tests.user_set(pagure.APP, user): + pagure.APP.config['WEBHOOK'] = True output = self.app.post('/test/hook_token') self.assertEqual(output.status_code, 400) @@ -1558,6 +1562,7 @@ index 0000000..fb7093d self.assertIn( '
  • New hook token generated
    • ', output.data) + pagure.APP.config['WEBHOOK'] = False repo = pagure.lib.get_project(self.session, 'test') self.assertNotEqual(repo.hook_token, 'aaabbbccc') From a241a96a0a0f22d66f96fef742c1441735ddf3d3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:54 +0000 Subject: [PATCH 17/26] redis is never necessary --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 763f9f2..18afe52 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -989,7 +989,7 @@ def new_issue(session, repo, title, content, user, ticketfolder, return issue -def drop_issue(session, issue, user, ticketfolder, redis): +def drop_issue(session, issue, user, ticketfolder, redis=None): ''' Delete a specified issue. ''' user_obj = __get_user(session, user) From e68a84d75c6eb62effb6300b9b72e564c6d55a0f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:55 +0000 Subject: [PATCH 18/26] Adjust the spec file to include and create the webhook sub-package --- diff --git a/files/pagure.spec b/files/pagure.spec index f56a67a..53da8e5 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -80,6 +80,7 @@ Currently, Pagure offers a web-interface for git repositories, a ticket system and possibilities to create new projects, fork existing ones and create/merge pull-requests across or within projects. + %package milters Summary: Milter to integrate pagure with emails BuildArch: noarch @@ -91,8 +92,6 @@ Requires(postun): systemd # It would work with sendmail but we configure things (like the tempfile) # to work with postfix Requires: postfix - - %description milters Milters (Mail filters) allowing the integration of pagure and emails. This is useful for example to allow commenting on a ticket by email. @@ -111,7 +110,23 @@ Requires(preun): systemd Requires(postun): systemd %description ev Pagure comes with an eventsource server allowing live update of the pages -supporting it. This packages provides it. +supporting it. This package provides it. + + +%package webhook +Summary: Web-Hook server for pagure +BuildArch: noarch + +BuildRequires: systemd-devel +Requires: python-redis +Requires: python-trollius +Requires: python-trollius-redis +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +%description ev +Pagure comes with an webhook server allowing http callbacks for any action +done on a project. This package provides it. %prep @@ -165,21 +180,33 @@ install -m 755 ev-server/pagure-stream-server.py \ install -m 644 ev-server/pagure_ev.service \ $RPM_BUILD_ROOT/%{_unitdir}/pagure_ev.service +# Install the web-hook +mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/pagure-webhook +install -m 755 webhook-server/pagure-webhook-server.py \ + $RPM_BUILD_ROOT/%{_libexecdir}/pagure-webhook/pagure-webhook-server.py +install -m 644 webhook-server/pagure_webhook.service \ + $RPM_BUILD_ROOT/%{_unitdir}/pagure_webhook.service %post milters %systemd_post pagure_milter.service %post ev %systemd_post pagure_ev.service +%post webhook +%systemd_post pagure_webhook.service %preun milters %systemd_preun pagure_milter.service %preun ev %systemd_preun pagure_ev.service +%preun webhook +%systemd_preun pagure_webhook.service %postun milters %systemd_postun_with_restart pagure_milter.service %postun ev %systemd_postun_with_restart pagure_ev.service +%postun webhook +%systemd_postun_with_restart pagure_webhook.service %files @@ -211,6 +238,12 @@ install -m 644 ev-server/pagure_ev.service \ %{_unitdir}/pagure_ev.service +%files webhook +%license LICENSE +%{_libexecdir}/pagure-webhook/ +%{_unitdir}/pagure_webhook.service + + %changelog * Fri Nov 20 2015 Pierre-Yves Chibon - 0.1.33-1 - Update to 0.1.33 From bdd7032a57319bba98ff783ef91a00f59949b19f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:55 +0000 Subject: [PATCH 19/26] Include the webhook-server in the releases --- diff --git a/MANIFEST.in b/MANIFEST.in index c107032..d7202fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,3 +7,4 @@ recursive-include tests * recursive-include doc * recursive-include alembic * recursive-include ev-server * +recursive-include webhook-server * From 2e464aa974e47fa29d4e2460bc1c0c01ece8935f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:55 +0000 Subject: [PATCH 20/26] Re-work the way the redis server is retrieved With this approach we set it once and for all at the start of the application and the connection remains then available to all the method in the internal library. --- diff --git a/pagure/__init__.py b/pagure/__init__.py index 9a6d2d2..d6b50e5 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -25,7 +25,6 @@ from logging.handlers import SMTPHandler import flask import pygit2 -import redis import werkzeug from pagure.flask_fas_openid import FAS from functools import wraps @@ -59,11 +58,11 @@ FAS = FAS(APP) SESSION = pagure.lib.create_session(APP.config['DB_URL']) REDIS = None if APP.config['EVENTSOURCE_SOURCE'] or APP.config['WEBHOOK']: - POOL = redis.ConnectionPool( + pagure.lib.set_redis( host=APP.config['REDIS_HOST'], port=APP.config['REDIS_PORT'], - db=APP.config['REDIS_DB']) - REDIS = redis.StrictRedis(connection_pool=POOL) + db=APP.config['REDIS_DB'] + ) if not APP.debug: APP.logger.addHandler(pagure.mail_logging.get_mail_handler( diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 18afe52..621227c 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -22,6 +22,7 @@ import urlparse import uuid import bleach +import redis import sqlalchemy import sqlalchemy.schema from datetime import timedelta @@ -44,6 +45,16 @@ from pagure.lib import model # pylint: disable=R0913 +REDIS = None + + +def set_redis(host, port, db): + """ Set the redis connection with the specified information. """ + global REDIS + pool = redis.ConnectionPool(host=host, port=port, db=db) + REDIS = redis.StrictRedis(connection_pool=pool) + + def __get_user(session, key): """ Searches for a user in the database for a given username or email. """ @@ -188,7 +199,7 @@ def create_user_ssh_keys_on_disk(user, gitolite_keydir): def add_issue_comment(session, issue, comment, user, ticketfolder, - notify=True, redis=None): + notify=True): ''' Add a comment to an issue. ''' user_obj = __get_user(session, user) @@ -216,10 +227,10 @@ def add_issue_comment(session, issue, comment, user, ticketfolder, project=issue.project.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) - if redis: + if REDIS: if issue.private: redis.publish(issue.uid, json.dumps({ 'issue': 'private', @@ -238,7 +249,7 @@ def add_issue_comment(session, issue, comment, user, ticketfolder, return 'Comment added' -def add_tag_obj(session, obj, tags, user, ticketfolder, redis=None): +def add_tag_obj(session, obj, tags, user, ticketfolder): ''' Add a tag to an object (either an issue or a project). ''' user_obj = __get_user(session, user) @@ -292,11 +303,11 @@ def add_tag_obj(session, obj, tags, user, ticketfolder, redis=None): tags=added_tags, agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) # Send notification for the event-source server - if redis: + if REDIS: redis.publish(obj.uid, json.dumps({'added_tags': added_tags})) if added_tags: @@ -305,8 +316,7 @@ def add_tag_obj(session, obj, tags, user, ticketfolder, redis=None): return 'Nothing to add' -def add_issue_assignee(session, issue, assignee, user, ticketfolder, - redis=None): +def add_issue_assignee(session, issue, assignee, user, ticketfolder): ''' Add an assignee to an issue, in other words, assigned an issue. ''' user_obj = __get_user(session, user) @@ -327,11 +337,11 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder, project=issue.project.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) # Send notification for the event-source server - if redis: + if REDIS: redis.publish(issue.uid, json.dumps({'unassigned': '-'})) return 'Assignee reset' @@ -360,11 +370,11 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder, project=issue.project.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) # Send notification for the event-source server - if redis: + if REDIS: redis.publish(issue.uid, json.dumps( {'assigned': assignee_obj.to_json(public=True)})) @@ -372,7 +382,7 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder, def add_pull_request_assignee( - session, request, assignee, user, requestfolder, redis=None): + session, request, assignee, user, requestfolder): ''' Add an assignee to a request, in other words, assigned an issue. ''' __get_user(session, assignee) user_obj = __get_user(session, user) @@ -393,7 +403,7 @@ def add_pull_request_assignee( project=request.project.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) return 'Request reset' @@ -421,14 +431,14 @@ def add_pull_request_assignee( project=request.project.to_json(public=True), agent=user_obj.username, ), - redis=redis + redis=REDIS, ) return 'Request assigned' def add_issue_dependency( - session, issue, issue_blocked, user, ticketfolder, redis=None): + session, issue, issue_blocked, user, ticketfolder): ''' Add a dependency between two issues. ''' user_obj = __get_user(session, user) @@ -467,11 +477,11 @@ def add_issue_dependency( added_dependency=issue_blocked.id, agent=user_obj.username, ), - redis=redis + redis=REDIS, ) # Send notification for the event-source server - if redis: + if REDIS: redis.publish(issue.uid, json.dumps({ 'added_dependency': issue_blocked.id, 'issue_uid': issue.uid, @@ -487,7 +497,7 @@ def add_issue_dependency( def remove_issue_dependency( - session, issue, issue_blocked, user, ticketfolder, redis=None): + session, issue, issue_blocked, user, ticketfolder): ''' Remove a dependency between two issues. ''' user_obj = __get_user(session, user) @@ -527,11 +537,11 @@ def remove_issue_dependency( removed_dependency=child_del, agent=user_obj.username, ), - redis=redis + redis=REDIS, ) # Send notification for the event-source server - if redis: + if REDIS: redis.publish(issue.uid, json.dumps({ 'removed_dependency': child_del, 'issue_uid': issue.uid, @@ -546,7 +556,7 @@ def remove_issue_dependency( return 'Dependency removed' -def remove_tags(session, project, tags, ticketfolder, user, redis=None): +def remove_tags(session, project, tags, ticketfolder, user): ''' Removes the specified tag of a project. ''' user_obj = __get_user(session, user) @@ -580,14 +590,14 @@ def remove_tags(session, project, tags, ticketfolder, user, redis=None): tags=removed_tags, agent=user_obj.username, ), - redis=redis + redis=REDIS, ) return msgs def remove_tags_obj( - session, obj, tags, ticketfolder, user, redis=None): + session, obj, tags, ticketfolder, user): ''' Removes the specified tag(s) of a given object. ''' user_obj = __get_user(session, user) @@ -614,19 +624,18 @@ def remove_tags_obj( tags=removed_tags, agent=user_obj.username, ), - redis=redis + redis=REDIS, ) # Send notification for the event-source server - if redis: + if REDIS: redis.publish(obj.uid, json.dumps( {'removed_tags': removed_tags})) return 'Removed tag: %s' % ', '.join(removed_tags) -def edit_issue_tags(session, project, old_tag, new_tag, ticketfolder, user, - redis=None): +def edit_issue_tags(session, project, old_tag, new_tag, ticketfolder, user): ''' Removes the specified tag of a project. ''' user_obj = __get_user(session, user) @@ -686,13 +695,13 @@ def edit_issue_tags(session, project, old_tag, new_tag, ticketfolder, user, new_tag=new_tag, agent=user_obj.username, ), - redis=redis + redis=REDIS, ) return msgs -def add_user_to_project(session, project, new_user, user, redis=None): +def add_user_to_project(session, project, new_user, user): ''' Add a specified user to a specified project. ''' new_user_obj = __get_user(session, new_user) user_obj = __get_user(session, user) @@ -720,13 +729,13 @@ def add_user_to_project(session, project, new_user, user, redis=None): new_user=new_user_obj.username, agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) return 'User added' -def add_group_to_project(session, project, new_group, user, redis=None): +def add_group_to_project(session, project, new_group, user): ''' Add a specified group to a specified project. ''' group_obj = search_groups(session, group_name=new_group) @@ -768,15 +777,14 @@ def add_group_to_project(session, project, new_group, user, redis=None): new_group=group_obj.group_name, agent=user, ), - redis=redis, + redis=REDIS, ) return 'Group added' def add_pull_request_comment(session, request, commit, filename, row, - comment, user, requestfolder, notify=True, - redis=None): + comment, user, requestfolder, notify=True): ''' Add a comment to a pull-request. ''' user_obj = __get_user(session, user) @@ -799,7 +807,7 @@ def add_pull_request_comment(session, request, commit, filename, row, pagure.lib.notify.notify_pull_request_comment(pr_comment, user_obj) # Send notification for the event-source server - if redis: + if REDIS: redis.publish(request.uid, json.dumps({ 'request_id': len(request.comments), 'comment_added': text2markdown(pr_comment.comment), @@ -818,14 +826,14 @@ def add_pull_request_comment(session, request, commit, filename, row, pullrequest=request.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) return 'Comment added' def add_pull_request_flag(session, request, username, percent, comment, url, - uid, user, requestfolder, redis=None): + uid, user, requestfolder): ''' Add a flag to a pull-request. ''' user_obj = __get_user(session, user) @@ -861,7 +869,7 @@ def add_pull_request_flag(session, request, username, percent, comment, url, flag=pr_flag.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) return 'Flag %s' % action @@ -947,7 +955,7 @@ def new_project(session, user, name, blacklist, def new_issue(session, repo, title, content, user, ticketfolder, issue_id=None, issue_uid=None, private=False, status=None, - notify=True, redis=None): + notify=True): ''' Create a new issue for the specified repo. ''' user_obj = __get_user(session, user) @@ -983,13 +991,13 @@ def new_issue(session, repo, title, content, user, ticketfolder, project=issue.project.to_json(public=True), agent=user_obj.username, ), - redis=redis + redis=REDIS, ) return issue -def drop_issue(session, issue, user, ticketfolder, redis=None): +def drop_issue(session, issue, user, ticketfolder): ''' Delete a specified issue. ''' user_obj = __get_user(session, user) @@ -1011,7 +1019,7 @@ def drop_issue(session, issue, user, ticketfolder, redis=None): project=issue.project.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) return issue @@ -1021,7 +1029,7 @@ def new_pull_request(session, branch_from, repo_to, branch_to, title, user, requestfolder, repo_from=None, remote_git=None, requestuid=None, requestid=None, - status='Open', notify=True, redis=None): + status='Open', notify=True): ''' Create a new pull request on the specified repo. ''' if not repo_from and not remote_git: raise pagure.exceptions.PagureException( @@ -1059,15 +1067,14 @@ def new_pull_request(session, branch_from, pullrequest=request.to_json(public=True), agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) return request def edit_issue(session, issue, ticketfolder, user, - title=None, content=None, status=None, private=False, - redis=None): + title=None, content=None, status=None, private=False): ''' Edit the specified issue. ''' user_obj = __get_user(session, user) @@ -1106,10 +1113,10 @@ def edit_issue(session, issue, ticketfolder, user, fields=edit, agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) - if redis and edit: + if REDIS and edit: if issue.private: redis.publish(issue.uid, json.dumps({ 'issue': 'private', @@ -1127,7 +1134,7 @@ def edit_issue(session, issue, ticketfolder, user, return 'Successfully edited issue #%s' % issue.id -def update_project_settings(session, repo, settings, user, redis=None): +def update_project_settings(session, repo, settings, user): ''' Update the settings of a project. ''' user_obj = __get_user(session, user) @@ -1165,7 +1172,7 @@ def update_project_settings(session, repo, settings, user, redis=None): fields=update, agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) return 'Edited successfully settings of repo: %s' % repo.fullname @@ -1699,8 +1706,7 @@ def search_pull_requests( return output -def close_pull_request(session, request, user, requestfolder, merged=True, - redis=None): +def close_pull_request(session, request, user, requestfolder, merged=True): ''' Close the provided pull-request. ''' user_obj = __get_user(session, user) @@ -1730,7 +1736,7 @@ def close_pull_request(session, request, user, requestfolder, merged=True, merged=merged, agent=user_obj.username, ), - redis=redis, + redis=REDIS, ) @@ -1938,7 +1944,7 @@ def avatar_url_from_openid(openid, size=64, default='retro', dns=False): hashhex, query) -def update_tags(session, obj, tags, username, ticketfolder, redis=None): +def update_tags(session, obj, tags, username, ticketfolder): """ Update the tags of a specified object (adding or removing them). This object can be either an issue or a project. @@ -1957,7 +1963,6 @@ def update_tags(session, obj, tags, username, ticketfolder, redis=None): tags=toadd, user=username, ticketfolder=ticketfolder, - redis=redis, ) ) @@ -1969,7 +1974,6 @@ def update_tags(session, obj, tags, username, ticketfolder, redis=None): tags=torm, user=username, ticketfolder=ticketfolder, - redis=redis, ) ) session.commit() @@ -1978,7 +1982,7 @@ def update_tags(session, obj, tags, username, ticketfolder, redis=None): def update_dependency_issue( - session, repo, issue, depends, username, ticketfolder, redis=None): + session, repo, issue, depends, username, ticketfolder): """ Update the dependency of a specified issue (adding or removing them) """ @@ -2005,7 +2009,6 @@ def update_dependency_issue( issue_blocked=issue, user=username, ticketfolder=ticketfolder, - redis=redis, ) ) @@ -2027,7 +2030,6 @@ def update_dependency_issue( issue_blocked=issue_depend, user=username, ticketfolder=ticketfolder, - redis=redis, ) ) @@ -2036,7 +2038,7 @@ def update_dependency_issue( def update_blocked_issue( - session, repo, issue, blocks, username, ticketfolder, redis=None): + session, repo, issue, blocks, username, ticketfolder): """ Update the upstream dependency of a specified issue (adding or removing them) @@ -2064,7 +2066,6 @@ def update_blocked_issue( issue_blocked=issue_block, user=username, ticketfolder=ticketfolder, - redis=redis, ) ) session.commit() @@ -2088,7 +2089,6 @@ def update_blocked_issue( issue_blocked=issue, user=username, ticketfolder=ticketfolder, - redis=redis, ) ) diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 7bbe082..e1277e9 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -399,7 +399,7 @@ def get_project_from_json( if tags: pagure.lib.add_tag_obj( session, project, tags=tags, user=user.username, - ticketfolder=None, redis=None) + ticketfolder=None) return project diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 2a423e3..2f55525 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -20,8 +20,7 @@ import pagure.exceptions import pagure.lib import pagure.lib.git import pagure.forms -from pagure import (APP, REDIS, SESSION, LOG, cla_required, - is_repo_admin) +from pagure import (APP, SESSION, LOG, cla_required, is_repo_admin) # pylint: disable=E1101 @@ -445,7 +444,6 @@ def pull_request_add_comment( comment=comment, user=flask.g.fas_user.username, requestfolder=APP.config['REQUESTS_FOLDER'], - redis=REDIS, ) SESSION.commit() if not is_js: diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 41a48cf..c28bc8b 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -21,7 +21,7 @@ import mimetypes import pagure.doc_utils import pagure.lib import pagure.forms -from pagure import (APP, SESSION, REDIS, LOG, __get_file_in_tree, +from pagure import (APP, SESSION, LOG, __get_file_in_tree, cla_required, is_repo_admin, authenticated) @@ -135,7 +135,6 @@ def update_issue(repo, issueid, username=None): comment=comment, user=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], - redis=REDIS, ) SESSION.commit() if message and not is_js: @@ -146,8 +145,8 @@ def update_issue(repo, issueid, username=None): messages = pagure.lib.update_tags( SESSION, issue, tags, username=flask.g.fas_user.username, - ticketfolder=APP.config['TICKETS_FOLDER'], - redis=REDIS) + ticketfolder=APP.config['TICKETS_FOLDER'] + ) if not is_js: for message in messages: flask.flash(message) @@ -159,7 +158,6 @@ def update_issue(repo, issueid, username=None): assignee=assignee or None, user=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], - redis=REDIS, ) if message and not is_js: SESSION.commit() @@ -175,7 +173,6 @@ def update_issue(repo, issueid, username=None): private=issue.private, user=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], - redis=REDIS, ) SESSION.commit() if message: @@ -186,7 +183,6 @@ def update_issue(repo, issueid, username=None): SESSION, repo, issue, depends, username=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], - redis=REDIS, ) if not is_js: for message in messages: @@ -197,7 +193,6 @@ def update_issue(repo, issueid, username=None): SESSION, repo, issue, blocks, username=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], - redis=REDIS, ) if not is_js: for message in messages: @@ -618,7 +613,6 @@ def edit_issue(repo, issueid, username=None): user=flask.g.fas_user.username, ticketfolder=APP.config['TICKETS_FOLDER'], private=private, - redis=REDIS, ) SESSION.commit() diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 8a0762b..63f316d 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -868,7 +868,7 @@ def update_project(repo, username=None): tags=[t.strip() for t in form.tags.data.split(',')], username=flask.g.fas_user.username, ticketfolder=None, - redis=None) + ) SESSION.add(repo) SESSION.commit() flask.flash('Project updated') diff --git a/tests/test_progit_flask_api_project.py b/tests/test_progit_flask_api_project.py index 3a320a5..d20b452 100644 --- a/tests/test_progit_flask_api_project.py +++ b/tests/test_progit_flask_api_project.py @@ -114,7 +114,7 @@ class PagureFlaskApiProjecttests(tests.Modeltests): # Adding a tag output = pagure.lib.update_tags( self.session, repo, 'infra', 'pingou', - ticketfolder=None, redis=None) + ticketfolder=None) self.assertEqual(output, ['Tag added: infra']) # Check after adding From 83cdec04cedc7b754fa965475bd30f787a902372 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:55 +0000 Subject: [PATCH 21/26] Fix publishing information on redis by using REDIS which is the redis connection --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 621227c..ec1b81b 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -232,12 +232,12 @@ def add_issue_comment(session, issue, comment, user, ticketfolder, if REDIS: if issue.private: - redis.publish(issue.uid, json.dumps({ + REDIS.publish(issue.uid, json.dumps({ 'issue': 'private', 'comment_id': issue_comment.id, })) else: - redis.publish(issue.uid, json.dumps({ + REDIS.publish(issue.uid, json.dumps({ 'comment_id': issue_comment.id, 'comment_added': text2markdown(issue_comment.comment), 'comment_user': issue_comment.user.user, @@ -308,7 +308,7 @@ def add_tag_obj(session, obj, tags, user, ticketfolder): # Send notification for the event-source server if REDIS: - redis.publish(obj.uid, json.dumps({'added_tags': added_tags})) + REDIS.publish(obj.uid, json.dumps({'added_tags': added_tags})) if added_tags: return 'Tag added: %s' % ', '.join(added_tags) @@ -342,7 +342,7 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder): # Send notification for the event-source server if REDIS: - redis.publish(issue.uid, json.dumps({'unassigned': '-'})) + REDIS.publish(issue.uid, json.dumps({'unassigned': '-'})) return 'Assignee reset' elif assignee is None and issue.assignee is None: @@ -375,7 +375,7 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder): # Send notification for the event-source server if REDIS: - redis.publish(issue.uid, json.dumps( + REDIS.publish(issue.uid, json.dumps( {'assigned': assignee_obj.to_json(public=True)})) return 'Issue assigned' @@ -482,12 +482,12 @@ def add_issue_dependency( # Send notification for the event-source server if REDIS: - redis.publish(issue.uid, json.dumps({ + REDIS.publish(issue.uid, json.dumps({ 'added_dependency': issue_blocked.id, 'issue_uid': issue.uid, 'type': 'children', })) - redis.publish(issue_blocked.uid, json.dumps({ + REDIS.publish(issue_blocked.uid, json.dumps({ 'added_dependency': issue.id, 'issue_uid': issue_blocked.uid, 'type': 'parent', @@ -542,12 +542,12 @@ def remove_issue_dependency( # Send notification for the event-source server if REDIS: - redis.publish(issue.uid, json.dumps({ + REDIS.publish(issue.uid, json.dumps({ 'removed_dependency': child_del, 'issue_uid': issue.uid, 'type': 'children', })) - redis.publish(issue_blocked.uid, json.dumps({ + REDIS.publish(issue_blocked.uid, json.dumps({ 'removed_dependency': issue.id, 'issue_uid': issue_blocked.uid, 'type': 'parent', @@ -629,7 +629,7 @@ def remove_tags_obj( # Send notification for the event-source server if REDIS: - redis.publish(obj.uid, json.dumps( + REDIS.publish(obj.uid, json.dumps( {'removed_tags': removed_tags})) return 'Removed tag: %s' % ', '.join(removed_tags) @@ -808,7 +808,7 @@ def add_pull_request_comment(session, request, commit, filename, row, # Send notification for the event-source server if REDIS: - redis.publish(request.uid, json.dumps({ + REDIS.publish(request.uid, json.dumps({ 'request_id': len(request.comments), 'comment_added': text2markdown(pr_comment.comment), 'comment_user': pr_comment.user.user, @@ -1118,12 +1118,12 @@ def edit_issue(session, issue, ticketfolder, user, if REDIS and edit: if issue.private: - redis.publish(issue.uid, json.dumps({ + REDIS.publish(issue.uid, json.dumps({ 'issue': 'private', 'fields': edit, })) else: - redis.publish(issue.uid, json.dumps({ + REDIS.publish(issue.uid, json.dumps({ 'fields': edit, 'issue': issue.to_json(public=True, with_comments=False), })) From fc2b269c60a7f574d438dd959a606356ea471426 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:55 +0000 Subject: [PATCH 22/26] Fix logging information while processing them --- diff --git a/webhook-server/pagure-webhook-server.py b/webhook-server/pagure-webhook-server.py index 50f7858..16a35d3 100644 --- a/webhook-server/pagure-webhook-server.py +++ b/webhook-server/pagure-webhook-server.py @@ -50,10 +50,9 @@ _i = 0 def call_web_hooks(project, topic, msg): ''' Sends the web-hook notification. ''' - log.info("Processing project %s - sending: %s" % ( - project.fullname, topic) - ) - log.debug('msg: %s' % msg) + log.info( + "Processing project: %s - topic: %s", project.fullname, topic) + log.debug('msg: %s', msg) # Send web-hooks notification global _i @@ -110,7 +109,9 @@ def handle_client(): # Inside a while loop, wait for incoming events. while True: reply = yield trollius.From(subscriber.next_published()) - print(u'Received: ', repr(reply.value), u'on channel', reply.channel) + log.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) data = json.loads(reply.value) username = None if '/' in data['project']: @@ -119,6 +120,7 @@ def handle_client(): projectname = data['project'] project = pagure.lib.get_project( session=pagure.SESSION, name=projectname, user=username) + log.info('Got the project, going to the webhooks') call_web_hooks(project, data['topic'], data['msg']) From 06af7f90057561db8d9f36532258407833b5fdca Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 10:38:55 +0000 Subject: [PATCH 23/26] Send information to redis for every message we send to fedmsg This will allow web-hook-based notifications via its own dedicated web-hook service if it is configured. --- diff --git a/pagure/lib/notify.py b/pagure/lib/notify.py index c6a959d..d1ed48d 100644 --- a/pagure/lib/notify.py +++ b/pagure/lib/notify.py @@ -51,6 +51,15 @@ def log(project, topic, msg, redis=None): # Send fedmsg notification (if fedmsg is there and set-up) fedmsg_publish(topic, msg) + if redis: + redis.publish( + 'hook', + json.dumps({ + 'project': project.fullname, + 'topic': topic, + 'msg': msg, + })) + def _clean_emails(emails, user): ''' Remove the email of the user doing the action if it is in the list. From 7d3eae2d96f2abd4dd0dbf8e0c0c12daf85b2cb2 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 11:02:37 +0000 Subject: [PATCH 24/26] Fix a typo and be more precise about optional packages in the documentation --- diff --git a/doc/install.rst b/doc/install.rst index 1765740..084387e 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -41,7 +41,8 @@ The ``pagure-webhook`` package contains the web-hook server. .. note:: The last three packages are optional, pagure would work fine without - them. + them but the live-update, the webhook and the comment by email + services will not work. * From the sources diff --git a/doc/install_webhooks.rst b/doc/install_webhooks.rst index a532941..2aec5e6 100644 --- a/doc/install_webhooks.rst +++ b/doc/install_webhooks.rst @@ -36,7 +36,7 @@ Configure your system The first file is the script of the web-hook server itself. -The secondg file is the systemd service file. +The second file is the systemd service file. * Activate the service and ensure it's started upon boot: From 8e41fbafe2bea9c6e0489c3ca150597a54471e85 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 11:32:04 +0000 Subject: [PATCH 25/26] Namespace the messages sent over redis --- diff --git a/ev-server/pagure-stream-server.py b/ev-server/pagure-stream-server.py index d636f1b..e6911f9 100644 --- a/ev-server/pagure-stream-server.py +++ b/ev-server/pagure-stream-server.py @@ -149,7 +149,7 @@ def handle_client(client_reader, client_writer): subscriber = yield trollius.From(connection.start_subscribe()) # Subscribe to channel. - yield trollius.From(subscriber.subscribe([obj.uid])) + yield trollius.From(subscriber.subscribe(['pagure.%s' % obj.uid])) # Inside a while loop, wait for incoming events. while True: diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index ec1b81b..4d20db1 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -232,12 +232,12 @@ def add_issue_comment(session, issue, comment, user, ticketfolder, if REDIS: if issue.private: - REDIS.publish(issue.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue.uid, json.dumps({ 'issue': 'private', 'comment_id': issue_comment.id, })) else: - REDIS.publish(issue.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue.uid, json.dumps({ 'comment_id': issue_comment.id, 'comment_added': text2markdown(issue_comment.comment), 'comment_user': issue_comment.user.user, @@ -308,7 +308,8 @@ def add_tag_obj(session, obj, tags, user, ticketfolder): # Send notification for the event-source server if REDIS: - REDIS.publish(obj.uid, json.dumps({'added_tags': added_tags})) + REDIS.publish('pagure.%s' % obj.uid, json.dumps( + {'added_tags': added_tags})) if added_tags: return 'Tag added: %s' % ', '.join(added_tags) @@ -342,7 +343,8 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder): # Send notification for the event-source server if REDIS: - REDIS.publish(issue.uid, json.dumps({'unassigned': '-'})) + REDIS.publish('pagure.%s' % issue.uid, json.dumps( + {'unassigned': '-'})) return 'Assignee reset' elif assignee is None and issue.assignee is None: @@ -375,7 +377,7 @@ def add_issue_assignee(session, issue, assignee, user, ticketfolder): # Send notification for the event-source server if REDIS: - REDIS.publish(issue.uid, json.dumps( + REDIS.publish('pagure.%s' % issue.uid, json.dumps( {'assigned': assignee_obj.to_json(public=True)})) return 'Issue assigned' @@ -482,12 +484,12 @@ def add_issue_dependency( # Send notification for the event-source server if REDIS: - REDIS.publish(issue.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue.uid, json.dumps({ 'added_dependency': issue_blocked.id, 'issue_uid': issue.uid, 'type': 'children', })) - REDIS.publish(issue_blocked.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue_blocked.uid, json.dumps({ 'added_dependency': issue.id, 'issue_uid': issue_blocked.uid, 'type': 'parent', @@ -542,12 +544,12 @@ def remove_issue_dependency( # Send notification for the event-source server if REDIS: - REDIS.publish(issue.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue.uid, json.dumps({ 'removed_dependency': child_del, 'issue_uid': issue.uid, 'type': 'children', })) - REDIS.publish(issue_blocked.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue_blocked.uid, json.dumps({ 'removed_dependency': issue.id, 'issue_uid': issue_blocked.uid, 'type': 'parent', @@ -629,7 +631,7 @@ def remove_tags_obj( # Send notification for the event-source server if REDIS: - REDIS.publish(obj.uid, json.dumps( + REDIS.publish('pagure.%s' % obj.uid, json.dumps( {'removed_tags': removed_tags})) return 'Removed tag: %s' % ', '.join(removed_tags) @@ -808,7 +810,7 @@ def add_pull_request_comment(session, request, commit, filename, row, # Send notification for the event-source server if REDIS: - REDIS.publish(request.uid, json.dumps({ + REDIS.publish('pagure.%s' % request.uid, json.dumps({ 'request_id': len(request.comments), 'comment_added': text2markdown(pr_comment.comment), 'comment_user': pr_comment.user.user, @@ -1118,12 +1120,12 @@ def edit_issue(session, issue, ticketfolder, user, if REDIS and edit: if issue.private: - REDIS.publish(issue.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue.uid, json.dumps({ 'issue': 'private', 'fields': edit, })) else: - REDIS.publish(issue.uid, json.dumps({ + REDIS.publish('pagure.%s' % issue.uid, json.dumps({ 'fields': edit, 'issue': issue.to_json(public=True, with_comments=False), })) diff --git a/pagure/lib/notify.py b/pagure/lib/notify.py index d1ed48d..46d4233 100644 --- a/pagure/lib/notify.py +++ b/pagure/lib/notify.py @@ -53,7 +53,7 @@ def log(project, topic, msg, redis=None): if redis: redis.publish( - 'hook', + 'pagure.hook', json.dumps({ 'project': project.fullname, 'topic': topic, diff --git a/webhook-server/pagure-webhook-server.py b/webhook-server/pagure-webhook-server.py index 16a35d3..1b4799d 100644 --- a/webhook-server/pagure-webhook-server.py +++ b/webhook-server/pagure-webhook-server.py @@ -104,7 +104,7 @@ def handle_client(): subscriber = yield trollius.From(connection.start_subscribe()) # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['hook'])) + yield trollius.From(subscriber.subscribe(['pagure.hook'])) # Inside a while loop, wait for incoming events. while True: From af4c2032a0213e193db24961aa243195185085f7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 30 2015 11:51:45 +0000 Subject: [PATCH 26/26] Rename handle_client as handle_messages as per puiterwijk's request --- diff --git a/webhook-server/pagure-webhook-server.py b/webhook-server/pagure-webhook-server.py index 1b4799d..5118c0a 100644 --- a/webhook-server/pagure-webhook-server.py +++ b/webhook-server/pagure-webhook-server.py @@ -96,7 +96,7 @@ def call_web_hooks(project, topic, msg): @trollius.coroutine -def handle_client(): +def handle_messages(): connection = yield trollius.From(trollius_redis.Connection.create( host='0.0.0.0', port=6379, db=0)) @@ -129,7 +129,7 @@ def main(): try: loop = trollius.get_event_loop() tasks = [ - trollius.async(handle_client()), + trollius.async(handle_messages()), ] loop.run_until_complete(trollius.wait(tasks)) loop.run_forever()