From fa5ca970b2566a0ced96c7dd89351e845b5ef1d0 Mon Sep 17 00:00:00 2001 From: farhaanbukhsh Date: Jul 26 2016 07:41:45 +0000 Subject: Integration PMCI --- diff --git a/pagure/consumer.py b/pagure/consumer.py new file mode 100644 index 0000000..b01496b --- /dev/null +++ b/pagure/consumer.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import fedmsg.consumers +from pagure.hooks import jenkins_hook +from pagure.lib import pagure_ci +from pagure.lib.model import BASE, Project, User +PAGURE_MAIN_REPO = '{base}{name}.git' +PAGURE_FORK_REPO = '{base}forks/{user}/{name}.git' + + +class Integrator(fedmsg.consumers.FedmsgConsumer): + topic = [ + 'io.pagure.prod.pagure.pull-request.comment.added', + 'org.fedoraproject.dev.pagure.pull-request.new', + 'org.fedoraproject.dev.pagure.pull-request.comment.added', + 'io.pagure.prod.pagure.pull-request.new', + 'org.fedoraproject.prod.jenkins.build', + ] + + config_key = 'integrator.enabled' + + def __init__(self, hub): + super(Integrator, self).__init__(hub) + pagure_ci.connect_db() + + def consume(self, msg): + topic, msg = msg['topic'], msg['body'] + self.log.info("Received %r, %r", topic, msg.get('msg_id', None)) + msg = msg['msg'] + try: + if topic.endswith('.pull-request.comment.added'): + if is_rebase(msg): + self.trigger_build(msg) + elif topic.endswith('.pull-request.new'): + self.trigger_build(msg) + else: + self.process_build(msg) + except jenkins_hook.ConfigNotFound as exc: + self.log.info('Unconfigured project %r', str(exc)) + + def trigger_build(self, msg): + pr_id = msg['pullrequest']['id'] + project = msg['pullrequest']['project']['name'] + branch = msg['pullrequest']['branch_from'] + print project, jenkins_hook.Service.PAGURE + for cfg in jenkins_hook.get_configs(project, jenkins_hook.Service.PAGURE): + repo = msg['pullrequest'].get('remote_git') or get_repo(cfg, msg) + + self.log.info("Trigger on %s PR #%s from %s: %s", + project, pr_id, repo, branch) + + pagure_ci.process_pr(self.log, cfg, pr_id, repo, branch) + + def process_build(self, msg): + for cfg in jenkins_hook.get_configs(msg['project'], jenkins_hook.Service.JENKINS): + pagure_ci.process_build(self.log, cfg, msg['build']) + + +def get_repo(cfg, msg): + url = PAGURE_MAIN_REPO + if msg['pullrequest']['repo_from']['parent']: + url = PAGURE_FORK_REPO + return url.format( + base=cfg.pagure_url, + user=msg['pullrequest']['repo_from']['user']['name'], + name=msg['pullrequest']['repo_from']['name']) + + +def is_rebase(msg): + if msg['pullrequest']['status'] != 'Open': + return False + try: + return msg['pullrequest']['comments'][-1]['notification'] + except (IndexError, KeyError): + return False diff --git a/pagure/hooks/jenkins_hook.py b/pagure/hooks/jenkins_hook.py index 1305078..902c87d 100644 --- a/pagure/hooks/jenkins_hook.py +++ b/pagure/hooks/jenkins_hook.py @@ -12,34 +12,32 @@ import os import sqlalchemy as sa import pygit2 -from wtforms import validators, TextField +from wtforms import validators, TextField, BooleanField from flask.ext import wtf from sqlalchemy.orm import relation from sqlalchemy.orm import backref +from sqlalchemy.ext.declarative import declarative_base from pagure.hooks import BaseHook, RequiredIf -from pagure.lib.model import BASE, Project +from pagure.lib.model import BASE, Project, User from pagure import get_repo_path - class PagureCI(BASE): + __tablename__ = 'hook_pagure_ci' - __table_args__ = {'extend_existing': True} id = sa.Column(sa.Integer, primary_key=True) project_id = sa.Column( - sa.Integer, + sa.Integer, sa.ForeignKey('projects.id', onupdate='CASCADE'), nullable=False, - unique=True, + unique=False, index=True) + active = sa.Column(sa.Boolean, nullable=False, default=False) - name = sa.Column(sa.String(64), primary_key=True, unique=True) - display_name = sa.Column(sa.String(64), nullable=False, default='Jenkins') - owner = sa.Column(sa.String(64)) - + name = sa.Column(sa.String(64)) pagure_name = sa.Column(sa.String(255)) pagure_url = sa.Column(sa.String(255)) pagure_token = sa.Column(sa.String(64)) @@ -47,19 +45,67 @@ class PagureCI(BASE): jenkins_name = sa.Column(sa.String(255)) jenkins_url = sa.Column(sa.String(255)) jenkins_token = sa.Column(sa.String(64)) - - hook_token = sa.Column(sa.String(64)) + hook_token = sa.Column(sa.String(64), + nullable=True, + unique=True, + index=True) project = relation( - 'Project', remote_side=[Project.id], + 'Project', + foreign_keys=[project_id], + remote_side=[Project.id], backref=backref( - 'jenkins_hook', cascade="delete, delete-orphan", + 'hook_pagure_ci', cascade="delete, delete-orphan", single_parent=True) ) + def __init__(self, name = None, display_name = None, owner = None, + pagure_name = None, pagure_url = None, pagure_token = None, + jenkins_name = None, jenkins_url = None, jenkins_token = None, + hook_token = None, active = False): + self.name = name + self.display_name = display_name + self.owner = owner + self.pagure_name = pagure_name + self.pagure_url = pagure_url + self.pagure_token = pagure_token + + self.jenkins_name = jenkins_name + self.jenkins_url = jenkins_url + self.jenkins_token = jenkins_token + + self.hook_token = hook_token + self.active = active + + def __repr__(self): + return ''.format(self) + +def init_db(db): + from sqlalchemy import create_engine + engine = create_engine(db, convert_unicode=True) + BASE.metadata.create_all(bind=engine) + +class ConfigNotFound(Exception): + pass + + +class Service(object): + PAGURE = PagureCI.pagure_name + JENKINS = PagureCI.jenkins_name + + +def get_configs(project_name, service): + """Returns all configurations with given name on a service. + + :raises ConfigNotFound: when no configuration matches + """ + cfg = BASE.query(PagureCI).filter(service == project_name).all() + if len(cfg) == 0: + raise ConfigNotFound(project_name) + return cfg class JenkinsForm(wtf.Form): - + '''Form to configure Jenkins hook''' name = TextField('Name', [validators.Required(), @@ -86,7 +132,8 @@ class JenkinsForm(wtf.Form): validators.Length(max=255)], default='http://jenkins.fedorainfracloud.org/') jenkins_token = TextField('Jenkins token', - [validators.Required()]) + [validators.Required()]) + active = BooleanField('Active',[validators.Optional()]) class Hook(BaseHook): @@ -97,10 +144,10 @@ class Hook(BaseHook): ' the changes made by the pushes to the git repository.' form = JenkinsForm db_object = PagureCI - backref = 'jenkins_hook' + backref = 'pagure_ci_hook' form_fields = [ - 'name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', - 'jenkins_url', 'jenkins_token' + 'display_name','name', 'pagure_name', 'pagure_url', 'pagure_token', 'jenkins_name', + 'jenkins_url', 'jenkins_token','active' ] @classmethod @@ -125,7 +172,7 @@ class Hook(BaseHook): if not os.path.exists(hook_file): os.symlink( hook_file, - os.path.join(repopath, 'hooks', 'post-receive.irc') + os.path.join(repopath, 'hooks', 'jenkins_hook.py') ) @classmethod diff --git a/pagure/lib/pagure_ci.py b/pagure/lib/pagure_ci.py new file mode 100644 index 0000000..b6e2723 --- /dev/null +++ b/pagure/lib/pagure_ci.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +import os +import flask +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy import create_engine +from pagure.hooks import jenkins_hook +from pagure.lib import model + +import json +import logging + +import requests +import jenkins + +os.environ.setdefault('INTEGRATOR_SETTINGS', '/etc/poormanci.conf') + +app = flask.Flask(__name__) +app.config.from_object('pagure.default_config') +app.config.from_envvar('INTEGRATOR_SETTINGS', silent=True) +app.logger.setLevel(logging.INFO) + +PAGURE_URL = '{base}api/0/{repo}/pull-request/{pr}/flag' +JENKINS_TRIGGER_URL = '{base}job/{project}/buildWithParameters' + + +db_session = None + +def connect_db(): + global db_session + engine = create_engine(app.config['DB_URL'], convert_unicode=True) + db_session = scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=engine)) + model.BASE.query = db_session.query_property() + + + +def process_pr(logger, cfg, pr_id, repo, branch): + post_data(logger, + JENKINS_TRIGGER_URL.format(base=cfg.jenkins_url, project=cfg.jenkins_name), + {'token': cfg.jenkins_token, + 'cause': pr_id, + 'REPO': repo, + 'BRANCH': branch}) + + +def process_build(logger, cfg, build_id): + # Get details from Jenkins + jenk = jenkins.Jenkins(cfg.jenkins_url) + build_info = jenk.get_build_info(cfg.jenkins_name, build_id) + result = build_info['result'] + url = build_info['url'] + + pr_id = None + + for action in build_info['actions']: + for cause in action.get('causes', []): + try: + pr_id = int(cause['note']) + except (KeyError, ValueError): + continue + + if not pr_id: + logger.info('Not a PR check') + return + + # Comment in Pagure + logger.info('Updating %s PR %d: %s', cfg.pagure_name, pr_id, result) + try: + post_flag(logger, cfg.display_name, cfg.pagure_url, cfg.pagure_token, + cfg.pagure_name, pr_id, result, url) + except KeyError as exc: + logger.warning('Unknown build status', exc_info=exc) + + +def post_flag(logger, name, base, token, repo, pr, result, url): + comment, percent = { + 'SUCCESS': ('Build successful', 100), + 'FAILURE': ('Build failed', 0), + }[result] + payload = { + 'username': name, + 'percent': percent, + 'comment': comment, + 'url': url, + } + post_data(logger, PAGURE_URL.format(base=base, repo=repo, pr=pr), payload, + headers={'Authorization': 'token ' + token}) + + +def post_data(logger, *args, **kwargs): + resp = requests.post(*args, **kwargs) + logger.debug('Received response status %s', resp.status_code) + if resp.status_code < 200 or resp.status_code >= 300: + logger.error('Network request failed: %d: %s', resp.status_code, resp.text) + + +@app.route('/hooks//build-finished', methods=['POST']) +def hook_finished(token): + try: + data = json.loads(flask.request.get_data()) + cfg = jenkins_hook.get_configs(data['name'], jenkins_hook.Service.JENKINS)[0] + build_id = data['build']['number'] + if token != cfg.hook_token: + raise ValueError('Token mismatch') + except (TypeError, ValueError, KeyError, models.ConfigNotFound) as exc: + app.logger.error('Error processing jenkins notification', exc_info=exc) + return ('Bad request...\n', 400, {'Content-Type': 'text/plain'}) + app.logger.info('Received jenkins notification') + process_build(app.logger, cfg, build_id) + return ('', 204) + + +@app.errorhandler(403) +def forbidden(e): + return flask.render_template('forbidden.html'), 403 + + +def cleanup_url(url): + """Make sure there is trailing slash.""" + return url.rstrip('/') + '/' + + +@app.before_request +def before_request(): + if db_session is None: + connect_db() + + +@app.teardown_appcontext +def shutdown_session(exception=None): + db_session.remove() diff --git a/pagure/ui/plugins.py b/pagure/ui/plugins.py index 88c9c8a..c7f5a17 100644 --- a/pagure/ui/plugins.py +++ b/pagure/ui/plugins.py @@ -9,6 +9,7 @@ """ import flask +import uuid from sqlalchemy.exc import SQLAlchemyError from straight.plugin import load @@ -94,12 +95,18 @@ def view_plugin(repo, plugin, username=None, full=True): else: dbobj = plugin.db_object() + print dir(dbobj) + + form = plugin.form(obj=dbobj) for field in plugin.form_fields: fields.append(getattr(form, field)) - + print "validate:", form.validate_on_submit(), dir(form), form.errors if form.validate_on_submit(): form.populate_obj(obj=dbobj) + if dbobj.__tablename__ == 'hook_pagure_ci': + dbobj.hook_token = uuid.uuid4().hex + if new: dbobj.project_id = repo.id SESSION.add(dbobj) @@ -121,6 +128,7 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + dbobj=dbobj, fields=fields) if form.active.data: @@ -154,4 +162,5 @@ def view_plugin(repo, plugin, username=None, full=True): username=username, plugin=plugin, form=form, + dbobj=dbobj, fields=fields)