PR#5511: [WIP] generic webhook like pagure ci plugin, reference implementation together with AWS CodePipeline - pagure

		`@@ -0,0 +1,245 @@`
		`+ # -- coding: utf-8 --`
		`+`
		`+ # SPDX-FileCopyrightText: 2024 Dominik Wombacher <dominik@wombacher.cc>`
		`+ #`
		`+ # SPDX-License-Identifier: GPL-2.0-or-later`
		`+`
		`+ from flask import request, g`
		`+ from cryptography.hazmat.primitives import constant_time`
		`+ from kitchen.text.converters import to_bytes`
		`+ import logging`
		`+ import json`
		`+`
		`+ import pagure.lib.query`
		`+ from pagure.config import config as pagure_config`
		`+ from pagure.exceptions import PagureException, APIError`
		`+ from pagure.lib import model, plugins`
		`+ from pagure.lib.query import get_authorized_project`
		`+ from pagure.api import API, APIERROR, api_method`
		`+ from pagure.api.ci import BUILD_STATS`
		`+`
		`+ _log = logging.getLogger(__name__)`
		`+`
		`+`
		`+ @API.route(`
		`+ "/ci/generic/<repo>", methods=["POST"]`
		`+ )`
		`+ @API.route(`
		`+ "/ci/generic/<namespace>/<repo>", methods=["POST"],`
		`+ )`
		`+ @API.route(`
		`+ "/ci/generic/forks/<username>/<repo>", methods=["POST"],`
		`+ )`
		`+ @API.route(`
		`+ "/ci/generic/forks/<username>/<namespace>/<repo>", methods=["POST"],`
		`+ )`
		`+ @api_method`
		`+ def generic_ci_notification(repo, username=None, namespace=None):`
		`+ """`
		`+ Example incoming Json payload:`
		`+ {`
		`+ "ci_token": <required>,`
		`+ "pull_request_uid": <required>,`
		`+ "build_status": <required>,`
		`+ "comment_display_name": <optional>,`
		`+ "build_url": <optional>,`
		`+ "build_id_external": <optional>,`
		`+ }`
		`+`
		`+ Expected values:`
		`+ ci_token = Has to match the token in the Project Pagure CI settings`
		`+ pull_request_uid = The same UID that was part of the outgoing webhook to identify the Pull Request`
		`+ build_status = The current status of the update`
		`+ comment_display_name = Name that shows up in the Pull Request comment, defaults to 'pagure ci - generic'`
		`+ build_url = URL where build logs, status, history or dashboard can be accessed, linked from the Pull Request comment`
		`+ build_id_external = Unique ID of the build, shows up as additional information in the Pull Request comment`
		`+`
		`+ Value restrictions:`
		`+ build_status = SUCCESS \| FAILURE \| ABORTED \| BUILDING`
		`+ """`
		`+ if request.is_json is False:`
		`+ _log.debug("Bad Request: No JSON payload received.")`
		`+ raise APIError(400, error_code=APIERROR.EINVALIDREQ)`
		`+`
		`+ data = request.json`
		`+ ci_token = data.get("ci_token")`
		`+ pull_request_uid = data.get("pull_request_uid")`
		`+ build_status = data.get("build_status")`
		`+ comment_display_name = data.get("comment_display_name")`
		`+ build_url = data.get("build_url")`
		`+ build_id_external = data.get("build_id_external")`
		`+`
		`+ if ci_token is None or ci_token == "":`
		`+ _log.debug("Bad Request: 'ci_token' not provided.")`
		`+ raise APIError(400, error_code=APIERROR.EINVALIDREQ)`
		`+`
		`+ if pull_request_uid is None or pull_request_uid == "":`
		`+ _log.debug("Bad Request: 'pull_request_uid' not provided.")`
		`+ raise APIError(400, error_code=APIERROR.EINVALIDREQ)`
		`+`
		`+ if build_status is None or build_status == "":`
		`+ _log.debug("Bad Request: 'build_status' not provided.")`
		`+ raise APIError(400, error_code=APIERROR.EINVALIDREQ)`
		`+`
		`+ project = get_authorized_project(`
		`+ g.session, repo, user=username, namespace=namespace`
		`+ )`
		`+ if not project:`
		`+ _log.debug(f"Not Found: Project / Repo %s doesn't exist." % repo)`
		`+ raise APIError(404, error_code=APIERROR.ENOPROJECT)`
		`+`
		`+ ci_hook = plugins.get_plugin("Pagure CI")`
		`+ ci_hook.db_object()`
		`+ if not constant_time.bytes_eq(`
		`+ to_bytes(ci_token), to_bytes(project.ci_hook.pagure_ci_token)`
		`+ ):`
		`+ _log.debug(f"Access Denied: The provided ci token is invalid for project %s." % project.fullname)`
		`+ raise APIError(401, error_code=APIERROR.EINVALIDTOK)`
		`+`
		`+ if build_status not in BUILD_STATS:`
		`+ _log.debug(f"Bad Request: Invalid build status retrieved: %s" % build_status)`
		`+ raise APIError(400, error_code=APIERROR.EINVALIDREQ)`
		`+`
		`+ pull_request = pagure.lib.query.get_request_by_uid(`
		`+ g.session, pull_request_uid`
		`+ )`
		`+ if not pull_request:`
		`+ _log.debug(f"Not Found: Pull Request with UID %s doesn't exist." % pull_request_uid)`
		`+ raise APIError(404, error_code=APIERROR.ENOREQ)`
		`+`
		`+ # Usage in https://pagure.io/pagure/blob/master/f/pagure/lib/query.py#_1451`
		`+ # makes 'pull_request.commit_stop' mandatory, otherwise 'pagure.lib.query.add_pull_request_flag'`
		`+ # fails down the road with:`
		`+ # "sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) NOT NULL constraint failed: commit_flags.commit_hash"`
		`+ if not pull_request.commit_stop:`
		`+ _log.debug(f"Internal Server Error: Value for 'commit_stop' mandatory but not found in Pull Request with UID %s." % pull_request_uid)`
		`+ # Nothing a user can do to recover, treat it as fatal error`
		`+ raise APIError(500, error_code=APIERROR.ENOCOMMIT)`
		`+`
		`+ # build_url is mandatory: https://pagure.io/pagure/blob/master/f/pagure/lib/model.py#_2503`
		`+ # Fallback to link to the PR if not provided to allow use-cases were no url is available`
		`+ if build_url is None or build_url == "":`
		`+ build_url = pull_request.full_url`
		`+`
		`+ comment, state, percent = BUILD_STATS[build_status]`
		`+`
		`+ # build_id_external is optional, if not provided fallback to 'N/A'`
		`+ if build_id_external is None or build_id_external == "":`
		`+ build_id_external = ("N/A"`
		`+ "")`
		`+ comment = comment % build_id_external`
		`+`
		`+ if comment_display_name is None or comment_display_name == "":`
		`+ comment_display_name = "pagure ci - %s" % project.ci_hook.ci_type`
		`+`
		`+ if pull_request.commit_stop:`
		`+ comment += " (commit: %s)" % (pull_request.commit_stop[:8])`
		`+`
		`+ flag_uid = None`
		`+ for flag in pull_request.flags:`
		`+ if (`
		`+ flag.status == pagure_config["FLAG_PENDING"]`
		`+ and flag.username == comment_display_name`
		`+ ):`
		`+ flag_uid = flag.uid`
		`+ break`
		`+`
		`+ _log.debug("Flag's UID: %s", flag_uid)`
		`+`
		`+ pagure.lib.query.add_pull_request_flag(`
		`+ g.session,`
		`+ request=pull_request,`
		`+ username=comment_display_name,`
		`+ percent=percent,`
		`+ comment=comment,`
		`+ url=build_url,`
		`+ status=state,`
		`+ uid=flag_uid,`
		`+ user=project.user.username,`
		`+ token=None,`
		`+ )`
		`+ g.session.commit()`
		`+`
		`+ ret = {`
		`+ "pull_request_uid": pull_request.uid,`
		`+ "build_status": build_status,`
		`+ "result": "Status update successful."`
		`+ }`
		`+`
		`+ return ret, 200`
		`+`
		`+`
		`+ def trigger_build(`
		`+ project: model.Project,`
		`+ ci_url: str,`
		`+ ci_job: str,`
		`+ ci_token: str,`
		`+ branch: str,`
		`+ branch_to: str,`
		`+ is_pull_request: bool = False,`
		`+ is_commit: bool = False,`
		`+ commit_hash: str = None,`
		`+ pull_request: model.PullRequest = None,`
		`+ ci_username = None,`
		`+ ci_password = None,`
		`+ ) -> None:`
		`+`
		`+ _log.info("pagure ci - generic - trigger_build")`
		`+`
		`+ if is_commit and is_pull_request:`
		`+ raise PagureException("pagure ci - generic - Invalid: Event Type can't be commit AND pull request")`
		`+`
		`+ if is_commit and commit_hash is None or commit_hash == "":`
		`+ raise PagureException("pagure ci - generic - Invalid: Commit needs to contain a commit hash")`
		`+`
		`+ if is_pull_request and pull_request is None:`
		`+ raise PagureException("pagure ci - generic - Invalid: Pull Request needs to contain a model.PullRequest object")`
		`+`
		`+`
		`+ def _outgoing_payload(`
		`+ project: model.Project,`
		`+ branch: str,`
		`+ branch_to: str,`
		`+ is_pull_request: bool = False,`
		`+ is_commit: bool = False,`
		`+ commit_hash: str = None,`
		`+ pull_request: model.PullRequest = None`
		`+ ) -> json:`
		`+`
		`+ if project.is_fork:`
		`+ fork_name = project.name`
		`+ fork_url = project.full_url`
		`+ repo_name = project.parent.name`
		`+ repo_url = project.parent.full_url`
		`+ else:`
		`+ repo_name = project.name`
		`+ repo_url = project.full_url`
		`+ fork_name = None`
		`+ fork_url = None`
		`+`
		`+ if pull_request:`
		`+ pull_request_url = pull_request.full_url`
		`+ pull_request_id = pull_request.id`
		`+ pull_request_uid = pull_request.uid`
		`+ else:`
		`+ pull_request_url = None`
		`+ pull_request_id = None`
		`+ pull_request_uid = None`
		`+`
		`+ payload = {`
		`+ "is_pull_request": is_pull_request,`
		`+ "is_commit": is_commit,`
		`+ "pull_request_id": pull_request_id,`
		`+ "pull_request_uid": pull_request_uid,`
		`+ "pull_request_url": pull_request_url,`
		`+ "commit_hash": commit_hash,`
		`+ "repo_name": repo_name,`
		`+ "repo_url": repo_url,`
		`+ "fork_name": fork_name,`
		`+ "fork_url": fork_url,`
		`+ "branch": branch,`
		`+ "branch_to": branch_to,`
		`+ "response_url": f"%s/api/0/ci/generic/%s" % (pagure_config["APP_URL"][:-1], project.url_path),`
		`+ }`
		`+`
		`+ return json.dumps(payload)`
		`\ No newline at end of file`

pagure/flask_app.py

file modified

+10

		`@@ -11,6 +11,8 @@`
		`from __future__ import absolute_import, unicode_literals`

		`import datetime`
		`+ import importlib`
		`+`
		`import gc`
		`import logging`
		`import os`
		`@@ -135,6 +137,14 @@`

		`from pagure.api import API # noqa: E402`

		`+ # Load all configured pagure ci plugins once to initiate routes`
		`+ # Required before Blueprint registration`
		`+ ci_plugins = {}`
		`+ for ci_type in pagure_config["PAGURE_CI_SERVICES"]:`
		`+ ci_plugins[ci_type] = importlib.import_module(`
		`+ "pagure.api.ci." + ci_type`
		`+ )`
		`+`
		`app.register_blueprint(API)`

		`from pagure.ui import UI_NS # noqa: E402`

pagure/hooks/pagure_ci.py

file modified

+2 -2

		`@@ -121,7 +121,7 @@`
		`[wtforms.validators.Optional(), wtforms.validators.Length(max=255)],`
		`)`

		`- ci_password = wtforms.StringField(`
		`+ ci_password = wtforms.PasswordField(`
		`"Password to authenticate with if needed",`
		`[wtforms.validators.Optional(), wtforms.validators.Length(max=255)],`
		`)`
		`@@ -174,7 +174,7 @@`
		`form = PagureCiForm`
		`db_object = PagureCITable`
		`backref = "ci_hook"`
		`- form_fields = ["ci_type", "ci_url", "ci_job", "active_commit", "active_pr"]`
		`+ form_fields = ["ci_type", "ci_url", "ci_job", "ci_username", "ci_password", "active_commit", "active_pr"]`
		`runner = PagureCIRunner`

		`@classmethod`

pagure/lib/tasks_services.py

file modified

+19 -5

		`@@ -430,6 +430,16 @@`
		`else:`
		`project_name = pr.project_from.fullname`

		`+ is_pull_request = True`
		`+ pull_request = pr`
		`+ is_commit = False`
		`+ commit_hash = None`
		`+ else:`
		`+ is_commit = True`
		`+ commit_hash = cause`
		`+ is_pull_request = False`
		`+ pull_request = None`
		`+`
		`user, namespace, project_name = split_project_fullname(project_name)`

		`_log.info("Pagure-CI: Looking for project: %s", project_name)`
		`@@ -478,16 +488,20 @@`
		`ci_project = project.parent`
		`ci = importlib.import_module(f"pagure.api.ci.{ci_type}")`
		`ci.trigger_build(`
		`- project_path=project.path,`
		`- url=ci_project.ci_hook.ci_url,`
		`- job=ci_project.ci_hook.ci_job,`
		`- token=ci_project.ci_hook.pagure_ci_token,`
		`+ project=project,`
		`+ ci_url=ci_project.ci_hook.ci_url,`
		`+ ci_job=ci_project.ci_hook.ci_job,`
		`+ ci_token=ci_project.ci_hook.pagure_ci_token,`
		`branch=branch,`
		`branch_to=branch_to,`
		`- cause=cause,`
		`+ is_pull_request=is_pull_request,`
		`+ is_commit=is_commit,`
		`+ commit_hash=commit_hash,`
		`+ pull_request=pull_request,`
		`ci_username=ci_project.ci_hook.ci_username,`
		`ci_password=ci_project.ci_hook.ci_password,`
		`)`
		`+`
		`except Exception as e:`
		`_log.error(`
		`f"Pagure-CI: Un-supported CI type {ci_type}. "`

requirements.txt

file modified

		`@@ -19,6 +19,7 @@`
		`flask-wtf <= 1.2.1`
		`kitchen == 1.2.6`
		`markdown <= 3.5.2`
		`+ mock <= 4.0.3`
		`munch <= 2.5.0`
		`Pillow <= 10.3.0`
		`psutil <= 5.9.8`

tests/test_pagure_api_ci_generic.py

file added

+750

		`@@ -0,0 +1,750 @@`
		`+ # -- coding: utf-8 --`
		`+`
		`+ # SPDX-FileCopyrightText: 2024 Dominik Wombacher <dominik@wombacher.cc>`
		`+ #`
		`+ # SPDX-License-Identifier: GPL-2.0-or-later`
		`+`
		`+ import json`
		`+ import os`
		`+`
		`+ import sys`
		`+ from mock import patch, Mock, ANY`
		`+`
		`+ sys.path.insert(`
		`+ 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")`
		`+ )`
		`+`
		`+ from pagure.config import config as pagure_config`
		`+ from pagure.lib import model`
		`+ import pagure.api.ci.generic`
		`+ import pagure.lib.tasks_services`
		`+ import pagure.lib.query`
		`+ import tests`
		`+ from pagure.api.ci import BUILD_STATS`
		`+`
		`+ from pagure.lib.query import get_authorized_project, get_request_by_uid`
		`+ _unpatched_get_authorized_project = get_authorized_project`
		`+ _unpatched_get_request_by_uid = get_request_by_uid`
		`+`
		`+`
		`+ class PagureApiCiGenerictests(tests.Modeltests):`
		`+ """Tests for pagure.lib.task_services"""`
		`+`
		`+ maxDiff = None`
		`+`
		`+`
		`+ def setUp(self):`
		`+ """Set up the environnment, ran before every tests."""`
		`+ super(PagureApiCiGenerictests, self).setUp()`
		`+`
		`+ pagure.config.config["REQUESTS_FOLDER"] = None`
		`+ self.sshkeydir = os.path.join(self.path, "sshkeys")`
		`+ pagure.config.config["MIRROR_SSHKEYS_FOLDER"] = self.sshkeydir`
		`+`
		`+ tests.create_projects(self.session)`
		`+`
		`+ # Use of '_unpatched_get_authorized_project' because`
		`+ # 'pagure.lib.query.get_authorized_project' is mocked for some tests`
		`+ project = _unpatched_get_authorized_project(self.session, "test")`
		`+`
		`+ # Install the plugin at the DB level`
		`+ plugin = pagure.lib.plugins.get_plugin("Pagure CI")`
		`+ dbobj = plugin.db_object()`
		`+ dbobj.ci_url = "https://ci.example.com/"`
		`+ dbobj.ci_job = "pagure"`
		`+ dbobj.pagure_ci_token = "random_token"`
		`+ dbobj.project_id = project.id`
		`+ dbobj.ci_type = "generic"`
		`+ self.session.add(dbobj)`
		`+ self.session.commit()`
		`+`
		`+ # Create a fork of test for foo`
		`+ item = pagure.lib.model.Project(`
		`+ user_id=2, # foo`
		`+ name="test",`
		`+ is_fork=True,`
		`+ parent_id=1,`
		`+ description="test project #1",`
		`+ hook_token="aaabbbccc_foo",`
		`+ )`
		`+ item.close_status = [`
		`+ "Invalid",`
		`+ "Insufficient data",`
		`+ "Fixed",`
		`+ "Duplicate",`
		`+ ]`
		`+ self.session.add(item)`
		`+ self.session.commit()`
		`+`
		`+ # Create Pull Request in test project`
		`+ pr_test = pagure.lib.model.PullRequest(`
		`+ id=1,`
		`+ uid="720a0568c1274e74966e54b433b2003e",`
		`+ title="pr1",`
		`+ project_id=item.id,`
		`+ project_id_from=project.id,`
		`+ branch="main",`
		`+ branch_from="feature",`
		`+ user_id=project.user_id,`
		`+ commit_start="96df1145c3466fb33edbbde327c6f8705627d2eb",`
		`+ commit_stop="96df1145c3466fb33edbbde327c6f8705627d2eb",`
		`+ )`
		`+ self.session.add(pr_test)`
		`+ self.session.commit()`
		`+`
		`+`
		`+ def _response_url(self, project: model.Project) -> str:`
		`+ return f"%s/api/0/ci/generic/%s" % (pagure_config["APP_URL"][:-1], project.url_path)`
		`+`
		`+`
		`+ def _values_for_required_parameters(self):`
		`+ # Use of '_unpatched_get_authorized_project' because`
		`+ # 'pagure.lib.query.get_authorized_project' is mocked for some tests`
		`+ project = _unpatched_get_authorized_project(self.session, "test")`
		`+ ci_url = "https://ci.example.com/"`
		`+ ci_job = "pagure"`
		`+ ci_token = "random_token"`
		`+ branch = "feature"`
		`+ branch_to = "main"`
		`+`
		`+ return project, ci_url, ci_job, ci_token, branch, branch_to`
		`+`
		`+`
		`+ def _values_for_required_parameters_fork(self, user: str):`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+`
		`+ # Use of '_unpatched_get_authorized_project' because`
		`+ # 'pagure.lib.query.get_authorized_project' is mocked for some tests`
		`+ project = _unpatched_get_authorized_project(self.session, "test", user=user)`
		`+`
		`+ return project, ci_url, ci_job, ci_token, branch, branch_to`
		`+`
		`+`
		`+ def test_trigger_ci_build_without_required_args(self,):`
		`+ """`
		`+ If the mandatory args are not passed, there should be a TypeError exception`
		`+ when 'pagure.lib.task_services.trigger_ci_build' eventually calls`
		`+ 'pagure.api.ci.generic.trigger_build'`
		`+ """`
		`+ self.assertRaises(`
		`+ TypeError,`
		`+ pagure.lib.tasks_services.trigger_ci_build`
		`+ )`
		`+`
		`+`
		`+ def test_trigger_ci_build_pull_request(self,):`
		`+ """`
		`+ Test that pagure.lib.tasks_services.trigger_ci_build calls`
		`+ pagure.api.ci.generic.trigger_build without throwing an Exception.`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ cause = "1"`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+`
		`+ output = pagure.lib.tasks_services.trigger_ci_build(`
		`+ cause=cause,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ ci_type="generic",`
		`+ project_name = project.name,`
		`+ pr_uid = pr_uid,`
		`+ )`
		`+`
		`+ self.assertIsNone(output)`
		`+`
		`+`
		`+ def test_trigger_ci_build_commit(self):`
		`+ """`
		`+ Test that pagure.lib.tasks_services.trigger_ci_build calls`
		`+ pagure.api.ci.generic.trigger_build without throwing an Exception.`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ cause = "9c0110e671ffb1e76b629fd23d6a08414ffb44ab"`
		`+`
		`+ output = pagure.lib.tasks_services.trigger_ci_build(`
		`+ cause=cause,`
		`+ branch=branch,`
		`+ branch_to=None,`
		`+ ci_type="generic",`
		`+ project_name = project.name,`
		`+ pr_uid = None,`
		`+ )`
		`+`
		`+ self.assertIsNone(output)`
		`+`
		`+`
		`+ @patch("pagure.lib.query.get_request_by_uid")`
		`+ @patch("pagure.lib.query.get_authorized_project")`
		`+ @patch("pagure.api.ci.generic.trigger_build")`
		`+ def test_trigger_ci_build_pull_request_called_once(self, trigger_build, get_authorized_project, get_request_by_uid):`
		`+ """`
		`+ Test that pagure.lib.tasks_services.trigger_ci_build calls`
		`+ pagure.api.ci.generic.trigger_build once.`
		`+ No further evaluation of failure or success.`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ cause = "1"`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ pull_request = _unpatched_get_request_by_uid(self.session, pr_uid)`
		`+`
		`+ # Ensure that the same project obj is used across the test`
		`+ get_authorized_project.return_value = project`
		`+`
		`+ # Ensure that the same pull_request obj is used across the test`
		`+ get_request_by_uid.return_value = pull_request`
		`+`
		`+ output = pagure.lib.tasks_services.trigger_ci_build(`
		`+ cause=cause,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ ci_type="generic",`
		`+ project_name = project.name,`
		`+ pr_uid = pr_uid,`
		`+ )`
		`+`
		`+ self.assertIsNone(output)`
		`+`
		`+ trigger_build.assert_called_once_with(`
		`+ project=project,`
		`+ ci_url=project.ci_hook.ci_url,`
		`+ ci_job=project.ci_hook.ci_job,`
		`+ ci_token=project.ci_hook.pagure_ci_token,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_pull_request=True,`
		`+ is_commit=False,`
		`+ commit_hash=None,`
		`+ pull_request=pull_request,`
		`+ ci_username=None,`
		`+ ci_password=None`
		`+ )`
		`+`
		`+`
		`+ @patch("pagure.lib.query.get_authorized_project")`
		`+ @patch("pagure.api.ci.generic.trigger_build")`
		`+ def test_trigger_ci_build_commit_called_once(self, trigger_build, get_authorized_project):`
		`+ """`
		`+ Test that pagure.lib.tasks_services.trigger_ci_build calls`
		`+ pagure.api.ci.generic.trigger_build once.`
		`+ No further evaluation of failure or success.`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ cause = "9c0110e671ffb1e76b629fd23d6a08414ffb44ab"`
		`+`
		`+ # Ensure that the same project obj is used across the test`
		`+ get_authorized_project.return_value = project`
		`+`
		`+ output = pagure.lib.tasks_services.trigger_ci_build(`
		`+ cause=cause,`
		`+ branch=branch,`
		`+ branch_to=None,`
		`+ ci_type="generic",`
		`+ project_name = project.name,`
		`+ pr_uid = None,`
		`+ )`
		`+`
		`+ self.assertIsNone(output)`
		`+`
		`+ trigger_build.assert_called_once_with(`
		`+ project=project,`
		`+ ci_url=project.ci_hook.ci_url,`
		`+ ci_job=project.ci_hook.ci_job,`
		`+ ci_token=project.ci_hook.pagure_ci_token,`
		`+ branch=branch,`
		`+ branch_to=None,`
		`+ is_pull_request=False,`
		`+ is_commit=True,`
		`+ commit_hash=cause,`
		`+ pull_request=None,`
		`+ ci_username=None,`
		`+ ci_password=None`
		`+ )`
		`+`
		`+`
		`+ def test_outgoing_payload_pull_request(self):`
		`+ """`
		`+ Validated generated payload for a pull request`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ is_pull_request = True`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ pull_request = _unpatched_get_request_by_uid(self.session, pr_uid)`
		`+`
		`+ want = {`
		`+ "is_pull_request": is_pull_request,`
		`+ "is_commit": False,`
		`+ "pull_request_id": pull_request.id,`
		`+ "pull_request_uid": pull_request.uid,`
		`+ "pull_request_url": pull_request.full_url,`
		`+ "commit_hash": None,`
		`+ "repo_name": project.name,`
		`+ "repo_url": project.full_url,`
		`+ "fork_name": None,`
		`+ "fork_url": None,`
		`+ "branch": branch,`
		`+ "branch_to": branch_to,`
		`+ "response_url": self._response_url(project),`
		`+ }`
		`+`
		`+ got = pagure.api.ci.generic._outgoing_payload(`
		`+ project=project,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_pull_request=is_pull_request,`
		`+ pull_request=pull_request,`
		`+ )`
		`+`
		`+ self.assertJSONEqual(got, json.dumps(want))`
		`+`
		`+`
		`+ def test_outgoing_payload_pull_request_fork(self):`
		`+ """`
		`+ Validated generated payload for a pull request from a fork`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters_fork(user="foo")`
		`+ is_pull_request = True`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ pull_request = _unpatched_get_request_by_uid(self.session, pr_uid)`
		`+`
		`+ want = {`
		`+ "is_pull_request": is_pull_request,`
		`+ "is_commit": False,`
		`+ "pull_request_id": pull_request.id,`
		`+ "pull_request_uid": pull_request.uid,`
		`+ "pull_request_url": pull_request.full_url,`
		`+ "commit_hash": None,`
		`+ "repo_name": project.parent.name,`
		`+ "repo_url": project.parent.full_url,`
		`+ "fork_name": project.name,`
		`+ "fork_url": project.full_url,`
		`+ "branch": branch,`
		`+ "branch_to": branch_to,`
		`+ "response_url": self._response_url(project),`
		`+ }`
		`+`
		`+ got = pagure.api.ci.generic._outgoing_payload(`
		`+ project=project,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_pull_request=is_pull_request,`
		`+ pull_request=pull_request,`
		`+ )`
		`+`
		`+ self.assertJSONEqual(got, json.dumps(want))`
		`+`
		`+`
		`+ def test_outgoing_payload_commit(self):`
		`+ """`
		`+ Validated generated payload for a commit`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ is_commit = True`
		`+ commit_hash = "96df1145c3466fb33edbbde327c6f8705627d2eb"`
		`+`
		`+ want = {`
		`+ "is_pull_request": False,`
		`+ "is_commit": is_commit,`
		`+ "pull_request_id": None,`
		`+ "pull_request_uid": None,`
		`+ "pull_request_url": None,`
		`+ "commit_hash": commit_hash,`
		`+ "repo_name": project.name,`
		`+ "repo_url": project.full_url,`
		`+ "fork_name": None,`
		`+ "fork_url": None,`
		`+ "branch": branch,`
		`+ "branch_to": branch_to,`
		`+ "response_url": self._response_url(project),`
		`+ }`
		`+`
		`+ got = pagure.api.ci.generic._outgoing_payload(`
		`+ project=project,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_commit=is_commit,`
		`+ commit_hash=commit_hash`
		`+ )`
		`+`
		`+ self.assertJSONEqual(got, json.dumps(want))`
		`+`
		`+`
		`+ def test_outgoing_payload_commit_fork(self):`
		`+ """`
		`+ Validated generated payload for a commit from a fork`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters_fork(user="foo")`
		`+ is_commit = True`
		`+ commit_hash = "96df1145c3466fb33edbbde327c6f8705627d2eb"`
		`+`
		`+ want = {`
		`+ "is_pull_request": False,`
		`+ "is_commit": is_commit,`
		`+ "pull_request_id": None,`
		`+ "pull_request_uid": None,`
		`+ "pull_request_url": None,`
		`+ "commit_hash": commit_hash,`
		`+ "repo_name": project.parent.name,`
		`+ "repo_url": project.parent.full_url,`
		`+ "fork_name": project.name,`
		`+ "fork_url": project.full_url,`
		`+ "branch": branch,`
		`+ "branch_to": branch_to,`
		`+ "response_url": self._response_url(project),`
		`+ }`
		`+`
		`+ got = pagure.api.ci.generic._outgoing_payload(`
		`+ project=project,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_commit=is_commit,`
		`+ commit_hash=commit_hash`
		`+ )`
		`+`
		`+ self.assertJSONEqual(got, json.dumps(want))`
		`+`
		`+`
		`+ def test_trigger_build_commit_and_pull_request(self):`
		`+ """`
		`+ Failing with PagureException expected when 'is_commit' and 'is_pull_request' are True`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ is_commit = True`
		`+ commit_hash = "96df1145c3466fb33edbbde327c6f8705627d2eb"`
		`+ is_pull_request = True`
		`+`
		`+ self.assertRaises(`
		`+ pagure.exceptions.PagureException,`
		`+ pagure.api.ci.generic.trigger_build,`
		`+ project=project,`
		`+ ci_url=ci_url,`
		`+ ci_job=ci_job,`
		`+ ci_token=ci_token,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_commit=is_commit,`
		`+ commit_hash=commit_hash,`
		`+ is_pull_request=is_pull_request`
		`+ )`
		`+`
		`+`
		`+ def test_trigger_build_commit_without_hash(self):`
		`+ """`
		`+ Failing with PagureException expected when 'is_commit' but no 'commit_hash' was provided`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ is_commit = True`
		`+`
		`+ self.assertRaises(`
		`+ pagure.exceptions.PagureException,`
		`+ pagure.api.ci.generic.trigger_build,`
		`+ project=project,`
		`+ ci_url=ci_url,`
		`+ ci_job=ci_job,`
		`+ ci_token=ci_token,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_commit=is_commit,`
		`+ )`
		`+`
		`+`
		`+ def test_trigger_build_pull_request_without_pull_request_object(self):`
		`+ """`
		`+ Failing with PagureException expected when 'is_pull_request' but no 'pull_request' object was provided`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ is_pull_request = True`
		`+`
		`+ self.assertRaises(`
		`+ pagure.exceptions.PagureException,`
		`+ pagure.api.ci.generic.trigger_build,`
		`+ project=project,`
		`+ ci_url=ci_url,`
		`+ ci_job=ci_job,`
		`+ ci_token=ci_token,`
		`+ branch=branch,`
		`+ branch_to=branch_to,`
		`+ is_pull_request=is_pull_request,`
		`+ )`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_no_json(self):`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ want = 400`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_json_missing_arg_ci_token(self):`
		`+ """`
		`+ Expecting a bad request response when mandatory arg 'ci_token' is missing.`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ build_status = "SUCCESS"`
		`+`
		`+ want = 400`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ "build_id_external": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_json_missing_arg_pull_request_uid(self):`
		`+ """`
		`+ Expecting a bad request response when mandatory arg 'pull_request_uid' is missing.`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ ci_token = "random_token"`
		`+ build_status = "SUCCESS"`
		`+`
		`+ want = 400`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "ci_token": ci_token,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ "build_id_external": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_json_missing_arg_build_status(self):`
		`+ """`
		`+ Expecting a bad request response when mandatory arg 'build_status' is missing.`
		`+ """`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ ci_token = "random_token"`
		`+`
		`+ want = 400`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "pull_request_uid": pr_uid,`
		`+ "ci_token": ci_token,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ "build_id_external": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_no_project_found(self):`
		`+ ci_token = "random_token"`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ build_status = "SUCCESS"`
		`+`
		`+ want = 404`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/invalid_repo_name",`
		`+ json={`
		`+ "ci_token": ci_token,`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ "build_id_external": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_invalid_ci_token(self):`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ ci_token = "InvalidToken"`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ build_status = "SUCCESS"`
		`+`
		`+ want = 401`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "ci_token": ci_token,`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ "build_id_external": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_no_pull_request_found(self):`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ ci_token = "random_token"`
		`+ pr_uid = "InvalidPullRequestUID"`
		`+ build_status = "SUCCESS"`
		`+`
		`+ want = 404`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "ci_token": ci_token,`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ "build_id_external": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ def test_generic_ci_notification_post_invalid_build_status(self):`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ ci_token = "random_token"`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ build_status = "InvalidBuildStatus"`
		`+`
		`+ want = 400`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "ci_token": ci_token,`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ "build_id_external": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+`
		`+ @patch("pagure.lib.query.get_authorized_project")`
		`+ @patch("pagure.lib.query.get_request_by_uid")`
		`+ @patch("pagure.lib.query.add_pull_request_flag")`
		`+ def test_generic_ci_add_pull_request_flag_called_once(self, add_pull_request_flag, get_request_by_uid, get_authorized_project):`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ ci_token = "random_token"`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ build_status = "SUCCESS"`
		`+ build_id_external = "1234"`
		`+ comment_display_name = "pagure ci - generic"`
		`+ build_url = "https://ci.example.com/1234"`
		`+`
		`+ pull_request = _unpatched_get_request_by_uid(self.session, pr_uid)`
		`+`
		`+ # Needs some improvement, quite some duplicate code and logic from the plugin`
		`+ comment, build_state, build_percent = BUILD_STATS[build_status]`
		`+ comment = comment % build_id_external`
		`+ if pull_request.commit_stop:`
		`+ comment += " (commit: %s)" % (pull_request.commit_stop[:8])`
		`+`
		`+ flag_uid = None`
		`+ for flag in pull_request.flags:`
		`+ if (`
		`+ flag.status == pagure_config["FLAG_PENDING"]`
		`+ and flag.username == comment_display_name`
		`+ ):`
		`+ flag_uid = flag.uid`
		`+ break`
		`+`
		`+ # Ensure that the same project obj is used across the test`
		`+ get_authorized_project.return_value = project`
		`+`
		`+ # Ensure that the same pull_request obj is used across the test`
		`+ get_request_by_uid.return_value = pull_request`
		`+`
		`+ want = 200`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "ci_token": ci_token,`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": comment_display_name,`
		`+ "build_url": build_url,`
		`+ "build_id_external": build_id_external,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want)`
		`+`
		`+ add_pull_request_flag.assert_called_once_with(`
		`+ ANY,`
		`+ request=pull_request,`
		`+ username=comment_display_name,`
		`+ percent=build_percent,`
		`+ comment=comment,`
		`+ url=build_url,`
		`+ status=build_state,`
		`+ uid=flag_uid,`
		`+ user=project.user.username,`
		`+ token=None`
		`+ )`
		`+`
		`+`
		`+ @patch("pagure.lib.query.get_authorized_project")`
		`+ @patch("pagure.lib.query.get_request_by_uid")`
		`+ def test_generic_ci_notification_post_response(self, get_request_by_uid, get_authorized_project):`
		`+ project, ci_url, ci_job, ci_token, branch, branch_to = self._values_for_required_parameters()`
		`+ ci_token = "random_token"`
		`+ pr_uid = "720a0568c1274e74966e54b433b2003e"`
		`+ build_status = "SUCCESS"`
		`+`
		`+ pull_request = _unpatched_get_request_by_uid(self.session, pr_uid)`
		`+`
		`+ # Ensure that the same project obj is used across the test`
		`+ get_authorized_project.return_value = project`
		`+`
		`+ # Ensure that the same pull_request obj is used across the test`
		`+ get_request_by_uid.return_value = pull_request`
		`+`
		`+ want = {`
		`+ "status_code": 200,`
		`+ "body": {`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "result": "Status update successful."`
		`+ }`
		`+ }`
		`+`
		`+ got = self.app.post(`
		`+ f"/api/0/ci/generic/%s" % project.url_path,`
		`+ json={`
		`+ "ci_token": ci_token,`
		`+ "pull_request_uid": pr_uid,`
		`+ "build_status": build_status,`
		`+ "comment_display_name": None,`
		`+ "build_url": None,`
		`+ }`
		`+ )`
		`+`
		`+ self.assertEqual(got.status_code, want["status_code"])`
		`+`
		`+ self.assertJSONEqual(`
		`+ got.get_data(as_text=True),`
		`+ json.dumps(want["body"])`
		`+ )`
		`\ No newline at end of file`

wombelix commented 9 days ago

It started with the idea of a pagure ci plugin for AWS CodePipeline but then I realized a more generic approach might be better.
AWS CodePipeline and / or AWS CodeBuild require a bit of custom implementation within the individual AWS Account, for example with AWS Lambda.
So I have to send some json payload via HTTPs to trigger that and can't move all AWS related logic into pagure.
There will be an AWS based reference implementation, but the plugin can be leverage by other implementations as well.

The related SUSE Hack Week project, the majority of work was done as part of it: https://hackweek.opensuse.org/projects/aws-codepipeline-ci-plugin-for-pagure-on-code-dot-opensuse-dot-org

I wrote some stuff down in my SUSE Hack Week recaps:

wombelix commented 9 days ago

The plugin is still work in progress, not ready to merge yet, current status:

Completed:

Outgoing webhook json generation
Incoming generic_ci_notification
Trigger build (mandatory value validation)

Pending:

Sending the webhook as part of Trigger build
SIGv4, Basic and Token authentication for outgoing requests

Full test coverage because of TDD approach. I forced myself this time to not write a single line actual code without writing the test first.

I tried to keep the changes to functionality that's shared with the jenkins plugin.
But right now that's broken and I'm aware of the following four failing jenkins related tests that I have to fix:

FAILED tests/test_pagure_lib_task_services.py::PagureLibTaskServicesJenkinsCItests::test_trigger_ci_build_valid_project - AssertionError: expected call not found.
FAILED tests/test_pagure_lib_task_services.py::PagureLibTaskServicesJenkinsCIAuthtests::test_trigger_ci_build_valid_project - AssertionError: expected call not found.
FAILED tests/test_pagure_lib_task_services.py::PagureLibTaskServicesJenkinsCIAuthtests::test_trigger_ci_build_valid_project_fork - AssertionError: expected call not found.
FAILED tests/test_pagure_lib_task_services.py::PagureLibTaskServicesJenkinsCItests::test_trigger_ci_build_valid_project_fork - AssertionError: expected call not found.