From 46fa2d89cbc1d73146e576d9303125b082bfb9af Mon Sep 17 00:00:00 2001
From: Pavel Raiskup <praiskup@redhat.com>
Date: Jan 12 2018 18:42:48 +0000
Subject: [PATCH 1/3] [frontend] new generic web-hook


This webhook is theoretically useable for the existing --srpm
build methods, but we need to get the webhook payload parsers (for
Github and Gitlab) concentrated on one place, and get the combo
parser applied here, too (TODO task).

The webhook payload is for now only stored in temporary data
directory, so the end-user (via the future srpm build method) will
have the payload in hand and can parse it manually.

---

diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py
index 3a4430f..422893e 100644
--- a/frontend/coprs_frontend/coprs/logic/builds_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py
@@ -697,11 +697,12 @@ GROUP BY
     @classmethod
     def delete_local_source(cls, build):
         """
-        Deletes the source (rpm or .spec) locally stored for upload (if exists)
+        Deletes the locally stored data for build purposes.  This is typically
+        uploaded srpm file, uploaded spec file or webhook POST content.
         """
         # is it hosted on the copr frontend?
-        if build.source_type == helpers.BuildSourceEnum("upload"):
-            data = json.loads(build.source_json)
+        data = json.loads(build.source_json)
+        if 'tmp' in data:
             tmp = data["tmp"]
             storage_path = app.config["SRPM_STORAGE_DIR"]
             try:
diff --git a/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py b/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py
index e3e35d4..5b0e143 100644
--- a/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py
+++ b/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py
@@ -1,4 +1,5 @@
 import flask
+from functools import wraps
 
 from coprs import db, app
 from coprs import helpers
@@ -15,10 +16,65 @@ from coprs.views.misc import page_not_found, access_restricted
 
 import logging
 import os
+import tempfile
+import shutil
 
 log = logging.getLogger(__name__)
 
 
+def skip_invalid_calls(route):
+    """
+    A best effort attempt to drop hook callswhich should not obviously end up
+    with new build request (thus allocated build-id).
+    """
+    @wraps(route)
+    def decorated_function(*args, **kwargs):
+        if 'X-GitHub-Event' in flask.request.headers:
+            event = flask.request.headers["X-GitHub-Event"]
+            if event == "ping":
+                return "SKIPPED\n", 200
+        return route(*args, **kwargs)
+
+    return decorated_function
+
+
+def copr_id_and_uuid_required(route):
+    @wraps(route)
+    def decorated_function(**kwargs):
+        if not 'copr_id' in kwargs or not 'uuid' in kwargs:
+            return 'COPR_ID_OR_UUID_TOKEN_MISSING\n', 400
+
+        copr_id = kwargs.pop('copr_id')
+        try:
+            copr = ComplexLogic.get_copr_by_id_safe(copr_id)
+        except ObjectNotFound:
+            return "PROJECT_NOT_FOUND\n", 404
+
+        if copr.webhook_secret != kwargs.pop('uuid'):
+            return "BAD_UUID\n", 403
+
+        return route(copr, **kwargs)
+
+    return decorated_function
+
+
+def package_name_required(route):
+    @wraps(route)
+    def decorated_function(copr, **kwargs):
+        if not 'package_name' in kwargs:
+            return 'PACKAGE_NAME_REQUIRED\n', 400
+
+        package_name = kwargs.pop('package_name')
+        try:
+            package = ComplexLogic.get_package_safe(copr, package_name)
+        except ObjectNotFound:
+            return "PACKAGE_NOT_FOUND\n", 404
+
+        return route(copr, package, **kwargs)
+
+    return decorated_function
+
+
 @webhooks_ns.route("/github/<copr_id>/<uuid>/", methods=["POST"])
 def webhooks_git_push(copr_id, uuid):
     if flask.request.headers["X-GitHub-Event"] == "ping":
@@ -99,3 +155,54 @@ def webhooks_gitlab_push(copr_id, uuid):
     db.session.commit()
 
     return "OK", 200
+
+
+class HookContentStorage(object):
+    tmp = None
+
+    def __init__(self):
+        if not flask.request.json:
+            return
+        self.tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"])
+        log.debug("storing hook content under %s", self.tmp)
+        try:
+            with open(os.path.join(self.tmp, 'hook_payload'), "w") as f:
+                # Do we need to dump http headers, too?
+                f.write(flask.request.data.decode('ascii'))
+
+        except Exception:
+            log.exception('can not store hook payload')
+            self.delete()
+
+    def rebuild_dict(self):
+        if self.tmp:
+            return {'tmp': os.path.basename(self.tmp), 'hook_data': True }
+        return {}
+
+    def delete(self):
+        if self.tmp:
+            shutil.rmtree(self.tmp)
+
+
+@webhooks_ns.route("/custom/<uuid>/<copr_id>/", methods=["POST"])
+@webhooks_ns.route("/custom/<uuid>/<copr_id>/<package_name>/", methods=["POST"])
+@copr_id_and_uuid_required
+@package_name_required
+@skip_invalid_calls
+def webhooks_package_custom(copr, package, flavor=None):
+    # Each source provider (github, gitlab, pagure, ...) provides different
+    # "payload" format for different events.  Parsing it here is burden we can
+    # do one day, but now just dump the hook contents somewhere so users can
+    # parse manually.
+    storage = HookContentStorage()
+    try:
+        build = BuildsLogic.rebuild_package(package, storage.rebuild_dict())
+        db.session.commit()
+    except Exception:
+        log.exception('can not submit build from webhook')
+        storage.delete()
+        return "BUILD_REQUEST_ERROR\n", 500
+
+    # Return the build ID, so (e.g.) the CI process (e.g. Travis job) knows
+    # what build results to wait for.
+    return str(build.id) + "\n", 200

From 446d95d971870e650dbcdf19ccb171458667eef4 Mon Sep 17 00:00:00 2001
From: Pavel Raiskup <praiskup@redhat.com>
Date: Jan 12 2018 18:42:52 +0000
Subject: [PATCH 2/3] [frontend] opt rename SRPM_STORAGE_DIR to STORAGE_DIR


Now we use that dir for uploading srpms, spec files and hook
contents.

---

diff --git a/frontend/coprs_frontend/config/copr.conf b/frontend/coprs_frontend/config/copr.conf
index 5a8a07b..deb2445 100644
--- a/frontend/coprs_frontend/config/copr.conf
+++ b/frontend/coprs_frontend/config/copr.conf
@@ -96,7 +96,7 @@ INTRANET_IPS = ["127.0.0.1", "192.168.1.0/24"]
 BUILDER_IPS = ["127.0.0.1"]
 
 # a place for storing srpms until they get uploaded
-SRPM_STORAGE_DIR = "/var/lib/copr/data/srpm_storage"
+STORAGE_DIR = "/var/lib/copr/data/srpm_storage"
 
 # no need to filter cla_* groups, they are already filtered by fedora openid
 BLACKLISTED_GROUPS = ['fedorabugs', 'packager', 'provenpackager']
diff --git a/frontend/coprs_frontend/config/copr_devel.conf b/frontend/coprs_frontend/config/copr_devel.conf
index c28720a..e52916b 100644
--- a/frontend/coprs_frontend/config/copr_devel.conf
+++ b/frontend/coprs_frontend/config/copr_devel.conf
@@ -56,7 +56,7 @@ INTRANET_IPS = ["127.0.0.1", "192.168.1.0/24"]
 REPO_GPGCHECK = 0
 
 # a place for storing srpms until they get uploaded
-SRPM_STORAGE_DIR = "/var/lib/copr/data/srpm_storage"
+STORAGE_DIR = "/var/lib/copr/data/srpm_storage"
 
 # no need to filter cla_* groups, they are already filtered by fedora openid
 BLACKLISTED_GROUPS = ['fedorabugs', 'packager', 'provenpackager']
diff --git a/frontend/coprs_frontend/config/copr_unit_test.conf b/frontend/coprs_frontend/config/copr_unit_test.conf
index a40d0ff..6c9a1e5 100644
--- a/frontend/coprs_frontend/config/copr_unit_test.conf
+++ b/frontend/coprs_frontend/config/copr_unit_test.conf
@@ -58,7 +58,7 @@ LOG_FILENAME = "copr_frontend.log"
 LOG_DIR = "/tmp/"
 
 # a place for storing srpms until they get uploaded
-SRPM_STORAGE_DIR = os.path.join(LOCAL_TMP_DIR, "srpm_storage")
+STORAGE_DIR = os.path.join(LOCAL_TMP_DIR, "srpm_storage")
 
 # %check section start redis-server on this port
 REDIS_HOST = "127.0.0.1"
diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py
index f3a7d84..ce4dd1f 100644
--- a/frontend/coprs_frontend/coprs/config.py
+++ b/frontend/coprs_frontend/coprs/config.py
@@ -55,7 +55,7 @@ class Config(object):
 
     REPO_GPGCHECK = 1
 
-    SRPM_STORAGE_DIR = "/var/lib/copr/data/srpm_storage/"
+    STORAGE_DIR = "/var/lib/copr/data/srpm_storage/"
 
     LAYOUT_OVERVIEW_HIDE_QUICK_ENABLE = False
 
diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py
index 422893e..a23f344 100644
--- a/frontend/coprs_frontend/coprs/logic/builds_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py
@@ -494,7 +494,7 @@ GROUP BY
         :param f_uploader(file_path): function which stores data at the given `file_path`
         :return:
         """
-        tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"])
+        tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"])
         tmp_name = os.path.basename(tmp)
         filename = secure_filename(orig_filename)
         file_path = os.path.join(tmp, filename)
@@ -704,7 +704,7 @@ GROUP BY
         data = json.loads(build.source_json)
         if 'tmp' in data:
             tmp = data["tmp"]
-            storage_path = app.config["SRPM_STORAGE_DIR"]
+            storage_path = app.config["STORAGE_DIR"]
             try:
                 shutil.rmtree(os.path.join(storage_path, tmp))
             except:
diff --git a/frontend/coprs_frontend/coprs/views/tmp_ns/tmp_general.py b/frontend/coprs_frontend/coprs/views/tmp_ns/tmp_general.py
index 8feda21..0e51be7 100644
--- a/frontend/coprs_frontend/coprs/views/tmp_ns/tmp_general.py
+++ b/frontend/coprs_frontend/coprs/views/tmp_ns/tmp_general.py
@@ -5,5 +5,5 @@ from coprs import app
 
 @tmp_ns.route("/<directory>/<file_path>")
 def give_srpm(directory, file_path):
-    path = os.path.join(app.config["SRPM_STORAGE_DIR"], directory)
+    path = os.path.join(app.config["STORAGE_DIR"], directory)
     return flask.send_from_directory(path, file_path)
diff --git a/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py b/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py
index 5b0e143..921d69c 100644
--- a/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py
+++ b/frontend/coprs_frontend/coprs/views/webhooks_ns/webhooks_general.py
@@ -163,7 +163,7 @@ class HookContentStorage(object):
     def __init__(self):
         if not flask.request.json:
             return
-        self.tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"])
+        self.tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"])
         log.debug("storing hook content under %s", self.tmp)
         try:
             with open(os.path.join(self.tmp, 'hook_payload'), "w") as f:
diff --git a/frontend/coprs_frontend/tests/coprs_test_case.py b/frontend/coprs_frontend/tests/coprs_test_case.py
index d08efab..09e9cbf 100644
--- a/frontend/coprs_frontend/tests/coprs_test_case.py
+++ b/frontend/coprs_frontend/tests/coprs_test_case.py
@@ -35,7 +35,7 @@ class CoprsTestCase(object):
         config = coprs.app.config
         for key in [
                 "LOCAL_TMP_DIR",
-                "SRPM_STORAGE_DIR",
+                "STORAGE_DIR",
             ]:
             if key in config:
                 path = os.path.abspath(config[key])

From 62e71a1d7bd17be837edaca5c618cfa27f2580e3 Mon Sep 17 00:00:00 2001
From: Pavel Raiskup <praiskup@redhat.com>
Date: Jan 12 2018 18:42:52 +0000
Subject: [PATCH 3/3] [frontend] add tests for custom webhook


---

diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index 59c4e9a..66733ce 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -395,7 +395,7 @@ def copr_permissions(copr):
 
 def render_copr_webhooks(copr):
     if not copr.webhook_secret:
-        copr.webhook_secret = uuid.uuid4()
+        copr.webhook_secret = str(uuid.uuid4())
         db.session.add(copr)
         db.session.commit()
 
diff --git a/frontend/coprs_frontend/tests/coprs_test_case.py b/frontend/coprs_frontend/tests/coprs_test_case.py
index 09e9cbf..21c7208 100644
--- a/frontend/coprs_frontend/tests/coprs_test_case.py
+++ b/frontend/coprs_frontend/tests/coprs_test_case.py
@@ -6,6 +6,7 @@ import time
 import glob
 from functools import wraps
 import datetime
+import uuid
 
 import pytest
 import decorator
@@ -277,6 +278,19 @@ class CoprsTestCase(object):
         self.db.session.add_all([self.b1, self.b2, self.b3, self.b4])
 
     @pytest.fixture
+    def f_hook_package(self):
+        self.f_users()
+        self.f_coprs()
+        self.f_mock_chroots()
+        self.f_builds()
+
+        self.c1.webhook_secret = str(uuid.uuid4())
+        self.db.session.add(self.c1)
+        self.pHook = models.Package(
+            copr=self.c1, name="hook-package",
+            source_type=helpers.BuildSourceEnum('scm'))
+
+    @pytest.fixture
     def f_build_few_chroots(self):
         """
             Requires fixture: f_mock_chroots_many
diff --git a/frontend/coprs_frontend/tests/test_webhooks.py b/frontend/coprs_frontend/tests/test_webhooks.py
new file mode 100644
index 0000000..c0908e9
--- /dev/null
+++ b/frontend/coprs_frontend/tests/test_webhooks.py
@@ -0,0 +1,80 @@
+import os
+import json
+import requests
+
+from tests.coprs_test_case import CoprsTestCase
+
+class TestCustomWebhook(CoprsTestCase):
+    def custom_post(self, data, token, copr_id, package_name=None):
+        url = "/webhooks/custom/{uuid}/{copr_id}/"
+        url = url.format(uuid=token, copr_id=copr_id)
+        if package_name:
+            url = "{0}{1}/".format(url, package_name)
+
+        return self.tc.post(
+            url,
+            content_type="application/json",
+            data=json.dumps(data) if data != None else None,
+        )
+
+
+    def test_package_not_found(self, f_hook_package, f_db):
+        r = self.custom_post(
+            None,
+            self.c1.webhook_secret,
+            self.c1.id,
+            'ddd',
+        )
+        assert r.data.decode('ascii') == "PACKAGE_NOT_FOUND\n"
+        assert r.status_code == 404
+
+
+    def test_hook_data_stored(self, f_hook_package, f_db):
+        package_name = self.pHook.name
+        hook_payload = {'some': 'data'}
+        r = self.custom_post(
+            hook_payload,
+            self.c1.webhook_secret,
+            self.c1.id,
+            package_name,
+        )
+
+        build_id = int(r.data)
+        builds = self.models.Build.query.filter_by(id=build_id).all()
+        assert len(builds) == 1
+
+        build = builds[0]
+        assert package_name == build.package.name
+
+        storage = self.app.config['STORAGE_DIR']
+        source = json.loads(build.source_json)
+        hook_file= os.path.join(storage, source['tmp'], 'hook_payload')
+        with open(hook_file, 'r') as f:
+            decoded = json.loads(f.read())
+            assert decoded == hook_payload
+
+
+    def test_bad_uuid(self, f_hook_package, f_db):
+        assert self.custom_post(
+            None,
+            "invalid-token",
+            self.c1.id,
+            self.pHook.name,
+        ).status_code == 403
+
+        assert self.custom_post(
+            None,
+            "invalid-token",
+            self.c1.id,
+            "invalid-pkg-name",
+        ).status_code == 403
+
+        resp = self.custom_post(
+            None,
+            "invalid-token",
+            '2',
+            "invalid-pkg-name",
+        )
+
+        assert resp.status_code == 403
+        assert resp.data.decode('ascii') == "BAD_UUID\n"