PR#52: Make tagging in Pagure generally available and some fixes for the Koji hub plugin - fedora-infra/rpmautospec

		`@@ -3,39 +3,41 @@`

		`import koji`
		`from koji.plugin import callback`
		`- import requests`

		`- from rpmautospec.py2compat import escape_tags`
		`+ from rpmautospec.py2compat import tagging`


		`CONFIG_FILE = "/etc/koji-hub/plugins/rpmautospec_hub.conf"`
		`CONFIG = None`

		`+ _log = logging.getLogger("koji.plugin.rpmautospec_hub")`
		`+`
		`+ git_filter = None`
		`+ git_filter_re = None`
		`+ pagure_proxy = None`
		`+`

		`@callback("postTag")`
		`def autotag_cb(cb_type, **kwargs):`
		`- global CONFIG`
		`-`
		`- log = logging.getLogger("koji.plugin.rpmautospec_hub")`
		`+ global CONFIG, pagure_proxy, git_filter, git_filter_re`

		`if not CONFIG:`
		`try:`
		`CONFIG = koji.read_config_files([(CONFIG_FILE, True)])`
		`except Exception:`
		`message = "While attempting to read config file %s, an exception occurred:"`
		`- log.exception(message, CONFIG_FILE)`
		`+ _log.exception(message, CONFIG_FILE)`
		`return`

		`- base_url = CONFIG.get("pagure", "url")`
		`- token = CONFIG.get("pagure", "token")`
		`+ git_filter = r".\.fedoraproject\.org/(?P<repo>rpms/.)\.git#(?P<commit>[a-f0-9]{40})$"`
		`+ if CONFIG.has_option("pagure", "git_filter"):`
		`+ git_filter = CONFIG.get("pagure", "git_filter",)`
		`+ git_filter_re = re.compile(git_filter)`

		`- git_filter = r".\.fedoraproject\.org/(?P<repo>rpms/.)\.git#(?P<commit>[a-f0-9]{40})$"`
		`- if CONFIG.has_option("pagure", "git_filter"):`
		`- git_filter = CONFIG.get(`
		`- "pagure",`
		`- "git_filter",`
		`- )`
		`- git_filter_re = re.compile(git_filter)`
		`+ if not pagure_proxy:`
		`+ base_url = CONFIG.get("pagure", "url")`
		`+ token = CONFIG.get("pagure", "token")`
		`+ pagure_proxy = tagging.PagureTaggingProxy(base_url=base_url, auth_token=token, logger=_log)`

		`build = kwargs["build"]`

		`@@ -45,38 +47,12 @@`
		`commit = match.group("commit")`
		`if not repo or not commit:`
		`info = "Could not parse repo and commit from {}, skipping."`
		`- log.info(info.format(build["source"]))`
		`+ _log.info(info.format(build["source"]))`
		`return`
		`else:`
		`- log.info("No source for this build, skipping.")`
		`+ _log.info("No source for this build, skipping.")`
		`return`

		`build["epoch"] = build.get("epoch") or 0`
		`-`
		`- nevr = "{name}-{epoch}-{version}-{release}".format(**build)`
		`-`
		`- if not build["epoch"]:`
		`- nevr = "{name}-{version}-{release}".format(**build)`
		`-`
		`- escaped_nevr = escape_tags.escape_tag(nevr)`
		`-`
		`- data = {`
		`- "tagname": escaped_nevr,`
		`- "commit_hash": commit,`
		`- "message": None,`
		`- "with_commits": True,`
		`- }`
		`-`
		`- endpoint_url = "{}/api/0/{}/git/tags".format(base_url, repo)`
		`- headers = {"Authorization": "token {}".format(token)}`
		`- try:`
		`- response = requests.post(endpoint_url, headers=headers, data=data)`
		`- except Exception:`
		`- error = "While attempting to create a tag in %s, an exception occurred:"`
		`- log.exception(error, endpoint_url)`
		`- return`
		`-`
		`- if not response.ok:`
		`- error = "While attempting to create a tag in %s, the request failed with: STATUS %s %s"`
		`- log.error(error, endpoint_url, response.status_code, response.text)`
		`- return`
		`+ tagname = "build/" + tagging.escape_tag("{name}-{epoch}-{version}-{release}".format(**build))`
		`+ pagure_proxy.create_tag(repository=repo, tagname=tagname, commit_hash=commit)`

python-rpmautospec.spec

file modified

+8 -7

		`@@ -11,6 +11,8 @@`

		`BuildArch: noarch`
		`BuildRequires: python3-devel >= 3.6.0`
		`+ # We need Python 2 macros for the Koji hub plugin`
		`+ BuildRequires: python2-devel`
		`# EPEL7 does not have python3-koji and the other dependencies here are only`
		`# needed in the buildroot for the tests, which can't run because of the lack of`
		`# python3-koji`
		`@@ -79,10 +81,8 @@`
		`A Koji plugin for tagging successful builds in their dist-git repository.`

		`%files -n koji-hub-plugin-rpmautospec`
		`+ %{python2_sitelib}/rpmautospec/`
		`%{_prefix}/lib/koji-hub-plugins/rpmautospec_hub.py`
		`- %{_prefix}/lib/koji-hub-plugins/rpmautospec/__init__.py`
		`- %{_prefix}/lib/koji-hub-plugins/rpmautospec/py2compat/__init__.py`
		`- %{_prefix}/lib/koji-hub-plugins/rpmautospec/py2compat/escape_tags.py`

		`%config(noreplace) %{_sysconfdir}/koji-hub/plugins/rpmautospec_hub.conf`

		`@@ -104,10 +104,11 @@`

		`# the hub-plugin py2 tagging library`
		`# Install the py2compat files to the koji-hub-plugin`
		`- mkdir -p %{buildroot}%{_prefix}/lib/koji-hub-plugins/rpmautospec/py2compat`
		`- touch %{buildroot}%{_prefix}/lib/koji-hub-plugins/rpmautospec/__init__.py \`
		`- %{buildroot}%{_prefix}/lib/koji-hub-plugins/rpmautospec/py2compat/__init__.py`
		`- install -m 0644 rpmautospec/py2compat/escape_tags.py %{buildroot}%{_prefix}/lib/koji-hub-plugins/rpmautospec/py2compat`
		`+ mkdir -p %{buildroot}%{python2_sitelib}/rpmautospec/py2compat/`
		`+ touch %{buildroot}%{python2_sitelib}/rpmautospec/__init__.py \`
		`+ %{buildroot}%{python2_sitelib}/rpmautospec/py2compat/__init__.py`
		`+ install -m 0644 rpmautospec/py2compat/tagging.py \`
		`+ %{buildroot}%{python2_sitelib}/rpmautospec/py2compat/`

		`# the hub-plugin config`
		`mkdir -p %{buildroot}%{_sysconfdir}/koji-hub/plugins/`

rpmautospec/changelog.py

file modified

+1 -1

		`@@ -9,7 +9,7 @@`
		`import typing`

		`from .misc import run_command`
		`- from .py2compat.escape_tags import unescape_tag`
		`+ from .py2compat.tagging import unescape_tag`

		`_log = logging.getLogger(__name__)`

rpmautospec/py2compat/tagging.py ~~rpmautospec/py2compat/escape_tags.py~~

file renamed

+56 -1

		`@@ -1,4 +1,7 @@`
		`- """Escape/unescape NEVRs for use as git tags.`
		`+ """Tagging functionality shared with the Koji hub plugin`
		`+`
		`+ - Escape/unescape NEVRs for use as git tags.`
		`+ - Tag commits via the Pagure API`

		`This module should work in Python 2.7 to be usable from a Koji plugin running`
		`in the context of the hub. Let's avoid Py2-isms where we can, though.`
		`@@ -8,6 +11,7 @@`
		`- remove monkey-patches for urllib.parse, the str type, re.Pattern`
		`- use f-strings instead of str.format()`
		`- revert to using Python 3 syntax for type hints`
		`+ - drop explicitly inheriting classes from object`
		`"""`

		`from __future__ import absolute_import`
		`@@ -15,9 +19,12 @@`
		`from __future__ import print_function`
		`from __future__ import unicode_literals`

		`+ import logging`
		`import re`
		`import sys`

		`+ import requests`
		`+`
		`try:`
		`from urllib import parse`
		`except ImportError: # pragma: no cover`
		`@@ -42,6 +49,8 @@`
		`else:`
		`PY = 3`

		`+ _log = logging.getLogger(__name__)`
		`+`
		`# See https://git-scm.com/docs/git-check-ref-format`
		`git_tag_seqs_to_escape = [`
		`"~",`
		`@@ -134,3 +143,49 @@`
		`Returns: An unescaped tag name.`
		`"""`
		`return parse.unquote(escaped_tagname)`
		`+`
		`+`
		`+ class PagureTaggingProxy(object):`
		`+ """A small proxy around remote tagging via the Pagure API`
		`+`
		`+ This is mainly to avoid passing the whole slew of Pagure instance`
		`+ configuration around for every single tagging operation.`
		`+ """`
		`+`
		`+ endpoint_url_tmpl = "{base_url}/api/0/{repository}/git/tags"`
		`+`
		`+ def __init__(self, base_url, auth_token, logger=None):`
		`+ # type: (str, str, str)`
		`+ self.base_url = base_url`
		`+ self.auth_token = auth_token`
		`+ self.log = logger or _log`
		`+`
		`+ def create_tag(`
		`+ self, repository, tagname, commit_hash, message=None, with_commits=True, force=False,`
		`+ ):`
		`+ # type: (str, str, str, str, bool, bool) -> requests.Response`
		`+ """Tag a commit remotely in a Pagure repository"""`
		`+ data = {`
		`+ "tagname": tagname,`
		`+ "commit_hash": commit_hash,`
		`+ "message": message,`
		`+ "with_commits": with_commits,`
		`+ "force": force,`
		`+ }`
		`+`
		`+ endpoint_url = self.endpoint_url_tmpl.format(base_url=self.base_url, repository=repository)`
		`+ headers = {"Authorization": "token {}".format(self.auth_token)}`
		`+`
		`+ try:`
		`+ response = requests.post(endpoint_url, headers=headers, data=data)`
		`+ except Exception:`
		`+ error = "While attempting to create a tag in %s, an exception occurred:"`
		`+ self.log.exception(error, endpoint_url)`
		`+ return`
		`+`
		`+ if not response.ok:`
		`+ error = "While attempting to create a tag in %s, the request failed with: STATUS %s %s"`
		`+ self.log.error(error, endpoint_url, response.status_code, response.text)`
		`+ return`
		`+`
		`+ return response`

rpmautospec/tag_package.py

file modified

+1 -1

		`@@ -4,7 +4,7 @@`
		`import os`

		`from .misc import get_package_builds, koji_init, run_command`
		`- from .py2compat.escape_tags import escape_tag, tag_prefix`
		`+ from .py2compat.tagging import escape_tag, tag_prefix`


		`_log = logging.getLogger(__name__)`

tests/koji_plugins/test_rpmautospec_hub.py

file modified

+18 -14

		`@@ -3,26 +3,29 @@`
		`from koji_plugins import rpmautospec_hub`


		`- test_config = {`
		`- "url": "src.fedoraproject.org",`
		`- "token": "aaabbbcc",`
		`- }`
		`-`
		`-`
		`class MockConfig:`
		`- def get(self, section, key, **kwargs):`
		`- return test_config.get(key, kwargs.get("fallback"))`
		`+ test_config = {`
		`+ "pagure": {"url": "src.fedoraproject.org", "token": "aaabbbcc"},`
		`+ }`
		`+`
		`+ def get(self, section, option, **kwargs):`
		`+ fallback = kwargs.pop("fallback", None)`
		`+ if fallback:`
		`+ return self.test_config.get(section, {}).get(option, fallback)`
		`+ else:`
		`+ return self.test_config[section][option]`

		`- def has_option(*args):`
		`- return False`
		`+ def has_option(self, section, option):`
		`+ return option in self.test_config.get(section, {})`


		`class TestRpmautospecHub:`
		`"""Test the rpmautospec hub plugin for Koji."""`

		`- @mock.patch("koji_plugins.rpmautospec_hub.requests.post")`
		`- @mock.patch("koji_plugins.rpmautospec_hub.CONFIG", MockConfig())`
		`- def test_tag_in_pagure(self, mock_post):`
		`+ @mock.patch("rpmautospec.py2compat.tagging.requests.post")`
		`+ @mock.patch("koji.read_config_files")`
		`+ def test_autotag_cb(self, read_config_files, mock_post):`
		`+ read_config_files.return_value = MockConfig()`
		`mock_post.return_value.ok = True`

		`cbtype = "postTag"`
		`@@ -49,7 +52,8 @@`
		`data={`
		`"commit_hash": "a0698fd21544880718d01a80ea19c91b13011235",`
		`"message": None,`
		`- "tagname": "deepin-wallpapers-1.7.6-4.fc32",`
		`+ "tagname": "build/deepin-wallpapers-0-1.7.6-4.fc32",`
		`"with_commits": True,`
		`+ "force": False,`
		`},`
		`)`

tests/rpmautospec/py2compat/test_tagging.py ~~tests/rpmautospec/py2compat/test_escape_tags.py~~

file renamed

+8 -8

		`@@ -2,11 +2,11 @@`

		`import pytest`

		`- from rpmautospec.py2compat import escape_tags`
		`+ from rpmautospec.py2compat import tagging`


		`- class TestEscapeTags:`
		`- """Test the rpmautospec.py2compat.escape_tags module."""`
		`+ class TestTagging:`
		`+ """Test the rpmautospec.py2compat.tagging module."""`

		`test_escape_sequences = {`
		`"%": "%25",`
		`@@ -42,21 +42,21 @@`
		`@pytest.mark.parametrize("sequence", test_escape_sequences)`
		`def test_escape_sequence(self, sequence):`
		`"""Test escape_sequence()"""`
		`- assert escape_tags.escape_sequence(sequence) == self.test_escape_sequences[sequence]`
		`+ assert tagging.escape_sequence(sequence) == self.test_escape_sequences[sequence]`

		`@pytest.mark.parametrize("unescaped_tag", test_escape_tags)`
		`def test_escape_tag(self, unescaped_tag):`
		`"""Test escape_tag()"""`
		`- assert escape_tags.escape_tag(unescaped_tag) == self.test_escape_tags[unescaped_tag]`
		`+ assert tagging.escape_tag(unescaped_tag) == self.test_escape_tags[unescaped_tag]`

		`- @mock.patch.object(escape_tags, "git_tag_seqs_to_escape", [b"Not a string"])`
		`+ @mock.patch.object(tagging, "git_tag_seqs_to_escape", [b"Not a string"])`
		`def test_escape_tag_broken_git_tag_seqs_to_escape(self):`
		`"""Test escape_tag() with garbage in git_tag_seqs_to_escape"""`
		`with pytest.raises(TypeError):`
		`- escape_tags.escape_tag("a string")`
		`+ tagging.escape_tag("a string")`

		`@pytest.mark.parametrize("unescaped_tag", test_escape_tags)`
		`def test_unescape_tag(self, unescaped_tag):`
		`"""Test escape_tag()"""`
		`escaped_tag = self.test_escape_tags[unescaped_tag]`
		`- assert escape_tags.unescape_tag(escaped_tag) == unescaped_tag`
		`+ assert tagging.unescape_tag(escaped_tag) == unescaped_tag`

no initial comment

zuul commented 4 years ago

Build failed.

pre-commit : FAILURE in 1m 14s
pytest : SUCCESS in 1m 04s
rpmbuild : SUCCESS in 1m 35s

rebased onto 1405ef6

4 years ago

pingou commented on line 16 of koji_plugins/rpmautospec_hub.py 4 years ago

Why not instantiating it here and thus drop the line 39 in this diff?

zuul commented 4 years ago

Build failed.

pre-commit : FAILURE in 1m 14s
pytest : SUCCESS in 1m 03s
rpmbuild : SUCCESS in 1m 10s

pingou commented on line 15 of python-rpmautospec.spec 4 years ago

We now know that this isn't working, so we can drop the py2compat from the koji-hub plugin subpackage.

nphilipp commented on line 16 of koji_plugins/rpmautospec_hub.py 4 years ago

Because every other Koji plugin I looked at instantiates the configuration during the callback (not sure about the reason) and we only know the filter regex when we can access the configuration.

4 new commits added