From 6800e6e29ae7f722168900e9d51a9ec4c35651d3 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Apr 22 2023 19:03:20 +0000 Subject: [PATCH 1/4] Add a tox config to make running tests more convenient Why make a venv for ourselves when tox can do it for us? Also split the runtime and test requirements, and update the README. Signed-off-by: Adam Williamson --- diff --git a/README.md b/README.md index 3135d4c..b698d09 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,12 @@ kedge apply -f kedge.yaml # Development Environment -To run compose_tracker's tests on your local machine you need to create a virtual environment +The easiest way to run compose_tracker's tests on your local machine is to use tox: just +ensure it's installed, and run `tox`. Alternatively, you can manually set up a virtual +environment: ``` -$ python -m venv .venv +$ python3 -m venv .venv $ source .venv/bin/activate (.venv) $ ``` @@ -55,7 +57,7 @@ $ source .venv/bin/activate Then you can install the dependencies using the requirements.txt file ``` -(.venv) $ pip install -r requirements.txt +(.venv) $ pip install -r requirements.txt -r requirements-test.txt ``` Finally you can run the tests diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..062d581 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +pytest +pytest-mock +toml diff --git a/requirements.txt b/requirements.txt index f42c790..bf6d27d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,3 @@ requests fedora_messaging ogr PyYAML -# Development -pytest -pytest-mock diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ca13902 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py{36,37,38,39,310,311} +skip_missing_interpreters = true +isolated_build = true + +[testenv] +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements-test.txt +commands = + py.test From 164bddc2810bdd7c9a9477582408572902b8a334 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Apr 22 2023 19:03:20 +0000 Subject: [PATCH 2/4] Fix test execution: missing args, missing mock config key The tests were never updated to pass the `eln` arg to `process` when it was added. Also, current fedora_messaging requires that we include a `publish_priority` key in the `config.conf` when we're mocking it (which we do several times). --- diff --git a/test_consumer.py b/test_consumer.py index 169e36b..e2ea623 100644 --- a/test_consumer.py +++ b/test_consumer.py @@ -80,7 +80,7 @@ def test_consumer_message_process(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) - con.process(msg) + con.process(msg, eln=False) assert "Fedora-Rawhide-20190619.n.0 DOOMED" in caplog.text @@ -90,7 +90,7 @@ def test_consumer_settings_ignore_compose(mocker, caplog): mocker.patch("compose_tracker.requests") mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}, "publish_priority": None}, ) con = Consumer() body = copy.copy(EXAMPLE_MESSAGE_BODY) @@ -98,7 +98,7 @@ def test_consumer_settings_ignore_compose(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=body, ) - con.process(msg) + con.process(msg, eln=False) assert "Skipping filing issues for IoT composes" in caplog.text @@ -108,13 +108,13 @@ def test_consumer_settings_ignore_compose_2_values(mocker, caplog): mocker.patch("compose_tracker.requests") mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}}, + {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}, "publish_priority": None}, ) con = Consumer() msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) - con.process(msg) + con.process(msg, eln=False) assert "Skipping filing issues for Rawhide composes" in caplog.text @@ -132,7 +132,7 @@ def test_consumer_ignore_good_composes(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=body, ) - con.process(msg) + con.process(msg, eln=False) assert "Fedora-Rawhide-20190619.n.0" not in caplog.text @@ -155,7 +155,7 @@ def test_consumer_logfile_time_parsing(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) - con.process(msg) + con.process(msg, eln=False) assert "- Compose run failed because: - [41350838]" in caplog.text assert "Compose Total time: 0:20:30" in caplog.text assert "Compose phase INIT time: 0:00:10." in caplog.text @@ -173,7 +173,7 @@ def test_consumer_logfile_parsing_failures(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}, "publish_priority": None}, ) # mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) @@ -188,7 +188,7 @@ def test_consumer_logfile_parsing_failures(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) - con.process(msg) + con.process(msg, eln=False) assert "Ostree (variant Silverblue, arch x86_64) failed, but going on anyway." in caplog.text assert "Ostree (variant Silverblue, arch ppc64le) failed, but going on anyway." in caplog.text @@ -203,7 +203,7 @@ def test_adding_labels(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}, "publish_priority": None}, ) req = mocker.patch("compose_tracker.requests.get", autospec=True) text_mock = mocker.MagicMock() @@ -217,7 +217,7 @@ def test_adding_labels(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) - con.process(msg) + con.process(msg, eln=False) assert "Adding Labels ['Fedora', 'Rawhide', 'DOOMED']" in caplog.text def test_consumer_maintainer_pings(mocker, caplog): @@ -231,7 +231,7 @@ def test_consumer_maintainer_pings(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}, "publish_priority": None}, ) # mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) @@ -246,7 +246,7 @@ def test_consumer_maintainer_pings(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) - con.process(msg) + con.process(msg, eln=False) assert "Variant: Spins, subvariant: SoaS task failed. Pinging maintainers: @pbrobinson" in caplog.text # Cannot assert particular maintainers since the set order is random assert "Variant: Spins, subvariant: xfce task failed. Pinging maintainers" in caplog.text and "@kevin" in caplog.text and '@nonamedotc' in caplog.text and '@maxamillion' in caplog.text @@ -262,7 +262,7 @@ def test_log_urls(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts", "mnt_koji_url": "https://kojipkgs.fedoraproject.org/"}, "publish_priority": None}, ) #mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) @@ -277,5 +277,5 @@ def test_log_urls(mocker, caplog): msg = fedora_messaging.api.Message( topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) - con.process(msg) + con.process(msg, eln=False) assert '/mnt/koji/compose/rawhide/Fedora-Rawhide-20200204.n.0/logs/x86_64/Silverblue/ostree-1/runroot.log' in caplog.text From 86e38f28c8a134e6218e99fe09b94ad3e5b5c244 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Apr 22 2023 19:04:58 +0000 Subject: [PATCH 3/4] Current Koji task IDs are now nine digits, not eight I'm...pretty sure it won't be my job to fix things when we get to 10. To my distant successor...hi! Did we get flying cars yet? Signed-off-by: Adam Williamson --- diff --git a/compose_tracker.py b/compose_tracker.py index c221f06..1b83b38 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -155,7 +155,7 @@ class Consumer(object): # then we'll get: # [FAIL] Image build (variant AtomicHost, arch *, subvariant AtomicHost) failed, but going on anyway. # ImageBuild task failed: 35659757. See /mnt/koji/compose/updates/Fedora-29-updates-testing-20190620.1/ - r = re.search(r'.*failed: (\d{8}).*', line) + r = re.search(r'.*failed: (\d{9}).*', line) if r: taskid = r.group(1) text = "- [%s](%s%s)" % (taskid, KOJI_TASK_URL, taskid) @@ -168,7 +168,7 @@ class Consumer(object): # [IMAGE_BUILD ] [INFO ] Hardlinking /mnt/koji/packages/Fedora-AtomicHost/29_Update/20190620.1/... # [IMAGE_BUILD ] [INFO ] Hardlinking /mnt/koji/packages/Fedora-AtomicHost/29_Update/20190620.1/... # [IMAGE_BUILD ] [INFO ] [DONE ] Creating image (formats: qcow2-raw-xz, arches: aarch64-ppc64le-x86_64, variant: AtomicHost, subvariant: AtomicHost) (task id: 35659753) - r = re.search(r'.*\[DONE \].*task id: (\d{8}).*', line) + r = re.search(r'.*\[DONE \].*task id: (\d{9}).*', line) if r: taskid = r.group(1) text = "- [%s](%s%s)" % (taskid, KOJI_TASK_URL, taskid) diff --git a/test_consumer.py b/test_consumer.py index e2ea623..7375848 100644 --- a/test_consumer.py +++ b/test_consumer.py @@ -34,15 +34,15 @@ EXAMPLE_PUNGI_LOG_DOOMED = """2020-02-04 06:15:09 [INFO ] [BEGIN] ---------- 2020-02-04 06:15:20 [INFO ] [BEGIN] ---------- PHASE: WEAVER ---------- 2020-02-04 06:15:21 [INFO ] [DONE ] ---------- PHASE: WEAVER ---------- 2020-02-04 06:15:21 [INFO ] [BEGIN] ---------- PHASE: CREATEISO ---------- -2020-02-04 06:35:39 [ERROR ] Compose run failed: ImageBuild task failed: 41350838. See /mnt/koji/compose/cloud/Fedora-Cloud-30-20200204.0/logs/aarch64-ppc64le-s390x-x86_64/imagebuild-Cloud-Cloud_Base-qcow2-raw-xz.aarch64-ppc64le-s390x-x86_64.log for more details. +2020-02-04 06:35:39 [ERROR ] Compose run failed: ImageBuild task failed: 413508381. See /mnt/koji/compose/cloud/Fedora-Cloud-30-20200204.0/logs/aarch64-ppc64le-s390x-x86_64/imagebuild-Cloud-Cloud_Base-qcow2-raw-xz.aarch64-ppc64le-s390x-x86_64.log for more details. 2020-02-04 06:35:39 [ERROR ] Extended traceback in: /mnt/koji/compose/cloud/Fedora-Cloud-30-20200204.0/logs/global/traceback.global.log """ EXAMPLE_PUNGI_LOG_INCOMPLETE = """2020-02-04 05:59:24 [INFO ] [BEGIN] ---------- PHASE: OSTREE ---------- 2020-02-04 06:03:08 [ERROR ] [FAIL] Ostree (variant Silverblue, arch x86_64) failed, but going on anyway. -2020-02-04 06:03:08 [ERROR ] Runroot task failed: 41350821. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20200204.n.0/logs/x86_64/Silverblue/ostree-1/runroot.log for more details. +2020-02-04 06:03:08 [ERROR ] Runroot task failed: 413508211. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20200204.n.0/logs/x86_64/Silverblue/ostree-1/runroot.log for more details. 2020-02-04 06:03:40 [ERROR ] [FAIL] Ostree (variant Silverblue, arch ppc64le) failed, but going on anyway. -2020-02-04 06:03:40 [ERROR ] Runroot task failed: 41350822. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20200204.n.0/logs/ppc64le/Silverblue/ostree-2/runroot.log for more details. +2020-02-04 06:03:40 [ERROR ] Runroot task failed: 413508221. See /mnt/koji/compose/rawhide/Fedora-Rawhide-20200204.n.0/logs/ppc64le/Silverblue/ostree-2/runroot.log for more details. 2020-02-04 06:04:44 [INFO ] [DONE ] ---------- PHASE: OSTREE ---------- """ @@ -156,7 +156,7 @@ def test_consumer_logfile_time_parsing(mocker, caplog): topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) con.process(msg, eln=False) - assert "- Compose run failed because: - [41350838]" in caplog.text + assert "- Compose run failed because: - [413508381]" in caplog.text assert "Compose Total time: 0:20:30" in caplog.text assert "Compose phase INIT time: 0:00:10." in caplog.text assert "Compose phase CREATEISO: FAILED." in caplog.text From 51ecc03a637f9f54d14916c2b5eb4be8b632196d Mon Sep 17 00:00:00 2001 From: Tomáš Hrčka Date: Apr 26 2023 09:26:23 +0000 Subject: [PATCH 4/4] Update Dockerfile --- diff --git a/Dockerfile b/Dockerfile index 719df70..56db979 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=true RUN dnf update -y && dnf clean all # Install pagure/fedmsg libraries -RUN dnf -y install python3-ogr fedora-messaging && dnf clean all +RUN dnf -y install python3-ogr fedora-messaging python3-toml && dnf clean all RUN mkdir /work WORKDIR /work