From b6590564c1b718a1cb579cebea01d2b91074547b Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: May 23 2021 12:44:26 +0000 Subject: builder, backend, frontend, python: provide build results via APIv3 Fix #1411 This change is not trivial. We need to generate `results.json` file containing the the built packages NEVRAs and we then download this file to backend. When build is marked as finished, we send these results to the frontend and store them to the database. The APIv3 endpoints are just a tip of the iceberg. We also provide `get_built_packages(...)` methods in our python client but we don't provide any `copr-cli` command (yet?) that would utilize them. --- diff --git a/backend/copr_backend/background_worker_build.py b/backend/copr_backend/background_worker_build.py index c738082..5f41466 100644 --- a/backend/copr_backend/background_worker_build.py +++ b/backend/copr_backend/background_worker_build.py @@ -9,6 +9,7 @@ import pipes import shutil import statistics import time +import json from packaging import version @@ -30,7 +31,7 @@ from copr_backend.vm_alloc import ResallocHostFactory MAX_HOST_ATTEMPTS = 3 MAX_SSH_ATTEMPTS = 5 -MIN_BUILDER_VERSION = "0.40.1.dev" +MIN_BUILDER_VERSION = "0.49.1.dev" CANCEL_CHECK_PERIOD = 5 MESSAGES = { @@ -281,6 +282,20 @@ class BuildBackgroundWorker(BackgroundWorker): self.frontend_client.update(data) self.sender.announce("build.end", self.job, self.last_hostname) + def _parse_results(self): + """ + Parse `results.json` and update the `self.job` object. + """ + if self.job.chroot == "srpm-builds": + # We care only about final RPMs + return + + path = os.path.join(self.job.results_dir, "results.json") + assert os.path.exists(path) + with open(path, "r") as f: + results = json.load(f) + self.job.results = results + def _wait_for_repo(self): """ Wait a while for initial createrepo, and eventually fail the build @@ -650,6 +665,7 @@ class BuildBackgroundWorker(BackgroundWorker): build_details = { "built_packages": self._collect_built_packages(job), } + self._parse_results() self.log.info("build details: %s", build_details) except Exception as e: raise BackendError( diff --git a/backend/copr_backend/job.py b/backend/copr_backend/job.py index 9f65f6f..b49dd7b 100644 --- a/backend/copr_backend/job.py +++ b/backend/copr_backend/job.py @@ -66,6 +66,8 @@ class BuildJob(object): self.uses_devel_repo = None self.sandbox = None + self.results = None + # TODO: validate update data, user marshmallow for key, val in task_data.items(): key = str(key) diff --git a/backend/test-data-copr-backend-1/build_results/00848963-example/results.json b/backend/test-data-copr-backend-1/build_results/00848963-example/results.json new file mode 100644 index 0000000..04ad4c9 --- /dev/null +++ b/backend/test-data-copr-backend-1/build_results/00848963-example/results.json @@ -0,0 +1,11 @@ +{ + "packages": [ + { + "name":"example", + "epoch":0, + "version":"1.0.14", + "release":"1.fc30", + "arch":"x86_64" + } + ] +} diff --git a/backend/tests/test_background_worker_build.py b/backend/tests/test_background_worker_build.py index a1335d9..a6d4626 100644 --- a/backend/tests/test_background_worker_build.py +++ b/backend/tests/test_background_worker_build.py @@ -268,7 +268,8 @@ def test_waiting_for_repo_success(mc_time, f_build_rpm_case_no_repodata, caplog) assert (logging.INFO, MESSAGES["repo_waiting"]) \ in [(r[1], r[2]) for r in caplog.record_tuples] -def test_full_rpm_build_no_sign(f_build_rpm_case, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_full_rpm_build_no_sign(_parse_results, f_build_rpm_case, caplog): """ Go through the whole (successful) build of a binary RPM """ @@ -321,7 +322,8 @@ def test_full_srpm_build(f_build_srpm): @mock.patch("copr_backend.sign.SIGN_BINARY", "tests/fake-bin-sign") @mock.patch("copr_backend.sign._sign_one") -def test_build_and_sign(mc_sign_one, f_build_rpm_sign_on, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_build_and_sign(_parse_results, mc_sign_one, f_build_rpm_sign_on, caplog): config = f_build_rpm_sign_on worker = config.bw worker.process() @@ -570,8 +572,9 @@ def test_cancel_before_start(f_build_rpm_sign_on, caplog): ], caplog) @_patch_bwbuild_object("CANCEL_CHECK_PERIOD", 0.5) +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") @mock.patch("copr_backend.sign.SIGN_BINARY", "tests/fake-bin-sign") -def test_build_retry(f_build_rpm_sign_on): +def test_build_retry(_parse_results, f_build_rpm_sign_on): config = f_build_rpm_sign_on worker = config.bw class _SideEffect(): @@ -680,7 +683,8 @@ def test_cancel_build_during_log_download(f_build_rpm_sign_on, caplog): COMMON_MSGS["not finished"], ], caplog) -def test_ssh_connection_error(f_build_rpm_case, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_ssh_connection_error(_parse_results, f_build_rpm_case, caplog): class _SideEffect: counter = 0 def __call__(self): @@ -707,7 +711,9 @@ def test_average_step(): @_patch_bwbuild_object("time.sleep", mock.MagicMock()) @_patch_bwbuild_object("time.time") -def test_retry_for_ssh_tail_failure(mc_time, f_build_rpm_case, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_retry_for_ssh_tail_failure(_parse_results, mc_time, f_build_rpm_case, + caplog): mc_time.side_effect = list(range(500)) class _SideEffect: counter = 0 @@ -766,7 +772,8 @@ def test_pkg_collect_failure(mc_pkg_evr, f_build_srpm, caplog): ], caplog) assert worker.job.status == 0 # fail -def test_existing_compressed_file(f_build_rpm_case, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_existing_compressed_file(_parse_results, f_build_rpm_case, caplog): config = f_build_rpm_case config.ssh.precreate_compressed_log_file = True worker = config.bw @@ -777,7 +784,8 @@ def test_existing_compressed_file(f_build_rpm_case, caplog): "Finished build: id=848963 failed=False ", # still success! ], caplog) -def test_tail_f_nonzero_exit(f_build_rpm_case, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_tail_f_nonzero_exit(_parse_results, f_build_rpm_case, caplog): config = f_build_rpm_case worker = config.bw class _SideEffect: diff --git a/beaker-tests/Sanity/copr-cli-basic-operations/runtest-build-results-json.sh b/beaker-tests/Sanity/copr-cli-basic-operations/runtest-build-results-json.sh new file mode 100755 index 0000000..29e5efd --- /dev/null +++ b/beaker-tests/Sanity/copr-cli-basic-operations/runtest-build-results-json.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Include Beaker environment +. /usr/bin/rhts-environment.sh || exit 1 +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +# Load config settings +HERE=$(dirname "$(realpath "$0")") +source "$HERE/config" +source "$HERE/helpers" + + +rlJournalStart + rlPhaseStartSetup + setup_checks + RESULTDIR=`mktemp -d` + rlPhaseEnd + + rlPhaseStartTest + rlRun "copr-cli create ${NAME_PREFIX}TestResultsJson --chroot $CHROOT" 0 + rlRun -s "copr-cli build ${NAME_PREFIX}TestResultsJson $HELLO --nowait" 0 + rlRun "parse_build_id" + rlRun "copr watch-build $BUILD_ID" + + checkResults() + { + rlAssertEquals "There should be 4 results" `cat $RESULTDIR/results.json |jq '.packages | length'` 4 + path=$1 + rlRun "cat $path |jq -e '.packages[0].name == \"hello-debugsource\"'" + rlRun "cat $path |jq -e '.packages[0].epoch == 0'" + rlRun "cat $path |jq -e '.packages[0].version == \"2.8\"'" + rlRun "cat $path |jq -e '.packages[0].release == \"1.fc$FEDORA_VERSION\"'" + rlRun "cat $path |jq -e '.packages[0].arch == \"x86_64\"'" + + rlRun "cat $path |jq -e '.packages[1].name == \"hello\"'" + rlRun "cat $path |jq -e '.packages[1].epoch == 0'" + rlRun "cat $path |jq -e '.packages[1].version == \"2.8\"'" + rlRun "cat $path |jq -e '.packages[1].release == \"1.fc$FEDORA_VERSION\"'" + rlRun "cat $path |jq -e '.packages[1].arch == \"x86_64\"'" + + rlRun "cat $path |jq -e '.packages[2].name == \"hello\"'" + rlRun "cat $path |jq -e '.packages[2].epoch == 0'" + rlRun "cat $path |jq -e '.packages[2].version == \"2.8\"'" + rlRun "cat $path |jq -e '.packages[2].release == \"1.fc$FEDORA_VERSION\"'" + rlRun "cat $path |jq -e '.packages[2].arch == \"src\"'" + + rlRun "cat $path |jq -e '.packages[3].name == \"hello-debuginfo\"'" + rlRun "cat $path |jq -e '.packages[3].epoch == 0'" + rlRun "cat $path |jq -e '.packages[3].version == \"2.8\"'" + rlRun "cat $path |jq -e '.packages[3].release == \"1.fc$FEDORA_VERSION\"'" + rlRun "cat $path |jq -e '.packages[3].arch == \"x86_64\"'" + } + + # Check the results.json file that is stored on backend + URL_PATH="results/${NAME_PREFIX}TestResultsJson/$CHROOT/$(build_id_with_leading_zeroes)-hello/results.json" + rlRun "wget -P $RESULTDIR $BACKEND_URL/$URL_PATH" + checkResults "$RESULTDIR/results.json" + + + # Check the /build-chroot/built-packages/ APIv3 route + python << END +from copr.v3 import Client +from copr_cli.util import json_dumps + +client = Client.create_from_config_file() +response = client.build_chroot_proxy.get_built_packages($BUILD_ID, "$CHROOT") +with open("$RESULTDIR/results-api-build-chroot.json", "w") as f: + f.write(json_dumps(response)) +END + checkResults "$RESULTDIR/results-api-build-chroot.json" + + + # Check the /build/built-packages/ APIv3 route + python << END +from copr.v3 import Client +from copr_cli.util import json_dumps + +client = Client.create_from_config_file() +response = client.build_proxy.get_built_packages($BUILD_ID) +with open("$RESULTDIR/results-api-build.json", "w") as f: + f.write(json_dumps(response["$CHROOT"])) +END + checkResults "$RESULTDIR/results-api-build.json" + + + rlPhaseEnd + + rlPhaseStartCleanup + cleanProject "${NAME_PREFIX}TestResultsJson" + rlRun "rm -rf $RESULTDIR/*" + rlPhaseEnd +rlJournalPrintText +rlJournalEnd diff --git a/frontend/coprs_frontend/alembic/versions/efec6b1aa9a2_add_buildchrootresults.py b/frontend/coprs_frontend/alembic/versions/efec6b1aa9a2_add_buildchrootresults.py new file mode 100644 index 0000000..c0c6a4e --- /dev/null +++ b/frontend/coprs_frontend/alembic/versions/efec6b1aa9a2_add_buildchrootresults.py @@ -0,0 +1,33 @@ +""" +Add BuildChrootResults + +Revision ID: efec6b1aa9a2 +Revises: d8a1062ee4cf +Create Date: 2021-05-13 16:48:05.569521 +""" + +import sqlalchemy as sa +from alembic import op + + +revision = 'efec6b1aa9a2' +down_revision = 'd8a1062ee4cf' + +def upgrade(): + op.create_table('build_chroot_result', + + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('build_chroot_id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=False), + sa.Column('epoch', sa.Integer(), default=0), + sa.Column('version', sa.Text(), nullable=False), + sa.Column('release', sa.Text(), nullable=False), + sa.Column('arch', sa.Text(), nullable=False), + + sa.ForeignKeyConstraint(['build_chroot_id'], ['build_chroot.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + +def downgrade(): + op.drop_table('build_chroot_result') diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 94e8fb3..8f9d275 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -965,6 +965,10 @@ class BuildsLogic(object): if upd_dict.get("status") in BuildsLogic.terminal_states: build_chroot.ended_on = upd_dict.get("ended_on") or time.time() + assert isinstance(upd_dict, dict) + assert not isinstance(upd_dict, str) + BuildChrootResultsLogic.create_from_dict( + build_chroot, upd_dict.get("results")) if upd_dict.get("status") == StatusEnum("starting"): build_chroot.started_on = upd_dict.get("started_on") or time.time() @@ -1302,6 +1306,24 @@ class BuildChrootsLogic(object): return query @classmethod + def get_by_results(cls, **kwargs): + """ + Query all `BuildChroot` instances whose `results` corresponds with the + specified `kwargs`. + + Supported parameter names: + + name, epoch, version, release, arch + + Example usage: + + cls.get_by_results(name="hello") + cls.get_by_results(name="foo", arch="x86_64") + """ + return (models.BuildChroot.query + .filter(models.BuildChroot.results.any(**kwargs))) + + @classmethod def filter_by_build_id(cls, query, build_id): return query.filter(models.Build.id == build_id) @@ -1342,6 +1364,39 @@ class BuildChrootsLogic(object): return cls.filter_by_copr_and_mock_chroot(BuildChroot.query, copr, mock_chroot) +class BuildChrootResultsLogic: + """ + High-level interface for working with `models.BuildChrootResult` objects + """ + + @classmethod + def create(cls, build_chroot, name, epoch, version, release, arch): + """ + Create a new record about a built package in some `BuildChroot` + """ + return models.BuildChrootResult( + build_chroot_id=build_chroot, + build_chroot=build_chroot, + name=name, + epoch=epoch, + version=version, + release=release, + arch=arch, + ) + + + @classmethod + def create_from_dict(cls, build_chroot, results): + """ + Parses a `dict` in the following format + + {"packages": [{"name": "foo", "epoch": 0, ...}]} + + and records all of the built packages for a given `BuildChroot`. + """ + return [cls.create(build_chroot, **result) + for result in results["packages"]] + class BuildsMonitorLogic(object): @classmethod diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 0cab10e..ed6a064 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -1464,6 +1464,13 @@ class Build(db.Model, helpers.Serializer): """ return [ch for ch in self.chroots if ch in self.copr.active_chroots] + @property + def results_dict(self): + """ + Built packages in each build chroot. + """ + return {bc.name: bc.results_dict for bc in self.build_chroots} + class DistGitBranch(db.Model, helpers.Serializer): """ @@ -1949,6 +1956,43 @@ class BuildChroot(db.Model, helpers.Serializer): logs.append(log) return logs + @property + def results_dict(self): + """ + Returns a `dict` containing all built packages in this chroot + """ + built_packages = [] + for result in self.results: + options = {"__columns_except__": ["id", "build_chroot_id"]} + result_dict= result.to_dict(options=options) + built_packages.append(result_dict) + return {"packages": built_packages} + + + +class BuildChrootResult(db.Model, helpers.Serializer): + """ + Represents a built package within some `BuildChroot` + """ + + id = db.Column(db.Integer, primary_key=True) + build_chroot_id = db.Column( + db.Integer, + db.ForeignKey("build_chroot.id"), + nullable=False + ) + + name = db.Column(db.Text, nullable=False) + epoch = db.Column(db.Integer, default=0) + version = db.Column(db.Text, nullable=False) + release = db.Column(db.Text, nullable=False) + arch = db.Column(db.Text, nullable=False) + + build_chroot = db.relationship( + "BuildChroot", + backref=db.backref("results", cascade="all, delete-orphan"), + ) + class LegalFlag(db.Model, helpers.Serializer): id = db.Column(db.Integer, primary_key=True) diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py index 203f23a..dc5ec44 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py @@ -64,3 +64,13 @@ def get_build_chroot_list(build_id, **kwargs): def get_build_chroot_config(build_id, chrootname): chroot = ComplexLogic.get_build_chroot(build_id, chrootname) return flask.jsonify(build_config(chroot)) + + +@apiv3_ns.route("/build-chroot/built-packages/", methods=GET) +@query_params() +def get_build_chroot_built_packages(build_id, chrootname): + """ + Return built packages (NEVRA dicts) for a given build chroot + """ + chroot = ComplexLogic.get_build_chroot(build_id, chrootname) + return flask.jsonify(chroot.results_dict) diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_builds.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_builds.py index 03ed223..1700afa 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_builds.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_builds.py @@ -115,6 +115,15 @@ def get_source_build_config(build_id): return flask.jsonify(to_source_build_config(build)) +@apiv3_ns.route("/build/built-packages//", methods=GET) +def get_build_built_packages(build_id): + """ + Return built packages (NEVRA dicts) for a given build + """ + build = ComplexLogic.get_build_safe(build_id) + return flask.jsonify(build.results_dict) + + @apiv3_ns.route("/build/cancel/", methods=PUT) @api_login_required def cancel_build(build_id): diff --git a/frontend/coprs_frontend/tests/test_apiv3/test_build_chroots.py b/frontend/coprs_frontend/tests/test_apiv3/test_build_chroots.py new file mode 100644 index 0000000..a6afa43 --- /dev/null +++ b/frontend/coprs_frontend/tests/test_apiv3/test_build_chroots.py @@ -0,0 +1,45 @@ +""" +Test all kind of build chroot request via APIv3 +""" + +import pytest +from tests.coprs_test_case import CoprsTestCase, TransactionDecorator +from coprs.logic.builds_logic import BuildChrootResultsLogic + + +class TestAPIv3BuildChrootsResults(CoprsTestCase): + """ + Tests related to build chroots results + """ + + @TransactionDecorator("u1") + @pytest.mark.usefixtures("f_users", "f_users_api", "f_coprs", + "f_mock_chroots", "f_builds", "f_db") + def test_build_chroot_built_packages(self): + """ + Test the endpoint for getting built packages (NEVRA dicts) for a given + build chroot. + """ + self.db.session.add(self.b1, self.b1_bc) + built_packages = { + "packages": [ + { + "name": "hello", + "epoch": 0, + "version": "2.8", + "release": "1.fc33", + "arch": "x86_64" + }, + ] + } + BuildChrootResultsLogic.create_from_dict( + self.b1.build_chroots[0], built_packages) + self.db.session.commit() + + endpoint = "/api_3/build-chroot/built-packages/" + endpoint += "?build_id={build_id}&chrootname={chrootname}" + params = {"build_id": self.b1.id, "chrootname": "fedora-18-x86_64"} + + result = self.tc.get(endpoint.format(**params)) + assert result.is_json + assert result.json == built_packages diff --git a/frontend/coprs_frontend/tests/test_apiv3/test_builds.py b/frontend/coprs_frontend/tests/test_apiv3/test_builds.py index 47143c1..2765f5a 100644 --- a/frontend/coprs_frontend/tests/test_apiv3/test_builds.py +++ b/frontend/coprs_frontend/tests/test_apiv3/test_builds.py @@ -9,6 +9,7 @@ import pytest from bs4 import BeautifulSoup from copr_common.enums import BuildSourceEnum +from coprs.logic.builds_logic import BuildChrootResultsLogic from tests.coprs_test_case import CoprsTestCase, TransactionDecorator @@ -238,3 +239,35 @@ class TestWebUIBuilds(CoprsTestCase): resp = self.test_client.get(route) assert get_selected(resp.data)["value"] == "simple" + + +class TestAPIv3BuildsResults(CoprsTestCase): + """ + Tests related to build results + """ + + @TransactionDecorator("u1") + @pytest.mark.usefixtures("f_users", "f_users_api", "f_coprs", + "f_mock_chroots", "f_builds", "f_db") + def test_build_built_packages(self): + """ + Test the endpoint for getting built packages (NEVRA dicts) for a given + build. + """ + self.db.session.add(self.b1, self.b1_bc) + nevra = { + "name": "hello", + "epoch": 0, + "version": "2.8", + "release": "1.fc33", + "arch": "x86_64" + } + built_packages = {"packages": [nevra]} + BuildChrootResultsLogic.create_from_dict( + self.b1.build_chroots[0], built_packages) + self.db.session.commit() + + endpoint = "/api_3/build/built-packages/{0}/".format(self.b1.id) + result = self.tc.get(endpoint) + assert result.is_json + assert result.json["fedora-18-x86_64"] == built_packages diff --git a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py index e7c9248..7a719a5 100644 --- a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py +++ b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py @@ -16,7 +16,11 @@ from coprs.exceptions import (ActionInProgressException, InsufficientStorage) from coprs.logic.actions_logic import ActionsLogic -from coprs.logic.builds_logic import BuildsLogic +from coprs.logic.builds_logic import ( + BuildsLogic, + BuildChrootsLogic, + BuildChrootResultsLogic, +) from tests.coprs_test_case import CoprsTestCase, TransactionDecorator @@ -510,3 +514,48 @@ class TestBuildsLogic(CoprsTestCase): assert len(build.build_chroots) == 1 assert build.source_status == StatusEnum("importing") assert build.package.name == "foo" + + @pytest.mark.usefixtures("f_users", "f_coprs", "f_mock_chroots", "f_builds", "f_db") + def test_build_results_filter(self): + """ + Test that we can query `BuildChroot` instances based on their `results`. + """ + defaults = { + "name": None, + "epoch": 0, + "version": "1.0", + "release": "1", + "arch": None, + } + + b1b2_chroots = self.b1.build_chroots + self.b2.build_chroots + for chroot in b1b2_chroots: + result = defaults | {"name": "foo", "arch": "x86_64"} + results = {"packages": [result]} + BuildChrootResultsLogic.create_from_dict(chroot, results) + + for chroot in self.b3.build_chroots: + result = defaults | {"name": "bar", "arch": "noarch"} + results = {"packages": [result]} + BuildChrootResultsLogic.create_from_dict(chroot, results) + + for chroot in self.b4.build_chroots: + result1 = defaults | {"name": "foobar", "arch": "ppc64le"} + result2 = defaults | {"name": "qux", "arch": "ppc64le"} + results = {"packages": [result1, result2]} + BuildChrootResultsLogic.create_from_dict(chroot, results) + + # Filter results by name + result = BuildChrootsLogic.get_by_results(name="foo").all() + assert set(result) == set(b1b2_chroots) + + # Filter results by multiple attributes + result = BuildChrootsLogic.get_by_results(name="foo", arch="noarch").all() + assert result == [] + + result = BuildChrootsLogic.get_by_results(name="foo", arch="x86_64").all() + assert set(result) == set(b1b2_chroots) + + # Filter results with multiple rows + result = BuildChrootsLogic.get_by_results(name="qux").all() + assert set(result) == set(self.b4.build_chroots) diff --git a/frontend/coprs_frontend/tests/test_views/test_backend_ns/test_backend_general.py b/frontend/coprs_frontend/tests/test_views/test_backend_ns/test_backend_general.py index 193ec0d..40116b1 100644 --- a/frontend/coprs_frontend/tests/test_views/test_backend_ns/test_backend_general.py +++ b/frontend/coprs_frontend/tests/test_views/test_backend_ns/test_backend_general.py @@ -147,6 +147,19 @@ class TestWaitingBuilds(CoprsTestCase): # status = 0 # failure # status = 1 # succeeded class TestUpdateBuilds(CoprsTestCase): + built_packages = """ +{ + "packages":[ + { + "name":"example", + "epoch":0, + "version":"1.0.14", + "release":"1.fc30", + "arch":"x86_64" + } + ] +}""" + data1 = """ { "builds":[ @@ -168,6 +181,17 @@ class TestUpdateBuilds(CoprsTestCase): "status": 1, "chroot": "fedora-18-x86_64", "result_dir": "bar", + "results": { + "packages":[ + { + "name":"example", + "epoch":0, + "version":"1.0.14", + "release":"1.fc30", + "arch":"x86_64" + } + ] + }, "ended_on": 1490866440 } ] @@ -190,6 +214,7 @@ class TestUpdateBuilds(CoprsTestCase): "status": 0, "chroot": "fedora-18-x86_64", "result_dir": "bar", + "results": {"packages": []}, "ended_on": 1390866440 }, { @@ -198,6 +223,7 @@ class TestUpdateBuilds(CoprsTestCase): "status": 0, "chroot": "fedora-18-x86_64", "result_dir": "bar", + "results": {"packages": []}, "ended_on": 1390866440 }, { diff --git a/python/copr/v3/proxies/build.py b/python/copr/v3/proxies/build.py index caf22c7..b6be321 100644 --- a/python/copr/v3/proxies/build.py +++ b/python/copr/v3/proxies/build.py @@ -45,6 +45,18 @@ class BuildProxy(BaseProxy): response = request.send() return munchify(response) + def get_built_packages(self, build_id): + """ + Return built packages (NEVRA dicts) for a given build + + :param int build_id: + :return: Munch + """ + endpoint = "/build/built-packages/{0}".format(build_id) + request = Request(endpoint, api_base_url=self.api_base_url) + response = request.send() + return munchify(response) + def get_list(self, ownername, projectname, packagename=None, status=None, pagination=None): """ Return a list of packages diff --git a/python/copr/v3/proxies/build_chroot.py b/python/copr/v3/proxies/build_chroot.py index 82c4a63..ce9b1a5 100644 --- a/python/copr/v3/proxies/build_chroot.py +++ b/python/copr/v3/proxies/build_chroot.py @@ -58,3 +58,20 @@ class BuildChrootProxy(BaseProxy): request = Request(endpoint, api_base_url=self.api_base_url, params=params) response = request.send() return munchify(response) + + def get_built_packages(self, build_id, chrootname): + """ + Return built packages (NEVRA dicts) for a given build chroot + + :param int build_id: + :param str chrootname: + :return: Munch + """ + endpoint = "/build-chroot/built-packages" + params = { + "build_id": build_id, + "chrootname": chrootname, + } + request = Request(endpoint, api_base_url=self.api_base_url, params=params) + response = request.send() + return munchify(response) diff --git a/rpmbuild/copr-rpmbuild.spec b/rpmbuild/copr-rpmbuild.spec index 7668b56..5e5163d 100644 --- a/rpmbuild/copr-rpmbuild.spec +++ b/rpmbuild/copr-rpmbuild.spec @@ -45,6 +45,7 @@ BuildRequires: %{python}-pytest BuildRequires: %{python_pfx}-munch BuildRequires: %{python}-requests BuildRequires: %{python_pfx}-jinja2 +BuildRequires: %{python_pfx}-simplejson %if 0%{?fedora} || 0%{?rhel} > 7 BuildRequires: argparse-manpage diff --git a/rpmbuild/copr_rpmbuild/automation/__init__.py b/rpmbuild/copr_rpmbuild/automation/__init__.py index 0abb6d3..4d0d29d 100644 --- a/rpmbuild/copr_rpmbuild/automation/__init__.py +++ b/rpmbuild/copr_rpmbuild/automation/__init__.py @@ -4,6 +4,7 @@ This package contains support for running (mainly) static analysis tools such as """ from copr_rpmbuild.automation.fedora_review import FedoraReview +from copr_rpmbuild.automation.rpm_results import RPMResults def run_automation_tools(task, resultdir, mock_config_file, log): @@ -11,7 +12,7 @@ def run_automation_tools(task, resultdir, mock_config_file, log): Iterate over all supported post-build tools (e.g. `fedora-review`, `rpmlint`, etc) and run the desired ones for a given task. """ - tools = [FedoraReview] + tools = [FedoraReview, RPMResults] for _class in tools: tool = _class(task, resultdir, mock_config_file, log) if not tool.enabled: diff --git a/rpmbuild/copr_rpmbuild/automation/rpm_results.py b/rpmbuild/copr_rpmbuild/automation/rpm_results.py new file mode 100644 index 0000000..e21c730 --- /dev/null +++ b/rpmbuild/copr_rpmbuild/automation/rpm_results.py @@ -0,0 +1,75 @@ +""" +Create `results.json` file +""" + +import os +import rpm +import simplejson +from copr_rpmbuild.automation.base import AutomationTool + + +class RPMResults(AutomationTool): + """ + Create `results.json` file containing NEVRAs for all built RPM files + """ + + @property + def enabled(self): + """ + Do this for every build + """ + return True + + def run(self): + """ + Create `results.json` + """ + nevras = self.find_results_nevras_dicts() + packages = {"packages": nevras} + path = os.path.join(self.resultdir, "results.json") + with open(path, "w") as dst: + simplejson.dump(packages, dst, indent=4) + + def find_results_nevras_dicts(self): + """ + Find all RPM packages in the `resultdir` and return their NEVRAs + as `dicts` + """ + nevras = [] + for result in os.listdir(self.resultdir): + if not result.endswith(".rpm"): + continue + package = os.path.join(self.resultdir, result) + nevras.append(self.get_nevra_dict(package)) + return nevras + + @classmethod + def get_nevra_dict(cls, path): + """ + Takes a package path and returns its NEVRA as a `dict` + """ + filename = os.path.basename(path) + if not filename.endswith(".rpm"): + msg = "File name doesn't end with '.rpm': {}".format(path) + raise ValueError(msg) + + hdr = cls.get_rpm_header(path) + arch = "src" if filename.endswith(".src.rpm") else hdr["arch"] + return { + "name": hdr["name"], + "epoch": hdr["epoch"] or 0, + "version": hdr["version"], + "release": hdr["release"], + "arch": arch, + } + + @staticmethod + def get_rpm_header(path): + """ + Examine a RPM package file and return its header + See docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s04.html + """ + ts = rpm.TransactionSet() + with open(path, "r") as f: + hdr = ts.hdrFromFdno(f.fileno()) + return hdr