From 706ac3b1af012d0fab859e6517d1be28f02ce877 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Apr 24 2019 08:20:42 +0000 Subject: Merge branch 'master' of ssh://pagure.io/freshmaker --- diff --git a/Dockerfile b/Dockerfile index e0b8b44..911154a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:28 +FROM fedora:29 LABEL \ name="Freshmaker application" \ vendor="Freshmaker developers" \ @@ -11,7 +11,7 @@ COPY $freshmaker_rpm /tmp RUN cd /etc/yum.repos.d/ \ && dnf -v -y install 'dnf-command(config-manager)' \ - && dnf config-manager --add-repo http://download-ipv4.eng.brq.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-F-28/compose/Everything/x86_64/os/ \ + && dnf config-manager --add-repo http://download-ipv4.eng.brq.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-F-29/compose/Everything/x86_64/os/ \ && dnf -y clean all \ && dnf -v --nogpg -y install \ httpd mod_wsgi mod_auth_gssapi python2-rhmsg mod_ssl \ diff --git a/Jenkinsfile b/Jenkinsfile index e5ab93a..47e84ac 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,19 +42,24 @@ node('master'){ timestamps { -node('fedora-28') { - checkout scm +node('fedora-29') { stage('Prepare') { + checkout scm sh 'sudo rm -f rpmbuild-output/*.src.rpm' sh 'mkdir -p rpmbuild-output' sh 'make -f .copr/Makefile srpm outdir=./rpmbuild-output/' sh 'sudo dnf -y builddep ./rpmbuild-output/freshmaker-*.src.rpm' - sh 'sudo dnf -y install python2-tox python3-tox' - /* Needed to get the latest /etc/mock/fedora-28-x86_64.cfg */ + // TODO: some of the deps here should probably be in BuildRequires + sh 'sudo dnf -y install \ + gcc \ + krb5-devel \ + openldap-devel \ + python3-tox' + /* Needed to get the latest mock configs */ sh 'sudo dnf -y update mock-core-configs' } stage('Run unit tests') { - sh 'tox -e flake8' + sh 'tox' } /* We take a flock on the mock configs, to avoid multiple unrelated jobs on * the same Jenkins slave trying to use the same mock root at the same @@ -69,16 +74,27 @@ node('fedora-28') { """ archiveArtifacts artifacts: 'mock-result/f28/**' }, + 'F29': { + sh """ + mkdir -p mock-result/f29 + flock /etc/mock/fedora-29-x86_64.cfg \ + /usr/bin/mock -v --enable-network --resultdir=mock-result/f29 -r fedora-29-x86_64 --clean --rebuild rpmbuild-output/*.src.rpm + """ + archiveArtifacts artifacts: 'mock-result/f29/**' + }, ) } } +if ("${env.JOB_NAME}" != 'freshmaker-prs') { node('docker') { - checkout scm stage('Build Docker container') { - unarchive mapping: ['mock-result/f28/': '.'] - def f28_rpm = findFiles(glob: 'mock-result/f28/**/*.noarch.rpm')[0] + checkout scm + // Remember to reflect the version change in the Dockerfile in the future. + sh 'grep -q "FROM fedora:29" Dockerfile' + unarchive mapping: ['mock-result/f29/': '.'] + def f29_rpm = findFiles(glob: 'mock-result/f29/**/*.noarch.rpm')[0] def appversion = sh(returnStdout: true, script: """ - rpm2cpio ${f28_rpm} | \ + rpm2cpio ${f29_rpm} | \ cpio --quiet --extract --to-stdout ./usr/lib/python\\*/site-packages/freshmaker\\*.egg-info/PKG-INFO | \ awk '/^Version: / {print \$2}' """).trim() @@ -92,7 +108,7 @@ node('docker') { /* Note that the docker.build step has some magic to guess the * Dockerfile used, which will break if the build directory (here ".") * is not the final argument in the string. */ - def image = docker.build "factory2/freshmaker:internal-${appversion}", "--build-arg freshmaker_rpm=$f28_rpm --build-arg cacert_url=https://password.corp.redhat.com/RH-IT-Root-CA.crt ." + def image = docker.build "factory2/freshmaker:internal-${appversion}", "--build-arg freshmaker_rpm=$f29_rpm --build-arg cacert_url=https://password.corp.redhat.com/RH-IT-Root-CA.crt ." /* Pushes to the internal registry can sometimes randomly fail * with "unknown blob" due to a known issue with the registry * storage configuration. So we retry up to 3 times. */ @@ -104,7 +120,7 @@ node('docker') { docker.withRegistry( 'https://quay.io/', 'quay-io-factory2-builder-sa-credentials') { - def image = docker.build "factory2/freshmaker:${appversion}", "--build-arg freshmaker_rpm=$f28_rpm ." + def image = docker.build "factory2/freshmaker:${appversion}", "--build-arg freshmaker_rpm=$f29_rpm ." image.push() } /* Save container version for later steps (this is ugly but I can't find anything better...) */ @@ -113,9 +129,9 @@ node('docker') { } } node('docker') { - checkout scm if (scmVars.GIT_BRANCH == 'origin/master') { stage('Tag "latest".') { + checkout scm unarchive mapping: ['appversion': 'appversion'] def appversion = readFile('appversion').trim() docker.withRegistry( @@ -138,6 +154,7 @@ node('docker') { } } } +} } // end timestamps } catch (e) { diff --git a/freshmaker/errata.py b/freshmaker/errata.py index e4cb02a..58f5560 100644 --- a/freshmaker/errata.py +++ b/freshmaker/errata.py @@ -33,6 +33,7 @@ from freshmaker.events import ( FreshmakerManualRebuildEvent) from freshmaker import conf, log from freshmaker.bugzilla import BugzillaAPI +from freshmaker.utils import retry class ErrataAdvisory(object): @@ -122,19 +123,19 @@ class Errata(object): xmlrpc_url = self.server_url + '/errata/xmlrpc.cgi' self.xmlrpc = ServerProxy(xmlrpc_url, transport=SafeCookieTransport()) + @retry(wait_on=(requests.exceptions.RequestException,), logger=log) def _errata_authorized_get(self, *args, **kwargs): - r = requests.get( - *args, - auth=HTTPKerberosAuth(principal=conf.krb_auth_principal), - **kwargs) - if r.status_code == 401: - log.info("CCache file expired, removing it.") - os.unlink(conf.krb_auth_ccache_file) + try: r = requests.get( *args, auth=HTTPKerberosAuth(principal=conf.krb_auth_principal), **kwargs) - r.raise_for_status() + r.raise_for_status() + except requests.exceptions.RequestException as e: + if e.response is not None and e.response.status_code == 401: + log.info("CCache file probably expired, removing it.") + os.unlink(conf.krb_auth_ccache_file) + raise return r.json() def _errata_rest_get(self, endpoint): diff --git a/freshmaker/events.py b/freshmaker/events.py index 32fc922..ded8806 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -316,7 +316,8 @@ class ManualRebuildWithAdvisoryEvent(ErrataAdvisoryRPMsSignedEvent): from advisory. """ - def __init__(self, msg_id, advisory, container_images, **kwargs): + def __init__(self, msg_id, advisory, container_images, + requester_metadata_json=None, **kwargs): """ Creates new ManualRebuildWithAdvisoryEvent. @@ -328,6 +329,7 @@ class ManualRebuildWithAdvisoryEvent(ErrataAdvisoryRPMsSignedEvent): super(ManualRebuildWithAdvisoryEvent, self).__init__( msg_id, advisory, **kwargs) self.container_images = container_images + self.requester_metadata_json = requester_metadata_json class BrewSignRPMEvent(BaseEvent): @@ -379,3 +381,32 @@ class FreshmakerManualRebuildEvent(BaseEvent): super(FreshmakerManualRebuildEvent, self).__init__( msg_id, dry_run=dry_run) self.errata_id = errata_id + + +class FreshmakerManageEvent(BaseEvent): + """ + Event triggered by an internal message for managing Freshmaker itself. + """ + _max_tries = 3 + + def __init__(self, msg_body, **kwargs): + super(FreshmakerManageEvent, self).__init__(None, manual=True, **kwargs) + self.body = msg_body + + def __new__(cls, msg_body, *args, **kwargs): + # The intention here is to balance control over retries. We want + # to allow handlers to implement their own logic depending on + # `last_try`, when they *SHALL* return an empty list. But, we also + # want to avoid endless loops and guarantee some higher control. If + # handler(s) don't stop their tries (by returning new events), + # then the unhandleable `None` is returned here as last resort, + # instead of `FreshmakerManageEvent`. + instance = super(FreshmakerManageEvent, cls).__new__(cls) + instance.action = msg_body['action'] + instance.try_count = msg_body['try'] + instance.try_count += 1 + instance.last_try = instance.try_count == FreshmakerManageEvent._max_tries + + if instance.try_count > FreshmakerManageEvent._max_tries: + return None + return instance diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 44855f2..7ea08e3 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -81,37 +81,46 @@ def fail_event_on_handler_exception(func): return decorator -def fail_artifact_build_on_handler_exception(func): +def fail_artifact_build_on_handler_exception(whitelist=None): """ Decorator which marks the models.ArtifactBuild associated with handler by BaseHandler.set_context() as FAILED in case the `func` raises an exception. The exception is re-raised by this decorator once its finished. - """ - @wraps(func) - def decorator(handler, *args, **kwargs): - try: - return func(handler, *args, **kwargs) - except Exception as e: - err = 'Could not process message handler. See the traceback.' - log.exception(err) - - # In case the exception interrupted the database transaction, - # rollback it. - db.session.rollback() - # Mark the event as failed. - build_id = handler.current_db_artifact_build_id - build = db.session.query(ArtifactBuild).filter_by( - id=build_id).first() - if build: - build.transition( - ArtifactBuildState.FAILED.value, "Handling of " - "build failed with traceback: %s" % (str(e))) - db.session.commit() - raise - return decorator + :param list/set whitelist: When set, defines the whitelist of Exception + subclasses which do not cause the ArtifactBuild to fail but are instead + just re-raised. + """ + def wrapper(func): + @wraps(func) + def decorator(handler, *args, **kwargs): + try: + return func(handler, *args, **kwargs) + except Exception as e: + if whitelist and type(e) in whitelist: + raise + + err = 'Could not process message handler. See the traceback.' + log.exception(err) + + # In case the exception interrupted the database transaction, + # rollback it. + db.session.rollback() + + # Mark the event as failed. + build_id = handler.current_db_artifact_build_id + build = db.session.query(ArtifactBuild).filter_by( + id=build_id).first() + if build: + build.transition( + ArtifactBuildState.FAILED.value, "Handling of " + "build failed with traceback: %s" % (str(e))) + db.session.commit() + raise + return decorator + return wrapper class BaseHandler(object): @@ -445,7 +454,7 @@ class ContainerBuildHandler(BaseHandler): scratch=conf.koji_container_scratch_build, compose_ids=compose_ids) - @fail_artifact_build_on_handler_exception + @fail_artifact_build_on_handler_exception(whitelist=[ODCSComposeNotReady]) def build_image_artifact_build(self, build, repo_urls=[]): """ Submits ArtifactBuild of 'image' type to Koji. diff --git a/freshmaker/handlers/internal/__init__.py b/freshmaker/handlers/internal/__init__.py index 17571d6..c20cac8 100644 --- a/freshmaker/handlers/internal/__init__.py +++ b/freshmaker/handlers/internal/__init__.py @@ -23,3 +23,4 @@ from .update_db_on_advisory_change import UpdateDBOnAdvisoryChange # noqa from .update_db_on_module_build import UpdateDBOnModuleBuild # noqa from .generate_advisory_signed_event_on_rpm_sign import GenerateAdvisorySignedEventOnRPMSign # noqa from .update_db_on_odcs_compose_fail import UpdateDBOnODCSComposeFail # noqa +from .cancel_event_on_freshmaker_manage_request import CancelEventOnFreshmakerManageRequest # noqa diff --git a/freshmaker/handlers/internal/cancel_event_on_freshmaker_manage_request.py b/freshmaker/handlers/internal/cancel_event_on_freshmaker_manage_request.py new file mode 100644 index 0000000..e8a2e2c --- /dev/null +++ b/freshmaker/handlers/internal/cancel_event_on_freshmaker_manage_request.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Filip Valder + +from freshmaker import conf, log, db +from freshmaker.models import ArtifactBuild +from freshmaker.handlers import BaseHandler +from freshmaker.events import FreshmakerManageEvent +from freshmaker.kojiservice import koji_service + + +class CancelEventOnFreshmakerManageRequest(BaseHandler): + name = "CancelEventOnFreshmakerManageRequest" + order = 0 + + def can_handle(self, event): + if isinstance(event, FreshmakerManageEvent) and event.action == 'eventcancel': + return True + + return False + + def handle(self, event): + """ + Handle Freshmaker manage request to cancel actions triggered by + event, given by event_id in the event.body. This especially + means to cancel running Koji builds. If some of the builds + couldn't be canceled for some reason, there's ongoing event + containing only those builds (by DB id). + """ + + failed_to_cancel_builds_id = [] + log_fail = log.error if event.last_try else log.warning + with koji_service( + conf.koji_profile, log, dry_run=event.dry_run) as session: + builds = db.session.query(ArtifactBuild).filter( + ArtifactBuild.id.in_(event.body['builds_id'])).all() + for build in builds: + if session.cancel_build(build.build_id): + build.state_reason = 'Build canceled in external build system.' + continue + if event.last_try: + build.state_reason = ('Build was NOT canceled in external build system.' + ' Max number of tries reached!') + failed_to_cancel_builds_id.append(build.id) + db.session.commit() + + if failed_to_cancel_builds_id: + log_fail("Builds which failed to cancel in external build system," + " by DB id: %s; try #%s", + failed_to_cancel_builds_id, event.try_count) + if event.last_try or not failed_to_cancel_builds_id: + return [] + + event.body['builds_id'] = failed_to_cancel_builds_id + return [event] diff --git a/freshmaker/handlers/koji/rebuild_images_on_parent_image_build.py b/freshmaker/handlers/koji/rebuild_images_on_parent_image_build.py index 918488a..4bcaa43 100644 --- a/freshmaker/handlers/koji/rebuild_images_on_parent_image_build.py +++ b/freshmaker/handlers/koji/rebuild_images_on_parent_image_build.py @@ -154,12 +154,12 @@ class RebuildImagesOnParentImageBuild(ContainerBuildHandler): db_event.transition( EventState.COMPLETE, 'Advisory %s: %d of %d container image(s) failed to rebuild.' % ( - db_event.search_key, num_failed, len(db_event.builds),)) + db_event.search_key, num_failed, len(db_event.builds.all()),)) else: db_event.transition( EventState.COMPLETE, 'Advisory %s: All %s container images have been rebuilt.' % ( - db_event.search_key, len(db_event.builds),)) + db_event.search_key, len(db_event.builds.all()),)) def _verify_advisory_rpms_in_container_build(self, errata_id, container_build_id): """ diff --git a/freshmaker/handlers/koji/rebuild_images_on_rpm_advisory_change.py b/freshmaker/handlers/koji/rebuild_images_on_rpm_advisory_change.py index 7c58248..c92b6e4 100644 --- a/freshmaker/handlers/koji/rebuild_images_on_rpm_advisory_change.py +++ b/freshmaker/handlers/koji/rebuild_images_on_rpm_advisory_change.py @@ -124,7 +124,7 @@ class RebuildImagesOnRPMAdvisoryChange(ContainerBuildHandler): db_event.get_image_builds_in_first_batch(db.session)) msg = 'Advisory %s: Rebuilding %d container images.' % ( - db_event.search_key, len(db_event.builds)) + db_event.search_key, len(db_event.builds.all())) db_event.transition(EventState.BUILDING, msg) return [] @@ -140,7 +140,7 @@ class RebuildImagesOnRPMAdvisoryChange(ContainerBuildHandler): batch = 0 printed = [] while (len(printed) != len(builds.values()) or - len(printed) != len(db_event.builds)): + len(printed) != len(db_event.builds.all())): self.log_info(' Batch %d:', batch) old_printed_count = len(printed) for build in builds.values(): @@ -370,13 +370,13 @@ class RebuildImagesOnRPMAdvisoryChange(ContainerBuildHandler): private_key=conf.lightblue_private_key) # Check if we are allowed to rebuild unpublished images and clear - # published and release_category if so. + # published and release_categories if so. if self.event.is_allowed(self, published=True): published = True - release_category = "Generally Available" + release_categories = ("Generally Available", "Tech Preview", "Beta",) else: published = None - release_category = None + release_categories = None # Limit the Lightblue query to particular leaf images if set in Event. leaf_container_images = None @@ -392,6 +392,6 @@ class RebuildImagesOnRPMAdvisoryChange(ContainerBuildHandler): batches = lb.find_images_to_rebuild( srpm_nvrs, content_sets, filter_fnc=self._filter_out_not_allowed_builds, - published=published, release_category=release_category, + published=published, release_categories=release_categories, leaf_container_images=leaf_container_images) return batches diff --git a/freshmaker/kojiservice.py b/freshmaker/kojiservice.py index 41ecffa..2b57366 100644 --- a/freshmaker/kojiservice.py +++ b/freshmaker/kojiservice.py @@ -178,6 +178,9 @@ class KojiService(object): return task_id + def cancel_build(self, build_id): + return self.session.cancelBuild(build_id) + def get_build_rpms(self, build_nvr, arches=None): build_info = self.session.getBuild(build_nvr) return self.session.listRPMs(buildID=build_info['id'], diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 44e01f0..0f2b8f3 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -675,18 +675,15 @@ class LightBlue(object): return images def _set_container_repository_filters( - self, request, published=True, deprecated=False, - release_category="Generally Available"): + self, request, published=True, + release_categories=("Generally Available", "Tech Preview", "Beta",)): """ Sets the additional filters to containerRepository request - based on the self.published, self.deprecated and self.release_category - attributes. + based on the self.published, self.release_categories attributes. :param bool published: whether to limit queries to published repositories - :param bool deprecated: set to True to limit results to deprecated - repositories - :param str release_category: filter only repositories with specific - release category (options: Deprecated, Generally Available, Beta, Tech Preview) + :param tuple release_categories: filter only repositories with specific + release categories (options: Deprecated, Generally Available, Beta, Tech Preview) """ if published is not None: request["query"]["$and"].append({ @@ -695,35 +692,28 @@ class LightBlue(object): "rvalue": published }) - if deprecated is not None: - request["query"]["$and"].append({ - "field": "deprecated", - "op": "=", - "rvalue": deprecated - }) - - if release_category: + if release_categories: # Check if release_categories is None or empty request["query"]["$and"].append({ - "field": "release_categories.*", - "op": "=", - "rvalue": release_category + "$or": [{ + "field": "release_categories.*", + "op": "=", + "rvalue": category + } for category in release_categories] }) return request def find_all_container_repositories( - self, published=True, deprecated=False, - release_category="Generally Available"): + self, published=True, + release_categories=("Generally Available", "Tech Preview", "Beta",)): """ Returns dict with repository name as key and ContainerRepository as value. :param bool published: whether to limit queries to published repositories - :param bool deprecated: set to True to limit results to deprecated - repositories - :param str release_category: filter only repositories with specific - release category (options: Deprecated, Generally Available, Beta, + :param tuple release_categories: filter only repositories with specific + release categories (options: Deprecated, Generally Available, Beta, Tech Preview) :rtype: dict :return: Dict with repository name as key and ContainerRepository as @@ -740,7 +730,7 @@ class LightBlue(object): ] } repo_request = self._set_container_repository_filters( - repo_request, published, deprecated, release_category) + repo_request, published, release_categories) repositories = self.find_container_repositories(repo_request) return {r["repository"]: r for r in repositories} @@ -1215,9 +1205,8 @@ class LightBlue(object): images.append(image) def find_images_with_packages_from_content_set( - self, srpm_nvrs, content_sets, filter_fnc=None, - published=True, deprecated=False, - release_category="Generally Available", + self, srpm_nvrs, content_sets, filter_fnc=None, published=True, + release_categories=("Generally Available", "Tech Preview", "Beta",), leaf_container_images=None): """Query lightblue and find containers which contain given package from one of content sets @@ -1233,10 +1222,8 @@ class LightBlue(object): Freshmaker configuration. :param bool published: whether to limit queries to published repositories - :param bool deprecated: set to True to limit results to deprecated - repositories - :param str release_category: filter only repositories with specific - release category (options: Deprecated, Generally Available, Beta, Tech Preview) + :param str release_categories: filter only repositories with specific + release categories (options: Deprecated, Generally Available, Beta, Tech Preview) :param list leaf_container_images: List of NVRs of leaf images to consider for the rebuild. If not set, all images found in Lightblue will be considered for rebuild. @@ -1248,8 +1235,7 @@ class LightBlue(object): the given image - can be used for comparisons if needed :rtype: list """ - repos = self.find_all_container_repositories( - published, deprecated, release_category) + repos = self.find_all_container_repositories(published, release_categories) if not repos: return [] if not leaf_container_images: @@ -1508,9 +1494,9 @@ class LightBlue(object): return batches def find_images_to_rebuild( - self, srpm_nvrs, content_sets, published=True, deprecated=False, - release_category="Generally Available", filter_fnc=None, - leaf_container_images=None): + self, srpm_nvrs, content_sets, published=True, + release_categories=("Generally Available", "Tech Preview", "Beta",), + filter_fnc=None, leaf_container_images=None): """ Find images to rebuild through image build layers @@ -1525,10 +1511,8 @@ class LightBlue(object): when looking for the packages :param bool published: whether to limit queries to published repositories - :param bool deprecated: set to True to limit results to deprecated - repositories - :param str release_category: filter only repositories with specific - release category (options: Deprecated, Generally Available, Beta, Tech Preview) + :param tuple release_categories: filter only repositories with specific + release categories (options: Deprecated, Generally Available, Beta, Tech Preview) :param function filter_fnc: Function called as filter_fnc(container_image) with container_image being ContainerImage instance. If this function returns True, the image @@ -1541,8 +1525,8 @@ class LightBlue(object): is not respected when `leaf_container_images` are used. """ images = self.find_images_with_packages_from_content_set( - srpm_nvrs, content_sets, filter_fnc, published, deprecated, - release_category, leaf_container_images=leaf_container_images) + srpm_nvrs, content_sets, filter_fnc, published, + release_categories, leaf_container_images=leaf_container_images) srpm_names = [koji.parse_NVR(srpm_nvr)["name"] for srpm_nvr in srpm_nvrs] diff --git a/freshmaker/migrations/versions/5bdd5566615a_.py b/freshmaker/migrations/versions/5bdd5566615a_.py new file mode 100644 index 0000000..c970161 --- /dev/null +++ b/freshmaker/migrations/versions/5bdd5566615a_.py @@ -0,0 +1,22 @@ +"""Add requester_metadata to Events table. + +Revision ID: 5bdd5566615a +Revises: 5a555923da42 +Create Date: 2019-02-25 15:02:13.847086 + +""" + +# revision identifiers, used by Alembic. +revision = '5bdd5566615a' +down_revision = '5a555923da42' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('events', sa.Column('requester_metadata', sa.String(), nullable=True)) + + +def downgrade(): + op.drop_column('events', 'requester_metadata') diff --git a/freshmaker/models.py b/freshmaker/models.py index cec9e93..c709b2c 100644 --- a/freshmaker/models.py +++ b/freshmaker/models.py @@ -145,8 +145,10 @@ class Event(FreshmakerBase): state = db.Column(db.Integer, nullable=False) state_reason = db.Column(db.String, nullable=True) time_created = db.Column(db.DateTime, nullable=True) - # List of builds associated with this Event. - builds = relationship("ArtifactBuild", back_populates="event") + # AppenderQuery for getting builds associated with this Event. + builds = relationship("ArtifactBuild", back_populates="event", + lazy="dynamic", cascade="all, delete-orphan", + passive_deletes=True) # True if the even should be handled in dry run mode. dry_run = db.Column(db.Boolean, default=False) # For manual rebuilds, set to user requesting the rebuild. Otherwise null. @@ -155,6 +157,9 @@ class Event(FreshmakerBase): # (for example NVR of container images) to rebuild if passed using the # REST API. requested_rebuilds = db.Column(db.String, nullable=True) + # For manual rebuilds, contains the serialized JSON optionally submitted + # by the requester to track the context of this event. + requester_metadata = db.Column(db.String, nullable=True) manual_triggered = db.Column( db.Boolean, @@ -201,8 +206,11 @@ class Event(FreshmakerBase): instance = cls.get(session, message_id) if instance: return instance - return cls.create(session, message_id, search_key, event_type, - released=released, manual=manual, dry_run=dry_run) + instance = cls.create( + session, message_id, search_key, event_type, + released=released, manual=manual, dry_run=dry_run) + session.commit() + return instance @classmethod def get_or_create_from_event(cls, session, event, released=True): @@ -282,13 +290,23 @@ class Event(FreshmakerBase): return db.session.query(ArtifactBuild).filter_by( event_id=self.id).filter(state != state).count() == 0 - def builds_transition(self, state, reason): + def builds_transition(self, state, reason, filters=None): """ Calls transition(state, reason) for all builds associated with this event. + + :param dict filters: Filter only specific builds to transition. + :return: list of build ids which were transitioned """ - for build in self.builds: - build.transition(state, reason) + + if not self.builds: + return [] + + builds_to_transition = self.builds.filter_by( + **filters).all() if isinstance(filters, dict) else self.builds + + return [build.id + for build in builds_to_transition if build.transition(state, reason)] def transition(self, state, state_reason=None): """ @@ -296,6 +314,7 @@ class Event(FreshmakerBase): :param state: EventState value :param state_reason: Reason why this state has been set. + :return: True/False, whether state was changed """ # Log the state and state_reason @@ -307,7 +326,7 @@ class Event(FreshmakerBase): self, EventState(state).name, state_reason)) if self.state == state: - return + return False self.state = state if EventState(state).counter: @@ -319,6 +338,8 @@ class Event(FreshmakerBase): messaging.publish('event.state.changed', self.json()) messaging.publish('event.state.changed.min', self.json_min()) + return True + def __repr__(self): return "" % (self.message_id, self.event_type, self.search_key) @@ -329,6 +350,12 @@ class Event(FreshmakerBase): type_name = "UnknownEventType %d" % self.event_type_id return "<%s, search_key=%s>" % (type_name, self.search_key) + @property + def requester_metadata_json(self): + if not self.requester_metadata: + return {} + return json.loads(self.requester_metadata) + def json(self): data = self._common_json() data['builds'] = [b.json() for b in self.builds] @@ -336,7 +363,7 @@ class Event(FreshmakerBase): def json_min(self): builds_summary = defaultdict(int) - builds_summary['total'] = len(self.builds) + builds_summary['total'] = len(self.builds.all()) for build in self.builds: state_name = ArtifactBuildState(build.state).name builds_summary[state_name] += 1 @@ -361,6 +388,7 @@ class Event(FreshmakerBase): "requester": self.requester, "requested_rebuilds": (self.requested_rebuilds.split(" ") if self.requested_rebuilds else []), + "requester_metadata": self.requester_metadata_json, } def find_dependent_events(self): @@ -513,6 +541,7 @@ class ArtifactBuild(FreshmakerBase): :param state: ArtifactBuildState value :param state_reason: Reason why this state has been set. + :return: True/False, whether state was changed """ # Log the state and state_reason @@ -524,7 +553,7 @@ class ArtifactBuild(FreshmakerBase): self, ArtifactBuildState(state).name, state_reason)) if self.state == state: - return + return False self.state = state if ArtifactBuildState(state).counter: @@ -548,6 +577,8 @@ class ArtifactBuild(FreshmakerBase): messaging.publish('build.state.changed', self.json()) + return True + def __repr__(self): return "" % ( self.name, ArtifactType(self.type).name, diff --git a/freshmaker/monitor.py b/freshmaker/monitor.py index 3d93c6c..abaa112 100644 --- a/freshmaker/monitor.py +++ b/freshmaker/monitor.py @@ -118,6 +118,10 @@ freshmaker_event_skipped_counter = Counter( 'freshmaker_event_skipped', 'Number of events, for which no action was taken', registry=registry) +freshmaker_event_canceled_counter = Counter( + 'freshmaker_event_canceled', + 'Number of events canceled during their handling', + registry=registry) freshmaker_build_api_latency = Histogram( 'build_api_latency', diff --git a/freshmaker/parsers/internal/__init__.py b/freshmaker/parsers/internal/__init__.py index 9d44c11..1f08864 100644 --- a/freshmaker/parsers/internal/__init__.py +++ b/freshmaker/parsers/internal/__init__.py @@ -20,3 +20,4 @@ # SOFTWARE. from .manual_rebuild import FreshmakerManualRebuildParser # noqa +from .freshmaker_manage_request import FreshmakerManageRequestParser # noqa diff --git a/freshmaker/parsers/internal/freshmaker_manage_request.py b/freshmaker/parsers/internal/freshmaker_manage_request.py new file mode 100644 index 0000000..49d8bf7 --- /dev/null +++ b/freshmaker/parsers/internal/freshmaker_manage_request.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Filip Valder + +from freshmaker.parsers import BaseParser +from freshmaker.events import FreshmakerManageEvent + + +class FreshmakerManageRequestParser(BaseParser): + """Parser parsing freshmaker.manage.* events""" + + name = "FreshmakerManageRequestParser" + topic_suffixes = ["freshmaker.manage.eventcancel"] + + def can_parse(self, topic, msg): + return any([topic.endswith(s) for s in self.topic_suffixes]) + + def parse(self, topic, msg): + """ + Parse message and call specific method according to the action + defined within the message. + """ + action_from_topic = topic.split('.')[-1] + inner_msg = msg.get('msg') + + if 'action' not in inner_msg: + raise ValueError("Action is not defined within the message.") + + if inner_msg['action'] != action_from_topic: + raise ValueError("Last part of 'Freshmaker manage' message topic" + " must match the action defined within the message.") + + if 'try' not in inner_msg: + inner_msg['try'] = 0 + + try: + getattr(self, action_from_topic)(inner_msg) + except AttributeError: + raise NotImplementedError("The message contains unsupported action.") + + return FreshmakerManageEvent(inner_msg) + + def eventcancel(self, inner_msg): + """ + Parse message for event cancelation request + """ + try: + inner_msg['event_id'] + inner_msg['builds_id'] + except KeyError: + raise ValueError("Message doesn't contain all required information.") + + return True diff --git a/freshmaker/parsers/internal/manual_rebuild.py b/freshmaker/parsers/internal/manual_rebuild.py index 00bc5d0..69e5aaf 100644 --- a/freshmaker/parsers/internal/manual_rebuild.py +++ b/freshmaker/parsers/internal/manual_rebuild.py @@ -51,8 +51,8 @@ class FreshmakerManualRebuildParser(BaseParser): advisory = ErrataAdvisory.from_advisory_id(errata, errata_id) event = ManualRebuildWithAdvisoryEvent( - msg_id, advisory, data.get("container_images", []), manual=True, - dry_run=dry_run) + msg_id, advisory, data.get("container_images", []), data.get("metadata", None), + manual=True, dry_run=dry_run) return event diff --git a/freshmaker/pulp.py b/freshmaker/pulp.py index 1d89b05..c111754 100644 --- a/freshmaker/pulp.py +++ b/freshmaker/pulp.py @@ -62,7 +62,7 @@ class Pulp(object): 'filters': { 'id': {'$in': repo_ids}, }, - 'fields': ['notes.content_set'], + 'fields': ['notes'], } } repos = self._rest_post('repositories/search/', json.dumps(query_data)) diff --git a/freshmaker/types.py b/freshmaker/types.py index d3e041d..bcb5fbd 100644 --- a/freshmaker/types.py +++ b/freshmaker/types.py @@ -25,7 +25,7 @@ from freshmaker.monitor import ( freshmaker_artifact_build_failed_counter, freshmaker_artifact_build_canceled_counter, freshmaker_event_complete_counter, freshmaker_event_failed_counter, - freshmaker_event_skipped_counter) + freshmaker_event_skipped_counter, freshmaker_event_canceled_counter) class ArtifactType(Enum): @@ -70,7 +70,8 @@ class EventState(Enum): None, freshmaker_event_complete_counter, freshmaker_event_failed_counter, - freshmaker_event_skipped_counter + freshmaker_event_skipped_counter, + freshmaker_event_canceled_counter ] if type(value) == int: @@ -87,3 +88,5 @@ class EventState(Enum): FAILED = 3 # no action to take upon the event SKIPPED = 4 + # handling of the event has been canceled (also canceling builds etc.) + CANCELED = 5 diff --git a/freshmaker/views.py b/freshmaker/views.py index 3b9cc83..fbd39ec 100644 --- a/freshmaker/views.py +++ b/freshmaker/views.py @@ -21,6 +21,7 @@ # # Written by Jan Kaluza +import json import six from flask import request, jsonify from flask.views import MethodView @@ -33,6 +34,7 @@ from freshmaker import types from freshmaker import db from freshmaker import conf from freshmaker import version +from freshmaker import log from freshmaker.api_utils import filter_artifact_builds from freshmaker.api_utils import filter_events from freshmaker.api_utils import json_error @@ -42,6 +44,7 @@ from freshmaker.parsers.internal.manual_rebuild import FreshmakerManualRebuildPa from freshmaker.monitor import ( monitor_api, freshmaker_build_api_latency, freshmaker_event_api_latency) from freshmaker.image_verifier import ImageVerifier +from freshmaker.types import ArtifactBuildState, EventState api_v1 = { 'event_types': { @@ -100,7 +103,7 @@ api_v1 = { 'event': { 'url': '/api/1/events/', 'options': { - 'methods': ['GET'], + 'methods': ['GET', 'PATCH'], } }, }, @@ -216,6 +219,9 @@ class BuildStateAPI(MethodView): class EventAPI(MethodView): + + _freshmaker_manage_prefix = 'event' + @freshmaker_event_api_latency.time() def get(self, id): if id is None: @@ -235,6 +241,50 @@ class EventAPI(MethodView): else: return json_error(404, "Not Found", "No such event found.") + @login_required + @requires_role('admins') + def patch(self, id): + """ + Manage event + + Accepts JSON with following key/value pairs: + - "action" - one of currently supported actions: 'cancel' + """ + data = request.get_json(force=True) + if 'action' not in data: + return json_error( + 400, "Bad Request", "Missing action in request." + " Don't know what to do with the event.") + + if data['action'] == 'cancel': + event = models.Event.query.filter_by(id=id).first() + if not event: + return json_error(400, "Not Found", "No such event found.") + + msg = "Event id %s requested for canceling by user %s" % \ + (event.id, g.user.username) + log.info(msg) + + event.transition(EventState.CANCELED, msg) + event.builds_transition( + ArtifactBuildState.CANCELED.value, + "Build canceled before running on external build system.", + filters={'state': ArtifactBuildState.PLANNED.value}) + builds_id = event.builds_transition( + ArtifactBuildState.CANCELED.value, None, + filters={'state': ArtifactBuildState.BUILD.value}) + db.session.commit() + + data["action"] = self._freshmaker_manage_prefix + data["action"] + data["event_id"] = event.id + data["builds_id"] = builds_id + messaging.publish("manage.eventcancel", data) + # Return back the JSON representation of Event to client. + return jsonify(event.json()), 200 + else: + return json_error( + 400, "Bad Request", "Unsupported action requested.") + class BuildAPI(MethodView): @freshmaker_build_api_latency.time() @@ -285,6 +335,8 @@ class BuildAPI(MethodView): db_event = models.Event.get_or_create_from_event(db.session, event) db_event.requester = g.user.username db_event.requested_rebuilds = " ".join(event.container_images) + if event.requester_metadata_json: + db_event.requester_metadata = json.dumps(event.requester_metadata_json) db.session.commit() # Forward the POST data (including the msg_id of the database event we diff --git a/tests/fedmsgs/freshmaker_manage_eventcancel b/tests/fedmsgs/freshmaker_manage_eventcancel new file mode 100644 index 0000000..b121175 --- /dev/null +++ b/tests/fedmsgs/freshmaker_manage_eventcancel @@ -0,0 +1,18 @@ +{ + "i": 85, + "source_version": "0.1.1", + "username": "freshmaker", + "signature": "123", + "timestamp": 1552038054.3143399, + "topic": "org.fedoraproject.prod.freshmaker.manage.eventcancel", + "source_name": "unittest", + "msg": { + "action": "eventcancel", + "event_id": 3, + "builds_id": [ + 1, + 2 + ] + }, + "msg_id": "2019-4d21d4c1-22b0-4182-a680-8de1b1374e0c" +} diff --git a/tests/fedmsgs/freshmaker_manage_mismatched_action b/tests/fedmsgs/freshmaker_manage_mismatched_action new file mode 100644 index 0000000..44f62ef --- /dev/null +++ b/tests/fedmsgs/freshmaker_manage_mismatched_action @@ -0,0 +1,13 @@ +{ + "i": 86, + "source_version": "0.1.1", + "username": "freshmaker", + "signature": "124", + "timestamp": 1552038054.3143400, + "topic": "org.fedoraproject.prod.freshmaker.manage.eventcancel", + "source_name": "unittest", + "msg": { + "action": "unsupported" + }, + "msg_id": "2019-4483a0ba-1234-46a1-be7e-d2bf4104ed46" +} diff --git a/tests/fedmsgs/freshmaker_manage_missing_action b/tests/fedmsgs/freshmaker_manage_missing_action new file mode 100644 index 0000000..d7fab76 --- /dev/null +++ b/tests/fedmsgs/freshmaker_manage_missing_action @@ -0,0 +1,11 @@ +{ + "i": 86, + "source_version": "0.1.1", + "username": "freshmaker", + "signature": "124", + "timestamp": 1552038054.3143400, + "topic": "org.fedoraproject.prod.freshmaker.manage.eventcancel", + "source_name": "unittest", + "msg": {}, + "msg_id": "2019-4483a0ba-1234-46a1-be7e-d2bf4104ed46" +} diff --git a/tests/handlers/internal/test_freshmaker_manage_request.py b/tests/handlers/internal/test_freshmaker_manage_request.py new file mode 100644 index 0000000..c117036 --- /dev/null +++ b/tests/handlers/internal/test_freshmaker_manage_request.py @@ -0,0 +1,133 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Filip Valder + +import unittest + +from mock import patch + +from freshmaker import events, models, db +from freshmaker.handlers.internal import CancelEventOnFreshmakerManageRequest +from freshmaker.parsers.internal import FreshmakerManageRequestParser +from freshmaker.types import ArtifactBuildState + +from tests import helpers, get_fedmsg + + +class ErroneousFreshmakerManageRequestsTest(helpers.ModelsTestCase): + def setUp(self): + super(ErroneousFreshmakerManageRequestsTest, self).setUp() + events.BaseEvent.register_parser(FreshmakerManageRequestParser) + + def test_freshmaker_manage_mismatched_action(self): + msg = get_fedmsg('freshmaker_manage_mismatched_action') + with self.assertRaises(ValueError) as err: + self.get_event_from_msg(msg) + self.assertEqual( + err.exception.args[0], 'Last part of \'Freshmaker manage\' message' + ' topic must match the action defined within the message.') + + def test_freshmaker_manage_missing_action(self): + msg = get_fedmsg('freshmaker_manage_missing_action') + with self.assertRaises(ValueError) as err: + self.get_event_from_msg(msg) + self.assertEqual( + err.exception.args[0], 'Action is not defined within the message.') + + def test_more_than_max_tries_on_freshmaker_manage_request(self): + msg = get_fedmsg('freshmaker_manage_eventcancel') + msg['body']['msg']['try'] = events.FreshmakerManageEvent._max_tries + event = self.get_event_from_msg(msg) + + handler = CancelEventOnFreshmakerManageRequest() + self.assertFalse(handler.can_handle(event)) + + +class CancelEventOnFreshmakerManageRequestTest(helpers.ModelsTestCase): + def setUp(self): + super(CancelEventOnFreshmakerManageRequestTest, self).setUp() + events.BaseEvent.register_parser(FreshmakerManageRequestParser) + + self.db_event = models.Event.create( + db.session, "2017-00000000-0000-0000-0000-000000000003", "RHSA-2018-103", + events.TestingEvent) + models.ArtifactBuild.create( + db.session, self.db_event, "mksh", "module", build_id=1237, + state=ArtifactBuildState.CANCELED.value) + models.ArtifactBuild.create( + db.session, self.db_event, "bash", "module", build_id=1238, + state=ArtifactBuildState.CANCELED.value) + + @patch('freshmaker.kojiservice.KojiService.cancel_build') + def test_cancel_event_on_freshmaker_manage_request(self, mocked_cancel_build): + msg = get_fedmsg('freshmaker_manage_eventcancel') + event = self.get_event_from_msg(msg) + + handler = CancelEventOnFreshmakerManageRequest() + self.assertTrue(handler.can_handle(event)) + retval = handler.handle(event) + self.assertEqual(retval, []) + + mocked_cancel_build.assert_any_call(1237) + mocked_cancel_build.assert_any_call(1238) + self.assertEqual([b.state_reason for b in self.db_event.builds.all()].count( + "Build canceled in external build system."), 2) + + def test_can_not_handle_other_action_than_eventcancel(self): + msg = get_fedmsg('freshmaker_manage_eventcancel') + msg['body']['topic'] = 'freshmaker.manage.someotheraction' + msg['body']['msg']['action'] = 'someotheraction' + event = self.get_event_from_msg(msg) + + handler = CancelEventOnFreshmakerManageRequest() + self.assertFalse(handler.can_handle(event)) + + @patch('freshmaker.kojiservice.KojiService.cancel_build', side_effect=[False, False]) + def test_max_tries_reached_on_cancel_event(self, mocked_cancel_build): + msg = get_fedmsg('freshmaker_manage_eventcancel') + msg['body']['msg']['try'] = events.FreshmakerManageEvent._max_tries - 1 + event = self.get_event_from_msg(msg) + + handler = CancelEventOnFreshmakerManageRequest() + retval = handler.handle(event) + self.assertEqual(retval, []) + + self.assertEqual([b.state_reason for b in self.db_event.builds.all()].count( + "Build was NOT canceled in external build system. Max number of tries reached!"), 2) + + @patch('freshmaker.kojiservice.KojiService.cancel_build', + side_effect=[False, False, True, True]) + def test_retry_failed_cancel_event_with_success(self, mocked_cancel_build): + msg = get_fedmsg('freshmaker_manage_eventcancel') + event = self.get_event_from_msg(msg) + + handler = CancelEventOnFreshmakerManageRequest() + new_event = handler.handle(event) + self.assertTrue(isinstance(new_event, list) and len(new_event)) + retval = handler.handle(new_event[0]) + self.assertEqual(retval, []) + + self.assertEqual([b.state_reason for b in self.db_event.builds.all()].count( + "Build canceled in external build system."), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/handlers/koji/test_rebuild_images_on_rpm_advisory_change.py b/tests/handlers/koji/test_rebuild_images_on_rpm_advisory_change.py index c090acc..6efefa2 100644 --- a/tests/handlers/koji/test_rebuild_images_on_rpm_advisory_change.py +++ b/tests/handlers/koji/test_rebuild_images_on_rpm_advisory_change.py @@ -439,7 +439,7 @@ class TestFindImagesToRebuild(helpers.FreshmakerTestCase): self.find_images_to_rebuild.assert_called_once_with( set(['httpd-2.4-11.el7']), ['content-set-1'], filter_fnc=self.handler._filter_out_not_allowed_builds, - published=True, release_category='Generally Available', + published=True, release_categories=('Generally Available', 'Tech Preview', 'Beta'), leaf_container_images=None) @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ @@ -456,7 +456,7 @@ class TestFindImagesToRebuild(helpers.FreshmakerTestCase): self.find_images_to_rebuild.assert_called_once_with( set(['httpd-2.4-11.el7', 'httpd-2.2-11.el6']), ['content-set-1'], filter_fnc=self.handler._filter_out_not_allowed_builds, - published=True, release_category='Generally Available', + published=True, release_categories=('Generally Available', 'Tech Preview', 'Beta'), leaf_container_images=None) @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ @@ -475,7 +475,7 @@ class TestFindImagesToRebuild(helpers.FreshmakerTestCase): self.find_images_to_rebuild.assert_called_once_with( set(['httpd-2.4-11.el7']), ['content-set-1'], filter_fnc=self.handler._filter_out_not_allowed_builds, - published=None, release_category=None, + published=None, release_categories=None, leaf_container_images=None) @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ @@ -492,7 +492,7 @@ class TestFindImagesToRebuild(helpers.FreshmakerTestCase): self.find_images_to_rebuild.assert_called_once_with( set(['httpd-2.4-11.el7']), ['content-set-1'], filter_fnc=self.handler._filter_out_not_allowed_builds, - published=True, release_category='Generally Available', + published=True, release_categories=('Generally Available', 'Tech Preview', 'Beta'), leaf_container_images=None) @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ @@ -510,5 +510,5 @@ class TestFindImagesToRebuild(helpers.FreshmakerTestCase): self.find_images_to_rebuild.assert_called_once_with( set(['httpd-2.4-11.el7']), ['content-set-1'], filter_fnc=self.handler._filter_out_not_allowed_builds, - published=True, release_category='Generally Available', + published=True, release_categories=('Generally Available', 'Tech Preview', 'Beta'), leaf_container_images=["foo", "bar"]) diff --git a/tests/test_errata.py b/tests/test_errata.py index 57c1424..e805b6a 100644 --- a/tests/test_errata.py +++ b/tests/test_errata.py @@ -20,7 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from mock import patch +from mock import patch, MagicMock +from requests_kerberos.exceptions import MutualAuthenticationError +from requests.exceptions import HTTPError from freshmaker.errata import Errata, ErrataAdvisory from freshmaker.events import ( @@ -352,3 +354,49 @@ class TestErrata(helpers.FreshmakerTestCase): xmlrpc.get_advisory_cdn_docker_file_list.return_value = None repo_tags = self.errata.get_docker_repo_tags(28484) self.assertEqual(repo_tags, None) + + +class TestErrataAuthorizedGet(helpers.FreshmakerTestCase): + def setUp(self): + super(TestErrataAuthorizedGet, self).setUp() + self.errata = Errata("https://localhost/") + + self.patcher = helpers.Patcher( + 'freshmaker.errata.') + self.requests_get = self.patcher.patch("requests.get") + self.response = MagicMock() + self.response.json.return_value = {"foo": "bar"} + self.unlink = self.patcher.patch("os.unlink") + + def tearDown(self): + super(TestErrataAuthorizedGet, self).tearDown() + self.patcher.unpatch_all() + + def test_errata_authorized_get(self): + self.requests_get.return_value = self.response + data = self.errata._errata_authorized_get("http://localhost/test") + self.assertEqual(data, {"foo": "bar"}) + + def test_errata_authorized_get_kerberos_exception(self): + # Test that MutualAuthenticationError is retried. + self.requests_get.side_effect = [MutualAuthenticationError, self.response] + + data = self.errata._errata_authorized_get("http://localhost/test") + + self.assertEqual(data, {"foo": "bar"}) + self.assertEqual(len(self.requests_get.mock_calls), 2) + + def test_errata_authorized_get_kerberos_exception_401(self): + # Test that 401 error response is retried with kerberos ccache file + # removed. + error_response = MagicMock() + error_response.status_code = 401 + error_response.raise_for_status.side_effect = HTTPError( + "Expected exception", response=error_response) + self.requests_get.side_effect = [error_response, self.response] + + data = self.errata._errata_authorized_get("http://localhost/test") + + self.assertEqual(data, {"foo": "bar"}) + self.assertEqual(len(self.requests_get.mock_calls), 2) + self.unlink.assert_called_once_with(helpers.AnyStringWith("freshmaker_cc")) diff --git a/tests/test_handler.py b/tests/test_handler.py index 66cfdfb..6d7be04 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -243,6 +243,8 @@ class TestGetRepoURLs(helpers.ModelsTestCase): handler = MyHandler() handler.build_image_artifact_build(self.build_1, ["http://localhost/x.repo"]) + self.assertEqual(self.build_1.state, ArtifactBuildState.PLANNED.value) + class TestAllowBuildBasedOnWhitelist(helpers.FreshmakerTestCase): """Test BaseHandler.allow_build""" diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index 5124152..d47f34a 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -923,15 +923,11 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): "rvalue": True }, { - "field": "deprecated", - "op": "=", - "rvalue": False + "$or": [ + {"field": "release_categories.*", "rvalue": "Generally Available", "op": "="}, + {"field": "release_categories.*", "rvalue": "Tech Preview", "op": "="}, + {"field": "release_categories.*", "rvalue": "Beta", "op": "="}] }, - { - "field": "release_categories.*", - "op": "=", - "rvalue": "Generally Available" - } ] }, "projection": [ diff --git a/tests/test_models.py b/tests/test_models.py index 5d2194a..c6d089b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -31,6 +31,14 @@ from tests import helpers class TestModels(helpers.ModelsTestCase): + def test_get_or_create_from_event(self): + event = events.TestingEvent('msg-1') + # First call creates new event, second call returns the same one. + for i in range(2): + db_event = Event.get_or_create_from_event(db.session, event) + self.assertEqual(db_event.id, 1) + self.assertEqual(db_event.message_id, 'msg-1') + def test_creating_event_and_builds(self): event = Event.create(db.session, "test_msg_id", "RHSA-2017-284", events.TestingEvent) build = ArtifactBuild.create(db.session, event, "ed", "module", 1234) @@ -42,7 +50,7 @@ class TestModels(helpers.ModelsTestCase): self.assertEqual(e.message_id, "test_msg_id") self.assertEqual(e.search_key, "RHSA-2017-284") self.assertEqual(e.event_type, events.TestingEvent) - self.assertEqual(len(e.builds), 2) + self.assertEqual(len(e.builds.all()), 2) self.assertEqual(e.builds[0].name, "ed") self.assertEqual(e.builds[0].type, 2) @@ -176,6 +184,7 @@ class TestModels(helpers.ModelsTestCase): 'state_reason': None, 'url': 'http://localhost:5001/api/1/events/1', 'requested_rebuilds': [], + 'requester_metadata': {}, }) diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 04a1436..e87169e 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -29,7 +29,7 @@ from six.moves import reload_module from freshmaker import app, db, events, models, login_manager from tests import helpers -num_of_metrics = 44 +num_of_metrics = 46 @login_manager.user_loader diff --git a/tests/test_pulp.py b/tests/test_pulp.py index 205d911..cc62829 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -90,7 +90,7 @@ class TestPulp(helpers.FreshmakerTestCase): 'filters': { 'id': {'$in': repo_ids}, }, - 'fields': ['notes.content_set'], + 'fields': ['notes'], } }), auth=(self.username, self.password)) @@ -151,7 +151,7 @@ class TestPulp(helpers.FreshmakerTestCase): 'filters': { 'id': {'$in': repo_ids}, }, - 'fields': ['notes.content_set'], + 'fields': ['notes'], } }), auth=(self.username, self.password)) diff --git a/tests/test_views.py b/tests/test_views.py index b4bc55b..3d2fdf6 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -350,6 +350,48 @@ class TestViews(helpers.ModelsTestCase): for id, build in zip([2, 1], evs): self.assertEqual(id, build['id']) + def test_patch_event_missing_action(self): + resp = self.client.patch( + '/api/1/events/1', + data=json.dumps({})) + data = json.loads(resp.get_data(as_text=True)) + self.assertEqual(data['error'], 'Bad Request') + self.assertTrue(data['message'].startswith('Missing action in request.')) + + def test_patch_event_unsupported_action(self): + resp = self.client.patch( + '/api/1/events/1', + data=json.dumps({'action': 'unsupported'})) + data = json.loads(resp.get_data(as_text=True)) + self.assertEqual(data['error'], 'Bad Request') + self.assertTrue(data['message'].startswith('Unsupported action requested.')) + + def test_patch_event_cancel(self): + event = models.Event.create(db.session, "2017-00000000-0000-0000-0000-000000000003", + "RHSA-2018-103", events.TestingEvent) + models.ArtifactBuild.create(db.session, event, "mksh", "module", build_id=1237, + state=ArtifactBuildState.PLANNED.value) + models.ArtifactBuild.create(db.session, event, "bash", "module", build_id=1238, + state=ArtifactBuildState.PLANNED.value) + models.ArtifactBuild.create(db.session, event, "dash", "module", build_id=1239, + state=ArtifactBuildState.BUILD.value) + models.ArtifactBuild.create(db.session, event, "tcsh", "module", build_id=1240, + state=ArtifactBuildState.DONE.value) + db.session.commit() + + resp = self.client.patch( + '/api/1/events/{}'.format(event.id), + data=json.dumps({'action': 'cancel'})) + data = json.loads(resp.get_data(as_text=True)) + + self.assertEqual(data['id'], event.id) + self.assertEqual(len(data['builds']), 4) + self.assertEqual(data['state_name'], 'CANCELED') + self.assertTrue(data['state_reason'].startswith( + 'Event id {} requested for canceling by user '.format(event.id))) + self.assertEqual(len([b for b in data['builds'] if b['state_name'] == 'CANCELED']), 3) + self.assertEqual(len([b for b in data['builds'] if b['state_name'] == 'DONE']), 1) + def test_query_event_types(self): resp = self.client.get('/api/1/event-types/') event_types = json.loads(resp.get_data(as_text=True))['items'] @@ -507,7 +549,8 @@ class TestManualTriggerRebuild(helpers.ModelsTestCase): u'url': u'/api/1/events/1', u'dry_run': False, u'requester': 'tester1', - u'requested_rebuilds': []}) + u'requested_rebuilds': [], + u'requester_metadata': {}}) publish.assert_called_once_with( 'manual.rebuild', {'msg_id': 'manual_rebuild_123', u'errata_id': 1}) @@ -554,6 +597,28 @@ class TestManualTriggerRebuild(helpers.ModelsTestCase): {'msg_id': 'manual_rebuild_123', u'errata_id': 1, 'container_images': ["foo-1-1", "bar-1-1"]}) + @patch('freshmaker.messaging.publish') + @patch('freshmaker.parsers.internal.manual_rebuild.ErrataAdvisory.' + 'from_advisory_id') + @patch('freshmaker.parsers.internal.manual_rebuild.time.time') + def test_manual_rebuild_metadata(self, time, from_advisory_id, publish): + time.return_value = 123 + from_advisory_id.return_value = ErrataAdvisory( + 123, 'name', 'REL_PREP', ['rpm']) + + resp = self.client.post( + '/api/1/builds/', data=json.dumps({ + 'errata_id': 1, 'metadata': {"foo": ["bar"]}}), + content_type='application/json') + data = json.loads(resp.get_data(as_text=True)) + + # Other fields are predictible. + self.assertEqual(data['requester_metadata'], {"foo": ["bar"]}) + publish.assert_called_once_with( + 'manual.rebuild', + {'msg_id': 'manual_rebuild_123', u'errata_id': 1, + 'metadata': {"foo": ["bar"]}}) + class TestOpenIDCLogin(ViewBaseTest): """Test that OpenIDC login""" diff --git a/tox.ini b/tox.ini index 91b90ff..d6e5f69 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ commands = -W "ignore:You do not have a working installation:UserWarning" \ -W "ignore:inspect.getargspec:DeprecationWarning" \ -W "ignore:This method will be removed in future versions. Use 'parser.read_file()':DeprecationWarning" \ + -W "ignore:Use .persist_selectable:DeprecationWarning" \ {posargs} [testenv:flake8]