PR#41: Add event type and search_key to Event table. They should be used to query for particular events using the JSON API. - freshmaker

freshmaker

#41 Add event type and search_key to Event table. They should be used to query for particular events using the JSON API.

Merged 8 years ago by jkaluza. Opened 8 years ago by jkaluza.

jkaluza/freshmaker searchable-events into master

Add event type and search_key to Event table. They should be used to query for particular events using the JSON API.

Jan Kaluza • 8 years ago

a2e55da

freshmaker/events.py

file modified

+32

		`@@ -121,6 +121,14 @@`

		`return None`

		`+ @property`
		`+ def search_key(self):`
		`+ """`
		`+ Returns the searchable key which is used to query for particular`
		`+ events using the JSON API.`
		`+ """`
		`+ return self.msg_id`
		`+`

		`class MBSModuleStateChangeEvent(BaseEvent):`
		`""" A class that inherits from BaseEvent to provide an event`
		`@@ -136,6 +144,10 @@`
		`self.build_id = build_id`
		`self.build_state = build_state`

		`+ @property`
		`+ def search_key(self):`
		`+ return str(self.build_id)`
		`+`

		`class GitModuleMetadataChangeEvent(BaseEvent):`
		`"""`
		`@@ -149,6 +161,10 @@`
		`self.branch = branch`
		`self.rev = rev`

		`+ @property`
		`+ def search_key(self):`
		`+ return "%s/%s?#%s" % (self.module, self.branch, self.rev)`
		`+`

		`class GitRPMSpecChangeEvent(BaseEvent):`
		`"""`
		`@@ -164,6 +180,10 @@`
		`self.branch = branch`
		`self.rev = rev`

		`+ @property`
		`+ def search_key(self):`
		`+ return "%s/%s?#%s" % (self.rpm, self.branch, self.rev)`
		`+`

		`class TestingEvent(BaseEvent):`
		`"""`
		`@@ -182,6 +202,10 @@`
		`self.branch = branch`
		`self.rev = rev`

		`+ @property`
		`+ def search_key(self):`
		`+ return "%s/%s?#%s" % (self.container, self.branch, self.rev)`
		`+`

		`class BodhiUpdateCompleteStableEvent(BaseEvent):`
		`"""Event when RPMs are available in Fedora master mirrors`
		`@@ -210,6 +234,10 @@`
		`self.builds = builds`
		`self.release = release`

		`+ @property`
		`+ def search_key(self):`
		`+ return str(self.update_id)`
		`+`

		`class KojiTaskStateChangeEvent(BaseEvent):`
		`"""`
		`@@ -228,3 +256,7 @@`
		`def __init__(self, msg_id, nvr):`
		`super(BrewRPMSignEvent, self).__init__(msg_id)`
		`self.nvr = nvr`
		`+`
		`+ @property`
		`+ def search_key(self):`
		`+ return str(self.task_id)`

freshmaker/handlers/__init__.py

file modified

+2 -1

		`@@ -124,7 +124,8 @@`
		`:param build_id: id of the build in build system.`
		`:param def_of: the artifact which this one depends on.`
		`"""`
		`- ev = models.Event.get_or_create(db.session, event.msg_id)`
		`+ ev = models.Event.get_or_create(db.session, event.msg_id,`
		`+ event.search_key, event.__class__)`
		`models.ArtifactBuild.create(db.session, ev, name, type, build_id, dep_of)`
		`db.session.commit()`

freshmaker/migrations/versions/85906960545_event_search_key_and_type.py

file added

+28

		`@@ -0,0 +1,28 @@`
		`+ """Add search_key and event_type_id to events`
		`+`
		`+ Revision ID: 85906960545`
		`+ Revises: 1529069af28e`
		`+ Create Date: 2017-06-01 08:09:30.207251`
		`+`
		`+ """`
		`+`
		`+ # revision identifiers, used by Alembic.`
		`+ revision = '85906960545'`
		`+ down_revision = '1529069af28e'`
		`+`
		`+ from alembic import op`
		`+ import sqlalchemy as sa`
		`+`
		`+`
		`+ def upgrade():`
		`+ ### commands auto generated by Alembic - please adjust! ###`
		`+ op.add_column('events', sa.Column('event_type_id', sa.Integer(), nullable=False))`
		`+ op.add_column('events', sa.Column('search_key', sa.String(), nullable=False))`
		`+ ### end Alembic commands ###`
		`+`
		`+`
		`+ def downgrade():`
		`+ ### commands auto generated by Alembic - please adjust! ###`
		`+ op.drop_column('events', 'search_key')`
		`+ op.drop_column('events', 'event_type_id')`
		`+ ### end Alembic commands ###`

freshmaker/models.py

file modified

+35 -4

		`@@ -28,6 +28,10 @@`
		`from sqlalchemy.orm import (validates, relationship)`

		`from freshmaker import db`
		`+ from freshmaker.events import (`
		`+ MBSModuleStateChangeEvent, GitModuleMetadataChangeEvent,`
		`+ GitRPMSpecChangeEvent, TestingEvent, GitDockerfileChangeEvent,`
		`+ BodhiUpdateCompleteStableEvent, KojiTaskStateChangeEvent)`

		`# BUILD_STATES for the builds submitted by Freshmaker`
		`BUILD_STATES = {`
		`@@ -51,6 +55,18 @@`

		`INVERSE_ARTIFACT_TYPES = {v: k for k, v in ARTIFACT_TYPES.items()}`

		`+ EVENT_TYPES = {`
		`+ MBSModuleStateChangeEvent: 0,`
		`+ GitModuleMetadataChangeEvent: 1,`
		`+ GitRPMSpecChangeEvent: 2,`
		`+ TestingEvent: 3,`
		`+ GitDockerfileChangeEvent: 4,`
		`+ BodhiUpdateCompleteStableEvent: 5,`
		`+ KojiTaskStateChangeEvent: 6,`
		`+ }`
		`+`
		`+ INVERSE_EVENT_TYPES = {v: k for k, v in EVENT_TYPES.items()}`
		`+`

		`class FreshmakerBase(db.Model):`
		`__abstract__ = True`
		`@@ -59,28 +75,43 @@`
		`class Event(FreshmakerBase):`
		`__tablename__ = "events"`
		`id = db.Column(db.Integer, primary_key=True)`
		`+ # ID of message generating the rebuild event.`
		`message_id = db.Column(db.String, nullable=False)`
		`+ # Searchable key for the event - used when searching for events from the JSON`
		`+ # API.`
		`+ search_key = db.Column(db.String, nullable=False)`
		`+ # Event type id defined in EVENT_TYPES - ID of class inherited from`
		`+ # BaseEvent class - used when searching for events of particular type.`
		`+ event_type_id = db.Column(db.Integer, nullable=False)`

		`# List of builds associated with this Event.`
		`builds = relationship("ArtifactBuild", back_populates="event")`

		`@classmethod`
		`- def create(cls, session, message_id):`
		`+ def create(cls, session, message_id, search_key, event_type):`
		`+ if event_type in EVENT_TYPES:`
		`+ event_type = EVENT_TYPES[event_type]`
		`event = cls(`
		`message_id=message_id,`
		`+ search_key=search_key,`
		`+ event_type_id=event_type`
		`)`
		`session.add(event)`
		`return event`

		`@classmethod`
		`- def get_or_create(cls, session, message_id):`
		`+ def get_or_create(cls, session, message_id, search_key, event_type):`
		`instance = session.query(cls).filter_by(message_id=message_id).first()`
		`if instance:`
		`return instance`
		`- return cls.create(session, message_id)`
		`+ return cls.create(session, message_id, search_key, event_type)`
		`+`
		`+ @property`
		`+ def event_type(self):`
		`+ return INVERSE_EVENT_TYPES[self.event_type_id]`

		`def __repr__(self):`
		`- return "<Event %s>" % (self.message_id)`
		`+ return "<Event %s, %r, %s>" % (self.message_id, self.event_type, self.search_key)`


		`class ArtifactBuild(FreshmakerBase):`

start_frontend_from_here

file modified

+1 -1

		`@@ -1,3 +1,3 @@`
		`#!/bin/bash`

		`- FRESHMAKER_DEVELOPER_ENV=1 python freshmaker/manage.py runssl`
		`+ PYTHON_PATH=. FRESHMAKER_DEVELOPER_ENV=1 python freshmaker/manage.py runssl`

tests/test_koji_task_state_change_handler.py

file modified

+2 -1

		`@@ -59,7 +59,8 @@`
		`Tests build state will be updated when receives koji task state changed message`
		`"""`
		`task_id = 123`
		`- ev = models.Event.create(db.session, 'test_msg_id')`
		`+ ev = models.Event.create(db.session, 'test_msg_id', "event-name",`
		`+ events.KojiTaskStateChangeEvent)`
		`build = models.ArtifactBuild.create(db.session,`
		`ev,`
		`'testimage',`

tests/test_models.py

file modified

+4 -1

		`@@ -24,6 +24,7 @@`

		`from freshmaker import db`
		`from freshmaker.models import Event, ArtifactBuild`
		`+ from freshmaker.events import TestingEvent`


		`class TestModels(unittest.TestCase):`
		`@@ -39,7 +40,7 @@`
		`db.session.commit()`

		`def test_creating_event_and_builds(self):`
		`- event = Event.create(db.session, "test_msg_id")`
		`+ event = Event.create(db.session, "test_msg_id", "RHSA-2017-284", TestingEvent)`
		`build = ArtifactBuild.create(db.session, event, "ed", "module", 1234)`
		`ArtifactBuild.create(db.session, event, "mksh", "module", 1235, build)`
		`db.session.commit()`
		`@@ -47,6 +48,8 @@`

		`e = db.session.query(Event).filter(event.id == 1).one()`
		`self.assertEqual(e.message_id, "test_msg_id")`
		`+ self.assertEqual(e.search_key, "RHSA-2017-284")`
		`+ self.assertEqual(e.event_type, TestingEvent)`
		`self.assertEqual(len(e.builds), 2)`

		`self.assertEqual(e.builds[0].name, "ed")`

jkaluza commented 8 years ago

The main goal of this PR is to allow JSON API to query for events by type and by the search_key. It should be possible to do queries like this:

Get all the modules which have been rebuilt as dependency of module build with id 123
Get all the Docker images which have been rebuilt because of errata advisory RHBA-2017-123 update.
...

In order to do these queries, we have to extend the Event table by Event type (so you can limit the query to just events triggered by Errata or MBS, or dist-git and so on) and add a key by which the events can be searched. The key is different per each event type. For dist-git, it is branch and commit hash, for MBS it is build ID and so on.

There is new modules.EVENT_TYPES list which gives each BaseEvent subclass database id. There is new BaseEvent.search_key property, which returns the key used for searching for every BaseEvent instance.

I'm not sure if the search_key is enough generally, but for the current events and other events I have in my mind currently it should be enough to just have single key to query on.