| |
@@ -21,200 +21,183 @@
|
| |
|
| |
from datetime import datetime
|
| |
import logging
|
| |
+ from typing import Optional, Any, Iterable, Sequence
|
| |
|
| |
from fedora.client import ServerError
|
| |
from bodhi.client.bindings import BodhiClient
|
| |
+ import flask_sqlalchemy
|
| |
|
| |
from blockerbugs import app
|
| |
from blockerbugs.models.update import Update
|
| |
from blockerbugs.models.bug import Bug
|
| |
-
|
| |
- bodhi_baseurl = app.config['BODHI_URL']
|
| |
+ from blockerbugs.models.release import Release
|
| |
|
| |
|
| |
class UpdateSync(object):
|
| |
- def __init__(self, db, bodhi_interface=None):
|
| |
+ """The main class for perfoming Update synchronization with Bodhi."""
|
| |
+
|
| |
+ def __init__(self, db: flask_sqlalchemy.SQLAlchemy, bodhiclient: Optional[BodhiClient] = None
|
| |
+ ) -> None:
|
| |
self.db = db
|
| |
- if bodhi_interface:
|
| |
- self.bodhi_interface = bodhi_interface()
|
| |
- else:
|
| |
- # disable saving session on disk by cache_session=False
|
| |
- self.bodhi_interface = BodhiClient(base_url=bodhi_baseurl, cache_session=False)
|
| |
+ # disable saving session on disk by cache_session=False
|
| |
+ self.bodhi = bodhiclient or BodhiClient(base_url=app.config['BODHI_URL'],
|
| |
+ cache_session=False)
|
| |
self.log = logging.getLogger('update_sync')
|
| |
- self._releases = [] # all known releases
|
| |
+ self._releases: list[dict[str, Any]] = []
|
| |
+ """All releases known to Bodhi"""
|
| |
|
| |
@property
|
| |
- def releases(self):
|
| |
- '''All releases known to Bodhi, as retrieved from /releases/ endpoint'''
|
| |
+ def releases(self) -> list[dict[str, Any]]:
|
| |
+ """All releases known to Bodhi, as retrieved from /releases/ endpoint"""
|
| |
if self._releases:
|
| |
# already retrieved
|
| |
return self._releases
|
| |
|
| |
- self._releases = self.bodhi_interface.get_releases(
|
| |
- rows_per_page=100)['releases']
|
| |
-
|
| |
- self.log.debug('Retrieved %d known releases from Bodhi' %
|
| |
- (len(self._releases)))
|
| |
+ self._releases = self.bodhi.get_releases(rows_per_page=100)['releases']
|
| |
+ self.log.debug('Retrieved %d known releases from Bodhi', len(self._releases))
|
| |
return self._releases
|
| |
|
| |
- def extract_information(self, update):
|
| |
- updateinfo = {}
|
| |
- date_pushed = None
|
| |
- if update.date_pushed:
|
| |
- date_pushed = datetime.strptime(update.date_pushed,
|
| |
- '%Y-%m-%d %H:%M:%S')
|
| |
- updateinfo['date_pushed_testing'] = None
|
| |
- updateinfo['date_pushed_stable'] = None
|
| |
- updateinfo['pending'] = False
|
| |
- # this will be None if there is no request
|
| |
- updateinfo['request'] = update.request
|
| |
- if update.status == 'pending':
|
| |
- updateinfo['pending'] = True
|
| |
-
|
| |
- if update.request == 'stable':
|
| |
- updateinfo['status'] = 'stable'
|
| |
- updateinfo['date_pushed_testing'] = date_pushed
|
| |
- elif update.request == 'testing':
|
| |
- updateinfo['status'] = 'testing'
|
| |
- else:
|
| |
- updateinfo['status'] = 'undefined'
|
| |
- else:
|
| |
- updateinfo['status'] = str(update.status)
|
| |
- if update.status == 'testing':
|
| |
- updateinfo['date_pushed_testing'] = date_pushed
|
| |
- elif update.status == 'stable':
|
| |
- updateinfo['date_pushed_stable'] = date_pushed
|
| |
-
|
| |
- updateinfo['title'] = str(update.title)
|
| |
- updateinfo['karma'] = update.karma
|
| |
- updateinfo['stable_karma'] = update.stable_karma
|
| |
- updateinfo['url'] = update.url
|
| |
-
|
| |
- updateinfo['date_submitted'] = datetime.strptime(update.date_submitted,
|
| |
+ def extract_information(self, update: dict) -> dict[str, Any]:
|
| |
+ """Create a dict with extracted Update information. See the code to learn the dict keyvals.
|
| |
+
|
| |
+ :param update: the update object as retrieved from Bodhi API
|
| |
+ """
|
| |
+ updateinfo: dict[str, Any] = {}
|
| |
+ updateinfo['updateid'] = update['updateid']
|
| |
+ updateinfo['status'] = update['status']
|
| |
+ updateinfo['karma'] = update['karma']
|
| |
+ updateinfo['url'] = update['url']
|
| |
+ updateinfo['date_submitted'] = datetime.strptime(update['date_submitted'],
|
| |
'%Y-%m-%d %H:%M:%S')
|
| |
- updateinfo['bugs'] = []
|
| |
- if len(update.bugs) > 0:
|
| |
- for buginfo in update.bugs:
|
| |
- updateinfo['bugs'].append(buginfo.bug_id)
|
| |
+ updateinfo['title'] = update['title']
|
| |
+ updateinfo['request'] = update['request']
|
| |
+ updateinfo['stable_karma'] = update['stable_karma']
|
| |
+ updateinfo['bugs'] = [buginfo['bug_id'] for buginfo in update['bugs']]
|
| |
|
| |
return updateinfo
|
| |
|
| |
- def get_update(self, envr):
|
| |
- updates = self.bodhi_interface.query(package=envr)
|
| |
- return self.extract_information(updates['updates'][0])
|
| |
-
|
| |
- def is_watched_bug(self, bugnum):
|
| |
- watched_bug = Bug.query.filter_by(bugid=bugnum).first()
|
| |
- if watched_bug:
|
| |
- return True
|
| |
- else:
|
| |
- return False
|
| |
-
|
| |
- def clean_updates(self, updates, relid):
|
| |
- """Remove updates for this release which are no longer related
|
| |
- to any blocker or freeze exception bug from the database.
|
| |
+ def clean_updates(self, updateids: Iterable[str], release: Release) -> None:
|
| |
+ """Remove all updates from the database which are related to a particular release and are
|
| |
+ not listed among ``updateids``.
|
| |
"""
|
| |
- query_updates = Update.query.filter(
|
| |
- Update.release_id == relid,
|
| |
- ).all()
|
| |
- db_updates = set(update.title for update in query_updates)
|
| |
- bodhi_updates = set(update['title'] for update in updates)
|
| |
- unneeded_updates = db_updates.difference(bodhi_updates)
|
| |
-
|
| |
- for update in query_updates:
|
| |
- if update.title in unneeded_updates:
|
| |
- self.log.debug("Removing no longer relevant update %s" %
|
| |
- update.title)
|
| |
+ db_updates: list[Update] = Update.query.filter_by(release=release).all()
|
| |
+ db_updateids = set(update.updateid for update in db_updates)
|
| |
+ unneeded_updateids = db_updateids.difference(updateids)
|
| |
+
|
| |
+ for update in db_updates:
|
| |
+ if update.updateid in unneeded_updateids:
|
| |
+ self.log.debug("Removing no longer relevant %r", update)
|
| |
self.db.session.delete(update)
|
| |
- self.db.session.commit()
|
| |
+ self.db.session.commit()
|
| |
|
| |
- def search_updates(self, bugids, release_num):
|
| |
- # not all releases exist all the time (before branching, before bodhi
|
| |
- # activation point, etc), so drop those which Bodhi doesn't currently
|
| |
- # know of
|
| |
- known_releases = [rel['name'].lower() for rel in self.releases]
|
| |
+ def search_updates(self, bugids: Sequence[int], release_num: int) -> list[dict[str, Any]]:
|
| |
+ """Find all Bodhi updates in a particular release which fix one of the bugs specified.
|
| |
+
|
| |
+ :param bugids: Bugzilla ticket numbers
|
| |
+ :param release_num: the version of a release to query
|
| |
+ :return: a list of update info dictionaries, as provided by ``extract_information()``
|
| |
+ """
|
| |
query_releases = [
|
| |
- 'f%d' % release_num, # standard repo
|
| |
+ 'f%d' % release_num, # rpms
|
| |
'f%df' % release_num, # flatpaks
|
| |
- 'f%dm' % release_num, # modularity
|
| |
+ 'f%dm' % release_num, # modules
|
| |
+ 'f%dc' % release_num, # containers
|
| |
]
|
| |
+ # not all releases exist all the time (before branching, before Bodhi activation point,
|
| |
+ # etc), so drop those which Bodhi doesn't currently know of
|
| |
+ known_releases = [rel['name'].lower() for rel in self.releases]
|
| |
for rel in query_releases.copy():
|
| |
if rel not in known_releases:
|
| |
- self.log.warning("Release %s not found in Bodhi (might be "
|
| |
- "normal depending on release life cycle)" % rel)
|
| |
+ self.log.debug("Release %s not found in Bodhi (might be normal depending on the "
|
| |
+ "release life cycle)", rel)
|
| |
query_releases.remove(rel)
|
| |
-
|
| |
- queries_data = dict(
|
| |
- bugs=[str(bug_id) for bug_id in bugids],
|
| |
- release=query_releases,
|
| |
- limit=100,
|
| |
- )
|
| |
- result = self.bodhi_interface.query(**queries_data)
|
| |
-
|
| |
- if u'status' in result.keys():
|
| |
- raise ServerError('', 200, result['errors'][0].description)
|
| |
-
|
| |
- updates = {}
|
| |
- for update in result.updates:
|
| |
- updates[update.title] = update
|
| |
-
|
| |
- while result.page < result.pages:
|
| |
- result = self.bodhi_interface.query(page=result.page + 1,
|
| |
- **queries_data)
|
| |
- for update in result.updates:
|
| |
- updates[update.title] = update
|
| |
-
|
| |
- updates = updates.values() # updates without duplicates
|
| |
- self.log.info('found %d updates from bodhi for %d bugs in f%d' %
|
| |
- (len(updates), len(bugids), release_num))
|
| |
+ if not query_releases:
|
| |
+ self.log.warning("No releases related to F%d found in Bodhi! Nothing to query.",
|
| |
+ release_num)
|
| |
+ return []
|
| |
+
|
| |
+ queries_data = {
|
| |
+ 'bugs': [str(bug_id) for bug_id in bugids],
|
| |
+ 'release': query_releases,
|
| |
+ 'limit': 100,
|
| |
+ 'status': ['pending', 'testing', 'stable'],
|
| |
+ }
|
| |
+ updates_dict = {}
|
| |
+ # Bodhi counts pages from 1
|
| |
+ page = pages = 1
|
| |
+ while page <= pages:
|
| |
+ result = self.bodhi.query(page=page, **queries_data)
|
| |
+
|
| |
+ if 'status' in result:
|
| |
+ raise ServerError('', 400, result['errors'][0]['description'])
|
| |
+
|
| |
+ for update in result['updates']:
|
| |
+ assert update['release']['version'] == str(release_num)
|
| |
+ assert update['status'] in ['pending', 'testing', 'stable']
|
| |
+ updates_dict[update['updateid']] = update
|
| |
+
|
| |
+ page += 1
|
| |
+ pages = result['pages']
|
| |
+
|
| |
+ updates = updates_dict.values() # updates without duplicates
|
| |
+ self.log.info('Found %d updates in Bodhi for %d bugs in release F%d', len(updates),
|
| |
+ len(bugids), release_num)
|
| |
return [self.extract_information(update) for update in updates]
|
| |
|
| |
- def get_release_bugs(self, release):
|
| |
+ def get_release_bugs(self, release: Release) -> list[Bug]:
|
| |
+ """Get all open proposed/accepted bugs related to a certain release (i.e. to all its
|
| |
+ active milestones).
|
| |
+ """
|
| |
buglist = []
|
| |
- for milestone in release.milestones:
|
| |
- buglist.extend(milestone.bugs.filter_by(active=True).all())
|
| |
+ for milestone in release.milestones.filter_by(active=True): # type: ignore[attr-defined]
|
| |
+ buglist.extend(
|
| |
+ milestone.bugs.filter( # type: ignore[attr-defined]
|
| |
+ Bug.active == True, # noqa: E712
|
| |
+ Bug.is_proposed_accepted == True,
|
| |
+ ).all()
|
| |
+ )
|
| |
+
|
| |
return buglist
|
| |
|
| |
- def sync_bug_updates(self, release, bugs):
|
| |
- starttime = datetime.utcnow()
|
| |
- bugs_ids = [bug.bugid for bug in bugs]
|
| |
- self.log.debug('searching for updates for bugs %s' % str(bugs_ids))
|
| |
- try:
|
| |
- updates = self.search_updates(bugs_ids, release.number)
|
| |
- except ServerError as ex:
|
| |
- self.log.error(
|
| |
- 'f{r.number} sync updates failed: {e.code} {e.msg}'.format(
|
| |
- e=ex, r=release))
|
| |
- return
|
| |
+ def sync_updates(self, release: Release) -> None:
|
| |
+ """Synchronize all updates for a particular release. That means pulling new updates from
|
| |
+ Bodhi (related to all bugs which we track in the release), and removing no-longer-relevant
|
| |
+ updates from the database.
|
| |
+ """
|
| |
+ self.log.info('Syncing updates for release F%d ...', release.number)
|
| |
+
|
| |
+ bugs = self.get_release_bugs(release)
|
| |
+ self.log.debug('Found %d relevant bugs in release F%d', len(bugs), release.number)
|
| |
+
|
| |
+ updateinfos = []
|
