| |
@@ -0,0 +1,140 @@
|
| |
+ #
|
| |
+ # watchdog_utils.py - Functions for oraculum WatchDog
|
| |
+ #
|
| |
+ # Copyright 2020, Red Hat, Inc
|
| |
+ #
|
| |
+ # This program is free software; you can redistribute it and/or modify
|
| |
+ # it under the terms of the GNU General Public License as published by
|
| |
+ # the Free Software Foundation; either version 2 of the License, or
|
| |
+ # (at your option) any later version.
|
| |
+ #
|
| |
+ # This program is distributed in the hope that it will be useful,
|
| |
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
|
| |
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
| |
+ # GNU General Public License for more details.
|
| |
+ #
|
| |
+ # You should have received a copy of the GNU General Public License along
|
| |
+ # with this program; if not, write to the Free Software Foundation, Inc.,
|
| |
+ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
| |
+ #
|
| |
+ # Authors:
|
| |
+ # Frantisek Zatloukal <fzatlouk@redhat.com>
|
| |
+
|
| |
+ import re
|
| |
+ import datetime
|
| |
+
|
| |
+ from oraculum.models.watchdog import WatchDogData, WatchDogStartPoint
|
| |
+ from oraculum.models.db_cache import CachedData
|
| |
+ from oraculum.utils.mailing_utils import send_mail
|
| |
+
|
| |
+ from oraculum import app, db
|
| |
+
|
| |
+ def push_to_watchdog(kind, what, message):
|
| |
+ row = WatchDogData(kind, what, message)
|
| |
+ db.session.add(row)
|
| |
+ db.session.commit()
|
| |
+
|
| |
+ def is_task_complete(started_task, ended_tasks):
|
| |
+ """
|
| |
+ Finds out if given started_task has ending counterpart in ended_tasks
|
| |
+ """
|
| |
+ for ended_task in ended_tasks:
|
| |
+ if (started_task.what == ended_task.what) and (started_task.id < ended_task.id):
|
| |
+ return True
|
| |
+ return False
|
| |
+
|
| |
+ def process_queue():
|
| |
+ from oraculum.utils.celery_utils import get_users_for_sync
|
| |
+ from oraculum import CACHE
|
| |
+
|
| |
+ # Find out what db line id to start with
|
| |
+ start_point = WatchDogStartPoint.query.first()
|
| |
+
|
| |
+ if not start_point:
|
| |
+ start_point = WatchDogStartPoint(0)
|
| |
+ db.session.add(start_point)
|
| |
+ db.session.commit()
|
| |
+ start_point = start_point.last_seen_id
|
| |
+
|
| |
+ # Prepare users and packages who meet criteria for sync
|
| |
+ cached_users = get_users_for_sync()
|
| |
+ cached_packages = set()
|
| |
+ for cached_user in cached_users:
|
| |
+ packages = CACHE.get("packager-dashboard_user_data_static", cached_user)["packages"]
|
| |
+ cached_packages.update(packages)
|
| |
+
|
| |
+ # Calculate time for "how old is too old"
|
| |
+ cache_cutoff = datetime.datetime.utcnow() - datetime.timedelta(seconds=max(app.config["SYNC_INTERVALS"].values())) - datetime.timedelta(seconds=3600)
|
| |
+
|
| |
+ # Prepare data from db for watchdog emailing
|
| |
+ data_too_old = CachedData.query.filter(CachedData.time_created <= cache_cutoff).all()
|
| |
+
|
| |
+ # WatchDog Table
|
| |
+ # Store last id we saw in the database to process it from there next time
|
| |
+ end_point = WatchDogData.query.order_by(WatchDogData.id.desc()).first()
|
| |
+ if not end_point:
|
| |
+ send_mail("No data in oraculum WatchDog")
|
| |
+ return
|
| |
+ end_point = end_point.id
|
| |
+
|
| |
+ sync_failures = WatchDogData.query.filter_by(kind="sync_failed").filter(WatchDogData.id > start_point).filter(WatchDogData.id <= end_point).all()
|
| |
+ started_tasks = WatchDogData.query.filter_by(kind="sync_started").filter(WatchDogData.id > start_point).filter(WatchDogData.id <= end_point).all()
|
| |
+ planned_tasks = WatchDogData.query.filter_by(kind="sync_planned").filter(WatchDogData.id > start_point).filter(WatchDogData.id <= end_point).all()
|
| |
+ ended_tasks = WatchDogData.query.filter_by(kind="sync_ended").filter(WatchDogData.id > start_point).filter(WatchDogData.id <= end_point).all()
|
| |
+
|
| |
+ message = "<h1>Oraculum Failures Report</h1>\n"
|
| |
+ message += "<h3>Sync Failures</h3>"
|
| |
+ for sync_failure in sync_failures:
|
| |
+ message += "<b>What: </b>" + sync_failure.what + "<br />"
|
| |
+ message += "<b>When: </b>" + str(sync_failure.when) + "<br />"
|
| |
+ message += "<b>Message: </b><br />"
|
| |
+ message += sync_failure.message.replace("\n", "<br />")
|
| |
+ message += "<hr>"
|
| |
+
|
| |
+ message += "<h3>Old data in DB</h3>"
|
| |
+ for old_data in data_too_old:
|
| |
+ # Filter data we don't cache (inactive users and their packages)
|
| |
+ if "packager-dashboard_user_data_static" in old_data.provider:
|
| |
+ # Regexp down gets args from "what"
|
| |
+ # Eg. packager-dashboard_bugs [('fedora-easy-karma',), {}] > 'fedora-easy-karma'
|
| |
+ if re.search('\[\(\'(.*)\',\)', old_data.provider).groups()[0] not in cached_users:
|
| |
+ continue
|
| |
+ if "packager-dashboard_pull_requests" in old_data.provider or "packager-dashboard_bugs" in old_data.provider:
|
| |
+ if re.search('\[\(\'(.*)\',\)', old_data.provider).groups()[0] not in cached_packages:
|
| |
+ continue
|
| |
+ message += "<b>What: </b>" + old_data.provider + "<br />"
|
| |
+ message += "<b>Last Sync: </b>" + str(old_data.time_created) + "<br />"
|
| |
+ message += "<hr>"
|
| |
+
|
| |
+ message += "<h3>Rotting Started Tasks</h3>"
|
| |
+ for started_task in started_tasks:
|
| |
+ # Filter out tasks that ended in some way (either fail or success, we are here just for the rotting ones)
|
| |
+ if is_task_complete(started_task, ended_tasks) or is_task_complete(started_task, sync_failures):
|
| |
+ continue
|
| |
+
|
| |
+ # Find out tasks that were planned or started too long ago
|
| |
+ if started_task.when + datetime.timedelta(seconds=app.config["TASKS_ROT_AFTER"]) <= datetime.datetime.utcnow():
|
| |
+ message += "<b>What: </b>" + started_task.what + "<br />"
|
| |
+ message += "<b>Start Time: </b>" + str(started_task.when) + "<br />"
|
| |
+
|
| |
+ message += "<h3>Rotting Planned Tasks</h3>"
|
| |
+ for planned_task in planned_tasks:
|
| |
+ # Filter out tasks that ended in some way (either fail or success, we are here just for the rotting ones)
|
| |
+ if is_task_complete(planned_task, ended_tasks) or is_task_complete(planned_task, sync_failures):
|
| |
+ continue
|
| |
+
|
| |
+ # Find out tasks that were planned or started too long ago
|
| |
+ if planned_task.when + datetime.timedelta(seconds=app.config["TASKS_ROT_AFTER"]) <= datetime.datetime.utcnow():
|
| |
+ message += "<b>What: </b>" + planned_task.what + "<br />"
|
| |
+ message += "<b>Plan Time: </b>" + str(planned_task.when) + "<br />"
|
| |
+
|
| |
+ try:
|
| |
+ if not app.config["SEND_ERROR_EMAILS"]:
|
| |
+ raise
|
| |
+ send_mail(message)
|
| |
+ # Insert last processed id into the db so we can start from there next time
|
| |
+ start_point = WatchDogStartPoint.query.first()
|
| |
+ start_point.last_seen_id = end_point
|
| |
+ db.session.commit()
|
| |
+ except Exception:
|
| |
+ app.logger.error("Watchdog: Mail report hasn't been sent due to mail error")
|
| |