#1229 Add tests for notification/deletion of outdated chroots
Merged 4 years ago by praiskup. Opened 4 years ago by frostyx.
copr/ frostyx/copr outdated-chroots-tests  into  master

@@ -1,5 +1,6 @@ 

  import click

  from coprs import db

+ from coprs import app

  from coprs.logic import coprs_logic, actions_logic

  

  
@@ -10,6 +11,10 @@ 

      help="Do not actually remove any data, but rather print information on stdout"

  )

  def delete_outdated_chroots(dry_run):

+     return delete_outdated_chroots_function(dry_run)

+ 

+ 

+ def delete_outdated_chroots_function(dry_run):

      """

      Delete data in all chroots that are considered as outdated. That means, the chroot is EOL

      and the preservation period is over because admin of the project didn't extend its duration.
@@ -20,6 +25,13 @@ 

      chroots = coprs_logic.CoprChrootsLogic \

          .filter_outdated_to_be_deleted(coprs_logic.CoprChrootsLogic.get_multiple())

      for i, chroot in enumerate(chroots, start=1):

+ 

+         # This shouldn't happen but we should play it safe, not just hope

+         if not chroot.delete_notify:

I dislike one particular thing about this condition. Let's say nobody was notified many many months about anything. This constraint will disallow deleting chroots, that wasn't notified about. However, once we send a notification, this script will delete it on next run, making the notification basically useless. Also, once there are no remaining days and the chroot should have already been deleted, it cannot longer be extended by another 180 days, IMHO it won't even appear in the table.

I think we could do something like this after printing error.

chroot.delete_after = datetime.today() + timedelta(app.config["EOL_CHROOTS_NOTIFICATION_PERIOD"])`

Shouldn't we bump chroot.delete_after when we set chroot.delete_notify?

+             app.logger.error("Refusing to delete {0}/{1} because any notification was sent about its deletion",

+                              chroot.copr.full_name, chroot.name)

+             continue

+ 

          # This command will possibly delete a lot of chroots and can be a performance issue when committing

          # all at once. We are going to commit every x actions to avoid that.

          if i % 1000 == 0:

@@ -22,6 +22,11 @@ 

      help="Notify all (even the recently notified) relevant people"

  )

  def notify_outdated_chroots(dry_run, email_filter, all):

+     with app.app_context():

+         return notify_outdated_chroots_function(dry_run, email_filter, all)

+ 

+ 

+ def notify_outdated_chroots_function(dry_run, email_filter, all):

      """

      Notify all admins of projects with builds in outdated chroots about upcoming deletion.

      """
@@ -31,13 +36,20 @@ 

  

      notifier = DryRunNotifier() if dry_run else Notifier()

      outdated = coprs_logic.CoprChrootsLogic.filter_outdated(coprs_logic.CoprChrootsLogic.get_multiple())

-     for user, chroots in get_user_chroots_map(outdated, email_filter).items():

+     user_chroots_map = get_user_chroots_map(outdated, email_filter).items()

+     for i, (user, chroots) in enumerate(user_chroots_map, start=1):

          chroots = filter_chroots([chroot for chroot in chroots], all)

          if not chroots:

              continue

          chroots.sort(key=lambda x: x.copr.full_name)

          notifier.notify(user, chroots)

-         notifier.store_timestamp(chroots)

+ 

+         # This command will possibly update a lot of chroots and can be a performance issue when committing

+         # all at once. We are going to commit every x actions to avoid that.

+         if i % 1000 == 0:

+             notifier.commit()

+ 

+     notifier.commit()

Would empty transaction traceback here?

  

  def get_user_chroots_map(chroots, email_filter):

      user_chroot_map = {}
@@ -62,7 +74,7 @@ 

  

          # Skip the chroot if was notified in less than `n` days

          now = datetime.datetime.now()

-         if (now - chroot.delete_notify).days >= 80:

+         if (now - chroot.delete_notify).days >= app.config["EOL_CHROOTS_NOTIFICATION_PERIOD"]:

              filtered.append(chroot)

  

      return filtered
@@ -79,9 +91,12 @@ 

          msg = OutdatedChrootMessage(chroots)

          send_mail([user.mail], msg)

  

-     def store_timestamp(self, chroots):

+         # If `send_mail` didn't raise any exception,

+         # we consider the email to be sent correctly

          for chroot in chroots:

              chroot.delete_notify = datetime.datetime.now()

+ 

+     def commit(self):

          db.session.commit()

  

  
@@ -90,5 +105,5 @@ 

          about = ["{0} ({1})".format(chroot.copr.full_name, chroot.name) for chroot in chroots]

          print("Notify {} about {}".format(user.mail, about))

  

-     def store_timestamp(self, chroots):

+     def commit(self):

          pass

@@ -81,6 +81,9 @@ 

      # When the data in EOL chroots should be deleted (in days)

      DELETE_EOL_CHROOTS_AFTER = 180

  

+     # Days between notification emails about a chroot

+     EOL_CHROOTS_NOTIFICATION_PERIOD = 80

+ 

      # We may have a (temporary) chroot that doesn't correspond with /etc/os-release

      # on a client system, e.g. "rhelbeta-8" chroots in Copr which doesn't match to

      # any real system, instead it is a temporary alias for "epel-8". In such case,

@@ -0,0 +1,46 @@ 

+ from unittest.mock import patch

+ from datetime import datetime, timedelta

+ from tests.coprs_test_case import CoprsTestCase

+ from coprs.logic.actions_logic import ActionsLogic

+ from copr_common.enums import ActionTypeEnum

+ from commands.delete_outdated_chroots import delete_outdated_chroots_function

+ 

+ class TestDeleteOutdatedChroots(CoprsTestCase):

+ 

+     def test_delete_outdated(self, f_users, f_coprs, f_mock_chroots, f_db):

+         delete_outdated_chroots_function(dry_run=False)

+         actions = ActionsLogic.get_many(ActionTypeEnum("delete")).all()

+         assert len(actions) == 0

+ 

+     def test_delete_outdated_yesterday(self, f_users, f_coprs, f_mock_chroots, f_db):

+         # This chroot expired yesterday and should be removed

+         self.c2.copr_chroots[0].delete_after = datetime.today() - timedelta(days=1)

+         self.c2.copr_chroots[0].delete_notify = datetime.fromtimestamp(123)

+         delete_outdated_chroots_function(dry_run=False)

+         actions = ActionsLogic.get_many(ActionTypeEnum("delete")).all()

+         assert len(actions) == 1

+         assert self.c2.copr_chroots[0].delete_after is None

+ 

+     def test_delete_outdated_not_yet(self, f_users, f_coprs, f_mock_chroots, f_db):

+         # This chroot will expire tomorrow, so don't remove it yet

+         self.c2.copr_chroots[0].delete_after = datetime.today() + timedelta(days=1)

+         self.c2.copr_chroots[0].delete_notify = datetime.fromtimestamp(123)

+         delete_outdated_chroots_function(dry_run=False)

+         actions = ActionsLogic.get_many(ActionTypeEnum("delete")).all()

+         assert len(actions) == 0

+         assert self.c2.copr_chroots[0].delete_after is not None

+ 

+     @patch("commands.delete_outdated_chroots.app.logger.error")

+     def test_delete_outdated_handle_none_notify(self, logerror, f_users, f_coprs, f_mock_chroots, f_db):

+         self.c2.copr_chroots[0].delete_after = datetime.today() - timedelta(days=1)

+         delete_outdated_chroots_function(dry_run=False)

+ 

+         assert logerror.call_count == 1

+         msg, full_name, chroot = logerror.call_args[0]

+         assert "Refusing to delete" in msg

+         assert full_name == "user2/foocopr"

+         assert chroot == "fedora-17-x86_64"

+ 

+         actions = ActionsLogic.get_many(ActionTypeEnum("delete")).all()

+         assert len(actions) == 0

+         assert self.c2.copr_chroots[0].delete_after is not None

@@ -0,0 +1,94 @@ 

+ from unittest.mock import patch

+ from datetime import datetime, timedelta

+ from tests.coprs_test_case import CoprsTestCase

+ from coprs import app

+ from coprs.logic import coprs_logic

+ from coprs.mail import OutdatedChrootMessage

+ from commands.notify_outdated_chroots import get_user_chroots_map, filter_chroots, notify_outdated_chroots_function

+ 

+ class TestNotifyOutdatedChroots(CoprsTestCase):

+ 

+     def test_user_chroots_map(self, f_users, f_coprs, f_mock_chroots, f_db):

+         chroots = coprs_logic.CoprChrootsLogic.get_multiple().all()

+         assert get_user_chroots_map(chroots, None) == {

+             self.u1: self.c1.copr_chroots,

+             self.u2: self.c2.copr_chroots + self.c3.copr_chroots,

+         }

+ 

+     def test_user_chroots_map_permissions(self, f_users, f_coprs, f_mock_chroots, f_copr_permissions, f_db):

+         # With `f_copr_permissions`, `u1` is now one of the admis of `c3`

+         chroots = coprs_logic.CoprChrootsLogic.get_multiple().all()

+         assert get_user_chroots_map(chroots, None) == {

+             self.u1: self.c1.copr_chroots + self.c3.copr_chroots,

+             self.u2: self.c2.copr_chroots + self.c3.copr_chroots,

+         }

+ 

+     def test_user_chroots_map_email(self, f_users, f_coprs, f_mock_chroots, f_db):

+         chroots = coprs_logic.CoprChrootsLogic.get_multiple().all()

+         assert get_user_chroots_map(chroots, "user2@spam.foo") == \

+             {self.u2: self.c2.copr_chroots + self.c3.copr_chroots}

+ 

+ 

+     def test_filter_chroots(self, f_users, f_coprs, f_mock_chroots, f_db):

+         chroots = self.c1.copr_chroots + self.c2.copr_chroots

+ 

+         # Do not care how recently was those chroots notified

+         assert filter_chroots(chroots, all=True) == chroots

+ 

+         # None of these chroots have been notified yet, hence we want to notify all

+         assert filter_chroots(chroots, all=False) == chroots

+ 

+         # We sent notification about this chroot yesterday, so don't spam about it

+         chroots[0].delete_notify = datetime.today() - timedelta(days=1)

+         assert filter_chroots(chroots, all=False) == chroots[1:]

+ 

+         # The notification was sent ~3 monts ago, we want to send a new one

+         chroots[0].delete_notify = datetime.today() - timedelta(days=90)

+         assert filter_chroots(chroots, all=False) == chroots

+ 

+     @patch("commands.notify_outdated_chroots.dev_instance_warning")

+     @patch("commands.notify_outdated_chroots.send_mail")

+     def test_notify_outdated_chroots(self, send_mail, dev_instance_warning, f_users, f_coprs, f_mock_chroots, f_db):

+         app.config["SERVER_NAME"] = "localhost"

+         with app.app_context():

+ 

+             # Any copr chroots are marked to be deleted, hence there is nothing to be notified about

+             notify_outdated_chroots_function(dry_run=False, email_filter=None, all=False)

+             assert send_mail.call_count == 0

+ 

+             # Mark a copr chroot to be deleted, we should send a notification

+             self.c2.copr_chroots[0].delete_after = datetime.today() + timedelta(days=150)

+             assert self.c2.copr_chroots[0].delete_notify is None

+             notify_outdated_chroots_function(dry_run=False, email_filter=None, all=False)

+             assert self.c2.copr_chroots[0].delete_notify is not None

+ 

+             assert send_mail.call_count == 1

+             recipients, message = send_mail.call_args[0]

+             assert isinstance(message, OutdatedChrootMessage)

+             assert recipients == ["user2@spam.foo"]

+             assert "Project: user2/foocopr"

+             assert "Chroot: fedora-17-x86_64"

+             assert "Remaining: 149 days"

+ 

+             # Run notifications immediately once more

+             # Nothing should change, we have a mechanism to not spam users

+             previous_delete_notify = self.c2.copr_chroots[0].delete_notify

+             notify_outdated_chroots_function(dry_run=False, email_filter=None, all=False)

+             assert send_mail.call_count == 1  # No new calls

+             assert self.c2.copr_chroots[0].delete_notify == previous_delete_notify

+ 

+             # Now, don't care when we sent last notifications. Notify everyone again

+             notify_outdated_chroots_function(dry_run=False, email_filter=None, all=True)

+             assert send_mail.call_count == 2

+             assert self.c2.copr_chroots[0].delete_notify != previous_delete_notify

+ 

+     @patch("commands.notify_outdated_chroots.dev_instance_warning")

+     @patch("commands.notify_outdated_chroots.send_mail")

+     def test_notify_outdated_chroots_email_filter(self, send_mail, dev_instance_warning,

+                                                   f_users, f_coprs, f_mock_chroots, f_db):

+         # Make sure that if `email_filter` is set, nobody else is going to be affected

+         email_filter = ["somebody@nonexistent.ex"]

+         self.c2.copr_chroots[0].delete_after = datetime.today() + timedelta(days=150)

+         notify_outdated_chroots_function(dry_run=False, email_filter=email_filter, all=True)

+         assert send_mail.call_count == 0

+         assert not self.c2.copr_chroots[0].delete_notify

@@ -1,7 +1,7 @@ 

  import json

- import datetime

  import pytest

  

+ from datetime import datetime, timedelta, date

  from flask_whooshee import Whooshee

  

  from sqlalchemy import desc
@@ -136,6 +136,40 @@ 

          # However, it should not be removed from the Copr

          assert [ch.name for ch in self.c2.copr_chroots] == ["fedora-17-x86_64", "fedora-17-i386"]

  

+     def test_filter_outdated(self, f_users, f_coprs, f_mock_chroots, f_db):

+         outdated = CoprChrootsLogic.filter_outdated(CoprChrootsLogic.get_multiple())

+         assert outdated.all() == []

+ 

+         # A chroot is supposed to be removed today (without a time specification)

+         # Do not notify anyone, it is already too late. For all intents and purposes,

+         # the data is already gone.

+         self.c2.copr_chroots[0].delete_after = date.today()

+         assert outdated.all() == []

+ 

+         # A chroot will be deleted tomorrow

+         self.c2.copr_chroots[0].delete_after = datetime.today() + timedelta(days=1)

+         assert outdated.all() == [self.c2.copr_chroots[0]]

+ 

+         # A chroot was deleted yesterday

+         self.c2.copr_chroots[0].delete_after = datetime.today() - timedelta(days=1)

+         assert outdated.all() == []

+ 

+     def test_filter_outdated_to_be_deleted(self, f_users, f_coprs, f_mock_chroots, f_db):

+         outdated = CoprChrootsLogic.filter_outdated_to_be_deleted(CoprChrootsLogic.get_multiple())

+         assert outdated.all() == []

+ 

+         # A chroot is supposed to be removed today (without a time specification)

+         self.c2.copr_chroots[0].delete_after = date.today()

+         assert outdated.all() == [self.c2.copr_chroots[0]]

+ 

+         # A chroot should be deleted tomorrow, don't touch it yet

+         self.c2.copr_chroots[0].delete_after = datetime.today() + timedelta(days=1)

+         assert outdated.all() == []

+ 

+         # A chroot was supposed to be deleted yesterday, delete it

+         self.c2.copr_chroots[0].delete_after = datetime.today() - timedelta(days=1)

+         assert outdated.all() == [self.c2.copr_chroots[0]]

+ 

  

  class TestPinnedCoprsLogic(CoprsTestCase):

  

... and disallow to remove chroots, which for some reason, haven't been notified about. This PR is a reaction to this message on a mailing list

For what it's worth, I never got the promised notification for my Coprs.
The legacy chroots are just gone forever with no warning whatsoever.
-- Kevin Kofler

I haven't found any cause of this error (yet) but this feature desperately needed to be tested. Also, I've added a constraint, that chroots, that we haven't sent a notification about, cannot be removed.

rebased onto 1526b7ea47bb32c9497ffac950179c5b73506bfb

4 years ago

rebased onto 87e905db19f1376f0b9a9cff2e86523e96d4daaa

4 years ago

I've rebased the PR, there were some errors.

I haven't found any cause of this error (yet)

I found a possible cause of this issue. Currently, when you run

# This is a command executed via cron, just adding the `-e --all` params to be sure
runuser -c 'copr-frontend notify_outdated_chroots -e jkadlcik@redhat.com -all' - copr-fe

it fails with

 RuntimeError: Attempted to generate a URL without the application
    context being pushed. This has to be executed when application context
    is available.

(It doesn't happen with --dry-run). There is a fix in this PR.

I dislike one particular thing about this condition. Let's say nobody was notified many many months about anything. This constraint will disallow deleting chroots, that wasn't notified about. However, once we send a notification, this script will delete it on next run, making the notification basically useless. Also, once there are no remaining days and the chroot should have already been deleted, it cannot longer be extended by another 180 days, IMHO it won't even appear in the table.

I think we could do something like this after printing error.

chroot.delete_after = datetime.today() + timedelta(app.config["EOL_CHROOTS_NOTIFICATION_PERIOD"])`

Shouldn't we bump chroot.delete_after when we set chroot.delete_notify?

2 new commits added

  • frontend: update delete_notify timestamp implicitly
  • frontend: do not commit after each notified chroot
4 years ago

I made a few small changes proposed by @praiskup off-list.
PTAL

Would empty transaction traceback here?

No.

Name                                                       Stmts   Miss  Cover   Missing
----------------------------------------------------------------------------------------
commands/delete_outdated_chroots.py                           29      4    86%   14, 38, 54, 57

... looks good to me, missing are not worth testing. +1

1 new commit added

  • frontend: make sure nobody else is affected when email_filter is set
4 years ago

rebased onto 6ccc7e7

4 years ago

Pull-Request has been merged by praiskup

4 years ago