+ .. _copr_outdated_chroots_removal_policy:

+ 

+ Outdated chroots removal policy

+ ===============================

+ 

+ This page describes the process of deleting build results in outdated chroots.

+ 

+ 

+ When a distribution (e.g. Fedora 26) officially reaches the end of its life, all its chroots

+ (e.g. fedora-26-x86_64, fedora-26-i386) get also disabled in Copr. This doesn't happen simultaneously,

+ instead, there is a small adjustment period of undocumented length, to give users enough time to migrate.

+ 

+ Once such chroots are disabled, new builds can no longer be submitted in them. Already existing build results

+ are still available and don't get removed at this point. By default, they are preserved for the next 180 days

+ after chroot disablement and then automatically removed. This can be suppressed for any project by any of its admins.

+ 

+ The remaining time can be restored back to 180 days at any moment, so the preservation period can be

+ periodically extended for an unlimited amount of time.

+ 

+ # When the data in EOL chroots should be deleted (in days)

+ DELETE_EOL_CHROOTS_AFTER = 180

+ """Add delete_after and delete_notify columns

+ 

+ Revision ID: 69c5f19841a5

+ Revises: c28451aaed50

+ Create Date: 2018-10-09 00:25:40.725051

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = '69c5f19841a5'

+ down_revision = 'c28451aaed50'

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     op.add_column('copr_chroot', sa.Column('delete_after', sa.DateTime(), nullable=True))

+     op.add_column('copr_chroot', sa.Column('delete_notify', sa.DateTime(), nullable=True))

+ 

+ 

+ def downgrade():

+     op.drop_column('copr_chroot', 'delete_notify')

+     op.drop_column('copr_chroot', 'delete_after')

+     # When the data in EOL chroots should be deleted (in days)

+     DELETE_EOL_CHROOTS_AFTER = 180

+ 

+ 

+ class CoprChrootExtend(FlaskForm):

+     name = wtforms.StringField("Chroot name", validators=[wtforms.validators.DataRequired])

+ 

+ 

+ import datetime

+     def get_admins_for_copr(cls, copr):

+         permissions = cls.get_for_copr(copr)

+         return [copr.user] + [p.user for p in permissions if p.copr_admin == helpers.PermissionEnum("approved")]

+ 

+     @classmethod

+     def get_multiple(cls):

+         return models.CoprChroot.query

+ 

+     @classmethod

+     def get_by_name(cls, copr, chroot_name, active_only=True):

+         mc = MockChrootsLogic.get_from_name(chroot_name, active_only=active_only).one()

+     def create_chroot(cls, user, copr, mock_chroot, buildroot_pkgs=None, repos=None, comps=None, comps_name=None,

+                       module_md=None, module_md_name=None, with_opts="", without_opts="",

+                       delete_after=None, delete_notify=None):

+         cls._update_chroot(buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name, chroot,

+                            with_opts, without_opts, delete_after, delete_notify)

+     def update_chroot(cls, user, copr_chroot, buildroot_pkgs=None, repos=None, comps=None, comps_name=None,

+                       module_md=None, module_md_name=None, with_opts="", without_opts="",

+                       delete_after=None, delete_notify=None):

+         cls._update_chroot(buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name,

+                            copr_chroot, with_opts, without_opts, delete_after, delete_notify)

+     def _update_chroot(cls, buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name,

+                        copr_chroot, with_opts, without_opts, delete_after, delete_notify):

+         if delete_after is not None:

+             copr_chroot.delete_after = delete_after

+ 

+         if delete_notify is not None:

+             copr_chroot.delete_notify = delete_notify

+ 

+     @classmethod

+     def filter_outdated(cls, query):

+         return query.filter(models.CoprChroot.delete_after >= datetime.datetime.now())

+ 

+ from coprs import app, helpers

+ class OutdatedChrootMessage(Message):

+     def __init__(self, copr_chroots):

+         """

+         :param models.Copr copr:

+         :param list copr_chroots: list of models.CoprChroot instances

+         """

+         self.subject = "Upcoming deletion of outdated chroots in your projects"

+         self.text = ("You have been notified because you are an admin of projects,"

+                      "that have some builds in outdated chroots\n\n"

+ 

+                      "According to the 'Copr outdated chroots removal policy'\n"

+                      "https://docs.pagure.org/copr.copr/copr_outdated_chroots_removal_policy.html\n"

+                      "data are going to be preserved {0} days after the chroot is EOL"

+                      "and then automatically deleted, unless you decide to prolong the expiration period.\n\n"

+ 

+                      "Please, visit the projects settings if you want to extend the time.\n\n"

+                      .format(app.config["DELETE_EOL_CHROOTS_AFTER"]))

+ 

+         for chroot in copr_chroots:

+             self.text += (

+                 "Project: {0}\n"

+                 "Chroot: {1}\n"

+                 "Remaining: {2} days\n"

+                 "{3}\n\n".format(chroot.copr.full_name, chroot.name, chroot.delete_after_days,

+                                  helpers.copr_url('coprs_ns.copr_repositories', chroot.copr, _external=True)))

+ 

+ 

+     def outdated_chroots(self):

+         return [chroot for chroot in self.copr_chroots if chroot.delete_after]

+ 

+     @property

+     # Once mock_chroot gets EOL, copr_chroots are going to be deleted

+     # if their owner doesn't extend their time span

+     delete_after = db.Column(db.DateTime)

+     delete_notify = db.Column(db.DateTime)

+ 

+     @property

+     def delete_after_days(self):

+         if not self.delete_after:

+             return None

+         now = datetime.datetime.now()

+         return (self.delete_after - now).days

+ 

+     {{ tab("repositories", "Repositories", copr_url('coprs_ns.copr_repositories', copr)) }}

+   {% endif %}

+ 

+   {% if g.user.can_edit(copr) %}

+ {% extends "coprs/detail/settings.html" %}

+ 

+ {% set selected_monitor_tab = "repositories" %}

+ {%block settings_breadcrumb %}Repositores{% endblock %}

+ 

+ 

+ {% block tab_content %}

+ 

+ <h2>Repositories</h2>

+ 

+ <div class="col-sm-8 col-md-9">

+     <h3>Active repositories</h3>

+     <p>See Active Releases in project <a href="">Overview</a>.</p>

+ 

+     <h3>Outdated repositories</h3>

+     {% if not outdated_chroots %}

+     <p>This project has no repositories for outdated distributions.</p>

+     {% else %}

+     <p>

+         This project has following repositories for outdated distributions which are going to be removed unless you

+         extend the time for they should be preserved. Please see

+         <a href="#">Outdated repos removal policy</a>

+         in Copr Documentation.

+     </p>

+     <form action="" method="POST">

+         <table class="table table-bordered">

+             <thead><tr><th>Release</th><th>Architecture</th><th>Remaining time</th><th>Extend</th></tr></thead>

+             <tbody>

+                 {% for chroot in outdated_chroots %}

+                 <tr>

+                     <td>{{ chroot.mock_chroot.os.capitalize() }}</td>

+                     <td>{{ chroot.mock_chroot.arch }}</td>

+                     <td>{% if chroot.delete_after_days %}{{ chroot.delete_after_days }} days{% else %} Unknown {% endif %}</td>

+                     <td>

+                         <button name="name" class="btn btn-default" type="submit" value="{{ chroot.mock_chroot.name }}">Extend</button>

+                     </td>

+                 </tr>

+             </tbody>

+             {% endfor %}

+         </table>

+     {% endif %}

+     </form>

+ </div>

+ 

+ {% endblock %}

+ import datetime

+ @coprs_ns.route("/<username>/<coprname>/repositories/")

+ @coprs_ns.route("/g/<group_name>/<coprname>/repositories/")

+ @login_required

+ @req_with_copr

+ def copr_repositories(copr):

+     if not flask.g.user.can_edit(copr):

+         flask.flash("You don't have access to this page.", "error")

+         return flask.redirect(url_for_copr_details(copr))

+ 

+     return render_copr_repositories(copr)

+ 

+ 

+ def render_copr_repositories(copr):

+     outdated_chroots = copr.outdated_chroots

+     return flask.render_template("coprs/detail/settings/repositories.html", copr=copr,

+                                  outdated_chroots=outdated_chroots)

+ 

+ 

+ @coprs_ns.route("/<username>/<coprname>/repositories/", methods=["POST"])

+ @coprs_ns.route("/g/<group_name>/<coprname>/repositories/", methods=["POST"])

+ @login_required

+ @req_with_copr

+ def copr_repositories_post(copr):

+     if not flask.g.user.can_edit(copr):

+         flask.flash("You don't have access to this page.", "error")

+         return flask.redirect(url_for_copr_details(copr))

+ 

+     form = forms.CoprChrootExtend()

+     copr_chroot = coprs_logic.CoprChrootsLogic.get_by_name(copr, form.name.data, active_only=False).one()

+     delete_after_days = app.config["DELETE_EOL_CHROOTS_AFTER"] + 1

+     delete_after_timestamp = datetime.datetime.now() + datetime.timedelta(days=delete_after_days)

+     coprs_logic.CoprChrootsLogic.update_chroot(flask.g.user, copr_chroot,

+                                                delete_after=delete_after_timestamp)

+     db.session.commit()

+     return render_copr_repositories(copr)

+ 

+ 

+ import datetime

+ from coprs.mail import send_mail, OutdatedChrootMessage

+ from coprs.helpers import chroot_to_branch, PermissionEnum

+                 mock_chroot = coprs_logic.MockChrootsLogic.edit_by_name(

+ 

+                 if action != "eol":

+                     continue

+ 

+                 for copr_chroot in mock_chroot.copr_chroots:

+                     delete_after_days = app.config["DELETE_EOL_CHROOTS_AFTER"] + 1

+                     delete_after_timestamp = datetime.datetime.now() + datetime.timedelta(delete_after_days)

+                     # Workarounding an auth here

+                     coprs_logic.CoprChrootsLogic.update_chroot(copr_chroot.copr.user, copr_chroot,

+                                                                delete_after=delete_after_timestamp)

+                choices=["activate", "deactivate", "eol"],

+ class NotifyOutdatedChrootsCommand(Command):

+     """

+     Notify all admins of projects with builds in outdated chroots about upcoming deletion.

+     """

+     option_list = [

+         Option("--dry-run", action="store_true",

+                help="Do not actually notify the people, but rather print information on stdout"),

+         Option("-e", "--email", action="append", dest="email_filter",

+                help="Notify only "),

+         Option("-a", "--all", action="store_true",

+                help="Notify all (even the recently notified) relevant people"),

+     ]

+ 

+     def run(self, dry_run, email_filter, all):

+         self.dry_run = dry_run

+         self.email_filter = email_filter

+         self.all = all

+ 

+         outdated = coprs_logic.CoprChrootsLogic.filter_outdated(coprs_logic.CoprChrootsLogic.get_multiple())

+         for user, chroots in self.get_user_chroots_map(outdated).items():

+             chroots = self.filter_chroots([chroot for chroot in chroots])

+             self.notify(user, chroots)

+             self.store_notify_timesamp(chroots)

+ 

+     def get_user_chroots_map(self, chroots):

+         user_chroot_map = {}

+         for chroot in chroots:

+             for admin in coprs_logic.CoprPermissionsLogic.get_admins_for_copr(chroot.copr):

+                 if self.email_filter and admin.mail not in self.email_filter:

+                     continue

+                 if admin not in user_chroot_map:

+                     user_chroot_map[admin] = []

+                 user_chroot_map[admin].append(chroot)

+         return user_chroot_map

+ 

+     def filter_chroots(self, chroots):

+         if self.all:

+             return chroots

+ 

+         filtered = []

+         for chroot in chroots:

+             if not chroot.delete_notify:

+                 filtered.append(chroot)

+                 continue

+ 

+             now = datetime.datetime.now()

+             if (now - chroot.delete_notify).days >= 14:

+                 filtered.append(chroot)

+ 

+         return filtered

+ 

+     def notify(self, user, chroots):

+         if self.dry_run:

+             about = ["{0} ({1})".format(chroot.copr.full_name, chroot.name) for chroot in chroots]

+             print("Notify {} about {}".format(user.mail, about))

+         else:

+             msg = OutdatedChrootMessage(chroots)

+             send_mail(user.mail, msg)

+ 

+     def store_notify_timesamp(self, chroots):

+         if self.dry_run:

+             return

+         for chroot in chroots:

+             chroot.delete_after_notify = datetime.datetime.now()

+         db.session.commit()

+ 

+ 

+ manager.add_command("notify_outdated_chroots", NotifyOutdatedChrootsCommand())

+ import datetime

+ from coprs.mail import PermissionRequestMessage, PermissionChangeMessage, LegalFlagMessage, OutdatedChrootMessage

+ 

+     def test_outdated_chroot_message(self, f_users, f_coprs, f_mock_chroots, f_db):

+         chroots = [self.c1.copr_chroots[0], self.c2.copr_chroots[0]]

+         for chroot in chroots:

+             chroot.delete_after = datetime.datetime.now() + datetime.timedelta(days=7 + 1) # 7 days = 6d, 23h, 59m, ...

+ 

+         app.config["SERVER_NAME"] = "localhost"

+         app.config["DELETE_EOL_CHROOTS_AFTER"] = 123

+         with app.app_context():

+             msg = OutdatedChrootMessage(chroots)

+         assert msg.subject == "Upcoming deletion of outdated chroots in your projects"

+         assert msg.text == ("You have been notified because you are an admin of projects,"

+                             "that have some builds in outdated chroots\n\n"

+ 

+                             "According to the 'Copr outdated chroots removal policy'\n"

+                             "https://docs.pagure.org/copr.copr/copr_outdated_chroots_removal_policy.html\n"

+                             "data are going to be preserved 123 days after the chroot is EOL"

+                             "and then automatically deleted, unless you decide to prolong the expiration period.\n\n"

+ 

+                             "Please, visit the projects settings if you want to extend the time.\n\n"

+ 

+                             "Project: user1/foocopr\n"

+                             "Chroot: fedora-18-x86_64\n"

+                             "Remaining: 7 days\n"

+                             "http://localhost/coprs/user1/foocopr/repositories/\n\n"

+ 

+                             "Project: user2/foocopr\n"

+                             "Chroot: fedora-17-x86_64\n"

+                             "Remaining: 7 days\n"

+                             "http://localhost/coprs/user2/foocopr/repositories/\n\n")