From 7685853ac36ecc5f25f417690dde2939ab4bb024 Mon Sep 17 00:00:00 2001 From: Miroslav Suchý Date: Nov 27 2018 13:37:40 +0000 Subject: Merge #451 `Notifications for outdated chroots` --- diff --git a/doc/copr_outdated_chroots_removal_policy.rst b/doc/copr_outdated_chroots_removal_policy.rst new file mode 100644 index 0000000..8681ebb --- /dev/null +++ b/doc/copr_outdated_chroots_removal_policy.rst @@ -0,0 +1,18 @@ +.. _copr_outdated_chroots_removal_policy: + +Outdated chroots removal policy +=============================== + +This page describes the process of deleting build results in outdated chroots. + + +When a distribution (e.g. Fedora 26) officially reaches the end of its life, all its chroots +(e.g. fedora-26-x86_64, fedora-26-i386) get also disabled in Copr. This doesn't happen simultaneously, +instead, there is a small adjustment period of undocumented length, to give users enough time to migrate. + +Once such chroots are disabled, new builds can no longer be submitted in them. Already existing build results +are still available and don't get removed at this point. By default, they are preserved for the next 180 days +after chroot disablement and then automatically removed. This can be suppressed for any project by any of its admins. + +The remaining time can be restored back to 180 days at any moment, so the preservation period can be +periodically extended for an unlimited amount of time. diff --git a/docker/frontend/files/etc/copr/copr.conf b/docker/frontend/files/etc/copr/copr.conf index d5bcbcf..798dfd8 100644 --- a/docker/frontend/files/etc/copr/copr.conf +++ b/docker/frontend/files/etc/copr/copr.conf @@ -121,3 +121,6 @@ NEWS_URL = "https://fedora-copr.github.io/" NEWS_FEED_URL = "https://fedora-copr.github.io/feed.xml" OPENID_PROVIDER_URL = "https://id.fedoraproject.org" + +# When the data in EOL chroots should be deleted (in days) +DELETE_EOL_CHROOTS_AFTER = 180 diff --git a/frontend/coprs_frontend/alembic/schema/versions/69c5f19841a5_add_delete_after_and_delete_notify_.py b/frontend/coprs_frontend/alembic/schema/versions/69c5f19841a5_add_delete_after_and_delete_notify_.py new file mode 100644 index 0000000..a2b89a5 --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/69c5f19841a5_add_delete_after_and_delete_notify_.py @@ -0,0 +1,24 @@ +"""Add delete_after and delete_notify columns + +Revision ID: 69c5f19841a5 +Revises: c28451aaed50 +Create Date: 2018-10-09 00:25:40.725051 + +""" + +# revision identifiers, used by Alembic. +revision = '69c5f19841a5' +down_revision = 'c28451aaed50' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('copr_chroot', sa.Column('delete_after', sa.DateTime(), nullable=True)) + op.add_column('copr_chroot', sa.Column('delete_notify', sa.DateTime(), nullable=True)) + + +def downgrade(): + op.drop_column('copr_chroot', 'delete_notify') + op.drop_column('copr_chroot', 'delete_after') diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py index 2abecc4..1cd16f8 100644 --- a/frontend/coprs_frontend/coprs/config.py +++ b/frontend/coprs_frontend/coprs/config.py @@ -76,6 +76,9 @@ class Config(object): NEWS_URL = "https://fedora-copr.github.io/" NEWS_FEED_URL = "https://fedora-copr.github.io/feed.xml" + # When the data in EOL chroots should be deleted (in days) + DELETE_EOL_CHROOTS_AFTER = 180 + class ProductionConfig(Config): DEBUG = False diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index cab0d35..5c1c414 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -892,6 +892,11 @@ class ChrootForm(FlaskForm): with_opts = wtforms.TextField("With options") without_opts = wtforms.TextField("Without options") + +class CoprChrootExtend(FlaskForm): + name = wtforms.StringField("Chroot name", validators=[wtforms.validators.DataRequired]) + + class CoprLegalFlagForm(FlaskForm): comment = wtforms.TextAreaField("Comment") diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py index 618e53e..2decfda 100644 --- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py +++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py @@ -1,5 +1,6 @@ import os import time +import datetime from sqlalchemy import and_ from sqlalchemy.sql import func @@ -387,6 +388,11 @@ class CoprPermissionsLogic(object): return query @classmethod + def get_admins_for_copr(cls, copr): + permissions = cls.get_for_copr(copr) + return [copr.user] + [p.user for p in permissions if p.copr_admin == helpers.PermissionEnum("approved")] + + @classmethod def new(cls, copr_permission): db.session.add(copr_permission) @@ -497,6 +503,10 @@ class BranchesLogic(object): class CoprChrootsLogic(object): @classmethod + def get_multiple(cls): + return models.CoprChroot.query + + @classmethod def mock_chroots_from_names(cls, names): db_chroots = models.MockChroot.query.all() @@ -508,8 +518,8 @@ class CoprChrootsLogic(object): return mock_chroots @classmethod - def get_by_name(cls, copr, chroot_name): - mc = MockChrootsLogic.get_from_name(chroot_name, active_only=True).one() + def get_by_name(cls, copr, chroot_name, active_only=True): + mc = MockChrootsLogic.get_from_name(chroot_name, active_only=active_only).one() query = ( models.CoprChroot.query.join(models.MockChroot) .filter(models.CoprChroot.copr_id == copr.id) @@ -538,8 +548,9 @@ class CoprChrootsLogic(object): models.CoprChroot(copr=copr, mock_chroot=mock_chroot)) @classmethod - def create_chroot(cls, user, copr, mock_chroot, - buildroot_pkgs=None, repos=None, comps=None, comps_name=None, module_md=None, module_md_name=None, with_opts="", without_opts=""): + def create_chroot(cls, user, copr, mock_chroot, buildroot_pkgs=None, repos=None, comps=None, comps_name=None, + module_md=None, module_md_name=None, with_opts="", without_opts="", + delete_after=None, delete_notify=None): """ :type user: models.User :type mock_chroot: models.MockChroot @@ -553,12 +564,14 @@ class CoprChrootsLogic(object): "Only owners and admins may update their projects.") chroot = models.CoprChroot(copr=copr, mock_chroot=mock_chroot) - cls._update_chroot(buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name, chroot, with_opts, without_opts) + cls._update_chroot(buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name, chroot, + with_opts, without_opts, delete_after, delete_notify) return chroot @classmethod - def update_chroot(cls, user, copr_chroot, - buildroot_pkgs=None, repos=None, comps=None, comps_name=None, module_md=None, module_md_name=None, with_opts="", without_opts=""): + def update_chroot(cls, user, copr_chroot, buildroot_pkgs=None, repos=None, comps=None, comps_name=None, + module_md=None, module_md_name=None, with_opts="", without_opts="", + delete_after=None, delete_notify=None): """ :type user: models.User :type copr_chroot: models.CoprChroot @@ -567,11 +580,13 @@ class CoprChrootsLogic(object): user, copr_chroot.copr, "Only owners and admins may update their projects.") - cls._update_chroot(buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name, copr_chroot, with_opts, without_opts) + cls._update_chroot(buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name, + copr_chroot, with_opts, without_opts, delete_after, delete_notify) return copr_chroot @classmethod - def _update_chroot(cls, buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name, copr_chroot, with_opts, without_opts): + def _update_chroot(cls, buildroot_pkgs, repos, comps, comps_name, module_md, module_md_name, + copr_chroot, with_opts, without_opts, delete_after, delete_notify): if buildroot_pkgs is not None: copr_chroot.buildroot_pkgs = buildroot_pkgs @@ -594,6 +609,12 @@ class CoprChrootsLogic(object): copr_chroot.module_md_name = module_md_name ActionsLogic.send_update_module_md(copr_chroot) + if delete_after is not None: + copr_chroot.delete_after = delete_after + + if delete_notify is not None: + copr_chroot.delete_notify = delete_notify + db.session.add(copr_chroot) @classmethod @@ -653,6 +674,10 @@ class CoprChrootsLogic(object): db.session.delete(copr_chroot) + @classmethod + def filter_outdated(cls, query): + return query.filter(models.CoprChroot.delete_after >= datetime.datetime.now()) + class MockChrootsLogic(object): @classmethod diff --git a/frontend/coprs_frontend/coprs/mail.py b/frontend/coprs_frontend/coprs/mail.py index 323bd69..d7565b4 100644 --- a/frontend/coprs_frontend/coprs/mail.py +++ b/frontend/coprs_frontend/coprs/mail.py @@ -2,7 +2,7 @@ import flask import platform import smtplib from email.mime.text import MIMEText -from coprs import helpers +from coprs import app, helpers class Message(object): @@ -73,6 +73,33 @@ class LegalFlagMessage(Message): reporter.mail)) +class OutdatedChrootMessage(Message): + def __init__(self, copr_chroots): + """ + :param models.Copr copr: + :param list copr_chroots: list of models.CoprChroot instances + """ + self.subject = "Upcoming deletion of outdated chroots in your projects" + self.text = ("You have been notified because you are an admin of projects," + "that have some builds in outdated chroots\n\n" + + "According to the 'Copr outdated chroots removal policy'\n" + "https://docs.pagure.org/copr.copr/copr_outdated_chroots_removal_policy.html\n" + "data are going to be preserved {0} days after the chroot is EOL" + "and then automatically deleted, unless you decide to prolong the expiration period.\n\n" + + "Please, visit the projects settings if you want to extend the time.\n\n" + .format(app.config["DELETE_EOL_CHROOTS_AFTER"])) + + for chroot in copr_chroots: + self.text += ( + "Project: {0}\n" + "Chroot: {1}\n" + "Remaining: {2} days\n" + "{3}\n\n".format(chroot.copr.full_name, chroot.name, chroot.delete_after_days, + helpers.copr_url('coprs_ns.copr_repositories', chroot.copr, _external=True))) + + def send_mail(recipient, message, sender=None): """ :param str/list recipient: One recipient email as a string or multiple emails in a list diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index e4c2172..a91eb3f 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -312,6 +312,10 @@ class Copr(db.Model, helpers.Serializer, CoprSearchRelatedData): return sorted(self.active_chroots, key=lambda ch: ch.name) @property + def outdated_chroots(self): + return [chroot for chroot in self.copr_chroots if chroot.delete_after] + + @property def active_chroots_grouped(self): """ Return list of active mock_chroots of this copr @@ -1044,6 +1048,11 @@ class CoprChroot(db.Model, helpers.Serializer): with_opts = db.Column(db.Text, default="", server_default="", nullable=False) without_opts = db.Column(db.Text, default="", server_default="", nullable=False) + # Once mock_chroot gets EOL, copr_chroots are going to be deleted + # if their owner doesn't extend their time span + delete_after = db.Column(db.DateTime) + delete_notify = db.Column(db.DateTime) + def update_comps(self, comps_xml): if isinstance(comps_xml, str): data = comps_xml.encode("utf-8") @@ -1098,6 +1107,13 @@ class CoprChroot(db.Model, helpers.Serializer): def is_active(self): return self.mock_chroot.is_active + @property + def delete_after_days(self): + if not self.delete_after: + return None + now = datetime.datetime.now() + return (self.delete_after - now).days + def to_dict(self): options = {"__columns_only__": [ "buildroot_pkgs", "repos", "comps_name", "copr_id", "with_opts", "without_opts" diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/settings.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/settings.html index 97b96af..f9fe205 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/settings.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/settings.html @@ -47,6 +47,10 @@ {% endif %} {% if g.user.can_edit(copr) %} + {{ tab("repositories", "Repositories", copr_url('coprs_ns.copr_repositories', copr)) }} + {% endif %} + + {% if g.user.can_edit(copr) %} {{ tab("delete", "Delete Project", copr_url('coprs_ns.copr_delete', copr)) }} {% endif %} {% endif %} diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/settings/repositories.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/settings/repositories.html new file mode 100644 index 0000000..4c521cb --- /dev/null +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/settings/repositories.html @@ -0,0 +1,45 @@ +{% extends "coprs/detail/settings.html" %} + +{% set selected_monitor_tab = "repositories" %} +{%block settings_breadcrumb %}Repositores{% endblock %} + + +{% block tab_content %} + +

Repositories

+ +
+

Active repositories

+

See Active Releases in project Overview.

+ +

Outdated repositories

+ {% if not outdated_chroots %} +

This project has no repositories for outdated distributions.

+ {% else %} +

+ This project has following repositories for outdated distributions which are going to be removed unless you + extend the time for they should be preserved. Please see + Outdated repos removal policy + in Copr Documentation. +

+
+ + + + {% for chroot in outdated_chroots %} + + + + + + + + {% endfor %} +
ReleaseArchitectureRemaining timeExtend
{{ chroot.mock_chroot.os.capitalize() }}{{ chroot.mock_chroot.arch }}{% if chroot.delete_after_days %}{{ chroot.delete_after_days }} days{% else %} Unknown {% endif %} + +
+ {% endif %} +
+
+ +{% endblock %} diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 64968d1..524aced 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -5,6 +5,7 @@ import time import fnmatch import subprocess import json +import datetime from six.moves.urllib.parse import urljoin @@ -606,6 +607,43 @@ def copr_update_permissions(copr): return flask.redirect(url_for_copr_details(copr)) +@coprs_ns.route("///repositories/") +@coprs_ns.route("/g///repositories/") +@login_required +@req_with_copr +def copr_repositories(copr): + if not flask.g.user.can_edit(copr): + flask.flash("You don't have access to this page.", "error") + return flask.redirect(url_for_copr_details(copr)) + + return render_copr_repositories(copr) + + +def render_copr_repositories(copr): + outdated_chroots = copr.outdated_chroots + return flask.render_template("coprs/detail/settings/repositories.html", copr=copr, + outdated_chroots=outdated_chroots) + + +@coprs_ns.route("///repositories/", methods=["POST"]) +@coprs_ns.route("/g///repositories/", methods=["POST"]) +@login_required +@req_with_copr +def copr_repositories_post(copr): + if not flask.g.user.can_edit(copr): + flask.flash("You don't have access to this page.", "error") + return flask.redirect(url_for_copr_details(copr)) + + form = forms.CoprChrootExtend() + copr_chroot = coprs_logic.CoprChrootsLogic.get_by_name(copr, form.name.data, active_only=False).one() + delete_after_days = app.config["DELETE_EOL_CHROOTS_AFTER"] + 1 + delete_after_timestamp = datetime.datetime.now() + datetime.timedelta(days=delete_after_days) + coprs_logic.CoprChrootsLogic.update_chroot(flask.g.user, copr_chroot, + delete_after=delete_after_timestamp) + db.session.commit() + return render_copr_repositories(copr) + + @coprs_ns.route("/id//createrepo/", methods=["POST"]) @login_required def copr_createrepo(copr_id): diff --git a/frontend/coprs_frontend/manage.py b/frontend/coprs_frontend/manage.py index 803fb01..285ea67 100755 --- a/frontend/coprs_frontend/manage.py +++ b/frontend/coprs_frontend/manage.py @@ -4,6 +4,7 @@ import argparse import os import subprocess import sqlalchemy +import datetime import time import flask @@ -15,11 +16,12 @@ from coprs import app from coprs import db from coprs import exceptions from coprs import models +from coprs.mail import send_mail, OutdatedChrootMessage from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic, users_logic from coprs.views.misc import create_user_wrapper from coprs.whoosheers import CoprWhoosheer from sqlalchemy import and_, or_ -from coprs.helpers import chroot_to_branch +from coprs.helpers import chroot_to_branch, PermissionEnum class TestCommand(Command): @@ -265,8 +267,18 @@ class AlterChrootCommand(ChrootCommand): activate = (action == "activate") for chroot_name in chroot_names: try: - coprs_logic.MockChrootsLogic.edit_by_name( + mock_chroot = coprs_logic.MockChrootsLogic.edit_by_name( chroot_name, activate) + + if action != "eol": + continue + + for copr_chroot in mock_chroot.copr_chroots: + delete_after_days = app.config["DELETE_EOL_CHROOTS_AFTER"] + 1 + delete_after_timestamp = datetime.datetime.now() + datetime.timedelta(delete_after_days) + # Workarounding an auth here + coprs_logic.CoprChrootsLogic.update_chroot(copr_chroot.copr.user, copr_chroot, + delete_after=delete_after_timestamp) db.session.commit() except exceptions.MalformedArgumentException: self.print_invalid_format(chroot_name) @@ -278,7 +290,7 @@ class AlterChrootCommand(ChrootCommand): "-a", dest="action", help="Action to take - currently activate or deactivate", - choices=["activate", "deactivate"], + choices=["activate", "deactivate", "eol"], required=True), ) @@ -507,6 +519,73 @@ class RemoveGraphsDataCommand(Command): db.session.commit() +class NotifyOutdatedChrootsCommand(Command): + """ + Notify all admins of projects with builds in outdated chroots about upcoming deletion. + """ + option_list = [ + Option("--dry-run", action="store_true", + help="Do not actually notify the people, but rather print information on stdout"), + Option("-e", "--email", action="append", dest="email_filter", + help="Notify only "), + Option("-a", "--all", action="store_true", + help="Notify all (even the recently notified) relevant people"), + ] + + def run(self, dry_run, email_filter, all): + self.dry_run = dry_run + self.email_filter = email_filter + self.all = all + + outdated = coprs_logic.CoprChrootsLogic.filter_outdated(coprs_logic.CoprChrootsLogic.get_multiple()) + for user, chroots in self.get_user_chroots_map(outdated).items(): + chroots = self.filter_chroots([chroot for chroot in chroots]) + self.notify(user, chroots) + self.store_notify_timesamp(chroots) + + def get_user_chroots_map(self, chroots): + user_chroot_map = {} + for chroot in chroots: + for admin in coprs_logic.CoprPermissionsLogic.get_admins_for_copr(chroot.copr): + if self.email_filter and admin.mail not in self.email_filter: + continue + if admin not in user_chroot_map: + user_chroot_map[admin] = [] + user_chroot_map[admin].append(chroot) + return user_chroot_map + + def filter_chroots(self, chroots): + if self.all: + return chroots + + filtered = [] + for chroot in chroots: + if not chroot.delete_notify: + filtered.append(chroot) + continue + + now = datetime.datetime.now() + if (now - chroot.delete_notify).days >= 14: + filtered.append(chroot) + + return filtered + + def notify(self, user, chroots): + if self.dry_run: + about = ["{0} ({1})".format(chroot.copr.full_name, chroot.name) for chroot in chroots] + print("Notify {} about {}".format(user.mail, about)) + else: + msg = OutdatedChrootMessage(chroots) + send_mail(user.mail, msg) + + def store_notify_timesamp(self, chroots): + if self.dry_run: + return + for chroot in chroots: + chroot.delete_after_notify = datetime.datetime.now() + db.session.commit() + + manager = Manager(app) manager.add_command("test", TestCommand()) manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) @@ -526,6 +605,7 @@ manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) manager.add_command("update_graphs", UpdateGraphsDataCommand()) manager.add_command("vacuum_graphs", RemoveGraphsDataCommand()) +manager.add_command("notify_outdated_chroots", NotifyOutdatedChrootsCommand()) if __name__ == "__main__": manager.run() diff --git a/frontend/coprs_frontend/tests/test_mail.py b/frontend/coprs_frontend/tests/test_mail.py index 1b70583..8ff6758 100644 --- a/frontend/coprs_frontend/tests/test_mail.py +++ b/frontend/coprs_frontend/tests/test_mail.py @@ -1,4 +1,5 @@ -from coprs.mail import PermissionRequestMessage, PermissionChangeMessage, LegalFlagMessage +import datetime +from coprs.mail import PermissionRequestMessage, PermissionChangeMessage, LegalFlagMessage, OutdatedChrootMessage from tests.coprs_test_case import CoprsTestCase from coprs import app @@ -31,3 +32,33 @@ class TestMail(CoprsTestCase): "Navigate to http://localhost/admin/legal-flag/\n" "Contact on owner is: user1 \n" "Reported by user2 ") + + def test_outdated_chroot_message(self, f_users, f_coprs, f_mock_chroots, f_db): + chroots = [self.c1.copr_chroots[0], self.c2.copr_chroots[0]] + for chroot in chroots: + chroot.delete_after = datetime.datetime.now() + datetime.timedelta(days=7 + 1) # 7 days = 6d, 23h, 59m, ... + + app.config["SERVER_NAME"] = "localhost" + app.config["DELETE_EOL_CHROOTS_AFTER"] = 123 + with app.app_context(): + msg = OutdatedChrootMessage(chroots) + assert msg.subject == "Upcoming deletion of outdated chroots in your projects" + assert msg.text == ("You have been notified because you are an admin of projects," + "that have some builds in outdated chroots\n\n" + + "According to the 'Copr outdated chroots removal policy'\n" + "https://docs.pagure.org/copr.copr/copr_outdated_chroots_removal_policy.html\n" + "data are going to be preserved 123 days after the chroot is EOL" + "and then automatically deleted, unless you decide to prolong the expiration period.\n\n" + + "Please, visit the projects settings if you want to extend the time.\n\n" + + "Project: user1/foocopr\n" + "Chroot: fedora-18-x86_64\n" + "Remaining: 7 days\n" + "http://localhost/coprs/user1/foocopr/repositories/\n\n" + + "Project: user2/foocopr\n" + "Chroot: fedora-17-x86_64\n" + "Remaining: 7 days\n" + "http://localhost/coprs/user2/foocopr/repositories/\n\n")