From 09bbcd9514fee71cfb9b7b60e445ef9240ae55ac Mon Sep 17 00:00:00 2001 From: Dominik Turecek Date: Mar 12 2019 07:53:26 +0000 Subject: [frontend][backend] make copr_prune_results skip already pruned outdated chroots --- diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py index e360ad0..1b0ff2e 100755 --- a/backend/run/copr_prune_results.py +++ b/backend/run/copr_prune_results.py @@ -7,6 +7,8 @@ import logging import subprocess import pwd +import json + log = logging.getLogger(__name__) from copr.exceptions import CoprException @@ -16,6 +18,7 @@ sys.path.append("/usr/share/copr/") from backend.helpers import BackendConfigReader from backend.helpers import get_auto_createrepo_status,get_persistent_status,get_auto_prune_status +from backend.frontend import FrontendClient DEF_DAYS = 14 @@ -56,8 +59,13 @@ class Pruner(object): def __init__(self, opts): self.opts = opts self.prune_days = getattr(self.opts, "prune_days", DEF_DAYS) + self.chroots = {} + self.frontend_client = FrontendClient(self.opts) def run(self): + response = self.frontend_client._post_to_frontend_repeatedly("", "chroots-prunerepo-status") + self.chroots = json.loads(response.content) + results_dir = self.opts.destdir loginfo("Pruning results dir: {} ".format(results_dir)) user_dir_names, user_dirs = list_subdir(results_dir) @@ -73,6 +81,14 @@ class Pruner(object): self.prune_project(project_path, username, projectdir) loginfo("--------------------------------------------") + loginfo("Setting final_prunerepo_done for deactivated chroots") + chroots_to_prune = [] + for chroot, active in self.chroots.items(): + if not active: + chroots_to_prune.append(chroot) + self.frontend_client._post_to_frontend_repeatedly(chroots_to_prune, "final-prunerepo-done") + + loginfo("--------------------------------------------") loginfo("Pruning finished") def prune_project(self, project_path, username, projectdir): @@ -108,6 +124,10 @@ class Pruner(object): if not os.path.isdir(chroot_path): continue + if sub_dir_name not in self.chroots: + loginfo("Final pruning already done for chroot {}/{}:{}".format(username, projectdir, sub_dir_name)) + continue + try: cmd = ['prunerepo', '--verbose', '--days={0}'.format(self.prune_days), '--cleancopr', chroot_path] stdout = runcmd(cmd) diff --git a/frontend/coprs_frontend/alembic/schema/versions/b64659389c54_add_column_final_prunerepo_done_to_mock_.py b/frontend/coprs_frontend/alembic/schema/versions/b64659389c54_add_column_final_prunerepo_done_to_mock_.py new file mode 100644 index 0000000..a37528f --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/b64659389c54_add_column_final_prunerepo_done_to_mock_.py @@ -0,0 +1,22 @@ +"""add column final_prunerepo_done to mock_chroot table + +Revision ID: b64659389c54 +Revises: ca76b7902c2f +Create Date: 2019-02-28 11:57:48.674072 + +""" + +# revision identifiers, used by Alembic. +revision = 'b64659389c54' +down_revision = 'ca76b7902c2f' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('mock_chroot', sa.Column('final_prunerepo_done', sa.Boolean, nullable=False, server_default='0')) + + +def downgrade(): + op.drop_column('mock_chroot', 'final_prunerepo_done') diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py index 841e3d9..5ec56c5 100644 --- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py +++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py @@ -791,3 +791,23 @@ class MockChrootsLogic(object): split_name.append(None) return tuple(split_name) + + @classmethod + def prunerepo_finished(cls, chroots_pruned): + for chroot_name in chroots_pruned: + chroot = cls.get_from_name(chroot_name).one() + if not chroot.is_active: + chroot.final_prunerepo_done = True + + db.session.commit() + return True + + @classmethod + def chroots_prunerepo_status(cls): + query = models.MockChroot.query + chroots = {} + for chroot in query: + if chroot.is_active or not chroot.final_prunerepo_done: + chroots[chroot.name] = chroot.is_active + + return chroots diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 1dd125a..bc518e1 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -978,6 +978,10 @@ class MockChroot(db.Model, helpers.Serializer): distgit_branch = db.relationship("DistGitBranch", backref=db.backref("chroots")) + # After a mock_chroot is EOLed, this is set to true so that copr_prune_results + # will skip all projects using this chroot + final_prunerepo_done = db.Column(db.Boolean, default=False, server_default="0", nullable=False) + @classmethod def latest_fedora_branched_chroot(cls, arch='x86_64'): return (cls.query diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py index fe11426..bed6289 100644 --- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py +++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py @@ -10,6 +10,8 @@ from coprs.logic import actions_logic from coprs.logic.builds_logic import BuildsLogic from coprs.logic.complex_logic import ComplexLogic from coprs.logic.packages_logic import PackagesLogic +from coprs.logic.coprs_logic import MockChrootsLogic +from coprs.exceptions import MalformedArgumentException from coprs.views import misc from coprs.views.backend_ns import backend_ns @@ -342,3 +344,13 @@ def reschedule_build_chroot(): response["msg"] = "build chroot is not in running states, ignoring" return flask.jsonify(response) + +@backend_ns.route("/chroots-prunerepo-status/", methods=["POST", "PUT"]) +def chroots_prunerepo_status(): + return flask.jsonify(MockChrootsLogic.chroots_prunerepo_status()) + +@backend_ns.route("/final-prunerepo-done/", methods=["POST", "PUT"]) +@misc.backend_authenticated +def final_prunerepo_done(): + chroots_pruned = flask.request.get_json() + return flask.jsonify(MockChrootsLogic.prunerepo_finished(chroots_pruned)) \ No newline at end of file