+ %global copr_common_version 0.11.1.dev

+ 

+ BuildRequires: python3-copr-common >= %copr_common_version

+ Requires:   python3-copr-common >= %copr_common_version

+             ActionType.REMOVE_DIRS: RemoveDirs,

+ class RemoveDirs(Action):

+     """

+     Delete outdated CoprDir instances.  Frontend gives us only a list of

+     sub-directories to remove.

+     """

+     def _run_internal(self):

+         copr_dirs = json.loads(self.data["data"])

+         for copr_dir in copr_dirs:

+             assert len(copr_dir.split('/')) == 2

+             assert ':pr:' in copr_dir

+             directory = os.path.join(self.destdir, copr_dir)

+             self.log.info("RemoveDirs: removing %s", directory)

+             try:

+                 shutil.rmtree(directory)

+             except FileNotFoundError:

+                 self.log.error("RemoveDirs: %s not found", directory)

+ 

+     def run(self):

+         result = ActionResult.FAILURE

+         try:

+             self._run_internal()

+             result = ActionResult.SUCCESS

+         except OSError:

+             self.log.exception("RemoveDirs OSError")

+         return result

+ 

+ 

+     REMOVE_DIRS = 11

+ MIN_FE_BE_API = 2

+ 

+     @mock.patch("copr_backend.actions.shutil.rmtree")

+     def test_remove_dirs(self, mock_rmtree, mc_time):

+         _unused = mc_time

+         test_action = Action.create_from(

+             opts=self.opts,

+             action={

+                 "action_type": ActionType.REMOVE_DIRS,

+                 "data": json.dumps([

+                     "@python/python3.8:pr:11",

+                     "jdoe/some:pr:123",

+                 ]),

+             },

+         )

+         assert test_action.run() == ActionResult.SUCCESS

+         assert mock_rmtree.call_args_list == [

+             mock.call('/var/lib/copr/public_html/results/@python/python3.8:pr:11'),

+             mock.call('/var/lib/copr/public_html/results/jdoe/some:pr:123'),

+         ]

+         ctx.return_value.headers = {

+             "Copr-FE-BE-API-Version": "666",

+         }

+         "remove_dirs": 11,

+ Version:    0.11.1.dev

+ runuser -c '/usr/share/copr/coprs_frontend/manage.py delete-dirs' - copr-fe

+ %global copr_common_version 0.11.1.dev

+ 

+ BuildRequires: python3-copr-common >= %copr_common_version

+ Requires: python3-copr-common >= %copr_common_version

+ """

+ Admin/Cron logic for removing outdated CoprDir objects

+ """

+ 

+ import click

+ 

+ from coprs import db

+ from coprs.logic.coprs_logic import CoprDirsLogic

+ 

+ def _delete_dirs_function():

+     CoprDirsLogic.send_delete_dirs_action()

+     db.session.commit()

+ 

+ @click.command()

+ def delete_dirs():

+     """

+     Delete outdated pull request directories (e.g. like copr-dev:pr:123)

+     from the database and generate an action so the data are removed from

+     backend, too.

+     """

+     _delete_dirs_function()

+   'io.pagure.prod.pagure.pull-request.updated' : 'https://pagure.io/',

+   'org.fedoraproject.prod.pagure.git.receive': "https://src.fedoraproject.org/",

+   'org.fedoraproject.prod.pagure.pull-request.new': "https://src.fedoraproject.org/",

+   'org.fedoraproject.prod.pagure.pull-request.comment.added': "https://src.fedoraproject.org/",

+   'org.fedoraproject.prod.pagure.pull-request.updated': "https://src.fedoraproject.org/",

+     # We remove pull-request directories after some time.

+     KEEP_PR_DIRS_DAYS = 40

+ 

+             query = query.filter(models.Build.copr_dir_id==copr_dir.id)

+ import locale

+ import json

+ from functools import cmp_to_key

+ from itertools import zip_longest

+ 

+ from coprs import app, db

+     def delete_with_builds(cls, copr_dir):

+         """

+         Delete CoprDir istance from database, and transitively delete all

+         assigned Builds and BuildChroots.  No Backend action is generated.

+         """

+         models.Build.query.filter(models.Build.copr_dir_id==copr_dir.id)\

+                 .delete()

+         cls.delete(copr_dir)

+ 

+     @classmethod

+     @staticmethod

+     def _compare_dir_pairs(pair1, pair2):

+         """

+         Compare list of pairs from get_all_with_latest_submitted_build_query

+         """

+ 

+         dirname1 = pair1.CoprDir.name

+         dirname2 = pair2.CoprDir.name

+ 

+         if ':' not in dirname1:

+             return -1

+         if ':' not in dirname2:

+             return 1

+ 

+         pr = False

+         for left, right in zip_longest(dirname1.split(':'), dirname2.split(':')):

+             if None in [left, right]:

+                 return 1 if left is None else -1

+             if pr:

+                 try:

+                     # _try_ to compare as numbers now

+                     return int(right) - int(left)

+                 except ValueError:

+                     # let's fallback to string comparison

+                     pr = False

+             rc = locale.strcoll(left, right)

+             if rc != 0:

+                 return rc

+ 

+             # go down to another field

+             if left == 'pr':

+                 pr = True

+         return 0

+ 

+     @classmethod

+     def get_all_with_latest_submitted_build_query(cls, copr_id):

+         """

+         Get query returning list of pairs (CoprDir, latest_build_submitted_on)

+         """

+         subquery = (

+             db.session.query(

+                 func.max(models.Build.submitted_on)

+             )

+             .filter(models.Build.copr_dir_id==models.CoprDir.id)

+             .group_by(models.Build.copr_dir_id)

+             .label("latest_build_submitted_on")

+         )

+         return (

+             db.session.query(models.CoprDir, subquery)

+             .filter(models.CoprDir.copr_id==int(copr_id))

+         )

+ 

+     @classmethod

+     def get_all_with_latest_submitted_build(cls, copr_id):

+         """

+         Return a list of pairs like [(CoprDir, latest_build_submitted_on), ...]

+         ordered by name.

+         """

+ 

+         keep_days = app.config["KEEP_PR_DIRS_DAYS"]

+         now = time.time()

+ 

+         pairs = list(cls.get_all_with_latest_submitted_build_query(copr_id))

+ 

+         results = []

+         for pair in sorted(pairs, key=cmp_to_key(cls._compare_dir_pairs)):

+             last = pair.latest_build_submitted_on

+             copr_dir = pair.CoprDir

+ 

+             removal_candidate = True

+             if ':pr:' not in copr_dir.name:

+                 removal_candidate = False

+                 delete = False  # never ever remove this!

+                 remaining_days = 'infinity'

+                 days_since_last = 0

+ 

+             elif last is None:

+                 delete = True

+                 remaining_days = 0

+                 days_since_last = float('inf')

+ 

+             else:

+                 seconds_since_last = now - last

+                 days_since_last = seconds_since_last/3600/24

+                 delete = days_since_last > keep_days

+                 remaining_days = int(keep_days - days_since_last)

+                 if remaining_days < 0:

+                     remaining_days = 0

+ 

+             results += [{

+                 'copr_dir': copr_dir,

+                 'warn': days_since_last > keep_days / 2,

+                 'delete': delete,

+                 'remaining_days': remaining_days,

+                 'removal_candidate': removal_candidate,

+             }]

+ 

+         return results

+ 

+     @classmethod

+     def get_copr_ids_with_pr_dirs(cls):

+         """

+         Get query returning all Copr instances that have some PR directories

+         """

+         return (

+             db.session.query(models.CoprDir.copr_id)

+             .filter(models.CoprDir.name.like("%:pr:%"))

+             .group_by(models.CoprDir.copr_id)

+         )

+ 

+     @classmethod

+     def send_delete_dirs_action(cls):

+         """

+         Go through all projects, and check if there are some PR directories to

+         be removed.  Generate delete action for them, and drop them.

+         """

+         copr_ids = cls.get_copr_ids_with_pr_dirs().all()

+ 

+         remove_dirs = []

+         for copr_id in copr_ids:

+             copr_id = copr_id[0]

+             all_dirs = cls.get_all_with_latest_submitted_build(copr_id)

+             for copr_dir in all_dirs:

+                 dir_object = copr_dir["copr_dir"]

+                 if not copr_dir['delete']:

+                     continue

+ 

+                 dirname = "{}/{}".format(

+                     dir_object.copr.owner_name,

+                     dir_object.name,

+                 )

+                 print("{} is going to be deleted".format(dirname))

+                 cls.delete_with_builds(dir_object)

+                 remove_dirs.append(dirname)

+ 

+         action = models.Action(

+             action_type=ActionTypeEnum("remove_dirs"),

+             object_type="copr",

+             data=json.dumps(remove_dirs),

+             created_on=int(time.time()))

+ 

+         db.session.add(action)

+                  unique=True, postgresql_where=(main),

+                  sqlite_where=(main)),

+         {% for copr_dir_info in copr_dirs %}

+           {% set copr_dir = copr_dir_info.copr_dir %}

+             <a href="{{ copr_url('coprs_ns.copr_builds', copr) }}?dirname={{ copr_dir.name }}" class="btn btn-default btn-sm {% if current_dirname == copr_dir.name %}active{% endif %}"

+                 {% if copr_dir_info.removal_candidate %}

+                     title="Will be removed in {{ copr_dir_info.remaining_days }} days if no other build is done here"

+                 {% endif %}

+                 {% if copr_dir_info.delete %}

+                     style="color: red;"

+                 {% elif copr_dir_info.warn %}

+                     style="color: orange;"

+                 {% endif %}

+             >

+                 {{ copr_dir.name }}

+     response.headers['Copr-FE-BE-API-Version'] = '2'

+ from coprs.logic.coprs_logic import CoprDirsLogic

+     dirs = CoprDirsLogic.get_all_with_latest_submitted_build(copr.id)

+ 

+                               copr_dirs=dirs,

+ import commands.delete_dirs

+     "delete_dirs",

+ from tests.lib.pagure_pull_requests import PullRequestTrigger

+         self.pr_trigger = PullRequestTrigger(self)

+ """

+ Helper methods for generating PR builds

+ """

+ 

+ import copy

+ 

+ from unittest import mock

+ 

+ from coprs import db, models

+ from copr_common.enums import StatusEnum

+ 

+ from pagure_events import build_on_fedmsg_loop

+ 

+ 

+ class _Message:

+     def __init__(self, topic, body):

+         self.topic = topic

+         self.body = body

+ 

+ 

+ class PullRequestTrigger:

+     """ Trigger builds using "Pagure-like" message """

+     test_object = None

+ 

+     data = {

+         "msg": {

+             "agent": "test",

+             "pullrequest": {

+                 "branch": "master",

+                 "branch_from": "test_PR",

+                 "id": 1,

+                 "commit_start": "78a74b02771506daf8927b3391a669cbc32ccf10",

+                 "commit_stop": "da3d120f2ff24fa730067735c19a0b52c8bc1a44",

+                 "repo_from": {

+                     "fullname": "test/copr/copr",

+                     "url_path": "test/copr/copr",

+                 },

+                 "project": {

+                     "fullname": "test/copr/copr",

+                     "url_path": "test/copr/copr",

+                 },

+                 'status': 'Open',

+                 "comments": [],

+                 'user': {

+                     "fullname": "John Doe",

+                     "url_path": "user/jdoe",

+                     "full_url": "https://src.fedoraproject.org/user/jdoe",

+                     "name": "jdoe"

+                 },

+             }

+         }

+     }

+ 

+     def __init__(self, test_object):

+         self.test_object = test_object

+         self.build = build_on_fedmsg_loop()

+ 

+     @staticmethod

+     def _get_the_package(project, pkgname):

+         return models.Package.query.join(models.CoprDir).filter(

+             models.Package.copr_id==project.id,

+             models.Package.name==pkgname,

+             models.CoprDir.main).one_or_none()

+ 

+     @mock.patch('pagure_events.get_repeatedly', mock.Mock())

+     def build_package(self, project_name, pkgname, pr_id):

+         """ Trigger a build in Copr with name """

+         project = models.Copr.query.filter(

+             models.Copr.name == project_name).one()

+ 

+         package = self._get_the_package(project, pkgname)

+ 

+         build_count = models.Build.query.count()

+ 

+         if not package:

+             self.test_object.api3.create_distgit_package(

+                 project.name,

+                 pkgname,

+                 {"webhook_rebuild": True},

+             )

+             db.session.commit()

+ 

+         package = self._get_the_package(project, pkgname)

+ 

+         data = copy.deepcopy(self.data['msg'])

+         data['pullrequest']['project'] = {

+             "fullname": "rpms/" + pkgname,

+             "url_path": "rpms/" + pkgname,

+         }

+         data['pullrequest']['id'] = int(pr_id)

+         message = _Message(

+             'org.fedoraproject.prod.pagure.pull-request.updated',

+             data,

+         )

+         with mock.patch('pagure_events.helpers.raw_commit_changes') as patch:

+             patch.return_value = {

+                 'tests/integration/conftest.py @@ -28,6 +28,16 @@ def test_env(): return env',

+                 'tests/integration/conftest.py b/tests/integration/conftest.py index '

+                 '1747874..a2b81f6 100644 --- a/tests/integration/conftest.py +++'}

+             self.build(message)

+ 

+         builds = models.Build.query.order_by("id").all()

+ 

+         build_count_new = models.Build.query.count()

+         assert build_count_new == build_count + 1

+ 

+         build = builds[-1] # last build

+ 

+         build.source_status = StatusEnum("succeeded")

+         for bch in build.build_chroots:

+             bch.status = StatusEnum("succeeded")

+ 

+         return build

+ 

+     def build_package_with_args(self, project_name, pkgname, pr_id,

+                                 submitted_on=None):

+         """ Trigger a build in Copr with name and additional args """

+         build = self.build_package(project_name, pkgname, pr_id)

+         if submitted_on is not None:

+             build.submitted_on = submitted_on

+         db.session.commit()

+     def create_distgit_package(self, project, pkgname, options=None):

+     def create_distgit_package(self, project, pkgname, options=None):

+         if options:

+             raise NotImplementedError

+     def create_distgit_package(self, project, pkgname, options=None):

+         data = {"package_name": pkgname}

+         if options is not None:

+             data.update(options)

+         resp = self.post(route, data)

+         return self.client.post(

+ """

+ Tests for CoprDirsLogic

+ """

+ 

+ from datetime import datetime, timedelta

+ import json

+ import pytest

+ from coprs import db, models

+ from tests.coprs_test_case import (

+     CoprsTestCase,

+     TransactionDecorator,

+     new_app_context

+ )

+ from commands.delete_dirs import _delete_dirs_function

+ 

+ 

+ class TestCoprDirsLogic(CoprsTestCase):

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_no_removal(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 1)

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 2)

+         self.pr_trigger.build_package("test-pr-dirs", "other", 1)

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 1)

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 3)

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 5

+         assert models.CoprDir.query.count() == 4  # main + 3 PRs

+ 

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_no_build(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 1)

+         build = models.Build.query.get(1)

+         db.session.delete(build)

+         models.Action.query.delete()

+         db.session.commit()

+         assert models.Build.query.count() == 0

+         assert models.CoprDir.query.count() == 2

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 0

+         assert models.CoprDir.query.count() == 1

+         action = models.Action.query.one()

+         assert action.action_type == 11

+         assert json.loads(action.data) == ["user1/test-pr-dirs:pr:1"]

+ 

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_old_pr(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         old_build_on = datetime.timestamp(datetime.now() - timedelta(days=100))

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "testpkg", 1,

+                                                 old_build_on)

+         models.Action.query.delete()

+         db.session.commit()

+         assert models.Build.query.count() == 1

+         assert models.CoprDir.query.count() == 2

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 0

+         assert models.CoprDir.query.count() == 1

+         action = models.Action.query.one()

+         assert action.action_type == 11

+         assert json.loads(action.data) == ["user1/test-pr-dirs:pr:1"]

+ 

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_one_prs(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         old_build_on = datetime.timestamp(datetime.now() - timedelta(days=100))

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "testpkg", 1,

+                                                 old_build_on)

+         # this assures the pr:1 is kept

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "another", 1)

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "pr-2-package", 2,

+                                                 old_build_on)

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "pr-3-package", 3,

+                                                 old_build_on)

+         models.Action.query.delete()

+         db.session.commit()

+         assert models.Build.query.count() == 4

+         assert models.CoprDir.query.count() == 4

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 2

+         assert models.CoprDir.query.count() == 2

+         action = models.Action.query.one()

+         assert action.action_type == 11

+         assert set(json.loads(action.data)) == set(["user1/test-pr-dirs:pr:2",

+                                                     "user1/test-pr-dirs:pr:3"])