#1837 Automatically remove CoprDir instances
Merged 3 years ago by praiskup. Opened 3 years ago by praiskup.
Unknown source auto-delete-copr-dirs  into  main

file modified
+4 -1
@@ -6,6 +6,8 @@

  %global tests_version 2

  %global tests_tar test-data-copr-backend

  

+ %global copr_common_version 0.11.1.dev

+ 

  Name:       copr-backend

  Version:    1.148

  Release:    1%{?dist}
@@ -35,6 +37,7 @@

  BuildRequires: python3-setuptools

  

  BuildRequires: python3-copr

+ BuildRequires: python3-copr-common >= %copr_common_version

  BuildRequires: python3-copr-messaging

  BuildRequires: python3-daemon

  BuildRequires: python3-dateutil
@@ -73,7 +76,7 @@

  Requires:   openssh-clients

  Requires:   prunerepo >= %prunerepo_version

  Requires:   python3-copr

- Requires:   python3-copr-common >= 0.10.1.dev

+ Requires:   python3-copr-common >= %copr_common_version

  Requires:   python3-copr-messaging

  Requires:   python3-daemon

  Requires:   python3-dateutil

@@ -63,6 +63,7 @@

              ActionType.FORK: Fork,

              ActionType.BUILD_MODULE: BuildModule,

              ActionType.DELETE: Delete,

+             ActionType.REMOVE_DIRS: RemoveDirs,

          }.get(action_type, None)

  

          if action_type == ActionType.DELETE:
@@ -506,6 +507,33 @@

          return result

  

  

+ class RemoveDirs(Action):

+     """

+     Delete outdated CoprDir instances.  Frontend gives us only a list of

+     sub-directories to remove.

+     """

+     def _run_internal(self):

+         copr_dirs = json.loads(self.data["data"])

+         for copr_dir in copr_dirs:

+             assert len(copr_dir.split('/')) == 2

+             assert ':pr:' in copr_dir

+             directory = os.path.join(self.destdir, copr_dir)

+             self.log.info("RemoveDirs: removing %s", directory)

+             try:

+                 shutil.rmtree(directory)

+             except FileNotFoundError:

+                 self.log.error("RemoveDirs: %s not found", directory)

+ 

+     def run(self):

+         result = ActionResult.FAILURE

+         try:

+             self._run_internal()

+             result = ActionResult.SUCCESS

+         except OSError:

+             self.log.exception("RemoveDirs OSError")

+         return result

+ 

+ 

  # TODO: sync with ActionTypeEnum from common

  class ActionType(object):

      DELETE = 0
@@ -519,6 +547,7 @@

      UPDATE_MODULE_MD = 8  # Deprecated

      BUILD_MODULE = 9

      CANCEL_BUILD = 10

+     REMOVE_DIRS = 11

  

  

  class ActionQueueTask(QueueTask):

@@ -8,7 +8,7 @@

  from copr_common.request import SafeRequest, RequestError

  from copr_backend.exceptions import FrontendClientException

  

- MIN_FE_BE_API = 1

+ MIN_FE_BE_API = 2

  

  class FrontendClient:

      """

@@ -973,3 +973,22 @@

          with open(file, "r") as fd:

              lines = fd.readlines()

              assert lines == [text]

+ 

+     @mock.patch("copr_backend.actions.shutil.rmtree")

+     def test_remove_dirs(self, mock_rmtree, mc_time):

+         _unused = mc_time

+         test_action = Action.create_from(

+             opts=self.opts,

+             action={

+                 "action_type": ActionType.REMOVE_DIRS,

+                 "data": json.dumps([

+                     "@python/python3.8:pr:11",

+                     "jdoe/some:pr:123",

+                 ]),

+             },

+         )

+         assert test_action.run() == ActionResult.SUCCESS

+         assert mock_rmtree.call_args_list == [

+             mock.call('/var/lib/copr/public_html/results/@python/python3.8:pr:11'),

+             mock.call('/var/lib/copr/public_html/results/jdoe/some:pr:123'),

+         ]

@@ -20,6 +20,9 @@

  def f_request_method(request):

      'mock the requests.{get,post,put} method'

      with mock.patch("copr_common.request.{}".format(request.param)) as ctx:

+         ctx.return_value.headers = {

+             "Copr-FE-BE-API-Version": "666",

+         }

          yield (request.param, ctx)

  

  

@@ -35,6 +35,7 @@

          "update_module_md": 8,

          "build_module": 9,

          "cancel_build": 10,

+         "remove_dirs": 11,

      }

  

  

@@ -16,7 +16,7 @@

  %endif

  

  Name:       python-copr-common

- Version:    0.11

+ Version:    0.11.1.dev

  Release:    1%{?dist}

  Summary:    Python code used by Copr

  

@@ -7,3 +7,4 @@

  runuser -c '/usr/share/copr/coprs_frontend/manage.py vacuum-graphs' - copr-fe

  runuser -c '/usr/share/copr/coprs_frontend/manage.py clean-expired-projects' - copr-fe

  runuser -c '/usr/share/copr/coprs_frontend/manage.py clean-old-builds' - copr-fe

+ runuser -c '/usr/share/copr/coprs_frontend/manage.py delete-dirs' - copr-fe

file modified
+4 -2
@@ -6,6 +6,8 @@

  # https://fedoraproject.org/wiki/Packaging:Guidelines#Packaging_of_Additional_RPM_Macros

  %global macrosdir       %(d=%{_rpmconfigdir}/macros.d; [ -d $d ] || d=%{_sysconfdir}/rpm; echo $d)

  

+ %global copr_common_version 0.11.1.dev

+ 

  # Please bump the %%flavor_guard version every-time some incompatible change

  # happens (since the last release) in %%flavor_files set of files.  Those files

  # are basically replaced by third-party flavor providers, and any file removal,
@@ -75,7 +77,7 @@

  BuildRequires: python3-CommonMark

  BuildRequires: python3-blinker

  BuildRequires: python3-beautifulsoup4

- BuildRequires: python3-copr-common >= 0.7

+ BuildRequires: python3-copr-common >= %copr_common_version

  BuildRequires: python3-email-validator

  BuildRequires: python3-dateutil

  BuildRequires: python3-decorator
@@ -134,7 +136,7 @@

  Requires: python3-CommonMark

  Requires: python3-alembic

  Requires: python3-blinker

- Requires: python3-copr-common >= 0.7

+ Requires: python3-copr-common >= %copr_common_version

  Requires: python3-dateutil

  Requires: python3-email-validator

  Requires: python3-flask

@@ -0,0 +1,21 @@

+ """

+ Admin/Cron logic for removing outdated CoprDir objects

+ """

+ 

+ import click

+ 

+ from coprs import db

+ from coprs.logic.coprs_logic import CoprDirsLogic

+ 

+ def _delete_dirs_function():

+     CoprDirsLogic.send_delete_dirs_action()

+     db.session.commit()

+ 

+ @click.command()

+ def delete_dirs():

+     """

+     Delete outdated pull request directories (e.g. like copr-dev:pr:123)

+     from the database and generate an action so the data are removed from

+     backend, too.

+     """

+     _delete_dirs_function()

@@ -71,5 +71,9 @@

    'io.pagure.prod.pagure.git.receive' : 'https://pagure.io/',

    'io.pagure.prod.pagure.pull-request.new' : 'https://pagure.io/',

    'io.pagure.prod.pagure.pull-request.comment.added' : 'https://pagure.io/',

-   'io.pagure.prod.pagure.pull-request.updated' : 'https://pagure.io/'

+   'io.pagure.prod.pagure.pull-request.updated' : 'https://pagure.io/',

+   'org.fedoraproject.prod.pagure.git.receive': "https://src.fedoraproject.org/",

+   'org.fedoraproject.prod.pagure.pull-request.new': "https://src.fedoraproject.org/",

+   'org.fedoraproject.prod.pagure.pull-request.comment.added': "https://src.fedoraproject.org/",

+   'org.fedoraproject.prod.pagure.pull-request.updated': "https://src.fedoraproject.org/",

  }

@@ -125,6 +125,9 @@

      # Setting this to True requires special installation

      PROFILER = False

  

+     # We remove pull-request directories after some time.

+     KEEP_PR_DIRS_DAYS = 40

+ 

  class ProductionConfig(Config):

      DEBUG = False

      # SECRET_KEY = "put_some_secret_here"

@@ -394,9 +394,7 @@

          query = models.Build.query.filter(models.Build.copr_id==copr.id)

          if dirname:

              copr_dir = coprs_logic.CoprDirsLogic.get_by_copr(copr, dirname).one()

-         else:

-             copr_dir = copr.main_dir

-         query = query.filter(models.Build.copr_dir_id==copr_dir.id)

+             query = query.filter(models.Build.copr_dir_id==copr_dir.id)

          query = query.options(selectinload('build_chroots'), selectinload('package'))

          return query

  

@@ -1,6 +1,11 @@

+ import locale

+ import json

  import os

  import time

  import datetime

+ from functools import cmp_to_key

+ from itertools import zip_longest

+ 

  import flask

  

  from sqlalchemy import and_, not_
@@ -12,7 +17,7 @@

  from sqlalchemy.orm.attributes import get_history

  

  from copr_common.enums import ActionTypeEnum, BackendResultEnum, ActionPriorityEnum

- from coprs import db

+ from coprs import app, db

  from coprs import exceptions

  from coprs import helpers

  from coprs import models
@@ -559,10 +564,162 @@

          db.session.delete(copr_dir)

  

      @classmethod

+     def delete_with_builds(cls, copr_dir):

+         """

+         Delete CoprDir istance from database, and transitively delete all

+         assigned Builds and BuildChroots.  No Backend action is generated.

+         """

+         models.Build.query.filter(models.Build.copr_dir_id==copr_dir.id)\

+                 .delete()

+         cls.delete(copr_dir)

+ 

+     @classmethod

      def delete_all_by_copr(cls, copr):

          for copr_dir in copr.dirs:

              db.session.delete(copr_dir)

  

+     @staticmethod

+     def _compare_dir_pairs(pair1, pair2):

+         """

+         Compare list of pairs from get_all_with_latest_submitted_build_query

+         """

+ 

+         dirname1 = pair1.CoprDir.name

+         dirname2 = pair2.CoprDir.name

+ 

+         if ':' not in dirname1:

+             return -1

+         if ':' not in dirname2:

+             return 1

+ 

+         pr = False

+         for left, right in zip_longest(dirname1.split(':'), dirname2.split(':')):

+             if None in [left, right]:

+                 return 1 if left is None else -1

+             if pr:

+                 try:

+                     # _try_ to compare as numbers now

+                     return int(right) - int(left)

+                 except ValueError:

+                     # let's fallback to string comparison

+                     pr = False

+             rc = locale.strcoll(left, right)

+             if rc != 0:

+                 return rc

+ 

+             # go down to another field

+             if left == 'pr':

+                 pr = True

+         return 0

+ 

+     @classmethod

+     def get_all_with_latest_submitted_build_query(cls, copr_id):

+         """

+         Get query returning list of pairs (CoprDir, latest_build_submitted_on)

+         """

+         subquery = (

+             db.session.query(

+                 func.max(models.Build.submitted_on)

+             )

+             .filter(models.Build.copr_dir_id==models.CoprDir.id)

+             .group_by(models.Build.copr_dir_id)

+             .label("latest_build_submitted_on")

+         )

+         return (

+             db.session.query(models.CoprDir, subquery)

+             .filter(models.CoprDir.copr_id==int(copr_id))

+         )

+ 

+     @classmethod

+     def get_all_with_latest_submitted_build(cls, copr_id):

+         """

+         Return a list of pairs like [(CoprDir, latest_build_submitted_on), ...]

+         ordered by name.

+         """

+ 

+         keep_days = app.config["KEEP_PR_DIRS_DAYS"]

+         now = time.time()

+ 

+         pairs = list(cls.get_all_with_latest_submitted_build_query(copr_id))

+ 

+         results = []

+         for pair in sorted(pairs, key=cmp_to_key(cls._compare_dir_pairs)):

+             last = pair.latest_build_submitted_on

+             copr_dir = pair.CoprDir

+ 

+             removal_candidate = True

+             if ':pr:' not in copr_dir.name:

+                 removal_candidate = False

+                 delete = False  # never ever remove this!

+                 remaining_days = 'infinity'

+                 days_since_last = 0

+ 

+             elif last is None:

+                 delete = True

+                 remaining_days = 0

+                 days_since_last = float('inf')

+ 

+             else:

+                 seconds_since_last = now - last

+                 days_since_last = seconds_since_last/3600/24

+                 delete = days_since_last > keep_days

+                 remaining_days = int(keep_days - days_since_last)

+                 if remaining_days < 0:

+                     remaining_days = 0

+ 

+             results += [{

+                 'copr_dir': copr_dir,

+                 'warn': days_since_last > keep_days / 2,

+                 'delete': delete,

+                 'remaining_days': remaining_days,

+                 'removal_candidate': removal_candidate,

+             }]

+ 

+         return results

+ 

+     @classmethod

+     def get_copr_ids_with_pr_dirs(cls):

+         """

+         Get query returning all Copr instances that have some PR directories

+         """

+         return (

+             db.session.query(models.CoprDir.copr_id)

+             .filter(models.CoprDir.name.like("%:pr:%"))

+             .group_by(models.CoprDir.copr_id)

+         )

+ 

+     @classmethod

+     def send_delete_dirs_action(cls):

+         """

+         Go through all projects, and check if there are some PR directories to

+         be removed.  Generate delete action for them, and drop them.

+         """

+         copr_ids = cls.get_copr_ids_with_pr_dirs().all()

+ 

+         remove_dirs = []

+         for copr_id in copr_ids:

+             copr_id = copr_id[0]

+             all_dirs = cls.get_all_with_latest_submitted_build(copr_id)

+             for copr_dir in all_dirs:

+                 dir_object = copr_dir["copr_dir"]

+                 if not copr_dir['delete']:

+                     continue

+ 

+                 dirname = "{}/{}".format(

+                     dir_object.copr.owner_name,

+                     dir_object.name,

+                 )

+                 print("{} is going to be deleted".format(dirname))

+                 cls.delete_with_builds(dir_object)

+                 remove_dirs.append(dirname)

+ 

+         action = models.Action(

+             action_type=ActionTypeEnum("remove_dirs"),

+             object_type="copr",

+             data=json.dumps(remove_dirs),

+             created_on=int(time.time()))

+ 

+         db.session.add(action)

  

  @listens_for(models.Copr.auto_createrepo, 'set')

  def on_auto_createrepo_change(target_copr, value_acr, old_value_acr, initiator):

@@ -743,11 +743,9 @@

      copr = db.relationship("Copr", backref=db.backref("dirs"))

  

      __table_args__ = (

-         # TODO: Drop the PG only index.  Add (a) normal unique index, (b) add a

-         # check index for 'main == true' (NULLs are not calculated), and (c)

-         # flip all the false values to NULL.

          db.Index('only_one_main_copr_dir', copr_id, main,

-                  unique=True, postgresql_where=(main==True)),

+                  unique=True, postgresql_where=(main),

+                  sqlite_where=(main)),

  

          db.UniqueConstraint('ownername', 'name',

                              name='ownername_copr_dir_uniq'),

@@ -35,10 +35,20 @@

          <a href="{{ copr_url('coprs_ns.copr_builds', copr) }}" class="btn btn-default btn-sm {% if not current_dirname %}active{% endif %}">

            all builds

          </a>

-         {% for copr_dir in copr.dirs|sort(attribute='name') %}

+         {% for copr_dir_info in copr_dirs %}

+           {% set copr_dir = copr_dir_info.copr_dir %}

            {% if copr_dir.main or (copr_dir.packages | count) > 0 %}

-             <a href="{{ copr_url('coprs_ns.copr_builds', copr) }}?dirname={{ copr_dir.name }}" class="btn btn-default btn-sm {% if current_dirname == copr_dir.name %}active{% endif %}">

-               {{ copr_dir.name }}

+             <a href="{{ copr_url('coprs_ns.copr_builds', copr) }}?dirname={{ copr_dir.name }}" class="btn btn-default btn-sm {% if current_dirname == copr_dir.name %}active{% endif %}"

+                 {% if copr_dir_info.removal_candidate %}

+                     title="Will be removed in {{ copr_dir_info.remaining_days }} days if no other build is done here"

+                 {% endif %}

+                 {% if copr_dir_info.delete %}

+                     style="color: red;"

+                 {% elif copr_dir_info.warn %}

+                     style="color: orange;"

+                 {% endif %}

+             >

+                 {{ copr_dir.name }}

              </a>

            {% endif %}

          {% endfor %}

@@ -27,7 +27,7 @@

      do something new with the protocol.  On the Backend/builder side we can

      setup the version according to our needs.

      """

-     response.headers['Copr-FE-BE-API-Version'] = '1'

+     response.headers['Copr-FE-BE-API-Version'] = '2'

      return response

  

  

@@ -10,6 +10,7 @@

  from coprs.logic import builds_logic

  from coprs.logic.builds_logic import BuildsLogic

  from coprs.logic.complex_logic import ComplexLogic

+ from coprs.logic.coprs_logic import CoprDirsLogic

  

  from coprs.views.misc import (login_required, req_with_copr, send_build_icon)

  from coprs.views.coprs_ns import coprs_ns
@@ -59,10 +60,13 @@

      dirname = flask.request.args.get('dirname')

      builds_query = builds_logic.BuildsLogic.get_copr_builds_list(copr, dirname)

      builds = builds_query.yield_per(1000)

+     dirs = CoprDirsLogic.get_all_with_latest_submitted_build(copr.id)

+ 

      response = flask.Response(stream_with_context(helpers.stream_template("coprs/detail/builds.html",

                                copr=copr,

                                builds=builds,

                                current_dirname=dirname,

+                               copr_dirs=dirs,

                                flashes=flashes)))

  

      flask.session.pop('_flashes', [])

@@ -16,6 +16,7 @@

  import commands.create_chroot

  import commands.alter_chroot

  import commands.display_chroots

+ import commands.delete_dirs

  import commands.drop_chroot

  import commands.branch_fedora

  import commands.comment_chroot
@@ -82,6 +83,7 @@

      "clean_expired_projects",

      "clean_old_builds",

      "delete_orphans",

+     "delete_dirs",

  ]

  

  

@@ -22,6 +22,7 @@

  from coprs.logic.dist_git_logic import DistGitLogic

  

  from tests.request_test_api import WebUIRequests, API3Requests, BackendRequests

+ from tests.lib.pagure_pull_requests import PullRequestTrigger

  

  class CoprsTestCase(object):

  
@@ -75,6 +76,7 @@

          self.web_ui = WebUIRequests(self)

          self.api3 = API3Requests(self)

          self.backend = BackendRequests(self)

+         self.pr_trigger = PullRequestTrigger(self)

  

      def teardown_method(self, method):

          # delete just data, not the tables

empty or binary file added
@@ -0,0 +1,121 @@

+ """

+ Helper methods for generating PR builds

+ """

+ 

+ import copy

+ 

+ from unittest import mock

+ 

+ from coprs import db, models

+ from copr_common.enums import StatusEnum

+ 

+ from pagure_events import build_on_fedmsg_loop

+ 

+ 

+ class _Message:

+     def __init__(self, topic, body):

+         self.topic = topic

+         self.body = body

+ 

+ 

+ class PullRequestTrigger:

+     """ Trigger builds using "Pagure-like" message """

+     test_object = None

+ 

+     data = {

+         "msg": {

+             "agent": "test",

+             "pullrequest": {

+                 "branch": "master",

+                 "branch_from": "test_PR",

+                 "id": 1,

+                 "commit_start": "78a74b02771506daf8927b3391a669cbc32ccf10",

+                 "commit_stop": "da3d120f2ff24fa730067735c19a0b52c8bc1a44",

+                 "repo_from": {

+                     "fullname": "test/copr/copr",

+                     "url_path": "test/copr/copr",

+                 },

+                 "project": {

+                     "fullname": "test/copr/copr",

+                     "url_path": "test/copr/copr",

+                 },

+                 'status': 'Open',

+                 "comments": [],

+                 'user': {

+                     "fullname": "John Doe",

+                     "url_path": "user/jdoe",

+                     "full_url": "https://src.fedoraproject.org/user/jdoe",

+                     "name": "jdoe"

+                 },

+             }

+         }

+     }

+ 

+     def __init__(self, test_object):

+         self.test_object = test_object

+         self.build = build_on_fedmsg_loop()

+ 

+     @staticmethod

+     def _get_the_package(project, pkgname):

+         return models.Package.query.join(models.CoprDir).filter(

+             models.Package.copr_id==project.id,

+             models.Package.name==pkgname,

+             models.CoprDir.main).one_or_none()

+ 

+     @mock.patch('pagure_events.get_repeatedly', mock.Mock())

+     def build_package(self, project_name, pkgname, pr_id):

+         """ Trigger a build in Copr with name """

+         project = models.Copr.query.filter(

+             models.Copr.name == project_name).one()

+ 

+         package = self._get_the_package(project, pkgname)

+ 

+         build_count = models.Build.query.count()

+ 

+         if not package:

+             self.test_object.api3.create_distgit_package(

+                 project.name,

+                 pkgname,

+                 {"webhook_rebuild": True},

+             )

+             db.session.commit()

+ 

+         package = self._get_the_package(project, pkgname)

+ 

+         data = copy.deepcopy(self.data['msg'])

+         data['pullrequest']['project'] = {

+             "fullname": "rpms/" + pkgname,

+             "url_path": "rpms/" + pkgname,

+         }

+         data['pullrequest']['id'] = int(pr_id)

+         message = _Message(

+             'org.fedoraproject.prod.pagure.pull-request.updated',

+             data,

+         )

+         with mock.patch('pagure_events.helpers.raw_commit_changes') as patch:

+             patch.return_value = {

+                 'tests/integration/conftest.py @@ -28,6 +28,16 @@ def test_env(): return env',

+                 'tests/integration/conftest.py b/tests/integration/conftest.py index '

+                 '1747874..a2b81f6 100644 --- a/tests/integration/conftest.py +++'}

+             self.build(message)

+ 

+         builds = models.Build.query.order_by("id").all()

+ 

+         build_count_new = models.Build.query.count()

+         assert build_count_new == build_count + 1

+ 

+         build = builds[-1] # last build

+ 

+         build.source_status = StatusEnum("succeeded")

+         for bch in build.build_chroots:

+             bch.status = StatusEnum("succeeded")

+ 

+         return build

+ 

+     def build_package_with_args(self, project_name, pkgname, pr_id,

+                                 submitted_on=None):

+         """ Trigger a build in Copr with name and additional args """

+         build = self.build_package(project_name, pkgname, pr_id)

+         if submitted_on is not None:

+             build.submitted_on = submitted_on

+         db.session.commit()

@@ -54,7 +54,7 @@

          """ Modify CoprChroot """

          raise NotImplementedError

  

-     def create_distgit_package(self, project, pkgname):

+     def create_distgit_package(self, project, pkgname, options=None):

          """ Modify CoprChroot """

          raise NotImplementedError

  
@@ -117,7 +117,9 @@

              assert resp.status_code == 302

          return resp

  

-     def create_distgit_package(self, project, pkgname):

+     def create_distgit_package(self, project, pkgname, options=None):

+         if options:

+             raise NotImplementedError

          data = {

              "package_name": pkgname,

          }
@@ -270,10 +272,13 @@

          resp = self.post(route, data)

          return resp

  

-     def create_distgit_package(self, project, pkgname):

+     def create_distgit_package(self, project, pkgname, options=None):

          route = "/api_3/package/add/{}/{}/{}/distgit".format(

              self.transaction_username, project, pkgname)

-         resp = self.post(route, {"package_name": pkgname})

+         data = {"package_name": pkgname}

+         if options is not None:

+             data.update(options)

+         resp = self.post(route, data)

          return resp

  

      def rebuild_package(self, project, pkgname, build_options=None):
@@ -300,7 +305,7 @@

  

      def update(self, data):

          """ Post to the "/backend/update/" using a dict """

-         self.client.post(

+         return self.client.post(

              "/backend/update/",

              content_type="application/json",

              headers=self.test_class_object.auth_header,

@@ -0,0 +1,98 @@

+ """

+ Tests for CoprDirsLogic

+ """

+ 

+ from datetime import datetime, timedelta

+ import json

+ import pytest

+ from coprs import db, models

+ from tests.coprs_test_case import (

+     CoprsTestCase,

+     TransactionDecorator,

+     new_app_context

+ )

+ from commands.delete_dirs import _delete_dirs_function

+ 

+ 

+ class TestCoprDirsLogic(CoprsTestCase):

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_no_removal(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 1)

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 2)

+         self.pr_trigger.build_package("test-pr-dirs", "other", 1)

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 1)

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 3)

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 5

+         assert models.CoprDir.query.count() == 4  # main + 3 PRs

+ 

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_no_build(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         self.pr_trigger.build_package("test-pr-dirs", "testpkg", 1)

+         build = models.Build.query.get(1)

+         db.session.delete(build)

+         models.Action.query.delete()

+         db.session.commit()

+         assert models.Build.query.count() == 0

+         assert models.CoprDir.query.count() == 2

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 0

+         assert models.CoprDir.query.count() == 1

+         action = models.Action.query.one()

+         assert action.action_type == 11

+         assert json.loads(action.data) == ["user1/test-pr-dirs:pr:1"]

+ 

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_old_pr(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         old_build_on = datetime.timestamp(datetime.now() - timedelta(days=100))

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "testpkg", 1,

+                                                 old_build_on)

+         models.Action.query.delete()

+         db.session.commit()

+         assert models.Build.query.count() == 1

+         assert models.CoprDir.query.count() == 2

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 0

+         assert models.CoprDir.query.count() == 1

+         action = models.Action.query.one()

+         assert action.action_type == 11

+         assert json.loads(action.data) == ["user1/test-pr-dirs:pr:1"]

+ 

+     @TransactionDecorator("u1")

+     @new_app_context

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_users_api", "f_db")

+     def test_coprdir_cleanup_one_prs(self):

+         self.api3.new_project("test-pr-dirs", ["fedora-17-i386"])

+         old_build_on = datetime.timestamp(datetime.now() - timedelta(days=100))

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "testpkg", 1,

+                                                 old_build_on)

+         # this assures the pr:1 is kept

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "another", 1)

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "pr-2-package", 2,

+                                                 old_build_on)

+         self.pr_trigger.build_package_with_args("test-pr-dirs", "pr-3-package", 3,

+                                                 old_build_on)

+         models.Action.query.delete()

+         db.session.commit()

+         assert models.Build.query.count() == 4

+         assert models.CoprDir.query.count() == 4

+         _delete_dirs_function()

+         # nothing got removed

+         assert models.Build.query.count() == 2

+         assert models.CoprDir.query.count() == 2

+         action = models.Action.query.one()

+         assert action.action_type == 11

+         assert set(json.loads(action.data)) == set(["user1/test-pr-dirs:pr:2",

+                                                     "user1/test-pr-dirs:pr:3"])

no initial comment

Build succeeded.

rebased onto 4127f388adf25adce0b02e5476075356d16b4256

3 years ago

Build succeeded.

the right way to test membership is if ':' not in item1

pair1 and pair2 I would like more than item1 and item2, but it's just an idea

When can a FileNotFoundError exception be thrown when trying to delete a directory? Wouldn't it be better to capture OSError here?

I tried the new command delete-dirs and the directories were deleted so this works fine.

I would rather fail hard if there was a different exception than FileNotFoundError.

rebased onto e55576c7eec64da152af7a0caa0d486aba335d87

3 years ago

Thank you for taking a look, I fixed the remarks (except for the exception).

4 new commits added

  • frontend: pg-only index => pg|sqlite-only index
  • backend, frontend, common: automatically delete PR CoprDirs
  • frontend: colorize CoprDir-buttons on builds page
  • frontend: by default we show builds from all dirs
3 years ago

Build succeeded.

Thank you for taking a look, I fixed the remarks (except for the exception).

Ok, thank you.

rebased onto be3e1ad

3 years ago

Merge Failed.

This change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.

Build succeeded.

Merging (per +1 on the meeting).

Pull-Request has been merged by praiskup

3 years ago
Metadata
Changes Summary 24