From d7615a65043438183b307f2868c4a347ae34656d Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 09 2019 12:59:59 +0000 Subject: [frontend][python][cli] support temporary projects Fixes: #615, PR#649 --- diff --git a/cli/copr_cli/main.py b/cli/copr_cli/main.py index 850dc46..498b5ad 100644 --- a/cli/copr_cli/main.py +++ b/cli/copr_cli/main.py @@ -354,6 +354,7 @@ class Commands(object): persistent=args.persistent, auto_prune=ON_OFF_MAP[args.auto_prune], use_bootstrap_container=ON_OFF_MAP[args.use_bootstrap_container], + delete_after_days=args.delete_after_days, ) print("New project was successfully created.") @@ -374,6 +375,7 @@ class Commands(object): auto_prune=ON_OFF_MAP[args.auto_prune], use_bootstrap_container=ON_OFF_MAP[args.use_bootstrap_container], chroots=args.chroots, + delete_after_days=args.delete_after_days, ) @requires_api_auth @@ -776,6 +778,8 @@ def setup_parser(): This option can only be specified by a COPR admin.") parser_create.add_argument("--use-bootstrap", choices=["on", "off"], dest="use_bootstrap_container", help="If mock bootstrap container is used to initialize the buildroot.") + parser_create.add_argument("--delete-after-days", default=None, metavar='DAYS', + help="Delete the project after the specfied period of time") parser_create.set_defaults(func="action_create") # create the parser for the "modify_project" command @@ -801,6 +805,10 @@ def setup_parser(): This option can only be specified by a COPR admin.") parser_modify.add_argument("--use-bootstrap", choices=["on", "off"], dest="use_bootstrap_container", help="If mock bootstrap container is used to initialize the buildroot.") + parser_modify.add_argument("--delete-after-days", default=None, metavar='DAYS', + help=("Delete the project after the specfied " + "period of time, empty or -1 disables, " + "(default is \"don't change\")")) parser_modify.set_defaults(func="action_modify_project") # create the parser for the "delete" command diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py index 2feacc8..945e76e 100644 --- a/cli/tests/test_cli.py +++ b/cli/tests/test_cli.py @@ -373,7 +373,8 @@ def test_create_project(config_from_file, project_proxy_add, capsys): "instructions": "instruction string", "chroots": ["f20", "f21"], "additional_repos": ["repo1", "repo2"], "unlisted_on_hp": None, "devel_mode": None, "enable_net": False, - "use_bootstrap_container": None + "use_bootstrap_container": None, + "delete_after_days": None, } assert stdout == "New project was successfully created.\n" diff --git a/doc/user_documentation.rst b/doc/user_documentation.rst index a48dc26..c540907 100644 --- a/doc/user_documentation.rst +++ b/doc/user_documentation.rst @@ -138,6 +138,15 @@ used to create SRPM). For more info, have a look at :ref:`custom_source_method`. +Temporary projects +------------------ + +If you want have your copr project deleted automatically after some time +(because it is some CI/CD project, some testing stuff, etc.) you can set the +"delete after days" option in web UI or on command-line: +``copr-cli create your-project ... --delete-after-days 10`` + + GitHub Webhooks --------------- diff --git a/frontend/conf/cron.daily/copr-frontend b/frontend/conf/cron.daily/copr-frontend index f8c8662..6d294e5 100644 --- a/frontend/conf/cron.daily/copr-frontend +++ b/frontend/conf/cron.daily/copr-frontend @@ -1,3 +1,4 @@ #!/usr/bin/sh runuser -c '/usr/share/copr/coprs_frontend/manage.py vacuum_graphs' - copr-fe +runuser -c '/usr/share/copr/coprs_frontend/manage.py clean_expired_project' - copr-fe diff --git a/frontend/coprs_frontend/alembic/schema/versions/b828274ddebf_temporary_project.py b/frontend/coprs_frontend/alembic/schema/versions/b828274ddebf_temporary_project.py new file mode 100644 index 0000000..d0abd8c --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/b828274ddebf_temporary_project.py @@ -0,0 +1,21 @@ +""" +temporary project + +Revision ID: b828274ddebf +Revises: b64659389c54 +Create Date: 2019-04-05 11:55:08.004627 +""" + +import sqlalchemy as sa +from alembic import op + +revision = 'b828274ddebf' +down_revision = 'b8a8a1345ed9' + +def upgrade(): + op.add_column('copr', sa.Column('delete_after', sa.DateTime(), nullable=True)) + op.create_index(op.f('ix_copr_delete_after'), 'copr', ['delete_after'], unique=False) + +def downgrade(): + op.drop_index(op.f('ix_copr_delete_after'), table_name='copr') + op.drop_column('copr', 'delete_after') diff --git a/frontend/coprs_frontend/commands/clean_expired_projects.py b/frontend/coprs_frontend/commands/clean_expired_projects.py new file mode 100644 index 0000000..c7316f8 --- /dev/null +++ b/frontend/coprs_frontend/commands/clean_expired_projects.py @@ -0,0 +1,15 @@ +from flask_script import Command +from coprs import db_session_scope +from coprs.logic.coprs_logic import CoprsLogic + + +class CleanExpiredProjectsCommand(Command): + """ + Clean all the expired temporary projects. This command is meant to be + executed by cron. + """ + + # pylint: disable=method-hidden + def run(self): + with db_session_scope(): + CoprsLogic.delete_expired_projects() diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index 4f2eb01..f82eb76 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -263,6 +263,13 @@ class CoprFormFactory(object): instructions = wtforms.TextAreaField("Instructions") + delete_after_days = wtforms.IntegerField( + "Delete after days", + validators=[ + wtforms.validators.Optional(), + wtforms.validators.NumberRange(min=0, max=60), + ]) + repos = wtforms.TextAreaField( "External Repositories", validators=[UrlRepoListValidator()], @@ -984,6 +991,11 @@ class CoprModifyForm(FlaskForm): auto_prune = wtforms.BooleanField(validators=[wtforms.validators.Optional()], false_values=FALSE_VALUES) use_bootstrap_container = wtforms.BooleanField(validators=[wtforms.validators.Optional()], false_values=FALSE_VALUES) follow_fedora_branching = wtforms.BooleanField(validators=[wtforms.validators.Optional()], false_values=FALSE_VALUES) + follow_fedora_branching = wtforms.BooleanField(default=True, false_values=FALSE_VALUES) + delete_after_days = wtforms.IntegerField( + validators=[wtforms.validators.Optional(), + wtforms.validators.NumberRange(min=-1, max=60)], + filters=[(lambda x : -1 if x is None else x)]) # Deprecated, use `enable_net` instead build_enable_net = wtforms.BooleanField(validators=[wtforms.validators.Optional()], false_values=FALSE_VALUES) diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py index dbfb671..5ad53b2 100644 --- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py +++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py @@ -370,6 +370,19 @@ class CoprsLogic(object): raise exceptions.InsufficientRightsException( "Only owners may delete their projects.") + @classmethod + def delete_expired_projects(cls): + query = ( + models.Copr.query + .filter(models.Copr.delete_after.isnot(None)) + .filter(models.Copr.delete_after < datetime.datetime.now()) + .filter(models.Copr.deleted.isnot(True)) + ) + for copr in query.all(): + print("deleting project '{}'".format(copr.full_name)) + CoprsLogic.delete_unsafe(copr.user, copr) + + class CoprPermissionsLogic(object): @classmethod diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 4b80db4..df9d153 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -256,6 +256,9 @@ class _CoprPublic(db.Model, helpers.Serializer, CoprSearchRelatedData): scm_repo_url = db.Column(db.Text) scm_api_type = db.Column(db.Text) + # temporary project if non-null + delete_after = db.Column(db.DateTime, index=True, nullable=True) + __mapper_args__ = { "order_by": created_on.desc() } @@ -470,6 +473,30 @@ class Copr(db.Model, helpers.Serializer): def new_webhook_secret(self): self.webhook_secret = str(uuid.uuid4()) + @property + def delete_after_days(self): + if self.delete_after is None: + return None + + delta = self.delete_after - datetime.datetime.now() + return delta.days if delta.days > 0 else 0 + + @delete_after_days.setter + def delete_after_days(self, days): + if days is None or days == -1: + self.delete_after = None + return + + delete_after = datetime.datetime.now() + datetime.timedelta(days=days+1) + delete_after = delete_after.replace(hour=0, minute=0, second=0, microsecond=0) + self.delete_after = delete_after + + @property + def delete_after_msg(self): + if self.delete_after_days == 0: + return "will be deleted ASAP" + return "will be deleted after {} days".format(self.delete_after_days) + class CoprPermission(db.Model, helpers.Serializer): """ diff --git a/frontend/coprs_frontend/coprs/static/css/custom-styles.css b/frontend/coprs_frontend/coprs/static/css/custom-styles.css index 9bea1cf..308b43b 100644 --- a/frontend/coprs_frontend/coprs/static/css/custom-styles.css +++ b/frontend/coprs_frontend/coprs/static/css/custom-styles.css @@ -34,3 +34,7 @@ span.padding { width: 5px; display: inline-block; } + +input.short-input-field { + width: 10em; +} diff --git a/frontend/coprs_frontend/coprs/templates/_helpers.html b/frontend/coprs_frontend/coprs/templates/_helpers.html index 259d9f6..6a1a85b 100644 --- a/frontend/coprs_frontend/coprs/templates/_helpers.html +++ b/frontend/coprs_frontend/coprs/templates/_helpers.html @@ -1,4 +1,4 @@ -{% macro render_field(field, label=None, class='', info=None) %} +{% macro render_field(field, label=None, class=None, info=None) %} {% if not kwargs['hidden'] %}
@@ -6,7 +6,7 @@ {{ label or field.label }}:
- {{ field(class="form-control", **kwargs)|safe }} + {{ field(class="form-control" + (" " + class if class else ""), **kwargs)|safe }}
    {% if info %} {% for line in (info if info is not string else [info]) %} diff --git a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html index cbead2d..4bfe160 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html @@ -44,8 +44,10 @@ {{ render_field(form.homepage, label='Homepage', placeholder='Optional - project homepage') }} {{ render_field(form.contact, label='Contact', placeholder='Optional - email address or contact url', info="Use e-mail address or a link to your issue tracker. This information will be shown publicly.") }} - - + {{ render_field(form.delete_after_days, + class="short-input-field", + placeholder='Optional', + info='Delete the project after the specfied period of time (empty = disabled)') }}
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail.html b/frontend/coprs_frontend/coprs/templates/coprs/detail.html index 6b30284..cc96233 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail.html @@ -29,6 +29,9 @@

{{ copr_title(copr) }}

+ {% if copr.delete_after %} + temporary project: {{ copr.delete_after_msg }} + {% endif %} {% if copr.forked_from %} ( forked from {{ copr_title(copr.forked_from) }}) {% endif %} diff --git a/frontend/coprs_frontend/coprs/templates/coprs/show.html b/frontend/coprs_frontend/coprs/templates/coprs/show.html index 3198f12..10ed4e8 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/show.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/show.html @@ -15,9 +15,14 @@ {% for copr in coprs %} -

- {{ copr_name(copr) }} +
+

+ {{ copr_name(copr) }}

+ {% if copr.delete_after %} + (temporary project, {{ copr.delete_after_msg }}) + {% endif %} +
{{ copr.description|markdown|remove_anchor|default('Description not filled in by author. Very likely personal repository for testing purpose, which you should not use.', true) }}
    diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_projects.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_projects.py index eb7af19..3dbb3e2 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_projects.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_projects.py @@ -125,6 +125,7 @@ def add_project(ownername): homepage=form.homepage.data, contact=form.contact.data, disable_createrepo=form.disable_createrepo.data, + delete_after_days=form.delete_after_days.data, ) db.session.commit() except (DuplicateException, diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index a7088ff..74ad01b 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -153,55 +153,21 @@ def copr_add(username=None, group_name=None): @coprs_ns.route("/g//new/", methods=["POST"]) @login_required def copr_new(username=None, group_name=None): - if group_name: - return process_group_copr_new(group_name) - return process_copr_new(username) - - -def process_group_copr_new(group_name): - group = ComplexLogic.get_group_by_name_safe(group_name) - form = forms.CoprFormFactory.create_form_cls(group=group)() - - if form.validate_on_submit(): - try: - copr = coprs_logic.CoprsLogic.add( - flask.g.user, - name=form.name.data, - homepage=form.homepage.data, - contact=form.contact.data, - repos=form.repos.data.replace("\n", " "), - selected_chroots=form.selected_chroots, - description=form.description.data, - instructions=form.instructions.data, - disable_createrepo=form.disable_createrepo.data, - build_enable_net=form.build_enable_net.data, - unlisted_on_hp=form.unlisted_on_hp.data, - group=group, - persistent=form.persistent.data, - auto_prune=(form.auto_prune.data if flask.g.user.admin else True), - use_bootstrap_container=form.use_bootstrap_container.data, - follow_fedora_branching=form.follow_fedora_branching.data, - ) - except (exceptions.DuplicateException, exceptions.NonAdminCannotCreatePersistentProject) as e: - flask.flash(str(e), "error") - return flask.render_template("coprs/group_add.html", form=form, group=group) - - db.session.add(copr) - db.session.commit() - after_the_project_creation(copr, form) - - return flask.redirect(url_for_copr_details(copr)) - else: - return flask.render_template("coprs/group_add.html", form=form, group=group) + return process_copr_new(group_name) -def process_copr_new(username): +def process_copr_new(group_name=None): """ - Receive information from the user on how to create its new copr + Receive information from the user (and group) on how to create its new copr and create it accordingly. """ + group = None + redirect = "coprs/add.html" + if group_name: + group = ComplexLogic.get_group_by_name_safe(group_name) + redirect = "coprs/group_add.html" - form = forms.CoprFormFactory.create_form_cls()() + form = forms.CoprFormFactory.create_form_cls(group=group)() if form.validate_on_submit(): try: copr = coprs_logic.CoprsLogic.add( @@ -216,21 +182,23 @@ def process_copr_new(username): disable_createrepo=form.disable_createrepo.data, build_enable_net=form.build_enable_net.data, unlisted_on_hp=form.unlisted_on_hp.data, + group=group, persistent=form.persistent.data, auto_prune=(form.auto_prune.data if flask.g.user.admin else True), use_bootstrap_container=form.use_bootstrap_container.data, follow_fedora_branching=form.follow_fedora_branching.data, + delete_after_days=form.delete_after_days.data, ) except (exceptions.DuplicateException, exceptions.NonAdminCannotCreatePersistentProject) as e: flask.flash(str(e), "error") - return flask.render_template("coprs/add.html", form=form) + return flask.render_template(redirect, form=form) db.session.commit() after_the_project_creation(copr, form) return flask.redirect(url_for_copr_details(copr)) else: - return flask.render_template("coprs/add.html", form=form) + return flask.render_template(redirect, form=form) def after_the_project_creation(copr, form): @@ -485,6 +453,7 @@ def process_copr_update(copr, form): copr.unlisted_on_hp = form.unlisted_on_hp.data copr.use_bootstrap_container = form.use_bootstrap_container.data copr.follow_fedora_branching = form.follow_fedora_branching.data + copr.delete_after_days = form.delete_after_days.data if flask.g.user.admin: copr.auto_prune = form.auto_prune.data else: diff --git a/frontend/coprs_frontend/manage.py b/frontend/coprs_frontend/manage.py index 9d34405..e133129 100755 --- a/frontend/coprs_frontend/manage.py +++ b/frontend/coprs_frontend/manage.py @@ -40,6 +40,7 @@ commands = { "vacuum_graphs": "RemoveGraphsDataCommand", "notify_outdated_chroots": "NotifyOutdatedChrootsCommand", "delete_outdated_chroots": "DeleteOutdatedChrootsCommand", + "clean_expired_projects": "CleanExpiredProjectsCommand", } if os.getuid() == 0: diff --git a/frontend/coprs_frontend/tests/test_logic/test_coprs_logic.py b/frontend/coprs_frontend/tests/test_logic/test_coprs_logic.py index 595ac7a..5681fa6 100644 --- a/frontend/coprs_frontend/tests/test_logic/test_coprs_logic.py +++ b/frontend/coprs_frontend/tests/test_logic/test_coprs_logic.py @@ -1,4 +1,5 @@ import json +import datetime from flask_whooshee import Whooshee @@ -80,3 +81,22 @@ class TestCoprsLogic(CoprsTestCase): assert data["projectname"] == name + def test_delete_expired_coprs(self, f_users, f_mock_chroots, f_coprs, f_db): + query = self.db.session.query(models.Copr) + + # nothing is deleted at the beginning + assert len([c for c in query.all() if c.deleted]) == 0 + + # one is to be deleted in the future + self.c1.delete_after_days = 2 + # one is already to be deleted + self.c2.delete_after = datetime.datetime.now() - datetime.timedelta(days=1) + + # and one is not to be temporary at all (c3) + + CoprsLogic.delete_expired_projects() + self.db.session.commit() + + query = self.db.session.query(models.Copr) + assert len(query.all()) == 3 # we only set deleted=true + assert len([c for c in query.all() if c.deleted]) == 1 diff --git a/python/copr/v3/proxies/project.py b/python/copr/v3/proxies/project.py index 3701176..79020a2 100644 --- a/python/copr/v3/proxies/project.py +++ b/python/copr/v3/proxies/project.py @@ -59,7 +59,8 @@ class ProjectProxy(BaseProxy): def add(self, ownername, projectname, chroots, description=None, instructions=None, homepage=None, contact=None, additional_repos=None, unlisted_on_hp=False, enable_net=True, persistent=False, - auto_prune=True, use_bootstrap_container=False, devel_mode=False): + auto_prune=True, use_bootstrap_container=False, devel_mode=False, + delete_after_days=None): """ Create a project @@ -77,6 +78,7 @@ class ProjectProxy(BaseProxy): :param bool auto_prune: if backend auto-deletion script should be run for the project :param bool use_bootstrap_container: if mock bootstrap container is used to initialize the buildroot :param bool devel_mode: if createrepo should run automatically + :param int delete_after_days: delete the project after the specfied period of time :return: Munch """ endpoint = "/project/add/{ownername}" @@ -97,6 +99,7 @@ class ProjectProxy(BaseProxy): "auto_prune": auto_prune, "use_bootstrap_container": use_bootstrap_container, "devel_mode": devel_mode, + "delete_after_days": delete_after_days, } request = Request(endpoint, api_base_url=self.api_base_url, method=POST, params=params, data=data, auth=self.auth) @@ -105,7 +108,8 @@ class ProjectProxy(BaseProxy): def edit(self, ownername, projectname, chroots=None, description=None, instructions=None, homepage=None, contact=None, additional_repos=None, unlisted_on_hp=None, enable_net=None, - auto_prune=None, use_bootstrap_container=None, devel_mode=None): + auto_prune=None, use_bootstrap_container=None, devel_mode=None, + delete_after_days=None): """ Edit a project @@ -122,6 +126,7 @@ class ProjectProxy(BaseProxy): :param bool auto_prune: if backend auto-deletion script should be run for the project :param bool use_bootstrap_container: if mock bootstrap container is used to initialize the buildroot :param bool devel_mode: if createrepo should run automatically + :param int delete_after_days: delete the project after the specfied period of time :return: Munch """ endpoint = "/project/edit/{ownername}/{projectname}" @@ -141,6 +146,7 @@ class ProjectProxy(BaseProxy): "auto_prune": auto_prune, "use_bootstrap_container": use_bootstrap_container, "devel_mode": devel_mode, + "delete_after_days": delete_after_days, } request = Request(endpoint, api_base_url=self.api_base_url, method=POST, params=params, data=data, auth=self.auth)