From 6f3f028e0a21e1437e212148994caac4cb148f22 Mon Sep 17 00:00:00 2001 From: Miroslav Suchý Date: Nov 07 2018 11:37:50 +0000 Subject: Merge #438 `Allow per-package chroot-blacklisting by wildcard patterns` --- diff --git a/backend/backend/actions.py b/backend/backend/actions.py index 87de475..404ba36 100644 --- a/backend/backend/actions.py +++ b/backend/backend/actions.py @@ -499,6 +499,7 @@ class Action(object): self.log.exception(e) +# TODO: sync with ActionTypeEnum from common class ActionType(object): DELETE = 0 RENAME = 1 diff --git a/backend/backend/daemons/worker.py b/backend/backend/daemons/worker.py index 61932cb..49c7df3 100644 --- a/backend/backend/daemons/worker.py +++ b/backend/backend/daemons/worker.py @@ -7,11 +7,11 @@ import pipes import glob from setproctitle import setproctitle -from ..exceptions import MockRemoteError, CoprWorkerError, VmError +from ..exceptions import MockRemoteError, CoprWorkerError, VmError, CoprBackendSrpmError from ..mockremote import MockRemote from ..constants import BuildStatus, build_log_format from ..helpers import register_build_result, get_redis_logger, \ - local_file_logger, run_cmd + local_file_logger, run_cmd, pkg_name_evr from ..msgbus import MsgBusStomp, MsgBusFedmsg from ..sshcmd import SSHConnectionError @@ -264,17 +264,17 @@ class Worker(multiprocessing.Process): self.log.info("Built packages:\n%s", built_packages) return built_packages - def get_srpm_url(self, job): - self.log.info("Retrieving srpm URL for %s", job.results_dir) - try: - pattern = os.path.join(job.results_dir, '*.src.rpm') - srpm_name = os.path.basename(glob.glob(pattern)[0]) - srpm_url = os.path.join(job.results_dir_url, srpm_name) - except IndexError: - srpm_url = "" - + def get_srpm_build_details(self, job): + build_details = {'srpm_url': ''} + self.log.info("Retrieving srpm URL from %s", job.results_dir) + pattern = os.path.join(job.results_dir, '*.src.rpm') + srpm_file = glob.glob(pattern)[0] + srpm_name = os.path.basename(srpm_file) + srpm_url = os.path.join(job.results_dir_url, srpm_name) + build_details['pkg_name'], build_details['pkg_version'] = pkg_name_evr(srpm_file) + build_details['srpm_url'] = srpm_url self.log.info("SRPM URL: %s", srpm_url) - return srpm_url + return build_details def get_build_details(self, job): """ @@ -283,7 +283,7 @@ class Worker(multiprocessing.Process): """ try: if job.chroot == "srpm-builds": - build_details = { "srpm_url": self.get_srpm_url(job) } + build_details = self.get_srpm_build_details(job) else: build_details = { "built_packages": self.collect_built_packages(job) } self.log.info("build details: %s", build_details) diff --git a/backend/backend/exceptions.py b/backend/backend/exceptions.py index 551a7fe..4972e09 100644 --- a/backend/backend/exceptions.py +++ b/backend/backend/exceptions.py @@ -88,6 +88,8 @@ class CoprWorkerError(CoprBackendError): class CoprSpawnFailError(CoprBackendError): pass +class CoprBackendSrpmError(CoprBackendError): + pass class VmError(CoprBackendError): """ diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index 6e2203b..1e8c373 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -32,7 +32,7 @@ from redis import StrictRedis from copr.client import CoprClient from backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT, DEF_CONSECUTIVE_FAILURE_THRESHOLD, \ CONSECUTIVE_FAILURE_REDIS_KEY, default_log_format -from backend.exceptions import CoprBackendError +from backend.exceptions import CoprBackendError, CoprBackendSrpmError from . import constants @@ -503,3 +503,31 @@ def local_file_logger(name, path, fmt): # to the previous project for h in build_logger.handlers[:]: build_logger.removeHandler(h) + +def pkg_name_evr(srpm_path): + """ + Queries a package for its name and evr (epoch:version-release) + """ + cmd = ['rpm', '-qp', '--nosignature', '--qf', + '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', srpm_path] + + try: + result = run_cmd(cmd) + except OSError as e: + raise CoprBackendSrpmError(str(e)) + + if result.returncode != 0: + raise CoprBackendSrpmError('Error querying srpm: %s' % error) + + try: + name, epoch, version, release = result.stdout.split(" ") + except ValueError as e: + raise CoprBackendSrpmError(str(e)) + + # Epoch is an integer or '(none)' if not set + if epoch.isdigit(): + evr = "{}:{}-{}".format(epoch, version, release) + else: + evr = "{}-{}".format(version, release) + + return name, evr diff --git a/dist-git/dist_git/helpers.py b/dist-git/dist_git/helpers.py index 147d01f..11140de 100644 --- a/dist-git/dist_git/helpers.py +++ b/dist-git/dist_git/helpers.py @@ -198,36 +198,3 @@ def run_cmd(cmd, cwd='.', raise_on_error=True): raise RunCommandException(result.stderr) return result - - -def pkg_name_evr(srpm_path): - """ - Queries a package for its name and evr (epoch:version-release) - """ - log.debug("Obtaining package name and version.") - cmd = ['rpm', '-qp', '--nosignature', '--qf', - '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', srpm_path] - - try: - proc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8') - output, error = proc.communicate() - except OSError as e: - raise SrpmQueryException(str(e)) - - if proc.returncode != 0: - raise SrpmQueryException('Error querying srpm: %s' % error) - - try: - name, epoch, version, release = output.split(" ") - except ValueError as e: - raise SrpmQueryException(str(e)) - - # Epoch is an integer or '(none)' if not set - if epoch.isdigit(): - evr = "{}:{}-{}".format(epoch, version, release) - else: - evr = "{}-{}".format(version, release) - - return name, evr diff --git a/dist-git/dist_git/import_task.py b/dist-git/dist_git/import_task.py index b1a79ae..5520ebe 100644 --- a/dist-git/dist_git/import_task.py +++ b/dist-git/dist_git/import_task.py @@ -20,6 +20,7 @@ class ImportTask(object): task.project = task_dict["project"] task.branches = task_dict["branches"] task.srpm_url = task_dict["srpm_url"] + task.pkg_name = task_dict["pkg_name"] except (KeyError, ValueError) as e: raise PackageImportException(str(e)) diff --git a/dist-git/dist_git/importer.py b/dist-git/dist_git/importer.py index e3f4234..de94b54 100644 --- a/dist-git/dist_git/importer.py +++ b/dist-git/dist_git/importer.py @@ -80,7 +80,8 @@ class Importer(object): self.opts, task.repo_namespace, task.branches, - srpm_path + srpm_path, + task.pkg_name, )) except PackageImportException as e: log.exception("Exception raised during package import.") diff --git a/dist-git/dist_git/package_import.py b/dist-git/dist_git/package_import.py index 3eaad82..6e50618 100644 --- a/dist-git/dist_git/package_import.py +++ b/dist-git/dist_git/package_import.py @@ -153,7 +153,7 @@ def cleanup_repo(repo_path): @helpers.single_run(import_lock) -def import_package(opts, namespace, branches, srpm_path): +def import_package(opts, namespace, branches, srpm_path, pkg_name): """ Import package into a DistGit repo for the given branches. @@ -163,11 +163,8 @@ def import_package(opts, namespace, branches, srpm_path): :param str srpm_path: path to the srpm file :return Munch: resulting import data: - (branch_commits, reponame, pkg_name, pkg_evr) + (branch_commits, reponame, pkg_name) """ - pkg_name, pkg_evr = helpers.pkg_name_evr(srpm_path) - log.debug("pkg_name: " + str(pkg_name)) - log.debug("pkg_evr: " + str(pkg_evr)) reponame = "{}/{}".format(namespace, pkg_name) setup_git_repo(reponame, branches) @@ -212,8 +209,6 @@ def import_package(opts, namespace, branches, srpm_path): helpers.run_cmd(['git', 'config', 'user.email', opts.git_user_email]) message = "automatic import of {}".format(pkg_name) - if pkg_evr: - message += " {}".format(pkg_evr) branch_commits = {} for branch in branches: @@ -253,6 +248,4 @@ def import_package(opts, namespace, branches, srpm_path): return munch.Munch( branch_commits=branch_commits, reponame=reponame, - pkg_name=pkg_name, - pkg_evr=pkg_evr, ) diff --git a/dist-git/tests/base.py b/dist-git/tests/base.py index 21bf4bc..5517f1e 100644 --- a/dist-git/tests/base.py +++ b/dist-git/tests/base.py @@ -49,6 +49,7 @@ class Base(object): "branches": [ self.BRANCH ], "srpm_url": "http://example.com/pkg.src.rpm", + "pkg_name": "pkg", } self.upload_task_data = { "build_id": 124, @@ -57,6 +58,7 @@ class Base(object): "branches": [ self.BRANCH ], "srpm_url": "http://front/tmp/tmp_2/pkg_2.src.rpm", + "pkg_name": "pkg_2", } self.url_task = import_task.ImportTask.from_dict(self.url_task_data) diff --git a/dist-git/tests/test_crazy_merging.py b/dist-git/tests/test_crazy_merging.py index d961140..c68610b 100644 --- a/dist-git/tests/test_crazy_merging.py +++ b/dist-git/tests/test_crazy_merging.py @@ -188,8 +188,9 @@ def initial_commit_everywhere(request, branches, mc_group, mc_chroot, opts_basic class TestMerging(object): def commit_to_branches(self, to_branches, opts, version): - srpm_path, packge_desc = get_srpm(version) - import_result = import_package(opts, 'bob/blah', to_branches, srpm_path) + srpm_path, package_desc = get_srpm(version) + name = package_desc['package_name'] + import_result = import_package(opts, 'bob/blah', to_branches, srpm_path, name) return import_result['branch_commits'] def setup_method(self, method): diff --git a/dist-git/tests/test_package_import.py b/dist-git/tests/test_package_import.py index 15b0f4e..1ff068c 100644 --- a/dist-git/tests/test_package_import.py +++ b/dist-git/tests/test_package_import.py @@ -93,26 +93,21 @@ class TestPackageImport(Base): mc_pyrpkg_commands.return_value = mc_cmd mc_cmd.commithash = '1234' - mc_helpers.pkg_name_evr = MagicMock(return_value = ('pkg_name', '1.2')) - namespace = 'somenamespace' branches = ['f25', 'f26'] - result = import_package(self.opts, namespace, branches, 'some_srpm_path') + result = import_package(self.opts, namespace, branches, + 'some_srpm_path', 'pkg_name') expected_result = Munch({ 'branch_commits': {'f26': '1234', 'f25': '1234'}, 'reponame': 'somenamespace/pkg_name', - 'pkg_name': 'pkg_name', - 'pkg_evr': '1.2', }) assert (result == expected_result) mc_cmd.push.side_effect = rpkgError - result = import_package(self.opts, namespace, branches, 'some_srpm_path') + result = import_package(self.opts, namespace, branches, 'some_srpm_path', 'pkg_name') expected_result = Munch({ 'branch_commits': {}, 'reponame': 'somenamespace/pkg_name', - 'pkg_name': 'pkg_name', - 'pkg_evr': '1.2', }) assert (result == expected_result) diff --git a/frontend/coprs_frontend/alembic/schema/versions/51716ab39d37_package_chroot_blacklist.py b/frontend/coprs_frontend/alembic/schema/versions/51716ab39d37_package_chroot_blacklist.py new file mode 100644 index 0000000..b146ac5 --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/51716ab39d37_package_chroot_blacklist.py @@ -0,0 +1,22 @@ +""" +package.chroot_blacklist + +Revision ID: 51716ab39d37 +Revises: c28451aaed50 +Create Date: 2018-10-04 21:24:41.498242 +""" + +# revision identifiers, used by Alembic. +revision = '51716ab39d37' +down_revision = 'c28451aaed50' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('package', sa.Column('chroot_blacklist_raw', sa.Text(), nullable=True)) + + +def downgrade(): + op.drop_column('package', 'chroot_blacklist_raw') diff --git a/frontend/coprs_frontend/coprs/__init__.py b/frontend/coprs_frontend/coprs/__init__.py index 1bca3bd..62af04d 100644 --- a/frontend/coprs_frontend/coprs/__init__.py +++ b/frontend/coprs_frontend/coprs/__init__.py @@ -74,7 +74,7 @@ from coprs.views.webhooks_ns import webhooks_ns from coprs.views.webhooks_ns import webhooks_general -from .context_processors import include_banner, inject_fedmenu +from .context_processors import include_banner, inject_fedmenu, counter_processor setup_log() diff --git a/frontend/coprs_frontend/coprs/context_processors.py b/frontend/coprs_frontend/coprs/context_processors.py index c7443c2..17680ce 100644 --- a/frontend/coprs_frontend/coprs/context_processors.py +++ b/frontend/coprs_frontend/coprs/context_processors.py @@ -69,3 +69,16 @@ def login_menu(): }) return dict(login_menu=menu) + +@app.context_processor +def counter_processor(): + def counter(name): + if not 'counters' in flask.g: + flask.g.counters = {} + if not name in flask.g.counters: + flask.g.counters[name] = 0 + + flask.g.counters[name] += 1 + return str(flask.g.counters[name]) + + return dict(counter=counter) diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index bdae1ca..cab0d35 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -6,6 +6,7 @@ import wtforms import json from flask_wtf.file import FileRequired, FileField +from fnmatch import fnmatch try: # get rid of deprecation warning with newer flask_wtf from flask_wtf import FlaskForm @@ -15,7 +16,7 @@ except ImportError: from coprs import constants from coprs import helpers from coprs import models -from coprs.logic.coprs_logic import CoprsLogic +from coprs.logic.coprs_logic import CoprsLogic, MockChrootsLogic from coprs.logic.users_logic import UsersLogic from coprs import exceptions @@ -170,13 +171,10 @@ class ChrootsValidator(object): return selected = set(field.data.split()) - enabled = set(self.chroots_list()) + enabled = set(MockChrootsLogic.active_names()) - if not (selected <= enabled): - raise wtforms.ValidationError("Such chroot is not enabled: {}".format(", ".join(selected - enabled))) - - def chroots_list(self): - return [c.name for c in models.MockChroot.query.filter(models.MockChroot.is_active).all()] + if selected - enabled: + raise wtforms.ValidationError("Such chroot is not available: {}".format(", ".join(selected - enabled))) class NameNotNumberValidator(object): @@ -312,17 +310,13 @@ class CoprFormFactory(object): have_any = True return have_any - F.chroots_list = list(map(lambda x: x.name, - models.MockChroot.query.filter( - models.MockChroot.is_active == True - ).all())) + F.chroots_list = MockChrootsLogic.active_names() F.chroots_list.sort() # sets of chroots according to how we should print them in columns F.chroots_sets = {} for ch in F.chroots_list: checkbox_default = False - if mock_chroots and ch in map(lambda x: x.name, - mock_chroots): + if mock_chroots and ch in [x.name for x in mock_chroots]: checkbox_default = True setattr(F, ch, wtforms.BooleanField(ch, default=checkbox_default, false_values=FALSE_VALUES)) @@ -404,11 +398,48 @@ class RebuildPackageFactory(object): return form +def cleanup_chroot_blacklist(string): + if not string: + return string + fields = [x.lstrip().rstrip() for x in string.split(',')] + return ', '.join(fields) + + +def validate_chroot_blacklist(form, field): + if field.data: + string = field.data + fields = [x.lstrip().rstrip() for x in string.split(',')] + for field in fields: + pattern = r'^[a-z0-9-*]+$' + if not re.match(pattern, field): + raise wtforms.ValidationError('Pattern "{0}" does not match "{1}"'.format(field, pattern)) + + matched = set() + all_chroots = MockChrootsLogic.active_names() + for chroot in all_chroots: + if fnmatch(chroot, field): + matched.add(chroot) + + if not matched: + raise wtforms.ValidationError('no chroot matched by pattern "{0}"'.format(field)) + + if matched == all_chroots: + raise wtforms.ValidationError('patterns are black-listing all chroots') + + class BasePackageForm(FlaskForm): package_name = wtforms.StringField( "Package name", validators=[wtforms.validators.DataRequired()]) webhook_rebuild = wtforms.BooleanField(default=False, false_values=FALSE_VALUES) + chroot_blacklist = wtforms.StringField( + "Chroot blacklist", + filters=[cleanup_chroot_blacklist], + validators=[ + wtforms.validators.Optional(), + validate_chroot_blacklist, + ], + ) class PackageFormScm(BasePackageForm): @@ -696,7 +727,7 @@ class RebuildAllPackagesFormFactory(object): class BaseBuildFormFactory(object): - def __new__(cls, active_chroots, form): + def __new__(cls, active_chroots, form, package=None): class F(form): @property def selected_chroots(self): @@ -730,11 +761,18 @@ class BaseBuildFormFactory(object): # overrides BasePackageForm.package_name and is unused for building F.package_name = wtforms.StringField() - F.chroots_list = list(map(lambda x: x.name, active_chroots)) + # fill chroots based on project settings + F.chroots_list = [x.name for x in active_chroots] F.chroots_list.sort() F.chroots_sets = {} + + package_chroots = set(F.chroots_list) + if package: + package_chroots = set([ch.name for ch in package.chroots]) + for ch in F.chroots_list: - setattr(F, ch, wtforms.BooleanField(ch, default=True, false_values=FALSE_VALUES)) + default = ch in package_chroots + setattr(F, ch, wtforms.BooleanField(ch, default=default, false_values=FALSE_VALUES)) if ch[0] in F.chroots_sets: F.chroots_sets[ch[0]].append(ch) else: @@ -743,8 +781,8 @@ class BaseBuildFormFactory(object): class BuildFormScmFactory(object): - def __new__(cls, active_chroots): - return BaseBuildFormFactory(active_chroots, PackageFormScm) + def __new__(cls, active_chroots, package=None): + return BaseBuildFormFactory(active_chroots, PackageFormScm, package) class BuildFormTitoFactory(object): @@ -764,13 +802,13 @@ class BuildFormMockFactory(object): class BuildFormPyPIFactory(object): - def __new__(cls, active_chroots): - return BaseBuildFormFactory(active_chroots, PackageFormPyPI) + def __new__(cls, active_chroots, package=None): + return BaseBuildFormFactory(active_chroots, PackageFormPyPI, package) class BuildFormRubyGemsFactory(object): - def __new__(cls, active_chroots): - return BaseBuildFormFactory(active_chroots, PackageFormRubyGems) + def __new__(cls, active_chroots, package=None): + return BaseBuildFormFactory(active_chroots, PackageFormRubyGems, package) class BuildFormDistGitFactory(object): @@ -788,8 +826,8 @@ class BuildFormUploadFactory(object): class BuildFormCustomFactory(object): - def __new__(cls, active_chroots): - return BaseBuildFormFactory(active_chroots, PackageFormCustom) + def __new__(cls, active_chroots, package=None): + return BaseBuildFormFactory(active_chroots, PackageFormCustom, package) class BuildFormUrlFactory(object): diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 3189724..efd013d 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -28,6 +28,7 @@ from coprs.logic import users_logic from coprs.logic.actions_logic import ActionsLogic from coprs.models import BuildChroot from .coprs_logic import MockChrootsLogic +from coprs.logic.packages_logic import PackagesLogic log = app.logger @@ -648,9 +649,8 @@ GROUP BY :type batch: models.Batch :rtype: models.Build """ - if chroot_names is None: - chroots = [c for c in copr.active_chroots] - else: + chroots = None + if chroot_names: chroots = [] for chroot in copr.active_chroots: if chroot.name in chroot_names: @@ -708,9 +708,6 @@ GROUP BY if skip_import and srpm_url: chroot_status = StatusEnum("pending") source_status = StatusEnum("succeeded") - elif srpm_url: - chroot_status = StatusEnum("waiting") - source_status = StatusEnum("importing") else: chroot_status = StatusEnum("waiting") source_status = StatusEnum("pending") @@ -735,12 +732,8 @@ GROUP BY db.session.add(build) - # add BuildChroot object for each active (or selected) chroot - # this copr is assigned to - if not chroots: - chroots = copr.active_chroots - for chroot in chroots: + # Chroots were explicitly set per-build. git_hash = None if git_hashes: git_hash = git_hashes.get(chroot.name) @@ -785,9 +778,8 @@ GROUP BY ) db.session.add(build) - chroots = package.copr.active_chroots status = StatusEnum("waiting") - for chroot in chroots: + for chroot in package.chroots: buildchroot = models.BuildChroot( build=build, status=status, @@ -858,20 +850,50 @@ GROUP BY """ log.info("Updating build {} by: {}".format(build.id, upd_dict)) - # update build - for attr in ["built_packages", "srpm_url"]: + # create the package if it doesn't exist + pkg_name = upd_dict.get('pkg_name', None) + if pkg_name: + if not PackagesLogic.get(build.copr_dir.id, pkg_name).first(): + try: + package = PackagesLogic.add( + build.copr.user, build.copr_dir, + pkg_name, build.source_type, build.source_json) + db.session.add(package) + db.session.commit() + except (sqlalchemy.exc.IntegrityError, exceptions.DuplicateException) as e: + app.logger.exception(e) + db.session.rollback() + return + build.package = PackagesLogic.get(build.copr_dir.id, pkg_name).first() + + for attr in ["built_packages", "srpm_url", "pkg_version"]: value = upd_dict.get(attr, None) if value: setattr(build, attr, value) # update source build status - if upd_dict.get("task_id") == build.task_id: + if str(upd_dict.get("task_id")) == str(build.task_id): build.result_dir = upd_dict.get("result_dir", "") - if upd_dict.get("status") == StatusEnum("succeeded"): + new_status = upd_dict.get("status") + if new_status == StatusEnum("succeeded"): new_status = StatusEnum("importing") - else: - new_status = upd_dict.get("status") + chroot_status=StatusEnum("waiting") + if not build.build_chroots: + # create the BuildChroots from Package setting, if not + # already set explicitly for concrete build + for chroot in build.package.chroots: + buildchroot = models.BuildChroot( + build=build, + status=chroot_status, + mock_chroot=chroot, + git_hash=None, + ) + db.session.add(buildchroot) + else: + for buildchroot in build.build_chroots: + buildchroot.status = chroot_status + db.session.add(buildchroot) build.source_status = new_status if new_status == StatusEnum("failed") or \ diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py index 562579a..618e53e 100644 --- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py +++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py @@ -700,6 +700,10 @@ class MockChrootsLogic(object): return new_chroot @classmethod + def active_names(cls): + return [ch.name for ch in cls.get_multiple(active_only=True).all()] + + @classmethod def new(cls, mock_chroot): db.session.add(mock_chroot) diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 288061c..d5c21ca 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -4,6 +4,7 @@ import os import json import base64 import uuid +from fnmatch import fnmatch from sqlalchemy.ext.associationproxy import association_proxy from six.moves.urllib.parse import urljoin @@ -522,6 +523,10 @@ class Package(db.Model, helpers.Serializer, CoprSearchRelatedData): copr_dir_id = db.Column(db.Integer, db.ForeignKey("copr_dir.id"), index=True) copr_dir = db.relationship("CoprDir", backref=db.backref("packages")) + # comma-separated list of wildcards of chroot names that this package should + # not be built against, e.g. "fedora-*, epel-*-i386" + chroot_blacklist_raw = db.Column(db.Text) + @property def dist_git_repo(self): return "{}/{}".format(self.copr_dir.full_name, self.name) @@ -582,6 +587,55 @@ class Package(db.Model, helpers.Serializer, CoprSearchRelatedData): return self.copr.id + @property + def chroot_blacklist(self): + if not self.chroot_blacklist_raw: + return [] + + blacklisted = [] + for pattern in self.chroot_blacklist_raw.split(','): + pattern = pattern.strip() + if not pattern: + continue + blacklisted.append(pattern) + + return blacklisted + + + @staticmethod + def matched_chroot(chroot, patterns): + for pattern in patterns: + if fnmatch(chroot.name, pattern): + return True + return False + + + @property + def main_pkg(self): + if self.copr_dir.main: + return self + + main_pkg = Package.query.filter_by( + name=self.name, + copr_dir_id=self.copr.main_dir.id + ).first() + return main_pkg + + + @property + def chroots(self): + chroots = list(self.copr.active_chroots) + if not self.chroot_blacklist_raw: + # no specific blacklist + if self.copr_dir.main: + return chroots + return self.main_pkg.chroots + + filtered = [c for c in chroots if not self.matched_chroot(c, self.chroot_blacklist)] + # We never want to filter everything, this is a misconfiguration. + return filtered if filtered else chroots + + class Build(db.Model, helpers.Serializer): """ Representation of one build in one copr @@ -713,9 +767,6 @@ class Build(db.Model, helpers.Serializer): def import_log_urls(self): backend_log = self.import_log_url_backend types = [helpers.BuildSourceEnum("upload"), helpers.BuildSourceEnum("link")] - if self.source_type in types: - if json.loads(self.source_json).get("url", "").endswith(".src.rpm"): - backend_log = None return filter(None, [backend_log, self.import_log_url_distgit]) @property @@ -817,6 +868,10 @@ class Build(db.Model, helpers.Serializer): if self.canceled: return StatusEnum("canceled") + use_src_statuses = ["starting", "pending", "running"] + if self.source_status in [StatusEnum(s) for s in use_src_statuses]: + return self.source_status + for state in ["running", "starting", "pending", "failed", "succeeded", "skipped", "forked", "waiting"]: if StatusEnum(state) in self.chroot_states: if state == "waiting": diff --git a/frontend/coprs_frontend/coprs/templates/_helpers.html b/frontend/coprs_frontend/coprs/templates/_helpers.html index f305ee9..ba51e43 100644 --- a/frontend/coprs_frontend/coprs/templates/_helpers.html +++ b/frontend/coprs_frontend/coprs/templates/_helpers.html @@ -132,10 +132,16 @@ {% if build.canceled %} {{ build_state_text("canceled") }} {% else %} - {% if build.status|state_from_num == "waiting" %} + {% if build.status is not none %} + {% if build.status|state_from_num == "waiting" %} + {{ build_state_text(build.source_status|state_from_num) }} + {% else %} + {{ build_state_text(build.status|state_from_num) }} + {% endif %} + {% elif build.source_status %} {{ build_state_text(build.source_status|state_from_num) }} {% else %} - {{ build_state_text(build.status|state_from_num) }} + {{ build_state_text(0|state_from_num) }} {% endif %} {% endif %} {% endmacro %} @@ -532,14 +538,14 @@ https://admin.fedoraproject.org/accounts/group/view/{{name}} {% macro render_srpm_build_method_box(form) %} -
-

3. How to build SRPM from the source

+

{{ counter('instructions') }}. How to build SRPM from the source

diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html index 5bbfae3..964ac94 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/_builds_forms.html @@ -1,4 +1,4 @@ -{% from "_helpers.html" import render_field, render_form_errors, copr_url, render_pypi_python_versions_field, render_additional_build_options, render_srpm_build_method_box %} +{% from "_helpers.html" import render_field, render_form_errors, copr_url, render_pypi_python_versions_field, render_additional_build_options, render_srpm_build_method_box with context %} {% from "coprs/detail/_method_forms.html" import copr_method_form_fileds_custom %} {# This file contains forms for the "New Build" action @@ -39,20 +39,20 @@ {% if not hide_panels %}
-

2. Provide the source

+

{{ counter('instructions') }}. Provide the source

{% endif %} {% endmacro %} -{% macro copr_build_form_end(form, view, copr, hide_panels=False, numbering=3) %} +{% macro copr_build_form_end(form, view, copr, hide_panels=False) %} {% if not hide_panels %}
-

{{ numbering }}. Select chroots and other options

+

{{ counter('instructions') }}. Select chroots and other options

{% endif %} @@ -129,7 +129,7 @@ {{ render_srpm_build_method_box(form) }} - {{ copr_build_form_end(form, view, copr, numbering=4) }} + {{ copr_build_form_end(form, view, copr) }} {% endmacro %} diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/_package_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/_package_forms.html index 5a8f583..d28b1dd 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/_package_forms.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/_package_forms.html @@ -1,4 +1,4 @@ -{% from "_helpers.html" import render_field, render_form_errors, copr_url, render_pypi_python_versions_field, render_additional_build_options, render_srpm_build_method_box %} +{% from "_helpers.html" import render_field, render_form_errors, copr_url, render_pypi_python_versions_field, render_additional_build_options, render_srpm_build_method_box with context %} {% from "coprs/detail/_method_forms.html" import copr_method_form_fileds_custom %} {% macro copr_package_form_begin(form, view, copr, package) %} @@ -7,7 +7,7 @@
-

2. Provide the source

+

{{ counter('instructions') }}. Provide the source

{{ render_field(form.package_name) }} @@ -16,7 +16,7 @@ method="post" enctype="multipart/form-data">
-

2. Provide the source

+

{{ counter('instructions') }}. Provide the source

@@ -50,6 +50,20 @@
{% endmacro %} +{% macro render_generic_pkg_form(form) %} +
+
+
+
+

{{ counter('instructions') }}. Generic package setup

+
+
+{{ render_field(form.chroot_blacklist, + placeholder="Optional - comma-separated list of wildcard-patterns, e.g. fedora-*, *-i386", + info="What chroots should be skipped for this package, by default we build for all.", +)}} +{% endmacro %} + {% macro render_anitya_autorebuild(form) %}