From 041d651926a1f539bfe3d9a1021192308f79160a Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:15 +0000 Subject: [PATCH 1/26] [frontend] add build to module relation --- diff --git a/frontend/coprs_frontend/alembic/schema/versions/3b0851cb25fc_add_build_to_module_relation.py b/frontend/coprs_frontend/alembic/schema/versions/3b0851cb25fc_add_build_to_module_relation.py new file mode 100644 index 0000000..b54fa33 --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/3b0851cb25fc_add_build_to_module_relation.py @@ -0,0 +1,24 @@ +"""Add build to module relation + +Revision ID: 3b0851cb25fc +Revises: 7bb0c7762df0 +Create Date: 2017-12-27 13:46:07.774167 + +""" + +# revision identifiers, used by Alembic. +revision = '3b0851cb25fc' +down_revision = '7bb0c7762df0' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('build', sa.Column('module_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'build', 'module', ['module_id'], ['id']) + + +def downgrade(): + op.drop_constraint(None, 'build', type_='foreignkey') + op.drop_column('build', 'module_id') diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index b7c6337..9642a75 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -553,6 +553,9 @@ class Build(db.Model, helpers.Serializer): batch_id = db.Column(db.Integer, db.ForeignKey("batch.id")) batch = db.relationship("Batch", backref=db.backref("builds")) + module_id = db.Column(db.Integer, db.ForeignKey("module.id")) + module = db.relationship("Module", backref=db.backref("builds")) + @property def user_name(self): return self.user.name From 62ce2ac1003876117b413650c38628101ecf9e24 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:15 +0000 Subject: [PATCH 2/26] [frontend][python][cli] require to specify project when building module --- diff --git a/cli/copr_cli/main.py b/cli/copr_cli/main.py index e02fb10..6a0de33 100644 --- a/cli/copr_cli/main.py +++ b/cli/copr_cli/main.py @@ -621,7 +621,7 @@ class Commands(object): """ Build module via Copr MBS """ - ownername, projectname = parse_name(args.copr or "") + ownername, projectname = parse_name(args.copr) modulemd = open(args.yaml, "rb") if args.yaml else args.url response = self.client.build_module(modulemd, ownername, projectname) print(response.message if response.output == "ok" else response.error) @@ -1057,7 +1057,7 @@ def setup_parser(): # module building parser_build_module = subparsers.add_parser("build-module", help="Builds a given module in Copr") - parser_build_module.add_argument("copr", help="The copr repo to list the packages of. Can be just name of project or even in format owner/project.", nargs="?") + parser_build_module.add_argument("copr", help="The copr repo to build module in. Can be just name of project or even in format owner/project.") parser_build_module_mmd_source = parser_build_module.add_mutually_exclusive_group(required=True) parser_build_module_mmd_source.add_argument("--url", help="SCM with modulemd file in yaml format") parser_build_module_mmd_source.add_argument("--yaml", help="Path to modulemd file in yaml format") diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index 886c6f9..9abf39a 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -704,8 +704,6 @@ class ModuleBuildForm(FlaskForm): modulemd = FileField("modulemd") scmurl = wtforms.StringField() branch = wtforms.StringField() - copr_owner = wtforms.StringField() - copr_project = wtforms.StringField() class ChrootForm(FlaskForm): diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index fada4c4..2d1f8aa 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -990,9 +990,10 @@ def copr_build_package(copr, package_name): }) -@api_ns.route("/module/build/", methods=["POST"]) +@api_ns.route("/coprs///module/build/", methods=["POST"]) @api_login_required -def copr_build_module(): +@api_req_with_copr +def copr_build_module(copr): form = forms.ModuleBuildForm(csrf_enabled=False) if not form.validate_on_submit(): raise LegacyApiError(form.errors) diff --git a/python/copr/client/client.py b/python/copr/client/client.py index f893a9d..d97f7e8 100644 --- a/python/copr/client/client.py +++ b/python/copr/client/client.py @@ -1599,14 +1599,14 @@ class CoprClient(UnicodeMixin): return response def build_module(self, modulemd, ownername=None, projectname=None): - endpoint = "module/build" - url = "{}/{}/".format(self.api_url, endpoint) + url = "{0}/coprs/{1}/{2}/module/build/".format( + self.api_url, ownername, projectname + ) - data = {"copr_owner": ownername, "copr_project": projectname} if isinstance(modulemd, io.BufferedIOBase): - data.update({"modulemd": (os.path.basename(modulemd.name), modulemd, "application/x-rpm")}) + data = {"modulemd": (os.path.basename(modulemd.name), modulemd, "application/x-rpm")} else: - data.update({"scmurl": modulemd, "branch": "master"}) + data = {"scmurl": modulemd, "branch": "master"} def fetch(url, data, method): m = MultipartEncoder(data) From 54eaa094424e5b79ab872a1ed13ef58670c0b49e Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:15 +0000 Subject: [PATCH 3/26] [frontend] build a module without using MBS --- diff --git a/frontend/coprs_frontend/coprs/logic/actions_logic.py b/frontend/coprs_frontend/coprs/logic/actions_logic.py index 6a48f24..c2cf7f7 100644 --- a/frontend/coprs_frontend/coprs/logic/actions_logic.py +++ b/frontend/coprs_frontend/coprs/logic/actions_logic.py @@ -257,15 +257,12 @@ class ActionsLogic(object): db.session.add(action) @classmethod - def send_build_module(cls, user, copr, module): + def send_build_module(cls, copr, module): """ :type copr: models.Copr :type modulemd: str content of module yaml file """ - if not user.can_build_in(copr): - raise exceptions.InsufficientRightsException("You don't have permissions to build in this copr.") - data = { "chroots": [c.name for c in copr.active_chroots], } diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 3a4430f..252e622 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -758,6 +758,13 @@ GROUP BY db.session.add(build_chroot) + # If the last package of a module was successfully built, + # then send an action to create module repodata on backend + if (build.module + and upd_dict.get("status") == StatusEnum("succeeded") + and all(b for b in build.module.builds if b.status == StatusEnum("succeeded"))): + ActionsLogic.send_build_module(build.copr, build.module) + for attr in ["results", "built_packages", "srpm_url"]: value = upd_dict.get(attr, None) if value: diff --git a/frontend/coprs_frontend/coprs/logic/modules_logic.py b/frontend/coprs_frontend/coprs/logic/modules_logic.py index a5ab46b..14f3d3c 100644 --- a/frontend/coprs_frontend/coprs/logic/modules_logic.py +++ b/frontend/coprs_frontend/coprs/logic/modules_logic.py @@ -43,10 +43,9 @@ class ModulesLogic(object): return mmd @classmethod - def from_modulemd(cls, yaml): - mmd = cls.yaml2modulemd(yaml) + def from_modulemd(cls, mmd): return models.Module(name=mmd.name, stream=mmd.stream, version=mmd.version, summary=mmd.summary, - description=mmd.description, yaml_b64=base64.b64encode(yaml)) + description=mmd.description, yaml_b64=base64.b64encode(mmd.dumps())) @classmethod def validate(cls, yaml): diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index 2d1f8aa..78688ef 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -998,30 +998,35 @@ def copr_build_module(copr): if not form.validate_on_submit(): raise LegacyApiError(form.errors) + yaml = form.modulemd.data.read() try: - common = {"owner": flask.g.user.name, - "copr_owner": form.copr_owner.data, - "copr_project": form.copr_project.data} - if form.scmurl.data: - kwargs = {"json": dict({"scmurl": form.scmurl.data, "branch": form.branch.data}, **common)} - else: - kwargs = {"data": common, "files": {"yaml": (form.modulemd.data.filename, form.modulemd.data)}} + ModulesLogic.validate(yaml) + + modulemd = ModulesLogic.yaml2modulemd(yaml) + module = ModulesLogic.add(flask.g.user, copr, ModulesLogic.from_modulemd(modulemd)) + + batch = models.Batch() + db.session.add(batch) - response = requests.post(flask.current_app.config["MBS_URL"], verify=False, **kwargs) - if response.status_code == 500: - raise LegacyApiError("Error from MBS: {} - {}".format(response.status_code, response.reason)) + for pkgname, rpm in modulemd.components.rpms.items(): + # @TODO move to better place + default_distgit = "https://src.fedoraproject.org/rpms/{pkgname}" + chroot_name = "fedora-rawhide-x86_64" + clone_url = rpm.repository if rpm.repository else default_distgit.format(pkgname=pkgname) - resp = json.loads(response.content) - if response.status_code != 201: - raise LegacyApiError("Error from MBS: {}".format(resp["message"])) + build = BuildsLogic.create_new_from_scm(flask.g.user, copr, scm_type="git", clone_url=clone_url, + committish=rpm.ref, chroot_names=[chroot_name]) + build.batch_id = batch.id + build.module_id = module.id + db.session.commit() return flask.jsonify({ "output": "ok", - "message": "Created module {}-{}-{}".format(resp["name"], resp["stream"], resp["version"]), + "message": "Created module {}".format(module.nsv), }) - except requests.ConnectionError: - raise LegacyApiError("Can't connect to MBS instance") + except ValidationError as ex: + raise LegacyApiError({"nsv": [ex.message]}) @api_ns.route("/coprs///module/make/", methods=["POST"]) From 2b405e529420c1e8cc3b366a6e30380d81edd12d Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:15 +0000 Subject: [PATCH 4/26] [frontend] have unique module nsv per project --- diff --git a/frontend/coprs_frontend/alembic/schema/versions/f61a5c930abf_unique_constraint_on_modules.py b/frontend/coprs_frontend/alembic/schema/versions/f61a5c930abf_unique_constraint_on_modules.py new file mode 100644 index 0000000..4db06a3 --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/f61a5c930abf_unique_constraint_on_modules.py @@ -0,0 +1,22 @@ +"""Unique constraint on modules + +Revision ID: f61a5c930abf +Revises: 3b0851cb25fc +Create Date: 2017-12-27 16:32:51.786669 + +""" + +# revision identifiers, used by Alembic. +revision = 'f61a5c930abf' +down_revision = '3b0851cb25fc' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_unique_constraint('copr_name_stream_version_uniq', 'module', ['copr_id', 'name', 'stream', 'version']) + + +def downgrade(): + op.drop_constraint('copr_name_stream_version_uniq', 'module', type_='unique') diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 9642a75..a7ac67f 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -1225,6 +1225,10 @@ class Module(db.Model, helpers.Serializer): copr_id = db.Column(db.Integer, db.ForeignKey("copr.id")) copr = db.relationship("Copr", backref=db.backref("modules")) + __table_args__ = ( + db.UniqueConstraint("copr_id", "name", "stream", "version", name="copr_name_stream_version_uniq"), + ) + @property def yaml(self): return base64.b64decode(self.yaml_b64) diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index 78688ef..ff268ab 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -999,10 +999,9 @@ def copr_build_module(copr): raise LegacyApiError(form.errors) yaml = form.modulemd.data.read() + modulemd = ModulesLogic.yaml2modulemd(yaml) try: ModulesLogic.validate(yaml) - - modulemd = ModulesLogic.yaml2modulemd(yaml) module = ModulesLogic.add(flask.g.user, copr, ModulesLogic.from_modulemd(modulemd)) batch = models.Batch() @@ -1028,6 +1027,10 @@ def copr_build_module(copr): except ValidationError as ex: raise LegacyApiError({"nsv": [ex.message]}) + except sqlalchemy.exc.IntegrityError: + raise LegacyApiError("Module {}-{}-{} already exists".format( + modulemd.name, modulemd.stream, modulemd.version)) + @api_ns.route("/coprs///module/make/", methods=["POST"]) @api_login_required From 62c6337905e9072882a6828a78db84accbba65b8 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 5/26] [frontend] always have a known state of a module --- diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index a7ac67f..f180abb 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -1252,13 +1252,20 @@ class Module(db.Model, helpers.Serializer): return Action.query.filter(Action.object_type == "module").filter(Action.object_id == self.id).first() @property + def status(self): + """ + Return numeric representation of status of this build + """ + if any(b for b in self.builds if b.status == StatusEnum("failed")): + return helpers.ModuleStatusEnum("failed") + return self.action.result if self.action else helpers.ModuleStatusEnum("pending") + + @property def state(self): """ Return text representation of status of this build """ - if self.action is not None: - return helpers.ModuleStatusEnum(self.action.result) - return "-" + return helpers.ModuleStatusEnum(self.status) def repo_url(self, arch): # @TODO Use custom chroot instead of fedora-24 diff --git a/frontend/coprs_frontend/coprs/templates/_helpers.html b/frontend/coprs_frontend/coprs/templates/_helpers.html index bd09f74..4d769e8 100644 --- a/frontend/coprs_frontend/coprs/templates/_helpers.html +++ b/frontend/coprs_frontend/coprs/templates/_helpers.html @@ -135,11 +135,7 @@ {% endmacro %} {% macro module_state(module) %} - {% if module.action %} - {{ build_state_text(module.action.result | module_state_from_num) }} - {% else %} - - - {% endif %} + {{ build_state_text(module.status | module_state_from_num) }} {% endmacro %} {% macro chroot_to_os_logo(chroot) %} From b7dcd6bdb396f7c8a0deac58381dd1ca0c8df894 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 6/26] [frontend] show related builds and module --- diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html index a0780aa..72728a3 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/build.html @@ -110,6 +110,14 @@ - {% endif %} + {% if build.module %} +
Module:
+
+ + {{ build.module.nsv }} + +
+ {% endif %}
Version:
{% if build.pkg_version %} diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html index d5e4e4c..a07279b 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html @@ -2,6 +2,7 @@ {% from "coprs/detail/_builds_forms.html" import copr_build_cancel_form, copr_build_repeat_form, copr_build_delete_form %} {% from "coprs/detail/_describe_source.html" import describe_source %} {% from "coprs/detail/_describe_failure.html" import describe_failure %} +{% from "coprs/detail/_package_table.html" import package_table with context %} {% from "_helpers.html" import chroot_to_os_logo, build_state_text, build_state, copr_name %} {% block title %}Module {{ module.id }} in {{ copr_name(copr) }}{% endblock %} @@ -165,8 +166,9 @@ dnf module install {{ module.name }}/default {{ yaml | safe }} - + {{package_table(module.builds)}} + From ac8c382c600d5a7e16cd9856d4c62c44bb096cc2 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 7/26] [frontend] temporarily disable building via web UI --- diff --git a/frontend/coprs_frontend/coprs/templates/coprs/create_module.html b/frontend/coprs_frontend/coprs/templates/coprs/create_module.html index b30d763..90d4a33 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/create_module.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/create_module.html @@ -35,6 +35,18 @@ {{ render_form_errors(form) }} +
+

This feature is not yet available due to recent modularity changes. + You can build your module using the command line

+ 
+          
+            copr-cli build-module --yaml /path/to/yourmodule.yaml {{ copr.full_name }}
+          
+
+

Learn more

+
+ + {#
@@ -159,6 +171,7 @@

Copr will generate the modulemd yaml file and create a modular RPM repository for you.

+ #}
From 80d92112524a3f55e6fccf11c556286923821dd8 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 8/26] [frontend] change module version to bigint Module version is typically in %Y%m%d%H%M%S format (e.g.20171023061952), so we should be able to store such big integers --- diff --git a/frontend/coprs_frontend/alembic/schema/versions/1f94b22f70a1_change_module_version_to_bigint.py b/frontend/coprs_frontend/alembic/schema/versions/1f94b22f70a1_change_module_version_to_bigint.py new file mode 100644 index 0000000..281c663 --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/1f94b22f70a1_change_module_version_to_bigint.py @@ -0,0 +1,22 @@ +"""Change module version to bigint + +Revision ID: 1f94b22f70a1 +Revises: f61a5c930abf +Create Date: 2017-12-29 22:22:09.256634 + +""" + +# revision identifiers, used by Alembic. +revision = '1f94b22f70a1' +down_revision = 'f61a5c930abf' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.alter_column('module', 'version', existing_type=sa.Integer(), type_=sa.BigInteger()) + + +def downgrade(): + op.alter_column('module', 'version', existing_type=sa.BigInteger(), type_=sa.Integer()) diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index f180abb..9661dd3 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -1208,7 +1208,7 @@ class Module(db.Model, helpers.Serializer): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(100), nullable=False) stream = db.Column(db.String(100), nullable=False) - version = db.Column(db.Integer, nullable=False) + version = db.Column(db.BigInteger, nullable=False) summary = db.Column(db.String(100), nullable=False) description = db.Column(db.Text) created_on = db.Column(db.Integer, nullable=True) From 9bdbf645caed88ca8d99d07e697b79d399e53b08 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 9/26] [frontend] set default values for optional modulemd params --- diff --git a/frontend/coprs_frontend/coprs/logic/modules_logic.py b/frontend/coprs_frontend/coprs/logic/modules_logic.py index 14f3d3c..d810f1b 100644 --- a/frontend/coprs_frontend/coprs/logic/modules_logic.py +++ b/frontend/coprs_frontend/coprs/logic/modules_logic.py @@ -5,6 +5,7 @@ import json import requests import modulemd from sqlalchemy import and_ +from datetime import datetime from coprs import models from coprs import db from coprs import exceptions @@ -48,8 +49,7 @@ class ModulesLogic(object): description=mmd.description, yaml_b64=base64.b64encode(mmd.dumps())) @classmethod - def validate(cls, yaml): - mmd = cls.yaml2modulemd(yaml) + def validate(cls, mmd): if not all([mmd.name, mmd.stream, mmd.version]): raise ValidationError("Module should contain name, stream and version") @@ -65,6 +65,12 @@ class ModulesLogic(object): db.session.add(module) return module + @classmethod + def set_defaults_for_optional_params(cls, mmd, filename=None): + mmd.name = mmd.name or str(os.path.splitext(filename)[0]) + mmd.stream = mmd.stream or "master" + mmd.version = mmd.version or int(datetime.now().strftime("%Y%m%d%H%M%S")) + class ModulemdGenerator(object): def __init__(self, name="", stream="", version=0, summary="", config=None): diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index ff268ab..fdcd1c2 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -1000,8 +1000,9 @@ def copr_build_module(copr): yaml = form.modulemd.data.read() modulemd = ModulesLogic.yaml2modulemd(yaml) + ModulesLogic.set_defaults_for_optional_params(modulemd, filename=form.modulemd.data.filename) try: - ModulesLogic.validate(yaml) + ModulesLogic.validate(modulemd) module = ModulesLogic.add(flask.g.user, copr, ModulesLogic.from_modulemd(modulemd)) batch = models.Batch() From 6a7d3610ee4f72e6ad589e0b8ed434f83bb8afc9 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 10/26] [frontend] implement submitting modules via URL --- diff --git a/frontend/coprs_frontend/coprs/logic/modules_logic.py b/frontend/coprs_frontend/coprs/logic/modules_logic.py index d810f1b..2eb831f 100644 --- a/frontend/coprs_frontend/coprs/logic/modules_logic.py +++ b/frontend/coprs_frontend/coprs/logic/modules_logic.py @@ -177,3 +177,29 @@ class MBSResponse(object): if self.response.status_code != 201: return "Error from MBS: {}".format(resp["message"]) return "Created module {}-{}-{}".format(resp["name"], resp["stream"], resp["version"]) + + +class ModuleProvider(object): + def __init__(self, filename, yaml): + self.filename = filename + self.yaml = yaml + + @classmethod + def from_input(cls, obj): + if type(obj) in [str, unicode]: + return cls.from_url(obj) + return cls.from_file(obj) + + @classmethod + def from_file(cls, ref): + return cls(ref.filename, ref.read()) + + @classmethod + def from_url(cls, url): + if not url.endswith(".yaml"): + raise ValidationError("This URL doesn't point to a .yaml file") + + request = requests.get(url) + if request.status_code != 200: + raise requests.RequestException("This URL seems to be wrong") + return cls(os.path.basename(url), request.text) diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index fdcd1c2..b53053c 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -22,7 +22,7 @@ from coprs.logic.builds_logic import BuildsLogic from coprs.logic.complex_logic import ComplexLogic from coprs.logic.users_logic import UsersLogic from coprs.logic.packages_logic import PackagesLogic -from coprs.logic.modules_logic import ModulesLogic +from coprs.logic.modules_logic import ModulesLogic, ModuleProvider from coprs.views.misc import login_required, api_login_required @@ -998,13 +998,14 @@ def copr_build_module(copr): if not form.validate_on_submit(): raise LegacyApiError(form.errors) - yaml = form.modulemd.data.read() - modulemd = ModulesLogic.yaml2modulemd(yaml) - ModulesLogic.set_defaults_for_optional_params(modulemd, filename=form.modulemd.data.filename) + modulemd = None try: + mod_info = ModuleProvider.from_input(form.modulemd.data or form.scmurl.data) + modulemd = ModulesLogic.yaml2modulemd(mod_info.yaml) + ModulesLogic.set_defaults_for_optional_params(modulemd, filename=mod_info.filename) ModulesLogic.validate(modulemd) - module = ModulesLogic.add(flask.g.user, copr, ModulesLogic.from_modulemd(modulemd)) + module = ModulesLogic.add(flask.g.user, copr, ModulesLogic.from_modulemd(modulemd)) batch = models.Batch() db.session.add(batch) @@ -1026,7 +1027,10 @@ def copr_build_module(copr): }) except ValidationError as ex: - raise LegacyApiError({"nsv": [ex.message]}) + raise LegacyApiError(ex.message) + + except requests.RequestException as ex: + raise LegacyApiError(ex.message) except sqlalchemy.exc.IntegrityError: raise LegacyApiError("Module {}-{}-{} already exists".format( From de78dfa39d9b74473a7042efed089904d808a736 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 11/26] [frontend] build modules in all enabled chroots --- diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index b53053c..ffc3b21 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -1012,11 +1012,10 @@ def copr_build_module(copr): for pkgname, rpm in modulemd.components.rpms.items(): # @TODO move to better place default_distgit = "https://src.fedoraproject.org/rpms/{pkgname}" - chroot_name = "fedora-rawhide-x86_64" clone_url = rpm.repository if rpm.repository else default_distgit.format(pkgname=pkgname) - build = BuildsLogic.create_new_from_scm(flask.g.user, copr, scm_type="git", clone_url=clone_url, - committish=rpm.ref, chroot_names=[chroot_name]) + build = BuildsLogic.create_new_from_scm(flask.g.user, copr, scm_type="git", + clone_url=clone_url, committish=rpm.ref) build.batch_id = batch.id build.module_id = module.id From 49bcb404c6911e2998ce4d50ed9321b3e8c32742 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 16:59:16 +0000 Subject: [PATCH 12/26] [frontend] fix baseurl for module repofile --- diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index 163be68..1e69364 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -10,6 +10,7 @@ from textwrap import dedent import re import flask +import posixpath from flask import url_for from dateutil import parser as dt_parser from netaddr import IPAddress, IPNetwork @@ -293,7 +294,7 @@ def generate_repo_url(mock_chroot, url): if mock_chroot.os_version != "rawhide": os_version = "$releasever" - url = urljoin( + url = posixpath.join( url, "{0}-{1}-{2}/".format(mock_chroot.os_release, os_version, "$basearch")) diff --git a/frontend/coprs_frontend/coprs/logic/modules_logic.py b/frontend/coprs_frontend/coprs/logic/modules_logic.py index 2eb831f..cf13282 100644 --- a/frontend/coprs_frontend/coprs/logic/modules_logic.py +++ b/frontend/coprs_frontend/coprs/logic/modules_logic.py @@ -30,6 +30,11 @@ class ModulesLogic(object): models.Module.copr_id == copr.id)) @classmethod + def get_by_nsv_str(cls, copr, nsv): + name, stream, version = nsv.split("-") + return cls.get_by_nsv(copr, name, stream, version) + + @classmethod def get_multiple(cls): return models.Module.query.order_by(models.Module.id.desc()) diff --git a/frontend/coprs_frontend/coprs/templates/coprs/copr-modules.cfg b/frontend/coprs_frontend/coprs/templates/coprs/copr-modules.cfg index 1d32ed6..97af256 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/copr-modules.cfg +++ b/frontend/coprs_frontend/coprs/templates/coprs/copr-modules.cfg @@ -1,4 +1,4 @@ -[{{ copr.repo_id }}] -name = Copr modules repo for {{ copr.full_name }} -url = {{ copr.modules_url | fix_url_https_backend }} +[{{ copr.repo_id }}_{{ module.nsv }}] +name = Copr modules repo for {{ module.full_name }} +baseurl = {{ baseurl | fix_url_https_backend }} enabled = 1 diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 59c4e9a..96bc800 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -819,19 +819,23 @@ def render_generate_repo_file(copr, name_release): ### Module repo files ### ######################################################### -@coprs_ns.route("///repo/modules/") -@coprs_ns.route("/@//repo/modules/") -@coprs_ns.route("/g///repo/modules/") +@coprs_ns.route("///module_repo//") +@coprs_ns.route("/g///module_repo//") @req_with_copr -def generate_module_repo_file(copr): +def generate_module_repo_file(copr, name_release, module_nsv): """ Generate module repo file for a given project. """ - return render_generate_module_repo_file(copr) + return render_generate_module_repo_file(copr, name_release, module_nsv) -def render_generate_module_repo_file(copr): +def render_generate_module_repo_file(copr, name_release, module_nsv): + module = ModulesLogic.get_by_nsv_str(copr, module_nsv).one() + mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(name_release, noarch=True).first() url = os.path.join(copr.repo_url, '') # adds trailing slash + repo_url = generate_repo_url(mock_chroot, copr.modules_url) + baseurl = "{}+{}/latest/$basearch".format(repo_url.rstrip("/"), module_nsv) pubkey_url = urljoin(url, "pubkey.gpg") response = flask.make_response( - flask.render_template("coprs/copr-modules.cfg", copr=copr, url=url, pubkey_url=pubkey_url)) + flask.render_template("coprs/copr-modules.cfg", copr=copr, module=module, + baseurl=baseurl, pubkey_url=pubkey_url)) response.mimetype = "text/plain" response.headers["Content-Disposition"] = \ "filename={0}.cfg".format(copr.repo_name) From e3abedcd5e895235a5b911a748cbee635852f560 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 13/26] [frontend][python] remove outdated modularity code --- diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 9661dd3..1edc58c 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -1266,10 +1266,3 @@ class Module(db.Model, helpers.Serializer): Return text representation of status of this build """ return helpers.ModuleStatusEnum(self.status) - - def repo_url(self, arch): - # @TODO Use custom chroot instead of fedora-24 - # @TODO Get rid of OS name from module path, see how koji does it - # https://kojipkgs.stg.fedoraproject.org/repos/module-base-runtime-0.25-9/latest/x86_64/toplink/packages/module-build-macros/0.1/ - module_dir = "fedora-24-{}+{}-{}-{}".format(arch, self.name, self.stream, self.version) - return "/".join([self.copr.repo_url, "modules", module_dir, "latest", arch]) diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index ffc3b21..02ac43b 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -1036,48 +1036,6 @@ def copr_build_module(copr): modulemd.name, modulemd.stream, modulemd.version)) -@api_ns.route("/coprs///module/make/", methods=["POST"]) -@api_login_required -@api_req_with_copr -def copr_make_module(copr): - form = forms.ModuleFormUploadFactory(csrf_enabled=False) - if not form.validate_on_submit(): - # @TODO Prettier error - raise LegacyApiError(form.errors) - - modulemd = form.modulemd.data.read() - module = ModulesLogic.from_modulemd(modulemd) - try: - ModulesLogic.validate(modulemd) - msg = "Nothing happened" - if form.create.data: - module = ModulesLogic.add(flask.g.user, copr, module) - db.session.flush() - msg = "Module was created" - - if form.build.data: - if not module.id: - module = ModulesLogic.get_by_nsv(copr, module.name, module.stream, module.version).one() - ActionsLogic.send_build_module(flask.g.user, copr, module) - msg = "Module build was submitted" - db.session.commit() - - return flask.jsonify({ - "output": "ok", - "message": msg, - "modulemd": modulemd.decode("utf-8"), - }) - - except sqlalchemy.exc.IntegrityError: - raise LegacyApiError({"nsv": ["Module {} already exists".format(module.nsv)]}) - - except sqlalchemy.orm.exc.NoResultFound: - raise LegacyApiError({"nsv": ["Module {} doesn't exist. You need to create it first".format(module.nsv)]}) - - except ValidationError as ex: - raise LegacyApiError({"nsv": [ex.message]}) - - @api_ns.route("/coprs///build-config//", methods=["GET"]) @api_ns.route("/g///build-config//", methods=["GET"]) @api_req_with_copr @@ -1094,21 +1052,3 @@ def copr_build_config(copr, chroot): raise LegacyApiError('Chroot not found.') return flask.jsonify(output) - - -@api_ns.route("/module/repo/", methods=["POST"]) -def copr_module_repo(): - """ - :return: URL to a DNF repository for the module - """ - form = forms.ModuleRepo(csrf_enabled=False) - if not form.validate_on_submit(): - raise LegacyApiError(form.errors) - - copr = ComplexLogic.get_copr_by_owner_safe(form.owner.data, form.copr.data) - nvs = [form.name.data, form.stream.data, form.version.data] - module = ModulesLogic.get_by_nsv(copr, *nvs).first() - if not module: - raise LegacyApiError("No module {}".format("-".join(nvs))) - - return flask.jsonify({"output": "ok", "repo": module.repo_url(form.arch.data)}) diff --git a/python/copr/client/client.py b/python/copr/client/client.py index d97f7e8..286b5d8 100644 --- a/python/copr/client/client.py +++ b/python/copr/client/client.py @@ -1616,27 +1616,3 @@ class CoprClient(UnicodeMixin): # @TODO Refactor process_package_action to be general purpose response = self.process_package_action(url, None, None, data=data, fetch_functor=fetch) return response - - def make_module(self, projectname, modulemd, username=None, create=True, build=True): - api_endpoint = "module/make" - ownername = username if username else self.username - f = open(modulemd, "rb") - data = { - "modulemd": (os.path.basename(f.name), f, "application/x-rpm"), - "username": self.username, - "create": "y" if create else "", - "build": "y" if build else "", - } - - url = "{0}/coprs/{1}/{2}/{3}/".format( - self.api_url, ownername, projectname, api_endpoint - ) - - def fetch(url, data, method): - m = MultipartEncoder(data) - monit = MultipartEncoderMonitor(m, lambda x: x) - return self._fetch(url, monit, method="post", headers={'Content-Type': monit.content_type}) - - # @TODO Refactor process_package_action to be general general purpose - response = self.process_package_action(url, ownername, projectname, data=data, fetch_functor=fetch) - return response From bbca18bda0a9aecece68f5a6b4d34a3d8a9c97d0 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 14/26] [frontend] show module repofiles --- diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html index a07279b..067e590 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/module.html @@ -149,6 +149,13 @@ dnf module install {{ module.name }}/default

For dnf module documentation, see the "Module Command" section in man dnf. + +

+ Repofiles: + {% for chroot in unique_chroots %} + {{ chroot.name_release }} + {% endfor %} + {% endif %} diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 96bc800..e2a8ef1 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -1024,7 +1024,20 @@ def copr_module(copr, id): module = ModulesLogic.get(id).first() formatter = HtmlFormatter(style="autumn", linenos=False, noclasses=True) pretty_yaml = highlight(module.yaml, get_lexer_by_name("YAML"), formatter) - return flask.render_template("coprs/detail/module.html", copr=copr, module=module, yaml=pretty_yaml) + + # Get the list of chroots with unique name_release attribute + # Once we use jinja in 2.10 version, we can simply use + # {{ copr.active_chroots |unique(attribute='name_release') }} + unique_chroots = [] + unique_name_releases = set() + for chroot in copr.active_chroots_sorted: + if chroot.name_release in unique_name_releases: + continue + unique_chroots.append(chroot) + unique_name_releases.add(chroot.name_release) + + return flask.render_template("coprs/detail/module.html", copr=copr, module=module, + yaml=pretty_yaml, unique_chroots=unique_chroots) @coprs_ns.route("///module//raw") From deda574f6a80de970916742f2fd91a5a2d79a3ba Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 15/26] [frontend] re-enable webform for building modules --- diff --git a/frontend/coprs_frontend/coprs/logic/modules_logic.py b/frontend/coprs_frontend/coprs/logic/modules_logic.py index cf13282..5c7f52b 100644 --- a/frontend/coprs_frontend/coprs/logic/modules_logic.py +++ b/frontend/coprs_frontend/coprs/logic/modules_logic.py @@ -77,6 +77,51 @@ class ModulesLogic(object): mmd.version = mmd.version or int(datetime.now().strftime("%Y%m%d%H%M%S")) +class ModuleBuildFacade(object): + def __init__(self, user, copr, yaml, filename=None): + self.user = user + self.copr = copr + self.yaml = yaml + self.filename = filename + + self.modulemd = ModulesLogic.yaml2modulemd(yaml) + ModulesLogic.set_defaults_for_optional_params(self.modulemd, filename=filename) + ModulesLogic.validate(self.modulemd) + + def submit_build(self): + module = ModulesLogic.add(self.user, self.copr, ModulesLogic.from_modulemd(self.modulemd)) + self.add_builds(self.modulemd.components.rpms, module) + return module + + def get_build_batches(self, rpms): + """ + Determines Which component should be built in which batch. Returns an ordered list of grouped components, + first group of components should be built as a first batch, second as second and so on. + Particular components groups are represented by lists and can by built in a random order within the batch. + :return: list of lists + """ + return [rpms] + + def add_builds(self, rpms, module): + for group in self.get_build_batches(rpms): + batch = models.Batch() + db.session.add(batch) + for pkgname, rpm in group.items(): + clone_url = self.get_clone_url(pkgname, rpm) + build = builds_logic.BuildsLogic.create_new_from_scm(self.user, self.copr, scm_type="git", + clone_url=clone_url, committish=rpm.ref) + build.batch_id = batch.id + build.module_id = module.id + + def get_clone_url(self, pkgname, rpm): + return rpm.repository if rpm.repository else self.default_distgit.format(pkgname=pkgname) + + @property + def default_distgit(self): + # @TODO move to better place + return "https://src.fedoraproject.org/rpms/{pkgname}" + + class ModulemdGenerator(object): def __init__(self, name="", stream="", version=0, summary="", config=None): self.config = config @@ -137,11 +182,6 @@ class ModulemdGenerator(object): def add_buildrequires(self, module, stream): self.mmd.add_buildrequires(module, stream) - def add_base_runtime(self): - name, stream = "base-runtime", "master" - self.add_requires(name, stream) - self.add_buildrequires(name, stream) - def generate(self): return self.mmd.dumps() diff --git a/frontend/coprs_frontend/coprs/templates/coprs/create_module.html b/frontend/coprs_frontend/coprs/templates/coprs/create_module.html index 90d4a33..b30d763 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/create_module.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/create_module.html @@ -35,18 +35,6 @@ {{ render_form_errors(form) }} -
-

This feature is not yet available due to recent modularity changes. - You can build your module using the command line

- 
-          
-            copr-cli build-module --yaml /path/to/yourmodule.yaml {{ copr.full_name }}
-          
-
-

Learn more

-
- - {#
@@ -171,7 +159,6 @@

Copr will generate the modulemd yaml file and create a modular RPM repository for you.

- #}
diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index 02ac43b..20bee43 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -22,7 +22,7 @@ from coprs.logic.builds_logic import BuildsLogic from coprs.logic.complex_logic import ComplexLogic from coprs.logic.users_logic import UsersLogic from coprs.logic.packages_logic import PackagesLogic -from coprs.logic.modules_logic import ModulesLogic, ModuleProvider +from coprs.logic.modules_logic import ModulesLogic, ModuleProvider, ModuleBuildFacade from coprs.views.misc import login_required, api_login_required @@ -998,28 +998,13 @@ def copr_build_module(copr): if not form.validate_on_submit(): raise LegacyApiError(form.errors) - modulemd = None + facade = None try: mod_info = ModuleProvider.from_input(form.modulemd.data or form.scmurl.data) - modulemd = ModulesLogic.yaml2modulemd(mod_info.yaml) - ModulesLogic.set_defaults_for_optional_params(modulemd, filename=mod_info.filename) - ModulesLogic.validate(modulemd) - - module = ModulesLogic.add(flask.g.user, copr, ModulesLogic.from_modulemd(modulemd)) - batch = models.Batch() - db.session.add(batch) - - for pkgname, rpm in modulemd.components.rpms.items(): - # @TODO move to better place - default_distgit = "https://src.fedoraproject.org/rpms/{pkgname}" - clone_url = rpm.repository if rpm.repository else default_distgit.format(pkgname=pkgname) - - build = BuildsLogic.create_new_from_scm(flask.g.user, copr, scm_type="git", - clone_url=clone_url, committish=rpm.ref) - build.batch_id = batch.id - build.module_id = module.id - + facade = ModuleBuildFacade(flask.g.user, copr, mod_info.yaml, mod_info.filename) + module = facade.submit_build() db.session.commit() + return flask.jsonify({ "output": "ok", "message": "Created module {}".format(module.nsv), @@ -1033,7 +1018,7 @@ def copr_build_module(copr): except sqlalchemy.exc.IntegrityError: raise LegacyApiError("Module {}-{}-{} already exists".format( - modulemd.name, modulemd.stream, modulemd.version)) + facade.modulemd.name, facade.modulemd.stream, facade.modulemd.version)) @api_ns.route("/coprs///build-config//", methods=["GET"]) diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index e2a8ef1..d482470 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -16,6 +16,7 @@ import sqlalchemy import modulemd from email.mime.text import MIMEText from itertools import groupby +from wtforms import ValidationError from pygments import highlight from pygments.lexers import get_lexer_by_name @@ -33,7 +34,7 @@ from coprs.logic.coprs_logic import CoprsLogic from coprs.logic.packages_logic import PackagesLogic from coprs.logic.stat_logic import CounterStatLogic from coprs.logic.users_logic import UsersLogic -from coprs.logic.modules_logic import ModulesLogic, ModulemdGenerator, MBSProxy +from coprs.logic.modules_logic import ModulesLogic, ModulemdGenerator, ModuleBuildFacade from coprs.rmodels import TimedStatEvents from coprs.logic.complex_logic import ComplexLogic @@ -1001,20 +1002,26 @@ def build_module(copr, form): generator.add_api(form.api.data) generator.add_profiles(enumerate(zip(form.profile_names.data, form.profile_pkgs.data))) generator.add_components(form.packages.data, form.filter.data, form.builds.data) - generator.add_base_runtime() - tmp = tempfile.mktemp() - generator.dump(tmp) + yaml = generator.generate() - proxy = MBSProxy(mbs_url=flask.current_app.config["MBS_URL"], user_name=flask.g.user.name) - with open(tmp) as tmp_handle: - response = proxy.build_module(copr.owner_name, copr.name, generator.nsv, tmp_handle) - os.remove(tmp) + facade = None + try: + facade = ModuleBuildFacade(flask.g.user, copr, yaml) + module = facade.submit_build() + db.session.commit() + + flask.flash("Modulemd yaml file successfully generated and submitted to be build as {}" + .format(module.nsv), "success") + return flask.redirect(url_for_copr_details(copr)) - if response.failed: - flask.flash(response.message, "error") + except ValidationError as ex: + flask.flash(ex.message, "error") + return render_create_module(copr, form, len(form.profile_names)) + + except sqlalchemy.exc.IntegrityError: + flask.flash("Module {}-{}-{} already exists".format( + facade.modulemd.name, facade.modulemd.stream, facade.modulemd.version), "error") return render_create_module(copr, form, len(form.profile_names)) - flask.flash("Modulemd yaml file successfully generated and submitted to be build", "success") - return flask.redirect(url_for_copr_details(copr)) @coprs_ns.route("///module/") From 0e94049995ca500d5c5e943bef0b26df6ff9e07b Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 16/26] [frontend] fix condition that all module packages were successfully built The previous version actually filtered successful packages first, and then checked whether the objects are True. This is not the intended logic. We rather want to get status of all module packages and then check if all of them are successful. --- diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 252e622..7ac0f58 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -762,7 +762,7 @@ GROUP BY # then send an action to create module repodata on backend if (build.module and upd_dict.get("status") == StatusEnum("succeeded") - and all(b for b in build.module.builds if b.status == StatusEnum("succeeded"))): + and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): ActionsLogic.send_build_module(build.copr, build.module) for attr in ["results", "built_packages", "srpm_url"]: From 13a4ace243f9cfb1e3d0ff957c51769a3bf91cdc Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 17/26] [frontend] generate the module NSV rather than asking for it --- diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index 9abf39a..1cd6092 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -881,9 +881,6 @@ class ActivateFasGroupForm(FlaskForm): class CreateModuleForm(FlaskForm): - name = wtforms.StringField("Name") - stream = wtforms.StringField("Stream") - version = wtforms.IntegerField("Version") builds = wtforms.FieldList(wtforms.StringField("Builds ID list")) packages = wtforms.FieldList(wtforms.StringField("Packages list")) filter = wtforms.FieldList(wtforms.StringField("Package Filter")) @@ -899,12 +896,6 @@ class CreateModuleForm(FlaskForm): if not FlaskForm.validate(self): return False - module = ModulesLogic.get_by_nsv(self.copr, self.name.data, self.stream.data, self.version.data).first() - if module: - self.errors["nsv"] = [Markup("Module {} already exists".format( - helpers.copr_url("coprs_ns.copr_module", module.copr, id=module.id), module.full_name))] - return False - # Profile names should be unique names = filter(None, self.profile_names.data) if len(set(names)) < len(names): diff --git a/frontend/coprs_frontend/coprs/templates/coprs/create_module.html b/frontend/coprs_frontend/coprs/templates/coprs/create_module.html index b30d763..b6d1c46 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/create_module.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/create_module.html @@ -37,30 +37,6 @@
- -
-
-
-
-
- - - -
-
- - {{ render_field(form.stream, id="moduleStream") }} -
-
- - {{ render_field(form.version, id="moduleVersion") }} -
-
-
-
-
- - {% for package, build in built_packages %} diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index d482470..675cce5 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -996,8 +996,7 @@ def build_module(copr, form): return render_create_module(copr, form, profiles=len(form.profile_names)) summary = "Module from Copr repository: {}".format(copr.full_name) - generator = ModulemdGenerator(str(copr.name), str(form.stream.data), - form.version.data, summary, app.config) + generator = ModulemdGenerator(str(copr.name), summary=summary, config=app.config) generator.add_filter(form.filter.data) generator.add_api(form.api.data) generator.add_profiles(enumerate(zip(form.profile_names.data, form.profile_pkgs.data))) @@ -1021,6 +1020,7 @@ def build_module(copr, form): except sqlalchemy.exc.IntegrityError: flask.flash("Module {}-{}-{} already exists".format( facade.modulemd.name, facade.modulemd.stream, facade.modulemd.version), "error") + db.session.rollback() return render_create_module(copr, form, len(form.profile_names)) From bec6f75d498476d1f70d9c0323c7c98296c389fc Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 18/26] [frontend][backend] copy only module builds into the repo directory --- diff --git a/backend/backend/actions.py b/backend/backend/actions.py index 95ea225..0d3e73e 100644 --- a/backend/backend/actions.py +++ b/backend/backend/actions.py @@ -442,8 +442,16 @@ class Action(object): if os.path.exists(destdir): self.log.warning("Module {0} already exists. Omitting.".format(destdir)) else: - self.log.info("Copy directory: {} as {}".format(srcdir, destdir)) - shutil.copytree(srcdir, destdir) + # We want to copy just the particular module builds + # into the module destdir, not the whole chroot + os.makedirs(destdir) + prefixes = ["{:08d}-".format(x) for x in data["builds"]] + dirs = [d for d in os.listdir(srcdir) if d.startswith(tuple(prefixes))] + for folder in dirs: + shutil.copytree(os.path.join(srcdir, folder), os.path.join(destdir, folder)) + self.log.info("Copy directory: {} as {}".format( + os.path.join(srcdir, folder), os.path.join(destdir, folder))) + modulemd.dump_all(os.path.join(destdir, "modules.yaml"), [mmd]) createrepo(path=destdir, front_url=self.front_url, username=ownername, projectname=projectname, diff --git a/frontend/coprs_frontend/coprs/logic/actions_logic.py b/frontend/coprs_frontend/coprs/logic/actions_logic.py index c2cf7f7..06d1867 100644 --- a/frontend/coprs_frontend/coprs/logic/actions_logic.py +++ b/frontend/coprs_frontend/coprs/logic/actions_logic.py @@ -265,6 +265,7 @@ class ActionsLogic(object): data = { "chroots": [c.name for c in copr.active_chroots], + "builds": [b.id for b in module.builds], } action = models.Action( From 2db786742503661f7ec0400093508853896a8d95 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 19/26] [frontend] use exists instead of all, it is faster --- diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 7ac0f58..62d2ffe 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -11,7 +11,7 @@ from sqlalchemy import or_ from sqlalchemy import and_ from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound -from sqlalchemy.sql import false,true +from sqlalchemy.sql import false, true, exists from werkzeug.utils import secure_filename from sqlalchemy import desc,asc, bindparam, Integer from collections import defaultdict @@ -762,7 +762,7 @@ GROUP BY # then send an action to create module repodata on backend if (build.module and upd_dict.get("status") == StatusEnum("succeeded") - and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): + and not exists(b.status != StatusEnum("succeeded") for b in build.module.builds)): ActionsLogic.send_build_module(build.copr, build.module) for attr in ["results", "built_packages", "srpm_url"]: From 47124a815b872dd4b4c46a9058f6440302a10eca Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:11 +0000 Subject: [PATCH 20/26] [frontend] add index to build module_id --- diff --git a/frontend/coprs_frontend/alembic/schema/versions/e183e12563ee_add_index_to_module_id.py b/frontend/coprs_frontend/alembic/schema/versions/e183e12563ee_add_index_to_module_id.py new file mode 100644 index 0000000..14cfb79 --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/e183e12563ee_add_index_to_module_id.py @@ -0,0 +1,22 @@ +"""Add index to module_id + +Revision ID: e183e12563ee +Revises: 1f94b22f70a1 +Create Date: 2018-01-07 11:03:29.885276 + +""" + +# revision identifiers, used by Alembic. +revision = 'e183e12563ee' +down_revision = '1f94b22f70a1' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_index(op.f('ix_build_module_id'), 'build', ['module_id'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_build_module_id'), table_name='build') diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 1edc58c..77e2446 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -553,7 +553,7 @@ class Build(db.Model, helpers.Serializer): batch_id = db.Column(db.Integer, db.ForeignKey("batch.id")) batch = db.relationship("Batch", backref=db.backref("builds")) - module_id = db.Column(db.Integer, db.ForeignKey("module.id")) + module_id = db.Column(db.Integer, db.ForeignKey("module.id"), index=True) module = db.relationship("Module", backref=db.backref("builds")) @property From 563542692701fbbaa616b9d46f94b0f97be3eb7c Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:40 +0000 Subject: [PATCH 21/26] [frontend] remove outdated tests, see 3f62873 --- diff --git a/frontend/coprs_frontend/tests/test_views/test_api_ns/test_api_general.py b/frontend/coprs_frontend/tests/test_views/test_api_ns/test_api_general.py index 89b0c3d..2b0e436 100644 --- a/frontend/coprs_frontend/tests/test_views/test_api_ns/test_api_general.py +++ b/frontend/coprs_frontend/tests/test_views/test_api_ns/test_api_general.py @@ -103,62 +103,3 @@ class TestCreateCopr(CoprsTestCase): # self.db.session.add_all([self.u1, self.mc1]) # # - - -class TestModuleRepo(CoprsTestCase): - endpoint = "/api/module/repo/" - - def test_api_module_repo(self, f_users, f_coprs, f_modules, f_db): - data = {"owner": self.u1.name, "copr": self.c1.name, "name": "first-module", - "stream": "foo", "version": 1, "arch": "x86_64"} - - r = self.tc.post(self.endpoint, data=data) - response = json.loads(r.data.decode("utf-8")) - assert response["output"] == "ok" - assert response["repo"] == "http://copr-be-dev.cloud.fedoraproject.org/results/user1/foocopr/modules/"\ - "fedora-24-x86_64+first-module-foo-1/latest/x86_64" - - def test_api_module_repo_no_params(self): - error = "This field is required." - r = self.tc.post(self.endpoint, data={}) - response = json.loads(r.data.decode("utf-8")) - assert response["output"] == "notok" - for key in ["owner", "copr", "name", "stream", "version", "arch"]: - assert error in response["error"][key] - - -class TestBuildModule(CoprsTestCase): - @TransactionDecorator("u1") - def test_api_build_module_basic(self, f_users, f_coprs, f_db): - self.db.session.add_all([self.u1, self.c1]) - self.tc.post("/api/new/") - - fd, filename = tempfile.mkstemp() - os.write(fd, b""" - data: - api: - rpms: [example-debuginfo] - components: {} - description: '' - filter: - rpms: [example-debuginfo, example] - license: - module: [] - name: project - profiles: - default: - rpms: [example] - stream: test - summary: 'Module from Copr repository: clime/project' - version: 1 - document: modulemd - version: 1 - """) - os.close(fd) - - f = open(filename, "rb") - data = {"modulemd": (filename, f.read().decode("utf-8"), "application/yaml")} - api_endpoint = '/api/coprs/{}/{}/module/make/'.format(self.u1.name, self.c1.name) - r = self.post_api_with_auth(api_endpoint, content=data, user=self.u1) - assert r.status_code == 200 - os.remove(filename) From 2a0da14ec5bf940244026414161bbdee27c32413 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:40 +0000 Subject: [PATCH 22/26] [frontend] when passing URL with path, expect it in result; see ad9c3b4cc --- diff --git a/frontend/coprs_frontend/tests/test_helpers.py b/frontend/coprs_frontend/tests/test_helpers.py index 4ed29a0..13881fe 100644 --- a/frontend/coprs_frontend/tests/test_helpers.py +++ b/frontend/coprs_frontend/tests/test_helpers.py @@ -38,8 +38,9 @@ class TestHelpers(CoprsTestCase): def test_generate_repo_url(self): test_sets = [] - http_url = "http://example.com/repo" - https_url = "https://example.com/repo" + + http_url = "http://example.com/path" + https_url = "https://example.com/path" mock_chroot = mock.MagicMock() mock_chroot.os_release = "fedora" @@ -47,18 +48,18 @@ class TestHelpers(CoprsTestCase): test_sets.extend([ dict(args=(mock_chroot, http_url), - expected="http://example.com/fedora-$releasever-$basearch/"), + expected="http://example.com/path/fedora-$releasever-$basearch/"), dict(args=(mock_chroot, https_url), - expected="https://example.com/fedora-$releasever-$basearch/")]) + expected="https://example.com/path/fedora-$releasever-$basearch/")]) m2 = deepcopy(mock_chroot) m2.os_version = "rawhide" test_sets.extend([ dict(args=(m2, http_url), - expected="http://example.com/fedora-rawhide-$basearch/"), + expected="http://example.com/path/fedora-rawhide-$basearch/"), dict(args=(m2, https_url), - expected="https://example.com/fedora-rawhide-$basearch/")]) + expected="https://example.com/path/fedora-rawhide-$basearch/")]) m3 = deepcopy(mock_chroot) m3.os_release = "rhel7" @@ -66,9 +67,9 @@ class TestHelpers(CoprsTestCase): test_sets.extend([ dict(args=(m3, http_url), - expected="http://example.com/rhel7-7.1-$basearch/"), + expected="http://example.com/path/rhel7-7.1-$basearch/"), dict(args=(m3, https_url), - expected="https://example.com/rhel7-7.1-$basearch/")]) + expected="https://example.com/path/rhel7-7.1-$basearch/")]) app.config["USE_HTTPS_FOR_RESULTS"] = True for test_set in test_sets: From d435300d926ff438f51b2079cff60d542d37ccc9 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:40 +0000 Subject: [PATCH 23/26] [python] use username from config if nothing is explicitly specified --- diff --git a/python/copr/client/client.py b/python/copr/client/client.py index 286b5d8..22cc04a 100644 --- a/python/copr/client/client.py +++ b/python/copr/client/client.py @@ -1599,6 +1599,9 @@ class CoprClient(UnicodeMixin): return response def build_module(self, modulemd, ownername=None, projectname=None): + if not ownername: + ownername = self.username + url = "{0}/coprs/{1}/{2}/module/build/".format( self.api_url, ownername, projectname ) From 634ec4f3f029cc02597fc168d4743c93d74db3dc Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:00:40 +0000 Subject: [PATCH 24/26] Revert "[frontend] use exists instead of all, it is faster" This reverts commit b66fef178ab638716672122f65b95dd1fb4abfe6. --- diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 62d2ffe..7ac0f58 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -11,7 +11,7 @@ from sqlalchemy import or_ from sqlalchemy import and_ from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound -from sqlalchemy.sql import false, true, exists +from sqlalchemy.sql import false,true from werkzeug.utils import secure_filename from sqlalchemy import desc,asc, bindparam, Integer from collections import defaultdict @@ -762,7 +762,7 @@ GROUP BY # then send an action to create module repodata on backend if (build.module and upd_dict.get("status") == StatusEnum("succeeded") - and not exists(b.status != StatusEnum("succeeded") for b in build.module.builds)): + and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): ActionsLogic.send_build_module(build.copr, build.module) for attr in ["results", "built_packages", "srpm_url"]: From e52c84e55d18e744c4fe1efbd131f7e77ed9569a Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 17:37:12 +0000 Subject: [PATCH 25/26] [frontend] rewrite condition because unicode is undefined in python3 --- diff --git a/frontend/coprs_frontend/coprs/logic/modules_logic.py b/frontend/coprs_frontend/coprs/logic/modules_logic.py index 5c7f52b..85a495f 100644 --- a/frontend/coprs_frontend/coprs/logic/modules_logic.py +++ b/frontend/coprs_frontend/coprs/logic/modules_logic.py @@ -231,9 +231,9 @@ class ModuleProvider(object): @classmethod def from_input(cls, obj): - if type(obj) in [str, unicode]: - return cls.from_url(obj) - return cls.from_file(obj) + if hasattr(obj, "read"): + return cls.from_file(obj) + return cls.from_url(obj) @classmethod def from_file(cls, ref): From acdc59191e7c9158d709efed4de1fc59800aea65 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Jan 13 2018 22:53:47 +0000 Subject: [PATCH 26/26] [frontend] b64encode requires bytes in python3 --- diff --git a/frontend/coprs_frontend/coprs/logic/modules_logic.py b/frontend/coprs_frontend/coprs/logic/modules_logic.py index 85a495f..fd32243 100644 --- a/frontend/coprs_frontend/coprs/logic/modules_logic.py +++ b/frontend/coprs_frontend/coprs/logic/modules_logic.py @@ -50,8 +50,9 @@ class ModulesLogic(object): @classmethod def from_modulemd(cls, mmd): + yaml_b64 = base64.b64encode(mmd.dumps().encode("utf-8")).decode("utf-8") return models.Module(name=mmd.name, stream=mmd.stream, version=mmd.version, summary=mmd.summary, - description=mmd.description, yaml_b64=base64.b64encode(mmd.dumps())) + description=mmd.description, yaml_b64=yaml_b64) @classmethod def validate(cls, mmd):