#82 Mark module as failed when it has failed builds
Closed 6 years ago by frostyx. Opened 6 years ago by frostyx.
copr/ frostyx/copr mark-module-failed  into  master

@@ -0,0 +1,23 @@ 

+ """Add reference between builds and modules

+ 

+ Revision ID: e5a41d2bada

+ Revises: bf4b5dc74740

+ Create Date: 2017-06-11 09:42:59.526921

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = 'e5a41d2bada'

+ down_revision = 'bf4b5dc74740'

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     op.add_column('build', sa.Column('module_id', sa.Integer(), nullable=True))

+     op.drop_index('build_copr_id_package_id', table_name='build')

+ 

+ 

+ def downgrade():

+     op.drop_column('build', 'module_id')

@@ -518,6 +518,7 @@ 

  

          F.enable_net = wtforms.BooleanField()

          F.background = wtforms.BooleanField(default=False)

+         F.module = wtforms.StringField()

          F.package_name = wtforms.StringField()

  

          F.chroots_list = list(map(lambda x: x.name, active_chroots))

@@ -337,7 +337,7 @@ 

  

      @classmethod

      def create_new_from_other_build(cls, user, copr, source_build,

-                                     chroot_names=None, **build_options):

+                                     chroot_names=None, module=None, **build_options):

          skip_import = False

          git_hashes = {}

  
@@ -357,34 +357,38 @@ 

                  git_hashes[chroot.name] = chroot.git_hash

  

          build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names,

-                                     pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, **build_options)

+                                     pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import,

+                                     module=module, **build_options)

          build.package_id = source_build.package_id

          build.pkg_version = source_build.pkg_version

          return build

  

      @classmethod

      def create_new_from_url(cls, user, copr, srpm_url,

-                             chroot_names=None, **build_options):

+                             chroot_names=None, module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr

  

          :type chroot_names: List[str]

+         :type module: models.Module

  

          :rtype: models.Build

          """

          source_type = helpers.BuildSourceEnum("srpm_link")

          source_json = json.dumps({"url": srpm_url})

-         return cls.create_new(user, copr, source_type, source_json, chroot_names, pkgs=srpm_url, **build_options)

+         return cls.create_new(user, copr, source_type, source_json, chroot_names,

+                               pkgs=srpm_url, module=module, **build_options)

  

      @classmethod

      def create_new_from_tito(cls, user, copr, git_url, git_dir, git_branch, tito_test,

-                             chroot_names=None, **build_options):

+                             chroot_names=None, module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr

  

          :type chroot_names: List[str]

+         :type module: models.Module

  

          :rtype: models.Build

          """
@@ -393,16 +397,17 @@ 

                                    "git_dir": git_dir,

                                    "git_branch": git_branch,

                                    "tito_test": tito_test})

-         return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)

+         return cls.create_new(user, copr, source_type, source_json, chroot_names, module=module, **build_options)

  

      @classmethod

      def create_new_from_mock(cls, user, copr, scm_type, scm_url, scm_branch, spec,

-                              chroot_names=None, **build_options):

+                              chroot_names=None, module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr

  

          :type chroot_names: List[str]

+         :type module: models.Module

  

          :rtype: models.Build

          """
@@ -411,11 +416,11 @@ 

                                    "scm_url": scm_url,

                                    "scm_branch": scm_branch,

                                    "spec": spec})

-         return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)

+         return cls.create_new(user, copr, source_type, source_json, chroot_names, module=module, **build_options)

  

      @classmethod

      def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions,

-                              chroot_names=None, **build_options):

+                              chroot_names=None, module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr
@@ -424,6 +429,7 @@ 

          :type python_versions: List[str]

  

          :type chroot_names: List[str]

+         :type module: models.Module

  

          :rtype: models.Build

          """
@@ -431,31 +437,33 @@ 

          source_json = json.dumps({"pypi_package_name": pypi_package_name,

                                    "pypi_package_version": pypi_package_version,

                                    "python_versions": python_versions})

-         return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)

+         return cls.create_new(user, copr, source_type, source_json, chroot_names, module=module, **build_options)

  

      @classmethod

      def create_new_from_rubygems(cls, user, copr, gem_name,

-                                  chroot_names=None, **build_options):

+                                  chroot_names=None, module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr

          :type gem_name: str

          :type chroot_names: List[str]

+         :type module: models.Module

          :rtype: models.Build

          """

          source_type = helpers.BuildSourceEnum("rubygems")

          source_json = json.dumps({"gem_name": gem_name})

-         return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)

+         return cls.create_new(user, copr, source_type, source_json, chroot_names, module=module, **build_options)

  

      @classmethod

      def create_new_from_distgit(cls, user, copr, clone_url, branch,

-                                  chroot_names=None, **build_options):

+                                  chroot_names=None, module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr

          :type clone_url: str

          :type branch: str

          :type chroot_names: List[str]

+         :type module: models.Module

          :rtype: models.Build

          """

          source_type = helpers.BuildSourceEnum("distgit")
@@ -463,14 +471,15 @@ 

              "clone_url": clone_url,

              "branch": branch

          })

-         return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)

+         return cls.create_new(user, copr, source_type, source_json, chroot_names, module=module, **build_options)

  

      @classmethod

      def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,

-                                chroot_names=None, **build_options):

+                                chroot_names=None, module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr

+         :type module: models.Module

          :param f_uploader(file_path): function which stores data at the given `file_path`

          :return:

          """
@@ -491,7 +500,7 @@ 

          source_json = json.dumps({"tmp": tmp_name, "pkg": filename})

          try:

              build = cls.create_new(user, copr, source_type, source_json,

-                                         chroot_names, pkgs=pkg_url, **build_options)

+                                         chroot_names, pkgs=pkg_url, module=module, **build_options)

          except Exception:

              shutil.rmtree(tmp)  # todo: maybe we should delete in some cleanup procedure?

              raise
@@ -500,7 +509,8 @@ 

  

      @classmethod

      def create_new(cls, user, copr, source_type, source_json, chroot_names=None,

-                         pkgs="", git_hashes=None, skip_import=False, background=False, **build_options):

+                         pkgs="", git_hashes=None, skip_import=False, background=False,

+                         module=None, **build_options):

          """

          :type user: models.User

          :type copr: models.Copr
@@ -511,6 +521,7 @@ 

          :type git_hashes: dict

          :type skip_import: bool

          :type background: bool

+         :type module: models.Module

          :rtype: models.Build

          """

          if chroot_names is None:
@@ -525,6 +536,7 @@ 

              user=user,

              pkgs=pkgs,

              copr=copr,

+             module=module,

              chroots=chroots,

              source_type=source_type,

              source_json=source_json,
@@ -542,7 +554,7 @@ 

      @classmethod

      def add(cls, user, pkgs, copr, source_type=None, source_json=None,

              repos=None, chroots=None, timeout=None, enable_net=True,

-             git_hashes=None, skip_import=False, background=False):

+             git_hashes=None, skip_import=False, background=False, module=None):

          if chroots is None:

              chroots = []

  
@@ -569,6 +581,7 @@ 

              user=user,

              pkgs=pkgs,

              copr=copr,

+             module=module,

              repos=repos,

              source_type=source_type,

              source_json=source_json,

@@ -29,6 +29,14 @@ 

                   models.Module.copr_id == copr.id))

  

      @classmethod

+     def get_by_nsv_str(cls, copr, nsv):

+         try:

+             name, stream, version = nsv.rsplit("-", 2)

+             return cls.get_by_nsv(copr, name, stream, version)

+         except ValueError:

+             raise ValidationError("Incorrect module NSV format")

+ 

+     @classmethod

      def get_multiple(cls):

          return models.Module.query.order_by(models.Module.id.desc())

  

@@ -526,6 +526,8 @@ 

      copr = db.relationship("Copr", backref=db.backref("builds"))

      package_id = db.Column(db.Integer, db.ForeignKey("package.id"))

      package = db.relationship("Package")

+     module_id = db.Column(db.Integer, db.ForeignKey("module.id"))

+     module = db.relationship("Module", backref=db.backref("builds"))

  

      chroots = association_proxy("build_chroots", "mock_chroot")

  
@@ -1200,9 +1202,12 @@ 

          """

          Return text representation of status of this build

          """

+ 

+         if any(build.status == StatusEnum("failed") for build in self.builds):

+             return helpers.ModuleStatusEnum("failed")

          if self.action is not None:

              return helpers.ModuleStatusEnum(self.action.result)

-         return "-"

+         return None

  

      def repo_url(self, arch):

          # @TODO Use custom chroot instead of fedora-24

@@ -135,8 +135,8 @@ 

  {% endmacro %}

  

  {% macro module_state(module) %}

-   {% if module.action %}

-     {{ build_state_text(module.action.result | module_state_from_num) }}

+   {% if module.state %}

+     {{ build_state_text(module.state | module_state_from_num) }}

    {% else %}

      -

    {% endif %}

@@ -346,6 +346,7 @@ 

              srpm_url=pkg,

              chroot_names=form.selected_chroots,

              background=form.background.data,

+             module=ModulesLogic.get_by_nsv_str(copr, form.module.data).one_or_none(),

          ) for pkg in pkgs]

      return process_creating_new_build(copr, form, create_new_build)

  
@@ -363,6 +364,7 @@ 

              orig_filename=secure_filename(form.pkgs.data.filename),

              chroot_names=form.selected_chroots,

              background=form.background.data,

+             module=ModulesLogic.get_by_nsv_str(copr, form.module.data).one_or_none(),

          )

      return process_creating_new_build(copr, form, create_new_build)

  
@@ -386,6 +388,7 @@ 

              form.python_versions.data,

              form.selected_chroots,

              background=form.background.data,

+             module=ModulesLogic.get_by_nsv_str(copr, form.module.data).one_or_none(),

          )

      return process_creating_new_build(copr, form, create_new_build)

  
@@ -406,6 +409,7 @@ 

              form.tito_test.data,

              form.selected_chroots,

              background=form.background.data,

+             module=ModulesLogic.get_by_nsv_str(copr, form.module.data).one_or_none(),

          )

      return process_creating_new_build(copr, form, create_new_build)

  
@@ -426,6 +430,7 @@ 

              form.spec.data,

              form.selected_chroots,

              background=form.background.data,

+             module=ModulesLogic.get_by_nsv_str(copr, form.module.data).one_or_none(),

          )

      return process_creating_new_build(copr, form, create_new_build)

  
@@ -443,6 +448,7 @@ 

              form.gem_name.data,

              form.selected_chroots,

              background=form.background.data,

+             module=ModulesLogic.get_by_nsv_str(copr, form.module.data).one_or_none(),

          )

      return process_creating_new_build(copr, form, create_new_build)

  
@@ -461,6 +467,7 @@ 

              form.branch.data,

              form.selected_chroots,

              background=form.background.data,

+             module=ModulesLogic.get_by_nsv_str(copr, form.module.data).one_or_none(),

          )

      return process_creating_new_build(copr, form, create_new_build)

  

file modified
+26 -13
@@ -355,7 +355,7 @@ 

  

      def create_new_build(self, projectname, pkgs, username=None,

                           timeout=None, memory=None, chroots=None,

-                          background=False, progress_callback=None):

+                          background=False, module=None, progress_callback=None):

          """ Creates new build

  

              :param projectname: name of Copr project (without user namespace)
@@ -365,6 +365,7 @@ 

              :param memory: [optional] amount of required memory for build process

              :param chroots: [optional] build only with given chroots

              :param background: [optional] mark the build as a background job.

+             :param module: [optional] str NSV of a module

              :param progress_callback: [optional] a function that received a

              MultipartEncoderMonitor instance for each chunck of uploaded data

  
@@ -389,11 +390,11 @@ 

                  raise CoprRequestException(e)

  

          return self.process_creating_new_build(projectname, data, api_endpoint, username, chroots, background=background,

-                                                progress_callback=progress_callback, multipart=True)

+                                                module=module, progress_callback=progress_callback, multipart=True)

  

      def create_new_build_pypi(self, projectname, pypi_package_name, pypi_package_version=None,

                           python_versions=[3, 2], username=None, timeout=None, memory=None,

-                          chroots=None, background=False, progress_callback=None):

+                          chroots=None, background=False, module=None, progress_callback=None):

          """ Creates new build from PyPI

  

              :param projectname: name of Copr project (without user namespace)
@@ -405,6 +406,7 @@ 

              :param memory: [optional] amount of required memory for build process

              :param chroots: [optional] build only with given chroots

              :param background: [optional] mark the build as a background job.

+             :param module: [optional] str NSV of a module

              :param progress_callback: [optional] a function that received a

              MultipartEncoderMonitor instance for each chunck of uploaded data

  
@@ -421,10 +423,11 @@ 

          }

          api_endpoint = "new_build_pypi"

          return self.process_creating_new_build(projectname, data, api_endpoint, username,

-                                                chroots, background=background)

+                                                chroots, background=background, module=module)

  

      def create_new_build_tito(self, projectname, git_url, git_dir=None, git_branch=None, tito_test=None, username=None,

-                               timeout=None, memory=None, chroots=None, background=False, progress_callback=None):

+                               timeout=None, memory=None, chroots=None, background=False,

+                               module=None, progress_callback=None):

          """ Creates new build from PyPI

  

              :param projectname: name of Copr project (without user namespace)
@@ -437,6 +440,7 @@ 

              :param memory: [optional] amount of required memory for build process

              :param chroots: [optional] build only with given chroots

              :param background: [optional] mark the build as a background job.

+             :param module: [optional] str NSV of a module

              :param progress_callback: [optional] a function that received a

              MultipartEncoderMonitor instance for each chunck of uploaded data

  
@@ -454,10 +458,11 @@ 

          }

          api_endpoint = "new_build_tito"

          return self.process_creating_new_build(projectname, data, api_endpoint, username,

-                                                chroots, background=background)

+                                                chroots, background=background, module=module)

  

      def create_new_build_mock(self, projectname, scm_url, spec, scm_type="git", scm_branch=None, username=None,

-                               timeout=None, memory=None, chroots=None, background=False, progress_callback=None):

+                               timeout=None, memory=None, chroots=None, background=False,

+                               module=None, progress_callback=None):

          """ Creates new build from PyPI

  

              :param projectname: name of Copr project (without user namespace)
@@ -470,6 +475,7 @@ 

              :param memory: [optional] amount of required memory for build process

              :param chroots: [optional] build only with given chroots

              :param background: [optional] mark the build as a background job.

+             :param module: [optional] str NSV of a module

              :param progress_callback: [optional] a function that received a

              MultipartEncoderMonitor instance for each chunck of uploaded data

  
@@ -487,10 +493,11 @@ 

          }

          api_endpoint = "new_build_mock"

          return self.process_creating_new_build(projectname, data, api_endpoint, username,

-                                                chroots, background=background)

+                                                chroots, background=background, module=module)

  

      def create_new_build_rubygems(self, projectname, gem_name, username=None,

-                               timeout=None, memory=None, chroots=None, background=False, progress_callback=None):

+                                   timeout=None, memory=None, chroots=None, background=False,

+                                   module=None, progress_callback=None):

          """ Creates new build from RubyGems.org

  

              :param projectname: name of Copr project (without user namespace)
@@ -500,6 +507,7 @@ 

              :param memory: [optional] amount of required memory for build process

              :param chroots: [optional] build only with given chroots

              :param background: [optional] mark the build as a background job.

+             :param module: [optional] str NSV of a module

              :param progress_callback: [optional] a function that received a

              MultipartEncoderMonitor instance for each chunck of uploaded data

  
@@ -514,10 +522,11 @@ 

          }

          api_endpoint = "new_build_rubygems"

          return self.process_creating_new_build(projectname, data, api_endpoint, username,

-                                                chroots, background=background)

+                                                chroots, background=background, module=module)

  

      def create_new_build_distgit(self, projectname, clone_url, branch=None, username=None,

-                               timeout=None, memory=None, chroots=None, background=False, progress_callback=None):

+                                  timeout=None, memory=None, chroots=None, background=False,

+                                  module=None, progress_callback=None):

          """ Creates new build from a dist-git repository

  

              :param projectname: name of Copr project (without user namespace)
@@ -528,6 +537,7 @@ 

              :param memory: [optional] amount of required memory for build process

              :param chroots: [optional] build only with given chroots

              :param background: [optional] mark the build as a background job.

+             :param module: [optional] str NSV of a module

              :param progress_callback: [optional] a function that received a

              MultipartEncoderMonitor instance for each chunck of uploaded data

  
@@ -543,10 +553,10 @@ 

          }

          api_endpoint = "new_build_distgit"

          return self.process_creating_new_build(projectname, data, api_endpoint, username,

-                                                chroots, background=background)

+                                                chroots, background=background, module=module)

  

      def process_creating_new_build(self, projectname, data, api_endpoint, username=None, chroots=None,

-                                    background=False, progress_callback=None, multipart=False):

+                                    background=False, module=None, progress_callback=None, multipart=False):

          if not username:

              username = self.username

          data["username"] = username
@@ -558,6 +568,9 @@ 

          if background:

              data["background"] = "y"

  

+         if module:

+             data["module"] = module

+ 

          for chroot in chroots or []:

              data[chroot] = "y"

  

I've added DB relationship between builds and modules in a sense that build can belong to module and some module can has a list of modules.

There are several reasons for this

  1. It might be useful information in the future
  2. But mainly, there is currently no way how to mark module as failed when some of it builds fails. This PR is imho nice way how to fix it.

I've not tested this on fe-dev machine yet.

It might need an little PR for MBS after resolving this. It might be something like

diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py
index 2efb8aa..f671503 100644
--- a/module_build_service/builder/CoprModuleBuilder.py
+++ b/module_build_service/builder/CoprModuleBuilder.py
@@ -253,7 +253,9 @@ class CoprModuleBuilder(GenericBuilder):
             raise RuntimeError("Buildroot is not prep-ed")

         # Build package from `source`
-        response = self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username)
+        nsv = "-".join([self.module.name, self.module.stream, self.module.version])
+        response = self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username,
+                                                module=nsv)
         if response.output != "ok":
             log.error(response.error)

4 new commits added

  • [frontend] mark module as failed when it has failed builds
  • [frontend][python] allow to create builds with relations to modules
  • [frontend] allow to create builds with relation to modules
  • [frontend] add relation between builds and modules
6 years ago

This is obsoleted by PR#186, closing ...

Pull-Request has been closed by frostyx

6 years ago