#1722 frontend: allow disabling of appstream metadata
Opened 2 months ago by schlupov. Modified 4 days ago
copr/ schlupov/copr disabling_of_appstream_metadata  into  main

@@ -114,6 +114,7 @@ 

          projectname = data["projectname"]

          project_dirnames = data["project_dirnames"]

          chroots = data["chroots"]

+         appstream = data["appstream"]

  

          result = ActionResult.SUCCESS

  
@@ -130,7 +131,7 @@ 

                  except FileExistsError:

                      pass

  

-                 if not call_copr_repo(repo, logger=self.log):

+                 if not call_copr_repo(repo, appstream=appstream, logger=self.log):

                      result = ActionResult.FAILURE

  

          return result
@@ -157,6 +158,7 @@ 

          old_path = os.path.join(self.destdir, self.data["old_value"])

          new_path = os.path.join(self.destdir, self.data["new_value"])

          builds_map = json.loads(self.data["data"])["builds_map"]

+         appstream = data["appstream"]

  

          if not os.path.exists(old_path):

              self.log.info("Source copr directory doesn't exist: %s", old_path)
@@ -211,7 +213,7 @@ 

  

              result = ActionResult.SUCCESS

              for chroot_path in chroot_paths:

-                 if not call_copr_repo(chroot_path, logger=self.log):

+                 if not call_copr_repo(chroot_path, appstream=appstream, logger=self.log):

                      result = ActionResult.FAILURE

  

          except (CoprSignError, CreateRepoError, CoprRequestException, IOError) as ex:
@@ -228,7 +230,7 @@ 

      """

      # pylint: disable=abstract-method

      def _handle_delete_builds(self, ownername, projectname, project_dirname,

-                               chroot_builddirs, build_ids):

+                               chroot_builddirs, build_ids, appstream):

          """ call /bin/copr-repo --delete """

          devel = uses_devel_repo(self.front_url, ownername, projectname)

          result = ActionResult.SUCCESS
@@ -247,7 +249,7 @@ 

              # repodata temporarily pointing at non-existing files)!

              if chroot != "srpm-builds":

                  # In srpm-builds we don't create repodata at all

-                 if not call_copr_repo(chroot_path, delete=subdirs, devel=devel,

+                 if not call_copr_repo(chroot_path, delete=subdirs, devel=devel, appstream=appstream,

                                        logger=self.log):

                      result = ActionResult.FAILURE

  
@@ -341,13 +343,14 @@ 

          projectname = ext_data["projectname"]

          project_dirnames = ext_data["project_dirnames"]

          build_ids = ext_data["build_ids"]

+         appstream = ext_data["appstream"]

  

          result = ActionResult.SUCCESS

          for project_dirname, chroot_builddirs in project_dirnames.items():

              if ActionResult.FAILURE == \

                 self._handle_delete_builds(ownername, projectname,

                                            project_dirname, chroot_builddirs,

-                                           build_ids):

+                                           build_ids, appstream):

                  result = ActionResult.FAILURE

          return result

  
@@ -371,13 +374,14 @@ 

              projectname = ext_data["projectname"]

              project_dirname = ext_data["project_dirname"]

              chroot_builddirs = ext_data["chroot_builddirs"]

+             appstream = ext_data["appstream"]

          except KeyError:

              self.log.exception("Invalid action data")

              return ActionResult.FAILURE

  

          return self._handle_delete_builds(ownername, projectname,

                                            project_dirname, chroot_builddirs,

-                                           build_ids)

+                                           build_ids, appstream)

  

  

  class DeleteChroot(Delete):
@@ -414,6 +418,7 @@ 

  class RawhideToRelease(Action):

      def run(self):

          data = json.loads(self.data["data"])

+         appstream = data ["appstream"]

          result = ActionResult.SUCCESS

          try:

              chrootdir = os.path.join(self.opts.destdir, data["ownername"], data["projectname"], data["dest_chroot"])
@@ -432,7 +437,7 @@ 

                      with open(os.path.join(destdir, "build.info"), "a") as f:

                          f.write("\nfrom_chroot={}".format(data["rawhide_chroot"]))

  

-             if not call_copr_repo(chrootdir, logger=self.log):

+             if not call_copr_repo(chrootdir, appstream=appstream, logger=self.log):

                  result = ActionResult.FAILURE

          except:

              result = ActionResult.FAILURE
@@ -449,6 +454,7 @@ 

              projectname = data["projectname"]

              chroots = data["chroots"]

              project_path = os.path.join(self.opts.destdir, ownername, projectname)

+             appstream = data["appstream"]

  

              mmd_yaml = base64.b64decode(data["modulemd_b64"]).decode("utf-8")

              mmd_yaml = modulemd_tools.yaml.upgrade(mmd_yaml, 2)
@@ -491,7 +497,7 @@ 

                      mmd_yaml = modulemd_tools.yaml.update(mmd_yaml, rpms_nevras=artifacts)

                      self.log.info("Module artifacts: %s", artifacts)

                      modulemd_tools.yaml.dump(mmd_yaml, destdir)

-                     if not call_copr_repo(destdir, logger=self.log):

+                     if not call_copr_repo(destdir, appstream=appstream, logger=self.log):

                          result = ActionResult.FAILURE

  

          except Exception:

@@ -596,6 +596,7 @@ 

          project_owner = self.job.project_owner

          project_name = self.job.project_name

          devel = self.job.uses_devel_repo

+         appstream = self.job.appstream

  

          base_url = "/".join([self.opts.results_baseurl, project_owner,

                               project_name, self.job.chroot])
@@ -605,7 +606,8 @@ 

                        base_url, not devel)

          if not call_copr_repo(self.job.chroot_dir, devel=devel,

                                add=[self.job.target_dir_name],

-                               logger=self.log):

+                               logger=self.log,

+                               appstream=appstream):

              raise BackendError("createrepo failed")

  

      def _get_srpm_build_details(self, job):

@@ -616,7 +616,7 @@ 

  

  

  def call_copr_repo(directory, rpms_to_remove=None, devel=False, add=None, delete=None, timeout=None,

-                    logger=None):

+                    logger=None, appstream=True):

      """

      Execute 'copr-repo' tool, and return True if the command succeeded.

      """
@@ -635,6 +635,8 @@ 

      cmd += opt_multiply('--add', add)

      cmd += opt_multiply('--delete', delete)

      cmd += opt_multiply('--rpms-to-remove', rpms_to_remove)

+     if not appstream:

+         cmd += ['--no-appstream-metadata']

      if devel:

          cmd += ['--devel']

  

@@ -66,6 +66,8 @@ 

          self.uses_devel_repo = None

          self.sandbox = None

  

+         self.appstream = None

+ 

          # TODO: validate update data, user marshmallow

          for key, val in task_data.items():

              key = str(key)
@@ -77,6 +79,9 @@ 

          if str(self.task_id) == str(self.build_id):

              self.chroot = 'srpm-builds'

  

+         if task_data["appstream"]:

+             self.appstream = task_data["appstream"]

+ 

          self.destdir = os.path.normpath(os.path.join(

              worker_opts.destdir,

              task_data["project_owner"],

file modified
+4 -5
@@ -217,13 +217,12 @@ 

  

  

  def add_appdata(opts):

-     if opts.devel or not opts.appstream:

-         opts.log.info("appstream-builder skipped, /devel subdir or "

-                       "--no-appstream-metadata specified")

+     if opts.devel:

+         opts.log.info("appstream-builder skipped, /devel subdir")

          return

  

-     if os.path.exists(os.path.join(opts.projectdir, ".disable-appstream")):

-         opts.log.info("appstream-builder skipped, .disable-appstream file")

+     if not opts.appstream:

+         opts.log.info("appstream-builder skipped")

          return

  

      path = opts.directory

file modified
+10 -1
@@ -39,6 +39,7 @@ 

              "src_pkg_name": self.pkgs_stripped[0],

              "ownername": "foo",

              "projectname": "bar",

+             "appstream": True,

              "project_dirname": "bar",

              "chroot_builddirs": {

                  "fedora20": ["00001-foo"],
@@ -158,7 +159,8 @@ 

                                  '00000002-pkg1': '00000009-pkg1', '00000005-pkg2': '00000010-pkg2'}

                          },

                          "user": "thrnciar",

-                         "copr": "source-copr"

+                         "copr": "source-copr",

+                         "appstream": True,

                      }),

                  "old_value": "thrnciar/source-copr",

                  "new_value": "thrnciar/destination-copr",
@@ -307,6 +309,7 @@ 

                  "data": json.dumps({

                      "ownername": "foo",

                      "project_dirnames": ["bar", "baz"],

+                     "appstream": True,

                  }),

              },

          )
@@ -458,6 +461,7 @@ 

                      "ownername": "@copr",

                      "projectname": "prunerepo",

                      "project_dirname": "prunerepo",

+                     "appstream": True,

                      "chroot_builddirs": {

                          "fedora-23-x86_64": [builddir],

                      },
@@ -553,6 +557,7 @@ 

                  "data": json.dumps({

                      "ownername": "foo",

                      "projectname": "bar",

+                     "appstream": True,

                      "project_dirname": "bar",

                      "chroot_builddirs": {

                          "fedora-20-x86_64": ["rubygem-log4r-1.1.10-2.fc21"],
@@ -622,6 +627,7 @@ 

                  "data": json.dumps({

                      "ownername": "foo",

                      "projectname": "bar",

+                     "appstream": True,

                      "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"],

                      "project_dirname": "bar",

                      "chroot_builddirs": {
@@ -709,6 +715,7 @@ 

          ext_data = json.dumps({

              "ownername": "foo",

              "projectname": "bar",

+             "appstream": True,

              "project_dirnames": {

                  'bar': {

                      "fedora-20": ["01-foo", "02-foo"],
@@ -752,6 +759,7 @@ 

              "chroots": ["epel-6-i386", "fedora-20-x86_64"],

              "ownername": "foo",

              "projectname": "bar",

+             "appstream": True,

              "project_dirnames": ["bar"]

          })

          self.opts.destdir = tmp_dir
@@ -784,6 +792,7 @@ 

              "chroots": ["epel-6-i386", "fedora-20-x86_64"],

              "ownername": "foo",

              "projectname": "bar",

+             "appstream": True,

              "project_dirnames": ["bar"]

          })

          self.opts.destdir = tmp_dir

@@ -68,7 +68,7 @@ 

          self.request_createrepo.get(some_dir)

          # appstream=False has no effect, others beat it, appstream=False

          # makes it non-matching.

-         self.request_createrepo.get(some_dir, {"add": ["add_2"], "appstream": False})

+         self.request_createrepo.get(some_dir, {"add": ["add_2"], "appstream": True})

          self.request_createrepo.get(some_dir, {"add": [], "delete": ["del_1"]})

          self.request_createrepo.get(some_dir, {"add": [], "delete": ["del_2"]})

          assert not bcr.check_processed()

@@ -74,7 +74,8 @@ 

      "use_bootstrap_container": False,

      "uses_devel_repo": False,

      "with_opts": [],

-     "without_opts": []

+     "without_opts": [],

+     "appstream": True,

  }

  

  
@@ -95,7 +96,8 @@ 

      }),

      "source_type": 8,

      "submitter": "praiskup",

-     "task_id": "855954"

+     "task_id": "855954",

+     "appstream": True,

  }

  

  

@@ -0,0 +1,22 @@ 

+ """

+ add column appstream

+ 

+ Revision ID: 2318cc31444e

+ Revises: d8a1062ee4cf

+ Create Date: 2021-04-28 14:07:25.439393

+ """

+ 

+ import sqlalchemy as sa

+ from alembic import op

+ 

+ 

+ revision = '2318cc31444e'

+ down_revision = 'd8a1062ee4cf'

+ 

+ 

+ def upgrade():

+     op.add_column('copr', sa.Column('appstream', sa.Boolean(), server_default='1', nullable=False))

+ 

+ 

+ def downgrade():

+     op.drop_column('copr', 'appstream')

@@ -597,6 +597,12 @@ 

              Failing fedora-review will not fail the build itself.""",

              default=False, false_values=FALSE_VALUES)

  

+     appstream = wtforms.BooleanField(

+         "Generate appstream metadata",

+         description="""AppStream consists of a specification to describe individual

+                     software component metadata in XML. Disable or enable generating the metadata.""",

+         default=True, false_values=FALSE_VALUES)

+ 

      @property

      def errors(self):

          """

@@ -107,6 +107,7 @@ 

              "projectname": copr.name,

              "project_dirnames": dirnames,

              "chroots": chroots,

+             "appstream": copr.appstream,

          }

          action = models.Action(

              action_type=ActionTypeEnum("createrepo"),
@@ -123,6 +124,7 @@ 

          data_dict = {

              "ownername": copr.owner_name,

              "project_dirnames": [copr_dir.name for copr_dir in copr.dirs],

+             "appstream": copr.appstream,

          }

          action = models.Action(action_type=ActionTypeEnum("delete"),

                                 object_type="copr",
@@ -165,7 +167,8 @@ 

              "projectname": build.copr_name,

              "project_dirname":

                  build.copr_dirname if build.copr_dir else build.copr_name,

-             "chroot_builddirs": cls.get_chroot_builddirs(build)

+             "chroot_builddirs": cls.get_chroot_builddirs(build),

+             "appstream": build.appstream,

          }

  

      @classmethod
@@ -279,6 +282,7 @@ 

          data_dict = {

              "ownername": copr.owner_name,

              "projectname": copr.name,

+             "appstream": copr.appstream,

          }

  

          action = models.Action(
@@ -314,7 +318,8 @@ 

              object_type="copr",

              old_value="{0}".format(src.full_name),

              new_value="{0}".format(dst.full_name),

-             data=json.dumps({"user": dst.owner_name, "copr": dst.name, "builds_map": builds_map}),

+             data=json.dumps(

+                 {"user": dst.owner_name, "copr": dst.name, "builds_map": builds_map, "appstream": dst.appstream}),

              created_on=int(time.time()),

          )

          db.session.add(action)

@@ -219,7 +219,7 @@ 

      def add(cls, user, name, selected_chroots, repos=None, description=None,

              instructions=None, check_for_duplicates=False, group=None, persistent=False,

              auto_prune=True, bootstrap=None, follow_fedora_branching=False, isolation=None,

-             **kwargs):

+             appstream=True, **kwargs):

  

          if not flask.g.user.admin and flask.g.user != user:

              msg = ("You were authorized as '{0}' user without permissions to access "
@@ -246,6 +246,7 @@ 

                             bootstrap=bootstrap,

                             isolation=isolation,

                             follow_fedora_branching=follow_fedora_branching,

+                            appstream=appstream,

                             **kwargs)

  

  

@@ -337,6 +337,8 @@ 

      # optional tools to run after build

      fedora_review = db.Column(db.Boolean, default=False, nullable=False, server_default="0")

  

+     appstream = db.Column(db.Boolean, default=True, nullable=False, server_default="1")

+ 

  

  class _CoprPrivate(db.Model, helpers.Serializer):

      """
@@ -1464,6 +1466,11 @@ 

          """

          return [ch for ch in self.chroots if ch in self.copr.active_chroots]

  

+     @property

+     def appstream(self):

+         """Whether appstream metadata should be generated for a build."""

+         return self.copr.appstream

+ 

  

  class DistGitBranch(db.Model, helpers.Serializer):

      """

@@ -148,6 +148,7 @@ 

          [form.multilib],

          [form.module_hotfixes],

          [form.fedora_review],

+         [form.appstream],

      ])}}

  

      {{ render_field(form.delete_after_days,

@@ -33,6 +33,7 @@ 

          "bootstrap": copr.bootstrap,

          "isolation": copr.isolation,

          "module_hotfixes": copr.module_hotfixes,

+         "appstream": copr.appstream,

      }

  

  
@@ -163,6 +164,7 @@ 

              fedora_review=form.fedora_review.data,

              follow_fedora_branching=form.follow_fedora_branching.data,

              runtime_dependencies=form.runtime_dependencies.data,

+             appstream=form.appstream.data,

          )

          db.session.commit()

      except (DuplicateException,

@@ -114,6 +114,7 @@ 

              "project_owner": task.build.copr.owner_name,

              "sandbox": task.build.sandbox,

              "background": bool(task.build.is_background),

+             "appstream": bool(task.build.appstream),

              "chroot": task.mock_chroot.name,

          }

  
@@ -185,6 +186,7 @@ 

              "sandbox": task.sandbox,

              "background": bool(task.is_background),

              "chroot": chroot,

+             "appstream": bool(task.copr.appstream),

          }

  

          if for_backend:

@@ -250,6 +250,7 @@ 

                  runtime_dependencies=form.runtime_dependencies.data.replace("\n", " "),

                  bootstrap=form.bootstrap.data,

                  isolation=form.isolation.data,

+                 appstream=form.appstream.data,

              )

  

              db.session.commit()
@@ -574,6 +575,7 @@ 

      copr.runtime_dependencies = form.runtime_dependencies.data.replace("\n", " ")

      copr.bootstrap = form.bootstrap.data

      copr.isolation = form.isolation.data

+     copr.appstream = form.appstream.data

      if flask.g.user.admin:

          copr.auto_prune = form.auto_prune.data

      else:

@@ -77,7 +77,7 @@ 

              "chroots": chroots,

          }

  

-         for config in ['bootstrap', 'isolation', 'contact', 'homepage']:

+         for config in ['bootstrap', 'isolation', 'contact', 'homepage', 'appstream']:

              if not config in kwargs:

                  continue

              data[config] = kwargs[config]
@@ -201,7 +201,7 @@ 

              "chroots": chroots,

          }

  

-         for config in ['bootstrap', 'isolation', 'contact', 'homepage']:

+         for config in ['bootstrap', 'isolation', 'contact', 'homepage', 'appstream']:

              if not config in kwargs:

                  continue

              data[config] = kwargs[config]

@@ -69,7 +69,8 @@ 

          self.api3.new_project("test", ["fedora-rawhide-i386"],

                                bootstrap="default", isolation="simple",

                                contact="somebody@redhat.com",

-                               homepage="https://github.com/fedora-copr")

+                               homepage="https://github.com/fedora-copr",

+                               appstream=True)

          old_data = self._get_copr_id_data(1)

  

          # When new arguments are added to the Copr model, we should update this
@@ -93,7 +94,7 @@ 

              "test", delete_after_days=5, enable_net=True, devel_mode=True,

              repos=["http://example/repo/", "http://another/"],

              runtime_dependencies=["http://run1/repo/", "http://run2/"],

-             bootstrap_image="noop",

+             bootstrap_image="noop", appstream=True,

          )

          new_data = self._get_copr_id_data(1)

          delete_after = datetime.datetime.now() + datetime.timedelta(days=5)
@@ -138,6 +139,8 @@ 

          }, {

              "follow_fedora_branching": True,

          }, {

+         }, {

+             "appstream": True,

          }]

  

          for setup in easy_changes:

@@ -165,14 +165,15 @@ 

          #    the only Batch where we need to iterate through Builds (as there's

          #    only one tree of batches).

          # 5. Read BuildChroots (lazy) from ^^ to get statuses.

-         # 6. Large query for BuildChroots (get_pending_build_tasks).

+         # 6. Get info whether to generate appstream metadata.

+         # 7. Large query for BuildChroots (get_pending_build_tasks).

          #

          # The last batch (ID=2+more_bchs) contains one "ready" BuildChroot task

          # (the srpm upload emulation, see _prepare_project_with_batches()) which

          # is only blocked by parent batch.  But because we cache Batch objects

          # in pending_jobs() method - they are preloaded and we can be sure that

          # we don't have to re-load the batch data to check if that is finished.

-         expected = 6

+         expected = 7

          if expected != len(dq):

              print()

              for n, query in enumerate(dq):
@@ -185,6 +186,7 @@ 

          # First batch is done, second is processing and third is blocked.  Only

          # the builds from second batch are present.

          assert data == [{

+             'appstream': False,

              'background': False,

              'build_id': 2,

              'chroot': None,
@@ -192,6 +194,7 @@ 

              'sandbox': 'user1/test--user1',

              'task_id': '2',

          }, {

+             'appstream': False,

              'background': False,

              'build_id': 4,

              'chroot': None,
@@ -300,13 +303,13 @@ 

          asserts = [

              sql_alchemy_time < fill_time/2,

              query_time < fill_time/20,

-             # - for each project two queries (for batch => one build +batch_build)

+             # - for each project three queries (for batch => one build +batch_build, for appstream metadata info)

              # - two large queries (srpm + rpms)

              # - one query for self.tc initialization

              # Note that we only lazily load first Build in each unblocked batch,

              # because even that first Build in batch is not yet finished -

              # meaning that the whole batch is not yet finished as well.

-             len(dq) == len(projects)*2 + 2 + 1,

+             len(dq) == len(projects)*3 + 2 + 1,

          ]

  

          if not all(asserts):

@@ -1066,7 +1066,8 @@ 

          template = {

              "ownername": "user1",

              "projectname": "test",

-             "project_dirnames": ["test"]

+             "project_dirnames": ["test"],

+             "appstream": True,

          }

          def _expected(action, chroots):

              template["chroots"] = chroots

@@ -73,7 +73,7 @@ 

              auto_prune=True, use_bootstrap_container=None, devel_mode=False,

              delete_after_days=None, multilib=False, module_hotfixes=False,

              bootstrap=None, bootstrap_image=None, isolation=None,

-             fedora_review=None):

+             fedora_review=None, appstream=True):

          """

          Create a project

  
@@ -104,6 +104,7 @@ 

              Possible values are 'default', 'simple', 'nspawn'.

          :param bool fedora_review: Run fedora-review tool for packages

                                     in this project

+         :param bool appstream: Disable or enable generating the appstream metadata

          :return: Munch

          """

          endpoint = "/project/add/{ownername}"
@@ -130,6 +131,7 @@ 

              "multilib": multilib,

              "module_hotfixes": module_hotfixes,

              "fedora_review": fedora_review,

+             "appstream": appstream,

          }

  

          _compat_use_bootstrap_container(data, use_bootstrap_container)
@@ -144,7 +146,7 @@ 

               auto_prune=None, use_bootstrap_container=None, devel_mode=None,

               delete_after_days=None, multilib=None, module_hotfixes=None,

               bootstrap=None, bootstrap_image=None, isolation=None,

-              fedora_review=None):

+              fedora_review=None, appstream=None):

          """

          Edit a project

  
@@ -174,6 +176,7 @@ 

              This is a noop parameter and its value is ignored.

          :param bool fedora_review: Run fedora-review tool for packages

                                     in this project

+         :param bool appstream: Disable or enable generating the appstream metadata

          :return: Munch

          """

          endpoint = "/project/edit/{ownername}/{projectname}"
@@ -199,6 +202,7 @@ 

              "multilib": multilib,

              "module_hotfixes": module_hotfixes,

              "fedora_review": fedora_review,

+             "appstream": appstream,

          }

  

          _compat_use_bootstrap_container(data, use_bootstrap_container)

Metadata Update from @schlupov:
- Pull-request tagged with: needs-work

2 months ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

rebased onto 11b8c32554c1484307029d322e5744bee46a1342

2 months ago

Metadata Update from @schlupov:
- Pull-request untagged with: needs-work

2 months ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

rebased onto 095857ed75a521d794eb69117e4c546b75a8c3db

a month ago

Merge Failed.

This change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.

rebased onto 4e2b0af

a month ago

Merge Failed.

This change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

Metadata Update from @schlupov:
- Pull-request tagged with: needs-work

a month ago

I'm thinking about this, and instead of passing additional argument here we should probably stop calling add_appdata when appstream is false.

Here is reversed logic, if not* appstream.

Here is probably the problem, the flag in DB is named disable_appstream_metadata while in our api it is just appstream. (missing negation)

missing description field here, should tell the user when the checkbox might be useful

rebased onto 8026529

4 days ago

Metadata Update from @schlupov:
- Pull-request untagged with: needs-work

4 days ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

rebased onto 0fc78f6

4 days ago

Build succeeded.

Metadata