#442 Add scratch_build_tasks to generate composes including scratch builds.
Merged 3 years ago by lsedlar. Opened 3 years ago by jkaluza.
jkaluza/odcs scratch-builds  into  master

file modified
+5
@@ -154,6 +154,11 @@ 

      .. note::

          If ``target_dir`` is set to non-default value, then the ``result_repofile`` might be an empty string, because ODCS might not have enough data to generate the URL.

  

+ .. _scratch_build_tasks:

+ 

+ *scratch_build_tasks* - ``(white-space separated list of strings)``

+     List of Koji task IDs of RPM build scratch builds which will be included in the compose.

+ 

  .. _scratch_modules:

  

  *scratch_modules* - ``(white-space separated list of strings)``

file modified
+8
@@ -78,6 +78,14 @@ 

  }

  {%- endif %}

  

+ {%- if config.scratch_build_tasks %}

+ pkgset_koji_scratch_tasks = [

+ {%- for task_id in config.scratch_build_tasks %}

+         '{{ task_id }}',

+ {%- endfor %}

+     ]

+ {%- endif %}

+ 

  

  {%- if config.source_type_str in ["tag", "build"] and not config.packages %}

  # In case no package is requested, include all of them.

@@ -628,6 +628,24 @@ 

              )

              continue

  

+         scratch_build_tasks = (

+             set(compose.scratch_build_tasks.split(" "))

+             if compose.scratch_build_tasks

+             else set()

+         )

+         old_scratch_build_tasks = (

+             set(old_compose.scratch_build_tasks.split(" "))

+             if old_compose.scratch_build_tasks

+             else set()

+         )

+         if scratch_build_tasks != old_scratch_build_tasks:

+             log.debug(

+                 "%r: Cannot reuse %r - scratch_build_tasks not same",

+                 compose,

+                 old_compose,

+             )

+             continue

+ 

          parent_pungi_compose_ids = (

              set(compose.parent_pungi_compose_ids.split(" "))

              if compose.parent_pungi_compose_ids
@@ -945,6 +963,7 @@ 

                  modular_koji_tags=compose.modular_koji_tags,

                  module_defaults_url=compose.module_defaults_url,

                  scratch_modules=compose.scratch_modules,

+                 scratch_build_tasks=compose.scratch_build_tasks,

              )

              if compose.flags & COMPOSE_FLAGS["no_deps"]:

                  pungi_cfg.gather_method = "nodeps"

@@ -0,0 +1,24 @@ 

+ """Add scratch_build_tasks field.

+ 

+ Revision ID: ca08065687c4

+ Revises: 7b143656694f

+ Create Date: 2020-09-03 11:20:54.035095

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = "ca08065687c4"

+ down_revision = "7b143656694f"

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     op.add_column(

+         "composes", sa.Column("scratch_build_tasks", sa.String(), nullable=True)

+     )

+ 

+ 

+ def downgrade():

+     op.drop_column("composes", "scratch_build_tasks")

@@ -166,6 +166,8 @@ 

      _target_dir = db.Column("target_dir", db.String, nullable=True)

      # White-space separated list of scratch modules (N:S:V:C) to include in a compose.

      scratch_modules = db.Column(db.String, nullable=True)

+     # White-space separated list of RPM scratch builds to include in a compose.

+     scratch_build_tasks = db.Column(db.String, nullable=True)

      # White-space separated list of parent Pungi compose IDs (pungi_compose_id).

      parent_pungi_compose_ids = db.Column(db.String, nullable=True)

  
@@ -215,6 +217,7 @@ 

          target_dir=None,

          scratch_modules=None,

          parent_pungi_compose_ids=None,

+         scratch_build_tasks=None,

      ):

          now = datetime.utcnow()

          compose = cls(
@@ -241,6 +244,7 @@ 

              target_dir=target_dir or conf.target_dir,

              scratch_modules=scratch_modules,

              parent_pungi_compose_ids=parent_pungi_compose_ids,

+             scratch_build_tasks=scratch_build_tasks,

          )

          session.add(compose)

          return compose
@@ -288,6 +292,7 @@ 

              target_dir=compose.target_dir,

              scratch_modules=compose.scratch_modules,

              parent_pungi_compose_ids=compose.parent_pungi_compose_ids,

+             scratch_build_tasks=compose.scratch_build_tasks,

          )

          session.add(compose)

          return compose
@@ -424,6 +429,7 @@ 

              "target_dir": target_dir,

              "scratch_modules": self.scratch_modules,

              "parent_pungi_compose_ids": self.parent_pungi_compose_ids,

+             "scratch_build_tasks": self.scratch_build_tasks,

          }

  

          if full:

@@ -229,6 +229,7 @@ 

          modular_koji_tags=None,

          module_defaults_url=None,

          scratch_modules=None,

+         scratch_build_tasks=None,

      ):

          super(PungiConfig, self).__init__()

          self.release_name = release_name
@@ -247,6 +248,9 @@ 

          self.packages = packages or []

          self.builds = builds or []

          self.scratch_modules = scratch_modules.split(" ") if scratch_modules else []

+         self.scratch_build_tasks = (

+             scratch_build_tasks.split(" ") if scratch_build_tasks else []

+         )

  

          # Store results as list of strings, so it can be used by jinja2

          # templates.

@@ -282,6 +282,7 @@ 

          :jsonparam list source["koji_event"]: Number defining the :ref:`koji_event<koji_event>`.

          :jsonparam list source["modular_koji_tags"]: List defining the :ref:`modular_koji_tags<modular_koji_tags>`.

          :jsonparam list source["scratch_modules"]: List defining the :ref:`scratch_modules<scratch_modules>`.

+         :jsonparam list source["scratch_build_tasks"]: List defining the :ref:`scratch_build_tasks<scratch_build_tasks>`.

  

          :statuscode 200: Compose request created and returned.

          :statuscode 401: Request not in valid format.
@@ -451,6 +452,10 @@ 

          if "scratch_modules" in source_data:

              scratch_modules = " ".join(source_data["scratch_modules"])

  

+         scratch_build_tasks = None

+         if "scratch_build_tasks" in source_data:

+             scratch_build_tasks = " ".join(source_data["scratch_build_tasks"])

+ 

          label = data.get("label", None)

          compose_type = data.get("compose_type", "test")

  
@@ -496,6 +501,7 @@ 

              target_dir=target_dir,

              scratch_modules=scratch_modules,

              parent_pungi_compose_ids=parent_pungi_compose_ids,

+             scratch_build_tasks=scratch_build_tasks,

          )

          db.session.add(compose)

          # Flush is needed, because we use `before_commit` SQLAlchemy event to

@@ -93,6 +93,7 @@ 

              "toplevel_url": "http://localhost/odcs/odcs-1",

              "scratch_modules": None,

              "parent_pungi_compose_ids": None,

+             "scratch_build_tasks": None,

          }

          self.assertEqual(c.json(True), expected_json)

  

@@ -242,6 +242,25 @@ 

                  cfg["additional_packages"], [(u"^Temporary$", {u"*": [u"*"]})]

              )

  

+     def test_get_pungi_conf_scratch_build_tasks(self):

+         _, mock_path = tempfile.mkstemp()

+         template_path = os.path.abspath(os.path.join(test_dir, "../conf/pungi.conf"))

+         shutil.copy2(template_path, mock_path)

+ 

+         with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path):

+             pungi_cfg = PungiConfig(

+                 "MBS-512",

+                 "1",

+                 PungiSourceType.KOJI_TAG,

+                 "f26",

+                 scratch_build_tasks="123456 123457",

+             )

+ 

+             cfg = pungi_cfg.get_pungi_config()

+             self.assertEqual(

+                 set(cfg["pkgset_koji_scratch_tasks"]), set(["123456", "123457"])

+             )

+ 

      def test_get_pungi_conf_modular_koji_tags(self):

          _, mock_path = tempfile.mkstemp()

          template_path = os.path.abspath(os.path.join(test_dir, "../conf/pungi.conf"))

@@ -364,6 +364,7 @@ 

              "scratch_modules": None,

              "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"],

              "parent_pungi_compose_ids": None,

+             "scratch_build_tasks": None,

          }

          self.assertEqual(data, expected_json)

  
@@ -1247,6 +1248,33 @@ 

              "User unknown not allowed to operate with compose with source_types=tag.",

          )

  

+     def test_submit_scratch_build_tasks(self):

+         with self.test_request_context(user="dev"):

+             flask.g.oidc_scopes = [

+                 "{0}{1}".format(conf.oidc_base_namespace, "new-compose")

+             ]

+ 

+             rv = self.client.post(

+                 "/api/1/composes/",

+                 data=json.dumps(

+                     {

+                         "source": {

+                             "type": "module",

+                             "source": "testmodule:master",

+                             "scratch_build_tasks": [

+                                 "123456",

+                                 "123457",

+                             ],

+                         }

+                     }

+                 ),

+             )

+             data = json.loads(rv.get_data(as_text=True))

+             self.assertEqual(

+                 data["scratch_build_tasks"],

+                 "123456 123457",

+             )

+ 

      def test_submit_build_scratch_modules(self):

          with self.test_request_context(user="dev"):

              flask.g.oidc_scopes = [
@@ -1763,6 +1791,7 @@ 

              "scratch_modules": None,

              "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"],

              "parent_pungi_compose_ids": None,

+             "scratch_build_tasks": None,

          }

          self.assertEqual(data, expected_json)