#431 Add `parent_pungi_compose_ids` as pass it to pungi-koji using --parent-compose-id.
Merged 3 years ago by lsedlar. Opened 3 years ago by jkaluza.
jkaluza/odcs parent-compose-ids  into  master

file modified
+5
@@ -123,6 +123,11 @@ 

  *packages* - ``(white-space separated list of strings or null)``

      List of names of RPMs (packages) which should appear in the compose. The list of packages to choose from is defined by the content of Koji builds defined in ``builds``. If ``null``, all packages from ``builds`` will be included in a compose.

  

+ .. _parent_pungi_compose_ids:

+ 

+ *parent_pungi_compose_ids* - ``(list of strings)``

+     Pungi compose IDs of parent composes.

+ 

  .. _pungi_config_dump:

  

  *pungi_config_dump* - ``(string)``

file modified
+12 -10
@@ -193,21 +193,23 @@ 

  

  

  Regenerating expired ``raw_config`` compose

- ---------------------------------------

+ -------------------------------------------

  

  When regenerating expired ``raw_config`` compose using the HTTP REST API,

- there is a difference between the composes with ``production`` :ref:`compose_type`

- and other :ref:`compose_types<compose_type>`.

+ there is a difference between the composes with ``production``

+ :ref:`compose_type<compose_type>` and other :ref:`compose_types<compose_type>`.

  

- For ``production`` :ref:`compose_type`, ODCS stores the :ref:`pungi_config_dump`

- and regeneration such compose will result in the very same configuration file

- to be used. The regenerated compose should therefore be identical to original compose.

+ For ``production`` :ref:`compose_type<compose_type>`, ODCS stores

+ the :ref:`pungi_config_dump<pungi_config_dump>` and regeneration such compose

+ will result in the very same configuration file to be used. The regenerated

+ compose should therefore be identical to original compose.

  

  For other :ref:`compose_types<compose_type>`, the generated compose only

- uses the same the same :ref:`source` and :ref:`koji_event` to point to original

- Raw config configuration and Koji tags. Depending on the Raw config compose configuration,

- this can mean that the resulting compose is not 100% identical with the original one. The

- input packages should always be identical thanks to the same koji_event, but the configuration

+ uses the same the same :ref:`source<source>` and :ref:`koji_event<koji_event>`

+ to point to original Raw config configuration and Koji tags. Depending on

+ the Raw config compose configuration, this can mean that the resulting

+ compose is not 100% identical with the original one. The input packages

+ should always be identical thanks to the same koji_event, but the configuration

  can differ for example in case when it points to other external git repositories

  and their commits using branch name. This branch name can be resolved to very different 

  commit hash when such compose is regenerated in the future.

@@ -623,6 +623,24 @@ 

              )

              continue

  

+         parent_pungi_compose_ids = (

+             set(compose.parent_pungi_compose_ids.split(" "))

+             if compose.parent_pungi_compose_ids

+             else set()

+         )

+         old_parent_pungi_compose_ids = (

+             set(old_compose.parent_pungi_compose_ids.split(" "))

+             if old_compose.parent_pungi_compose_ids

+             else set()

+         )

+         if parent_pungi_compose_ids != old_parent_pungi_compose_ids:

+             log.debug(

+                 "%r: Cannot reuse %r - parent_pungi_compose_ids not same",

+                 compose,

+                 old_compose,

+             )

+             continue

+ 

          target_dir = compose.target_dir

          old_target_dir = old_compose.target_dir

          if target_dir != old_target_dir:

@@ -165,6 +165,8 @@ 

      _target_dir = db.Column("target_dir", db.String, nullable=True)

      # White-space separated list of scratch modules (N:S:V:C) to include in a compose.

      scratch_modules = db.Column(db.String, nullable=True)

+     # White-space separated list of parent Pungi compose IDs (pungi_compose_id).

+     parent_pungi_compose_ids = db.Column(db.String, nullable=True)

  

      @property

      def on_default_target_dir(self):
@@ -211,6 +213,7 @@ 

          compose_type=None,

          target_dir=None,

          scratch_modules=None,

+         parent_pungi_compose_ids=None,

      ):

          now = datetime.utcnow()

          compose = cls(
@@ -236,6 +239,7 @@ 

              compose_type=compose_type,

              target_dir=target_dir or conf.target_dir,

              scratch_modules=scratch_modules,

+             parent_pungi_compose_ids=parent_pungi_compose_ids,

          )

          session.add(compose)

          return compose
@@ -282,6 +286,7 @@ 

              celery_task_id=None,

              target_dir=compose.target_dir,

              scratch_modules=compose.scratch_modules,

+             parent_pungi_compose_ids=compose.parent_pungi_compose_ids,

          )

          session.add(compose)

          return compose
@@ -413,6 +418,7 @@ 

              "pungi_compose_id": self.pungi_compose_id,

              "target_dir": target_dir,

              "scratch_modules": self.scratch_modules,

+             "parent_pungi_compose_ids": self.parent_pungi_compose_ids,

          }

  

          if full:

@@ -497,6 +497,9 @@ 

  

          if compose.label:

              pungi_cmd.append("--label=%s" % str(compose.label))

+         if compose.parent_pungi_compose_ids:

+             for compose_id in compose.parent_pungi_compose_ids.split(" "):

+                 pungi_cmd.append("--parent-compose-id=%s" % str(compose_id))

  

          if self.koji_event:

              pungi_cmd += ["--koji-event", str(self.koji_event)]

@@ -270,6 +270,9 @@ 

          :jsonparam string label: String defining the :ref:`label<label>`.

          :jsonparam string compose_type: String defining the :ref:`compose_type<compose_type>`.

          :jsonparam string target_dir: String defining the :ref:`target_dir<target_dir>`.

+         :jsonparam list parent_pungi_compose_ids: Pungi compose IDs of parent composes associated

+             with this compose. They will be stored with this compose in the Compose Tracking

+             Service.

          :jsonparam object source: The JSON object defining the source of compose.

          :jsonparam string source["type"]: String defining the :ref:`source type<source_type>`.

          :jsonparam string source["source"]: String defining the :ref:`source<source>`.
@@ -411,6 +414,10 @@ 

          if "lookaside_repos" in data:

              lookaside_repos = " ".join(data["lookaside_repos"])

  

+         parent_pungi_compose_ids = None

+         if "parent_pungi_compose_ids" in data:

+             parent_pungi_compose_ids = " ".join(data["parent_pungi_compose_ids"])

+ 

          multilib_method = MULTILIB_METHODS["none"]

          if "multilib_method" in data:

              for name in data["multilib_method"]:
@@ -488,6 +495,7 @@ 

              compose_type=compose_type,

              target_dir=target_dir,

              scratch_modules=scratch_modules,

+             parent_pungi_compose_ids=parent_pungi_compose_ids,

          )

          db.session.add(compose)

          # Flush is needed, because we use `before_commit` SQLAlchemy event to

@@ -92,6 +92,7 @@ 

              "target_dir": "default",

              "toplevel_url": "http://localhost/odcs/odcs-1",

              "scratch_modules": None,

+             "parent_pungi_compose_ids": None,

          }

          self.assertEqual(c.json(True), expected_json)

  

@@ -452,6 +452,37 @@ 

          )

  

      @patch("odcs.server.utils.execute_cmd")

+     def test_pungi_run_parent_pungi_compose_ids(self, execute_cmd):

+         self.compose.parent_pungi_compose_ids = "Fedora-1-1 Fedora-2-2"

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.MODULE, "testmodule:master:1:1"

+         )

+         pungi = Pungi(1, pungi_cfg)

+         pungi.run(self.compose)

+ 

+         self.makedirs.assert_called_with(AnyStringWith("test_composes/odcs-1/"))

+         self.makedirs.assert_called_with(AnyStringWith("work/global"))

+         self.ci_dump.assert_called_once_with(

+             AnyStringWith("work/global/composeinfo-base.json")

+         )

+ 

+         execute_cmd.assert_called_once_with(

+             [

+                 "pungi-koji",

+                 AnyStringWith("pungi.json"),

+                 "--no-latest-link",

+                 AnyStringWith("--compose-dir="),

+                 "--test",

+                 "--parent-compose-id=Fedora-1-1",

+                 "--parent-compose-id=Fedora-2-2",

+             ],

+             cwd=AnyStringWith("/tmp/"),

+             timeout=3600,

+             stderr=AnyStringWith("pungi-stderr.log"),

+             stdout=AnyStringWith("pungi-stdout.log"),

+         )

+ 

+     @patch("odcs.server.utils.execute_cmd")

      @patch("odcs.server.pungi.PyConfigParser")

      def test_pungi_run_cts(self, py_config_parser, execute_cmd):

          self.patch_ci_dump.stop()

@@ -363,6 +363,7 @@ 

              "target_dir": "default",

              "scratch_modules": None,

              "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"],

+             "parent_pungi_compose_ids": None,

          }

          self.assertEqual(data, expected_json)

  
@@ -1273,6 +1274,26 @@ 

                  "foo:bar:20200806:abcdefgh fooo:bar:20200810:abcdefgh",

              )

  

+     def test_submit_build_parent_pungi_compose_ids(self):

+         with self.test_request_context(user="dev"):

+             flask.g.oidc_scopes = [

+                 "{0}{1}".format(conf.oidc_base_namespace, "new-compose")

+             ]

+ 

+             rv = self.client.post(

+                 "/api/1/composes/",

+                 data=json.dumps(

+                     {

+                         "parent_pungi_compose_ids": ["Fedora-1-1", "Fedora-2-2"],

+                         "source": {"type": "module", "source": "testmodule:master"},

+                     }

+                 ),

+             )

+             data = json.loads(rv.get_data(as_text=True))

+             self.assertEqual(

+                 data["parent_pungi_compose_ids"], "Fedora-1-1 Fedora-2-2",

+             )

+ 

      def test_query_compose(self):

          resp = self.client.get("/api/1/composes/1")

          data = json.loads(resp.get_data(as_text=True))
@@ -1740,6 +1761,7 @@ 

              "target_dir": "default",

              "scratch_modules": None,

              "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"],

+             "parent_pungi_compose_ids": None,

          }

          self.assertEqual(data, expected_json)