#1624 osbuild: add support for building ostree artifacts
Merged 2 years ago by lsedlar. Opened 2 years ago by obudai.
obudai/pungi osbuild-ostree  into  master

file modified
+5
@@ -1610,6 +1610,11 @@ 

      * ``arches`` -- list of architectures for which to build the image. By

        default, the variant arches are used. This option can only restrict it,

        not add a new one.

+     * ``ostree_url`` -- URL of the repository that's used to fetch the parent

+       commit from.

+     * ``ostree_ref`` -- name of the ostree branch

+     * ``ostree_parent`` -- commit hash or a a branch-like reference to the

+       parent commit.

  

  .. note::

     There is initial support for having this task as failable without aborting

file modified
+3
@@ -1175,6 +1175,9 @@ 

                                  "repo": {"$ref": "#/definitions/list_of_strings"},

                                  "failable": {"$ref": "#/definitions/list_of_strings"},

                                  "subvariant": {"type": "string"},

+                                 "ostree_url": {"type": "string"},

+                                 "ostree_ref": {"type": "string"},

+                                 "ostree_parent": {"type": "string"},

                              },

                              "required": ["name", "distro", "image_types"],

                              "additionalProperties": False,

file modified
+11
@@ -113,8 +113,19 @@ 

          koji = kojiwrapper.KojiWrapper(compose)

          koji.login()

  

+         ostree = {}

+         if config.get("ostree_url"):

+             ostree["url"] = config["ostree_url"]

+         if config.get("ostree_ref"):

+             ostree["ref"] = config["ostree_ref"]

+         if config.get("ostree_parent"):

+             ostree["parent"] = config["ostree_parent"]

+ 

          # Start task

          opts = {"repo": repo}

+         if ostree:

+             opts["ostree"] = ostree

+ 

          if release:

              opts["release"] = release

          task_id = koji.koji_proxy.osbuildImage(

file modified
+133 -1
@@ -178,7 +178,6 @@ 

  

          # Verify two Koji instances were created.

          self.assertEqual(len(KojiWrapper.call_args), 2)

-         print(koji.mock_calls)

          # Verify correct calls to Koji

          self.assertEqual(

              koji.mock_calls,
@@ -252,6 +251,139 @@ 

      @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024)

      @mock.patch("pungi.phases.osbuild.Linker")

      @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")

+     def test_process_ostree(self, KojiWrapper, Linker):

+         cfg = {

+             "name": "test-image",

+             "distro": "rhel-8",

+             "image_types": ["edge-raw-disk"],

+             "ostree_url": "http://edge.example.com/repo",

+             "ostree_ref": "test/iot",

+             "ostree_parent": "test/iot-parent",

+         }

+         build_id = 5678

+         koji = KojiWrapper.return_value

+         koji.watch_task.side_effect = self.make_fake_watch(0)

+         koji.koji_proxy.osbuildImage.return_value = 1234

+         koji.koji_proxy.getTaskResult.return_value = {

+             "composer": {"server": "https://composer.osbuild.org", "id": ""},

+             "koji": {"build": build_id},

+         }

+         koji.koji_proxy.getBuild.return_value = {

+             "build_id": build_id,

+             "name": "test-image",

+             "version": "1",

+             "release": "1",

+         }

+         koji.koji_proxy.listArchives.return_value = [

+             {

+                 "extra": {"image": {"arch": "aarch64"}},

+                 "filename": "image.aarch64.raw.xz",

+                 "type_name": "raw-xz",

+             },

+             {

+                 "extra": {"image": {"arch": "x86_64"}},

+                 "filename": "image.x86_64.raw.xz",

+                 "type_name": "raw-xz",

+             },

+         ]

+         koji.koji_module.pathinfo = orig_koji.pathinfo

+ 

+         self.t.process(

+             (

+                 self.compose,

+                 self.compose.variants["Everything"],

+                 cfg,

+                 ["aarch64", "x86_64"],

+                 "1",  # version

+                 "15",  # release

+                 "image-target",

+                 [self.topdir + "/compose/Everything/$arch/os"],

+                 ["x86_64"],

+             ),

+             1,

+         )

+ 

+         # Verify two Koji instances were created.

+         self.assertEqual(len(KojiWrapper.call_args), 2)

+         # Verify correct calls to Koji

+         self.assertEqual(

+             koji.mock_calls,

+             [

+                 mock.call.login(),

+                 mock.call.koji_proxy.osbuildImage(

+                     "test-image",

+                     "1",

+                     "rhel-8",

+                     ["edge-raw-disk"],

+                     "image-target",

+                     ["aarch64", "x86_64"],

+                     opts={

+                         "release": "15",

+                         "repo": [self.topdir + "/compose/Everything/$arch/os"],

+                         "ostree": {

+                             "url": "http://edge.example.com/repo",

+                             "ref": "test/iot",

+                             "parent": "test/iot-parent",

+                         },

+                     },

+                 ),

+                 mock.call.save_task_id(1234),

+                 mock.call.watch_task(1234, mock.ANY),

+                 mock.call.koji_proxy.getTaskResult(1234),

+                 mock.call.koji_proxy.getBuild(build_id),

+                 mock.call.koji_proxy.listArchives(buildID=build_id),

+             ],

+         )

+ 

+         # Assert there are 2 images added to manifest and the arguments are sane

+         self.assertEqual(

+             self.compose.im.add.call_args_list,

+             [

+                 mock.call(arch="aarch64", variant="Everything", image=mock.ANY),

+                 mock.call(arch="x86_64", variant="Everything", image=mock.ANY),

+             ],

+         )

+         for call in self.compose.im.add.call_args_list:

+             _, kwargs = call

+             image = kwargs["image"]

+             self.assertEqual(kwargs["variant"], "Everything")

+             self.assertIn(kwargs["arch"], ("aarch64", "x86_64"))

+             self.assertEqual(kwargs["arch"], image.arch)

+             self.assertEqual(

+                 "Everything/%(arch)s/images/image.%(arch)s.raw.xz"

+                 % {"arch": image.arch},

+                 image.path,

+             )

+             self.assertEqual("raw.xz", image.format)

+             self.assertEqual("raw-xz", image.type)

+             self.assertEqual("Everything", image.subvariant)

+ 

+         self.assertTrue(

+             os.path.isdir(self.topdir + "/compose/Everything/aarch64/images")

+         )

+         self.assertTrue(

+             os.path.isdir(self.topdir + "/compose/Everything/x86_64/images")

+         )

+ 

+         self.assertEqual(

+             Linker.return_value.mock_calls,

+             [

+                 mock.call.link(

+                     "/mnt/koji/packages/test-image/1/1/images/image.%(arch)s.raw.xz"

+                     % {"arch": arch},

+                     self.topdir

+                     + "/compose/Everything/%(arch)s/images/image.%(arch)s.raw.xz"

+                     % {"arch": arch},

+                     link_type="hardlink-or-copy",

+                 )

+                 for arch in ["aarch64", "x86_64"]

+             ],

+         )

+ 

+     @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536)

+     @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024)

+     @mock.patch("pungi.phases.osbuild.Linker")

+     @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")

      def test_process_without_release(self, KojiWrapper, Linker):

          cfg = {"name": "test-image", "distro": "rhel-8", "image_types": ["qcow2"]}

          build_id = 5678

This is currently a draft, I would firstly like feedback from @pwhalen and @gicmo. Also, I'm not sure about code duplication in the test suite, test_process, test_process_ostree and test_process_without_release are basically the same thing. Happy to hear your suggestions.


In order to start building Fedora IoT images with osbuild, we need to be able
to pass ostree options from pungi to the koji's osbuildImage task.

This commit adds support for it via new configuration options: ostree_url,
ostree_url and ostree_parent.

A test was added to cover these new options and they are were also added
into the documentation.

JIRA: COMPOSER-1702

rebased onto 866a8409e7d1967d162d519d76dee81b2df8d487

2 years ago

FWIW this looks good to me

Jenkins is failing on this line being too long. Black suggests this change:

-                "Everything/%(arch)s/images/image.%(arch)s.raw.xz" % {"arch": image.arch},
+                "Everything/%(arch)s/images/image.%(arch)s.raw.xz"
+                % {"arch": image.arch},

Where is the new ostree commit created? The parent specified by ostree_parent is pulled from ostree_url. Will OSBuild manage the new commit in a separate location?

Generally the patch looks good to me. Once ready to be merged, please remove the WIP from the title.

rebased onto df246421d26be9af31dd71defbf466bc5f3d7db6

2 years ago

rebased onto d0ace66

2 years ago

I believe that this is now ready to be merged.

Where is the new ostree commit created? The parent specified by ostree_parent is pulled from ostree_url. Will OSBuild manage the new commit in a separate location?

Generally the patch looks good to me. Once ready to be merged, please remove the WIP from the title.

This PR is needed for switching Fedora IoT to build some of their artifacts using osbuild, see this PR: https://pagure.io/fedora-iot/pungi-iot/pull-request/4

As you can see in the linked PR, building the ostree commits and managing the ostree repo is currently out of scope. Other tooling will still be responsible for those. Osbuild will just use the results to build a raw disk and an installer out of them.

We are planning to also build the ostree commits using osbuild but that's something for the future.

We are planning to also build the ostree commits using osbuild but that's something for the future.

Yes, that needs the ability to push the new trees from osbuild to a repo. We decided to delay that for the time being and keep using the existing way of generating the ostrees

Ping on this, can we get it reviewed and merged?

The code looks good to me. I have not tested it, but I don't see how it could break something.
I'll rebase it to latest master and merge.

Commit 7797933 fixes this pull-request

Pull-Request has been merged by lsedlar

2 years ago

Any chance we could get a tagged release/build with this in it for IoT?

New release is now tagged and RPMs are pending push to F36/F37 updates-testing.