#455 Support local build of OSTree and OSTree installer
Closed 7 years ago by qwan. Opened 7 years ago by qwan.
qwan/pungi ostree-local-build  into  master

file modified
+4 -3
@@ -1068,8 +1068,8 @@ 

  OSTree Settings

  ===============

  

- The ``ostree`` phase of *Pungi* can create ostree repositories in a Koji

- runroot environment.

+ The ``ostree`` phase of *Pungi* can create ostree repositories. If runroot is

+ enabled, it will run in Koji as a ``runroot`` task, otherwise it will run locally.

  

  **ostree**

      (*dict*) -- a variant/arch mapping of configuration. The format should be
@@ -1115,7 +1115,8 @@ 

  =========================

  

  The ``ostree_installer`` phase of *Pungi* can produce installer image bundling

- an OSTree repository. This always runs in Koji as a ``runroot`` task.

+ an OSTree repository. If runroot is enabled, it will run in Koji as a ``runroot``

+ task, otherwise it will run locally.

  

  **ostree_installer**

      (*dict*) -- a variant/arch mapping of configuration. The format should be

file modified
+36 -19
@@ -2,8 +2,10 @@ 

  

  import json

  import os

+ from kobo.shortcuts import run

  from kobo.threads import ThreadPool, WorkerThread

  import re

+ import rpmUtils.arch

  

  from .base import ConfigGuardedPhase

  from .. import util
@@ -58,7 +60,16 @@ 

          # mount it.

          util.makedirs(config['ostree_repo'])

  

-         self._run_ostree_cmd(compose, variant, arch, config, repodir)

+         runroot = compose.conf.get("runroot")

+         if not runroot:

+             # check whether arch matches host arch

+             host_arch = rpmUtils.arch.getBaseArch()

+             if arch != host_arch:

+                 self.pool.log_warning(

+                     '[SKIP ] OSTree for arch %s, variant %s. Host arch is %s.'

+                     % (arch, variant, host_arch))

+                 return

+         self._run_ostree_cmd(compose, variant, arch, config, repodir, runroot=runroot)

          ref, commitid = self._get_commit_info(config, repodir)

          if config.get('tag_ref', True) and ref and commitid:

              # Let's write the tag out ourselves
@@ -98,10 +109,10 @@ 

              return None, None

          return ref, commitid

  

-     def _run_ostree_cmd(self, compose, variant, arch, config, config_repo):

+     def _run_ostree_cmd(self, compose, variant, arch, config, config_repo, runroot=True):

          cmd = [

              'pungi-make-ostree',

-             '--log-dir=%s' % os.path.join(self.logdir),

+             '--log-dir=%s' % self.logdir,

              '--treefile=%s' % os.path.join(config_repo, config['treefile']),

          ]

  
@@ -115,22 +126,28 @@ 

          # positional argument: ostree_repo

          cmd.append(config['ostree_repo'])

  

-         runroot_channel = compose.conf.get("runroot_channel")

-         runroot_tag = compose.conf["runroot_tag"]

- 

-         packages = ['pungi', 'ostree', 'rpm-ostree']

-         log_file = os.path.join(self.logdir, 'runroot.log')

-         mounts = [compose.topdir, config['ostree_repo']]

-         koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

-         koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

-                                         channel=runroot_channel,

-                                         use_shell=True, task_id=True,

-                                         packages=packages, mounts=mounts,

-                                         new_chroot=True)

-         output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

-         if output["retcode"] != 0:

-             raise RuntimeError("Runroot task failed: %s. See %s for more details."

-                                % (output["task_id"], log_file))

+         if runroot:

+             # run in a koji build root

+             runroot_channel = compose.conf.get("runroot_channel")

+             runroot_tag = compose.conf["runroot_tag"]

+ 

+             packages = ['pungi', 'ostree', 'rpm-ostree']

+             log_file = os.path.join(self.logdir, 'runroot.log')

+             mounts = [compose.topdir, config['ostree_repo']]

+             koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

+             koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

+                                             channel=runroot_channel,

+                                             use_shell=True, task_id=True,

+                                             packages=packages, mounts=mounts,

+                                             new_chroot=True)

+             output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

+             if output["retcode"] != 0:

+                 raise RuntimeError("Runroot task failed: %s. See %s for more details."

+                                    % (output["task_id"], log_file))

+         else:

+             # run locally

+             log_file = os.path.join(self.logdir, 'pungi-make-ostree.log')

+             run(cmd, show_cmd=True, logfile=log_file)

  

      def _clone_repo(self, repodir, url, branch):

          scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},

@@ -6,6 +6,7 @@ 

  from productmd import images

  import pipes

  from kobo import shortcuts

+ import rpmUtils.arch

  

  from .base import ConfigGuardedPhase

  from .. import util
@@ -54,7 +55,18 @@ 

          disc_type = compose.conf['disc_types'].get('ostree', 'ostree')

  

          volid = get_volid(compose, arch, variant, disc_type=disc_type)

-         self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir, volid)

+ 

+         runroot = compose.conf.get("runroot")

+         if not runroot:

+             # check whether arch matches host arch

+             host_arch = rpmUtils.arch.getBaseArch()

+             if arch != host_arch:

+                 self.pool.log_warning(

+                     '[SKIP ] OSTree installer for arch %s, variant %s. Host arch is %s.'

+                     % (arch, variant, host_arch))

+                 return

+ 

+         self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir, volid, runroot=runroot)

  

          filename = compose.get_image_name(arch, variant, disc_type=disc_type)

          self._copy_image(compose, variant, arch, filename, output_dir)
@@ -136,7 +148,7 @@ 

              templates.append(template)

          return templates

  

-     def _run_ostree_cmd(self, compose, variant, arch, config, source_repo, output_dir, volid):

+     def _run_ostree_cmd(self, compose, variant, arch, config, source_repo, output_dir, volid, runroot=True):

          lorax_wrapper = lorax.LoraxWrapper()

          cmd = lorax_wrapper.get_lorax_cmd(

              compose.conf['release_name'],
@@ -155,17 +167,21 @@ 

              is_final=compose.supported,

          )

  

-         runroot_channel = compose.conf.get("runroot_channel")

-         runroot_tag = compose.conf["runroot_tag"]

- 

-         packages = ['pungi', 'lorax', 'ostree']

-         log_file = os.path.join(self.logdir, 'runroot.log')

-         koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

-         koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

-                                         channel=runroot_channel,

-                                         use_shell=True, task_id=True,

-                                         packages=packages, mounts=[compose.topdir])

-         output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

-         if output["retcode"] != 0:

-             raise RuntimeError("Runroot task failed: %s. See %s for more details."

-                                % (output["task_id"], log_file))

+         if runroot:

+             runroot_channel = compose.conf.get("runroot_channel")

+             runroot_tag = compose.conf["runroot_tag"]

+ 

+             packages = ['pungi', 'lorax', 'ostree']

+             log_file = os.path.join(self.logdir, 'runroot.log')

+             koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

+             koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

+                                             channel=runroot_channel,

+                                             use_shell=True, task_id=True,

+                                             packages=packages, mounts=[compose.topdir])

+             output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

+             if output["retcode"] != 0:

+                 raise RuntimeError("Runroot task failed: %s. See %s for more details."

+                                    % (output["task_id"], log_file))

+         else:

+             # run locally

+             shortcuts.run(cmd)

@@ -51,6 +51,7 @@ 

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': True,

              'runroot_tag': 'rrt',

              'image_volid_formats': ['{release_short}-{variant}-{arch}'],

          })
@@ -114,8 +115,8 @@ 

      @mock.patch('pungi.phases.ostree_installer.iso')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run(self, KojiWrapper, link, iso,

-                  get_file_size, get_mtime, ImageCls, run):

+     def test_run_in_runroot(self, KojiWrapper, link, iso,

+                             get_file_size, get_mtime, ImageCls, run):

          self.compose.supported = False

          pool = mock.Mock()

          cfg = {
@@ -150,8 +151,8 @@ 

      @mock.patch('pungi.phases.ostree_installer.iso')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_external_source(self, KojiWrapper, link, iso,

-                                  get_file_size, get_mtime, ImageCls, run):

+     def test_run_in_runroot_with_external_source(self, KojiWrapper, link, iso,

+                                                  get_file_size, get_mtime, ImageCls, run):

          pool = mock.Mock()

          cfg = {

              'source_repo_from': 'http://example.com/repo/$arch/',
@@ -183,9 +184,9 @@ 

      @mock.patch('pungi.wrappers.iso')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_fail_with_relative_template_path_but_no_repo(self, KojiWrapper, link,

-                                                           iso, get_file_size,

-                                                           get_mtime, ImageCls, run):

+     def test_fail_in_runroot_with_relative_template_path_but_no_repo(self, KojiWrapper, link,

+                                                                      iso, get_file_size,

+                                                                      get_mtime, ImageCls, run):

          pool = mock.Mock()

          cfg = {

              'source_repo_from': 'Everything',
@@ -216,9 +217,9 @@ 

      @mock.patch('pungi.phases.ostree_installer.iso')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_clone_templates(self, KojiWrapper, link, iso,

-                                  get_file_size, get_mtime, ImageCls, run,

-                                  get_dir_from_scm):

+     def test_run_in_runroot_clone_templates(self, KojiWrapper, link, iso,

+                                             get_file_size, get_mtime, ImageCls, run,

+                                             get_dir_from_scm):

          pool = mock.Mock()

          cfg = {

              'source_repo_from': 'Everything',
@@ -264,8 +265,8 @@ 

      @mock.patch('pungi.phases.ostree_installer.iso')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_with_implicit_release(self, KojiWrapper, link, iso,

-                                        get_file_size, get_mtime, ImageCls, run):

+     def test_run_in_runroot_with_implicit_release(self, KojiWrapper, link, iso,

+                                                   get_file_size, get_mtime, ImageCls, run):

          pool = mock.Mock()

          cfg = {

              'source_repo_from': 'Everything',
@@ -322,8 +323,8 @@ 

      @mock.patch('pungi.phases.ostree_installer.iso')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_fail_crash(self, KojiWrapper, link, iso, get_file_size,

-                         get_mtime, ImageCls, run):

+     def test_fail_in_runroot_crash(self, KojiWrapper, link, iso, get_file_size,

+                                    get_mtime, ImageCls, run):

          pool = mock.Mock()

          cfg = {

              'source_repo_from': 'Everything',
@@ -372,6 +373,63 @@ 

                        % self.topdir)

          ])

  

+     @mock.patch('rpmUtils.arch.getBaseArch')

+     @mock.patch('kobo.shortcuts.run')

+     @mock.patch('productmd.images.Image')

+     @mock.patch('pungi.util.get_mtime')

+     @mock.patch('pungi.util.get_file_size')

+     @mock.patch('pungi.phases.ostree_installer.iso')

+     @mock.patch('os.link')

+     def test_run_locally(self, link, iso, get_file_size,

+                          get_mtime, ImageCls, run, get_base_arch):

+         get_base_arch.return_value = 'x86_64'

+         # runroot not enabled in config

+         compose = helpers.DummyCompose(self.topdir, {

+             'release_name': 'Fedora',

+             'release_version': 'Rawhide',

+             'koji_profile': 'koji',

+         })

+         pool = mock.Mock()

+         cfg = {

+             'source_repo_from': 'Everything',

+             'release': '20160321.n.0',

+         }

+ 

+         t = ostree.OstreeInstallerThread(pool)

+ 

+         t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)

+         self.assertEqual(run.call_args_list,

+                          [mock.call(['lorax', '--product=Fedora', '--version=Rawhide', '--release=20160321.n.0',

+                                      '--source=file://{0}/compose/Everything/x86_64/os'.format(self.topdir),

+                                      '--variant=Everything', '--nomacboot',

+                                      '--isfinal', '--volid=test-Rawhide Everything.x86_64',

+                                      '{0}/work/x86_64/Everything/ostree_installer'.format(self.topdir)]),

+                           mock.call('cp -av {0}/work/x86_64/Everything/ostree_installer/* {0}/compose/Everything/x86_64/os/'.format(self.topdir))])

+ 

+     @mock.patch('rpmUtils.arch.getBaseArch')

+     @mock.patch('kobo.shortcuts.run')

+     def test_run_locally_skip_not_match_arch(self, run, get_base_arch):

+         get_base_arch.return_value = 'x86_64'

+         # runroot not enabled in config

+         compose = helpers.DummyCompose(self.topdir, {

+             'release_name': 'Fedora',

+             'release_version': 'Rawhide',

+             'koji_profile': 'koji',

+         })

+         pool = mock.Mock()

+         cfg = {

+             'source_repo_from': 'Everything',

+             'release': '20160321.n.0',

+         }

+ 

+         t = ostree.OstreeInstallerThread(pool)

+ 

+         variant = compose.variants['Everything']

+         t.process((compose, variant, 'ppc64', cfg), 1)

+         pool.log_warning.assert_has_calls([

+             mock.call('[SKIP ] OSTree installer for arch ppc64, variant %s. Host arch is x86_64.' % variant)

+         ])

+         self.assertFalse(run.called)

  

  if __name__ == '__main__':

      unittest.main()

file modified
+54 -3
@@ -59,6 +59,7 @@ 

          }

          self.compose = helpers.DummyCompose(self.topdir, {

              'koji_profile': 'koji',

+             'runroot': True,

              'runroot_tag': 'rrt',

              'translate_paths': [

                  (self.topdir + '/compose', 'http://example.com')
@@ -92,7 +93,7 @@ 

  

      @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run(self, KojiWrapper, get_dir_from_scm):

+     def test_run_in_runroot(self, KojiWrapper, get_dir_from_scm):

          get_dir_from_scm.side_effect = self._dummy_config_repo

  

          koji = KojiWrapper.return_value
@@ -128,7 +129,7 @@ 

  

      @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_fail(self, KojiWrapper, get_dir_from_scm):

+     def test_run_in_runroot_fail(self, KojiWrapper, get_dir_from_scm):

          get_dir_from_scm.side_effect = self._dummy_config_repo

  

          self.cfg['failable'] = ['*']
@@ -147,7 +148,7 @@ 

  

      @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_handle_exception(self, KojiWrapper, get_dir_from_scm):

+     def test_run_in_runroot_handle_exception(self, KojiWrapper, get_dir_from_scm):

          get_dir_from_scm.side_effect = self._dummy_config_repo

  

          self.cfg['failable'] = ['*']
@@ -164,6 +165,56 @@ 

          ])

  

      @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

+     @mock.patch('pungi.phases.ostree.run')

+     def test_run_locally(self, run, get_dir_from_scm):

+         get_dir_from_scm.side_effect = self._dummy_config_repo

+ 

+         compose = helpers.DummyCompose(self.topdir, {

+             'koji_profile': 'koji',

+             'translate_paths': [

+                 (self.topdir + '/compose', 'http://example.com')

+             ]

+         })

+         pool = mock.Mock()

+ 

+         t = ostree.OSTreeThread(pool)

+ 

+         t.process((compose, compose.variants['Everything'], 'x86_64', self.cfg), 1)

+         cmd = ['pungi-make-ostree',

+                '--log-dir=%s' % (self.topdir + '/logs/x86_64/Everything/ostree-1'),

+                '--treefile=%s' % (self.topdir + '/work/ostree-1/config_repo/' + self.cfg['treefile']),

+                self.cfg['ostree_repo']]

+ 

+         self.assertEqual(

+             run.call_args_list,

+             [mock.call(cmd, show_cmd=True,

+              logfile=self.topdir + '/logs/x86_64/Everything/ostree-1/pungi-make-ostree.log')])

+ 

+     @mock.patch('rpmUtils.arch.getBaseArch')

+     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

+     @mock.patch('pungi.phases.ostree.run')

+     def test_run_locally_skip_not_match_arch(self, run, get_dir_from_scm, get_base_arch):

+         get_dir_from_scm.side_effect = self._dummy_config_repo

+         get_base_arch.return_value = 'x86_64'

+ 

+         compose = helpers.DummyCompose(self.topdir, {

+             'koji_profile': 'koji',

+             'translate_paths': [

+                 (self.topdir + '/compose', 'http://example.com')

+             ]

+         })

+         pool = mock.Mock()

+ 

+         t = ostree.OSTreeThread(pool)

+ 

+         variant = compose.variants['Everything']

+         t.process((compose, variant, 'ppc64', self.cfg), 1)

+         pool.log_warning.assert_has_calls([

+             mock.call('[SKIP ] OSTree for arch ppc64, variant %s. Host arch is x86_64.' % variant)

+         ])

+         self.assertFalse(run.called)

+ 

+     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

      def test_run_send_message(self, KojiWrapper, get_dir_from_scm):

          get_dir_from_scm.side_effect = self._dummy_config_repo

no initial comment

For testing with fake rpms, I'm still investigating on it, and I think we can add that in a separate patch.

There's no checks to see if the architecture that is being requested to be built is the same as the architecture on the local host where it would be built.

Why is there hard wired versions in the test phase? It should pull that from a test config with a default not be hard wired into the code.

Why is there hard wired versions in the test phase? It should pull that from a test config with a default not be hard wired into the code.

Hi @pbrobinson:
Which versions do you mean?

This should be split out into a test config rather than be included inline in the code.

The architecture shouldn't be hard coded, it should be taken from the test config and in the case of a local OSTree build there should be checks the config aligns with the local arch and appropriate arch checks to ensure they match.

Arch specific checks, it might not be run on x86_64 and the config might not be for x86_64.

This needs to check it's running the right lorax command for the local architecture.

Hi @pbrobinson:
Which versions do you mean?

I put comments in line to make it explicit.

rebased

7 years ago

Patches improved per @pbrobinson 's comments.

Added arch checking for local build to skip unmatched arch. Arch check testing is covered in a new test case.

For the configs in test_ostree_installer_phase.py, most of them have different values, so have each test case to maintain its own config can be much more convenient. The one in test_ostree_phase.py has been updated.

One comment I'm unsure about is:

commented on line 107 of tests/test_ostree_installer_phase.py.
This needs to check it's running the right lorax command for the local architecture.

Is the lorax command diff from arches? I don't have ppc64 and aarch64 systems to check the exact lorax commands.

commented on line 107 of tests/test_ostree_installer_phase.py.
This needs to check it's running the right lorax command for the local architecture.

Is the lorax command diff from arches? I don't have ppc64 and aarch64 systems to check the exact lorax commands.

Yes, for example the "--nomacboot" used in the command I commented on is x86 specific. You might want to use anaconda templates (which is I think how pungi deals with it) for the different architectures.

pungi already deals with the different architectures so you should be able to look at how the rest of the code handles this and adjust as appropriate.

commented on line 107 of tests/test_ostree_installer_phase.py.
This needs to check it's running the right lorax command for the local architecture.
Is the lorax command diff from arches? I don't have ppc64 and aarch64 systems to check the exact lorax commands.

Yes, for example the "--nomacboot" used in the command I commented on is x86 specific. You might want to use anaconda templates (which is I think how pungi deals with it) for the different architectures.

--nomacboot is not x86 specific, it just happens that it does nothing on other arches


[root@athosian ~]# uname -m
aarch64
[root@athosian ~]# lorax --help|grep mac
             [--buildarch ARCH] [--volid VOLID] [--macboot] [--nomacboot]
  --macboot
  --nomacboot

pungi-make-ostree has been tested on armhfp, aarch64 and x86_64 I would honestly suggest that you just call it

pungi-make-ostree has been tested on armhfp, aarch64 and x86_64 I would honestly suggest that you just call it

lorax is used in ostree_installer phase here, which is not supported by pungi-make-ostree.

pungi-make-ostree has been tested on armhfp, aarch64 and x86_64 I would honestly suggest that you just call it

lorax is used in ostree_installer phase here, which is not supported by pungi-make-ostree.

right. I was just talking about the ostree repo part.

honestly I want to close this as invalid.

We do not provide functionality in pungi-koji to make install DVD's I suggest that you extend the pungi command to be able to make the ostree repo and installer locally, or we introduce a new program for that purpose

We do not provide functionality in pungi-koji to make install DVD's I suggest that you extend the pungi command to be able to make the ostree repo and installer locally, or we introduce a new program for that purpose

sorry, I'm a little confused, isn't ostree_installer phase provide that functionality of building installer images? What I'm trying to add is extending the ostree and ostree_installer phase to be able to run locally, which can be helpful for testing and in some cases for easy use while composing ostree stuff by manual with special configurations.

Yes, for example the "--nomacboot" used in the command I commented on is x86 specific. You might want to use anaconda templates (which is I think how pungi deals with it) for the different architectures.

pungi already deals with the different architectures so you should be able to look at how the rest of the code handles this and adjust as appropriate.

Hi @pbrobinson, The lorax command is constructed with lorax.LoraxWrapper's get_lorax_cmd which is used in runroot case too, so in theory, it should work just same as in the koji runroot env since runroot also cares about the arch. My understanding is the lorax command is same as the one in runroot with the same arch, so if runroot with x86_64 arch can works well, how could the local x86_64 breaks? The difference with runroot is runroot can have multi arches and our local build only take the arch which is same as host.

Could you point me the code of handling the multiple arches while constructing a lorax (or other stuff) command in pungi?

@pbrobinson @ausil I submitted #PR479 to add the support of building OSTree tree/installer locally with pungi-make-ostree instead of adding it to the phase chain, could you help to give a review?

And dropping this PR.

Pull-Request has been closed by qwan

7 years ago