#382 Build ostree and ostree-installer locally, and new options
Closed 7 years ago by lsedlar. Opened 7 years ago by qwan.
qwan/pungi ostree  into  master

file modified
+6 -1
@@ -1091,6 +1091,9 @@ 

        ``master``.

      * ``failable`` -- (*[str]*) List of architectures for which this

        deliverable is not release blocking.

+     * ``update_summary`` -- (*bool*) Update summary metadata after tree composing.

+       Defaults to ``False``.

+     * ``version`` -- (*str*) Version string to be added as versioning metadata.

  

  

  Example config
@@ -1103,7 +1106,9 @@ 

                  "treefile": "fedora-atomic-docker-host.json",

                  "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",

                  "source_repo_from": "Everything",

-                 "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/"

+                 "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/",

+                 "update_summary": True,

+                 "version": "24"

How is this option expected to be used? Should it only be set for production composes? For nightly composes the config can not be updated daily, so the version would not change much. Should the option be disabled in such case?

qwan commented 7 years ago

@lsedlar it doesn't matter, it's optional, by default, it doesn't add the versioning metadata if it's not present, and can be set by updating the config for both production and nightly composes if necessary.

This has been submit as a seperate PR: https://pagure.io/pungi/pull-request/454

              }

          })

      ]

file modified
+2
@@ -657,6 +657,8 @@ 

                      "source_repo_from": {"type": "string"},

                      "ostree_repo": {"type": "string"},

                      "failable": {"$ref": "#/definitions/list_of_strings"},

+                     "update_summary": {"type": "boolean"},

+                     "version": {"type": "string"},

                      "config_branch": {"type": "string"},

                  },

                  "required": ["treefile", "config_url", "source_repo_from", "ostree_repo"],

file modified
+19 -3
@@ -37,15 +37,27 @@ 

                        show_cmd=True, stdout=True, logfile=log_file)

  

  

- def make_ostree_repo(repo, config, log_dir=None):

+ def make_ostree_repo(repo, config, version=None, log_dir=None):

      log_file = make_log_file(log_dir, 'create-ostree-repo')

-     shortcuts.run(['rpm-ostree', 'compose', 'tree', '--repo=%s' % repo, config],

+     cmd = ['rpm-ostree', 'compose', 'tree', '--repo=%s' % repo]

+     if version:

+         # Add versioning metadata

+         cmd.append('--add-metadata-string=version=%s' % version)

+     cmd.append(config)

+     shortcuts.run(cmd, show_cmd=True, stdout=True, logfile=log_file)

+ 

+ 

+ def update_ostree_summary(repo, log_dir=None):

+     log_file = make_log_file(log_dir, 'ostree-summary')

+     shortcuts.run(['ostree', 'summary', '-u', '--repo=%s' % repo],

                    show_cmd=True, stdout=True, logfile=log_file)

  

  

  def run(opts):

      init_ostree_repo(opts.ostree_repo, log_dir=opts.log_dir)

-     make_ostree_repo(opts.ostree_repo, opts.treefile, log_dir=opts.log_dir)

+     make_ostree_repo(opts.ostree_repo, opts.treefile, version=opts.version, log_dir=opts.log_dir)

+     if opts.update_summary:

+         update_ostree_summary(opts.ostree_repo, log_dir=opts.log_dir)

  

  

  def main(args=None):
@@ -57,6 +69,10 @@ 

                          help='where to put the ostree repo')

      parser.add_argument('--treefile', required=True,

                          help='treefile for rpm-ostree')

+     parser.add_argument('--version',

+                         help='version string to be added as versioning metadata')

+     parser.add_argument('--update-summary', action='store_true',

+                         help='update summary metadata')

  

      opts = parser.parse_args(args)

  

file modified
+34 -17
@@ -1,6 +1,7 @@ 

  # -*- coding: utf-8 -*-

  

  import os

+ from kobo.shortcuts import run

  from kobo.threads import ThreadPool, WorkerThread

  import re

  
@@ -78,26 +79,42 @@ 

      def _run_ostree_cmd(self, compose, variant, arch, config, config_repo):

          cmd = [

              'pungi-make-ostree',

-             '--log-dir=%s' % os.path.join(self.logdir),

+             '--log-dir=%s' % self.logdir,

              '--treefile=%s' % os.path.join(config_repo, config['treefile']),

-             config['ostree_repo']

          ]

  

-         runroot_channel = compose.conf.get("runroot_channel")

-         runroot_tag = compose.conf["runroot_tag"]

- 

-         packages = ['pungi', 'ostree', 'rpm-ostree']

-         log_file = os.path.join(self.logdir, 'runroot.log')

-         mounts = [compose.topdir, config['ostree_repo']]

-         koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

-         koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

-                                         channel=runroot_channel,

-                                         use_shell=True, task_id=True,

-                                         packages=packages, mounts=mounts)

-         output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

-         if output["retcode"] != 0:

-             raise RuntimeError("Runroot task failed: %s. See %s for more details."

-                                % (output["task_id"], log_file))

+         version = config.get('version', None)

+         if version:

+             cmd.append('--version=%s' % version)

+ 

+         if config.get('update_summary', False):

+             cmd.append('--update-summary')

+ 

+         # positional argument: ostree_repo

+         cmd.append(config['ostree_repo'])

+ 

+         runroot = compose.conf.get("runroot")

+         if runroot:

+             # run in a koji build root

+             runroot_channel = compose.conf.get("runroot_channel")

+             runroot_tag = compose.conf["runroot_tag"]

+ 

+             packages = ['pungi', 'ostree', 'rpm-ostree']

+             log_file = os.path.join(self.logdir, 'runroot.log')

+             mounts = [compose.topdir, config['ostree_repo']]

+             koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

+             koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

+                                             channel=runroot_channel,

+                                             use_shell=True, task_id=True,

+                                             packages=packages, mounts=mounts)

+             output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

+             if output["retcode"] != 0:

+                 raise RuntimeError("Runroot task failed: %s. See %s for more details."

+                                    % (output["task_id"], log_file))

+         else:

+             # run locally

+             log_file = os.path.join(self.logdir, 'pungi-make-ostree.log')

+             run(cmd, show_cmd=True, logfile=log_file)

  

      def _clone_repo(self, repodir, url, branch):

          scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},

@@ -152,17 +152,22 @@ 

              add_arch_template_var=config.get('add_arch_template_var')

          )

  

-         runroot_channel = compose.conf.get("runroot_channel")

-         runroot_tag = compose.conf["runroot_tag"]

- 

-         packages = ['pungi', 'lorax', 'ostree']

-         log_file = os.path.join(self.logdir, 'runroot.log')

-         koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

-         koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

-                                         channel=runroot_channel,

-                                         use_shell=True, task_id=True,

-                                         packages=packages, mounts=[compose.topdir])

-         output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

-         if output["retcode"] != 0:

-             raise RuntimeError("Runroot task failed: %s. See %s for more details."

-                                % (output["task_id"], log_file))

+         runroot = compose.conf.get("runroot")

+         if runroot:

+             runroot_channel = compose.conf.get("runroot_channel")

+             runroot_tag = compose.conf["runroot_tag"]

+ 

+             packages = ['pungi', 'lorax', 'ostree']

+             log_file = os.path.join(self.logdir, 'runroot.log')

+             koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

+             koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,

+                                             channel=runroot_channel,

+                                             use_shell=True, task_id=True,

+                                             packages=packages, mounts=[compose.topdir])

+             output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)

+             if output["retcode"] != 0:

+                 raise RuntimeError("Runroot task failed: %s. See %s for more details."

+                                    % (output["task_id"], log_file))

+         else:

+             # run locally

+             shortcuts.run(cmd)

@@ -67,12 +67,13 @@ 

      @mock.patch('pungi.wrappers.iso.IsoWrapper')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run(self, KojiWrapper, link, IsoWrapper,

-                  get_file_size, get_mtime, ImageCls, run):

+     def test_run_in_runroot(self, KojiWrapper, link, IsoWrapper,

+                             get_file_size, get_mtime, ImageCls, run):

          compose = helpers.DummyCompose(self.topdir, {

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': 'True',

              'runroot_tag': 'rrt',

          })

          pool = mock.Mock()
@@ -129,13 +130,43 @@ 

      @mock.patch('pungi.util.get_file_size')

      @mock.patch('pungi.wrappers.iso.IsoWrapper')

      @mock.patch('os.link')

+     def test_run_locally(self, link, IsoWrapper,

+                          get_file_size, get_mtime, ImageCls, run):

+         compose = helpers.DummyCompose(self.topdir, {

+             'release_name': 'Fedora',

+             'release_version': 'Rawhide',

+             'koji_profile': 'koji',

+         })

+         pool = mock.Mock()

+         cfg = {

+             'source_repo_from': 'Everything',

+             'release': '20160321.n.0',

+         }

+ 

+         t = ostree.OstreeInstallerThread(pool)

+ 

+         t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)

+         self.assertEqual(run.call_args_list,

+                          [mock.call(['lorax', '--product=Fedora', '--version=Rawhide', '--release=20160321.n.0',

+                                      '--source=file://{0}/compose/Everything/x86_64/os'.format(self.topdir),

+                                      '--variant=Everything', '--nomacboot',

+                                      '{0}/work/x86_64/Everything/ostree_installer'.format(self.topdir)]),

+                           mock.call('cp -av {0}/work/x86_64/Everything/ostree_installer/* {0}/compose/Everything/x86_64/os/'.format(self.topdir))])

+ 

+     @mock.patch('kobo.shortcuts.run')

+     @mock.patch('productmd.images.Image')

+     @mock.patch('pungi.util.get_mtime')

+     @mock.patch('pungi.util.get_file_size')

+     @mock.patch('pungi.wrappers.iso.IsoWrapper')

+     @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_external_source(self, KojiWrapper, link, IsoWrapper,

-                                  get_file_size, get_mtime, ImageCls, run):

+     def test_run_in_runroot_with_external_source(self, KojiWrapper, link, IsoWrapper,

+                                                  get_file_size, get_mtime, ImageCls, run):

          compose = helpers.DummyCompose(self.topdir, {

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': 'True',

              'runroot_tag': 'rrt',

          })

          pool = mock.Mock()
@@ -193,13 +224,14 @@ 

      @mock.patch('pungi.wrappers.iso.IsoWrapper')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_fail_with_relative_template_path_but_no_repo(self, KojiWrapper, link,

-                                                           IsoWrapper, get_file_size,

-                                                           get_mtime, ImageCls, run):

+     def test_fail_in_runroot_with_relative_template_path_but_no_repo(self, KojiWrapper, link,

+                                                                      IsoWrapper, get_file_size,

+                                                                      get_mtime, ImageCls, run):

          compose = helpers.DummyCompose(self.topdir, {

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': 'True',

              'runroot_tag': 'rrt',

          })

          pool = mock.Mock()
@@ -232,13 +264,14 @@ 

      @mock.patch('pungi.wrappers.iso.IsoWrapper')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_clone_templates(self, KojiWrapper, link, IsoWrapper,

-                                  get_file_size, get_mtime, ImageCls, run,

-                                  get_dir_from_scm):

+     def test_run_in_runroot_clone_templates(self, KojiWrapper, link, IsoWrapper,

+                                             get_file_size, get_mtime, ImageCls, run,

+                                             get_dir_from_scm):

          compose = helpers.DummyCompose(self.topdir, {

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': 'True',

              'runroot_tag': 'rrt',

          })

          pool = mock.Mock()
@@ -307,12 +340,13 @@ 

      @mock.patch('pungi.wrappers.iso.IsoWrapper')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_with_implicit_release(self, KojiWrapper, link, IsoWrapper,

-                                        get_file_size, get_mtime, ImageCls, run):

+     def test_run_in_runroot_with_implicit_release(self, KojiWrapper, link, IsoWrapper,

+                                                   get_file_size, get_mtime, ImageCls, run):

          compose = helpers.DummyCompose(self.topdir, {

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': 'True',

              'runroot_tag': 'rrt',

          })

          pool = mock.Mock()
@@ -386,12 +420,13 @@ 

      @mock.patch('pungi.wrappers.iso.IsoWrapper')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_fail_crash(self, KojiWrapper, link, IsoWrapper, get_file_size,

-                         get_mtime, ImageCls, run):

+     def test_fail_in_runroot_crash(self, KojiWrapper, link, IsoWrapper, get_file_size,

+                                    get_mtime, ImageCls, run):

          compose = helpers.DummyCompose(self.topdir, {

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': 'True',

              'runroot_tag': 'rrt',

          })

          pool = mock.Mock()
@@ -418,12 +453,13 @@ 

      @mock.patch('pungi.wrappers.iso.IsoWrapper')

      @mock.patch('os.link')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_fail_runroot_fail(self, KojiWrapper, link, IsoWrapper,

-                                get_file_size, get_mtime, ImageCls, run):

+     def test_fail_in_runroot_fail(self, KojiWrapper, link, IsoWrapper,

+                                   get_file_size, get_mtime, ImageCls, run):

          compose = helpers.DummyCompose(self.topdir, {

              'release_name': 'Fedora',

              'release_version': 'Rawhide',

              'koji_profile': 'koji',

+             'runroot': 'True',

              'runroot_tag': 'rrt',

          })

          pool = mock.Mock()

file modified
+105 -3
@@ -57,6 +57,7 @@ 

          }

          self.compose = helpers.DummyCompose(self.topdir, {

              'koji_profile': 'koji',

+             'runroot': True,

              'runroot_tag': 'rrt',

              'translate_paths': [

                  (self.topdir + '/compose', 'http://example.com')
@@ -85,7 +86,7 @@ 

  

      @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run(self, KojiWrapper, get_dir_from_scm):

+     def test_run_in_runroot(self, KojiWrapper, get_dir_from_scm):

          get_dir_from_scm.side_effect = self._dummy_config_repo

  

          koji = KojiWrapper.return_value
@@ -121,7 +122,7 @@ 

  

      @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_fail(self, KojiWrapper, get_dir_from_scm):

+     def test_run_in_runroot_fail(self, KojiWrapper, get_dir_from_scm):

          get_dir_from_scm.side_effect = self._dummy_config_repo

  

          self.cfg['failable'] = ['*']
@@ -140,7 +141,7 @@ 

  

      @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

      @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')

-     def test_run_handle_exception(self, KojiWrapper, get_dir_from_scm):

+     def test_run_in_runroot_handle_exception(self, KojiWrapper, get_dir_from_scm):

          get_dir_from_scm.side_effect = self._dummy_config_repo

  

          self.cfg['failable'] = ['*']
@@ -213,6 +214,107 @@ 

                            1)

          self.assertEqual(self.compose.notifier.send.mock_calls, [])

  

+     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

+     @mock.patch('pungi.phases.ostree.run')

+     def test_run_locally(self, run, get_dir_from_scm):

+         get_dir_from_scm.side_effect = self._dummy_config_repo

+ 

+         compose = helpers.DummyCompose(self.topdir, {

+             'koji_profile': 'koji',

+             'translate_paths': [

+                 (self.topdir + '/compose', 'http://example.com')

+             ]

+         })

+         pool = mock.Mock()

+         cfg = {

+             'source_repo_from': 'Everything',

+             'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',

+             'config_branch': 'f24',

+             'treefile': 'fedora-atomic-docker-host.json',

+             'ostree_repo': self.repo

+         }

+ 

+         t = ostree.OSTreeThread(pool)

+ 

+         t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)

+         cmd = ['pungi-make-ostree',

+                '--log-dir=%s' % (self.topdir + '/logs/x86_64/Everything/ostree-1'),

+                '--treefile=%s' % (self.topdir + '/work/ostree-1/config_repo/' + cfg['treefile']),

+                cfg['ostree_repo']]

+ 

+         self.assertEqual(

+             run.call_args_list,

+             [mock.call(cmd, show_cmd=True,

+              logfile=self.topdir + '/logs/x86_64/Everything/ostree-1/pungi-make-ostree.log')])

+ 

+     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

+     @mock.patch('pungi.phases.ostree.run')

+     def test_run_locally_with_update_summary(self, run, get_dir_from_scm):

+         get_dir_from_scm.side_effect = self._dummy_config_repo

+ 

+         compose = helpers.DummyCompose(self.topdir, {

+             'koji_profile': 'koji',

+             'translate_paths': [

+                 (self.topdir + '/compose', 'http://example.com')

+             ]

+         })

+         pool = mock.Mock()

+         cfg = {

+             'source_repo_from': 'Everything',

+             'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',

+             'config_branch': 'f24',

+             'treefile': 'fedora-atomic-docker-host.json',

+             'ostree_repo': self.repo,

+             'update_summary': True,

+         }

+ 

+         t = ostree.OSTreeThread(pool)

+ 

+         t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)

+         cmd = ['pungi-make-ostree',

+                '--log-dir=%s' % (self.topdir + '/logs/x86_64/Everything/ostree-1'),

+                '--treefile=%s' % (self.topdir + '/work/ostree-1/config_repo/' + cfg['treefile']),

+                '--update-summary', cfg['ostree_repo']]

+ 

+         self.assertEqual(

+             run.call_args_list,

+             [mock.call(cmd, show_cmd=True,

+              logfile=self.topdir + '/logs/x86_64/Everything/ostree-1/pungi-make-ostree.log')])

+ 

+     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')

+     @mock.patch('pungi.phases.ostree.run')

+     def test_run_locally_with_versioning_metadata(self, run, get_dir_from_scm):

+         get_dir_from_scm.side_effect = self._dummy_config_repo

+ 

+         compose = helpers.DummyCompose(self.topdir, {

+             'koji_profile': 'koji',

+             'translate_paths': [

+                 (self.topdir + '/compose', 'http://example.com')

+             ]

+         })

+         pool = mock.Mock()

+         cfg = {

+             'source_repo_from': 'Everything',

+             'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',

+             'config_branch': 'f24',

+             'treefile': 'fedora-atomic-docker-host.json',

+             'ostree_repo': self.repo,

+             'version': '24'

+         }

+ 

+         t = ostree.OSTreeThread(pool)

+ 

+         t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)

+         cmd = ['pungi-make-ostree',

+                '--log-dir=%s' % (self.topdir + '/logs/x86_64/Everything/ostree-1'),

+                '--treefile=%s' % (self.topdir + '/work/ostree-1/config_repo/' + cfg['treefile']),

+                '--version=24',

+                cfg['ostree_repo']]

+ 

+         self.assertEqual(

+             run.call_args_list,

+             [mock.call(cmd, show_cmd=True,

+              logfile=self.topdir + '/logs/x86_64/Everything/ostree-1/pungi-make-ostree.log')])

  

  if __name__ == '__main__':

      unittest.main()

@@ -76,6 +76,47 @@ 

                          self.topdir + '/fedora-atomic-docker-host.json'],

                         logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True, stdout=True)])

  

+     @mock.patch('kobo.shortcuts.run')

+     def test_update_summary(self, run):

+         repo = os.path.join(self.topdir, 'atomic')

  

+         ostree.main([

+             '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),

+             '--treefile=%s/fedora-atomic-docker-host.json' % self.topdir,

+             '--update-summary',

+             repo,

+         ])

+ 

+         self.maxDiff = None

+         self.assertItemsEqual(

+             run.call_args_list,

+             [mock.call(['ostree', 'init', '--repo=%s' % repo, '--mode=archive-z2'],

+                        logfile=self.topdir + '/logs/Atomic/init-ostree-repo.log', show_cmd=True, stdout=True),

+              mock.call(['rpm-ostree', 'compose', 'tree', '--repo=%s' % repo,

+                         self.topdir + '/fedora-atomic-docker-host.json'],

+                        logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True, stdout=True),

+              mock.call(['ostree', 'summary', '-u', '--repo=%s' % repo],

+                        logfile=self.topdir + '/logs/Atomic/ostree-summary.log', show_cmd=True, stdout=True)]),

+ 

+     @mock.patch('kobo.shortcuts.run')

+     def test_versioning_metadata(self, run):

+         repo = os.path.join(self.topdir, 'atomic')

+ 

+         ostree.main([

+             '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),

+             '--treefile=%s/fedora-atomic-docker-host.json' % self.topdir,

+             '--version=24',

+             repo,

+         ])

+ 

+         self.maxDiff = None

+         self.assertItemsEqual(

+             run.call_args_list,

+             [mock.call(['ostree', 'init', '--repo=%s' % repo, '--mode=archive-z2'],

+                        logfile=self.topdir + '/logs/Atomic/init-ostree-repo.log', show_cmd=True, stdout=True),

+              mock.call(['rpm-ostree', 'compose', 'tree', '--repo=%s' % repo,

+                         '--add-metadata-string=version=24',

+                         self.topdir + '/fedora-atomic-docker-host.json'],

+                        logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True, stdout=True)])

  if __name__ == '__main__':

      unittest.main()

[ostree] Allow adding versioning metadata

[ostree] New option to enable generating ostree summary file

[ostree-installer] Allow making ostree installer locally

[ostree] Allow making ostree repository locally

How will we support multiple architectures if we build it locally on the host where pungi is running?

How will we support multiple architectures if we build it locally on the host where pungi is running?

Currently Atomic OSTree supports x86_64 only, so we don't need to worry about that at this moment :)

How will we support multiple architectures if we build it locally on the host where pungi is running?

Currently Atomic OSTree supports x86_64 only, so we don't need to worry about that at this moment :)

That's not true, we're enabling OSTree on both aarch64/Power64 before Fedora 25 Beta, this is actively being supported by the OSTree/Atomic tree.

How will we support multiple architectures if we build it locally on the host where pungi is running?
Currently Atomic OSTree supports x86_64 only, so we don't need to worry about that at this moment :)

That's not true, we're enabling OSTree on both aarch64/Power64 before Fedora 25 Beta, this is actively being supported by the OSTree/Atomic tree.

Thanks for letting me know this, I'll update the patches to filter out the arches which are not same as host when build the OSTree artifacts locally, how about it?

Thanks for letting me know this, I'll update the patches to filter out the arches which are not same as host when build the OSTree artifacts locally, how about it?

I'm not sure what that provides us from running them all in a runroot? IMO it should all be done in the same way to ensure consistent results. What problem is this fixing?

I'm not sure what that provides us from running them all in a runroot? IMO it should all be done in the same way to ensure consistent results. What problem is this fixing?

One of the reasons is that we can't build OSTree live images (not supported yet, I'll just going to add it after the local support) in runroot env, since the koji workers are VMs but live images need to be built with VM. I think it make sense to provide local build support for OSTree and ostree-installer too. And it'd be more convenient if we can compose the OSTree artifacts just locally without talking with koji.

As I understand it, running it locally might be useful for testing. I'm not a big fan of filtering incompatible arches, though, as it could lead to problems in production. Raising an error might be safer (and during testing the config would be required to only do ostree with local arch).

Is this going to work if e.g. repo or config contains a space? Safer solution would be to prepare the command in a list and optionally append the --add-metadata-string argument.

One of the reasons is that we can't build OSTree live images (not supported yet, I'll just going to add it after the local support) in runroot env, since the koji workers are VMs but live images need to be built with VM. I think it make sense to provide local build support for OSTree and ostree-installer too. And it'd be more convenient if we can compose the OSTree artifacts just locally without talking with koji.

The live images don't need to be built with VMs, we build live images for spins etc now using live-media-creator. The hosts that currently run all pungi intsances run on VMs so you're not winning anything by attempting this locally.

I think this approach is wrong. We do it in koji for specific reasons.

The live images don't need to be built with VMs, we build live images for spins etc now using live-media-creator. The hosts that currently run all pungi intsances run on VMs so you're not winning anything by attempting this locally.
I think this approach is wrong. We do it in koji for specific reason

OSTree live image is built with 'rpm-ostree-toolbox liveimage', according to my testing, it need to start a vm to create the image, it always failed without clear message while running in a VM (even the vm can be started successfully within a kvm VM), and @ausil also pointed out that rpm-ostree-toolbox didn't work when he tried in VM, additionally there is no /dev/kvm in the buildroots, so nested VM is impossible.

OSTree live image is built with 'rpm-ostree-toolbox liveimage', according to my testing, it need to start a vm to create the image, it always failed without clear message while running in a VM (even the vm can be started successfully within a kvm VM), and @ausil also pointed out that rpm-ostree-toolbox didn't work when he tried in VM, additionally there is no /dev/kvm in the buildroots, so nested VM is impossible.

So the fix isn't to run this locally on the pungi host as it'll have the same issue.

So the fix isn't to run this locally on the pungi host as it'll have the same issue.

Not exactly, this PR enable the ability to run locally for OSTree and OSTree installer. The live image phase has not been implemented yet, for live image phase, you're right, it need to be ran on 'bare metal' first, and because our koji workers are VMs, providing local support is a solution or workaround for that. I'm not sure about the specific reason of running this in koji only, but enabling the locally support can provide us the convenience of testing, building testing or nigtly composes.

'bare metal' first, and because our koji workers are VMs, providing local support is a solution or

That's not entirely true. We do have baremetal koji builders used for running imagefactory to build cloud/docker base images.

'bare metal' first, and because our koji workers are VMs, providing local support is a solution or

That's not entirely true. We do have baremetal koji builders used for running imagefactory to build cloud/docker base images.

That's a good news, I can look into how to create runroot tasks with 'baremetal' builders only. Would you mind to show me more details on why we need to run with koji only?

The reason why as much stuff as possible is done in koji is reproducibility: with runroot task you know exactly which packages were installed and where they came from. You could technically get a list of packages installed locally, but rebuilding the same environment could be tricky. Also, the runroot environment is fairly minimal with only the packages that are needed and no other running tasks.

I think getting the task restricted only to hardware builders as opposed to VMs would have to be done at least partly in Koji. The only thing we can influence at Pungi level is setting a channel. Koji would need to be configured so that all builders in that channel are actual machines and not VMs. I have no idea if there is enough of them to make this viable, nor what else might change with such a change.

Last I looked at rpm-ostree-toolbox it did not provide sufficient logging. livemedia-creator provides an option to make pxe-to-live trees, please do not call them live as it confuses things with livecds. The only way to ensure builds end up on a bare metal box is inside of koji itself. by managing what hosts are in what channel. you can not run the tasks natively on the host as they are dependent on having specific versions of the tools installed, something that is not manageable on the host. the tasks really should be supported in koji as tasks, we need to support multiple architectures within the same pungi run. We are also investigating the use of nested virt currently. pungi does not need to care what hosts pick the tasks up. we will do that on the koji side

@lsedlar @ausil I agree on that getting the (OSTree liveimage) task restricted to bare metal builders should be done at Koji side. From pungi side, at least we need pungi to support different runroot_channel for phases, since only some (now the OSTree liveimage) of the phases need to be ran with the channel of bare metal builders.

I think add the support of building OSTree artifacts locally doesn't hurt, at least with this feature we can test these phases more easily without runroot env, and it's a workaround for us to support OSTree liveimage at this moment, before we figure out a solution from koji side.

rebased

7 years ago

hi @lsedlar @pbabinca Can we merge this PR since the local build support doesn't break our runroot workflow? And it can bring us the convenient of testing the OSTree stuff, release team can still run with koji's runroot tasks, and we can build locally for testing or any other purpose.

rebased

7 years ago

How is this option expected to be used? Should it only be set for production composes? For nightly composes the config can not be updated daily, so the version would not change much. Should the option be disabled in such case?

@lsedlar it doesn't matter, it's optional, by default, it doesn't add the versioning metadata if it's not present, and can be set by updating the config for both production and nightly composes if necessary.

This has been submit as a seperate PR: https://pagure.io/pungi/pull-request/454

Closing in favor of #455 which is the same change rebased on master.

Pull-Request has been closed by lsedlar

7 years ago