pungi

Clone

#1613 Create DVDs with xorriso
Merged 2 years ago by lsedlar. Opened 2 years ago by lsedlar.
lsedlar/pungi xorriso-createiso  into  master

file modified
+31 -2 
@@ -15,6 +15,7 @@ 
 

      "CreateIsoOpts",
 

      [
 

          "buildinstall_method",
 

+         "boot_iso",
 

          "arch",
 

          "output_dir",
 

          "jigdo_dir",
 
@@ -26,6 +27,7 @@ 
 

          "hfs_compat",
 

          "use_xorrisofs",
 

          "iso_level",
 

+         "script_dir",
 

      ],
 

  )
 

  CreateIsoOpts.__new__.__defaults__ = (None,) * len(CreateIsoOpts._fields)
 
@@ -116,6 +118,27 @@ 
 

      emit(f, cmd)
 

  

  

+ def write_xorriso_commands(opts):
 

+     script = os.path.join(opts.script_dir, "xorriso-%s.txt" % id(opts))
 

+     with open(script, "w") as f:
 

+         emit(f, "-indev %s" % opts.boot_iso)
 

+         emit(f, "-outdev %s" % os.path.join(opts.output_dir, opts.iso_name))
 

+         emit(f, "-boot_image any replay")
 

+         emit(f, "-volid %s" % opts.volid)
 

+ 

+         with open(opts.graft_points) as gp:
 

+             for line in gp:
 

+                 iso_path, fs_path = line.strip().split("=", 1)
 

+                 emit(f, "-map %s %s" % (fs_path, iso_path))
 

+ 

+         if opts.arch == "ppc64le":
 

+             # This is needed for the image to be bootable.
 

+             emit(f, "-as mkisofs -U --")
 

+ 

+         emit(f, "-end")
 

+     return script
 

+ 

+ 

  def write_script(opts, f):
 

      if bool(opts.jigdo_dir) != bool(opts.os_tree):
 

          raise RuntimeError("jigdo_dir must be used together with os_tree")
 
@@ -123,8 +146,14 @@ 
 

      emit(f, "#!/bin/bash")
 

      emit(f, "set -ex")
 

      emit(f, "cd %s" % opts.output_dir)
 

-     make_image(f, opts)
 

-     run_isohybrid(f, opts)
 

+ 

+     if opts.use_xorrisofs and opts.buildinstall_method:
 

+         script = write_xorriso_commands(opts)
 

+         emit(f, "xorriso -dialog on <%s" % script)
 

+     else:
 

+         make_image(f, opts)
 

+         run_isohybrid(f, opts)
 

+ 

      implant_md5(f, opts)
 

      make_manifest(f, opts)
 

      if opts.jigdo_dir:

file modified
+7 -5 
@@ -343,7 +343,10 @@ 
 

  

                      if bootable:
 

                          opts = opts._replace(
 

-                             buildinstall_method=self.compose.conf["buildinstall_method"]
 

+                             buildinstall_method=self.compose.conf[
 

+                                 "buildinstall_method"
 

+                             ],
 

+                             boot_iso=os.path.join(os_tree, "images", "boot.iso"),
 

                          )
 

  

                      if self.compose.conf["create_jigdo"]:
 
@@ -355,10 +358,9 @@ 
 

                          # Reuse was successful, go to next ISO
 

                          continue
 

  

-                     script_file = os.path.join(
 

-                         self.compose.paths.work.tmp_dir(arch, variant),
 

-                         "createiso-%s.sh" % filename,
 

-                     )
 

+                     script_dir = self.compose.paths.work.tmp_dir(arch, variant)
 

+                     opts = opts._replace(script_dir=script_dir)
 

+                     script_file = os.path.join(script_dir, "createiso-%s.sh" % filename)
 

                      with open(script_file, "w") as f:
 

                          createiso.write_script(opts, f)
 

                      cmd["cmd"] = ["bash", script_file]

file modified
+6 -5 
@@ -132,14 +132,15 @@ 
 

              use_xorrisofs=compose.conf.get("createiso_use_xorrisofs"),
 

              iso_level=compose.conf.get("iso_level"),
 

          )
 

+         os_tree = compose.paths.compose.os_tree(arch, variant)
 

          if compose.conf["create_jigdo"]:
 

              jigdo_dir = compose.paths.compose.jigdo_dir(arch, variant)
 

-             os_tree = compose.paths.compose.os_tree(arch, variant)
 

              opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree)
 

  

          if bootable:
 

              opts = opts._replace(
 

-                 buildinstall_method=compose.conf["buildinstall_method"]
 

+                 buildinstall_method=compose.conf["buildinstall_method"],
 

+                 boot_iso=os.path.join(os_tree, "images", "boot.iso"),
 

              )
 

  

          # Check if it can be reused.
 
@@ -148,9 +149,9 @@ 
 

          config_hash = hash.hexdigest()
 

  

          if not self.try_reuse(compose, variant, arch, config_hash, opts):
 

-             script_file = os.path.join(
 

-                 compose.paths.work.tmp_dir(arch, variant), "extraiso-%s.sh" % filename
 

-             )
 

+             script_dir = compose.paths.work.tmp_dir(arch, variant)
 

+             opts = opts._replace(script_dir=script_dir)
 

+             script_file = os.path.join(script_dir, "extraiso-%s.sh" % filename)
 

              with open(script_file, "w") as f:
 

                  createiso.write_script(opts, f)

file modified
+8 
@@ -124,6 +124,7 @@ 
 

                      os_tree=None,
 

                      hfs_compat=True,
 

                      use_xorrisofs=False,
 

+                     script_dir="%s/work/x86_64/tmp-Server" % self.topdir,
 

                  )
 

              ],
 

          )
 
@@ -240,6 +241,9 @@ 
 

              [
 

                  CreateIsoOpts(
 

                      output_dir="%s/compose/Server/x86_64/iso" % self.topdir,
 

+                     boot_iso=(
 

+                         "%s/compose/Server/x86_64/os/images/boot.iso" % self.topdir
 

+                     ),
 

                      iso_name="image-name",
 

                      volid="test-1.0 Server.x86_64",
 

                      graft_points="dummy-graft-points",
 
@@ -250,6 +254,7 @@ 
 

                      os_tree=None,
 

                      hfs_compat=True,
 

                      use_xorrisofs=False,
 

+                     script_dir="%s/work/x86_64/tmp-Server" % self.topdir,
 

                  ),
 

                  CreateIsoOpts(
 

                      output_dir="%s/compose/Server/source/iso" % self.topdir,
 
@@ -262,6 +267,7 @@ 
 

                      os_tree=None,
 

                      hfs_compat=True,
 

                      use_xorrisofs=False,
 

+                     script_dir="%s/work/src/tmp-Server" % self.topdir,
 

                  ),
 

              ],
 

          )
 
@@ -394,6 +400,7 @@ 
 

                      os_tree=None,
 

                      hfs_compat=True,
 

                      use_xorrisofs=False,
 

+                     script_dir="%s/work/src/tmp-Server" % self.topdir,
 

                  )
 

              ],
 

          )
 
@@ -501,6 +508,7 @@ 
 

                      os_tree=None,
 

                      hfs_compat=False,
 

                      use_xorrisofs=False,
 

+                     script_dir="%s/work/x86_64/tmp-Server" % self.topdir,
 

                  )
 

              ],
 

          )

Use a different approach for building DVDs when xorriso is enabled.

The default of using genisoimage is not changed at all. When the config option is set to use xorriso, the actual execution is different between bootable and non-bootable images.

The non-bootable images are still created by running xorrisofs (which is a compatibility tool with same UI as genisoimage). Since the image is not bootable, there should be no problems with boot options.

For bootable images, Pungi will instead take the boot.iso generated by Lorax, and use xorriso to inject all the extra files into the image.

The shell script that used to invoke all the commands to build the ISO now runs the xorriso command in interactive mode and feeds another file into it. The new file contains the xorriso commands to add the required files to the image.

The commands fed to xorriso boil down to:

-indev path/to/boot.iso
-outdev path/to/dvd.iso
-boot_image any replay
-volid VOLID
-update path/to/file/on/fs /path/to/file/on/iso
...
-end

So far it seems the commands are able to produce images for all arches except ppc64le. I have not yet tried to boot them though.

On ppc64le I'm getting xorriso : SORRY : Cannot make proposal to produce APM of loaded image after the -boot_image command. An option would be to set -return_with to ignore such severity, but it may be an actual problem.

rebased onto 1908883c15ec2afde2700677ffe4e4e1ad90c02b
2 years ago

So far it seems the commands are able to produce images for all arches except ppc64le. I have not yet tried to boot them though.

I have now tried to boot the x86_64 one and that works fine.

edit: never mind for now, I may have looked at the wrong bit of lorax...
Edited 2 years ago by adamwill

What image did you test with, exactly? I just grabbed https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20220607.n.0/compose/Everything/ppc64le/iso/Fedora-Everything-netinst-ppc64le-Rawhide-20220607.n.0.iso and tried this:

xorriso -indev Fedora-Everything-netinst-ppc64le-Rawhide-20220607.n.0.iso -outdev test.iso -boot_image any replay -volid TESTID

and it worked OK?

I think you will also need to use -map for new directories and files.

hmm, I'm no expert but it seems odd that would trigger the problem. oh well, I can poke at it a bit more. The condition that triggers the error seems to be:

if((apm_count > 0 && !cared_for_apm) && !(flag & 5))

From the xorriso info output, and just the form of the lorax mkisofs command, though, I don't see why there would be an APM involved anywhere here?
Edited 2 years ago by adamwill

Adding -update rpm-tmp.1wi2Rx /test.file doesn't trigger the failure either (that's just a file I happen to have lying around in /var/tmp where I'm testing this from).

I was testing it with RHEL, not Fedora.

When trying a similar command on boot ISOs downloaded from Fedora, I do not see the error, and it seems to work fine.

It's a something with the boot.iso itself:

  • RHEL with old lorax: Boot record : (system area only) , MBR CHRP cyl-align-off APM
  • Fedora with old lorax: Boot record : (system area only) , MBR CHRP cyl-align-off
  • Fedora with new lorax: Boot record : (system area only) , MBR CHRP cyl-align-off

I think you will also need to use -map for new directories and files.

I have used -update, but -map may be more efficient. IIUC, -map only adds new files to the ISO, while -update also supports modifying existing ones. That shouldn't be needed here. I'll test it.
Edited 2 years ago by lsedlar

rebased onto 21b71ffd2ae2de5bf35e9971423d4e621f6bb061
2 years ago

rebased onto 596d6298a1524f8b9779b1d12d6fde4cf5051178
2 years ago

-map vs. -update doesn't seem to have any observable impact. Let's keep -map to more clearly communicate files are only added, not updated.

The APM issue can probably be ignored. I tried the same test with RHEL 9 and that works fine. So it's likely only something in 8 that's causing the problem, and the code will not be used there anyway.

I'll make a scratch build that can be used to test the patch.

Here's a scratch build for testing:
https://koji.fedoraproject.org/koji/taskinfo?taskID=88019648
(It's created from this: https://src.fedoraproject.org/rpms/pungi/pull-request/8)

In order to actually use the new code, configuration for the compose must specify createiso_use_xorrisofs = True (yup, the name does not match anymore :/ )

With the patch, Pungi should no longer care about what lorax does, so it should work no matter how the ISO is set up to boot.

CC @kevin

Ah, I suspect the RHEL ISOs may still be baking in unnecessary bits intended for boot on PPC Macs. That stuff was dropped from the ppc64le templates on lorax main branch in 2019 - around Fedora 30, but if the git branches are accurate to what's actually used in RHEL, it has not made it to RHEL 7 or 8 yet: the commit is not on rhel7-branch or rhel8-branch.

AFAICT it's entirely pointless on ppc64le images. PPC Macs were all big-endian.

@kevin what do you want to do now? can we run a test compose with all the changes necessary to actually try this (newer lorax, scratch build of pungi, change to the compose config)? Or would we need to push everything to an official build?

There's a possible fix for the aarch64 cloud failures that have been breaking the compose. I'd like to try another compose as soon as that kernel finishes building. Then, after that we can land all this stuff and try and get it working?

Sure. We'd need an official build of pungi to do it that way, I guess.

Ok, here's an official build: https://koji.fedoraproject.org/koji/buildinfo?buildID=1980140
I'm not going to create a Bodhi update until we know it works though.

For the record, we're a bit blocked on this - @kevin wanted to get one compose through the old way before trying this, but after reverting everything we're now stuck on an aarch64 kernel bug :( We'll get back to it next week.

@kevin, is there a staging compose host? I may be able to try and get the patch running there for at least some testing. In the past I could use composer.stg.phx2.fedoraproject.org for that, but it's long gone now.

We have now: compose-x86-01.stg.iad2.fedoraproject.org

What the current status on this? Looks like we are getting close to the F37 infrastructure deadline.

I can not access the staging compose host. I think it's because in stg environment my account is not member of any group.

I made a new build for possible testing: https://koji.fedoraproject.org/koji/buildinfo?buildID=1985234
It is based on the latest release.

@lsedlar I've added you to sysadmin/sysadmin-releng in stg, so you should now have access I think.

That said, I think @adamwill and I will try and just run this in prod today.

ok. I tagged in lorax again and upgraded pungi on the composer and re-reverted the kickstart and comps changes.

That compose failed. ;(

https://pagure.io/releng/failed-composes/issue/3671

INFO backend.py:391:  Running in chroot: ['/bin/sh', '-c', '{ rm -f /var/lib/rpm/__db*; rm -rf /var/cache/yum/*; set -x; bash /mnt/koji/compose/rawhide/Fedora-Rawhide-20220615.n.1/work/x86_64/tmp-Server/createiso-Fedora-Server-dvd-x86_64-Rawhide-20220615.n.1.iso.sh; } < /dev/null 2>&1 | /usr/bin/tee /builddir/runroot.log; exit ${PIPESTATUS[0]}']
DEBUG util.py:622:  child environment: None
DEBUG util.py:540:  Executing command: ['/bin/sh', '-c', '{ rm -f /var/lib/rpm/__db*; rm -rf /var/cache/yum/*; set -x; bash /mnt/koji/compose/rawhide/Fedora-Rawhide-20220615.n.1/work/x86_64/tmp-Server/createiso-Fedora-Server-dvd-x86_64-Rawhide-20220615.n.1.iso.sh; } < /dev/null 2>&1 | /usr/bin/tee /builddir/runroot.log; exit ${PIPESTATUS[0]}'] with env {'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;<mock-chroot>\\007"', 'PS1': '<mock-chroot> \\s-\\v\\$ ', 'LANG': 'C.UTF-8'} and shell False
DEBUG util.py:445:  + bash /mnt/koji/compose/rawhide/Fedora-Rawhide-20220615.n.1/work/x86_64/tmp-Server/createiso-Fedora-Server-dvd-x86_64-Rawhide-20220615.n.1.iso.sh
DEBUG util.py:445:  + cd /mnt/koji/compose/rawhide/Fedora-Rawhide-20220615.n.1/compose/Server/x86_64/iso
DEBUG util.py:445:  +++ cut -c3-
DEBUG util.py:445:  ++++ which lorax
DEBUG util.py:445:  +++ head -n1 /usr/sbin/lorax
DEBUG util.py:445:  ++ /usr/bin/python3 -c 'import pylorax; print(pylorax.find_templates())'
DEBUG util.py:445:  + TEMPLATE=/usr/share/lorax/templates.d/99-generic
DEBUG util.py:445:  + /usr/bin/genisoimage -untranslated-filenames -volid Fedora-S-dvd-x86_64-rawh -J -joliet-long -rational-rock -translation-table -input-charset utf-8 -x ./lost+found -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot -o Fedora-Server-dvd-x86_64-Rawhide-20220615.n.1.iso -graft-points -path-list /mnt/koji/compose/rawhide/Fedora-Rawhide-20220615.n.1/work/x86_64/iso/Fedora-Server-dvd-x86_64-Rawhide-20220615.n.1.iso-graft-points
DEBUG util.py:445:  Warning: creating filesystem that does not conform to ISO-9660.
DEBUG util.py:445:  /usr/bin/genisoimage: Uh oh, I cant find the boot catalog directory 'isolinux'!
DEBUG util.py:596:  Child return code was: 255

well, that's pungi still trying to use isolinux. Oh. Back in https://pagure.io/pungi/pull-request/1613#comment-172383 , @lsedlar said we need to set a config option for pungi - createiso_use_xorrisofs = True.

ok I set that, lets see how tonights compose goes.

Compose failed again. Looks like it's because we still try to run isohybrid, which is part of syslinux and should not be needed in this situation. So we probably need to add something like this:

diff --git a/pungi/createiso.py b/pungi/createiso.py
index 4e1a5336..bc5e19f5 100644
--- a/pungi/createiso.py
+++ b/pungi/createiso.py
@@ -94,7 +94,7 @@ def run_isohybrid(f, opts):
     be booted when written to USB disk. This is done by running isohybrid on
     the image.
     """
-    if opts.buildinstall_method and opts.arch in ["x86_64", "i386"]:
+    if opts.buildinstall_method and opts.arch in ["x86_64", "i386"] and not opts.use_xorrisofs:
         cmd = iso.get_isohybrid_cmd(opts.iso_name, opts.arch)
         emit(f, cmd)

rebased onto bac30771ffa82351b85206e1ae5b0a8b0e053f96
2 years ago

I uploaded a new version of the patch. I see there is already a build though, so I'm not going to make another one.

Right, your change should be functionally equivalent to mine so we shouldn't really need to do a new build.

Hi there! I was looking to submit an issue/PR as it pertains to xorriso to hopefully help out with a potential move away from genisoimage, but I ended up finding a grub2 issue and this PR instead! Looking through the changes in this, I wanted to give some info that may help down the road.

In Rocky Linux 9, we decided to use xorrisofs to make our images, as we figured genisoimage may be going away at some point. While we were getting it to work, we ran into a gotcha along the way.

To preface this, when genisoimage is being used, current code in pungi shows that -J and -joliet-long is used. xorriso doesn't have these on by default and it'll ultimately affect how the image behaves. In lorax, I believe that -J is being passed into xorrisofs. Though this is the case, it does not carry over when you are adding onto the boot image (why this is, I'm not entirely sure). In the dialog, we had to add these two lines:

-joliet on
-compliance joliet_long_names

Without these, some strange behavior ends up occurring, like virt-install misbehaving (and perhaps other components might misbehave as a result). virt-install tends to run isoinfo when you pass it an ISO image, something like the below:

# isoinfo -J -i Rocky-9.0-x86_64-dvd.iso -f
isoinfo: Unable to find Joliet SVD

This is something that virt-install will run and the error that it actually receives (with a code of 255). And because of this error, the installation doesn't proceed.

I hope this little tidbit helps!

There's a problem with the ppc64le dvd -- it doesn't boot. See rhbz#2109095

I've narrowed it down to xorriso needing to include -U (this effects mkksiso too) on ppc64le, so you should add something like this PR: https://github.com/weldr/lorax/pull/1251

You need to add '-as mkisofs -U' to the end of the cmdline to switch it to xorrisofs command parsing and adding the -U argument.

rebased onto 27e39deafcec56947d46b023b3e9395d6042c479
2 years ago

I updated the patch to add -as mkisofs -U on ppc64le.
The change is included in https://koji.fedoraproject.org/koji/buildinfo?buildID=2031152 and can be tested from there.

I have updated to that version of pungi on compose-rawhide01. We should be able to look at tomorrow's compose.

Thanks, I'll try and remember to check it.

Indeed this does look better in today's compose, the ppc64le server DVD now boots where it didn't before. thanks!

rebased onto 13ea8e5
2 years ago

Pull-Request has been merged by lsedlar
2 years ago
Metadata
None
No Tags
Changes Summary 4
+31 -2
file changed
pungi/createiso.py
+7 -5
file changed
pungi/phases/createiso.py
+6 -5
file changed
pungi/phases/extra_isos.py
+8 -0
file changed
tests/test_createiso_phase.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.