PR#602: don't use /tmp in chroot - koji

koji

#602 don't use /tmp in chroot

Merged 7 years ago by mikem. Opened 7 years ago by tkopecek.

tkopecek/koji issue569 into master

don't use /tmp in chroot

Tomas Kopecek • 7 years ago

84ccc5b

builder/kojid

file modified

+39 -27

		`@@ -732,6 +732,16 @@`
		`def rootdir(self):`
		`return "%s/%s/root" % (self.options.mockdir, self.name)`

		`+ def tmpdir(self, within=False):`
		`+ # mock 1.4+ /tmp is tmpfs mounted on each run, different`
		`+ # directory is needed for persistency`
		`+ # 'within' is equivalent to broot.path_without_to_within(broot.tmpdir())`
		`+ base = "/builddir/tmp"`
		`+ if within:`
		`+ return base`
		`+ else:`
		`+ return "%s/%s" % (self.rootdir(), base)`
mikem commented 7 years ago relatively minor, but this will produce a double slash since base starts with one
		`+`
		`def expire(self):`
		`self.session.host.setBuildRootState(self.id,'EXPIRED')`

		`@@ -1747,7 +1757,7 @@`
		`buildroot.init()`

		`logfile = os.path.join(self.workdir, 'checkout.log')`
		`- scmdir = buildroot.rootdir() + '/tmp/scmroot'`
		`+ scmdir = buildroot.tmpdir() + '/scmroot'`
		`koji.ensuredir(scmdir)`
		`self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=opts.get('scratch'))`
		`specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile)`
		`@@ -2682,7 +2692,7 @@`
		`build_tag: build tag name`
		`@returns: absolute path to the retrieved kickstart file`
		`"""`
		`- scmdir = os.path.join(broot.rootdir(), 'tmp')`
		`+ scmdir = broot.tmpdir()`
		`koji.ensuredir(scmdir)`
		`self.logger.debug("ksfile = %s" % ksfile)`
		`if self.opts.get('ksurl'):`
		`@@ -2776,18 +2786,18 @@`

		`# Write out the new ks file. Note that things may not be in the same`
		`# order and comments in the original ks file may be lost.`
		`- kskoji = os.path.join('/tmp', 'koji-image-%s-%i.ks' %`
		`+ kskoji = os.path.join(broot.tmpdir(), 'koji-image-%s-%i.ks' %`
		`(target_info['build_tag_name'], self.id))`
		`- kojikspath = os.path.join(broot.rootdir(), kskoji[1:])`
		`- outfile = open(kojikspath, 'w')`
		`+ koji.ensuredir(broot.tmpdir())`
		`+ outfile = open(kskoji, 'w')`
		`outfile.write(str(self.ks.handler))`
		`outfile.close()`

		`# put the new ksfile in the output directory`
		`- if not os.path.exists(kojikspath):`
		`- raise koji.LiveCDError("KS file missing: %s" % kojikspath)`
		`- self.uploadFile(kojikspath)`
		`- return kskoji # absolute path within chroot`
		`+ if not os.path.exists(kskoji):`
		`+ raise koji.LiveCDError("KS file missing: %s" % kskoji)`
		`+ self.uploadFile(kskoji)`
		`+ return broot.path_without_to_within(kskoji) # absolute path within chroot`

		`def getImagePackages(self, cachepath):`
		`"""`
		`@@ -2875,9 +2885,10 @@`
		`# Figure out appliance-creator arguments, let it fail if something`
		`# is wrong.`
		`odir = 'app-output'`
		`- opath = os.path.join(broot.rootdir(), 'tmp', odir)`
		`- cachedir = '/tmp/koji-appliance' # arbitrary paths in chroot`
		`- app_log = '/tmp/appliance.log'`
		`+ opath = os.path.join(broot.tmpdir(), odir)`
		`+ # arbitrary paths in chroot`
		`+ cachedir = broot.tmpdir(within=True) + '/koji-appliance'`
		`+ app_log = broot.tmpdir(within=True) + '/appliance.log'`
		`os.mkdir(opath)`

		`cmd = ['/usr/bin/appliance-creator', '-c', kskoji, '-d', '-v',`
		`@@ -3022,24 +3033,25 @@`
		`self.readKickstart(kspath, opts)`
		`kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)`

		`- cachedir = '/tmp/koji-livecd' # arbitrary paths in chroot`
		`- livecd_log = '/tmp/livecd.log'`
		`+ # arbitrary paths in chroot`
		`+ cachedir = broot.tmpdir(within=True) + '/koji-livecd'`
		`+ livecd_log = broot.tmpdir(within=True) + '/livecd.log'`
		`cmd = ['/usr/bin/livecd-creator', '-c', kskoji, '-d', '-v',`
		`'--logfile', livecd_log, '--cache', cachedir]`
		`isoname = '%s-%s-%s' % (name, version, release)`
		`cmd.extend(['-f', self._shortenVolID(name, version, release)])`

		`# Run livecd-creator`
		`- rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)`
		`+ rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)`
		`self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:]))`
		`if rv:`
		`raise koji.LiveCDError(`
		`'Could not create LiveCD: %s' % parseStatus(rv, 'livecd-creator') + '; see root.log or livecd.log for more information')`

		`# Find the resultant iso`
		`- # The cwd of the livecd-creator process is /tmp in the chroot, so`
		`+ # The cwd of the livecd-creator process is tmpdir() in the chroot, so`
		`# that is where it writes the .iso`
		`- files = os.listdir(os.path.join(broot.rootdir(), 'tmp'))`
		`+ files = os.listdir(broot.tmpdir())`
		`isofile = None`
		`for afile in files:`
		`if afile.endswith('.iso'):`
		`@@ -3049,7 +3061,7 @@`
		`raise koji.LiveCDError('multiple .iso files found: %s and %s' % (isofile, afile))`
		`if not isofile:`
		`raise koji.LiveCDError('could not find iso file in chroot')`
		`- isosrc = os.path.join(broot.rootdir(), 'tmp', isofile)`
		`+ isosrc = os.path.join(broot.tmpdir(), isofile)`

		`# copy the iso out of the chroot. If we were given an isoname,`
		`# this is where the renaming happens.`
		`@@ -3113,7 +3125,7 @@`
		`scm = SCM(self.opts['lorax_url'])`
		`scm.assert_allowed(self.options.allowed_scms)`
		`logfile = os.path.join(self.workdir, 'lorax-templates-checkout.log')`
		`- checkout_dir = scm.checkout(os.path.join(build_root.rootdir(), 'tmp'),`
		`+ checkout_dir = scm.checkout(build_root.tmpdir(),`
		`self.session, self.getUploadDir(), logfile)`
		`return os.path.join(build_root.path_without_to_within(checkout_dir),`
		`self.opts['lorax_dir'])`
		`@@ -3197,9 +3209,9 @@`
		`self.readKickstart(kspath, opts)`
		`kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)`

		`- cachedir = '/tmp/koji-livemedia' # arbitrary paths in chroot`
		`- livemedia_log = '/tmp/lmc-logs/livemedia-out.log'`
		`- resultdir = '/tmp/lmc'`
		`+ # arbitrary paths in chroot`
		`+ livemedia_log = broot.tmpdir(within=True) + '/lmc-logs/livemedia-out.log'`
		`+ resultdir = broot.tmpdir(within=True) + '/lmc'`


		`# Common LMC command setup, needs extending`
		`@@ -3237,12 +3249,12 @@`


		`# Run livemedia-creator`
		`- rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)`
		`+ rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)`

		`# upload logs`
		`logdirs = [`
		`- os.path.join(broot.rootdir(), 'tmp/lmc-logs'),`
		`- os.path.join(broot.rootdir(), 'tmp/lmc-logs/anaconda'),`
		`+ os.path.join(broot.tmpdir(), 'lmc-logs'),`
		`+ os.path.join(broot.tmpdir(), 'lmc-logs/anaconda'),`
		`]`
		`for logdir in logdirs:`
		`if not os.path.isdir(logdir):`
		`@@ -3265,7 +3277,7 @@`
		`'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') + '; see root.log or livemedia-out.log for more information')`

		`# Find the resultant iso`
		`- # The cwd of the livemedia-creator process is /tmp in the chroot, so`
		`+ # The cwd of the livemedia-creator process is broot.tmpdir() in the chroot, so`
		`# that is where it writes the .iso`
		`rootresultsdir = os.path.join(broot.rootdir(), resultdir.lstrip('/'))`
		`files = os.listdir(rootresultsdir)`
		`@@ -4494,7 +4506,7 @@`
		`# Setup files and directories for SRPM creation`
		`# We can't put this under the mock homedir because that directory`
		`# is completely blown away and recreated on every mock invocation`
		`- scmdir = broot.rootdir() + '/tmp/scmroot'`
		`+ scmdir = broot.tmpdir() + '/scmroot'`
		`koji.ensuredir(scmdir)`
		`logfile = self.workdir + '/checkout.log'`
		`uploadpath = self.getUploadDir()`

tkopecek commented 7 years ago

mock 1.4+ uses tmpfs for /tmp, so it is no longer writable outside of mock.
Use /builddir/tmp instead of /tmp for persistent data.

ngompa commented 7 years ago

:thumbsup:

julian8628 commented 7 years ago

:thumbsup:

rebased onto 451c6d3a5573fcd20ec4cdbe2e391b3ea44d7814

7 years ago

rebased onto 41c7cc95245b147e083c930db32dd8cb90e5779a

7 years ago

rebased onto b5fb990fc72a38aa1539a8d8fa738ee6f03080f0

7 years ago

rebased onto 84ccc5b

7 years ago

mikem commented on line 13 of builder/kojid 7 years ago

relatively minor, but this will produce a double slash since base starts with one

mikem commented 7 years ago

May have missed one in ApplianceTask

for directory, subdirs, files in os.walk(opath):
    for f in files:
        results.append(os.path.join(broot.rootdir(), 'tmp',
                                    directory, f))

mikem commented 7 years ago

Also in same handler...

    # Run appliance-creator
    rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd)

mikem commented 7 years ago

fixes for above here:
https://github.com/mikem23/koji-playground/commits/pagure/pr/602

ngompa commented 7 years ago

Can we please have this fixed for koji 1.15? This is a serious problem for Mageia's usage of Koji as all we have is Mock 1.4+, and Fedora Infrastructure itself has been holding back Mock upgrades for a while now.

I'm probably going to go ahead and pull these changes into the rebase for Koji 1.14 in Mageia Cauldron, as the changes in @mikem's GitHub fork look okay to me.