From 4c579da5803aa93a3b9dec0600aaf7d08c0070ac Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Oct 02 2019 14:59:10 +0000 Subject: hub: use joinpath instead of os.path.join --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 7bff002..e6700d4 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -68,6 +68,7 @@ from koji.util import base64encode from koji.util import decode_bytes from koji.util import dslice from koji.util import md5_constructor +from koji.util import joinpath from koji.util import move_and_symlink from koji.util import multi_fnmatch from koji.util import safer_move @@ -2456,13 +2457,13 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa #generate pkglist files pkglist = {} for repoarch in repo_arches: - archdir = os.path.join(repodir, repoarch) + archdir = joinpath(repodir, repoarch) koji.ensuredir(archdir) # Make a symlink to our topdir top_relpath = os.path.relpath(koji.pathinfo.topdir, archdir) - top_link = os.path.join(archdir, 'toplink') + top_link = joinpath(archdir, 'toplink') os.symlink(top_relpath, top_link) - pkglist[repoarch] = open(os.path.join(archdir, 'pkglist'), 'w') + pkglist[repoarch] = open(joinpath(archdir, 'pkglist'), 'w') #NOTE - rpms is now an iterator for rpminfo in rpms: if not with_debuginfo and koji.is_debuginfo(rpminfo['name']): @@ -2491,7 +2492,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa #write blocked package lists for repoarch in repo_arches: - blocklist = open(os.path.join(repodir, repoarch, 'blocklist'), 'w') + blocklist = open(joinpath(repodir, repoarch, 'blocklist'), 'w') for pkg in blocks: blocklist.write(pkg['package_name']) blocklist.write('\n') @@ -2507,9 +2508,9 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa 'epoch': archive['build_epoch'], 'volume_name': archive['volume_name'], } - srcdir = os.path.join(koji.pathinfo.mavenbuild(buildinfo), + srcdir = joinpath(koji.pathinfo.mavenbuild(buildinfo), koji.pathinfo.mavenrepo(archive)) - destlink = os.path.join(repodir, 'maven', + destlink = joinpath(repodir, 'maven', koji.pathinfo.mavenrepo(archive)) dir_links.add((srcdir, destlink)) dest_parent = os.path.dirname(destlink) @@ -2562,7 +2563,7 @@ def _write_maven_repo_metadata(destdir, artifacts): """ % datetime.datetime.now().strftime('%Y%m%d%H%M%S') - with open(os.path.join(destdir, 'maven-metadata.xml'), 'w') as mdfile: + with open(joinpath(destdir, 'maven-metadata.xml'), 'w') as mdfile: mdfile.write(contents) _generate_maven_metadata(destdir) @@ -2590,7 +2591,7 @@ def dist_repo_init(tag, keys, task_opts): insert.execute() repodir = koji.pathinfo.distrepo(repo_id, tinfo['name'], volume=volume) for arch in arches: - koji.ensuredir(os.path.join(repodir, arch)) + koji.ensuredir(joinpath(repodir, arch)) if volume and volume != 'DEFAULT': # symlink from main volume to this one basedir = koji.pathinfo.distrepo(repo_id, tinfo['name']) @@ -2599,9 +2600,9 @@ def dist_repo_init(tag, keys, task_opts): os.symlink(relpath, basedir) # handle comps if task_opts.get('comps'): - groupsdir = os.path.join(repodir, 'groups') + groupsdir = joinpath(repodir, 'groups') koji.ensuredir(groupsdir) - shutil.copyfile(os.path.join(koji.pathinfo.work(), + shutil.copyfile(joinpath(koji.pathinfo.work(), task_opts['comps']), groupsdir + '/comps.xml') # write repo info to disk repo_info = { @@ -3774,7 +3775,7 @@ def get_build_logs(build): for dirpath, dirs, files in os.walk(logdir): subdir = os.path.relpath(dirpath, logdir) for fn in files: - filepath = os.path.join(dirpath, fn) + filepath = joinpath(dirpath, fn) if os.path.islink(filepath): logger.warning("Symlink under logdir: %s", filepath) continue @@ -4525,17 +4526,17 @@ def list_archive_files(archive_id, queryOpts=None, strict=False): if maven_info: maven_archive = get_maven_archive(archive_info['id'], strict=True) archive_info.update(maven_archive) - file_path = os.path.join(koji.pathinfo.mavenbuild(build_info), + file_path = joinpath(koji.pathinfo.mavenbuild(build_info), koji.pathinfo.mavenfile(archive_info)) elif win_info: win_archive = get_win_archive(archive_info['id'], strict=True) archive_info.update(win_archive) - file_path = os.path.join(koji.pathinfo.winbuild(build_info), + file_path = joinpath(koji.pathinfo.winbuild(build_info), koji.pathinfo.winfile(archive_info)) elif image_info: image_archive = get_image_archive(archive_info['id'], strict=True) archive_info.update(image_archive) - file_path = os.path.join(koji.pathinfo.imagebuild(build_info), + file_path = joinpath(koji.pathinfo.imagebuild(build_info), archive_info['filename']) else: # TODO: support other build types @@ -4638,9 +4639,9 @@ def list_task_output(taskID, stat=False, all_volumes=False, strict=False): for path, dirs, files in os.walk(taskDir): for filename in files: relpath = path[len(taskDir) + 1:] - relfilename = os.path.join(relpath, filename) + relfilename = joinpath(relpath, filename) if stat: - stat_info = os.stat(os.path.join(path, filename)) + stat_info = os.stat(joinpath(path, filename)) stat_map = {} for attr in dir(stat_info): if attr == 'st_size': @@ -5354,7 +5355,7 @@ def check_noarch_rpms(basepath, rpms, logs=None): for arch in logs: for log in logs[arch]: if os.path.basename(log) == 'noarch_rpmdiff.json': - task_hash = json.load(open(os.path.join(basepath, log), 'rt')) + task_hash = json.load(open(joinpath(basepath, log), 'rt')) for task_id in task_hash: hashes[task_id] = task_hash[task_id] @@ -5683,7 +5684,7 @@ class CG_Importer(object): if metadata.endswith('.json'): # handle uploaded metadata workdir = koji.pathinfo.work() - path = os.path.join(workdir, directory, metadata) + path = joinpath(workdir, directory, metadata) if not os.path.exists(path): raise koji.GenericError("No such file: %s" % metadata) with open(path, 'rt') as fo: @@ -5892,7 +5893,7 @@ class CG_Importer(object): # - doesn't fit with current major archive categories builddir = koji.pathinfo.build(self.buildinfo) koji.ensuredir(builddir) - path = os.path.join(builddir, 'metadata.json') + path = joinpath(builddir, 'metadata.json') with open(path, 'w') as fo: fo.write(self.raw_metadata) @@ -6056,7 +6057,7 @@ class CG_Importer(object): if fileinfo.get('metadata_only', False): self.metadata_only = True workdir = koji.pathinfo.work() - path = os.path.join(workdir, self.directory, fileinfo.get('relpath', ''), fileinfo['filename']) + path = joinpath(workdir, self.directory, fileinfo.get('relpath', ''), fileinfo['filename']) fileinfo['hub.path'] = path filesize = os.path.getsize(path) @@ -6302,14 +6303,14 @@ def _import_wrapper(task_id, build_info, rpm_results): rpm_task_dir = koji.pathinfo.task(task_id) for rpm_path in [rpm_results['srpm']] + rpm_results['rpms']: - rpm_path = os.path.join(rpm_task_dir, rpm_path) + rpm_path = joinpath(rpm_task_dir, rpm_path) rpm_info = import_rpm(rpm_path, build_info, rpm_buildroot_id, wrapper=True) import_rpm_file(rpm_path, build_info, rpm_info) add_rpm_sig(rpm_info['id'], koji.rip_rpm_sighdr(rpm_path)) for log in rpm_results['logs']: # assume we're only importing noarch packages - import_build_log(os.path.join(rpm_task_dir, log), + import_build_log(joinpath(rpm_task_dir, log), build_info, subdir='noarch') def merge_scratch(task_id): @@ -6411,12 +6412,12 @@ def merge_scratch(task_id): for task_id, info in tasks.items(): taskpath = koji.pathinfo.task(task_id) for filename in info['rpms']: - filepath = os.path.realpath(os.path.join(taskpath, filename)) + filepath = os.path.realpath(joinpath(taskpath, filename)) rpminfo = import_rpm(filepath, build, info['buildroot_id']) import_rpm_file(filepath, build, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(filepath)) for logname in info['logs']: - logpath = os.path.realpath(os.path.join(taskpath, logname)) + logpath = os.path.realpath(joinpath(taskpath, logname)) import_build_log(logpath, build, subdir=info['arch']) # flag tags whose content has changed, so relevant repos can be regen'ed @@ -6675,7 +6676,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No if not metadata_only: # move the file to it's final destination - mavendir = os.path.join(koji.pathinfo.mavenbuild(buildinfo), + mavendir = joinpath(koji.pathinfo.mavenbuild(buildinfo), koji.pathinfo.mavenrepo(typeInfo)) _import_archive_file(filepath, mavendir) _generate_maven_metadata(mavendir) @@ -6696,7 +6697,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No if not metadata_only: destdir = koji.pathinfo.winbuild(buildinfo) if relpath: - destdir = os.path.join(destdir, relpath) + destdir = joinpath(destdir, relpath) _import_archive_file(filepath, destdir) elif type == 'image': insert = InsertProcessor('image_archives') @@ -6704,7 +6705,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No insert.set(arch=typeInfo['arch']) insert.execute() if not metadata_only: - imgdir = os.path.join(koji.pathinfo.imagebuild(buildinfo)) + imgdir = joinpath(koji.pathinfo.imagebuild(buildinfo)) _import_archive_file(filepath, imgdir) # import log files? else: @@ -9000,8 +9001,8 @@ def rpmdiff(basepath, rpmlist, hashes): # ignore differences in file size, md5sum, and mtime # (files may have been generated at build time and contain # embedded dates or other insignificant differences) - d = koji.rpmdiff.Rpmdiff(os.path.join(basepath, first_rpm), - os.path.join(basepath, other_rpm), ignore='S5TN') + d = koji.rpmdiff.Rpmdiff(joinpath(basepath, first_rpm), + joinpath(basepath, other_rpm), ignore='S5TN') if d.differs(): raise koji.BuildError( 'The following noarch package built differently on different architectures: %s\n' @@ -9036,7 +9037,7 @@ def importImageInternal(task_id, build_id, imgdata): imgdata['relpath'] = koji.pathinfo.taskrelpath(imgdata['task_id']) archives = [] for imgfile in imgdata['files']: - fullpath = os.path.join(workpath, imgfile) + fullpath = joinpath(workpath, imgfile) archivetype = get_archive_type(imgfile) logger.debug('image type we are importing is: %s' % archivetype) if not archivetype: @@ -9046,17 +9047,17 @@ def importImageInternal(task_id, build_id, imgdata): # upload logs logs = [f for f in os.listdir(workpath) if f.endswith('.log')] for logfile in logs: - logsrc = os.path.join(workpath, logfile) + logsrc = joinpath(workpath, logfile) if tinfo['method'] == 'livemedia': # multiarch livemedia spins can have log name conflicts, so we # add the arch to the path - logdir = os.path.join(koji.pathinfo.build(build_info), + logdir = joinpath(koji.pathinfo.build(build_info), 'data/logs/image', imgdata['arch']) else: - logdir = os.path.join(koji.pathinfo.build(build_info), + logdir = joinpath(koji.pathinfo.build(build_info), 'data/logs/image') koji.ensuredir(logdir) - final_path = os.path.join(logdir, os.path.basename(logfile)) + final_path = joinpath(logdir, os.path.basename(logfile)) if os.path.exists(final_path): raise koji.GenericError("Error importing build log. %s already exists." % final_path) if os.path.islink(logsrc) or not os.path.isfile(logsrc): @@ -10028,13 +10029,13 @@ class RootExports(object): if not srpms: return _applyQueryOpts([], queryOpts) srpm_info = srpms[0] - srpm_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.rpm(srpm_info)) + srpm_path = joinpath(koji.pathinfo.build(build_info), koji.pathinfo.rpm(srpm_info)) elif taskID: if not filepath: raise koji.GenericError('filepath must be spcified with taskID') if filepath.startswith('/') or '../' in filepath: raise koji.GenericError('invalid filepath: %s' % filepath) - srpm_path = os.path.join(koji.pathinfo.work(), + srpm_path = joinpath(koji.pathinfo.work(), koji.pathinfo.taskrelpath(taskID), filepath) else: @@ -10502,7 +10503,7 @@ class RootExports(object): " because RPM: %s is not internal" % rpmID) return _applyQueryOpts([], queryOpts) build_info = get_build(rpm_info['build_id']) - rpm_path = os.path.join(koji.pathinfo.build(build_info), + rpm_path = joinpath(koji.pathinfo.build(build_info), koji.pathinfo.rpm(rpm_info)) if not os.path.exists(rpm_path): if strict: @@ -10545,7 +10546,7 @@ class RootExports(object): if not rpm_info or not rpm_info['build_id']: return _applyQueryOpts([], queryOpts) build_info = get_build(rpm_info['build_id']) - rpm_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.rpm(rpm_info)) + rpm_path = joinpath(koji.pathinfo.build(build_info), koji.pathinfo.rpm(rpm_info)) if not os.path.exists(rpm_path): return _applyQueryOpts([], queryOpts) @@ -10590,7 +10591,7 @@ class RootExports(object): if not rpm_info or not rpm_info['build_id']: return {} build_info = get_build(rpm_info['build_id']) - rpm_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.rpm(rpm_info)) + rpm_path = joinpath(koji.pathinfo.build(build_info), koji.pathinfo.rpm(rpm_info)) if not os.path.exists(rpm_path): return {} @@ -10626,7 +10627,7 @@ class RootExports(object): if not rpm_info or not rpm_info['build_id']: return {} build_info = get_build(rpm_info['build_id']) - rpm_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.rpm(rpm_info)) + rpm_path = joinpath(koji.pathinfo.build(build_info), koji.pathinfo.rpm(rpm_info)) if not os.path.exists(rpm_path): return {} elif taskID: @@ -10634,7 +10635,7 @@ class RootExports(object): raise koji.GenericError('filepath must be specified with taskID') if filepath.startswith('/') or '../' in filepath: raise koji.GenericError('invalid filepath: %s' % filepath) - rpm_path = os.path.join(koji.pathinfo.work(), + rpm_path = joinpath(koji.pathinfo.work(), koji.pathinfo.taskrelpath(taskID), filepath) else: @@ -12471,22 +12472,22 @@ class HostExports(object): task.assertHost(host.id) scratchdir = koji.pathinfo.scratch() username = get_user(task.getOwner())['name'] - destdir = os.path.join(scratchdir, username, 'task_%s' % task_id) + destdir = joinpath(scratchdir, username, 'task_%s' % task_id) for reldir, files in to_list(results['files'].items()) + [('', results['logs'])]: for filename in files: if reldir: - relpath = os.path.join(reldir, filename) + relpath = joinpath(reldir, filename) else: relpath = filename - src = os.path.join(koji.pathinfo.task(results['task_id']), relpath) - dest = os.path.join(destdir, relpath) + src = joinpath(koji.pathinfo.task(results['task_id']), relpath) + dest = joinpath(destdir, relpath) move_and_symlink(src, dest, create_dir=True) if rpm_results: for relpath in [rpm_results['srpm']] + rpm_results['rpms'] + \ rpm_results['logs']: - src = os.path.join(koji.pathinfo.task(rpm_results['task_id']), + src = joinpath(koji.pathinfo.task(rpm_results['task_id']), relpath) - dest = os.path.join(destdir, 'rpms', relpath) + dest = joinpath(destdir, 'rpms', relpath) move_and_symlink(src, dest, create_dir=True) def moveWinBuildToScratch(self, task_id, results, rpm_results): @@ -12499,17 +12500,17 @@ class HostExports(object): task.assertHost(host.id) scratchdir = koji.pathinfo.scratch() username = get_user(task.getOwner())['name'] - destdir = os.path.join(scratchdir, username, 'task_%s' % task_id) + destdir = joinpath(scratchdir, username, 'task_%s' % task_id) for relpath in to_list(results['output'].keys()) + results['logs']: - filename = os.path.join(koji.pathinfo.task(results['task_id']), relpath) - dest = os.path.join(destdir, relpath) + filename = joinpath(koji.pathinfo.task(results['task_id']), relpath) + dest = joinpath(destdir, relpath) move_and_symlink(filename, dest, create_dir=True) if rpm_results: for relpath in [rpm_results['srpm']] + rpm_results['rpms'] + \ rpm_results['logs']: - filename = os.path.join(koji.pathinfo.task(rpm_results['task_id']), + filename = joinpath(koji.pathinfo.task(rpm_results['task_id']), relpath) - dest = os.path.join(destdir, 'rpms', relpath) + dest = joinpath(destdir, 'rpms', relpath) move_and_symlink(filename, dest, create_dir=True) def moveImageBuildToScratch(self, task_id, results): @@ -12526,20 +12527,20 @@ class HostExports(object): workdir = koji.pathinfo.task(sub_results['task_id']) scratchdir = koji.pathinfo.scratch() username = get_user(task.getOwner())['name'] - destdir = os.path.join(scratchdir, username, + destdir = joinpath(scratchdir, username, 'task_%s' % sub_results['task_id']) for img in sub_results['files'] + sub_results['logs']: - src = os.path.join(workdir, img) - dest = os.path.join(destdir, img) + src = joinpath(workdir, img) + dest = joinpath(destdir, img) logger.debug('renaming %s to %s' % (src, dest)) move_and_symlink(src, dest, create_dir=True) if 'rpmresults' in sub_results: rpm_results = sub_results['rpmresults'] for relpath in [rpm_results['srpm']] + rpm_results['rpms'] + \ rpm_results['logs']: - src = os.path.join(koji.pathinfo.task( + src = joinpath(koji.pathinfo.task( rpm_results['task_id']), relpath) - dest = os.path.join(destdir, 'rpms', relpath) + dest = joinpath(destdir, 'rpms', relpath) move_and_symlink(src, dest, create_dir=True) def initBuild(self, data): @@ -12692,7 +12693,7 @@ class HostExports(object): # This directory has a .pom file, so get the Maven group_id, # artifact_id, and version from it and associate those with # the artifacts in this directory - pom_path = os.path.join(maven_task_dir, relpath, poms[0]) + pom_path = joinpath(maven_task_dir, relpath, poms[0]) pom_info = koji.parse_pom(pom_path) dir_maven_info = koji.pom_to_maven_info(pom_info) else: @@ -12702,13 +12703,13 @@ class HostExports(object): if os.path.splitext(filename)[1] in ('.md5', '.sha1'): # metadata, we'll recreate that ourselves continue - filepath = os.path.join(maven_task_dir, relpath, filename) + filepath = joinpath(maven_task_dir, relpath, filename) if filename == 'maven-metadata.xml': # We want the maven-metadata.xml to be present in the build dir # so that it's a valid Maven repo, but we don't want to track it # in the database because we regenerate it when creating tag repos. # So we special-case it here. - destdir = os.path.join(koji.pathinfo.mavenbuild(build_info), + destdir = joinpath(koji.pathinfo.mavenbuild(build_info), relpath) _import_archive_file(filepath, destdir) _generate_maven_metadata(destdir) @@ -12721,7 +12722,7 @@ class HostExports(object): # move the logs to their final destination for log_path in maven_results['logs']: - import_build_log(os.path.join(maven_task_dir, log_path), + import_build_log(joinpath(maven_task_dir, log_path), build_info, subdir='maven') if rpm_results: @@ -12856,7 +12857,7 @@ class HostExports(object): if not archivetype: # Unknown archive type, fail the build raise koji.BuildError('unsupported file type: %s' % relpath) - filepath = os.path.join(task_dir, relpath) + filepath = joinpath(task_dir, relpath) metadata['relpath'] = os.path.dirname(relpath) import_archive(filepath, build_info, 'win', metadata, buildroot_id=results['buildroot_id']) @@ -12865,8 +12866,8 @@ class HostExports(object): subdir = 'win' reldir = os.path.dirname(relpath) if reldir: - subdir = os.path.join(subdir, reldir) - import_build_log(os.path.join(task_dir, relpath), + subdir = joinpath(subdir, reldir) + import_build_log(joinpath(task_dir, relpath), build_info, subdir=subdir) if rpm_results: @@ -13375,7 +13376,7 @@ class HostExports(object): binfo = get_build(rpminfo['build_id']) builddir = koji.pathinfo.build(binfo) build_dirs[rpminfo['build_id']] = builddir - rpminfo['_fullpath'] = os.path.join(builddir, relpath) + rpminfo['_fullpath'] = joinpath(builddir, relpath) rpmdata[bnp] = rpminfo # move the uploaded files @@ -13392,9 +13393,9 @@ class HostExports(object): rpmpath = rpminfo['_fullpath'] bnp = fn bnplet = bnp[0].lower() - ddir = os.path.join(archdir, 'Packages', bnplet) + ddir = joinpath(archdir, 'Packages', bnplet) koji.ensuredir(ddir) - l_dst = os.path.join(ddir, bnp) + l_dst = joinpath(ddir, bnp) if os.path.exists(l_dst): raise koji.GenericError("File already in repo: %s", l_dst) logger.debug("os.link(%r, %r)", rpmpath, l_dst) @@ -13440,12 +13441,12 @@ def get_upload_path(reldir, name, create=False, volume=None): host.verify() Task(task_id).assertHost(host.id) check_user = False - udir = os.path.join(koji.pathinfo.work(volume=volume), reldir) + udir = joinpath(koji.pathinfo.work(volume=volume), reldir) if create: koji.ensuredir(udir) if check_user: # assuming login was asserted earlier - u_fn = os.path.join(udir, '.user') + u_fn = joinpath(udir, '.user') if os.path.exists(u_fn): user_id = int(open(u_fn, 'r').read()) if context.session.user_id != user_id: @@ -13453,7 +13454,7 @@ def get_upload_path(reldir, name, create=False, volume=None): else: with open(u_fn, 'w') as fo: fo.write(str(context.session.user_id)) - return os.path.join(udir, name) + return joinpath(udir, name) def get_verify_class(verify): if verify == 'md5':