| |
@@ -19,6 +19,8 @@
|
| |
import json
|
| |
import re
|
| |
from kobo.shortcuts import force_list
|
| |
+ import kobo.log
|
| |
+ import subprocess
|
| |
|
| |
import pungi.wrappers.kojiwrapper
|
| |
import pungi.phases.pkgset.pkgsets
|
| |
@@ -123,6 +125,50 @@
|
| |
return module
|
| |
|
| |
|
| |
+ class ShadowPkgDownloader(kobo.log.LoggingBase):
|
| |
+ def __init__(self, compose, koji_wrapper):
|
| |
+ super(ShadowPkgDownloader, self).__init__(logger=compose._logger)
|
| |
+
|
| |
+ self.koji_topdir = koji_wrapper.koji_module.config.topdir
|
| |
+ self.koji_topurl = koji_wrapper.koji_module.config.topurl
|
| |
+ self.shadow_dir = compose.conf["koji_shadow_dir"]
|
| |
+
|
| |
+ def download(self, rpm_path):
|
| |
+ if not rpm_path.startswith(self.koji_topdir):
|
| |
+ raise RuntimeError("Unexpected path returned from Koji")
|
| |
+
|
| |
+ partial_path = os.path.relpath(rpm_path, self.koji_topdir)
|
| |
+
|
| |
+ shadow_path = os.path.join(self.shadow_dir, partial_path)
|
| |
+ if os.path.isfile(shadow_path):
|
| |
+ return shadow_path
|
| |
+
|
| |
+ if self.koji_topurl.endswith('/'):
|
| |
+ url = self.koji_topurl + partial_path
|
| |
+ else:
|
| |
+ url = self.koji_topurl + '/' + partial_path
|
| |
+
|
| |
+ shadow_dir = os.path.dirname(shadow_path)
|
| |
+ if not os.path.isdir(shadow_dir):
|
| |
+ try:
|
| |
+ os.makedirs(shadow_dir)
|
| |
+ except OSError:
|
| |
+ pass # multiple threads can race, causing EEXIST
|
| |
+
|
| |
+ self.log_info("Downloading %s" % url)
|
| |
+ subprocess.check_call(['curl', '-s', '-S', '-o', shadow_path, url])
|
| |
+
|
| |
+ return shadow_path
|
| |
+
|
| |
+ def __getstate__(self):
|
| |
+ result = self.__dict__.copy()
|
| |
+ del result["_logger"]
|
| |
+ return result
|
| |
+
|
| |
+ def __setstate__(self, data):
|
| |
+ self._logger = None
|
| |
+ self.__dict__.update(data)
|
| |
+
|
| |
class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase):
|
| |
enabled = True
|
| |
|
| |
@@ -130,15 +176,23 @@
|
| |
compose = self.compose
|
| |
koji_profile = compose.conf["koji_profile"]
|
| |
self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiWrapper(koji_profile)
|
| |
+ if 'koji_shadow_dir' in compose.conf:
|
| |
+ path_prefix = compose.conf["koji_shadow_dir"]
|
| |
+ downloader = ShadowPkgDownloader(self.compose, self.koji_wrapper)
|
| |
+ else:
|
| |
+ path_prefix = self.koji_wrapper.koji_module.config.topdir
|
| |
+ downloader = None
|
| |
# path prefix must contain trailing '/'
|
| |
- path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
|
| |
- package_sets = get_pkgset_from_koji(self.compose, self.koji_wrapper, path_prefix)
|
| |
+ path_prefix = path_prefix.rstrip("/") + "/"
|
| |
+ package_sets = get_pkgset_from_koji(self.compose, self.koji_wrapper, path_prefix,
|
| |
+ downloader=downloader)
|
| |
return (package_sets, path_prefix)
|
| |
|
| |
|
| |
- def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
|
| |
+ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix, downloader=None):
|
| |
event_info = get_koji_event_info(compose, koji_wrapper)
|
| |
- pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
|
| |
+ pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info,
|
| |
+ downloader=downloader)
|
| |
package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global)
|
| |
package_sets["global"] = pkgset_global
|
| |
|
| |
@@ -150,7 +204,8 @@
|
| |
return package_sets
|
| |
|
| |
|
| |
- def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
|
| |
+ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id,
|
| |
+ downloader=None):
|
| |
all_arches = set(["src"])
|
| |
for arch in compose.get_arches():
|
| |
is_multilib = is_arch_multilib(compose.conf, arch)
|
| |
@@ -167,7 +222,7 @@
|
| |
for variant in compose.all_variants.values():
|
| |
variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
|
| |
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
|
| |
- arches=all_arches)
|
| |
+ arches=all_arches, downloader=downloader)
|
| |
variant_tags[variant] = []
|
| |
|
| |
# Find out all modules in every variant and add their compose tags
|
| |
@@ -209,7 +264,7 @@
|
| |
else:
|
| |
global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
|
| |
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
|
| |
- arches=all_arches)
|
| |
+ arches=all_arches, downloader=downloader)
|
| |
# Get package set for each compose tag and merge it to global package
|
| |
# list. Also prepare per-variant pkgset, because we do not have list
|
| |
# of binary RPMs in module definition - there is just list of SRPMs.
|
| |
@@ -218,7 +273,7 @@
|
| |
"'%s'" % compose_tag)
|
| |
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
|
| |
koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
|
| |
- arches=all_arches)
|
| |
+ arches=all_arches, downloader=downloader)
|
| |
# Create a filename for log with package-to-tag mapping. The tag
|
| |
# name is included in filename, so any slashes in it are replaced
|
| |
# with underscores just to be safe.
|
| |
@lsedler would like your thoughts but I think koji_local_dir is a better name