| |
@@ -205,86 +205,110 @@
|
| |
|
| |
return self.rpms_by_arch
|
| |
|
| |
- def subset(
|
| |
- self, primary_arch, arch_list, exclusive_noarch=True, inherit_to_noarch=True
|
| |
- ):
|
| |
- """Create a subset of this package set that only includes
|
| |
- packages compatible with"""
|
| |
- pkgset = PackageSetBase(
|
| |
- self.name, self.sigkey_ordering, logger=self._logger, arches=arch_list
|
| |
- )
|
| |
- pkgset.merge(
|
| |
- self,
|
| |
- primary_arch,
|
| |
- arch_list,
|
| |
- exclusive_noarch=exclusive_noarch,
|
| |
- inherit_to_noarch=inherit_to_noarch,
|
| |
- )
|
| |
- return pkgset
|
| |
-
|
| |
- def merge(
|
| |
- self,
|
| |
- other,
|
| |
- primary_arch,
|
| |
- arch_list,
|
| |
- exclusive_noarch=True,
|
| |
- inherit_to_noarch=True,
|
| |
- ):
|
| |
- """
|
| |
- Merge ``other`` package set into this instance.
|
| |
+ def subsets(self, arch_map, exclusive_noarch=True, inherit_to_noarch=True):
|
| |
+ """Create subsets of this package set according to arch_map
|
| |
+ dict. Keys of the dict are primary arch names. Value for each
|
| |
+ key is a list of compatible package arches. e.g.:
|
| |
+ {
|
| |
+ "x86_64": ["x86_64", "i686", "noarch", "src"],
|
| |
+ "aarch64": ["aarch64", "noarch", "src"]
|
| |
+ }
|
| |
+ Will create one package set per primary arch which contains
|
| |
+ all packages for any of the arches in its list, and return
|
| |
+ a dict whose keys are the primary arch names and values are
|
| |
+ the package sets.
|
| |
"""
|
| |
- msg = "Merging package sets for %s: %s" % (primary_arch, arch_list)
|
| |
+ msg = "Creating package subsets with arch map %s" % (arch_map)
|
| |
self.log_debug("[BEGIN] %s" % msg)
|
| |
|
| |
- # if "src" is present, make sure "nosrc" is included too
|
| |
- if "src" in arch_list and "nosrc" not in arch_list:
|
| |
- arch_list.append("nosrc")
|
| |
-
|
| |
- # make sure sources are processed last
|
| |
- for i in ("nosrc", "src"):
|
| |
- if i in arch_list:
|
| |
- arch_list.remove(i)
|
| |
- arch_list.append(i)
|
| |
-
|
| |
- seen_sourcerpms = set()
|
| |
- # {Exclude,Exclusive}Arch must match *tree* arch + compatible native
|
| |
- # arches (excluding multilib arches)
|
| |
- if primary_arch:
|
| |
- exclusivearch_list = get_valid_arches(
|
| |
- primary_arch, multilib=False, add_noarch=False, add_src=False
|
| |
+ pkgsets = {}
|
| |
+ eals = {}
|
| |
+ seen_sourcerpms = {}
|
| |
+
|
| |
+ for arch in arch_map.keys():
|
| |
+ # create a PackageSetBase per target primary arch
|
| |
+ pkgsets[arch] = PackageSetBase(
|
| |
+ self.name,
|
| |
+ self.sigkey_ordering,
|
| |
+ logger=self._logger,
|
| |
+ arches=arch_map[arch],
|
| |
+ )
|
| |
+ # set up an exclusivearchlist per target primary arch
|
| |
+ eals[arch] = get_valid_arches(
|
| |
+ arch, multilib=False, add_noarch=False, add_src=False
|
| |
)
|
| |
# We don't want to consider noarch: if a package is true noarch
|
| |
# build (not just a subpackage), it has to have noarch in
|
| |
# ExclusiveArch otherwise rpm will refuse to build it.
|
| |
# This should eventually become a default, but it could have a big
|
| |
# impact and thus it's hidden behind an option.
|
| |
- if not exclusive_noarch and "noarch" in exclusivearch_list:
|
| |
- exclusivearch_list.remove("noarch")
|
| |
- else:
|
| |
- exclusivearch_list = None
|
| |
- for arch in arch_list:
|
| |
- self.rpms_by_arch.setdefault(arch, [])
|
| |
- for i in other.rpms_by_arch.get(arch, []):
|
| |
- if i.file_path in self.file_cache:
|
| |
- # TODO: test if it really works
|
| |
- continue
|
| |
- if inherit_to_noarch and exclusivearch_list and arch == "noarch":
|
| |
- if is_excluded(i, exclusivearch_list, logger=self._logger):
|
| |
- continue
|
| |
+ if not exclusive_noarch and "noarch" in eals[arch]:
|
| |
+ eals[arch].remove("noarch")
|
| |
+
|
| |
+ for ourarch in self.rpms_by_arch:
|
| |
+ if ourarch in ("src", "nosrc"):
|
| |
+ # we do these at the end, separately
|
| |
+ continue
|
| |
+
|
| |
+ # find the package sets we should put these packages into
|
| |
+ tarches = [tarch for tarch in arch_map.keys() if ourarch in arch_map[tarch]]
|
| |
+ if not tarches:
|
| |
+ continue
|
| |
|
| |
- if arch in ("nosrc", "src"):
|
| |
- # include only sources having binary packages
|
| |
- if i.name not in seen_sourcerpms:
|
| |
+ self.log_debug("handling arch %s" % ourarch)
|
| |
+ # main work loop, putting packages into appropriate sets
|
| |
+ for i in self.rpms_by_arch[ourarch]:
|
| |
+ for tarch in tarches:
|
| |
+ target = pkgsets[tarch]
|
| |
+ if i.file_path in target.file_cache.file_cache:
|
| |
+ # TODO: test if it really works
|
| |
+ self.log_debug("cache hit (bin)")
|
| |
continue
|
| |
- else:
|
| |
+ if inherit_to_noarch and eals[tarch] and ourarch == "noarch":
|
| |
+ if is_excluded(i, eals[tarch], logger=target._logger):
|
| |
+ continue
|
| |
+
|
| |
sourcerpm_name = kobo.rpmlib.parse_nvra(i.sourcerpm)["name"]
|
| |
- seen_sourcerpms.add(sourcerpm_name)
|
| |
+ # we do this check *after* the exclusion check and
|
| |
+ # use the target arch not the package arch so we
|
| |
+ # don't pull the source RPMs of excluded noarch
|
| |
+ # packages into the pkgsets for arches they're
|
| |
+ # excluded from
|
| |
+ if sourcerpm_name in seen_sourcerpms:
|
| |
+ seen_sourcerpms[sourcerpm_name].add(tarch)
|
| |
+ else:
|
| |
+ seen_sourcerpms[sourcerpm_name] = {tarch}
|
| |
|
| |
- self.file_cache.file_cache[i.file_path] = i
|
| |
- self.rpms_by_arch[arch].append(i)
|
| |
+ target.file_cache.file_cache[i.file_path] = i
|
| |
+ if ourarch in target.rpms_by_arch:
|
| |
+ target.rpms_by_arch[ourarch].append(i)
|
| |
+ else:
|
| |
+ target.rpms_by_arch[ourarch] = [i]
|
| |
+ self.log_debug("done with arch %s" % ourarch)
|
| |
+
|
| |
+ # now do src/nosrc
|
| |
+ for srcarch in ("src", "nosrc"):
|
| |
+ self.log_debug("handling src arch %s" % srcarch)
|
| |
+ srcs = self.rpms_by_arch.get(srcarch, [])
|
| |
+ tarches = [tarch for tarch in arch_map.keys() if "src" in arch_map[tarch]]
|
| |
+ for i in srcs:
|
| |
+ for tarch in tarches:
|
| |
+ target = pkgsets[tarch]
|
| |
+ if i.file_path in target.file_cache.file_cache:
|
| |
+ self.log_debug("cache hit (src)")
|
| |
+ continue
|
| |
+ if tarch in seen_sourcerpms[i.name]:
|
| |
+ target.file_cache.file_cache[i.file_path] = i
|
| |
+ if srcarch in target.rpms_by_arch:
|
| |
+ target.rpms_by_arch[srcarch].append(i)
|
| |
+ else:
|
| |
+ target.rpms_by_arch[srcarch] = [i]
|
| |
+ self.log_debug("done with src arch %s" % srcarch)
|
| |
|
| |
self.log_debug("[DONE ] %s" % msg)
|
| |
|
| |
+ return pkgsets
|
| |
+
|
| |
def save_file_list(self, file_path, remove_path_prefix=None):
|
| |
with open(file_path, "w") as f:
|
| |
for arch in sorted(self.rpms_by_arch):
|
| |
adamwill
commented 24 days ago
This attempts to speed up the generation of per-arch subsets of
the global package set by refactoring how it happens. Instead of
generating one subset per primary arch at a time in sequence,
iterating over the global package list for each compatible arch
each time (which means in Fedora we go through noarch and src
four times each, since we have four primary arches and they all
list noarch and src as compatible arches), iterate over the global
package list for every arch only one time, and put each package
encountered into the subsets for all compatible primary arches.
Signed-off-by: Adam Williamson awilliam@redhat.com