#3 Start migration to depchase
Merged 6 years ago by ncoghlan. Opened 6 years ago by ncoghlan.
modularity/ ncoghlan/fedmod switch-to-depchase  into  master

file modified
-13
@@ -61,19 +61,6 @@ 

  

  Note: the tests will currently fail. Fixing that is a work in progress :)

  

- ### Additional preparation of the local mbs-build cache

- 

- Running the tests currently also depends on having previously built at least

- one module locally with `mbs-build`, as described in the [local module build

- instructions](https://docs.pagure.org/modularity/development/building-modules/building-local.html).

- 

-     $ fedpkg clone --branch f26 modules/haproxy

-     $ cd haproxy

-     $ mbs-build local

- 

- Note: These commands can be run in any directory, they don't have to be run

- from the `fedmod` source checkout.

- 

  ### Reviewing dependencies

  

  To see the Python level dependencies graph:

file added
+498
@@ -0,0 +1,498 @@ 

+ #!/usr/bin/python3

+ import configparser

+ import itertools

+ import logging

+ import os

+ import sys

+ import tempfile

+ import click

+ import smartcols

+ import solv

+ 

+ XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")

+ CACHEDIR = os.path.join(XDG_CACHE_HOME, "depchase")

+ 

+ logger = logging.getLogger("depchase")

+ 

+ class Repo(object):

+     def __init__(self, name, baseurl):

+         self.name = name

+         self.baseurl = baseurl

+         self.handle = None

+         self.cookie = None

+         self.extcookie = None

+         self.srcrepo = None

+ 

+     @staticmethod

+     def calc_cookie_fp(fp):

+         chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256)

+         chksum.add("1.1")

+         chksum.add_fp(fp)

+         return chksum.raw()

+ 

+     @staticmethod

+     def calc_cookie_ext(f, cookie):

+         chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256)

+         chksum.add("1.1")

+         chksum.add(cookie)

+         chksum.add_fstat(f.fileno())

+         return chksum.raw()

+ 

+     def cachepath(self, ext=None):

+         path = "{}-{}".format(self.name.replace(".", "_"), self.baseurl)

+         if ext:

+             path = "{}-{}.solvx".format(path, ext)

+         else:

+             path = "{}.solv".format(path)

+         return os.path.join(CACHEDIR, path.replace("/", "_"))

+ 

+     def usecachedrepo(self, ext, mark=False):

+         try:

+             repopath = self.cachepath(ext)

+             f = open(repopath, "rb")

+             f.seek(-32, os.SEEK_END)

+             fcookie = f.read(32)

+             if len(fcookie) != 32:

+                 return False

+             cookie = self.extcookie if ext else self.cookie

+             if cookie and fcookie != cookie:

+                 return False

+             if not ext:

+                 f.seek(-32 * 2, os.SEEK_END)

+                 fextcookie = f.read(32)

+                 if len(fextcookie) != 32:

+                     return False

+             f.seek(0)

+             f = solv.xfopen_fd(None, f.fileno())

+             flags = 0

+             if ext:

+                 flags = solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES

+                 if ext != "DL":

+                     flags |= solv.Repo.REPO_LOCALPOOL

+             if not self.handle.add_solv(f, flags):

+                 return False

+             if not ext:

+                 self.cookie = fcookie

+                 self.extcookie = fextcookie

+             if mark:

+                 # no futimes in python?

+                 try:

+                     os.utime(repopath, None)

+                 except Exception:

+                     pass

+         except IOError:

+             return False

+         return True

+ 

+     def writecachedrepo(self, ext, repodata=None):

+         tmpname = None

+         try:

+             if not os.path.isdir(CACHEDIR):

+                 os.mkdir(CACHEDIR, 0o755)

+             fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", dir=CACHEDIR)

+             os.fchmod(fd, 0o444)

+             f = os.fdopen(fd, "wb+")

+             f = solv.xfopen_fd(None, f.fileno())

+             if not repodata:

+                 self.handle.write(f)

+             elif ext:

+                 repodata.write(f)

+             else:

+                 # rewrite_repos case, do not write stubs

+                 self.handle.write_first_repodata(f)

+             f.flush()

+             if not ext:

+                 if not self.extcookie:

+                     self.extcookie = self.calc_cookie_ext(f, self.cookie)

+                 f.write(self.extcookie)

+             if not ext:

+                 f.write(self.cookie)

+             else:

+                 f.write(self.extcookie)

+             f.close

+             if self.handle.iscontiguous():

+                 # switch to saved repo to activate paging and save memory

+                 nf = solv.xfopen(tmpname)

+                 if not ext:

+                     # main repo

+                     self.handle.empty()

+                     flags = solv.Repo.SOLV_ADD_NO_STUBS

+                     if repodata:

+                         # rewrite repos case, recreate stubs

+                         flags = 0

+                     if not self.handle.add_solv(nf, flags):

+                         sys.exit("internal error, cannot reload solv file")

+                 else:

+                     # extension repodata

+                     # need to extend to repo boundaries, as this is how

+                     # repodata.write() has written the data

+                     repodata.extend_to_repo()

+                     flags = solv.Repo.REPO_EXTEND_SOLVABLES

+                     if ext != "DL":

+                         flags |= solv.Repo.REPO_LOCALPOOL

+                     repodata.add_solv(nf, flags)

+             os.rename(tmpname, self.cachepath(ext))

+         except (OSError, IOError):

+             if tmpname:

+                 os.unlink(tmpname)

+ 

+     def load(self, pool):

+         assert not self.handle

+         self.handle = pool.add_repo(self.name)

+         self.handle.appdata = self

+         f = self.download("repodata/repomd.xml", False, None)

+         if not f:

+             self.handle.free(True)

+             self.handle = None

+             return False

+         self.cookie = self.calc_cookie_fp(f)

+         if self.usecachedrepo(None, True):

+             return True

+         self.handle.add_repomdxml(f)

+         fname, fchksum = self.find("primary")

+         if not fname:

+             return False

+         f = self.download(fname, True, fchksum)

+         if not f:

+             return False

+         self.handle.add_rpmmd(f, None)

+         self.add_exts()

+         self.writecachedrepo(None)

+         # Must be called after writing the repo

+         self.handle.create_stubs()

+         return True

+ 

+     def download(self, fname, uncompress, chksum):

+         f = open("{}/{}".format(self.baseurl, fname))

+         return solv.xfopen_fd(fname if uncompress else None, f.fileno())

+ 

+     def find(self, what):

+         di = self.handle.Dataiterator_meta(solv.REPOSITORY_REPOMD_TYPE, what, solv.Dataiterator.SEARCH_STRING)

+         di.prepend_keyname(solv.REPOSITORY_REPOMD)

+         for d in di:

+             dp = d.parentpos()

+             filename = dp.lookup_str(solv.REPOSITORY_REPOMD_LOCATION)

+             chksum = dp.lookup_checksum(solv.REPOSITORY_REPOMD_CHECKSUM)

+             if filename:

+                 if not chksum:

+                     print("No {} file checksum!".format(filename))

+                 return filename, chksum

+         return None, None

+ 

+     def add_ext_keys(self, ext, repodata, handle):

+         if ext == "FL":

+             repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.SOLVABLE_FILELIST)

+             repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.REPOKEY_TYPE_DIRSTRARRAY)

+         else:

+             raise NotImplementedError

+ 

+     def add_ext(self, repodata, what, ext):

+         filename, chksum = self.find(what)

+         if not filename:

+             return

+         handle = repodata.new_handle()

+         repodata.set_poolstr(handle, solv.REPOSITORY_REPOMD_TYPE, what)

+         repodata.set_str(handle, solv.REPOSITORY_REPOMD_LOCATION, filename)

+         repodata.set_checksum(handle, solv.REPOSITORY_REPOMD_CHECKSUM, chksum)

+         self.add_ext_keys(ext, repodata, handle)

+         repodata.add_flexarray(solv.SOLVID_META, solv.REPOSITORY_EXTERNAL, handle)

+ 

+     def add_exts(self):

+         repodata = self.handle.add_repodata()

+         self.add_ext(repodata, "filelists", "FL")

+         repodata.internalize()

+ 

+     def load_ext(self, repodata):

+         repomdtype = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_TYPE)

+         if repomdtype == "filelists":

+             ext = "FL"

+         else:

+             assert False

+         if self.usecachedrepo(ext):

+             return True

+         filename = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_LOCATION)

+         filechksum = repodata.lookup_checksum(solv.SOLVID_META, solv.REPOSITORY_REPOMD_CHECKSUM)

+         f = self.download(filename, True, filechksum)

+         if not f:

+             return False

+         if ext == "FL":

+             self.handle.add_rpmmd(f, "FL", solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES | solv.Repo.REPO_LOCALPOOL)

+         self.writecachedrepo(ext, repodata)

+         return True

+ 

+     def updateaddedprovides(self, addedprovides):

+         if self.handle.isempty():

+             return

+         # make sure there's just one real repodata with extensions

+         repodata = self.handle.first_repodata()

+         if not repodata:

+             return

+         oldaddedprovides = repodata.lookup_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES)

+         if not set(addedprovides) <= set(oldaddedprovides):

+             for id in addedprovides:

+                 repodata.add_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES, id)

+             repodata.internalize()

+             self.writecachedrepo(None, repodata)

+ 

+ def load_stub(repodata):

+     repo = repodata.repo.appdata

+     if repo:

+         return repo.load_ext(repodata)

+     return False

+ 

+ def setup_repos(conffile):

+     conf = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())

+ 

+     with open(conffile, "r") as cfg:

+         conf.read_file(cfg)

+ 

+     repos = {}

+     for sect in conf.sections():

+         repos[sect] = Repo(sect, conf[sect]["path"])

+     for repo in repos.values():

+         if repo.name.endswith("-source"):

+             continue

+         repo.srcrepo = repos.get("{}-source".format(repo.name))

+         if repo.srcrepo is None:

+             raise RuntimeError("{}-source repo is not defined".format(repo.name))

+     return list(repos.values())

+ 

+ def setup_pool(arch, repos=()):

+     pool = solv.Pool()

+     #pool.set_debuglevel(2)

+     pool.setarch(arch)

+     pool.set_loadcallback(load_stub)

+ 

+     for repo in repos:

+         repo.baseurl = repo.baseurl.format(arch=arch)

+ 

+     for repo in repos:

+         assert repo.load(pool)

+         if "override" in repo.name:

+             repo.handle.priority = 99

+ 

+     addedprovides = pool.addfileprovides_queue()

+     if addedprovides:

+         for repo in repos:

+             repo.updateaddedprovides(addedprovides)

+ 

+     pool.createwhatprovides()

+ 

+     return pool

+ 

+ def fix_deps(pool):

+     to_fix = (

+         # Weak libcrypt-nss deps due to https://github.com/openSUSE/libsolv/issues/205

+         ("glibc", solv.Selection.SELECTION_NAME,

+          solv.SOLVABLE_RECOMMENDS, lambda s: s.startswith("libcrypt-nss"), solv.SOLVABLE_SUGGESTS),

+         # Shim is not buildable

+         ("shim", solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_WITH_SOURCE,

+          solv.SOLVABLE_REQUIRES, lambda s: s in ("gnu-efi = 3.0w", "gnu-efi-devel = 3.0w"), None),

+     )

+     for txt, flags, before, func, after in to_fix:

+         for s in pool.select(txt, flags).solvables():

+             deps = s.lookup_deparray(before)

+             fixing = [dep for dep in deps if func(str(dep))]

+             for dep in fixing:

+                 deps.remove(dep)

+                 if after is not None:

+                     s.add_deparray(after, dep)

+             # Use s.set_deparray() once will be available

+             s.unset(before)

+             for dep in deps:

+                 s.add_deparray(before, dep)

+ 

+ def get_sourcepkg(p, s=None, only_name=False):

+     if s is None:

+         s = p.lookup_sourcepkg()[:-4]

+     if only_name:

+         return s

+     # Let's try to find corresponding source

+     sel = p.pool.select(s, solv.Selection.SELECTION_CANON | solv.Selection.SELECTION_SOURCE_ONLY)

+     sel.filter(p.repo.appdata.srcrepo.handle.Selection())

+     assert not sel.isempty(), "Could not find source package for {}".format(s)

+     solvables = sel.solvables()

+     assert len(solvables) == 1

+     return solvables[0]

+ 

+ def print_transaction(pool, transaction):

+     candq = transaction.newpackages()

+     if logger.getEffectiveLevel() <= logging.INFO:

+         tb = smartcols.Table()

+         tb.title = "DEPENDENCY INFORMATION"

+         cl = tb.new_column("INFO")

+         cl.tree = True

+         cl_match = tb.new_column("MATCH")

+         for p in candq:

+             ln = tb.new_line()

+             ln[cl] = str(p)

+             for dep in p.lookup_deparray(solv.SOLVABLE_REQUIRES):

+                 lns = tb.new_line(ln)

+                 lns[cl] = str(dep)

+                 matches = set(s for s in candq if s.matchesdep(solv.SOLVABLE_PROVIDES, dep))

+                 if not matches and str(dep).startswith("/"):

+                     # Append provides by files

+                     # TODO: use Dataiterator for getting filelist

+                     matches = set(s for s in pool.select(str(dep), solv.Selection.SELECTION_FILELIST).solvables() if s in candq)

+                 # It was possible to resolve set, so something is wrong here

+                 assert matches

+                 first = True

+                 for m in matches:

+                     if first:

+                         lnc = lns

+                     else:

+                         lnss = tb.new_line(lns)

+                         lnc = lnss

+                         first = False

+                     lnc[cl_match] = str(m)

+         logger.info(tb)

+ 

+ def solve(solver, pkgnames, selfhost=False):

+     pool = solver.pool

+ 

+     # We have to =(

+     fix_deps(pool)

+ 

+     jobs = []

+     # Initial jobs, no conflicting packages

+     for n in pkgnames:

+         sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH)

+         assert not sel.isempty(), "Could not find package for {}".format(n)

+         jobs += sel.jobs(solv.Job.SOLVER_INSTALL)

+     problems = solver.solve(jobs)

+     if problems:

+         for problem in problems:

+             print(problem)

+         sys.exit(1)

+ 

+     print_transaction(pool, solver.transaction())

+     candq = [s for s in solver.transaction().newpackages() if s.arch not in ("src", "nosrc")]

+     sources = set(get_sourcepkg(s) for s in candq)

+ 

+     if not selfhost:

+         return set(candq), sources

+ 

+     # We already solved runtime requires, no need to do that twice

+     selfhosting = set(candq)

+     selfhosting_srcs = set()

+     candq = list(sources)

+     # We will store text-based view of processed srcs for better performance,

+     # because selections are not free

+     srcs_done = set()

+     while candq:

+         jobs = [pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE | solv.Job.SOLVER_WEAK, p.id) for p in candq]

+         solver.solve(jobs)

+         print_transaction(pool, solver.transaction())

+         # We are interested to operate only on really new packages below

+         newpkgs = set(solver.transaction().newpackages()) - selfhosting

+         for p in newpkgs.copy():

+             if p.arch in ("src", "nosrc"):

+                 srcs_done.add(str(p))

+                 selfhosting_srcs.add(p)

+                 newpkgs.remove(p)

+                 continue

+         selfhosting |= newpkgs

+ 

+         # SOLVER_FAVOR packages which we already solved which will help us to get small dependency chain

+         pool.setpooljobs(pool.getpooljobs() + [pool.Job(solv.Job.SOLVER_FAVOR | solv.Job.SOLVER_SOLVABLE, p.id) for p in newpkgs])

+ 

+         # In new queue only non-solvables are left

+         raw_decisions = solver.raw_decisions(1)

+         if not raw_decisions:

+             # At this point, nothing can be resolved anymore, so let's show problems

+             for p in candq:

+                 job = pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE, p.id)

+                 problems = solver.solve([job])

+                 # In some cases, even solvable jobs are disabled

+                 # https://github.com/openSUSE/libsolv/issues/204

+                 #assert not problems

+                 for problem in problems:

+                     print(problem)

+             sys.exit(1)

+         candq = [s for s in candq if s.id not in raw_decisions]

+ 

+         srcs_queued = set(str(p) for p in candq if p.arch in ("src", "nosrc"))

+         for p in newpkgs:

+             s = get_sourcepkg(p, only_name=True)

+             if s in srcs_done or s in srcs_queued:

+                 continue

+             src = get_sourcepkg(p, s)

+             srcs_queued.add(str(src))

+             candq.append(src)

+ 

+     return selfhosting, selfhosting_srcs

+ 

+ 

+ def make_pool(arch, config):

+     return setup_pool(arch, setup_repos(config))

+ 

+ '''

+ @click.option("--recommends/--no-recommends", default=False,

+               help="Do not process optional (aka weak) dependencies.")

+ @click.option("--hint", multiple=True,

+               help="""

+ Specify a package to have higher priority when more than one package could

+ satisfy a dependency. This option may be specified multiple times.

+ 

+ For example, it is recommended to use --hint=glibc-minimal-langpack.

+ """)

+ @click.option("--selfhost", is_flag=True,

+               help="Look up the build dependencies as well.")

+ '''

+ def resolve(pool, pkgnames, recommends, hint, selfhost):

+ 

+     # Set up initial hints

+     favorq = []

+     for n in hint:

+         sel = pool.select(n, solv.Selection.SELECTION_NAME)

+         favorq += sel.jobs(solv.Job.SOLVER_FAVOR)

+     pool.setpooljobs(favorq)

+ 

+     solver = pool.Solver()

+     if not recommends:

+         # Ignore weak deps

+         solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1)

+ 

+     binary, source = solve(solver, pkgnames, selfhost=selfhost)

+     for p in itertools.chain(binary, source or ()):

+         print(p)

+ 

+ def print_reldeps(pool, pkg):

+     sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH)

+     assert not sel.isempty(), "Package can't be found"

+     found = sel.solvables()

+     assert len(found) == 1, "More matching solvables were found, {}".format(found)

+     s = found[0]

+ 

+     reldep2str = {solv.SOLVABLE_REQUIRES: "requires",

+                   solv.SOLVABLE_RECOMMENDS: "recommends",

+                   solv.SOLVABLE_SUGGESTS: "suggests",

+                   solv.SOLVABLE_SUPPLEMENTS: "supplements",

+                   solv.SOLVABLE_ENHANCES: "enhances"}

+     for reltype, relstr in reldep2str.items():

+         for dep in s.lookup_deparray(reltype):

+             print("{}: {}".format(relstr, dep))

+ 

+ def print_sourcepkg(pool, pkg):

+     sel = pool.select(pkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH)

+     assert not sel.isempty(), "Package can't be found"

+     found = sel.solvables()

+     assert len(found) == 1, "More matching solvables were found, {}".format(found)

+     s = found[0]

+ 

+     print(get_sourcepkg(s, only_name=True))

+ 

+ def pkgs_by_sourcepkg(pool, sourcepkg):

+ 

+     sel = pool.select(sourcepkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE)

+     if not sel.isempty():

+         found = sel.solvables()

+         assert len(found) == 1, "More matching solvables were found, {}".format(found)

+         if found[0].arch in ("src", "nosrc"):

+             sourcepkg = str(found[0])

+         else:

+             sourcepkg = get_sourcepkg(found[0], only_name=True)

+ 

+     for p in (s for s in pool.solvables if s.arch not in ("src", "nosrc")):

+         if get_sourcepkg(p, only_name=True) == sourcepkg:

+             print(p)

file modified
+13 -24
@@ -1,5 +1,7 @@ 

  import os

  import logging

+ import requests

+ from six.moves import configparser

  

  import dnf

  
@@ -13,12 +15,20 @@ 

  REPO_F27_SOURCE = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/source/tree/"

  ARCH = 'x86_64'

  

+ 

  # TODO: add ability to solve more packages in one run

  # TODO: solve architecture! - as option?

  

  

  p = None

  

+ def download_repo_metadata(config_path, metadata_dir):

+     """TODO: Automate retrieving the required repo metadata"""

+     raise NotImplementedError(

+         "See https://github.com/fedora-modularity/depchase#installation "

+         "for download instructions"

+     )

+ 

  

  def get_pkgs_source_rpm_name(pkg):

      """
@@ -63,7 +73,7 @@ 

          return self.module_id_to_name.get(koji_tag_name, koji_tag_name)

  

      def obtain_module_names(self):

-         logging.info('Scanning built modules, this might take a while.')

+         logging.info('Getting module list from PDC, this might take a while.')

          j = self.p['unreleasedvariants'](

              variant_type="module",

              active=True,
@@ -90,29 +100,8 @@ 

          return base

  

      def _init_repo_bases(self):

-         """

-         version 1 is no longer usable since koji no longer creates repos for

-         modules this version iterates over repos present in

-         ~/modulebuild/cache/koji_tags, this means that you should do `mbs-build

-         local` to populate the directory

- 

-         Once we have real compose of boltron, we might utilize it here

- 

-         initialize repo objects for dnf to kick off queries

-         """

-         logging.warn("Getting module information from mbs cache." +

-                      " Please run mbs-build local with dependency modules" +

-                      " you are interested in before running this script.")

-         cache_path = os.path.expanduser("~/modulebuild/cache/koji_tags")

-         for module_name in os.listdir(cache_path):

-             # for now ignore all what is not in f27 or rawhide stream, remove when situation changes

-             if not self.module_id_to_name[module_name].endswith((':f27', ':rawhide')):

-                 continue

-             if module_name != "f27-modularity":  # bootstrap contains almost everything, so let's ignore it

-                 self.repos[module_name] = "file://{}".format(os.path.join(cache_path, module_name))

-         for reponame, repourl in self.repos.items():

-             base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl)

-             self.repo_bases[reponame] = base

+         # TODO: download the repo metadata containing module info

+         pass

  

      def get_package_requires(self):

          # runtime

  • add depchase as a private module
  • remove mbs-build cache dependency

Pull-Request has been merged by ncoghlan

6 years ago