From eb70db8eec2107f6a8fadb0881e09b165896bd29 Mon Sep 17 00:00:00 2001
From: Dominika Hodovska <dhodovsk@redhat.com>
Date: Apr 25 2017 20:33:40 +0000
Subject: [PATCH 1/5] Improve ./brt_dep_solver.sh to diff_module_deps.py


---

diff --git a/diff_module_deps.py b/diff_module_deps.py
new file mode 100755
index 0000000..b3ea5fa
--- /dev/null
+++ b/diff_module_deps.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+import sys
+from modularity.cli import CliModuleDepsDifferHelper
+
+if __name__ == "__main__":
+    sys.exit(CliModuleDepsDifferHelper.run())
\ No newline at end of file
diff --git a/modularity/cli.py b/modularity/cli.py
index d46cc5e..1cb93c8 100644
--- a/modularity/cli.py
+++ b/modularity/cli.py
@@ -6,7 +6,7 @@ import argparse
 from modularity.module_generator import ModuleGenerator
 
 from modularity.oc_template import OpenShiftTemplateGenerator
-
+from modularity.module_deps_differ import ModuleDepsDiffer
 
 class CLI(object):
     """ Class for processing data from commandline """
@@ -74,6 +74,50 @@ class CLIRpm2Module(object):
             return object.__getattribute__(self, name)
 
 
+class CliModuleDepsDiffer(object):
+
+    def __init__(self, args=None):
+        self.parser = CliModuleDepsDiffer.build_parser()
+        self.args = self.parser.parse_args(args)
+
+    @staticmethod
+    def build_parser():
+        parser = argparse.ArgumentParser(description='k;')
+        parser.add_argument(
+            "pkg",
+            metavar='PKG',
+            help="Name of package for dependency resolution"
+        )
+        parser.add_argument(
+            "--resolve",
+            action='store_true',
+            default=False
+        )
+
+        return parser
+
+    def __getattr__(self, name):
+        try:
+            return getattr(self.args, name)
+        except AttributeError:
+            return object.__getattribute__(self, name)
+
+
+class CliModuleDepsDifferHelper(object):
+    @staticmethod
+    def run():
+        try:
+            cli = CliModuleDepsDiffer(sys.argv[1:])
+            mdd = ModuleDepsDiffer(cli)
+            mdd.run()
+            if cli.resolve:
+                mdd.dump_pkgs_result()
+            else:
+                mdd.dump_caps_result()
+        except KeyboardInterrupt:
+            print('\nInterrupted by user')
+
+
 class CliRpm2ModuleHelper(object):
 
     @staticmethod
diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py
new file mode 100755
index 0000000..a359899
--- /dev/null
+++ b/modularity/module_deps_differ.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+from pdc_client import PDCClient
+import dnf
+import sys
+import re
+
+
+SUPPORTED_MODULES = ['perl', 'base-runtime', 'common-build-dependencies-bootstrap', 'common-build-dependencies']
+STREAM = 'master'
+REPO_F26 = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/"
+REPO_F26_SOURCE = "http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/source/tree/"
+ARCH = 'x86_64'
+
+# TODO: add ability to solve more packages in one run
+# TODO: solve architecture! - as option?
+
+
+class ModuleDepsDiffer(object):
+
+    def __init__(self, args):
+        self.repos = dict()
+        self.repo_provides = dict()
+        # TODO: get this from args
+        self.package = args.pkg
+        self.build_req = set()
+        self.runtime_req = set()
+        self.build_caps_classified = dict()
+        self.build_caps_classified['result'] = set()
+        self.runtime_caps_classified = dict()
+        self.runtime_caps_classified['result'] = set()
+
+    def get_repo_urls(self):
+        for repo in SUPPORTED_MODULES:
+            p = PDCClient('https://pdc.fedoraproject.org/rest_api/v1/', develop=True, ssl_verify=True)
+
+            module_name = repo
+            stream = STREAM
+            repo_url_template = "https://kojipkgs.fedoraproject.org/repos/{}"
+            j = p['unreleasedvariants'](
+                variant_name=module_name,
+                variant_type="module",
+                variant_version=stream,
+                active=True
+            )
+            # TODO: get latest
+            latest = max(j["results"], key=lambda x:x['variant_release'])
+            repo_url = repo_url_template.format(latest["koji_tag"])
+            self.repos[repo] = str(repo_url + '/latest/' + ARCH)
+
+    @staticmethod
+    def get_base_from_repo(reponame, repourl):
+        base = dnf.Base()
+        repo = dnf.repo.Repo(reponame, base.conf)
+        repo.baseurl = repourl
+        repo.load()
+        repo.enable()
+        base.repos.add(repo)
+        base.fill_sack(load_available_repos=True, load_system_repo=False)
+        return base
+
+    def get_repo_provides(self):
+        self.get_repo_urls()
+        for reponame, repourl in self.repos.items():
+            base = ModuleDepsDiffer.get_base_from_repo(reponame, repourl)
+            allpkgs = base.sack.query().filter()
+            provides_list = set()
+            for pkg in allpkgs:
+                for cap in getattr(pkg, 'provides'):
+                    provides_list.add(str(cap))
+
+            self.repo_provides[reponame] = provides_list
+
+    def get_package_requires(self):
+        # runtime
+        base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26)
+        pkgs = base.sack.query().filter(name=self.package)
+        ModuleDepsDiffer.package_unique(pkgs)
+        requires = getattr(pkgs[0], 'requires')
+        for p in requires:
+            self.runtime_req.add(str(p))
+
+        # build
+        base = ModuleDepsDiffer.get_base_from_repo('f26-source', REPO_F26_SOURCE)
+        pkgs = base.sack.query().filter(name=pkgs[0].source_name)
+        ModuleDepsDiffer.package_unique(pkgs)
+        requires = getattr(pkgs[0], 'requires')
+        for p in requires:
+            self.build_req.add(str(p))
+
+    def classify_caps(self, caps_classified, requirements):
+        for repo_name, repo_provides in self.repo_provides.items():
+            caps_classified[repo_name] = set()
+
+        for cap in requirements:
+            if " = " not in cap:
+                r = re.compile(str('^' + re.escape(cap) + '*'))
+            else:
+                r = re.compile(re.escape(cap))
+            found = False
+            for repo_name, repo_provides in self.repo_provides.items():
+                if list(filter(r.match, repo_provides)):
+                    found = True
+                    caps_classified[repo_name].add(cap)
+            if not found:
+                caps_classified['result'].add(cap)
+
+    @staticmethod
+    def package_unique(result):
+        if len(result) > 1:
+            raise ValueError('Name of package is not unique')
+        if len(result) == 0:
+            raise ValueError('No package found in repo')
+
+    @staticmethod
+    def whatprovides(caps):
+        result = set()
+        base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26)
+        q = base.sack.query()
+        for cap in caps:
+            pkg = q.filter(provides__glob=[cap], latest=True)
+            if pkg:
+                stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0]))
+                result.add(stripped)
+            else:
+                q = q.filter(file__glob=cap)
+                if len(q) == 0:
+                    raise ValueError('No package provides capability \'' + cap + '\'')
+                stripped = re.sub(r'-[^-]*-[^-]*$', '', str(q.result[0]))
+                result.add(stripped)
+        return result
+
+    @staticmethod
+    def _dump_caps_result(items):
+        for repo, caps in items:
+            if not caps:
+                continue
+            if repo == 'result':
+                print("Capabilities which aren't provided by any module:")
+            else:
+                print("Capabilities which are part of " + repo + " module:")
+            for cap in sorted(caps):
+                print(cap)
+            print('')
+
+    def dump_caps_result(self):
+        print('BUILD requirements:')
+        print('-------------------')
+        ModuleDepsDiffer._dump_caps_result(self.build_caps_classified.items())
+        print('\n')
+        print('RUNTIME requirements:')
+        print('---------------------')
+        ModuleDepsDiffer._dump_caps_result(self.runtime_caps_classified.items())
+
+    @staticmethod
+    def _dump_pkgs_result(items):
+        for repo, caps in items:
+            if not caps:
+                continue
+            if repo == 'result':
+                print("Components which aren't provided by any module:")
+            else:
+                print("Components which are part of " + repo + " module:")
+            for cap in sorted(ModuleDepsDiffer.whatprovides(caps)):
+                print(cap)
+            print('')
+
+    def dump_pkgs_result(self):
+        print('BUILD dependencies:')
+        print('-------------------')
+        self._dump_pkgs_result(self.build_caps_classified.items())
+        print('\n')
+        print('RUNTIME dependencies:')
+        print('---------------------')
+        self._dump_pkgs_result(self.runtime_caps_classified.items())
+
+    def run(self):
+        self.get_repo_provides()
+        self.get_package_requires()
+        self.classify_caps(self.build_caps_classified, self.build_req)
+        self.classify_caps(self.runtime_caps_classified, self.runtime_req)

From 5c09b8462bc670d6412943fca42c4a4adf441703 Mon Sep 17 00:00:00 2001
From: Dominika Hodovska <dhodovsk@redhat.com>
Date: Apr 25 2017 20:38:59 +0000
Subject: [PATCH 2/5] Remove unneeded ./brt_dep_solver.sh script


---

diff --git a/brt_dep_solver.sh b/brt_dep_solver.sh
deleted file mode 100755
index 22c76de..0000000
--- a/brt_dep_solver.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/bin/bash
-
-# set -ex
-pkg=${1}
-
-CLEAN=${CLEAN:-true}
-VERBOSE=${VERBOSE:-false}
-REPO_F26="http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/"
-REPO_F26_SOURCE="http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/source/tree/"
-
-function verbose_echo {
-    [ "$VERBOSE" != "true" ] && return
-
-    echo "$@"
-}
-
-# TODO: add other known dependency modules
-function get-baseruntime-caps {
-
-    provided_caps=brt-provides.txt
-
-    verbose_echo -n "Getting capabilities provided by base-runtime packages... "
-    dnf repoquery --disablerepo='*' --repofrompath brt,https://kojipkgs.stg.fedoraproject.org/compose/branched/jkaluza/latest-Boltron-26/compose/base-runtime/x86_64/os/ --enablerepo=brt --provides -q > ${provided_caps} && verbose_echo "DONE"
-
-}
-
-function cleanup {
-
-    cd ..
-    ${CLEAN} && rm ${DIR}/* && rmdir ${DIR}
-}
-
-function resolve-deps {
-    if [[ ${1} == "runtime" ]]; then
-      runtime=true
-    elif [[ ${1} == "build" ]]; then
-      runtime=false
-    else
-      echo "resolve-deps: Invalid operation ${1}"
-      exit -1
-    fi
-
-    if [ "${runtime}" = "true" ]; then
-      DIR=${pkg}-runtime-res-files
-    else
-      DIR=${pkg}-build-res-files
-    fi
-
-    mkdir ${DIR}
-    cd ${DIR}
-
-    get-baseruntime-caps
-
-    verbose_echo -n "Getting ${1} capabilities required by ${pkg}... "
-    # TODO: use remote path for fedora repository
-    SRPM_FLAG=
-    [ "${runtime}" = "true" ] || SRPM_FLAG=--srpm
-    dnf repoquery --requires $SRPM_FLAG -q \
-	        --disablerepo='*' \
-            --repofrompath=f26,${REPO_F26} \
-            --repofrompath=f26-source,${REPO_F26_SOURCE} \
-            --enablerepo=f26 \
-            ${pkg} > ${pkg}-caps.txt && verbose_echo "DONE"
-
-#    [ -s ${pkg}-caps.txt ] || cleanup && return
-    if [ ! -s ${pkg}-caps.txt ]; then
-      cleanup
-      verbose_echo "No ${1} dependencies found."
-      return
-    fi
-    while read cap; do
-      if [[ $cap != *" = "* ]]; then
-        grep -wq "^${cap}*" ${provided_caps} || echo ${cap} >> ${pkg}-filtered-caps.txt
-      else
-        grep -wq "${cap}" ${provided_caps} || echo ${cap} >> ${pkg}-filtered-caps.txt
-      fi
-    done < ${pkg}-caps.txt
-
-    if [ ! -s ${pkg}-filtered-caps.txt ]; then
-      cleanup
-      verbose_echo "No ${1} dependencies found after diff with baseruntime."
-      return
-    fi
-    while read cap; do
-      dnf repoquery -q --latest-limit=1 --srpm \
-        --disablerepo='*' \
-        --repofrompath f26,${REPO_F26} \
-        --repofrompath=f26-source,${REPO_F26_SOURCE} \
-        --enablerepo=f26 \
-        --whatprovides "${cap}" >> cap2rpm.txt
-    done < ${pkg}-filtered-caps.txt
-
-    if [ "$runtime" = "true" ]; then
-      sed -e "s/-[^-]*-[^-]*$//" cap2rpm.txt | sort -u > ../${pkg}-runtime-deps.txt
-    else
-      sed -e "s/-[^-]*-[^-]*$//" cap2rpm.txt | sort -u > ../${pkg}-build-deps.txt
-    fi
-
-    cleanup
-}
-
-resolve-deps "build"
-resolve-deps "runtime"

From 20995433481b395ce75e9ca9dcfcb24226d68ed4 Mon Sep 17 00:00:00 2001
From: Dominika Hodovska <dhodovsk@redhat.com>
Date: Apr 26 2017 14:04:57 +0000
Subject: [PATCH 3/5] Use ModuleDepsDiffer when creating modulemd


---

diff --git a/modularity/cli.py b/modularity/cli.py
index 1cb93c8..69db9b6 100644
--- a/modularity/cli.py
+++ b/modularity/cli.py
@@ -84,8 +84,8 @@ class CliModuleDepsDiffer(object):
     def build_parser():
         parser = argparse.ArgumentParser(description='k;')
         parser.add_argument(
-            "pkg",
-            metavar='PKG',
+            "pkgs",
+            metavar='PKGS',
             help="Name of package for dependency resolution"
         )
         parser.add_argument(
diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py
index a359899..2969144 100755
--- a/modularity/module_deps_differ.py
+++ b/modularity/module_deps_differ.py
@@ -21,7 +21,7 @@ class ModuleDepsDiffer(object):
         self.repos = dict()
         self.repo_provides = dict()
         # TODO: get this from args
-        self.package = args.pkg
+        self.package = args.pkgs
         self.build_req = set()
         self.runtime_req = set()
         self.build_caps_classified = dict()
@@ -119,13 +119,13 @@ class ModuleDepsDiffer(object):
         for cap in caps:
             pkg = q.filter(provides__glob=[cap], latest=True)
             if pkg:
-                stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0]))
+                stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0].source_name))
                 result.add(stripped)
             else:
                 q = q.filter(file__glob=cap)
                 if len(q) == 0:
                     raise ValueError('No package provides capability \'' + cap + '\'')
-                stripped = re.sub(r'-[^-]*-[^-]*$', '', str(q.result[0]))
+                stripped = re.sub(r'-[^-]*-[^-]*$', '', str(q.result[0].source_name))
                 result.add(stripped)
         return result
 
diff --git a/modularity/module_generator.py b/modularity/module_generator.py
index 0e0a966..2c1b630 100644
--- a/modularity/module_generator.py
+++ b/modularity/module_generator.py
@@ -1,7 +1,6 @@
-import os
-import subprocess
 import modulemd
 import dnf
+from module_deps_differ import ModuleDepsDiffer
 
 
 class ModuleGenerator(object):
@@ -12,6 +11,7 @@ class ModuleGenerator(object):
         self.mmd = modulemd.ModuleMetadata()
         self.build_deps = set()
         self.run_deps = set()
+        self.differ = ModuleDepsDiffer(args)
 
     def _save_module_md(self):
         """
@@ -71,18 +71,9 @@ class ModuleGenerator(object):
             Function gets build and runtime dependencies of package
             :return:
             """
-        subprocess.call(['/bin/bash', './brt_dep_solver.sh', self.args.pkgs])
-
-        self.build_deps_file = self.args.pkgs + '-build-deps.txt'
-        self.run_deps_file = self.args.pkgs + '-runtime-deps.txt'
-
-        if os.path.exists(self.build_deps_file):
-            with open(self.build_deps_file, 'r') as f:
-                self.build_deps = set([l.strip('\n') for l in f.readlines()])
-
-        if os.path.exists(self.run_deps_file):
-            with open(self.run_deps_file, 'r') as f:
-                self.run_deps = set([l.strip('\n') for l in f.readlines()])
+        self.differ.run()
+        self.build_deps = ModuleDepsDiffer.whatprovides(self.differ.build_caps_classified['result'])
+        self.run_deps = ModuleDepsDiffer.whatprovides(self.differ.runtime_caps_classified['result'])
 
     def run(self):
         self._get_pkg_info()

From 014f2fecf87a07f38f52c4129bd56cd8bd450ff2 Mon Sep 17 00:00:00 2001
From: Dominika Hodovska <dhodovsk@redhat.com>
Date: Apr 27 2017 07:10:24 +0000
Subject: [PATCH 4/5] Fill required dependency modules in modulemd


---

diff --git a/modularity/module_generator.py b/modularity/module_generator.py
index 2c1b630..bbe2c6d 100644
--- a/modularity/module_generator.py
+++ b/modularity/module_generator.py
@@ -49,6 +49,19 @@ class ModuleGenerator(object):
         for pkg in (self.run_deps - self.build_deps):
             self.mmd.components.add_rpm(pkg, "Runtime dependency.")
 
+        for mod, caps in self.differ.build_caps_classified.items():
+            if mod == 'result':
+                continue
+            if caps:
+                # TODO: use specific stream?
+                self.mmd.add_buildrequires(mod, 'master')
+
+        for mod, caps in self.differ.runtime_caps_classified.items():
+            if mod == 'result':
+                continue
+            if caps:
+                self.mmd.add_requires(mod, 'master')
+
     def _get_pkg_info(self):
         """
             Function loads package from dnf

From 168d1aba838996dbb815c77a9be9b8c079f01edd Mon Sep 17 00:00:00 2001
From: Dominika Hodovska <dhodovsk@redhat.com>
Date: Apr 27 2017 11:47:13 +0000
Subject: [PATCH 5/5] Fix issue with more packages in query result


---

diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py
index 2969144..a95dc36 100755
--- a/modularity/module_deps_differ.py
+++ b/modularity/module_deps_differ.py
@@ -73,7 +73,7 @@ class ModuleDepsDiffer(object):
     def get_package_requires(self):
         # runtime
         base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26)
-        pkgs = base.sack.query().filter(name=self.package)
+        pkgs = base.sack.query().filter(name=self.package, arch=['noarch', ARCH], latest=True)
         ModuleDepsDiffer.package_unique(pkgs)
         requires = getattr(pkgs[0], 'requires')
         for p in requires:
@@ -107,7 +107,7 @@ class ModuleDepsDiffer(object):
     @staticmethod
     def package_unique(result):
         if len(result) > 1:
-            raise ValueError('Name of package is not unique')
+            raise ValueError('Name of package is not unique: ' + str(result.result))
         if len(result) == 0:
             raise ValueError('No package found in repo')
 
@@ -117,7 +117,7 @@ class ModuleDepsDiffer(object):
         base = ModuleDepsDiffer.get_base_from_repo('f26', REPO_F26)
         q = base.sack.query()
         for cap in caps:
-            pkg = q.filter(provides__glob=[cap], latest=True)
+            pkg = q.filter(provides__glob=[cap], arch=['noarch', ARCH], latest=True)
             if pkg:
                 stripped = re.sub(r'-[^-]*-[^-]*$', '', str(pkg.result[0].source_name))
                 result.add(stripped)
@@ -174,7 +174,7 @@ class ModuleDepsDiffer(object):
         self._dump_pkgs_result(self.runtime_caps_classified.items())
 
     def run(self):
-        self.get_repo_provides()
         self.get_package_requires()
+        self.get_repo_provides()
         self.classify_caps(self.build_caps_classified, self.build_req)
         self.classify_caps(self.runtime_caps_classified, self.runtime_req)

