From 8d769774349f6d7a5257e533e354baa867ddaad2 Mon Sep 17 00:00:00 2001
From: mprahl <mprahl@redhat.com>
Date: Oct 01 2019 19:29:00 +0000
Subject: Load the DNF repos in parallel


In production, loading each repo can be quite slow. This approach
loads the repos in parallel.

---

diff --git a/module_build_service/scheduler/default_modules.py b/module_build_service/scheduler/default_modules.py
index 71038d8..4313e63 100644
--- a/module_build_service/scheduler/default_modules.py
+++ b/module_build_service/scheduler/default_modules.py
@@ -350,6 +350,8 @@ def _get_rpms_in_external_repo(repo_url, arches, cache_dir_name):
 
     # Tell DNF to use the cache directory
     dnf_conf.cachedir = cache_location
+    # Don't skip repos that can't be synchronized
+    dnf_conf.skip_if_unavailable = False
     # Get rid of everything to be sure it's a blank slate. This doesn't delete the cached repo data.
     base.reset(repos=True, goal=True, sack=True)
 
@@ -361,14 +363,14 @@ def _get_rpms_in_external_repo(repo_url, arches, cache_dir_name):
         repo_name = "repo_{}".format(canon_arch)
         repo_arch_url = repo_url.replace("$arch", canon_arch)
         base.repos.add_new_repo(repo_name, dnf_conf, baseurl=[repo_arch_url])
-        # Load one repo at a time instead of running `base.update_cache()` so that we know which
-        # repo fails to load if one does
-        try:
-            base.repos[repo_name].load()
-        except dnf.exceptions.RepoError:
-            msg = "Failed to load the external repo {}".format(repo_arch_url)
-            log.exception(msg)
-            raise RuntimeError(msg)
+
+    try:
+        # Load the repos in parallel
+        base.update_cache()
+    except dnf.exceptions.RepoError:
+        msg = "Failed to load the external repos"
+        log.exception(msg)
+        raise RuntimeError(msg)
 
     base.fill_sack(load_system_repo=False)
 
diff --git a/tests/test_scheduler/test_default_modules.py b/tests/test_scheduler/test_default_modules.py
index 4462a72..d2dee05 100644
--- a/tests/test_scheduler/test_default_modules.py
+++ b/tests/test_scheduler/test_default_modules.py
@@ -448,14 +448,11 @@ def test_get_rpms_in_external_repo_failed_to_load(mock_makedirs, mock_dnf_base):
         def add_new_repo(*args, **kwargs):
             pass
 
-    mock_repo = Mock()
-    mock_repo.load.side_effect = dnf.exceptions.RepoError("Failed")
-    mock_dnf_base.return_value.repos = FakeRepo(repo_aarch64=mock_repo)
-
+    mock_dnf_base.return_value.update_cache.side_effect = dnf.exceptions.RepoError("Failed")
     external_repo_url = "http://domain.local/repo/latest/$arch/"
     arches = ["aarch64", "x86_64"]
     cache_dir_name = "module-el-build-12"
-    expected = "Failed to load the external repo http://domain.local/repo/latest/aarch64/"
+    expected = "Failed to load the external repos"
     with pytest.raises(RuntimeError, match=expected):
         default_modules._get_rpms_in_external_repo(external_repo_url, arches, cache_dir_name)