From acf9b8058c1a2b73dfd6ca5afd7f923d6d21974f Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: May 26 2021 10:19:17 +0000 Subject: use importlib then fallback to imp --- diff --git a/koji/__init__.py b/koji/__init__.py index 0b4cd9d..c781fdb 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -28,7 +28,6 @@ import base64 import datetime import errno import hashlib -import imp import json import logging import logging.handlers @@ -49,6 +48,13 @@ import warnings import weakref import xml.sax import xml.sax.handler +try: + import importlib + # this is a trick that PY2 doesn't have machinery + import importlib.machinery +except ImportError: # pragma: no cover + import imp as imp + importlib = None from fnmatch import fnmatch import dateutil.parser @@ -2030,18 +2036,24 @@ def get_profile_module(profile_name, config=None): # Prepare module name mod_name = "__%s__%s" % (__name__, profile_name) - imp.acquire_lock() + if not importlib: + imp.acquire_lock() + try: # Check if profile module exists and if so return it if mod_name in PROFILE_MODULES: return PROFILE_MODULES[mod_name] # Load current module under a new name - koji_module_loc = imp.find_module(__name__) - mod = imp.load_module(mod_name, - None, - koji_module_loc[1], - koji_module_loc[2]) + if importlib: + koji_spec = importlib.util.find_spec(__name__) + mod_spec = importlib.util.spec_from_file_location(mod_name, koji_spec.origin) + mod = importlib.util.module_from_spec(mod_spec) + sys.modules[mod_name] = mod + mod_spec.loader.exec_module(mod) + else: + koji_module_loc = imp.find_module(__name__) + mod = imp.load_module(mod_name, None, koji_module_loc[1], koji_module_loc[2]) # Tweak config of the new module mod.config = config @@ -2053,7 +2065,8 @@ def get_profile_module(profile_name, config=None): PROFILE_MODULES[mod_name] = mod finally: - imp.release_lock() + if not importlib: + imp.release_lock() return mod diff --git a/koji/plugin.py b/koji/plugin.py index 4d2e9b6..fd12223 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -21,7 +21,6 @@ from __future__ import absolute_import -import imp import logging import sys import traceback @@ -31,6 +30,14 @@ import six import koji from koji.util import encode_datetime_recurse +try: + import importlib + # this is a trick that PY2 doesn't have machinery + import importlib.machinery +except ImportError: # pragma: no cover + import imp as imp + importlib = None + # the available callback hooks and a list # of functions to be called for each event callbacks = { @@ -86,15 +93,24 @@ class PluginTracker(object): path = self.searchpath if path is None: raise koji.PluginError("empty module search path") - file, pathname, description = imp.find_module(name, self.pathlist(path)) + file = None try: - plugin = imp.load_module(mod_name, file, pathname, description) + if importlib: + orig_spec = importlib.machinery.PathFinder().find_spec(name, self.pathlist(path)) + plugin_spec = importlib.util.spec_from_file_location(mod_name, orig_spec.origin) + plugin = importlib.util.module_from_spec(plugin_spec) + sys.modules[mod_name] = plugin + plugin_spec.loader.exec_module(plugin) + else: + file, pathname, description = imp.find_module(name, self.pathlist(path)) + plugin = imp.load_module(mod_name, file, pathname, description) except Exception: msg = 'Loading plugin %s failed' % name logging.getLogger('koji.plugin').error(msg) raise finally: - file.close() + if file: + file.close() self.plugins[name] = plugin return plugin diff --git a/tests/test_cli/loadcli.py b/tests/test_cli/loadcli.py index c2800ac..737adcc 100644 --- a/tests/test_cli/loadcli.py +++ b/tests/test_cli/loadcli.py @@ -1,19 +1,26 @@ from __future__ import absolute_import import os -#import sys +import sys + +try: + import importlib + import importlib.machinery +except: + import imp + importlib = None + +CLI_MOD = "koji_cli_fake" # We have to do this craziness because 'import koji' is ambiguous. Is it the # koji module, or the koji cli module. Jump through hoops accordingly. # https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path CLI_FILENAME = os.path.dirname(__file__) + "/../../cli/koji" -''' -if sys.version_info[0] >= 3: - import importlib.util - spec = importlib.util.spec_from_file_location("koji_cli", CLI_FILENAME) +if importlib: + importlib.machinery.SOURCE_SUFFIXES.append('') + spec = importlib.util.spec_from_file_location(CLI_MOD, CLI_FILENAME) cli = importlib.util.module_from_spec(spec) + sys.modules[CLI_MOD] = cli spec.loader.exec_module(cli) + importlib.machinery.SOURCE_SUFFIXES.pop() else: -''' - -import imp -cli = imp.load_source('koji_cli_fake', CLI_FILENAME) + cli = imp.load_source(CLI_MOD, CLI_FILENAME) diff --git a/tests/test_docs_version.py b/tests/test_docs_version.py index f99627b..f97b345 100644 --- a/tests/test_docs_version.py +++ b/tests/test_docs_version.py @@ -3,14 +3,25 @@ import os import six import subprocess import unittest +import sys + +try: + import importlib.util +except ImportError: # pragma: no cover + import imp as imp + importlib = None # docs version lives in docs/source/conf.py TOPDIR = os.path.dirname(__file__) + '/..' SPHINX_CONF = TOPDIR + '/docs/source/conf.py' -import imp -import os -sphinx_conf = imp.load_source('sphinx_conf', SPHINX_CONF) +if importlib: + spec = importlib.util.spec_from_file_location("sphinx_conf", SPHINX_CONF) + sphinx_conf = importlib.util.module_from_spec(spec) + sys.modules["sphinx_conf"] = sphinx_conf + spec.loader.exec_module(sphinx_conf) +else: + sphinx_conf = imp.load_source('sphinx_conf', SPHINX_CONF) class TestDocsVersion(unittest.TestCase): diff --git a/tests/test_kojira/loadkojira.py b/tests/test_kojira/loadkojira.py index c35567b..ff90825 100644 --- a/tests/test_kojira/loadkojira.py +++ b/tests/test_kojira/loadkojira.py @@ -1,8 +1,24 @@ from __future__ import absolute_import import os +import sys +try: + import importlib + import importlib.machinery +except: + import imp + importlib = None # TODO - libify kojira so we don't need this hack -CLI_FILENAME = os.path.dirname(__file__) + "/../../util/kojira" -import imp -kojira = imp.load_source('kojira', CLI_FILENAME) +KOJIRA_MOD = "kojira_" +KOJIRA_FILENAME = os.path.dirname(__file__) + "/../../util/kojira" + +if importlib: + importlib.machinery.SOURCE_SUFFIXES.append('') + spec = importlib.util.spec_from_file_location(KOJIRA_MOD, KOJIRA_FILENAME) + kojira = importlib.util.module_from_spec(spec) + sys.modules[KOJIRA_MOD] = kojira + spec.loader.exec_module(kojira) + importlib.machinery.SOURCE_SUFFIXES.pop() +else: + kojira = imp.load_source(KOJIRA_MOD, KOJIRA_FILENAME) diff --git a/tests/test_lib/test_plugin.py b/tests/test_lib/test_plugin.py index 710a5c0..d9d79b9 100644 --- a/tests/test_lib/test_plugin.py +++ b/tests/test_lib/test_plugin.py @@ -3,8 +3,16 @@ import copy import datetime import mock from six.moves import range +import sys import unittest +try: + import importlib + imp = None +except: + import imp + importlib = None + import koji import koji.util import koji.plugin @@ -195,70 +203,104 @@ class TestCallbacks(unittest.TestCase): class TestPluginTracker(unittest.TestCase): def setUp(self): - self.find_module = mock.patch('imp.find_module').start() - self.modfile = mock.MagicMock() - self.modpath = mock.MagicMock() - self.moddesc = mock.MagicMock() - self.find_module.return_value = (self.modfile, self.modpath, - self.moddesc) - self.load_module = mock.patch('imp.load_module').start() + self.tracker = koji.plugin.PluginTracker(path='/MODPATH') + if imp: + self.modfile = mock.MagicMock() + self.modpath = mock.MagicMock() + self.moddesc = mock.MagicMock() + self.find_module = mock.patch('imp.find_module').start() + + self.find_module.return_value = (self.modfile, self.modpath, self.moddesc) + + self.load_module = mock.patch('imp.load_module').start() + else: + self.finder = mock.patch('importlib.machinery.PathFinder').start() + self.find_spec = self.finder.return_value.find_spec + self.find_spec.return_value = mock.MagicMock() + self.spec_from_file_location = mock.patch('importlib.util.spec_from_file_location').start() + self.module_from_spec = mock.patch('importlib.util.module_from_spec').start() def tearDown(self): mock.patch.stopall() + for key in ['hello', 'hey', 'dup_module']: + if '_koji_plugin__' + key in sys.modules: + del sys.modules['_koji_plugin__' + key] + del self.tracker + + def _set_returned_module(self, mod): + if imp: + self.load_module.return_value = mod + else: + self.module_from_spec.return_value = mod + + def _set_module_side_effect(self, se): + if imp: + self.load_module.side_effect = se + else: + self.module_from_spec.side_effect = se def test_tracked_plugin(self): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.return_value = 'MODULE!' - tracker.load('hello') - self.assertEqual(tracker.get('hello'), 'MODULE!') - self.find_module.assert_called_once_with('hello', ['/MODPATH']) + self._set_returned_module('MODULE!') + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), 'MODULE!') + if imp: + self.find_module.assert_called_once_with('hello', ['/MODPATH']) + else: + self.find_spec.assert_called_once_with('hello', ['/MODPATH']) def test_plugin_reload(self): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.return_value = 'MODULE!' - tracker.load('hello') - self.assertEqual(tracker.get('hello'), 'MODULE!') + self._set_returned_module('MODULE!') + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), 'MODULE!') # should not reload if we don't ask - self.load_module.return_value = 'DUPLICATE!' - tracker.load('hello') - self.assertEqual(tracker.get('hello'), 'MODULE!') + self._set_returned_module('DUPLICATE!') + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), 'MODULE!') # should reload if we do ask - tracker.load('hello', reload=True) - self.assertEqual(tracker.get('hello'), 'DUPLICATE!') + self.tracker.load('hello', reload=True) + self.assertEqual(self.tracker.get('hello'), 'DUPLICATE!') # should throw exception if not reloading and duplicate in sys.modules module_mock = mock.MagicMock() with mock.patch.dict('sys.modules', _koji_plugin__dup_module=module_mock): with self.assertRaises(koji.PluginError): - tracker.load('dup_module') + self.tracker.load('dup_module') def test_no_plugin_path(self): tracker = koji.plugin.PluginTracker() with self.assertRaises(koji.PluginError): tracker.load('hello') - self.load_module.assert_not_called() + if imp: + self.load_module.assert_not_called() + else: + self.module_from_spec.assert_not_called() self.assertEqual(tracker.get('hello'), None) def test_plugin_path_list(self): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.return_value = 'MODULE!' - tracker.load('hello', path=['/PATH1', '/PATH2']) - self.assertEqual(tracker.get('hello'), 'MODULE!') - self.find_module.assert_called_once_with('hello', ['/PATH1', '/PATH2']) - - self.find_module.reset_mock() - tracker.load('hey', path='/PATH1') - self.assertEqual(tracker.get('hey'), 'MODULE!') - self.find_module.assert_called_once_with('hey', ['/PATH1']) + self._set_returned_module('MODULE!') + self.tracker.load('hello', path=['/PATH1', '/PATH2']) + self.assertEqual(self.tracker.get('hello'), 'MODULE!') + if imp: + self.find_module.assert_called_once_with('hello', ['/PATH1', '/PATH2']) + self.find_module.reset_mock() + else: + self.find_spec.assert_called_once_with('hello', ['/PATH1', '/PATH2']) + self.find_spec.reset_mock() + + self.tracker.load('hey', path='/PATH1') + self.assertEqual(self.tracker.get('hey'), 'MODULE!') + if imp: + self.find_module.assert_called_once_with('hey', ['/PATH1']) + else: + self.find_spec.assert_called_once_with('hey', ['/PATH1']) @mock.patch('logging.getLogger') def test_bad_plugin(self, getLogger): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.side_effect = TestError + self._set_module_side_effect(TestError) with self.assertRaises(TestError): - tracker.load('hello') - self.assertEqual(tracker.get('hello'), None) + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), None) getLogger.assert_called_once() getLogger.return_value.error.assert_called_once() diff --git a/tests/test_www/loadwebindex.py b/tests/test_www/loadwebindex.py index faaee54..1d16c81 100644 --- a/tests/test_www/loadwebindex.py +++ b/tests/test_www/loadwebindex.py @@ -1,9 +1,21 @@ import os -import imp +import sys -# We have to do this craziness because 'import koji' is ambiguous. Is it the -# koji module, or the koji cli module. Jump through hoops accordingly. -# https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path + +try: + import importlib + import importlib.machinery +except: + import imp + importlib = None + +INDEX_MOD = "index_fake" INDEX_FILENAME = os.path.dirname(__file__) + "/../../www/kojiweb/index.py" -webidx = imp.load_source('index_fake', INDEX_FILENAME) +if importlib: + spec = importlib.util.spec_from_file_location(INDEX_MOD, INDEX_FILENAME) + webidx = importlib.util.module_from_spec(spec) + sys.modules[INDEX_MOD] = webidx + spec.loader.exec_module(webidx) +else: + cli = imp.load_source(INDEX_MOD, INDEX_FILENAME)