From 9541c2a173b05971f25ac7b78158ce839fae1fa2 Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Jun 02 2021 09:19:33 +0000 Subject: importlib instead of imp Fixes: https://pagure.io/koji/issue/2822 --- diff --git a/koji/__init__.py b/koji/__init__.py index 0b4cd9d..df2487e 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -28,7 +28,6 @@ import base64 import datetime import errno import hashlib -import imp import json import logging import logging.handlers @@ -49,6 +48,13 @@ import warnings import weakref import xml.sax import xml.sax.handler +try: + import importlib + import importlib.machinery +except ImportError: # pragma: no cover + # importlib not available for PY2, so we fall back to using imp + import imp as imp + importlib = None from fnmatch import fnmatch import dateutil.parser @@ -2030,18 +2036,24 @@ def get_profile_module(profile_name, config=None): # Prepare module name mod_name = "__%s__%s" % (__name__, profile_name) - imp.acquire_lock() + if not importlib: + imp.acquire_lock() + try: # Check if profile module exists and if so return it if mod_name in PROFILE_MODULES: return PROFILE_MODULES[mod_name] # Load current module under a new name - koji_module_loc = imp.find_module(__name__) - mod = imp.load_module(mod_name, - None, - koji_module_loc[1], - koji_module_loc[2]) + if importlib: + koji_spec = importlib.util.find_spec(__name__) + mod_spec = importlib.util.spec_from_file_location(mod_name, koji_spec.origin) + mod = importlib.util.module_from_spec(mod_spec) + sys.modules[mod_name] = mod + mod_spec.loader.exec_module(mod) + else: + koji_module_loc = imp.find_module(__name__) + mod = imp.load_module(mod_name, None, koji_module_loc[1], koji_module_loc[2]) # Tweak config of the new module mod.config = config @@ -2053,7 +2065,8 @@ def get_profile_module(profile_name, config=None): PROFILE_MODULES[mod_name] = mod finally: - imp.release_lock() + if not importlib: + imp.release_lock() return mod diff --git a/koji/plugin.py b/koji/plugin.py index 4d2e9b6..6306e97 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -21,7 +21,6 @@ from __future__ import absolute_import -import imp import logging import sys import traceback @@ -31,6 +30,14 @@ import six import koji from koji.util import encode_datetime_recurse +try: + import importlib + import importlib.machinery +except ImportError: # pragma: no cover + # importlib not available for PY2, so we fall back to using imp + import imp as imp + importlib = None + # the available callback hooks and a list # of functions to be called for each event callbacks = { @@ -86,15 +93,24 @@ class PluginTracker(object): path = self.searchpath if path is None: raise koji.PluginError("empty module search path") - file, pathname, description = imp.find_module(name, self.pathlist(path)) + file = None try: - plugin = imp.load_module(mod_name, file, pathname, description) + if importlib: + orig_spec = importlib.machinery.PathFinder().find_spec(name, self.pathlist(path)) + plugin_spec = importlib.util.spec_from_file_location(mod_name, orig_spec.origin) + plugin = importlib.util.module_from_spec(plugin_spec) + sys.modules[mod_name] = plugin + plugin_spec.loader.exec_module(plugin) + else: + file, pathname, description = imp.find_module(name, self.pathlist(path)) + plugin = imp.load_module(mod_name, file, pathname, description) except Exception: msg = 'Loading plugin %s failed' % name logging.getLogger('koji.plugin').error(msg) raise finally: - file.close() + if file: + file.close() self.plugins[name] = plugin return plugin diff --git a/tests/test_cli/loadcli.py b/tests/test_cli/loadcli.py index c2800ac..485e7c3 100644 --- a/tests/test_cli/loadcli.py +++ b/tests/test_cli/loadcli.py @@ -1,19 +1,26 @@ from __future__ import absolute_import import os -#import sys +import sys + +try: + import importlib + import importlib.machinery +except ImportError: + import imp + importlib = None + +CLI_MOD = "koji_cli_fake" # We have to do this craziness because 'import koji' is ambiguous. Is it the # koji module, or the koji cli module. Jump through hoops accordingly. # https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path CLI_FILENAME = os.path.dirname(__file__) + "/../../cli/koji" -''' -if sys.version_info[0] >= 3: - import importlib.util - spec = importlib.util.spec_from_file_location("koji_cli", CLI_FILENAME) +if importlib: + importlib.machinery.SOURCE_SUFFIXES.append('') + spec = importlib.util.spec_from_file_location(CLI_MOD, CLI_FILENAME) cli = importlib.util.module_from_spec(spec) + sys.modules[CLI_MOD] = cli spec.loader.exec_module(cli) + importlib.machinery.SOURCE_SUFFIXES.pop() else: -''' - -import imp -cli = imp.load_source('koji_cli_fake', CLI_FILENAME) + cli = imp.load_source(CLI_MOD, CLI_FILENAME) diff --git a/tests/test_docs_version.py b/tests/test_docs_version.py index f99627b..aa163f0 100644 --- a/tests/test_docs_version.py +++ b/tests/test_docs_version.py @@ -3,14 +3,25 @@ import os import six import subprocess import unittest +import sys + +try: + import importlib.util +except ImportError: # pragma: no cover + import imp as imp + importlib = None # docs version lives in docs/source/conf.py TOPDIR = os.path.dirname(__file__) + '/..' SPHINX_CONF = TOPDIR + '/docs/source/conf.py' -import imp -import os -sphinx_conf = imp.load_source('sphinx_conf', SPHINX_CONF) +if importlib: + spec = importlib.util.spec_from_file_location("sphinx_conf", SPHINX_CONF) + sphinx_conf = importlib.util.module_from_spec(spec) + sys.modules["sphinx_conf"] = sphinx_conf + spec.loader.exec_module(sphinx_conf) +else: + sphinx_conf = imp.load_source('sphinx_conf', SPHINX_CONF) class TestDocsVersion(unittest.TestCase): @@ -35,4 +46,3 @@ class TestDocsVersion(unittest.TestCase): # docs 'version' is x.y instead of x.y.z dver = '.'.join(koji_version.split('.')[:-1]) self.assertEqual(dver, sphinx_conf.version) - diff --git a/tests/test_kojira/loadkojira.py b/tests/test_kojira/loadkojira.py index c35567b..9ed4ce8 100644 --- a/tests/test_kojira/loadkojira.py +++ b/tests/test_kojira/loadkojira.py @@ -1,8 +1,24 @@ from __future__ import absolute_import import os +import sys +try: + import importlib + import importlib.machinery +except ImportError: + import imp + importlib = None # TODO - libify kojira so we don't need this hack -CLI_FILENAME = os.path.dirname(__file__) + "/../../util/kojira" -import imp -kojira = imp.load_source('kojira', CLI_FILENAME) +KOJIRA_MOD = "kojira_" +KOJIRA_FILENAME = os.path.dirname(__file__) + "/../../util/kojira" + +if importlib: + importlib.machinery.SOURCE_SUFFIXES.append('') + spec = importlib.util.spec_from_file_location(KOJIRA_MOD, KOJIRA_FILENAME) + kojira = importlib.util.module_from_spec(spec) + sys.modules[KOJIRA_MOD] = kojira + spec.loader.exec_module(kojira) + importlib.machinery.SOURCE_SUFFIXES.pop() +else: + kojira = imp.load_source(KOJIRA_MOD, KOJIRA_FILENAME) diff --git a/tests/test_lib/test_plugin.py b/tests/test_lib/test_plugin.py index 710a5c0..b942755 100644 --- a/tests/test_lib/test_plugin.py +++ b/tests/test_lib/test_plugin.py @@ -3,8 +3,16 @@ import copy import datetime import mock from six.moves import range +import sys import unittest +try: + import importlib + imp = None +except ImportError: + import imp + importlib = None + import koji import koji.util import koji.plugin @@ -14,7 +22,7 @@ class TestCallbackDecorators(unittest.TestCase): def test_callback_decorator(self): def myfunc(a, b, c=None): - return [a,b,c] + return [a, b, c] callbacks = ('preImport', 'postImport') newfunc = koji.plugin.callback(*callbacks)(myfunc) self.assertEqual(newfunc.callbacks, callbacks) @@ -23,7 +31,7 @@ class TestCallbackDecorators(unittest.TestCase): def test_ignore_error_decorator(self): def myfunc(a, b, c=None): - return [a,b,c] + return [a, b, c] newfunc = koji.plugin.ignore_error(myfunc) self.assertEqual(newfunc.failure_is_an_option, True) self.assertEqual(newfunc(1, 2), [1, 2, None]) @@ -31,7 +39,7 @@ class TestCallbackDecorators(unittest.TestCase): def test_export_decorator(self): def myfunc(a, b, c=None): - return [a,b,c] + return [a, b, c] newfunc = koji.plugin.export(myfunc) self.assertEqual(newfunc.exported, True) self.assertEqual(newfunc(1, 2), [1, 2, None]) @@ -39,7 +47,7 @@ class TestCallbackDecorators(unittest.TestCase): def test_export_cli_decorator(self): def myfunc(a, b, c=None): - return [a,b,c] + return [a, b, c] newfunc = koji.plugin.export_cli(myfunc) self.assertEqual(newfunc.exported_cli, True) self.assertEqual(newfunc(1, 2), [1, 2, None]) @@ -47,7 +55,7 @@ class TestCallbackDecorators(unittest.TestCase): def test_export_as_decorator(self): def myfunc(a, b, c=None): - return [a,b,c] + return [a, b, c] alias = "ALIAS" newfunc = koji.plugin.export_as(alias)(myfunc) self.assertEqual(newfunc.exported, True) @@ -57,7 +65,7 @@ class TestCallbackDecorators(unittest.TestCase): def test_export_in_decorator_with_alias(self): def myfunc(a, b, c=None): - return [a,b,c] + return [a, b, c] newfunc = koji.plugin.export_in('MODULE', 'ALIAS')(myfunc) self.assertEqual(newfunc.exported, True) self.assertEqual(newfunc.export_alias, 'MODULE.ALIAS') @@ -67,7 +75,7 @@ class TestCallbackDecorators(unittest.TestCase): def test_export_in_decorator_no_alias(self): def myfunc(a, b, c=None): - return [a,b,c] + return [a, b, c] newfunc = koji.plugin.export_in('MODULE')(myfunc) self.assertEqual(newfunc.exported, True) self.assertEqual(newfunc.export_alias, 'MODULE.myfunc') @@ -141,8 +149,8 @@ class TestCallbacks(unittest.TestCase): def test_datetime_callback(self): dt1 = datetime.datetime.now() - dt2 = datetime.datetime(2001,1,1) - args = (dt1,"2",["three"], {4: dt2},) + dt2 = datetime.datetime(2001, 1, 1) + args = (dt1, "2", ["three"], {4: dt2},) kwargs = {'foo': [dt1, dt2]} cbtype = 'preTag' koji.plugin.register_callback(cbtype, self.datetime_callback) @@ -154,8 +162,8 @@ class TestCallbacks(unittest.TestCase): def test_multiple_datetime_callback(self): dt1 = datetime.datetime.now() - dt2 = datetime.datetime(2001,1,1) - args = (dt1,"2",["three"], {4: dt2},) + dt2 = datetime.datetime(2001, 1, 1) + args = (dt1, "2", ["three"], {4: dt2},) kwargs = {'foo': [dt1, dt2]} cbtype = 'preTag' koji.plugin.register_callback(cbtype, self.datetime_callback) @@ -195,70 +203,105 @@ class TestCallbacks(unittest.TestCase): class TestPluginTracker(unittest.TestCase): def setUp(self): - self.find_module = mock.patch('imp.find_module').start() - self.modfile = mock.MagicMock() - self.modpath = mock.MagicMock() - self.moddesc = mock.MagicMock() - self.find_module.return_value = (self.modfile, self.modpath, - self.moddesc) - self.load_module = mock.patch('imp.load_module').start() + self.tracker = koji.plugin.PluginTracker(path='/MODPATH') + if imp: + self.modfile = mock.MagicMock() + self.modpath = mock.MagicMock() + self.moddesc = mock.MagicMock() + self.find_module = mock.patch('imp.find_module').start() + + self.find_module.return_value = (self.modfile, self.modpath, self.moddesc) + + self.load_module = mock.patch('imp.load_module').start() + else: + self.finder = mock.patch('importlib.machinery.PathFinder').start() + self.find_spec = self.finder.return_value.find_spec + self.find_spec.return_value = mock.MagicMock() + self.spec_from_file_location = mock.patch( + 'importlib.util.spec_from_file_location').start() + self.module_from_spec = mock.patch('importlib.util.module_from_spec').start() def tearDown(self): mock.patch.stopall() + for key in ['hello', 'hey', 'dup_module']: + if '_koji_plugin__' + key in sys.modules: + del sys.modules['_koji_plugin__' + key] + del self.tracker + + def _set_returned_module(self, mod): + if imp: + self.load_module.return_value = mod + else: + self.module_from_spec.return_value = mod + + def _set_module_side_effect(self, se): + if imp: + self.load_module.side_effect = se + else: + self.module_from_spec.side_effect = se def test_tracked_plugin(self): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.return_value = 'MODULE!' - tracker.load('hello') - self.assertEqual(tracker.get('hello'), 'MODULE!') - self.find_module.assert_called_once_with('hello', ['/MODPATH']) + self._set_returned_module('MODULE!') + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), 'MODULE!') + if imp: + self.find_module.assert_called_once_with('hello', ['/MODPATH']) + else: + self.find_spec.assert_called_once_with('hello', ['/MODPATH']) def test_plugin_reload(self): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.return_value = 'MODULE!' - tracker.load('hello') - self.assertEqual(tracker.get('hello'), 'MODULE!') + self._set_returned_module('MODULE!') + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), 'MODULE!') # should not reload if we don't ask - self.load_module.return_value = 'DUPLICATE!' - tracker.load('hello') - self.assertEqual(tracker.get('hello'), 'MODULE!') + self._set_returned_module('DUPLICATE!') + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), 'MODULE!') # should reload if we do ask - tracker.load('hello', reload=True) - self.assertEqual(tracker.get('hello'), 'DUPLICATE!') + self.tracker.load('hello', reload=True) + self.assertEqual(self.tracker.get('hello'), 'DUPLICATE!') # should throw exception if not reloading and duplicate in sys.modules module_mock = mock.MagicMock() with mock.patch.dict('sys.modules', _koji_plugin__dup_module=module_mock): with self.assertRaises(koji.PluginError): - tracker.load('dup_module') + self.tracker.load('dup_module') def test_no_plugin_path(self): tracker = koji.plugin.PluginTracker() with self.assertRaises(koji.PluginError): tracker.load('hello') - self.load_module.assert_not_called() + if imp: + self.load_module.assert_not_called() + else: + self.module_from_spec.assert_not_called() self.assertEqual(tracker.get('hello'), None) def test_plugin_path_list(self): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.return_value = 'MODULE!' - tracker.load('hello', path=['/PATH1', '/PATH2']) - self.assertEqual(tracker.get('hello'), 'MODULE!') - self.find_module.assert_called_once_with('hello', ['/PATH1', '/PATH2']) - - self.find_module.reset_mock() - tracker.load('hey', path='/PATH1') - self.assertEqual(tracker.get('hey'), 'MODULE!') - self.find_module.assert_called_once_with('hey', ['/PATH1']) + self._set_returned_module('MODULE!') + self.tracker.load('hello', path=['/PATH1', '/PATH2']) + self.assertEqual(self.tracker.get('hello'), 'MODULE!') + if imp: + self.find_module.assert_called_once_with('hello', ['/PATH1', '/PATH2']) + self.find_module.reset_mock() + else: + self.find_spec.assert_called_once_with('hello', ['/PATH1', '/PATH2']) + self.find_spec.reset_mock() + + self.tracker.load('hey', path='/PATH1') + self.assertEqual(self.tracker.get('hey'), 'MODULE!') + if imp: + self.find_module.assert_called_once_with('hey', ['/PATH1']) + else: + self.find_spec.assert_called_once_with('hey', ['/PATH1']) @mock.patch('logging.getLogger') def test_bad_plugin(self, getLogger): - tracker = koji.plugin.PluginTracker(path='/MODPATH') - self.load_module.side_effect = TestError + self._set_module_side_effect(TestError) with self.assertRaises(TestError): - tracker.load('hello') - self.assertEqual(tracker.get('hello'), None) + self.tracker.load('hello') + self.assertEqual(self.tracker.get('hello'), None) getLogger.assert_called_once() getLogger.return_value.error.assert_called_once() diff --git a/tests/test_www/loadwebindex.py b/tests/test_www/loadwebindex.py index faaee54..7cf4d39 100644 --- a/tests/test_www/loadwebindex.py +++ b/tests/test_www/loadwebindex.py @@ -1,9 +1,20 @@ import os -import imp +import sys -# We have to do this craziness because 'import koji' is ambiguous. Is it the -# koji module, or the koji cli module. Jump through hoops accordingly. -# https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path +try: + import importlib + import importlib.machinery +except ImportError: + import imp + importlib = None + +INDEX_MOD = "index_fake" INDEX_FILENAME = os.path.dirname(__file__) + "/../../www/kojiweb/index.py" -webidx = imp.load_source('index_fake', INDEX_FILENAME) +if importlib: + spec = importlib.util.spec_from_file_location(INDEX_MOD, INDEX_FILENAME) + webidx = importlib.util.module_from_spec(spec) + sys.modules[INDEX_MOD] = webidx + spec.loader.exec_module(webidx) +else: + cli = imp.load_source(INDEX_MOD, INDEX_FILENAME)