From 978afc5123dffea388205f18e5d716cf936fe06f Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 00:19:34 +0000 Subject: [PATCH 1/10] Added base test case for CG_Importer --- diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py new file mode 100644 index 0000000..004d2b7 --- /dev/null +++ b/tests/test_hub/test_cg_importer.py @@ -0,0 +1,11 @@ +import unittest +import mock + +import kojihub + + +class TestCGImporter(unittest.TestCase): + def test_basic_instantiation(self): + # TODO -- this doesn't make sense. A query with no arguments should + # probably raise an exception saying "this doesn't make sense." + kojihub.CG_Importer() # No exception! From 47641fdbb55e1fc61c9023a3a85708617063ac3d Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 02:02:04 +0000 Subject: [PATCH 2/10] Added tests for get_metadata - Fixed Typo --- diff --git a/hub/kojihub.py b/hub/kojihub.py index c88946c..7a21f31 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4749,7 +4749,7 @@ class CG_Importer(object): self.raw_metadata = json.dumps(metadata, indent=2) except Exception: logger.exception("Cannot encode supplied metadata") - raise koji.GenericError("Invalid metada, cannot encode: %r" % metadata) + raise koji.GenericError("Invalid metadata, cannot encode: %r" % metadata) return metadata if metadata is None: #default to looking for uploaded file diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 004d2b7..9b6a4a6 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -2,6 +2,7 @@ import unittest import mock import kojihub +from koji import GenericError class TestCGImporter(unittest.TestCase): @@ -9,3 +10,15 @@ class TestCGImporter(unittest.TestCase): # TODO -- this doesn't make sense. A query with no arguments should # probably raise an exception saying "this doesn't make sense." kojihub.CG_Importer() # No exception! + + def test_get_metadata_is_instance(self): + mock_input_val = {'something': 'good val'} + x = kojihub.CG_Importer() + x.get_metadata(mock_input_val, '') + assert x.raw_metadata + assert isinstance(x.raw_metadata, str) + + def test_get_metadata_is_not_instance(self): + x = kojihub.CG_Importer() + with self.assertRaises(GenericError): + x.get_metadata(42, '') From 6f94cb034a5df8bb89daeb1fe0d3c369aa6cd9ba Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 06:29:24 +0000 Subject: [PATCH 3/10] Added default.json and finished get_metadata tests --- diff --git a/tests/test_hub/cg_importer_json/default.json b/tests/test_hub/cg_importer_json/default.json new file mode 100644 index 0000000..7a350e8 --- /dev/null +++ b/tests/test_hub/cg_importer_json/default.json @@ -0,0 +1,85 @@ +{"metadata_version": 1, + "build": {"name": "rhel-server-docker", + "version": "7.1", + "release": "4", + "source": "git://git.engineering.redhat.com/users/vpavlin/tdl_templates.git#a14f145244", + "start_time": 1423148398, + "end_time": 1423148828}, + "buildroots": [{"id": 1, + "host": {"os": "rhel-7", + "arch": "x86_64"}, + "content_generator": {"name": "osbs", + "version": "0.2"}, + "container": {"type": "docker", + "arch": "x86_64"}, + "tools": [{"name": "docker", + "version": "1.5.0"}], + "components": [{"type": "rpm", + "name": "glibc", + "version": "2.17", + "release": "75.el7", + "epoch": null, + "arch": "x86_64", + "sigmd5": "a1b2c3...", + "signature": "fd431d51"}, + {"type": "rpm", + "name": "openssl", + "version": "1.0.1e", + "release": "42.el7", + "epoch": null, + "arch": "x86_64", + "sigmd5": "d4e5f6...", + "signature": "fd431d51"}, + {"type": "rpm", + "name": "bind-libs", + "version": "9.9.4", + "release": "18.el7", + "epoch": 32, + "arch": "x86_64", + "sigmd5": "987abc...", + "signature": null}, + {"type": "rpm", + "name": "python-urllib3", + "version": "1.5", + "release": "8.el7", + "epoch": null, + "arch": "noarch", + "sigmd5": "123hgf...", + "signature": null}, + {"type": "file", + "filename": "jboss-eap-6.3.3-full-build.zip", + "filesize": 12345678, + "checksum": "5ec2f29c4e1c2e2aa6552836e236a158", + "checksum_type": "md5"}], + "extra": {"osbs": {"build_id": 12345, + "builder_image_id": 67890}} + }], + "output": [{"buildroot_id": 1, + "filename": "rhel-server-docker-7.1-4.x86_64.tar.xz", + "filesize": 34440656, + "arch": "x86_64", + "checksum_type": "md5", + "checksum": "275ae42a45cfedbdb0c0a1acc0b55a1b", + "type": "docker-image", + "components": "", + "extra": {"docker": {"id": "987654...", + "parent_id": "a1b2c3...", + "repositories": ["repository.example.com/username/imagename:tagname", + "repository.example.com/username/imagename:latest"]}}}, + {"buildroot_id": 1, + "filename": "checkout.log", + "filesize": 85724, + "arch": "noarch", + "checksum_type": "md5", + "checksum": "a1b2c3...", + "type": "log"}, + {"buildroot_id": 1, + "filename": "os-indirection.log", + "filesize": 27189, + "arch": "noarch", + "checksum_type": "md5", + "checksum": "d4f5g6...", + "type": "log"} + ] + +} diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 9b6a4a6..9132bf9 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -1,6 +1,8 @@ import unittest import mock +import os +import koji import kojihub from koji import GenericError @@ -22,3 +24,21 @@ class TestCGImporter(unittest.TestCase): x = kojihub.CG_Importer() with self.assertRaises(GenericError): x.get_metadata(42, '') + + def test_get_metadata_is_none(self): + x = kojihub.CG_Importer() + with self.assertRaises(GenericError): + x.get_metadata(None, '') + + @mock.patch("koji.pathinfo.work") + def test_get_metadata_missing_json_file(self, work): + work.return_value = os.path.dirname(__file__) + x = kojihub.CG_Importer() + with self.assertRaises(GenericError): + x.get_metadata('missing.json', 'cg_importer_json') + + @mock.patch("koji.pathinfo.work") + def test_get_metadata_is_json_file(self, work): + work.return_value = os.path.dirname(__file__) + x = kojihub.CG_Importer() + x.get_metadata('default.json', 'cg_importer_json') From 22271c691363f571f362fdacc31637ad0c923895 Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 07:06:54 +0000 Subject: [PATCH 4/10] Cleaned up test cases for get_metadata --- diff --git a/tests/test_hub/cg_importer_json/default.json b/tests/test_hub/cg_importer_json/default.json index 7a350e8..63d47ae 100644 --- a/tests/test_hub/cg_importer_json/default.json +++ b/tests/test_hub/cg_importer_json/default.json @@ -1,4 +1,4 @@ -{"metadata_version": 1, +{"metadata_version": 0, "build": {"name": "rhel-server-docker", "version": "7.1", "release": "4", diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 9132bf9..3b58e0a 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -14,7 +14,7 @@ class TestCGImporter(unittest.TestCase): kojihub.CG_Importer() # No exception! def test_get_metadata_is_instance(self): - mock_input_val = {'something': 'good val'} + mock_input_val = {'foo': 'bar'} x = kojihub.CG_Importer() x.get_metadata(mock_input_val, '') assert x.raw_metadata @@ -42,3 +42,5 @@ class TestCGImporter(unittest.TestCase): work.return_value = os.path.dirname(__file__) x = kojihub.CG_Importer() x.get_metadata('default.json', 'cg_importer_json') + assert x.raw_metadata + assert isinstance(x.raw_metadata, str) From e0ee4e3d26ec8f69f8d4f964ab79593c6af7582c Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 22:15:06 +0000 Subject: [PATCH 5/10] Added tests for assert_cg_access --- diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 3b58e0a..c811878 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -44,3 +44,17 @@ class TestCGImporter(unittest.TestCase): x.get_metadata('default.json', 'cg_importer_json') assert x.raw_metadata assert isinstance(x.raw_metadata, str) + + @mock.patch('kojihub.context') + @mock.patch("koji.pathinfo.work") + def test_assert_cg_access(self, work, context): + work.return_value = os.path.dirname(__file__) + cursor = mock.MagicMock() + context.session.user_id = 42 + context.cnx.cursor.return_value = cursor + cursor.fetchall.return_value = [(1, 'foo'), (2, 'bar')] + x = kojihub.CG_Importer() + x.get_metadata('default.json', 'cg_importer_json') + x.assert_cg_access() + assert x.cgs + assert isinstance(x.cgs, set) From 6263fb235d11e9ad739085e87bfcad871d870b27 Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 22:15:19 +0000 Subject: [PATCH 6/10] Added support for unicode in lookup_name - Need to make sure to sanitize this properly in the future. - read metadata as binary to be explicit --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 7a21f31..fd82c12 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2660,6 +2660,10 @@ def lookup_name(table,info,strict=False,create=False): if isinstance(info, int) or isinstance(info, long): q="""SELECT id,name FROM %s WHERE id=%%(info)d""" % table elif isinstance(info, str): + # TODO: Need to make sure to properly sanitize encodings + q="""SELECT id,name FROM %s WHERE name=%%(info)s""" % table + elif isinstance(info, unicode): + info = info.encode('UTF-8') q="""SELECT id,name FROM %s WHERE name=%%(info)s""" % table else: raise koji.GenericError, 'invalid type for id lookup: %s' % type(info) @@ -4762,7 +4766,7 @@ class CG_Importer(object): path = os.path.join(workdir, directory, metadata) if not os.path.exists(path): raise koji.GenericError("No such file: %s" % metadata) - fo = open(path, 'r') + fo = open(path, 'rb') metadata = fo.read() fo.close() self.raw_metadata = metadata From c5b9056e3a8c602a1889aafaba883994fccd5b07 Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 22:54:12 +0000 Subject: [PATCH 7/10] Added test for CG_Importer test_build --- diff --git a/tests/test_hub/cg_importer_json/default.json b/tests/test_hub/cg_importer_json/default.json index 63d47ae..c03c78f 100644 --- a/tests/test_hub/cg_importer_json/default.json +++ b/tests/test_hub/cg_importer_json/default.json @@ -3,6 +3,7 @@ "version": "7.1", "release": "4", "source": "git://git.engineering.redhat.com/users/vpavlin/tdl_templates.git#a14f145244", + "extra": {}, "start_time": 1423148398, "end_time": 1423148828}, "buildroots": [{"id": 1, diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index c811878..5027793 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -58,3 +58,16 @@ class TestCGImporter(unittest.TestCase): x.assert_cg_access() assert x.cgs assert isinstance(x.cgs, set) + + @mock.patch('kojihub.context') + @mock.patch("koji.pathinfo.work") + def test_prep_build(self, work, context): + work.return_value = os.path.dirname(__file__) + cursor = mock.MagicMock() + context.cnx.cursor.return_value = cursor + #cursor.fetchall.return_value = [(1, 'foo'), (2, 'bar')] + x = kojihub.CG_Importer() + x.get_metadata('default.json', 'cg_importer_json') + x.prep_build() + assert x.buildinfo + assert isinstance(x.buildinfo, dict) From 29433862ce5f5ddb7bc67a14fbdbca9f9dadcde1 Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 03 2016 23:56:24 +0000 Subject: [PATCH 8/10] Added more CG_Importer unit tests --- diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 5027793..641ba14 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -1,6 +1,7 @@ import unittest import mock import os +import shutil import koji import kojihub @@ -8,6 +9,16 @@ from koji import GenericError class TestCGImporter(unittest.TestCase): + TMP_PATH = os.path.join(os.path.dirname(__file__), 'tmptest') + + def setUp(self): + if not os.path.exists(self.TMP_PATH): + os.mkdir(self.TMP_PATH) + + def tearDown(self): + if os.path.exists(self.TMP_PATH): + shutil.rmtree(self.TMP_PATH) + def test_basic_instantiation(self): # TODO -- this doesn't make sense. A query with no arguments should # probably raise an exception saying "this doesn't make sense." @@ -65,9 +76,27 @@ class TestCGImporter(unittest.TestCase): work.return_value = os.path.dirname(__file__) cursor = mock.MagicMock() context.cnx.cursor.return_value = cursor - #cursor.fetchall.return_value = [(1, 'foo'), (2, 'bar')] x = kojihub.CG_Importer() x.get_metadata('default.json', 'cg_importer_json') x.prep_build() assert x.buildinfo assert isinstance(x.buildinfo, dict) + + @mock.patch('kojihub.get_build') + @mock.patch("koji.pathinfo.work") + def test_prep_build_exists(self, work, get_build): + work.return_value = os.path.dirname(__file__) + get_build.return_value = True + x = kojihub.CG_Importer() + x.get_metadata('default.json', 'cg_importer_json') + with self.assertRaises(GenericError): + x.prep_build() + + @mock.patch("koji.pathinfo.build") + @mock.patch("koji.pathinfo.work") + def test_import_metadata(self, work, build): + work.return_value = os.path.dirname(__file__) + build.return_value = self.TMP_PATH + x = kojihub.CG_Importer() + x.get_metadata('default.json', 'cg_importer_json') + x.import_metadata() From b746012fb42f4ada8c03d7325c686fa30d3dd3ad Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 04 2016 22:46:45 +0000 Subject: [PATCH 9/10] Added test cases for get_build --- diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 641ba14..9c94899 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -92,6 +92,38 @@ class TestCGImporter(unittest.TestCase): with self.assertRaises(GenericError): x.prep_build() + @mock.patch('kojihub.get_build') + @mock.patch('kojihub.new_build') + @mock.patch('kojihub.context') + @mock.patch("koji.pathinfo.work") + def test_get_build(self, work, context, new_build_id, get_build): + work.return_value = os.path.dirname(__file__) + cursor = mock.MagicMock() + context.cnx.cursor.return_value = cursor + new_build_id.return_value = 42 + get_build.return_value = False + x = kojihub.CG_Importer() + x.get_metadata('default.json', 'cg_importer_json') + x.prep_build() + get_build.return_value = {'id': 43, 'package_id': 1, + 'package_name': 'testpkg', + 'name': 'testpkg', 'version': '1.0.1e', + 'release': '42.el7', 'epoch': None, + 'nvr': 'testpkg-1.0.1-1.fc24', + 'state': 'complete', 'task_id': 1, + 'owner_id': 1, 'owner_name': 'jvasallo', + 'volume_id': 'id-1212', 'volume_name': 'testvolume', + 'creation_event_id': '', 'creation_time': '', + 'creation_ts': 424242424242, + 'start_time': None, 'start_ts': None, + 'completion_time': None, 'completion_ts': None, + 'source': 'https://example.com', 'extra': {} + } + new_build_id.return_value = 43 + x.get_build() + assert x.buildinfo + assert isinstance(x.buildinfo, dict) + @mock.patch("koji.pathinfo.build") @mock.patch("koji.pathinfo.work") def test_import_metadata(self, work, build): From aa37742d9122ce09ee073be0a2bb40d86b5b2040 Mon Sep 17 00:00:00 2001 From: Joel Vasallo Date: Jun 11 2016 04:36:49 +0000 Subject: [PATCH 10/10] Leveraging koji.fixEncoding() and cleaned up spacing --- diff --git a/hub/kojihub.py b/hub/kojihub.py index fd82c12..a5356d7 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2663,7 +2663,7 @@ def lookup_name(table,info,strict=False,create=False): # TODO: Need to make sure to properly sanitize encodings q="""SELECT id,name FROM %s WHERE name=%%(info)s""" % table elif isinstance(info, unicode): - info = info.encode('UTF-8') + info = koji.fixEncoding(info) q="""SELECT id,name FROM %s WHERE name=%%(info)s""" % table else: raise koji.GenericError, 'invalid type for id lookup: %s' % type(info) diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 9c94899..896f465 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -10,7 +10,7 @@ from koji import GenericError class TestCGImporter(unittest.TestCase): TMP_PATH = os.path.join(os.path.dirname(__file__), 'tmptest') - + def setUp(self): if not os.path.exists(self.TMP_PATH): os.mkdir(self.TMP_PATH) @@ -20,8 +20,6 @@ class TestCGImporter(unittest.TestCase): shutil.rmtree(self.TMP_PATH) def test_basic_instantiation(self): - # TODO -- this doesn't make sense. A query with no arguments should - # probably raise an exception saying "this doesn't make sense." kojihub.CG_Importer() # No exception! def test_get_metadata_is_instance(self): @@ -131,4 +129,4 @@ class TestCGImporter(unittest.TestCase): build.return_value = self.TMP_PATH x = kojihub.CG_Importer() x.get_metadata('default.json', 'cg_importer_json') - x.import_metadata() + x.import_metadata()