#101 Added CG_Importer tests
Merged 7 years ago by mikem. Opened 7 years ago by jvasallo.
jvasallo/koji master  into  master

file modified
+6 -2
@@ -2660,6 +2660,10 @@ 

      if isinstance(info, int) or isinstance(info, long):

          q="""SELECT id,name FROM %s WHERE id=%%(info)d""" % table

      elif isinstance(info, str):

+         # TODO: Need to make sure to properly sanitize encodings

+         q="""SELECT id,name FROM %s WHERE name=%%(info)s""" % table

+     elif isinstance(info, unicode):

+         info = koji.fixEncoding(info)

          q="""SELECT id,name FROM %s WHERE name=%%(info)s""" % table

      else:

          raise koji.GenericError, 'invalid type for id lookup: %s' % type(info)
@@ -4749,7 +4753,7 @@ 

                  self.raw_metadata = json.dumps(metadata, indent=2)

              except Exception:

                  logger.exception("Cannot encode supplied metadata")

-                 raise koji.GenericError("Invalid metada, cannot encode: %r" % metadata)

+                 raise koji.GenericError("Invalid metadata, cannot encode: %r" % metadata)

              return metadata

          if metadata is None:

              #default to looking for uploaded file
@@ -4762,7 +4766,7 @@ 

              path = os.path.join(workdir, directory, metadata)

              if not os.path.exists(path):

                  raise koji.GenericError("No such file: %s" % metadata)

-             fo = open(path, 'r')

+             fo = open(path, 'rb')

              metadata = fo.read()

              fo.close()

          self.raw_metadata = metadata

@@ -0,0 +1,86 @@ 

+ {"metadata_version": 0,

+  "build": {"name": "rhel-server-docker",

+            "version": "7.1",

+            "release": "4",

+            "source": "git://git.engineering.redhat.com/users/vpavlin/tdl_templates.git#a14f145244",

+            "extra": {},

+            "start_time": 1423148398,

+            "end_time": 1423148828},

+  "buildroots": [{"id": 1,

+                  "host": {"os": "rhel-7",

+                           "arch": "x86_64"},

+                  "content_generator": {"name": "osbs",

+                                        "version": "0.2"},

+                  "container": {"type": "docker",

+                                "arch": "x86_64"},

+                  "tools": [{"name": "docker",

+                             "version": "1.5.0"}],

+                  "components": [{"type": "rpm",

+                                  "name": "glibc",

+                                  "version": "2.17",

+                                  "release": "75.el7",

+                                  "epoch": null,

+                                  "arch": "x86_64",

+                                  "sigmd5": "a1b2c3...",

+                                  "signature": "fd431d51"},

+                                 {"type": "rpm",

+                                  "name": "openssl",

+                                  "version": "1.0.1e",

+                                  "release": "42.el7",

+                                  "epoch": null,

+                                  "arch": "x86_64",

+                                  "sigmd5": "d4e5f6...",

+                                  "signature": "fd431d51"},

+                                 {"type": "rpm",

+                                  "name": "bind-libs",

+                                  "version": "9.9.4",

+                                  "release": "18.el7",

+                                  "epoch": 32,

+                                  "arch": "x86_64",

+                                  "sigmd5": "987abc...",

+                                  "signature": null},

+                                 {"type": "rpm",

+                                  "name": "python-urllib3",

+                                  "version": "1.5",

+                                  "release": "8.el7",

+                                  "epoch": null,

+                                  "arch": "noarch",

+                                  "sigmd5": "123hgf...",

+                                  "signature": null},

+                                 {"type": "file",

+                                  "filename": "jboss-eap-6.3.3-full-build.zip",

+                                  "filesize": 12345678,

+                                  "checksum": "5ec2f29c4e1c2e2aa6552836e236a158",

+                                  "checksum_type": "md5"}],

+                  "extra": {"osbs": {"build_id": 12345,

+                                     "builder_image_id": 67890}}

+                  }],

+  "output": [{"buildroot_id": 1,

+             "filename": "rhel-server-docker-7.1-4.x86_64.tar.xz",

+             "filesize": 34440656,

+             "arch": "x86_64",

+             "checksum_type": "md5",

+             "checksum": "275ae42a45cfedbdb0c0a1acc0b55a1b",

+             "type": "docker-image",

+             "components": "",

+             "extra": {"docker": {"id": "987654...",

+                                  "parent_id": "a1b2c3...",

+                                  "repositories": ["repository.example.com/username/imagename:tagname",

+                                                   "repository.example.com/username/imagename:latest"]}}},

+            {"buildroot_id": 1,

+             "filename": "checkout.log",

+             "filesize": 85724,

+             "arch": "noarch",

+             "checksum_type": "md5",

+             "checksum": "a1b2c3...",

+             "type": "log"},

+            {"buildroot_id": 1,

+             "filename": "os-indirection.log",

+             "filesize": 27189,

+             "arch": "noarch",

+             "checksum_type": "md5",

+             "checksum": "d4f5g6...",

+             "type": "log"}

+            ]

+ 

+ }

@@ -0,0 +1,132 @@ 

+ import unittest

+ import mock

+ import os

+ import shutil

+ 

+ import koji

+ import kojihub

+ from koji import GenericError

+ 

+ 

+ class TestCGImporter(unittest.TestCase):

+     TMP_PATH = os.path.join(os.path.dirname(__file__), 'tmptest')

+ 

+     def setUp(self):

+         if not os.path.exists(self.TMP_PATH):

+             os.mkdir(self.TMP_PATH)

+ 

+     def tearDown(self):

+         if os.path.exists(self.TMP_PATH):

+             shutil.rmtree(self.TMP_PATH)

+ 

+     def test_basic_instantiation(self):

+         kojihub.CG_Importer()  # No exception!

+ 

+     def test_get_metadata_is_instance(self):

+         mock_input_val = {'foo': 'bar'}

+         x = kojihub.CG_Importer()

+         x.get_metadata(mock_input_val, '')

+         assert x.raw_metadata

+         assert isinstance(x.raw_metadata, str)

+ 

+     def test_get_metadata_is_not_instance(self):

+         x = kojihub.CG_Importer()

+         with self.assertRaises(GenericError):

+             x.get_metadata(42, '')

+ 

+     def test_get_metadata_is_none(self):

+         x = kojihub.CG_Importer()

+         with self.assertRaises(GenericError):

+             x.get_metadata(None, '')

+ 

+     @mock.patch("koji.pathinfo.work")

+     def test_get_metadata_missing_json_file(self, work):

+         work.return_value = os.path.dirname(__file__)

+         x = kojihub.CG_Importer()

+         with self.assertRaises(GenericError):

+             x.get_metadata('missing.json', 'cg_importer_json')

+ 

+     @mock.patch("koji.pathinfo.work")

+     def test_get_metadata_is_json_file(self, work):

+         work.return_value = os.path.dirname(__file__)

+         x = kojihub.CG_Importer()

+         x.get_metadata('default.json', 'cg_importer_json')

+         assert x.raw_metadata

+         assert isinstance(x.raw_metadata, str)

+ 

+     @mock.patch('kojihub.context')

+     @mock.patch("koji.pathinfo.work")

+     def test_assert_cg_access(self, work, context):

+         work.return_value = os.path.dirname(__file__)

+         cursor = mock.MagicMock()

+         context.session.user_id = 42

+         context.cnx.cursor.return_value = cursor

+         cursor.fetchall.return_value = [(1, 'foo'), (2, 'bar')]

+         x = kojihub.CG_Importer()

+         x.get_metadata('default.json', 'cg_importer_json')

+         x.assert_cg_access()

+         assert x.cgs

+         assert isinstance(x.cgs, set)

+ 

+     @mock.patch('kojihub.context')

+     @mock.patch("koji.pathinfo.work")

+     def test_prep_build(self, work, context):

+         work.return_value = os.path.dirname(__file__)

+         cursor = mock.MagicMock()

+         context.cnx.cursor.return_value = cursor

+         x = kojihub.CG_Importer()

+         x.get_metadata('default.json', 'cg_importer_json')

+         x.prep_build()

+         assert x.buildinfo

+         assert isinstance(x.buildinfo, dict)

+ 

+     @mock.patch('kojihub.get_build')

+     @mock.patch("koji.pathinfo.work")

+     def test_prep_build_exists(self, work, get_build):

+         work.return_value = os.path.dirname(__file__)

+         get_build.return_value = True

+         x = kojihub.CG_Importer()

+         x.get_metadata('default.json', 'cg_importer_json')

+         with self.assertRaises(GenericError):

+             x.prep_build()

+ 

+     @mock.patch('kojihub.get_build')

+     @mock.patch('kojihub.new_build')

+     @mock.patch('kojihub.context')

+     @mock.patch("koji.pathinfo.work")

+     def test_get_build(self, work, context, new_build_id, get_build):

+         work.return_value = os.path.dirname(__file__)

+         cursor = mock.MagicMock()

+         context.cnx.cursor.return_value = cursor

+         new_build_id.return_value = 42

+         get_build.return_value = False

+         x = kojihub.CG_Importer()

+         x.get_metadata('default.json', 'cg_importer_json')

+         x.prep_build()

+         get_build.return_value = {'id': 43, 'package_id': 1,

+                                   'package_name': 'testpkg',

+                                   'name': 'testpkg', 'version': '1.0.1e',

+                                   'release': '42.el7', 'epoch': None,

+                                   'nvr': 'testpkg-1.0.1-1.fc24',

+                                   'state': 'complete', 'task_id': 1,

+                                   'owner_id': 1, 'owner_name': 'jvasallo',

+                                   'volume_id': 'id-1212', 'volume_name': 'testvolume',

+                                   'creation_event_id': '', 'creation_time': '',

+                                   'creation_ts': 424242424242,

+                                   'start_time': None, 'start_ts': None,

+                                   'completion_time': None, 'completion_ts': None,

+                                   'source': 'https://example.com', 'extra': {}

+                                  }

+         new_build_id.return_value = 43

+         x.get_build()

+         assert x.buildinfo

+         assert isinstance(x.buildinfo, dict)

+ 

+     @mock.patch("koji.pathinfo.build")

+     @mock.patch("koji.pathinfo.work")

+     def test_import_metadata(self, work, build):

+         work.return_value = os.path.dirname(__file__)

+         build.return_value = self.TMP_PATH

+         x = kojihub.CG_Importer()

+         x.get_metadata('default.json', 'cg_importer_json')

+         x.import_metadata()

Working through test cases of CG_Importer, so far so good. Only change I did was adding unicode with @threebean to hub/kojihub.py for lookup_name. Might want to review this in the future for better serialization.

1 new commit added

  • Added test cases for get_build
7 years ago

side note: please be careful about submitting PRs from your master branch. Pagure will auto-update the PR if that branch changes

should we use koji.fixEncoding() here?

Thanks for the feedback mikem. Was a WIP while I was at PyCon this past weekend, forgot to branch off.

Ill look into koji.fixEncoding() as you suggested. This part was a debated part as I was under the impression some refactoring on koji might happen soon. I could also make it a TODO comment until we can determine a long term solution.

A couple minor things
* adds trailing whitespace on two lines
* invalid comment seemingly copied from PR#93

E.g. https://github.com/mikem23/koji-playground/commits/cg-importer-tests

I'm slightly concerned that the 'elif isinstance(info, unicode)' case doesn't show up as covered by the test. I assume you added it because you ran into an issue, so surprised the test case doesn't hit it.

Apologies for the delay on this pull request. I am still playing post PyCon catchup at work. I hope to tiddy up the request tonight! Thanks for the feedback, and yes, I did run into an issue requiring to check for unicode, but ill double check to make sure its tested (if I added it, I'd better make sure its tested).

1 new commit added

  • Leveraging koji.fixEncoding() and cleaned up spacing
7 years ago

Commit f3a8d6e fixes this pull-request

Pull-Request has been merged by mikem@redhat.com

7 years ago