#10 add image diff to changelog
Merged 8 years ago by adamwill. Opened 8 years ago by adamwill.
adamwill/compose-utils image-checks  into  master

@@ -205,12 +205,47 @@ 

                          )

          return result

  

+     def image_id(self, img):

+         """Get a unique identifier for an image. Currently we use a

+         3-tuple (subvariant, type, arch). Should produce a sensible

+         human-readable text when run through " ".join(). Should also

+         be hashable for use as a dict key.

+         """

+         return (img['subvariant'], img['type'], img['arch'])

+ 

+     def get_images(self, compose):

+         """Parse a compose's image list and return a dict whose keys

+         are image identifiers (whatever self.image_id gives) and whose

+         values are the original Image instances `serialize`d into

+         dicts.

+         """

+         imgs = {}

+         imgdict = compose.images.images

+         for variant in imgdict:

+             for arch in imgdict[variant]:

+                 for img in imgdict[variant][arch]:

+                     # the way Image.serialize works is a bit odd...

+                     parser = []

+                     img.serialize(parser)

+                     img = parser[0]

+                     imgs[self.image_id(img)] = img

+         return imgs

+ 

      def get_changelog(self, old_compose, new_compose, max_logs=-1):

          result = {}

  

          result["old_compose"] = old_compose.rpms.compose.id

          result["new_compose"] = new_compose.rpms.compose.id

  

+         imgs_old = self.get_images(old_compose)

+         imgs_new = self.get_images(new_compose)

+         imgs_old_ids = set(imgs_old.keys())

+         imgs_new_ids = set(imgs_new.keys())

+         result["added_images"] = [imgs_new[imid] for imid in

+                                   imgs_new_ids - imgs_old_ids]

+         result["dropped_images"] = [imgs_old[imid] for imid in

+                                     imgs_old_ids - imgs_new_ids]

+ 

          srpms_old = self.get_srpms(old_compose)

          srpms_new = self.get_srpms(new_compose)

  
@@ -235,6 +270,8 @@ 

          pkgs_changed = set(rpm_headers_old) & set(rpm_headers_new)

  

          result["summary"] = {

+             "added_images": len(result["added_images"]),

+             "dropped_images": len(result["dropped_images"]),

              "added_packages": 0,

              "dropped_packages": 0,

              "upgraded_packages": 0,
@@ -321,6 +358,8 @@ 

      def _get_summary(self, changelog_data):

          result = []

          result.append("===== SUMMARY =====")

+         result.append("Added images:        %s" % changelog_data["summary"]["added_images"])

+         result.append("Dropped images:      %s" % changelog_data["summary"]["dropped_images"])

          result.append("Added packages:      %s" % changelog_data["summary"]["added_packages"])

          result.append("Dropped packages:    %s" % changelog_data["summary"]["dropped_packages"])

          result.append("Upgraded packages:   %s" % changelog_data["summary"]["upgraded_packages"])
@@ -347,6 +386,16 @@ 

  

          result.extend(self._get_summary(changelog_data))

  

+         result.append("===== ADDED IMAGES =====")

+         for i in changelog_data["added_images"]:

+             result.append(" ".join(self.image_id(i)))

+         result.append("")

+ 

+         result.append("===== DROPPED IMAGES =====")

+         for i in changelog_data["dropped_images"]:

+             result.append(" ".join(self.image_id(i)))

+         result.append("")

+ 

          result.append("===== ADDED PACKAGES =====")

          for i in changelog_data["added_packages"]:

              result.append("%(name)s: %(nvr)s" % i)
@@ -377,6 +426,18 @@ 

  

          result.extend(self._get_summary(changelog_data))

  

+         result.append("===== ADDED IMAGES =====")

+         for i in changelog_data["added_images"]:

+             result.append("Image: %s" % " ".join(self.image_id(i)))

+             result.append("Path: %s" % i['path'])

+         result.append("")

+ 

+         result.append("===== DROPPED IMAGES =====")

+         for i in changelog_data["dropped_images"]:

+             result.append("Image: %s" % " ".join(self.image_id(i)))

+             result.append("Path: %s" % i['path'])

+         result.append("")

+ 

          result.append("===== ADDED PACKAGES =====")

          for i in changelog_data["added_packages"]:

              result.append("Package: %s" % i["nvr"])

@@ -28,7 +28,8 @@ 

                          "path": "Client/i386/iso/DP-1.0-20160315.t.0-Client-i386-dvd1.iso", 

                          "size": 507904, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Client.i386"

+                         "volume_id": "DP-1.0 Client.i386",

+                         "subvariant": "Client"

                      }

                  ], 

                  "src": [
@@ -48,7 +49,8 @@ 

                          "path": "Client/source/iso/DP-1.0-20160315.t.0-Client-source-dvd1.iso", 

                          "size": 473088, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Client.src"

+                         "volume_id": "DP-1.0 Client.src",

+                         "subvariant": "Client"

                      }

                  ], 

                  "x86_64": [
@@ -68,31 +70,12 @@ 

                          "path": "Client/x86_64/iso/DP-1.0-20160315.t.0-Client-x86_64-dvd1.iso", 

                          "size": 516096, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Client.x86_64"

+                         "volume_id": "DP-1.0 Client.x86_64",

+                         "subvariant": "Client"

                      }

                  ]

              }, 

              "Server": {

-                 "s390x": [
ausil commented 8 years ago

Why did you remove this?

-                     {

-                         "arch": "s390x", 

-                         "bootable": false, 

-                         "checksums": {

-                             "md5": "bcc53aed2cd8ada9a5a346d7856cc23d", 

-                             "sha1": "d462e78474411c78ae569a940ba6eb2e9132fa69", 

-                             "sha256": "54f25d0d3bf06f2e752746d310271f42e6680cf9262feddf9eb3384bb7dcaf8f"

-                         }, 

-                         "disc_count": 1, 

-                         "disc_number": 1, 

-                         "format": "iso", 

-                         "implant_md5": "f87289c658389c0457c6306d6aaaaf11", 

-                         "mtime": 1458031335, 

-                         "path": "Server/s390x/iso/DP-1.0-20160315.t.0-Server-s390x-dvd1.iso", 

-                         "size": 493568, 

-                         "type": "dvd", 

-                         "volume_id": "DP-1.0 Server.s390x"

-                     }

-                 ], 

                  "src": [

                      {

                          "arch": "src", 
@@ -110,7 +93,8 @@ 

                          "path": "Server/source/iso/DP-1.0-20160315.t.0-Server-source-dvd1.iso", 

                          "size": 509952, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Server.src"

+                         "volume_id": "DP-1.0 Server.src",

+                         "subvariant": "Server"

                      }

                  ], 

                  "x86_64": [
@@ -130,10 +114,11 @@ 

                          "path": "Server/x86_64/iso/DP-1.0-20160315.t.0-Server-x86_64-dvd1.iso", 

                          "size": 577536, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Server.x86_64"

+                         "volume_id": "DP-1.0 Server.x86_64",

+                         "subvariant": "Server"

                      }

                  ]

              }

          }

      }

- } 

\ No newline at end of file

+ }

@@ -28,7 +28,8 @@ 

                          "path": "Client/i386/iso/DP-1.0-20160315.t.1-Client-i386-dvd1.iso", 

                          "size": 507904, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Client.i386"

+                         "volume_id": "DP-1.0 Client.i386",

+                         "subvariant": "Client"

                      }

                  ], 

                  "src": [
@@ -48,27 +49,8 @@ 

                          "path": "Client/source/iso/DP-1.0-20160315.t.1-Client-source-dvd1.iso", 

                          "size": 475136, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Client.src"

-                     }

-                 ], 

-                 "x86_64": [
ausil commented 8 years ago

Why did we remove x86_64 here?

-                     {

-                         "arch": "x86_64", 

-                         "bootable": false, 

-                         "checksums": {

-                             "md5": "ca40a2f5ce5cf2e930dde4031b16428c", 

-                             "sha1": "69c0b27e3998a93c19f32fe77afcf547c70227fc", 

-                             "sha256": "2ec61698ed2f4fde41a5253ed0d0afaaad495ae4dba2fc974c1abfcd77dd7358"

-                         }, 

-                         "disc_count": 1, 

-                         "disc_number": 1, 

-                         "format": "iso", 

-                         "implant_md5": "6dee6f24138cd7feb1223e78dc62e72d", 

-                         "mtime": 1458031712, 

-                         "path": "Client/x86_64/iso/DP-1.0-20160315.t.1-Client-x86_64-dvd1.iso", 

-                         "size": 516096, 

-                         "type": "dvd", 

-                         "volume_id": "DP-1.0 Client.x86_64"

+                         "volume_id": "DP-1.0 Client.src",

+                         "subvariant": "Client"

                      }

                  ]

              }, 
@@ -90,7 +72,8 @@ 

                          "path": "Server/s390x/iso/DP-1.0-20160315.t.1-Server-s390x-dvd1.iso", 

                          "size": 493568, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Server.s390x"

+                         "volume_id": "DP-1.0 Server.s390x",

+                         "subvariant": "Server"

                      }

                  ], 

                  "src": [
@@ -110,7 +93,8 @@ 

                          "path": "Server/source/iso/DP-1.0-20160315.t.1-Server-source-dvd1.iso", 

                          "size": 509952, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Server.src"

+                         "volume_id": "DP-1.0 Server.src",

+                         "subvariant": "Server"

                      }

                  ], 

                  "x86_64": [
@@ -130,10 +114,11 @@ 

                          "path": "Server/x86_64/iso/DP-1.0-20160315.t.1-Server-x86_64-dvd1.iso", 

                          "size": 577536, 

                          "type": "dvd", 

-                         "volume_id": "DP-1.0 Server.x86_64"

+                         "volume_id": "DP-1.0 Server.x86_64",

+                         "subvariant": "Server"

                      }

                  ]

              }

          }

      }

- } 

\ No newline at end of file

+ }

file modified
+43
@@ -78,7 +78,47 @@ 

          self.assertItemsEqual(data['dropped_packages'], [DUMMY_TFTP])

          self.assertItemsEqual(data['upgraded_packages'], [DUMMY_FIREFOX])

          self.assertItemsEqual(data['downgraded_packages'], [])

+         self.assertDictEqual(data['added_images'][0], {

+             "arch": "s390x", 

+             "bootable": False, 

+             "checksums": {

+                 "md5": "2d0a0e02d6a2225fba2c4cdf41d45ff7", 

+                 "sha1": "8788b0aa4974440f7ca12c5dd0cc6dfc78314fdd", 

+                 "sha256": "1b756cddb139e57fd6a07af72a7d52a9dc01c5fb0b9d37ef16fdf73a1a0a19af"

+             }, 

+             "disc_count": 1, 

+             "disc_number": 1, 

+             "format": "iso", 

+             "implant_md5": "0a2fc4f0e6f926c860e5df6c5ff0df32", 

+             "mtime": 1458031712, 

+             "path": "Server/s390x/iso/DP-1.0-20160315.t.1-Server-s390x-dvd1.iso", 

+             "size": 493568, 

+             "type": "dvd", 

+             "volume_id": "DP-1.0 Server.s390x",

+             "subvariant": "Server"

+         })

+         self.assertDictEqual(data['dropped_images'][0], {

+             "arch": "x86_64", 

+             "bootable": False, 

+             "checksums": {

+                 "md5": "7ea0d315e4b51006eb7c1de3da763c55", 

+                 "sha1": "268bf671d5e62f35b70a2db5b1198b861d4b0eb8", 

+                 "sha256": "32fedd333df5a0c78f3dd8c2634f8a4ab2328c7e0c9904dd06ac865fa723134f"

+             }, 

+             "disc_count": 1, 

+             "disc_number": 1, 

+             "format": "iso", 

+             "implant_md5": "128abd11392887a8abb2bbf119889791", 

+             "mtime": 1458031335, 

+             "path": "Client/x86_64/iso/DP-1.0-20160315.t.0-Client-x86_64-dvd1.iso", 

+             "size": 516096, 

+             "type": "dvd", 

+             "volume_id": "DP-1.0 Client.x86_64",

+             "subvariant": "Client"

+         })

          self.assertDictEqual(data['summary'], {

+             'added_images': 1,

+             'dropped_images': 1,

              'added_packages': 1,

              'added_packages_size': DUMMY_ELINKS_SIZE,

              'upgraded_packages': 1,
@@ -90,3 +130,6 @@ 

              'downgraded_packages_size': 0,

              'downgraded_packages_size_change': 0,

          })

+ 

+ if __name__ == '__main__':

+     unittest.main()

no initial comment

this is a port of something currently done in my 'check-compose'
tool which produces the 'compose check report' emails. I think
it makes sense to do it here instead. It's pretty simple: we
consider the 3-tuple (subvariant, type, arch) for an image to
be a unique identifier for the image, and we compare the images
in each compose and produce 'added_images' and 'dropped_images'
from sets of the identifiers. We use the same identifier run
through " ".join() as the human-readable identifier and include
just a count of added/removed images in the summary, the id for
each added and removed image in the brief log, and the id plus
the path for each added and removed image in the verbose log.

I tried to future proof it a bit, in case we need to change the
identifier format at any point, the idea is that we can just
change the image_id method and the rest of the code should
keep working. At first I just got the tuples and stored those
as added_images and dropped_images, but then it's a bit messy
in future if we change the identifier format. It's also set up
such that we can also add check-compose's 'expected images'
feature quite easily using the get_images dict (planning that
as a separate PR).

Why? Well, so there's an image diff to test =) The image lists in the test metadata prior to this commit are identical, so they're useless for testing the image diff feature because there's no difference. In order to test it I have to remove one image from each compose, so there's one 'added' image and one 'removed' image when you diff one against the other.

The existing tests don't use the image data at all, I don't think, it just had to be there to prevent productmd from choking.

ah hum, just found a problem with this that the tests don't catch but running the real binary on the test data does...TypeError: <productmd.images.Image object at 0x7f6c63315c50> is not JSON serializable. I'll figure a way to deal with that.

Pull-Request has been rebased

8 years ago

Pull-Request has been rebased

8 years ago

OK, that solves the serialization problem by serializing the Image objects. Now the tests work and running bin/compose-changelog -p /tmp tests/composes/DP-1.0-20160315.t.0/ tests/composes/DP-1.0-20160315.t.1/ works too.

Looks good. Test case passes for me and when testing on actual data I also get expected results.

Commit d52446e fixes this pull-request

Pull-Request has been merged by awilliam@redhat.com

8 years ago

For the record I have merged this outside of pagure (I fixed some trailing whitespace in tests).

For the record I have merged this outside of pagure (I fixed some trailing whitespace in tests).

It just occurred to me that I didn't conditionalize anything for the case where there is no image diff. It shouldn't break anything but the report might look a bit weird? I guess we'll see when it happens how the report looks and we can tweak it if necessary.

Commit d52446e fixes this pull-request

Pull-Request has been merged by awilliam@redhat.com

8 years ago