#6544 Fix push-two-week-atomic.py to handle new AutoCloud data format
Merged 7 years ago by ausil. Opened 7 years ago by maxamillion.
maxamillion/releng fix-two-week  into  master

file modified
+95 -182
@@ -60,7 +60,6 @@ 

  # Full path will be:

  #   /pub/alt/stage/$VERSION-$DATE/$IMAGE_TYPE/x86_64/[Images|os]/

  # http://dl.fedoraproject.org/pub/alt/atomic/stable/

- ATOMIC_LINKDEST = "/pub/alt/atomic/testing/"

  ATOMIC_STABLE_BASEDIR = "/pub/alt/atomic/stable/"

  

  # the modname gets used to construct the fully qualified topic, like
@@ -102,7 +101,7 @@ 

  # delta = 2 weeks in seconds

  DATAGREPPER_DELTA = 1209600

  # category to filter on from datagrepper

- DATAGREPPER_TOPIC = "org.fedoraproject.prod.autocloud.image.success"

+ DATAGREPPER_TOPIC = "org.fedoraproject.prod.autocloud.compose.complete"

  

  

  SIGUL_SIGNED_TXT_PATH = "/tmp/signed"
@@ -116,15 +115,15 @@ 

  

      Takes an autocloud fedmsg message and returns the image name and final url.

      """

-     dest_dir = os.path.join(

-         ATOMIC_STABLE_BASEDIR,

-         msg[u'msg'][u'compose_id'],

-         'CloudImages/x86_64/images/'

-     )

-     image_name = msg[u'msg'][u'compose_url'].split('/')[-1]

-     image_url = os.path.join(dest_dir, image_name)

-     return image_name, image_url

+     iul = msg[u'image_url'].split('/')

+ 

+     # This isn't used in the path for the destination dir, it's in there twice

+     iul.remove('compose')

+     iul.remove('compose')

  

+     image_name = iul[-1]

+     image_url = os.path.join(ATOMIC_STABLE_BASEDIR, '/'.join(iul[4:]))

+     return image_name, image_url

  

  def get_latest_successful_autocloud_test_info(

          release,
@@ -161,177 +160,92 @@ 

              params=dict(page=rpage, **request_params)

          ).json()[u'raw_messages']

  

-     # FIXME - I would like to find a good way to extract the types from the

-     #         datagrepper query instead of specifying each artifact

-     atomic_qcow2 = [

-         s for s in autocloud_data

-         if s[u'msg'][u'status'] == u'success'

-         and fnmatch.fnmatch(s[u'msg'][u'image_name'], u'Fedora-Atomic-*')

-         and s[u'msg'][u'type'] == u"qcow2"

-         and not compose_manually_marked_bad(s[u'msg'][u'compose_id'])

-     ]

  

-     atomic_vagrant_libvirt = [

-         s for s in autocloud_data

-         if s[u'msg'][u'status'] == u'success'

-         and fnmatch.fnmatch(s[u'msg'][u'image_name'], u'Fedora-Atomic-*')

-         and s[u'msg'][u'type'] == u"vagrant-libvirt"

-         and not compose_manually_marked_bad(s[u'msg'][u'compose_id'])

+     # List comprehension that will return a list of compose information from

+     # AutoCloud (the [u'msg'] payload of autocloud.compose.complete fedmsg)

+     # such that the following criteria are true:

+     #

+     #   - Is an Atomic compose (i.e. 'Atomic' is in the compose id)

+     #   - No compose artifacts failed the tests

+     #   - This is the current Fedora release we want

+     #

+     #   OR:

+     #       - This compose was manually marked good

+     candidate_composes = [

+         compose[u'msg'] for compose in autocloud_data

+             if u'Atomic' in compose[u'msg'][u'id']

+                 and compose[u'msg'][u'results'][u'failed'] == 0

+                 and compose[u'msg'][u'release'] == str(release)

+                 or compose[u'msg'][u'id'] in MARK_ATOMIC_GOOD_COMPOSES

      ]

  

-     atomic_vagrant_vbox = [

-         s for s in autocloud_data

-         if s[u'msg'][u'status'] == u'success'

-         and fnmatch.fnmatch(s[u'msg'][u'image_name'], u'Fedora-Atomic-*')

-         and s[u'msg'][u'type'] == u"vagrant-virtualbox"

-         and not compose_manually_marked_bad(s[u'msg'][u'compose_id'])

-     ]

+     filtered_composes = list(candidate_composes)

+     for compose in candidate_composes:

+         if compose_manually_marked_bad(compose[u'id']):

+             filtered_composes.remove(compose)

  

-     # HACK

-     #

-     # Handle manually marked "good" builds that were false positives in

-     # AutoCloud

-     if MARK_ATOMIC_GOOD_COMPOSES:

-         release_cycle_time = time.time() - DATAGREPPER_DELTA

-         atomic_qcow2_failed = [

-             s for s in autocloud_data

-             if s[u'msg'][u'status'] == u'failed'

-             and fnmatch.fnmatch(s[u'msg'][u'image_name'], u'Fedora-Atomic-*')

-             and s[u'msg'][u'type'] == u"qcow2"

-             and not compose_manually_marked_bad(s[u'msg'][u'compose_id'])

-         ]

+     # sc = successful compose

+     sc = filtered_composes[0]

  

-         atomic_vagrant_libvirt_failed = [

-             s for s in autocloud_data

-             if s[u'msg'][u'status'] == u'failed'

-             and fnmatch.fnmatch(s[u'msg'][u'image_name'], u'Fedora-Atomic-*')

-             and s[u'msg'][u'type'] == u"vagrant-libvirt"

-             and not compose_manually_marked_bad(s[u'msg'][u'compose_id'])

-         ]

+     autocloud_info = {}

  

-         atomic_vagrant_vbox_failed = [

-             s for s in autocloud_data

-             if s[u'msg'][u'status'] == u'failed'

-             and fnmatch.fnmatch(s[u'msg'][u'image_name'], u'Fedora-Atomic-*')

-             and s[u'msg'][u'type'] == u"vagrant-virtualbox"

-             and not compose_manually_marked_bad(s[u'msg'][u'compose_id'])

-         ]

+     # qcow2 image

+     qcow_msg = [

+         sc[u'results'][u'artifacts'][img] for img in sc[u'results'][u'artifacts']

+             if sc[u'results'][u'artifacts'][img][u'family'] == u'Atomic'

+             and sc[u'results'][u'artifacts'][img][u'type'] == u'qcow2'

+     ][0]

+     image_name, image_url = construct_url(qcow_msg)

+     autocloud_info["atomic_qcow2"] = {

+         "compose_id": sc[u'id'],

+         "name": qcow_msg[u'name'],

+         "release": sc[u'release'],

+         "image_name": image_name,

+         "image_url": image_url,

+     }

  

-         # Define some helpers to deal with some of this badness

-         # Thanks to Ralph(threebean) for the suggestion/patch to try and

-         # bring sanity to chaos

-         url2image = lambda s: s.split('/')[-1]

-         image2release = lambda s: int(s.split('.')[0].split('-')[-1])

- 

-         for good_compose in MARK_ATOMIC_GOOD_COMPOSES:

-             good_compose_date = str(image2release(good_compose))

-             release_window = datetime.datetime.strptime(good_compose_date, "%Y%m%d")

- 

-             if time.mktime(release_window.timetuple()) > release_cycle_time:

- 

-                 # NOTE: By appending, we won't override an "organically"

-                 #       passed test from AutoCloud because the selector

-                 #       below gets index 0 of the different atomic_ lists

-                 #

-                 #       We then insert at index 0 when the manually entered

-                 #       successful build is newer than the latest

-                 #       "organically" passed test

- 

-                 # check against atomic_qcow2

-                 for check_build in atomic_qcow2_failed:

-                     if good_compose == url2image(check_build[u'msg'][u'compose_id']):

-                         if len(atomic_qcow2) > 0:

-                             candidate = url2image(atomic_qcow2[0][u'msg'][u'compose_url'])

-                             if image2release(candidate) < image2release(good_compose):

-                                 atomic_qcow2.insert(0, check_build)

-                             else:

-                                 atomic_qcow2.append(check_build)

-                         else:

-                             atomic_qcow2.append(check_build)

- 

-                 # check against vagrant_libvirt

-                 for check_build in atomic_vagrant_libvirt_failed:

-                     if good_compose == url2image(check_build[u'msg'][u'compose_url']):

-                         if len(atomic_vagrant_libvirt) > 0:

-                             candidate = url2image(atomic_vagrant_libvirt[0][u'msg'][u'compose_url'])

-                             if image2release(candidate) < image2release(good_compose):

-                                 atomic_vagrant_libvirt.insert(0, check_build)

-                             else:

-                                 atomic_vagrant_libvirt.append(check_build)

-                         else:

-                             atomic_vagrant_libvirt.append(check_build)

- 

-                 # check against vagrant_vbox

-                 for check_build in atomic_vagrant_vbox_failed:

-                     if good_compose == url2image(check_build[u'msg'][u'compose_url']):

-                         if len(atomic_vagrant_vbox) > 0:

-                             candidate = url2image(atomic_vagrant_vbox[0][u'msg'][u'compose_url'])

-                             if image2release(candidate) < image2release(good_compose):

-                                 atomic_vagrant_vbox.insert(0, check_build)

-                             else:

-                                 atomic_vagrant_vbox.append(check_build)

-                         else:

-                             atomic_vagrant_vbox.append(check_build)

+     # raw image

+     #

+     # FIXME - This is a bit of a hack right now, but the raw image is what

+     #         the qcow2 is made of so only qcow2 is tested and infers the

+     #         success of both qcow2 and raw.xz

+     autocloud_info["atomic_raw"] = {

+         "compose_id": sc[u'id'],

+         "name": qcow_msg[u'name'],

+         "release": sc[u'release'],

+         "image_name": image_name.replace('qcow2', 'raw.xz'),    # HACK

+         "image_url": image_url.replace('qcow2', 'raw.xz'),      # HACK

+     }

  

-     autocloud_info = {}

+     # vagrant libvirt image

+     vlibvirt_msg = [

+         sc[u'results'][u'artifacts'][img] for img in sc[u'results'][u'artifacts']

+             if sc[u'results'][u'artifacts'][img][u'family'] == u'Atomic'

+             and sc[u'results'][u'artifacts'][img][u'type'] == u'vagrant-libvirt'

+     ][0]

+     image_name, image_url = construct_url(vlibvirt_msg)

+     autocloud_info["atomic_vagrant_libvirt"] = {

+         "compose_id": sc[u'id'],

+         "name": vlibvirt_msg[u'name'],

+         "release": sc[u'release'],

+         "image_name": image_name,

+         "image_url": image_url,

+     }

  

-     # Yet another hack because AutoCloud still doesn't give us this information

-     # even though the rewrite was supposed to

-     extract_compose_id = lambda x: x[u'msg']['compose_id']

-     qcow2_composes = map(extract_compose_id, atomic_qcow2)

-     vagrant_libvirt_composes = map(extract_compose_id, atomic_vagrant_libvirt)

-     vagrant_vbox_composes = map(extract_compose_id, atomic_vagrant_vbox)

-     compose = [c for c in qcow2_composes

-                if c in vagrant_vbox_composes

-                and c in vagrant_libvirt_composes][0]

- 

-     # sc = successful compose because we've now coorelated the data to a single

-     #       compose that's successful across all images

-     sc_qcow2 = [i for i in atomic_qcow2 if i[u'msg'][u'compose_id'] == compose]

-     sc_vagrant_libvirt = [i for i in atomic_vagrant_libvirt if i[u'msg'][u'compose_id'] == compose]

-     sc_vagrant_vbox = [i for i in atomic_vagrant_vbox if i[u'msg'][u'compose_id'] == compose]

- 

- 

-     if sc_qcow2:

-         image_name, image_url = construct_url(sc_qcow2[0])

-         autocloud_info["atomic_qcow2"] = {

-             "compose_id": sc_qcow2[0][u'msg'][u'compose_id'],

-             "name": sc_qcow2[0][u'msg'][u'image_name'],

-             "release": sc_qcow2[0][u'msg'][u'release'],

-             "image_name": image_name,

-             "image_url": image_url,

-         }

- 

-         # FIXME - This is a bit of a hack right now, but the raw image is what

-         #         the qcow2 is made of so only qcow2 is tested and infers the

-         #         success of both qcow2 and raw.xz

-         autocloud_info["atomic_raw"] = {

-             "compose_id": sc_qcow2[0][u'msg'][u'compose_id'],

-             "name": sc_qcow2[0][u'msg'][u'image_name'] + '-Raw',

-             "release": sc_qcow2[0][u'msg'][u'release'],

-             "image_name": image_name.replace('qcow2', 'raw.xz'),    # HACK

-             "image_url": image_url.replace('qcow2', 'raw.xz'),      # HACK

-         }

- 

-     if sc_vagrant_libvirt:

-         image_name, image_url = construct_url(sc_vagrant_libvirt[0])

-         autocloud_info["atomic_vagrant_libvirt"] = {

-             "compose_id": sc_vagrant_libvirt[0][u'msg'][u'compose_id'],

-             "name": sc_vagrant_libvirt[0][u'msg'][u'image_name'],

-             "release": sc_vagrant_libvirt[0][u'msg'][u'release'],

-             "image_name": image_name,

-             "image_url": image_url,

-         }

- 

-     if sc_vagrant_vbox:

-         image_name, image_url = construct_url(sc_vagrant_vbox[0])

-         autocloud_info["atomic_vagrant_virtualbox"] = {

-             "compose_id": sc_vagrant_vbox[0][u'msg'][u'compose_id'],

-             "name": sc_vagrant_vbox[0][u'msg'][u'image_name'],

-             "release": sc_vagrant_vbox[0][u'msg'][u'release'],

-             "image_name": image_name,

-             "image_url": image_url,

-         }

+     # vagrant vbox image

+     vvbox_msg = [

+         sc[u'results'][u'artifacts'][img] for img in sc[u'results'][u'artifacts']

+             if sc[u'results'][u'artifacts'][img][u'family'] == u'Atomic'

+             and sc[u'results'][u'artifacts'][img][u'type'] == u'vagrant-virtualbox'

+     ][0]

+     image_name, image_url = construct_url(vvbox_msg)

+     autocloud_info["atomic_vagrant_virtualbox"] = {

+         "compose_id": sc[u'id'],

+         "name": vvbox_msg[u'name'],

+         "release": sc[u'release'],

+         "image_name": image_name,

+         "image_url": image_url,

+     }

  

      return autocloud_info

  
@@ -438,8 +352,7 @@ 

  def stage_atomic_release(

          compose_id,

          compose_basedir=COMPOSE_BASEDIR,

-         dest_base_dir=ATOMIC_STABLE_BASEDIR,

-         link_dest=ATOMIC_LINKDEST):

+         dest_base_dir=ATOMIC_STABLE_BASEDIR):

      """

      stage_atomic_release

  
@@ -455,12 +368,7 @@ 

      rsync_cmd = [

          'sudo',

          'rsync -avhHP --delete-after',

-         '--link-dest={}'.format(

-             os.path.join(

-                 link_dest,

-                 compose_id.split('-')[-1]

-             )

-         ),

+         '--exclude Cloud/',

          "{}/".format(source_loc),

          dest_dir

      ]
@@ -668,6 +576,10 @@ 

      tested_autocloud_info = get_latest_successful_autocloud_test_info(

          pargs.release

      )

+ 

+     # FIXME - DEBUGGING

+     log.info("{}\n{}".format("TESTED_AUTOCLOUD_INFO", json.dumps(tested_autocloud_info, indent=2)))

+ 

      log.info("Query to datagrepper complete")

      # If the dict is empty, there were no successful builds in the last two

      # weeks, error accordingly
@@ -716,8 +628,9 @@ 

  

      send_atomic_announce_email(set(email_filelist))

  

-     log.info("Pruning old Atomic test composes")

-     prune_old_composes(ATOMIC_STABLE_BASEDIR, 2)

+     # FIXME - The logic in this functioni is broken, leave it disabled for now

+     #log.info("Pruning old Atomic test composes")

+     #prune_old_composes(ATOMIC_STABLE_BASEDIR, 2)

  

      log.info("Two Week Atomic Release Complete!")

  

Pull-Request has been merged by ausil

7 years ago
Metadata