PR#7162: atomic script: prepare for f27, multi-arch - releng

		`@@ -12,6 +12,7 @@`
		`# Authors:`
		`# Adam Miller <maxamillion@fedoraproject.org>`
		`# Patrick Uiterwijk <puiterwijk@redhat.com>`
		`+ # Dusty Mabe <dusty@dustymabe.com>`
		`#`
		`# Exit codes:`
		`# 0 - Success`
		`@@ -45,9 +46,14 @@`
		`log = logging.getLogger(os.path.basename(sys.argv[0]))`

		`# Define "constants"`
		`- ATOMIC_HOST_DIR = "/mnt/koji/compose/updates/atomic"`
		`- PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT = None`
		`- TARGET_REF = "fedora/%s/x86_64/atomic-host"`
		`+ ATOMIC_HOST_DIR = "/mnt/koji/atomic/%s"`
		`+ ARCHES = ['x86_64', 'aarch64', 'ppc64le']`
		`+ PREVIOUS_MAJOR_RELEASE_FINAL_COMMITS = {`
		`+ 'aarch64': None,`
		`+ 'ppc64le': None,`
		`+ 'x86_64': 'c099633883cd8d06895e32a14c63f6672072430c151de882223e4abe20efa7ca',`
		`+ }`
		`+ TARGET_REF = "fedora/%s/%s/atomic-host" # example fedora/27/x86_64/atomic-host`
		`COMPOSE_BASEDIR = "/mnt/koji/compose/twoweek/"`

		`# FIXME ???? Do we need a real STMP server here?`
		`@@ -105,7 +111,10 @@`
		`# delta = 2 weeks in seconds`
		`DATAGREPPER_DELTA = 1209600`
		`# category to filter on from datagrepper`
		`- DATAGREPPER_TOPIC = "org.fedoraproject.prod.autocloud.compose.complete"`
		`+ # url: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.autocloud.compose.complete`
		`+ DATAGREPPER_AUTOCLOUD_TOPIC = "org.fedoraproject.prod.autocloud.compose.complete"`
		`+ # url: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.ostree`
		`+ DATAGREPPER_OSTREE_TOPIC = "org.fedoraproject.prod.pungi.compose.ostree"`


		`SIGUL_SIGNED_TXT_PATH = "/tmp/signed"`
		`@@ -129,11 +138,60 @@`
		`image_url = os.path.join(ATOMIC_HOST_STABLE_BASEDIR, '/'.join(iul[4:]))`
		`return image_name, image_url`

		`+ def get_ostree_compose_info(`
		`+ ostree_pungi_compose_id,`
		`+ datagrepper_url=DATAGREPPER_URL,`
		`+ delta=DATAGREPPER_DELTA,`
		`+ topic=DATAGREPPER_OSTREE_TOPIC):`
		`+ """`
		`+ get_ostree_compose_info`
		`+`
		`+ Query datagrepper for fedmsg information from the compose.`
		`+ We'll find the ostree commits from this compose.`
		`+`
		`+ return -> dict`
		`+ Will return arch->commit dictionariy`
		`+`
		`+ """`
		`+`
		`+ # rows_per_page is maximum 100 from Fedora's datagrepper`
		`+ request_params = {`
		`+ "delta": delta,`
		`+ "topic": 'org.fedoraproject.prod.pungi.compose.ostree',`
		`+ "rows_per_page": 100,`
		`+ }`
		`+ r = requests.get(datagrepper_url, params=request_params)`
		`+`
		`+ # Start with page 1 response from datagrepper, grab the raw messages`
		`+ # and then continue to populate the list with the rest of the pages of data`
		`+ autocloud_data = r.json()[u'raw_messages']`
		`+ for rpage in range(2, r.json()[u'pages']+1):`
		`+ autocloud_data += requests.get(`
		`+ datagrepper_url,`
		`+ params=dict(page=rpage, **request_params)`
		`+ ).json()[u'raw_messages']`
		`+`
		`+ ostree_composes = [`
		`+ compose[u'msg'] for compose in autocloud_data`
		`+ if ostree_pungi_compose_id in compose[u'msg'][u'compose_id']`
		`+ and 'atomic-host' in compose[u'msg'][u'ref']`
		`+ ]`
		`+`
		`+ ostree_compose_info = dict()`
		`+ for ostree_compose in ostree_composes:`
		`+ arch = ostree_compose[u'arch']`
		`+ commit = ostree_compose[u'commitid']`
		`+ ostree_compose_info[arch] = commit`
		`+ log.info("Found %s, %s", arch, commit)`
		`+`
		`+ return ostree_compose_info`
		`+`
		`def get_latest_successful_autocloud_test_info(`
		`release,`
		`+ pungi_compose_id,`
		`datagrepper_url=DATAGREPPER_URL,`
		`delta=DATAGREPPER_DELTA,`
		`- topic=DATAGREPPER_TOPIC):`
		`+ topic=DATAGREPPER_AUTOCLOUD_TOPIC):`
		`"""`
		`get_latest_successful_autocloud_test_info`

		`@@ -165,31 +223,45 @@`
		`).json()[u'raw_messages']`


		`+ # XXX ignore this way of doing things for now (see below)`
		`+ ##### List comprehension that will return a list of compose information from`
		`+ ##### AutoCloud (the [u'msg'] payload of autocloud.compose.complete fedmsg)`
		`+ ##### such that the following criteria are true:`
		`+ #####`
		`+ ##### - Is an Atomic Host compose (i.e. 'Atomic' is in the compose id)`
		`+ ##### - No compose artifacts failed the tests`
		`+ ##### - This is the current Fedora release we want`
		`+ #####`
		`+ ##### OR:`
		`+ ##### - This compose was manually marked good`
		`+ ####candidate_composes = [`
		`+ #### compose[u'msg'] for compose in autocloud_data`
		`+ #### if u'Atomic' in compose[u'msg'][u'id']`
		`+ #### and compose[u'msg'][u'results'][u'failed'] == 0`
		`+ #### and compose[u'msg'][u'release'] == str(release)`
		`+ #### or compose[u'msg'][u'id'] in MARK_ATOMIC_HOST_GOOD_COMPOSES`
		`+ ####]`
		`+`
		`+ ####filtered_composes = list(candidate_composes)`
		`+ ####for compose in candidate_composes:`
		`+ #### if compose_manually_marked_bad(compose[u'id']):`
		`+ #### filtered_composes.remove(compose)`
		`+`
		`+ ##### sc = successful compose`
		`+ ####sc = filtered_composes[0]`
		`+`
		`+ # XXX For now ignore the autocloud results and just use the`
		`+ # requested pungi compose id.`
		`+ #`
		`# List comprehension that will return a list of compose information from`
		`# AutoCloud (the [u'msg'] payload of autocloud.compose.complete fedmsg)`
		`# such that the following criteria are true:`
		`- #`
		`- # - Is an Atomic Host compose (i.e. 'Atomic' is in the compose id)`
		`- # - No compose artifacts failed the tests`
		`- # - This is the current Fedora release we want`
		`- #`
		`- # OR:`
		`- # - This compose was manually marked good`
		`candidate_composes = [`
		`compose[u'msg'] for compose in autocloud_data`
		`- if u'Atomic' in compose[u'msg'][u'id']`
		`- and compose[u'msg'][u'results'][u'failed'] == 0`
		`- and compose[u'msg'][u'release'] == str(release)`
		`- or compose[u'msg'][u'id'] in MARK_ATOMIC_HOST_GOOD_COMPOSES`
		`+ if pungi_compose_id in compose[u'msg'][u'id']`
		`]`
		`-`
		`- filtered_composes = list(candidate_composes)`
		`- for compose in candidate_composes:`
		`- if compose_manually_marked_bad(compose[u'id']):`
		`- filtered_composes.remove(compose)`
		`-`
		`# sc = successful compose`
		`- sc = filtered_composes[0]`
		`+ sc = candidate_composes[0]`

		`autocloud_info = {}`

		`@@ -273,11 +345,10 @@`

		`def send_atomic_announce_email(`
		`email_filelist,`
		`+ ostree_commit_data,`
		`mail_receivers=ATOMIC_HOST_EMAIL_RECIPIENTS,`
		`sender_email=ATOMIC_HOST_EMAIL_SENDER,`
		`- sender_smtp=ATOMIC_HOST_EMAIL_SMTP,`
		`- tree_commit=None,`
		`- tree_version=None):`
		`+ sender_smtp=ATOMIC_HOST_EMAIL_SMTP):`
		`"""`
		`send_atomic_announce_email`

		`@@ -296,6 +367,13 @@`
		`released_artifacts.append(`
		`"https://alt.fedoraproject.org{}".format(e_file)`
		`)`
		`+ released_artifacts.sort()`
		`+ released_checksums.sort()`
		`+`
		`+ commits_string =""`
		`+ for arch in ARCHES:`
		`+ commit = ostree_commit_data[arch]['commit']`
		`+ commits_string += "Commit(%s): %s\n" % (arch, commit)`

		`msg = MIMEMultipart()`
		`msg['To'] = "; ".join(mail_receivers)`
		`@@ -304,10 +382,14 @@`
		`msg.attach(`
		`MIMEText(`
		`"""`
		`- A new Fedora Atomic Host update is available via an OSTree commit:`
		`+ A new Fedora Atomic Host update is available via an OSTree update:`

		`- Commit: {}`
		`Version: {}`
		`+ {}`
		`+`
		`+ We are releasing images from multiple architectures but please note`
		`+ that x86_64 architecture is the only one that undergoes automated`
		`+ testing at this time.`

		Existing systems can be upgraded in place via e.g. `atomic host upgrade` or
		`atomic host deploy`.
		`@@ -316,6 +398,9 @@`

		`https://getfedora.org/en/atomic/download/`

		`+ Alternatively, image artifacts can be found at the following links:`
		`+ {}`
		`+`
		`Respective signed CHECKSUM files can be found here:`
		`{}`

		`@@ -334,9 +419,9 @@`
		`https://getfedora.org/atomic_vagrant_virtualbox_latest_filename`

		`For more information about the latest targets, please reference the Fedora`
		`- Cloud Wiki space.`
		`+ Atomic Wiki space.`

		`- https://fedoraproject.org/wiki/Cloud#Quick_Links`
		`+ https://fedoraproject.org/wiki/Atomic_WG#Fedora_Atomic_Image_Download_Links`

		`Do note that it can take some of the mirrors up to 12 hours to "check-in" at`
		`their own discretion.`
		`@@ -344,8 +429,9 @@`
		`Thank you,`
		`Fedora Release Engineering`
		`""".format(`
		`- tree_commit,`
		`- tree_version,`
		`+ ostree_commit_data.items()[0][1]['version'],`
		`+ commits_string,`
		`+ '\n'.join(released_artifacts),`
		`'\n'.join(released_checksums)`
		`)`
		`)`
		`@@ -363,7 +449,7 @@`
		`print("ERROR: Unable to send email:\n{}\n".format(e))`

		`def stage_atomic_release(`
		`- compose_id,`
		`+ pungi_compose_id,`
		`compose_basedir=COMPOSE_BASEDIR,`
		`dest_base_dir=ATOMIC_HOST_STABLE_BASEDIR):`
		`"""`
		`@@ -374,8 +460,8 @@`

		`"""`

		`- source_loc = os.path.join(compose_basedir, compose_id, "compose")`
		`- dest_dir = os.path.join(dest_base_dir, compose_id)`
		`+ source_loc = os.path.join(compose_basedir, pungi_compose_id, "compose")`
		`+ dest_dir = os.path.join(dest_base_dir, pungi_compose_id)`

		`# FIXME - need sudo until pungi perms are fixed`
		`rsync_cmd = [`
		`@@ -568,8 +654,8 @@`
		`"""`
		`diff_cmd = ["/usr/bin/sudo",`
		`"ostree", "static-delta", "generate", "--repo",`
		`- ATOMIC_HOST_DIR, "--if-not-exists", "--from", old_commit,`
		`- "--to", new_commit]`
		`+ ATOMIC_HOST_DIR % release, "--if-not-exists",`
		`+ "--from", old_commit, "--to", new_commit]`
		`log.info("Creating Static Delta from %s to %s" % (old_commit, new_commit))`
		`if subprocess.call(diff_cmd):`
		`log.error("generate_static_delta: diff generation failed: %s", diff_cmd)`
		`@@ -583,28 +669,42 @@`

		`:param release - the Fedora release to target (25,26,etc)`
		`"""`
		`- summary_cmd = ["/usr/bin/sudo",`
		`- "ostree", "summary", "-u", "--repo",`
		`- ATOMIC_HOST_DIR]`
		`+ # Run as apache user because the files we are editing/creating`
		`+ # need to be owned by the apache user`
		`+ summary_cmd = ["/usr/bin/sudo", "ostree", "summary", "-u",`
		`+ "--repo", ATOMIC_HOST_DIR % release]`
		`log.info("Updating Summary file")`
		`if subprocess.call(summary_cmd):`
		`log.error("update_ostree_summary_file: update failed: %s", summary_cmd)`
		`exit(3)`

		`- def move_tree_commit(release, old_commit, new_commit):`
		`- generate_static_delta(release=release,`
		`- old_commit=old_commit,`
		`- new_commit=new_commit)`
		`+ def update_ref(ref, release, old_commit, new_commit):`
		`+ """`
		`+ update_ref`
		`+`
		`+ Update the given ref and set it to new_commit`
		`+`
		`+ :param ref - the ref to update`
		`+ :param release - the Fedora release to target (25,26,etc)`
		`+ :param old_commit - where the ref currently is`
		`+ :param new_commit - where the ref should end up`
		`+ """`
		`+`
		`+ if old_commit == new_commit:`
		`+ log.info("ref %s is already at %s. Skipping update",`
		`+ ref, new_commit`
		`+ )`
		`+ return`
		`+`
		`+ log.info("Moving ref %s from %s => %s",`
		`+ ref, old_commit, new_commit)`

		`- log.info("Moving ref %s to commit %s" %(TARGET_REF, new_commit))`
		`- reset_cmd = ['/usr/bin/sudo',`
		`- 'ostree', 'reset', TARGET_REF % release,`
		`- new_commit, '--repo', ATOMIC_HOST_DIR]`
		`+ reset_cmd = ['/usr/bin/sudo', 'ostree', 'reset', ref,`
		`+ new_commit, '--repo', ATOMIC_HOST_DIR % release]`
		`if subprocess.call(reset_cmd):`
		`- log.error("move_tree_commit: resetting ref to new commit failed: %s", reset_cmd)`
		`- exit(3)`
		`+ log.error("update_ref: resetting ref to new commit failed: %s", reset_cmd)`
		`+ sys.exit(3)`

		`- update_ostree_summary_file(release)`



		`@@ -615,21 +715,36 @@`
		`parser.add_argument(`
		`"-k",`
		`"--key",`
		`+ required=True,`
		`help="signing key to use with sigul",`
		`)`
		`parser.add_argument(`
		`"-r",`
		`"--release",`
		`+ required=True,`
		`help="Fedora Release to target for release (Ex: 24, 25, rawhide)",`
		`)`
		`+ parser.add_argument(`
		`+ "--pungi-compose-id",`
		`+ dest='pungi_compose_id',`
		`+ required=True,`
		`+ help="The pungi compose that created the media (Ex: Fedora-27-20171110.n.1)."`
		`+ )`
		`+ parser.add_argument(`
		`+ "--ostree-pungi-compose-id",`
		`+ dest='ostree_pungi_compose_id',`
		`+ help="""`
		`+ The pungi compose that created the ostree (Ex: Fedora-27-20171110.n.1).`
		`+ This is optional and is only required if the pungi compose that ostree`
		`+ commit is different than the ostree compose that created the media.`
		`+ """`
		`+ )`
		`pargs = parser.parse_args()`

		`- if not pargs.key:`
		`- log.error("No key passed, see -h for help")`
		`- sys.exit(1)`
		`- if not pargs.release:`
		`- log.error("No release arg passed, see -h for help")`
		`- sys.exit(1)`
		`+ # This one is only specified if it differs from --pungi-compose-id`
		`+ # If missing just assign it the value of pungi-compose-id.`
		`+ if not pargs.ostree_pungi_compose_id:`
		`+ pargs.ostree_pungi_compose_id = pargs.pungi_compose_id`

		`log.info("Checking to make sure release is not currently blocked")`
		`if BLOCK_ATOMIC_HOST_RELEASE:`
		`@@ -639,10 +754,8 @@`
		`log.info("Querying datagrepper for latest AutoCloud successful tests")`
		`# Acquire the latest successful builds from datagrepper`
		`tested_autocloud_info = get_latest_successful_autocloud_test_info(`
		`- pargs.release`
		`+ pargs.release, pargs.pungi_compose_id`
		`)`
		`-`
		`- # FIXME - DEBUGGING`
		`log.info("{}\n{}".format("TESTED_AUTOCLOUD_INFO", json.dumps(tested_autocloud_info, indent=2)))`

		`log.info("Query to datagrepper complete")`
		`@@ -652,38 +765,80 @@`
		`log.error("No successful builds found")`
		`sys.exit(2)`

		`- log.info("Extracting compose_id from tested autocloud data")`
		`- compose_id = tested_autocloud_info['atomic_qcow2']['compose_id']`
		`-`
		`- # TODO: https://github.com/kushaldas/tunirtests/pull/59 will allow us to`
		`- # extract this from the autocloud test results.`
		`- print('Releasing compose %s' % compose_id)`
		`- tree_commit = None`
		`- tree_version = None`
		`- while not tree_commit:`
		`- tree_commit = raw_input('Tree commit: ').strip()`
		`- try:`
		`- print("Validating and finding version of {}".format(tree_commit))`
		`- tree_version = subprocess.check_output(['/usr/bin/ostree', '--repo=' + ATOMIC_HOST_DIR, 'show', '--print-metadata-key=version', tree_commit])`
		`- except subprocess.CalledProcessError as e:`
		`- print('Error when validating commit: %s. Try again.' % tree_commit)`
		`- tree_commit = None`
		`- continue`
		`- # It's in GVariant print format by default, we can make this less hacky when`
		`- # porting to use libostree.`
		`- tree_version = tree_version.replace("'", "")`
		`-`
		`- rev_parse_cmd = ['/usr/bin/ostree', 'rev-parse', '--repo',`
		`- ATOMIC_HOST_DIR, TARGET_REF % pargs.release]`
		`- previous_commit = subprocess.check_output(rev_parse_cmd).strip()`
		`-`
		`- # This could happen if there was a failure in this script sending the email`
		`- # or anything after the commit has already been moved.`
		`- if previous_commit == tree_commit:`
		`- answer = raw_input('ref is already at that commit, are you sure?: (y/n)').strip()`
		`- if answer.lower() != 'y':`
		`- sys.exit(4)`
		`-`
		`+ ## XXX Not needed since we are specifying compose ID`
		`+ #log.info("Extracting compose_id from tested autocloud data")`
		`+ #compose_id = tested_autocloud_info['atomic_qcow2']['compose_id']`
		`+ #`
		`+ ## TODO: https://github.com/kushaldas/tunirtests/pull/59 will allow us to`
		`+ ## extract this from the autocloud test results.`
		`+ #print('Releasing compose %s' % compose_id)`
		`+`
		`+ # Initialize a empty dict that we will populate with information`
		`+ # about each commit. We'll use this information to do the release.`
		`+ # Example: => {`
		`+ # 'x86_64' => {`
		`+ # 'ref' => 'fedora/27/x86_64/atomic-host',`
		`+ # 'commit' => 'xxyyzz',`
		`+ # 'version' => '27.1'`
		`+ # 'previous_commit' => 'aabbccdd'`
		`+ # }`
		`+ # }`
		`+ ostree_commit_data = dict()`
		`+ for arch in ARCHES:`
		`+ ostree_commit_data[arch] = dict()`
		`+`
		`+ # Get commit information from fedmsg for the specified ostree`
		`+ # compose. This will give us back a dict of arch->commit.`
		`+ ostree_compose_info = get_ostree_compose_info(pargs.ostree_pungi_compose_id)`
		`+`
		`+ # Verify there is a commit for each of the architectures.`
		`+ for arch in ARCHES:`
		`+ if arch not in ostree_compose_info.keys():`
		`+ log.error("No compose commit info for %s in %s",`
		`+ arch, pargs.ostree_pungi_compose_id)`
		`+ sys.exit(2)`
		`+`
		`+ # populate the ostree_commit_data dict`
		`+ for arch in ARCHES:`
		`+ commit = ostree_compose_info[arch]`
		`+ ref = TARGET_REF % (pargs.release, arch)`
		`+`
		`+ # Verify the commit exists in the tree, and find the version`
		`+ log.info("Verifying and finding version of %s", commit)`
		`+ cmd = ['/usr/bin/ostree', '--repo=' + ATOMIC_HOST_DIR % pargs.release,`
		`+ 'show', '--print-metadata-key=version', commit]`
		`+ version = subprocess.check_output(cmd).strip()`
		`+ # output will be in GVariant print format by default -> s/'//`
		`+ version = version.replace("'", "")`
		`+`
		`+ # Find the previous commit for this ref in the tree`
		`+ cmd = ['/usr/bin/ostree', '--repo=' + ATOMIC_HOST_DIR % pargs.release,`
		`+ 'rev-parse', ref]`
		`+ previous_commit = subprocess.check_output(cmd).strip()`
		`+`
		`+ # set info in ostree_commit_data dict`
		`+ ostree_commit_data[arch]['ref'] = ref`
		`+ ostree_commit_data[arch]['commit'] = commit`
		`+ ostree_commit_data[arch]['version'] = version`
		`+ ostree_commit_data[arch]['previous_commit'] = previous_commit`
		`+`
		`+ log.info("OSTREE COMMIT DATA INFORMATION")`
		`+ log.info("%s", json.dumps(ostree_commit_data, indent=2))`
		`+`
		`+ # Verify all versions match`
		`+ ostree_commit_version = ostree_commit_data.items()[0][1]['version']`
		`+ for arch in ARCHES:`
		`+ if ostree_commit_data[arch]['version'] != ostree_commit_version:`
		`+ log.error("Found mismatched versions for commits")`
		`+ log.error("Got %s for %s. Expected %s",`
		`+ ostree_commit_data[arch]['version'],`
		`+ ostree_commit_data[arch]['commit'],`
		`+ ostree_commit_version)`
		`+ sys.exit(1)`
		`+`
		`+ log.info("Releasing ostrees at version: %s", ostree_commit_version)`
		`+`
		`+ # url: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.releng.atomic.twoweek.begin`
		`log.info("Sending fedmsg releng.atomic.twoweek.begin")`
		`fedmsg_publish(`
		`topic="atomic.twoweek.begin",`
		`@@ -693,27 +848,41 @@`
		`log.info("Signing image metadata - compose")`
		`sign_checksum_files(`
		`pargs.key,`
		`- os.path.join(COMPOSE_BASEDIR, compose_id),`
		`+ os.path.join(COMPOSE_BASEDIR, pargs.pungi_compose_id),`
		`)`

		`- # If we are already at the new commit then there is nothing to do`
		`- if previous_commit == tree_commit:`
		`- log.info("Tree commit is already at %s. Skipping move", tree_commit)`
		`- else:`
		`- log.info("Moving tree commit %s => %s (%s)", previous_commit, tree_commit, tree_version)`
		`- move_tree_commit(pargs.release, previous_commit, tree_commit)`
		`-`
		`- # Also, if existing previous release commit is defined, then`
		`- # generate a static delta from it`
		`- if PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT is not None:`
		`- generate_static_delta(release=pargs.release,`
		`- old_commit=PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT,`
		`- new_commit=tree_commit)`
		`+ # Perform the necessary ostree repo manipulations for the release`
		`+ # for each arch:`
		`+ # - create static delta from previous release`
		`+ # - create static delta from previous major release`
		`+ # - update the ref in the repo to the new commit`
		`+ for arch in ARCHES:`
		`+ # Generate static delta from previous release`
		`+ generate_static_delta(`
		`+ old_commit=ostree_commit_data[arch]['previous_commit'],`
		`+ new_commit=ostree_commit_data[arch]['commit']`
		`+ )`
		`+ # Generate static delta from previous major release (if defined)`
		`+ old_commit = PREVIOUS_MAJOR_RELEASE_FINAL_COMMITS.get(arch, None)`
		`+ if old_commit is not None:`
		`+ generate_static_delta(`
		`+ old_commit=old_commit,`
		`+ new_commit=ostree_commit_data[arch]['commit'],`
		`+ )`
		`+ # Move the ref`
		`+ update_ref(`
		`+ ostree_commit_data[arch]['ref'],`
		`+ pargs.release,`
		`+ ostree_commit_data[arch]['previous_commit'],`
		`+ ostree_commit_data[arch]['commit']`
		`+ )`
		`+ # Update summary file`
		`update_ostree_summary_file(pargs.release)`

		`log.info("Staging release content in /pub/alt/atomic/stable/")`
		`- stage_atomic_release(compose_id)`
		`+ stage_atomic_release(pargs.pungi_compose_id)`

		`+ # url: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.releng.atomic.twoweek.complete`
		`log.info("Sending fedmsg releng.atomic.twoweek.complete")`
		`fedmsg_publish(`
		`topic="atomic.twoweek.complete",`
		`@@ -724,7 +893,8 @@`
		`# Find all the Atomic Host images and CHECKSUM files to include in the email`
		`email_filelist = []`
		`for full_dir_path, _, short_names in \`
		`- os.walk(os.path.join(ATOMIC_HOST_STABLE_BASEDIR, compose_id)):`
		`+ os.walk(os.path.join(ATOMIC_HOST_STABLE_BASEDIR,`
		`+ pargs.pungi_compose_id)):`
		`for sname in fnmatch.filter(short_names, 'Atomic'):`
		`email_filelist.append(`
		`os.path.join(`
		`@@ -735,8 +905,7 @@`
		`for c_file in glob.glob(os.path.join(full_dir_path, "*CHECKSUM")):`
		`email_filelist.append(c_file)`

		`- send_atomic_announce_email(set(email_filelist), tree_commit=tree_commit,`
		`- tree_version=tree_version)`
		`+ send_atomic_announce_email(set(email_filelist), ostree_commit_data)`

		`# FIXME - The logic in this functioni is broken, leave it disabled for now`
		`#log.info("Pruning old Atomic Host test composes")`

dustymabe commented 6 years ago

Quite a few changes in the push_two_week script. Mainly:

enable multi-arch
prepare for f27
ignore autocloud test results for now

Please review each commit individually as that will be easiest for the reviewer.

rebased onto e906f1008472ef8560141ea1d69a792e989904f6

6 years ago

rebased onto 2848a0a

6 years ago

14 new commits added

atomic: add multi-arch info to release email
atomic: add dusty to authors of script
atomic: set f26 major release commit for x86_64
atomic: add some hyperlink comments to datagrepper
atomic: rewrite the ostree manipulations code
atomic: rewrite checks, enable multi-arch
atomic: update global vars for multi-arch support
atomic: rename move_ostree_commit -> update_ref
atomic: add get_ostree_compose_info function
atomic: add --ostree-pungi-compose-id option
atomic: add --pungi-compose-id arg, ignore autocloud
atomic: make the parser check for required args
atomic: remove update_ostree_summary_file func
atomic: remove unused release arg

6 years ago

puiterwijk commented 6 years ago

This release branch should no longer be created in the /mnt/koji/compose/updates/atomic/ repo, but rather directly into the /mnt/koji/atomic/$release repo, which means that you can also forget about the sync script for the purposes of this script.

I also can't stop noticing you didn't take my suggestion that the Pungi compose (whose compose ID you pass in the CLI) writes out the exact commitid's, so that you wouldn't have to request this information from datagrepper.

mohanboddu commented on line 537 of scripts/push-two-week-atomic.py 6 years ago

I think this should be:
update_ref(ostree_commit_data[arch]['ref'], old_commit, ostree_commit_data[arch]['commit'])

Well, old_commit is not actually required, but you are doing some checks to actually move the ref or not in update_ref function. So, its better to update this one.

dustymabe commented on line 537 of scripts/push-two-week-atomic.py 6 years ago

thanks. I am indeed not passing the right number of args

1 new commit added

atomic: operate directly on prod ostree repo

6 years ago

dustymabe commented 6 years ago

added new commit that changes to use /mnt/koji/atomic/$release instead

15 new commits added

atomic: operate directly on prod ostree repo
atomic: add multi-arch info to release email
atomic: add dusty to authors of script
atomic: set f26 major release commit for x86_64
atomic: add some hyperlink comments to datagrepper
atomic: rewrite the ostree manipulations code
atomic: rewrite checks, enable multi-arch
atomic: update global vars for multi-arch support
atomic: rename move_ostree_commit -> update_ref
atomic: add get_ostree_compose_info function
atomic: add --ostree-pungi-compose-id option
atomic: add --pungi-compose-id arg, ignore autocloud
atomic: make the parser check for required args
atomic: remove update_ostree_summary_file func
atomic: remove unused release arg

6 years ago

dustymabe commented 6 years ago

amended that last commit to fix typo

dustymabe commented 6 years ago

from @puiterwijk
I also can't stop noticing you didn't take my suggestion that the Pungi compose (whose compose ID you pass in the CLI) writes out the exact commitid's, so that you wouldn't have to request this information from datagrepper.

https://kojipkgs.fedoraproject.org/compose/updates/Fedora-27-updates-20171111.1/logs/x86_64/Everything/ostree-3/commitid.log

There are two reasons why I would prefer to not use a hardcoded URL (like above) for now:

The full path depends on what type of pungi run created the ostree compose. For example the one that we are going to be using for the release today would be:

https://kojipkgs.fedoraproject.org/compose/branched/Fedora-27-20171110.n.1/logs/x86_64/Atomic/ostree-3/commitid.log

I am not really confident in the ostree-N/commitid.log numbering scheme being consistent. For that single compose we have:

logs/aarch64/Everything/ostree-1
logs/ppc64le/Everything/ostree-2
logs/x86_64/Everything/ostree-3

Also, I think when we start creating the workstation ostrees during the updates there will be more ostree-N values in that directory (for x86_64) and we won't know which one to use.

15 new commits added

atomic: operate directly on prod ostree repo
atomic: add multi-arch info to release email
atomic: add dusty to authors of script
atomic: set f26 major release commit for x86_64
atomic: add some hyperlink comments to datagrepper
atomic: rewrite the ostree manipulations code
atomic: rewrite checks, enable multi-arch
atomic: update global vars for multi-arch support
atomic: rename move_ostree_commit -> update_ref
atomic: add get_ostree_compose_info function
atomic: add --ostree-pungi-compose-id option
atomic: add --pungi-compose-id arg, ignore autocloud
atomic: make the parser check for required args
atomic: remove update_ostree_summary_file func
atomic: remove unused release arg

6 years ago

dustymabe commented 6 years ago

did a fixup for atomic: add multi-arch info to release email so that the links are sorted (makes for better viewing)

mohanboddu commented 6 years ago

LGTM

@puiterwijk if you can give a quick review(as you know the process better), then I will merge it.

puiterwijk commented 6 years ago

I don't like the ostree commitid lookup, but let's just get this over with.
If @mohanboddu is okay, let's just do this.

Pull-Request has been merged by puiterwijk

6 years ago

walters commented 6 years ago

I scanned this and it looks pretty good to me. Nice split commits! On the multi-arch thing, it feels like it'd make our life easier if pungi dropped that data in the compose json or so?

Metadata

Assignee

None

Tags

No Tags

Changes Summary 1

+279 -110

file changed

scripts/push-two-week-atomic.py

releng

Source Code

Documentation

#7162 atomic script: prepare for f27, multi-arch Merged 6 years ago by puiterwijk. Opened 6 years ago by dustymabe. dustymabe/releng dusty into master

Metadata

Changes Summary 1

#7162 atomic script: prepare for f27, multi-arch

Merged 6 years ago by puiterwijk. Opened 6 years ago by dustymabe.

dustymabe/releng dusty into master