From 898848fd55461a408ca9a6cb6d692dd0db08099b Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jul 10 2017 17:45:44 +0000 Subject: [PATCH 1/5] atomic: update ref for f26 In f26 and beyond we are using 'fedora/{release}/{arch}/atomic-host' Signed-off-by: Dusty Mabe --- diff --git a/scripts/push-two-week-atomic.py b/scripts/push-two-week-atomic.py index 2310f0d..61491d2 100755 --- a/scripts/push-two-week-atomic.py +++ b/scripts/push-two-week-atomic.py @@ -47,7 +47,7 @@ log = logging.getLogger(os.path.basename(sys.argv[0])) # Define "constants" ATOMIC_DIR = "/mnt/koji/mash/atomic/%s" -TARGET_REF = "fedora-atomic/%s/x86_64/docker-host" +TARGET_REF = "fedora/%s/x86_64/atomic-host" COMPOSE_BASEDIR = "/mnt/koji/compose/twoweek/" MASHER_LOCKFILE_GLOB = "/mnt/koji/mash/updates/MASHING*" From 9bf9f5150fa9512ee2b9b5132716d4144f63c67f Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jul 10 2017 17:46:00 +0000 Subject: [PATCH 2/5] atomic: create generate_static_delta function Signed-off-by: Dusty Mabe --- diff --git a/scripts/push-two-week-atomic.py b/scripts/push-two-week-atomic.py index 61491d2..ce4a7f0 100755 --- a/scripts/push-two-week-atomic.py +++ b/scripts/push-two-week-atomic.py @@ -557,19 +557,33 @@ def prune_old_composes(prune_base_dir, prune_limit): "prune_old_composes: command failed: {}".format(prune_cmd) ) +def generate_static_delta(release, old_commit, new_commit): + """ + generate_static_delta -def move_tree_commit(release, old_commit, new_commit): - log.info("Creating diff") + Generate a static delta betwee two commits + + :param release - the Fedora release to target (25,26,etc) + :param old_commit - starting point for delta + :param new_commit - ending point for delta + """ # Run as apache user because the files we are editing/creating # need to be owned by the apache user diff_cmd = ["/usr/bin/sudo", "-u", "apache", "ostree", "static-delta", "generate", "--repo", ATOMIC_DIR % release, "--if-not-exists", "--from", old_commit, "--to", new_commit] + log.info("Creating Static Delta from %s to %s" % (old_commit, new_commit)) if subprocess.call(diff_cmd): - log.error("move_tree_commit: diff generation failed: %s", diff_cmd) + log.error("generate_static_delta: diff generation failed: %s", diff_cmd) exit(3) +def move_tree_commit(release, old_commit, new_commit): + generate_static_delta(release=release, + old_commit=old_commit, + new_commit=new_commit) + + log.info("Moving ref %s to commit %s" %(TARGET_REF, new_commit)) reset_cmd = ['/usr/bin/sudo', '-u', 'apache', 'ostree', 'reset', TARGET_REF % release, new_commit, '--repo', ATOMIC_DIR % release] From 1e9c81aa2369367f2fa4936e6203838669d385e4 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jul 10 2017 17:46:07 +0000 Subject: [PATCH 3/5] atomic: create update_ostree_summary_file function Signed-off-by: Dusty Mabe --- diff --git a/scripts/push-two-week-atomic.py b/scripts/push-two-week-atomic.py index ce4a7f0..a109b6d 100755 --- a/scripts/push-two-week-atomic.py +++ b/scripts/push-two-week-atomic.py @@ -578,6 +578,24 @@ def generate_static_delta(release, old_commit, new_commit): log.error("generate_static_delta: diff generation failed: %s", diff_cmd) exit(3) +def update_ostree_summary_file(release): + """ + update_ostree_summary_file + + Update the summary file for the ostree repo + + :param release - the Fedora release to target (25,26,etc) + """ + # Run as apache user because the files we are editing/creating + # need to be owned by the apache user + summary_cmd = ["/usr/bin/sudo", "-u", "apache", + "ostree", "summary", "-u", "--repo", + ATOMIC_DIR % release] + log.info("Updating Summary file") + if subprocess.call(summary_cmd): + log.error("update_ostree_summary_file: update failed: %s", summary_cmd) + exit(3) + def move_tree_commit(release, old_commit, new_commit): generate_static_delta(release=release, old_commit=old_commit, @@ -591,12 +609,8 @@ def move_tree_commit(release, old_commit, new_commit): log.error("move_tree_commit: resetting ref to new commit failed: %s", reset_cmd) exit(3) - summary_cmd = ["/usr/bin/sudo", "-u", "apache", - "ostree", "summary", "-u", "--repo", - ATOMIC_DIR % release] - if subprocess.call(summary_cmd): - log.error("move_tree_commit: summary update failed: %s", summary_cmd) - exit(3) + update_ostree_summary_file(release) + if __name__ == '__main__': From 2e550c029db969d3c2fbc1d190073125e1e3f05c Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jul 10 2017 17:46:13 +0000 Subject: [PATCH 4/5] atomic: add code for delta creation of f25 commit For now we'll define a commit in the N repo to that is the last commit from N-1 and we'll generate a static delta from that commit to the current release. This is so that rebases/upgrades from the previous release are much faster. Signed-off-by: Dusty Mabe --- diff --git a/scripts/push-two-week-atomic.py b/scripts/push-two-week-atomic.py index a109b6d..118a587 100755 --- a/scripts/push-two-week-atomic.py +++ b/scripts/push-two-week-atomic.py @@ -47,6 +47,7 @@ log = logging.getLogger(os.path.basename(sys.argv[0])) # Define "constants" ATOMIC_DIR = "/mnt/koji/mash/atomic/%s" +PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT = None TARGET_REF = "fedora/%s/x86_64/atomic-host" COMPOSE_BASEDIR = "/mnt/koji/compose/twoweek/" MASHER_LOCKFILE_GLOB = "/mnt/koji/mash/updates/MASHING*" @@ -729,6 +730,14 @@ if __name__ == '__main__': log.info("Moving tree commit %s => %s (%s)", previous_commit, tree_commit, tree_version) move_tree_commit(pargs.release, previous_commit, tree_commit) + # Also, if existing previous release commit is defined, then + # generate a static delta from it + if PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT is not None: + generate_static_delta(release=pargs.release, + old_commit=PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT, + new_commit=tree_commit) + update_ostree_summary_file(pargs.release) + log.info("Staging release content in /pub/alt/atomic/stable/") stage_atomic_release(compose_id) From fc574dd36c1ec7322880cb343590ed1aed8b05fd Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jul 10 2017 18:53:53 +0000 Subject: [PATCH 5/5] atomic: define the last released commit for f25 Used for static delta generation. Signed-off-by: Dusty Mabe --- diff --git a/scripts/push-two-week-atomic.py b/scripts/push-two-week-atomic.py index 118a587..54eec68 100755 --- a/scripts/push-two-week-atomic.py +++ b/scripts/push-two-week-atomic.py @@ -47,7 +47,7 @@ log = logging.getLogger(os.path.basename(sys.argv[0])) # Define "constants" ATOMIC_DIR = "/mnt/koji/mash/atomic/%s" -PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT = None +PREVIOUS_MAJOR_RELEASE_FINAL_COMMIT = 'ce555fa89da934e6eef23764fb40e8333234b8b60b6f688222247c958e5ebd5b' TARGET_REF = "fedora/%s/x86_64/atomic-host" COMPOSE_BASEDIR = "/mnt/koji/compose/twoweek/" MASHER_LOCKFILE_GLOB = "/mnt/koji/mash/updates/MASHING*"