#504 Emit location keys in nightly compose fedmsgs, plus related cleanups (f27)
Merged 6 years ago by mohanboddu. Opened 6 years ago by adamwill.
adamwill/pungi-fedora 27-fedmsg-destination  into  f27

file modified
+8 -4
@@ -61,17 +61,21 @@ 

  

  NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')

- fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

- fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

  

  # Set this to use later for a few items include depcheck

  DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

+ # Public URL the compose will wind up at, we put it in fedmsgs

+ LOCATION=$(echo $DESTDIR|sed -e 's,/mnt/koji,https://kojipkgs.fedoraproject.org,g')

+ # Update fedmsg template

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID" "$LOCATION")

+ 

  $RSYNCPREFIX mkdir -p $DESTDIR

  # Tell interested persons that the rsync is starting (zomg!)

  #send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

  #for dir in CloudImage metadata ;

  #  do

- #    $RSYNCPREFIX rsync -avhH $TARGET_DIR/$NEWCOMPOSE_ID/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

+ #    $RSYNCPREFIX rsync -avhH $DESTDIR/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

  #  done

  # Tell interested persons that the rsync is done.

  #send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete
@@ -80,6 +84,6 @@ 

  # Tell everyone by email about the compose

  SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'

  #for tomail in $TOMAIL ; do

- #    cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

+ #    cat $DESTDIR/logs/*verbose $DESTDIR/logs/depcheck | \

  #         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

  #done

file modified
+9 -4
@@ -60,16 +60,21 @@ 

  

  NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')

- fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

- fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

  

+ # Set this to use later for a few items include depcheck

  DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

+ # Public URL the compose will wind up at, we put it in fedmsgs

+ LOCATION=$(echo $DESTDIR|sed -e 's,/mnt/koji,https://kojipkgs.fedoraproject.org,g')

+ # Update fedmsg template

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID" "$LOCATION")

+ 

  $RSYNCPREFIX mkdir -p $DESTDIR

  # Tell interested persons that the rsync is starting (zomg!)

  #send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

  #for dir in Docker metadata ;

  #  do

- #    $RSYNCPREFIX rsync -avhH $TARGET_DIR/$NEWCOMPOSE_ID/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

+ #    $RSYNCPREFIX rsync -avhH $DESTDIR/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

  #  done

  # Tell interested persons that the rsync is done.

  #send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete
@@ -77,6 +82,6 @@ 

  send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete

  SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'

  #for tomail in $TOMAIL ; do

- #    cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

+ #    cat $DESTDIR/logs/*verbose $DESTDIR/logs/depcheck | \

  #         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

  #done

file modified
+14 -10
@@ -18,6 +18,7 @@ 

  TOMAIL="devel@lists.fedoraproject.org test@lists.fedoraproject.org"

  FROM="Fedora Branched Report <rawhide@fedoraproject.org>"

  RSYNCPREFIX="sudo -u ftpsync"

+ RSYNCTARGET="/pub/fedora/linux/modular/development/$RELEASE"

  OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  

  # assume a releng dir is a git checkout of the releng repo
@@ -63,13 +64,16 @@ 

  

  NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-Modular.*-||g')

- fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

- fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

  

  # Set this to use later for a few items include depcheck

  DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

+ # Public URL the synced compose will wind up at, we put it in fedmsgs

+ LOCATION="https://dl.fedoraproject.org$RSYNCTARGET"

+ # Update fedmsg template

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID" "$LOCATION")

  

- compose-changelog -p $TARGET_DIR/$NEWCOMPOSE_ID/logs/ $TARGET_DIR/$OLDCOMPOSE_ID/ $TARGET_DIR/$NEWCOMPOSE_ID/

+ compose-changelog -p $DESTDIR/logs/ $TARGET_DIR/$OLDCOMPOSE_ID/ $DESTDIR/

  # Figure out a version for broken deps e-mail that goes to package maintainers.

  # For Bikeshed it's just bikeshed, for branched versions we prepend F- to the number.

  if [ "$RELEASE" = "bikeshed" ]; then
@@ -83,19 +87,19 @@ 

  send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

  

  # Sync the content to /pub/fedora

- if [ ! -d /pub/fedora/linux/modular/development/$RELEASE ]; then

-   mkdir /pub/fedora/linux/modular/development/$RELEASE

+ if [ ! -d "$RSYNCTARGET" ]; then

+   mkdir "$RSYNCTARGET"

  fi

  $RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 --arch=aarch64 --arch=ppc64 --arch=ppc64le --arch=s390x --arch src \

-     "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora/linux/modular/development/$RELEASE/ \

+     "$DESTDIR" "$RSYNCTARGET/" \

      --variant Server \

      --exclude=repodata

  $RSYNCPREFIX compose-partial-copy --arch=armhfp --arch=x86_64 --arch=aarch64 --arch=ppc64 --arch=ppc64le --arch=s390x --arch src \

-     "$TARGET_DIR/$NEWCOMPOSE_ID" /pub/fedora/linux/modular/development/$RELEASE/ \

+     "$DESTDIR" "$RSYNCTARGET/" \

      --variant Server \

      --delete-after

- $RSYNCPREFIX rm /pub/fedora/linux/modular/development/$RELEASE/.composeinfo

- $RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/fedora/linux/modular/development/$RELEASE/ --name "$NEWCOMPOSE_ID"

+ $RSYNCPREFIX rm "$RSYNCTARGET/.composeinfo"

+ $RSYNCPREFIX ./releng/scripts/build_composeinfo "$RSYNCTARGET/" --name "$NEWCOMPOSE_ID"

  

  # Tell interested persons that the rsync is done.

  send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete
@@ -106,7 +110,7 @@ 

  # Tell everyone by email about the compose

  SUBJECT='Fedora Modular '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'

  for tomail in $TOMAIL ; do

-     cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose | \

+     cat $DESTDIR/logs/*verbose | \

           mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

  done

  

file modified
+10 -5
@@ -20,6 +20,7 @@ 

  TOMAIL="devel@lists.fedoraproject.org test@lists.fedoraproject.org"

  FROM="Fedora Branched Report <rawhide@fedoraproject.org>"

  RSYNCPREFIX="sudo -u ftpsync"

+ RSYNCTARGET="/pub/alt/atomic/testing/$SHORTCOMPOSE_ID"

  OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  # uncomment and edit for resuming a failed compose

  #COMPOSE_ID="Fedora-23-20150530.n.0"
@@ -60,17 +61,21 @@ 

  

  NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')

- fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

- fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

  

  DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

+ # Public URL the synced compose will wind up at, we put it in fedmsgs

+ LOCATION="https://dl.fedoraproject.org$RSYNCTARGET"

+ # Update fedmsg template

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID" "$LOCATION")

+ 

  $RSYNCPREFIX mkdir -p $DESTDIR

  # Tell interested persons that the rsync is starting (zomg!)

  send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

  for dir in Atomic CloudImages Docker metadata ;

    do

-     $RSYNCPREFIX rsync -avhH $TARGET_DIR/$NEWCOMPOSE_ID/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

-     $RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/ --name $NEWCOMPOSE_ID

+     $RSYNCPREFIX rsync -avhH $DESTDIR/compose/$dir/ "$RSYNCTARGET/$dir/" ;

+     $RSYNCPREFIX ./releng/scripts/build_composeinfo "$RSYNCTARGET/" --name $NEWCOMPOSE_ID

    done

  # Tell interested persons that the rsync is done.

  send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete
@@ -78,6 +83,6 @@ 

  send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete

  SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'

  #for tomail in $TOMAIL ; do

- #    cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

+ #    cat $DESTDIR/logs/*verbose $DESTDIR/logs/depcheck | \

  #         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

  #done

This is basically https://pagure.io/pungi-fedora/pull-request/503 for the f27 branch. I dropped the changes to nightly.sh from it as nightly.sh on f27 branch is not up to date with master, so they don't apply, and we will never use that script on the f27 branch again so it's not worth the effort of bringing it up to date first.

Pull-Request has been merged by mohanboddu

6 years ago