#484 Improve fedmsgs and skip post-compose actions on failed composes for Atomic, Docker and Cloud compose types
Merged 6 years ago by mohanboddu. Opened 6 years ago by adamwill.
adamwill/pungi-fedora fix-all-the-things  into  f27

file modified
+23 -25
@@ -12,19 +12,16 @@ 

  SKIP_PHASES="--skip-phase=productimg --skip-phase=pkgset --skip-phase=gather --skip-phase=extra_files --skip-phase=creatrepo"

  DEST=$(pwd)

  DATE=$(date "+%Y%m%d")

- DIST="27"

- BRANCHED="27"

- BRANCH="branched"

+ # the Pungi 'shortname', which we will include in fedmsgs for disambiguation

+ SHORT="Fedora-Cloud"

+ RELEASE="27"

+ RELEASE_TITLE="27"

  COMPSFILE="comps-f27.xml"

- GIT_BRANCH="f27"

- TMPDIR=`mktemp -d /tmp/$DIST.$DATE.XXXX`

+ TMPDIR=`mktemp -d /tmp/$RELEASE.$DATE.XXXX`

  TOMAIL="devel@lists.fedoraproject.org test@lists.fedoraproject.org"

  FROM="Fedora Branched Report <rawhide@fedoraproject.org>"

  RSYNCPREFIX="sudo -u ftpsync"

- ATOMIC=$(mktemp -d /tmp/atomic.${DIST}.$DATE.XXXX)

- ATOMICREPO="/mnt/koji/compose/atomic/$BRANCHED/"

- ATOMICDEST="/mnt/koji/atomic/$BRANCHED/"

- OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora-27/COMPOSE_ID)

+ OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  # uncomment and edit for resuming a failed compose

  #COMPOSE_ID="Fedora-23-20150530.n.0"

  # assume a releng dir is a git checkout of the releng repo
@@ -37,13 +34,12 @@ 

      git clone https://pagure.io/releng.git

  fi

  # Set up our fedmsg function, using the releng repo definition

- fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

- fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s"}' "$RELEASE" "$ARCH" "$SHORT")

  FEDMSG_MODNAME="compose"

  FEDMSG_CERTPREFIX="releng"

  . ./releng/scripts/fedmsg-functions.sh

  # Announce that we are starting, even though we don't know the compose_id yet..

- send_fedmsg "${fedmsg_json_start}" ${DIST} start

+ send_fedmsg "${fedmsg_json_start}" ${RELEASE} start

  pushd $TMPDIR

  git clone https://pagure.io/fedora-comps.git && {

      pushd fedora-comps
@@ -59,29 +55,31 @@ 

      CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID"

  fi

  time $CMD "$@"

- if [ "$?" = "0" ]; then

-    export mail=0

+ if [ "$?" != "0" ]; then

+     exit 1

  fi

- NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora-27/COMPOSE_ID)

+ 

+ NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

+ 

  # Set this to use later for a few items include depcheck

  DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

  $RSYNCPREFIX mkdir -p $DESTDIR

  # Tell interested persons that the rsync is starting (zomg!)

- send_fedmsg "${fedmsg_json_start}" ${DIST} rsync.start

+ #send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

  #for dir in CloudImage metadata ;

  #  do

  #    $RSYNCPREFIX rsync -avhH $TARGET_DIR/$NEWCOMPOSE_ID/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

  #  done

  # Tell interested persons that the rsync is done.

- #send_fedmsg "${fedmsg_json_done}" ${DIST} rsync.complete

+ #send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete

  # Tell everyone by fedmsg about the compose

- send_fedmsg "${fedmsg_json_done}" ${DIST} complete

+ send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete

  # Tell everyone by email about the compose

- SUBJECT='Fedora '$DIST' compose report: '$SHORTCOMPOSE_ID' changes'

- #if [ "$mail" = "0" ]; then

- #    for tomail in $TOMAIL ; do

- #        cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

- #             mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

- #    done

- #fi

+ SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'

+ #for tomail in $TOMAIL ; do

+ #    cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

+ #         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

+ #done

file modified
+23 -25
@@ -11,19 +11,16 @@ 

  SKIP_PHASES="--skip-phase=productimg --skip-phase=pkgset --skip-phase=gather --skip-phase=extra_files --skip-phase=creatrepo"

  DEST=$(pwd)

  DATE=$(date "+%Y%m%d")

- DIST="26"

- BRANCHED="26"

- BRANCH="branched"

+ # the Pungi 'shortname', which we will include in fedmsgs for disambiguation

+ SHORT="Fedora-Docker"

+ RELEASE="26"
ausil commented 6 years ago

not that its a new bug but this should be 27

+ RELEASE_TITLE="26"

  COMPSFILE="comps-f26.xml"

- GIT_BRANCH="f26"

- TMPDIR=`mktemp -d /tmp/$DIST.$DATE.XXXX`

+ TMPDIR=`mktemp -d /tmp/$RELEASE.$DATE.XXXX`

  TOMAIL="devel@lists.fedoraproject.org test@lists.fedoraproject.org"

  FROM="Fedora Branched Report <rawhide@fedoraproject.org>"

  RSYNCPREFIX="sudo -u ftpsync"

- ATOMIC=$(mktemp -d /tmp/atomic.${DIST}.$DATE.XXXX)

- ATOMICREPO="/mnt/koji/compose/atomic/$BRANCHED/"

- ATOMICDEST="/pub/fedora/linux/atomic/$BRANCHED/"

- OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora_Docker-26/COMPOSE_ID)

+ OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  # uncomment and edit for resuming a failed compose

  #COMPOSE_ID="Fedora-23-20150530.n.0"

  # assume a releng dir is a git checkout of teh releng repo
@@ -36,13 +33,12 @@ 

      git clone https://pagure.io/releng.git

  fi

  # Set up our fedmsg function, using the releng repo definition

- fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

- fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s"}' "$RELEASE" "$ARCH" "$SHORT")

  FEDMSG_MODNAME="compose"

  FEDMSG_CERTPREFIX="releng"

  . ./releng/scripts/fedmsg-functions.sh

  # Announce that we are starting...

- send_fedmsg "${fedmsg_json_start}" ${DIST} start

+ send_fedmsg "${fedmsg_json_start}" ${RELEASE} start

  pushd $TMPDIR

  git clone https://pagure.io/fedora-comps.git && {

      pushd fedora-comps
@@ -58,27 +54,29 @@ 

      CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID"

  fi

  time $CMD "$@"

- if [ "$?" = "0" ]; then

-    export mail=0

+ if [ "$?" != "0" ]; then

+     exit 1

  fi

- NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora_Docker-26/COMPOSE_ID)

+ 

+ NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

+ 

  DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

  $RSYNCPREFIX mkdir -p $DESTDIR

  # Tell interested persons that the rsync is starting (zomg!)

- send_fedmsg "${fedmsg_json_start}" ${DIST} rsync.start

+ #send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

  #for dir in Docker metadata ;

  #  do

  #    $RSYNCPREFIX rsync -avhH $TARGET_DIR/$NEWCOMPOSE_ID/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

  #  done

  # Tell interested persons that the rsync is done.

- #send_fedmsg "${fedmsg_json_done}" ${DIST} rsync.complete

+ #send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete

  # Tell everyone by fedmsg about the compose

- send_fedmsg "${fedmsg_json_done}" ${DIST} complete

- SUBJECT='Fedora '$DIST' compose report: '$SHORTCOMPOSE_ID' changes'

- #if [ "$mail" = "0" ]; then

- #    for tomail in $TOMAIL ; do

- #        cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

- #             mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

- #    done

- #fi

+ send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete

+ SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'

+ #for tomail in $TOMAIL ; do

+ #    cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

+ #         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

+ #done

file modified
+23 -25
@@ -11,19 +11,16 @@ 

  SKIP_PHASES="--skip-phase=productimg --skip-phase=extra_files"

  DEST=$(pwd)

  DATE=$(date "+%Y%m%d")

- DIST="27"

- BRANCHED="27"

- BRANCH="branched"

+ # the Pungi 'shortname', which we will include in fedmsgs for disambiguation

+ SHORT="Fedora-Atomic"

+ RELEASE="27"

+ RELEASE_TITLE="27"

  COMPSFILE="comps-f27.xml"

- GIT_BRANCH="f27"

- TMPDIR=`mktemp -d /tmp/$DIST.$DATE.XXXX`

+ TMPDIR=`mktemp -d /tmp/$RELEASE.$DATE.XXXX`

  TOMAIL="devel@lists.fedoraproject.org test@lists.fedoraproject.org"

  FROM="Fedora Branched Report <rawhide@fedoraproject.org>"

  RSYNCPREFIX="sudo -u ftpsync"

- ATOMIC=$(mktemp -d /tmp/atomic.${DIST}.$DATE.XXXX)

- ATOMICREPO="/mnt/koji/compose/atomic/$BRANCHED/"

- ATOMICDEST="/pub/fedora/linux/atomic/$BRANCHED/"

- OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora-Atomic-27/COMPOSE_ID)

+ OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  # uncomment and edit for resuming a failed compose

  #COMPOSE_ID="Fedora-23-20150530.n.0"

  # assume a releng dir is a git checkout of teh releng repo
@@ -36,13 +33,12 @@ 

      git clone https://pagure.io/releng.git

  fi

  # Set up our fedmsg function, using the releng repo definition

- fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

- fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s"}' "$RELEASE" "$ARCH" "$SHORT")

  FEDMSG_MODNAME="compose"

  FEDMSG_CERTPREFIX="releng"

  . ./releng/scripts/fedmsg-functions.sh

  # Announce that we are starting...

- send_fedmsg "${fedmsg_json_start}" ${DIST} start

+ send_fedmsg "${fedmsg_json_start}" ${RELEASE} start

  pushd $TMPDIR

  git clone https://pagure.io/fedora-comps.git && {

      pushd fedora-comps
@@ -58,28 +54,30 @@ 

      CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID"

  fi

  time $CMD "$@"

- if [ "$?" = "0" ]; then

-    export mail=0

+ if [ "$?" != "0" ]; then

+     exit 1

  fi

- NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-Fedora-Atomic-27/COMPOSE_ID)

+ 

+ NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID)

  SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g')

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID")

+ 

  DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID

  $RSYNCPREFIX mkdir -p $DESTDIR

  # Tell interested persons that the rsync is starting (zomg!)

- send_fedmsg "${fedmsg_json_start}" ${DIST} rsync.start

+ send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start

  for dir in Atomic CloudImages Docker metadata ;

    do

      $RSYNCPREFIX rsync -avhH $TARGET_DIR/$NEWCOMPOSE_ID/compose/$dir/ /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/$dir/ ;

      $RSYNCPREFIX ./releng/scripts/build_composeinfo /pub/alt/atomic/testing/$SHORTCOMPOSE_ID/ --name $NEWCOMPOSE_ID

    done

  # Tell interested persons that the rsync is done.

- send_fedmsg "${fedmsg_json_done}" ${DIST} rsync.complete

+ send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete

  # Tell everyone by fedmsg about the compose

- send_fedmsg "${fedmsg_json_done}" ${DIST} complete

- SUBJECT='Fedora '$DIST' compose report: '$SHORTCOMPOSE_ID' changes'

- #if [ "$mail" = "0" ]; then

- #    for tomail in $TOMAIL ; do

- #        cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

- #             mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

- #    done

- #fi

+ send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete

+ SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'

+ #for tomail in $TOMAIL ; do

+ #    cat $TARGET_DIR/$NEWCOMPOSE_ID/logs/*verbose $DESTDIR/logs/depcheck | \

+ #         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail

+ #done

These commits make several fixes/improvements to the post-release compose types: Atomic, Docker and Cloud. This PR is for the f27 branch, but I will send a matching PR for master too (I'm not sure which branch is actually used in production for these composes). Together, these commits should improve the content of the fedmsgs for these composes so they include the compose ID where possible and can be distinguished from each other by shortname when the compose ID isn't available. They also prevent all post-compose actions happening when the compose fails, as we already do for pre-release nightly composes, and avoid sending rsync-related fedmsgs for composes where we do not in fact rsync anything anywhere.

3 new commits added

  • Add 'short' key to post-release compose fedmsgs
  • cloud, docker: don't send rsync.start fedmsgs
  • atomic, cloud, docker: correct fedmsgs and bail on fail
6 years ago

3 new commits added

  • Add 'short' key to post-release compose fedmsgs
  • cloud, docker: don't send rsync.start fedmsgs
  • atomic, cloud, docker: correct fedmsgs and bail on fail
6 years ago

not that its a new bug but this should be 27

@ausil well sure, but I figured it's not my job to bump that, and it's not related to what I'm doing in this commit.

Pull-Request has been merged by mohanboddu

6 years ago

@mohanboddu can you also merge #485 , so the fedmsgs from rawhide and f27 post-release nightlies are consistent (all have the 'short' key)? Thanks.

I will work on updating the fedmsg meta stuff.