#19 send_fedmsg refactor and add atomic fedmsgs
Closed 5 years ago by maxamillion. Opened 5 years ago by maxamillion.
maxamillion/releng fedmsg_atomic  into  master

file modified
+16 -21
@@ -55,20 +55,15 @@ 

      echo "$(date --utc) build${DIST}: ${message}" | tee -a "${logfile}" >&2

  }

  

- function send_fedmsg()

- {

-     local log="${1}"

-     local topic="${2}"

-     # Emit a message using bodhi's cert (since we should be running as "masher").

-     echo "{\"log\": \"${log}\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \

-         --cert-prefix bodhi \

-         --modname compose \

-         --topic "${DIST}.${topic}" \

-         --json-input

- }

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")

+ 

+ FEDMSG_MODNAME="compose"

+ FEDMSG_CERTPREFIX="bodhi"

+ . fedmsg-functions.sh

  

  log "started"

- send_fedmsg start start

+ send_fedmsg ${fedmsg_json_start} ${DIST} start

  

  log "blocking retired packages"

  if [ "$ENVIRONMENT" == "production" ]; then
@@ -111,14 +106,14 @@ 

  log "mock setup /etc/hosts"

  $MOCK -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works

  

- send_fedmsg start mash.start

+ send_fedmsg ${fedmsg_json_start} ${DIST} mash.start

  

  log "starting mash"

  # Drop privs here so that we run as the masher UID

  $MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "chown mockbuild:mockbuild /var/cache/mash" || exit 1

  $MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/$BRANCHED -o ${MASHDIR} --compsfile $logdir/${COMPSFILE} $BRANCHED$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1

  

- send_fedmsg done mash.complete

+ send_fedmsg ${fedmsg_json_done} ${DIST} mash.complete

  

  log "finished mash"

  log "starting hardlink"
@@ -158,7 +153,7 @@ 

  log "finished atomic tree creation"

  }

  

- send_fedmsg start pungify.start

+ send_fedmsg ${fedmsg_json_start} ${DIST} pungify.start

  

  log "starting critppath generation"

  #only run critpath on primary arch
@@ -175,7 +170,7 @@ 

  wait

  log "finished pungify"

  

- send_fedmsg done pungify.complete

+ send_fedmsg ${fedmsg_json_done} ${DIST} pungify.complete

  

  log "starting build_composeinfo"

  echo "Running build_composeinfo"
@@ -198,7 +193,7 @@ 

  echo >> $logdir/finish

  log "finished compose"

  

- send_fedmsg start rsync.start

+ send_fedmsg ${fedmsg_json_start} ${DIST} rsync.start

  

  log "started compose sync"

  # data
@@ -213,7 +208,7 @@ 

     export mail=0

  fi

  

- send_fedmsg done rsync.complete

+ send_fedmsg ${fedmsg_json_done} ${DIST} rsync.complete

  

  log "starting sending email report"

  if [ "$mail" = "0" ]; then
@@ -225,7 +220,7 @@ 

  log "finished sending email report"

  

  [ -z "$ARCH" ] && {

- send_fedmsg start image.start

+ send_fedmsg ${fedmsg_json_start} ${DIST} image.start

  

  log "started checking out spin-kickstarts"

  pushd ../
@@ -242,7 +237,7 @@ 

  popd

  popd

  

- send_fedmsg done image.complete

+ send_fedmsg ${fedmsg_json_done} ${DIST} image.complete

  }

  

  for koji in arm ppc s390
@@ -257,6 +252,6 @@ 

    scripts/srpm-excluded-arch.py -a $arches --path ${MASHDIR}/$BRANCHED$EXPANDARCH/source/SRPMS/\*/ >$logdir/excludearch-$koji.log

  done

  

- send_fedmsg done complete

+ send_fedmsg ${fedmsg_json_done} ${DIST} complete

  

  exit 0

@@ -0,0 +1,37 @@ 

+ # fedmsg-functions

+ #

+ # This is a bash shell script that is meant to be sourced by other scripts

+ # and aims to deliver functions common to sending messages to fedmsg[0]

+ # in rel-eng shell scripts.

+ #

+ # [0] - http://www.fedmsg.com/en/latest/

+ 

+ 

+ # NOTES:

+ #   - Scripts that source this should define at least the following:

+ #       FEDMSG_MODNAME

+ #       FEDMSG_CERTPREFIX

+ #

+ #       Example:

+ #

+ #           FEDMSG_MODNAME="compose"

+ #           FEDMSG_CERTPREFIX="bodhi"

+ #           source fedmsg-functions.sh

+ #

+ #           fedmsg_json_start=$(printf '{"log": "start", "branch": "f24", "arch": "x86_64"}')

+ #           send_fedmsg ${fedmsg_json_start} f24 start

+ 

+ function send_fedmsg()

+ {

+     jsoninput="${1}"

+     dist="${2}"

+     topic="${3}"

+ 

+     # Emit a message using bodhi's cert (since we should be running as "masher").

+     printf ${jsoninput} | fedmsg-logger \

+         --cert-prefix ${FEDMSG_CERTPREFIX} \

+         --modname ${FEDMSG_MODNAME} \

+         --topic "${dist}.${topic}" \

+         --json-input

+ }

+ 

file modified
+24 -2
@@ -14,6 +14,10 @@ 

  ATOMIC=/srv/pungi/fedora-atomic/

  ATOMICDEST=/srv/pungi/atomic/

  

+ # Set this for send_fedmsg, will need to handle more appropriately if/when

+ # we support more than just x86_64 for atomic composes/updates

+ DIST="x86_64"

+ 

  SHORTCOMPOSE=$(echo $COMPOSE|sed -e 's|_RC||g')

  SHORTVERSION=$VERSION

  [ -z "$COMPOSE" ] && {
@@ -21,13 +25,21 @@ 

          SHORTCOMPOSE=$(echo $VERSION|sed -e 's|.*_||g')

  }

  

+ FEDMSG_MODNAME="compose"

+ FEDMSG_CERTPREFIX="bodhi"

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s"}' "${BRANCH}")

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s"}' "${BRANCH}")

+ . fedmsg-functions.sh

+ 

  FULLPATH=$(pwd)

  pushd $KICKSTARTS

  git reset --hard

  git checkout f$VERSION 

  git pull --rebase

  

- $FULLPATH/build-cloud-images "$VERSION" "$DATE" "$BRANCH" "$COMPOSE"

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.make-updates.start

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.cloudimg-build.start

+ $FULLPATH/build-cloud-images "$VERSION" "$DATE" "${BRANCH}" "$COMPOSE"

  popd

  

  # use the latest updated kickstarts
@@ -39,6 +51,8 @@ 

  mock -r fedora-$VERSION-compose-x86_64 --init || exit 1

  mock -r fedora-$VERSION-compose-x86_64 --install lorax rpm-ostree || exit 1

  mock -r fedora-$VERSION-compose-x86_64 --shell "mkdir -p ${DESTDIR}/Cloud_Atomic/x86_64/iso; mkdir -p ${DESTDIR}/logs"

+ 

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.atomic-lorax.start

  mock -r fedora-$VERSION-compose-x86_64 --shell "lorax --nomacboot -p Fedora -v ${SHORTVERSION} -r ${SHORTCOMPOSE} \

                                                   -s http://infrastructure.fedoraproject.org/pub/fedora/linux/releases/$VERSION/Everything/x86_64/os/ \

                                                   -i fedora-productimg-atomic \
@@ -56,6 +70,7 @@ 

                                                   ${DESTDIR}/Cloud_Atomic/x86_64/os/ " || exit 1

  mock -r fedora-$VERSION-compose-x86_64 --shell "cp -l ${DESTDIR}/Cloud_Atomic/x86_64/os/images/boot.iso ${DESTDIR}/Cloud_Atomic/x86_64/iso/Fedora-Cloud_Atomic-x86_64-${VERSION}-${DATE}.iso" || exit 1

  mock -r fedora-$VERSION-compose-x86_64 --shell "pushd ${DESTDIR}/Cloud_Atomic/x86_64/iso/ ;sha256sum -b --tag *iso >Fedora-Cloud_Atomic-${VERSION}-x86_64-CHECKSUM; popd" || exit 1

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.atomic-lorax.done

  

  popd

  
@@ -69,6 +84,7 @@ 

      break

    fi

  done

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.cloudimg-build.done

  

  sg releng "mkdir -p $FINALDEST"

  pushd $FINALDEST/
@@ -87,6 +103,7 @@ 

  

  

  

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.cloudimg-checksum.start

  pushd $FINALDEST/Cloud-Images

  for arch in i386 x86_64; do pushd $arch/Images; sg releng "sha256sum --tag -b *qcow2 *raw.xz *box>  Fedora-Cloud-Images-$arch-$VERSION-CHECKSUM"; popd; done

  popd
@@ -98,11 +115,16 @@ 

  pushd $FINALDEST/Docker/x86_64

  sg releng "sha256sum --tag -b *tar.xz>  Fedora-Docker-x86_64-$VERSION-CHECKSUM"

  popd

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.cloudimg-checksum.done

+ 

  

  

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.cloudimg-staging.start

  # stage the composed tree to final locateion

  sg releng "mkdir /pub/alt/stage/$VERSION-$DATE/"

  sg releng "chmod 700 /pub/alt/stage/$VERSION-$DATE/"

- for type in Cloud_Atomic Docker Cloud-Images Cloud-Images-Untested; do sg releng "rsync -avhH $FINALDEST/$type/ /pub/alt/stage/$VERSION-$DATE/$type/"; done

+ for type in Cloud_Atomic Docker Cloud-Images; do sg releng "rsync -avhH $FINALDEST/$type/ /pub/alt/stage/$VERSION-$DATE/$type/"; done

  sg releng "chmod 755 /pub/alt/stage/$VERSION-$DATE/"

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.cloudimg-staging.done

  

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.make-updates.done

no initial comment

We need to add a lot more messages to make-updates for starting/ finishing the compose, making the images gathering the output, etc

we do i386 for cloud images also so this is a bit not right

These shoudl be defined in the parent script

Fixed a couple things and updated based on feedback. Awaiting further review.

I rally do not think we need to send arch specific messages here.

There are two commits that mention that something was merged into something. I guess the commit messages could be better reworded, since from my understanding your branch should be rebased on the master branch. Also pagure says there is a conflict so it cannot be merged.

Changed some things around based on feedback.

I messed this up with a merge and now the rebase is really unclean. Closing this in favor of a new PR

https://pagure.io/releng/pull-request/27