releng

Clone

#19 send_fedmsg refactor and add atomic fedmsgs
Closed 8 years ago by maxamillion. Opened 8 years ago by maxamillion.
maxamillion/releng fedmsg_atomic  into  master

file modified
+16 -21 
@@ -55,20 +55,15 @@ 
 

      echo "$(date --utc) build${DIST}: ${message}" | tee -a "${logfile}" >&2
 

  }
 

  

- function send_fedmsg()
 

- {
 

-     local log="${1}"
 

-     local topic="${2}"
 

-     # Emit a message using bodhi's cert (since we should be running as "masher").
 

-     echo "{\"log\": \"${log}\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
 

-         --cert-prefix bodhi \
 

-         --modname compose \
 

-         --topic "${DIST}.${topic}" \
 

-         --json-input
 

- }
 

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")
 

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s"}' "$BRANCHED" "$ARCH")
 

+ 

+ FEDMSG_MODNAME="compose"
 

+ FEDMSG_CERTPREFIX="bodhi"
 

+ . fedmsg-functions.sh
 

  

  log "started"
 

- send_fedmsg start start
 

+ send_fedmsg ${fedmsg_json_start} ${DIST} start
 

  

  log "blocking retired packages"
 

  if [ "$ENVIRONMENT" == "production" ]; then
 
@@ -111,14 +106,14 @@ 
 

  log "mock setup /etc/hosts"
 

  $MOCK -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works
 

  

- send_fedmsg start mash.start
 

+ send_fedmsg ${fedmsg_json_start} ${DIST} mash.start
 

  

  log "starting mash"
 

  # Drop privs here so that we run as the masher UID
 

  $MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "chown mockbuild:mockbuild /var/cache/mash" || exit 1
 

  $MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/$BRANCHED -o ${MASHDIR} --compsfile $logdir/${COMPSFILE} $BRANCHED$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1
 

  

- send_fedmsg done mash.complete
 

+ send_fedmsg ${fedmsg_json_done} ${DIST} mash.complete
 

  

  log "finished mash"
 

  log "starting hardlink"
 
@@ -158,7 +153,7 @@ 
 

  log "finished atomic tree creation"
 

  }
 

  

- send_fedmsg start pungify.start
 

+ send_fedmsg ${fedmsg_json_start} ${DIST} pungify.start
 

  

  log "starting critppath generation"
 

  #only run critpath on primary arch
 
@@ -175,7 +170,7 @@ 
 

  wait
 

  log "finished pungify"
 

  

- send_fedmsg done pungify.complete
 

+ send_fedmsg ${fedmsg_json_done} ${DIST} pungify.complete
 

  

  log "starting build_composeinfo"
 

  echo "Running build_composeinfo"
 
@@ -198,7 +193,7 @@ 
 

  echo >> $logdir/finish
 

  log "finished compose"
 

  

- send_fedmsg start rsync.start
 

+ send_fedmsg ${fedmsg_json_start} ${DIST} rsync.start
 

  

  log "started compose sync"
 

  # data
 
@@ -213,7 +208,7 @@ 
 

     export mail=0
 

  fi
 

  

- send_fedmsg done rsync.complete
 

+ send_fedmsg ${fedmsg_json_done} ${DIST} rsync.complete
 

  

  log "starting sending email report"
 

  if [ "$mail" = "0" ]; then
 
@@ -225,7 +220,7 @@ 
 

  log "finished sending email report"
 

  

  [ -z "$ARCH" ] && {
 

- send_fedmsg start image.start
 

+ send_fedmsg ${fedmsg_json_start} ${DIST} image.start
 

  

  log "started checking out spin-kickstarts"
 

  pushd ../
 
@@ -242,7 +237,7 @@ 
 

  popd
 

  popd
 

  

- send_fedmsg done image.complete
 

+ send_fedmsg ${fedmsg_json_done} ${DIST} image.complete
 

  }
 

  

  for koji in arm ppc s390
 
@@ -257,6 +252,6 @@ 
 

    scripts/srpm-excluded-arch.py -a $arches --path ${MASHDIR}/$BRANCHED$EXPANDARCH/source/SRPMS/\*/ >$logdir/excludearch-$koji.log
 

  done
 

  

- send_fedmsg done complete
 

+ send_fedmsg ${fedmsg_json_done} ${DIST} complete
 

  

  exit 0

file added
+37 
@@ -0,0 +1,37 @@ 
 

+ # fedmsg-functions
 

+ #
 

+ # This is a bash shell script that is meant to be sourced by other scripts
 

+ # and aims to deliver functions common to sending messages to fedmsg[0]
 

+ # in rel-eng shell scripts.
 

+ #
 

+ # [0] - http://www.fedmsg.com/en/latest/
 

+ 

+ 

+ # NOTES:
 

+ #   - Scripts that source this should define at least the following:
 

+ #       FEDMSG_MODNAME
 

+ #       FEDMSG_CERTPREFIX
 

+ #
 

+ #       Example:
 

+ #
 

+ #           FEDMSG_MODNAME="compose"
 

+ #           FEDMSG_CERTPREFIX="bodhi"
 

+ #           source fedmsg-functions.sh
 

+ #
 

+ #           fedmsg_json_start=$(printf '{"log": "start", "branch": "f24", "arch": "x86_64"}')
 

+ #           send_fedmsg ${fedmsg_json_start} f24 start
 

+ 

+ function send_fedmsg()
 

+ {
 

+     jsoninput="${1}"
 

+     dist="${2}"
 

+     topic="${3}"
 

+ 

+     # Emit a message using bodhi's cert (since we should be running as "masher").
 

+     printf ${jsoninput} | fedmsg-logger \
 

+         --cert-prefix ${FEDMSG_CERTPREFIX} \
 

+         --modname ${FEDMSG_MODNAME} \
 

+         --topic "${dist}.${topic}" \
 

+         --json-input
 

+ }
 

+

file modified
+24 -2 
@@ -14,6 +14,10 @@ 
 

  ATOMIC=/srv/pungi/fedora-atomic/
 

  ATOMICDEST=/srv/pungi/atomic/
 

  

+ # Set this for send_fedmsg, will need to handle more appropriately if/when
 

+ # we support more than just x86_64 for atomic composes/updates
 

+ DIST="x86_64"
 

+ 

  SHORTCOMPOSE=$(echo $COMPOSE|sed -e 's|_RC||g')
 

  SHORTVERSION=$VERSION
 

  [ -z "$COMPOSE" ] && {
 
@@ -21,13 +25,21 @@ 
 

          SHORTCOMPOSE=$(echo $VERSION|sed -e 's|.*_||g')
 

  }
 

  

+ FEDMSG_MODNAME="compose"
 

+ FEDMSG_CERTPREFIX="bodhi"
 

+ fedmsg_json_start=$(printf '{"log": "start", "branch": "%s"}' "${BRANCH}")
 

+ fedmsg_json_done=$(printf '{"log": "done", "branch": "%s"}' "${BRANCH}")
 

+ . fedmsg-functions.sh
 

+ 

  FULLPATH=$(pwd)
 

  pushd $KICKSTARTS
 

  git reset --hard
 

  git checkout f$VERSION 

  git pull --rebase
 

  

- $FULLPATH/build-cloud-images "$VERSION" "$DATE" "$BRANCH" "$COMPOSE"
 

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.make-updates.start
 

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.cloudimg-build.start
 

+ $FULLPATH/build-cloud-images "$VERSION" "$DATE" "${BRANCH}" "$COMPOSE"
 

  popd
 

  

  # use the latest updated kickstarts
 
@@ -39,6 +51,8 @@ 
 

  mock -r fedora-$VERSION-compose-x86_64 --init || exit 1
 

  mock -r fedora-$VERSION-compose-x86_64 --install lorax rpm-ostree || exit 1
 

  mock -r fedora-$VERSION-compose-x86_64 --shell "mkdir -p ${DESTDIR}/Cloud_Atomic/x86_64/iso; mkdir -p ${DESTDIR}/logs"
 

+ 

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.atomic-lorax.start
 

  mock -r fedora-$VERSION-compose-x86_64 --shell "lorax --nomacboot -p Fedora -v ${SHORTVERSION} -r ${SHORTCOMPOSE} \
 

                                                   -s http://infrastructure.fedoraproject.org/pub/fedora/linux/releases/$VERSION/Everything/x86_64/os/ \
 

                                                   -i fedora-productimg-atomic \
 
@@ -56,6 +70,7 @@ 
 

                                                   ${DESTDIR}/Cloud_Atomic/x86_64/os/ " || exit 1
 

  mock -r fedora-$VERSION-compose-x86_64 --shell "cp -l ${DESTDIR}/Cloud_Atomic/x86_64/os/images/boot.iso ${DESTDIR}/Cloud_Atomic/x86_64/iso/Fedora-Cloud_Atomic-x86_64-${VERSION}-${DATE}.iso" || exit 1
 

  mock -r fedora-$VERSION-compose-x86_64 --shell "pushd ${DESTDIR}/Cloud_Atomic/x86_64/iso/ ;sha256sum -b --tag *iso >Fedora-Cloud_Atomic-${VERSION}-x86_64-CHECKSUM; popd" || exit 1
 

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.atomic-lorax.done
 

  

  popd
 

  
@@ -69,6 +84,7 @@ 
 

      break
 

    fi
 

  done
 

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.cloudimg-build.done
 

  

  sg releng "mkdir -p $FINALDEST"
 

  pushd $FINALDEST/
 
@@ -87,6 +103,7 @@ 
 

  

  

  

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.cloudimg-checksum.start
 

  pushd $FINALDEST/Cloud-Images
 

  for arch in i386 x86_64; do pushd $arch/Images; sg releng "sha256sum --tag -b *qcow2 *raw.xz *box>  Fedora-Cloud-Images-$arch-$VERSION-CHECKSUM"; popd; done
 

  popd
 
@@ -98,11 +115,16 @@ 
 

  pushd $FINALDEST/Docker/x86_64
 

  sg releng "sha256sum --tag -b *tar.xz>  Fedora-Docker-x86_64-$VERSION-CHECKSUM"
 

  popd
 

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.cloudimg-checksum.done
 

+ 

  

  

+ send_fedmsg ${fedmsg_json_start} ${BRANCH} compose.cloudimg-staging.start
 

  # stage the composed tree to final locateion
 

  sg releng "mkdir /pub/alt/stage/$VERSION-$DATE/"
 

  sg releng "chmod 700 /pub/alt/stage/$VERSION-$DATE/"
 

- for type in Cloud_Atomic Docker Cloud-Images Cloud-Images-Untested; do sg releng "rsync -avhH $FINALDEST/$type/ /pub/alt/stage/$VERSION-$DATE/$type/"; done
 

+ for type in Cloud_Atomic Docker Cloud-Images; do sg releng "rsync -avhH $FINALDEST/$type/ /pub/alt/stage/$VERSION-$DATE/$type/"; done
 

  sg releng "chmod 755 /pub/alt/stage/$VERSION-$DATE/"
 

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.cloudimg-staging.done
 

  

+ send_fedmsg ${fedmsg_json_done} ${BRANCH} compose.make-updates.done

no initial comment

We need to add a lot more messages to make-updates for starting/ finishing the compose, making the images gathering the output, etc

we do i386 for cloud images also so this is a bit not right

These shoudl be defined in the parent script

Fixed a couple things and updated based on feedback. Awaiting further review.

I rally do not think we need to send arch specific messages here.

There are two commits that mention that something was merged into something. I guess the commit messages could be better reworded, since from my understanding your branch should be rebased on the master branch. Also pagure says there is a conflict so it cannot be merged.

Changed some things around based on feedback.

I messed this up with a merge and now the rebase is really unclean. Closing this in favor of a new PR

https://pagure.io/releng/pull-request/27
Metadata
None
No Tags
Changes Summary 3
+16 -21
file changed
scripts/build-functions.sh
+37
file added
scripts/fedmsg-functions.sh
+24 -2
file changed
scripts/make-updates
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.