#119 Setup rhel and centos sync scripts
Merged 3 years ago by smooge. Opened 3 years ago by mohanboddu.
fedora-infra/ mohanboddu/ansible batcave-sync  into  master

@@ -0,0 +1,19 @@ 

+ #!/bin/sh

+ 

+ RSYNC_DESTDIR=/mnt/fedora/app/fi-repo/centos/centos-8/

+ RSYNC_MOD="centos-epel/centos/8.1.1911/"

+ 

+ REPO_FROMDIR=/mnt/fedora/app/fi-repo/centos/centos-8/

+ 

+ mkdir -p ${RSYNC_DESTDIR}

+ 

+ RSYNC='/usr/bin/rsync'

+ RS_OPT="-avSHP  --numeric-ids"

+ #RS_DEADLY="--delete --delete-excluded --delete-delay --delay-updates"

+ RS_DEADLY="--delete --delete-delay --delay-updates"

+ CENT_EXCLUDES="--exclude=isos --exclude=kickstart --exclude=s390x"

+ 

+ SERVER=centos-master01.iad2.fedoraproject.org

+ 

+ ${RSYNC} ${RS_OPT} ${RS_DEADLY} ${CENT_EXCLUDES} ${SERVER}::${RSYNC_MOD} ${RSYNC_DESTDIR}

+ 

@@ -0,0 +1,47 @@ 

+ #!/bin/sh

+ 

+ RHEL6SYNCDIR=/srv/web/repo/rhel/rhel6

+ RHEL6CACHEDIR=/var/tmp/reposync-rhel6

+ 

+ cd $RHEL6SYNCDIR

+ 

+ 

+ mkdir -p ${RHEL6CACHEDIR}

+ for i in i686 x86_64 ppc64; do

+ mkdir -p ${RHEL6CACHEDIR}/${i}

+ done

+ 

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-i386 -p i386 -a i686 --metadata-path ${RHEL6CACHEDIR}/i686

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-optional-i386 -p i386 -a i686 --metadata-path ${RHEL6CACHEDIR}/i686

+ # 2019-09-17 RHEL does not have these anymore for i386

+ #reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-ha-i386 -p i386 -a i686

+ #reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-lb-i386 -p i386 -a i686

+ 

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-x86_64 -p x86_64  --metadata-path ${RHEL6CACHEDIR}/x86_64

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-optional-x86_64 -p x86_64  --metadata-path ${RHEL6CACHEDIR}/x86_64

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-ha-x86_64 -p x86_64  --metadata-path ${RHEL6CACHEDIR}/x86_64

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-lb-x86_64 -p x86_64  --metadata-path ${RHEL6CACHEDIR}/x86_64

+ 

+ #

+ # rhel6 didn't have lb or ha repos for ppc64

+ #

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-ppc64.conf -t -n --download-metadata --repoid rhel-6-server-ppc64 -p ppc64 -a ppc64  --metadata-path ${RHEL6CACHEDIR}/ppc64

+ reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-ppc64.conf -t -n --download-metadata --repoid rhel-6-server-optional-ppc64 -p ppc64 -a ppc64  --metadata-path ${RHEL6CACHEDIR}/ppc64

+ 

+ for i in `ls -d /srv/web/repo/rhel/rhel6/i386/*`

+ do

+         cd $i

+         createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 . 

+ done

+ 

+ for i in `ls -d /srv/web/repo/rhel/rhel6/x86_64/*`

+ do

+         cd $i

+         createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 . 

+ done

+ 

+ for i in `ls -d /srv/web/repo/rhel/rhel6/ppc64/*`

+ do

+         cd $i

+         createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 . 

+ done

@@ -0,0 +1,52 @@ 

+ #!/bin/sh

+ 

+ RHEL7SYNCDIR=/srv/web/repo/rhel/rhel7

+ SYNCDIR=/var/cache/reposync/

+ YUMDIR=/var/tmp/reposync-rhel7/

+ 

+ ## Need to download the metadata and the modular data

+ ROPTS='--download-metadata -m'

+ 

+ # s390 isnt working yet

+ ARCHES="ppc64le s390x x86_64"

+ #ARCHES="x86_64"

+ 

+ mkdir -vp ${SYNCDIR} ${RHEL7SYNCDIR} ${YUMDIR}

+ for A in ${ARCHES}; do

+     mkdir -vp ${YUMDIR}/${A}

+ done

+ 

+ cd $RHEL7SYNCDIR

+ 

+ ##

+ ## Sync out RHEL-7 for appropriate arches

+ for A in ${ARCHES}; do

+     rm -rf ${SYNCDIR}/${A}

+     CONF=yum-batcave-rhel7-${A}.conf

+     REPOS="rhel-7-for-${A}-server-rpms rhel-7-ha-for-${A}-server-rpms rhel-7-extras-for-${A}-server-rpms rhel-7-optional-for-${A}-server-rpms rhel-7-devtools-for-${A}-server-rpms rhel-7-rhscl-for-${A}-server-rpms"

+ #    REPOS="rhel-7-rhscl-for-${A}-server-rpms"

+     for R in ${REPOS}; do

+         rm -rf ${YUMDIR}/${R}

+         # Test to see if we have this repository in our tree

+         grep -q ${R} ${RHEL7SYNCDIR}/${CONF}

+         if [[ $? -eq 0 ]]; then 

+             echo "Syncing ${R} for ${A}"

+             reposync -c ${RHEL7SYNCDIR}/${CONF} --repoid ${R} -a ${A} -p ${A} --metadata-path ${YUMDIR}/${A}

+             if [[ $? -ne 0 ]]; then

+                 echo "Something went wrong with reposync -c ${RHEL7SYNCDIR}/${CONF} --repoid ${R} -a ${A} -a noarch -p ${A} --metadata-path ${YUMDIR}/${A}"

+                 continue

+             fi

+             ## Try to create repo

+             pushd ${RHEL7SYNCDIR}/${A}/${R} &> /dev/null

+             createrepo_c -q --update --workers 6 . 

+             if [[ $? -ne 0 ]]; then

+                 echo "Something went wrong with createrepo ${A} ${R}"

+                 continue

+             fi

+             popd &> /dev/null

+         fi

+     done

+ done

+ 

+ #ostree pull --repo=/mnt/fedora/app/fi-repo/rhel/rhel7/atomic/rhel7 --mirror redhat

+ ## This section is for all the trees which are not easily scriptable

@@ -0,0 +1,1 @@ 

+ 03 09  * * * root /usr/local/bin/rhel8-split.sh

@@ -0,0 +1,99 @@ 

+ #!/bin/bash

+ HOMEDIR=/mnt/fedora/app/fi-repo/rhel/rhel8

+ BINDIR=/usr/local/bin

+ 

+ ARCHES="aarch64 ppc64le s390x x86_64"

+ DATE=$(date -Ih | sed 's/+.*//')

+ 

+ DATEDIR=${HOMEDIR}/koji/${DATE}

+ 

+ if [ -d ${DATEDIR} ]; then

+     echo "Directory already exists. Please remove or fix"

+     exit

+ else

+ mkdir -p ${DATEDIR}

+ fi

+ 

+ for ARCH in ${ARCHES}; do

+     # The archdir is where we daily download updates for rhel8

+     ARCHDIR=${HOMEDIR}/${ARCH}

+     if [ ! -d ${ARCHDIR} ]; then

+         echo "Unable to find ${ARCHDIR}"

+         exit

+     fi

+ 

+     # We consolidate all of the default repositories and remerge them

+     # in a daily tree. This allows us to point koji at a particular

+     # day if we have specific build concerns.

+     OUTDIR=${DATEDIR}/${ARCH}

+     mkdir -p ${OUTDIR}

+     if [ ! -d ${OUTDIR} ]; then

+         echo "Unable to find ${ARCHDIR}"

+         exit

+     else

+         cd ${OUTDIR}

+     fi

+ 

+     # Begin splitting the various packages into their subtrees

+     ${BINDIR}/splitter.py --action hardlink --target RHEL-8-001 ${ARCHDIR}/rhel-8-for-${ARCH}-baseos-rpms/ --only-defaults &> /dev/null

+     if [ $? -ne 0 ]; then

+         echo "splitter ${ARCH} baseos failed"

+         exit

+     fi

+     ${BINDIR}/splitter.py --action hardlink --target RHEL-8-002 ${ARCHDIR}/rhel-8-for-${ARCH}-appstream-rpms/ --only-defaults &> /dev/null

+     if [ $? -ne 0 ]; then

+         echo "splitter ${ARCH} appstream failed"

+         exit

+     fi

+     ${BINDIR}/splitter.py --action hardlink --target RHEL-8-003 ${ARCHDIR}/codeready-builder-for-rhel-8-${ARCH}-rpms/ &> /dev/null

+     if [ $? -ne 0 ]; then

+         echo "splitter ${ARCH} codeready failed"

+         exit

+     fi

+ 

+     # Copy the various module trees into RHEL-8-001 where we want them

+     # to work.

+     echo "Moving data to ${ARCH}/RHEL-8-001"

+     cp -anlr RHEL-8-002/* RHEL-8-001

+     cp -anlr RHEL-8-003/* RHEL-8-001

+     # Go into the main tree

+     pushd RHEL-8-001

+ 

+     # Mergerepo didn't work so lets just createrepo in the top directory.

+     createrepo_c .  &> /dev/null

+     popd

+ 

+     # Cleanup the trash

+     rm -rf RHEL-8-002 RHEL-8-003

+ #loop to the next

+ done

+ 

+ ## Set up the builds so they are pointing to the last working version

+ cd ${HOMEDIR}/koji/

+ if [[ -e staged ]]; then

+     if [[ -h staged ]]; then

+         rm -f staged

+     else

+         echo "Unable to remove staged. it is not a symbolic link"

+         exit

+     fi

+ else

+     echo "No staged link found"

+ fi

+ 

+ echo "Linking ${DATE} to staged"

+ ln -s ${DATE} staged

+ 

+ 

+ for ARCH in ${ARCHES}; do

+     pushd latest/

+     mkdir -p ${ARCH}

+     dnf --disablerepo=\* --enablerepo=RHEL-8-001 --repofrompath=RHEL-8-001,https://infrastructure.fedoraproject.org/repo/rhel/rhel8/koji/staged/${ARCH}/RHEL-8-001/ reposync -a ${ARCH} -a noarch -p ${ARCH} --newest --delete  &> /dev/null

+     if [[ $? -eq 0 ]]; then

+         cd ${ARCH}/RHEL-8-001

+         createrepo_c .  &> /dev/null

+     else

+         echo "Unable to run createrepo on latest/${ARCH}"

+     fi

+     popd

+ done

@@ -0,0 +1,70 @@ 

+ #!/bin/sh

+ 

+ ## 

+ # Setup the directories we will sync down to.

+ RHEL8SYNCDIR=/srv/web/repo/rhel/rhel8

+ SYNCDIR=/var/cache/reposync/

+ 

+ ## 

+ # Need to download the metadata and the modular data

+ DNFOPTS='--download-metadata -m --setopt=module_platform_id="platform:el8"'

+ # Need to set the arches we are going to sync down

+ ARCHES="aarch64 ppc64le s390x x86_64"

+ 

+ 

+ ##

+ # Do the deeds

+ cd ${RHEL8SYNCDIR}

+ 

+ # Make the sync dir where dnf will drop stuff. This disappeared once

+ # and caused the script to die so we do this explicitely now.

+ mkdir -vp ${SYNCDIR}

+ 

+ ##

+ ## Sync Out RHEL-8 for appropriate arches

+ for A in ${ARCHES}; do

+ 

+     # In case we have a bad sync from before, make sure the old sync

+     # data is destroyed.

+     rm -rf ${SYNCDIR}/${A}

+ 

+     # Configuration files for yum4 should fit this pattern.

+     CONF=yum4-batcave-rhel8-${A}.conf

+ 

+     # Setup the repos you are going to sync out of the config file in

+     # this pattern. This makes it easier for koji configs later

+     REPOS="rhel-8-for-${A}-appstream-rpms rhel-8-for-${A}-baseos-rpms codeready-builder-for-rhel-8-${A}-rpms"

+ 

+     # We do a 2 level reposync because batcave is RHEL-7 and reposync

+     # is what is default. However reposync does not pull in certain

+     # data so we have to repeat this with dnf to get the metadata

+     # needed for modularity.

+     for R in ${REPOS}; do

+         echo "Syncing ${R} for ${A}"

+         reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -p ${A} &>  /dev/null

+         # Be verbose on errors.

+         if [[ $? -ne 0 ]]; then

+             echo "Something went wrong with reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -a noarch -p ${A}"

+         fi

+         # x86_64 is 'special' because we have to tell it for i686 also.

+         if [[ ${A} == 'x86_64' ]]; then

+            dnf reposync -c ${RHEL8SYNCDIR}/${CONF} ${DNFOPTS}  --repo=${R} -a ${A} -a noarch -a i686 -p ${A} &> /dev/null

+         else

+            dnf reposync -c ${RHEL8SYNCDIR}/${CONF} ${DNFOPTS}  --repo=${R} -a ${A} -a noarch -p ${A} &> /dev/null

+         fi

+         if [[ $? -ne 0 ]]; then

+             echo "Something went wrong with dnf reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -p ${A}"

+         fi

+     done

+ done

+ 

+ ### Here we put in syncs which do not fit in with the general do this

+ ### for all architectures. In this case we only copy down openstack

+ ### for x86_64 and source because we really only care about rabbitmq

+ ### We put this in an 'other' directory so that grobisplitter and any

+ ### other tools don't accidently pull it into their work.

+ 

+ OTHERCONF=yum4-batcave-rhel8-other.conf

+ 

+ reposync -c ${RHEL8SYNCDIR}/${OTHERCONF} -r rhel-8-x86_64-openstack-16-rpms -a x86_64 -p other &> /dev/null

+ reposync -c ${RHEL8SYNCDIR}/${OTHERCONF} -r rhel-8-srpms-openstack-16-rpms --source -a x86_64 -p other &> /dev/null

@@ -326,6 +326,37 @@ 

    when: inventory_hostname.startswith('batcave01')

  

  #

+ # Setup rhel6 sync script.

+ #

+ 

+ - name: setup rhel6 sync script

+   copy: src=rhel6-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel6/rhel6-sync mode=0775

+   tags:

+   - batcave

+   - config

+   when: inventory_hostname.startswith('batcave01')

+ #

+ # Setup rhel7 sync script.

+ #

+ 

+ - name: setup rhel7 sync copy script

+   copy: src=rhel7-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel7/rhel7-sync mode=0775

+   tags:

+   - batcave

+   - config

+   when: inventory_hostname.startswith('batcave01')

+ #

+ # Setup rhel8 sync script.

+ #

+ 

+ - name: setup rhel8 sync copy script

+   copy: src=rhel8-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel8/rhel8-sync mode=0775

+   tags:

+   - batcave

+   - config

+   when: inventory_hostname.startswith('batcave01')

+ 

+ #

  # sync-rhn cron job

  #

  - name: setup sync-rhn cron
@@ -335,6 +366,18 @@ 

    - config

    when: inventory_hostname.startswith('batcave01')

  

+ 

+ #

+ # Setup centos sync script.

+ #

+ 

+ - name: setup centos sync script

+   copy: src=sync-centos dest=/mnt/fedora/app/fi-repo/centos/centos-8-sync mode=0775

+   tags:

+   - batcave

+   - config

+   when: inventory_hostname.startswith('batcave01')

+ 

  #

  # sync-centos cron job

  #
@@ -345,6 +388,26 @@ 

    - config

    when: inventory_hostname.startswith('batcave01')

  

+ #

+ # Setup rhel8-split script.

+ #

+ 

+ - name: setup rhel8-split script

+   copy: src=rhel8-split.sh dest=/usr/local/bin/rhel8-split.sh mode=0755

+   tags:

+   - batcave

+   - config

+   when: inventory_hostname.startswith('batcave01')

+ 

+ #

+ # Setup rhel8-split cron job

+ #

+ - name: setup rhel8-split cron

+   copy: src=rhel8-split.cron dest=/etc/cron.d/rhel8-split.cron mode=0644

+   tags:

+   - batcave

+   - config

+   when: inventory_hostname.startswith('batcave01')

  

  #

  # Setup web server config

I went with batcave role since there are couple of files/crons that are setup on batcave using batcave role, so I thought of not splitting reposync's to its own role.

If its desired, let me know I can update this PR.

rebased onto 9fd6a3602a6d25492c86afa968ca3a8da53c7916

3 years ago

rebased onto 9fd6a3602a6d25492c86afa968ca3a8da53c7916

3 years ago

Hold on sec, I missed something here.

rebased onto 7fd87747e0371f65147f486b8b03e5e692f00ed1

3 years ago

Okay, this should be good now.

rebased onto bbd521401a91d1c618bc13cdc75613d7af7cf531

3 years ago

rebased onto bbd521401a91d1c618bc13cdc75613d7af7cf531

3 years ago

Should we add to the files or to the commit what these scripts are and do?

Should we add to the files or to the commit what these scripts are and do?

This might require its own infra doc.

rebased onto 8ba8429

3 years ago

rebased onto 8ba8429

3 years ago

Pull-Request has been merged by smooge

3 years ago