fedora-infra / ansible

Clone
Source Code
GIT

#775 Enable CentOS Stream 9 in MirrorManager
Merged 2 years ago by smooge. Opened 2 years ago by adrian.
fedora-infra/ adrian/ansible 2021-09-06-enable-stream-in-mm  into  main

file modified
+3 
@@ -18,6 +18,9 @@ 
 

  15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
 

  45 */2  * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log
 

  

+ # check if category CentOS needs updating once a day
 

+ 15 20 * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log
 

+ 

  # Sync netblocks list once a day
 

  30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
 

  # Run the script for Internet2 netblocks very late to

file modified
+14 -7 
@@ -30,6 +30,19 @@ 
 

  	CATEGORY="Archive"
 

  elif [ "${1}" ==  "codecs" ]; then
 

  	CATEGORY="Codecs"
 

+ 	/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
 

+ 	exit 0
 

+ elif [ "${1}" ==  "centos" ]; then
 

+ 	CATEGORY="CentOS"
 

+ 	SCANNER="/usr/local/bin/scan-primary-mirror -c /etc/mirrormanager/scan-primary-mirror-centos.toml"
 

+ 	/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\""
 

+ 	if [ "$?" -eq "0" ]; then
 

+ 		echo -n "Finished umdl for ${CATEGORY} successfully at "
 

+ 	else
 

+ 		echo -n "${SCANNER} for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
 

+ 	fi
 

+ 	date
 

+ 	exit 0
 

  fi
 

  

  if [ -e /var/run/mirrormanager/umdl-${1} ]; then
 
@@ -39,13 +52,7 @@ 
 

  	let LASTRUN=CURDATE-86400
 

  fi
 

  

- # FULLFILETIMELIST -> FFTL
 

- if [ "${1}" ==  "codecs" ]; then
 

-     /usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
 

-     exit 0
 

- else
 

-     FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
 

- fi
 

+ FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
 

  FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null`
 

  

  if [ "$?" -eq "1" ]; then

file modified
+9 -2 
@@ -161,6 +161,13 @@ 
 

      owner: mirrormanager
 

      group: mirrormanager
 

      mode: 0600
 

+ - name: install scan-primary-mirror centos configuration file
 

+   template:
 

+     src: scan-primary-mirror-centos.toml
 

+     dest: /etc/mirrormanager/scan-primary-mirror-centos.toml
 

+     owner: mirrormanager
 

+     group: mirrormanager
 

+     mode: 0600
 

  - name: create /srv/mirrorlist-server.git
 

    file: path=/srv/mirrorlist-server.git state=directory owner=mirrormanager group=mirrormanager mode=0755
 

  - name: create /srv/scan-primary-mirror.git
 
@@ -168,7 +175,7 @@ 
 

  - name: checkout mirrorlist-server
 

    git:
 

      repo: https://github.com/adrianreber/mirrorlist-server.git
 

-     version: 3.0.4
 

+     version: 3.0.5
 

      dest: /srv/mirrorlist-server.git
 

    become: yes
 

    become_user: mirrormanager
 
@@ -176,7 +183,7 @@ 
 

  - name: checkout scan-primary-mirror
 

    git:
 

      repo: https://github.com/adrianreber/scan-primary-mirror.git
 

-     version: 0.1.1
 

+     version: 0.2.0
 

      dest: /srv/scan-primary-mirror.git
 

    become: yes
 

    become_user: mirrormanager

file added
+41 
@@ -0,0 +1,41 @@ 
 

+ common_rsync_options="--no-motd --timeout 14400"
 

+ max_propagation_days = 2
 

+ max_stale_days = 3
 

+ skip_repository_paths = ["SIGs"]
 

+ 

+ [database]
 

+ url="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}"
 

+ 

+ [[category]]
 

+ name="CentOS"
 

+ type="rsync"
 

+ url="rsync://mirror.stream.centos.org/CentOS-Stream-All"
 

+ checksum_base="http://mirror.stream.centos.org/"
 

+ 

+ [[repository_mapping]]
 

+ regex="^\\d+(?:-stream)?/AppStream/"
 

+ prefix="centos-appstream"
 

+ 

+ [[repository_mapping]]
 

+ regex="^\\d+(?:-stream)?/BaseOS/"
 

+ prefix="centos-baseos"
 

+ 

+ [[repository_mapping]]
 

+ regex="^\\d+(?:-stream)?/CRB/"
 

+ prefix="centos-crb"
 

+ 

+ [[repository_mapping]]
 

+ regex="^\\d+(?:-stream)?/HighAvailability/"
 

+ prefix="centos-highavailability"
 

+ 

+ [[repository_mapping]]
 

+ regex="^\\d+(?:-stream)?/NFV/"
 

+ prefix="centos-nfv"
 

+ 

+ [[repository_mapping]]
 

+ regex="^\\d+(?:-stream)?/RT/"
 

+ prefix="centos-rt"
 

+ 

+ [[repository_mapping]]
 

+ regex="^\\d+(?:-stream)?/ResilientStorage/"
 

+ prefix="centos-resilientstorage"

file modified
+3 
@@ -33,3 +33,6 @@ 
 

  

  # Category: 'Fedora Other'; once a day, 10 threads
 

  0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
 

+ 

+ # Category: 'CentOS'; twice a day, 20 threads
 

+ 0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1

In addition to configuration file changes this also needs the latest
version of MirrorManager, scan-primary-mirror and mirrorlist-server.

The reason for the software updates are mainly that CentOS Stream is
using an empty topdir ('') and all around the code there were
assumptions that topdir always ends with a slash and an empty string
does not end with a slash.

Build succeeded.

Shouldn't this not say "Fedora" here?

rebased onto 4858a3b3fd49b26d8cc6d8df952fe70097809358
2 years ago

Shouldn't this not say "Fedora" here?

Right. I removed it. Thanks.

rebased onto 353449fc704c2ee40dfb0c72e1b811769fed6958
2 years ago

Build succeeded.

Does this need to be taged as Freeze Break Request or it could wait for end of the Fedora Beta freeze?

Does this need to be taged as Freeze Break Request or it could wait for end of the Fedora Beta freeze?

I requested a freeze break request here: https://lists.fedoraproject.org/archives/list/infrastructure@lists.fedoraproject.org/thread/VWPLTQ7SNQZJ2ROI6IYWNVW2M5WHKG36/

Right now I am waiting for additional feedback if we want to wait or not.

So, I don't see much more discussion on list... but I am +1 to freeze break.

+1 FBR.

I'm not subscribed on the infra list, so I'll comment here instead. Based on my current understanding, this code change is low risk, it works well in staging, and has a big benefit to unblocking the c9s mirroring. I'm +1 to the freeze break.

rebased onto 359e4550908489afa77bc1bc82a5fb664c2be244
2 years ago

Build succeeded.

I see three +1 but I am not sure if @ngompa and @carlgeorge are part of the sysadmin-main or sysadmin-releng group which I need to have two +1 from (according to the documentation).

Waiting for one more +1 from sysadmin-main or sysadmin-releng group.

Metadata Update from @zlopez:
- Pull-request tagged with: freeze-break-request
2 years ago

@mobrien @humaton Could you review this PR?

+1

The people who can give a real +1 need to be in sysadmin-main. I have looked this over and am giving it a +1 and will merge.

rebased onto f3c9f3b
2 years ago

rebased onto f3c9f3b
2 years ago

Build succeeded.

Pull-Request has been merged by smooge
2 years ago

Thanks. I will try to enable this tomorrow.
Metadata
None

freeze-break-request

Changes Summary 5
+3 -0
file changed
roles/mirrormanager/backend/files/backend.cron
+14 -7
file changed
roles/mirrormanager/backend/files/umdl-required
+9 -2
file changed
roles/mirrormanager/backend/tasks/main.yml
+41
file added
roles/mirrormanager/backend/templates/scan-primary-mirror-centos.toml
+3 -0
file changed
roles/mirrormanager/crawler/files/crawler.cron
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.