#1301 Clean /srv/pub/alt/linuxsystemroles/logs
Merged a year ago by kevin. Opened a year ago by jkucera.
fedora-infra/ jkucera/ansible RHELPLAN-104620  into  main

@@ -0,0 +1,68 @@ 

+ #!/bin/bash

+ #

+ # Contact:

+ #   Email: systemroles-owner@lists.fedorahosted.org

+ #   IRC/matrix: #systemroles - @all - irc.libera.chat

+ 

+ set -euo pipefail

+ 

+ AGE_IN_DAYS=${AGE_IN_DAYS:-183}

+ newest_date=$(date --date="${AGE_IN_DAYS} days ago" +%Y%m%d)

+ 

+ function normalize_image_name() {

+   if [[ "${1:-}" =~ ^(RHEL-[[:digit:]]+\.[[:digit:]]+)[.-]([[:alnum:]._-]+)?$ ]]; then

+     echo "${BASH_REMATCH[1]}"

+   else

+     echo "${1:-}"

+   fi

+ }

+ 

+ if [ -z "${1:-}" ]; then

+   echo "Need a directory to pushd in" >&2

+   exit 1

+ fi

+ 

+ pushd "${1}"

+ 

+ find -maxdepth 1 -type d | while read dir; do

+   # Case: linux-system-roles-certificate-pull-linux-system-roles_certificate-80-4f880f7-rhel-x-20210305-152227

+   if [[ "${dir}" =~ ^./linux-system-roles-[[:alnum:]._-]+-pull-linux-system-roles_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)-([[:alnum:]._-]+)-([[:digit:]]+)-([[:digit:]]+)$ ]]; then

+     echo "${BASH_REMATCH[1]}" "$(normalize_image_name "${BASH_REMATCH[3]}")" "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${dir}"

Does nested BASH_REMATCH work? That is, if BASH_REMATCH is a global variable, then calling normalize_image_name will reset it?

+   # Case: lsr-citool_bootloader-19-0f14842_20220104-080416

+   elif [[ "${dir}" =~ ^./lsr-citool_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)_([[:digit:]]+)-([[:digit:]]+)$ ]]; then

+     echo "${BASH_REMATCH[1]}" "unknown" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}" "${dir}"

+   # Case: lsr-citool_certificate-132-212741b_RHEL-9.1.0-20220814.1_20220818-223408

+   elif [[ "${dir}" =~ ^./lsr-citool_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)_([[:alnum:]._-]+)_([[:digit:]]+)-([[:digit:]]+)$ ]]; then

+     echo "${BASH_REMATCH[1]}" "$(normalize_image_name "${BASH_REMATCH[3]}")" "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${dir}"

+   # Case: lsr-citool_network-509-82dd06b_RHEL-6.10-updates-20201110.17

+   elif [[ "${dir}" =~ ^./lsr-citool_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)_([[:alnum:]._-]+)$ ]]; then

+     tmod="$(stat -c %Y "${dir}")"

+     tmod="$(date --date="@${tmod}" +'%Y%m%d %H%M%S')"

+     echo "${BASH_REMATCH[1]}" "$(normalize_image_name "${BASH_REMATCH[3]}")" "${tmod}" "${dir}"

+   # Case: lsr-citool_* (artifacts of early lsr-citool development)

+   elif [[ "${dir}" =~ ^./lsr-citool_.*$ ]]; then

+     tmod="$(stat -c %Y "${dir}")"

+     tmod="$(date --date="@${tmod}" +'%Y%m%d %H%M%S')"

+     echo unknown unknown "${tmod}" "${dir}"

+   fi

+ done | sort -n -r | while read role image date time dir; do

+   # `sort -n -r` ensures that the most recent log for $role $image is on the top

+   if [[ "${role}" = unknown && "${date}" -lt "${newest_date}" ]]; then

+     # Let not artifacts of early stage of lsr-citool development to rot forever

+     rm -rf "${dir}"

+     continue

+   fi

+   latest_file=".latest_${role}_${image}"

+   if [[ -f "${latest_file}" ]]; then

+     # Latest log for $role $image seen already

+     if [[ "${date}" -lt "${newest_date}" ]]; then

+       rm -rf "${dir}"

+     fi

+   else

+     echo keeping ${role} ${image} ${date} ${time} ${dir}

+     echo "${dir}" > "${latest_file}"

+   fi

+ done

+ rm -rf .latest_*

+ 

+ popd

@@ -0,0 +1,417 @@ 

+ #!/bin/bash

+ #

+ # Test `linuxsystemroles-logs-clean` script.

+ #

+ # Usage: ./linuxsystemroles-logs-clean.test

+ #

+ # Expects `linuxsystemroles-logs-clean` script in the same directory.

+ 

+ set -euo pipefail

+ 

+ : <<_EOF_

+ 

+ Roles (as found in log names at /srv/pub/alt/linuxsystemroles/logs):

+   "ad_integration"

+   "bootloader"

+   "certificate"

+   "ci-test"

+   "ci-testing"

+   "cockpit"

+   "crypto_policies"

+   "firewall"

+   "ha_cluster"

+   "kdump"

+   "kernel_settings"

+   "logging"

+   "metrics"

+   "mssql"

+   "nbde_client"

+   "nbde_server"

+   "network"

+   "podman"

+   "postfix"

+   "postgresql"

+   "rhc"

+   "selinux"

+   "ssh"

+   "storage"

+   "timesync"

+   "tlog"

+   "tuned"

+   "vpn"

+ 

+ Images (as found in log names at /srv/pub/alt/linuxsystemroles/logs):

+   "centos-6"

+   "centos-7"

+   "centos-8"

+   "fedora-33"

+   "fedora-34"

+   "fedora-35"

+   "rhel-6"

+   "rhel-7"

+   "rhel-8"

+   "rhel-8-y"

+   "rhel-x"

+   "CentOS-7-latest"

+   "CentOS-Stream-8"

+   "Fedora-36"

+   "Fedora-37"

+   "RHEL-6.10-updates-20201110.17"

+   "RHEL-7.9-updates-20221012.4"

+   "RHEL-8.8.0-20221211.0"

+   "RHEL-9.2.0-20221212.0"

+ 

+ _EOF_

+ 

+ ME="$(basename ${0})"

+ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"

+ NOW="$(date +'%Y-%m-%d %H:%M:%S')"

+ TEMPDIR="$(mktemp -d /var/tmp/${ME}.XXXXXXXXXX)"

+ STATUS=0

+ 

+ trap "rm -rf ${TEMPDIR}" ABRT EXIT HUP INT QUIT TERM

+ 

+ ##

+ # failure - report test's failure and set STATUS to 1

+ #

+ # Parameters:

+ #   $1 - message

+ #

+ function failure() {

+   echo $* >&2

+   STATUS=1

+ }

+ 

+ ##

+ # error - report an error and exit with 2

+ #

+ # Parameters:

+ #   $1 - message

+ #

+ function error() {

+   echo $* >&2

+   exit 2

+ }

+ 

+ ##

+ # log_1_name - generate a name of a directory with logs

+ #

+ # Parameters:

+ #   $1 - role name

+ #   $2 - pull request number

+ #   $3 - commit hash

+ #   $4 - image name

+ #   $5 - date

+ #   $6 - time

+ #

+ function log_1_name() {

+   echo "linux-system-roles-${1}-pull-linux-system-roles_${1}-${2}-${3}-${4}-${5}-${6}"

+ }

+ 

+ ##

+ # log_2_name - see log_1_name

+ #

+ function log_2_name() {

+   if [[ -z "${4:-}" ]]; then

+     echo "lsr-citool_${1}-${2}-${3}_${5}-${6}"

+   elif [[ -z "${5:-}" || -z "${6:-}" ]]; then

+     echo "lsr-citool_${1}-${2}-${3}_${4}"

+   else

+     echo "lsr-citool_${1}-${2}-${3}_${4}_${5}-${6}"

+   fi

+ }

+ 

+ ##

+ # log_3_name - see log_1_name

+ #

+ function log_3_name() {

+   # Produced by early stage of development of lsr-citool

+   echo "lsr-citool_ci-tt"

+ }

+ 

+ ##

+ # log_4_name - see log_1_name

+ #

+ function log_4_name() {

+   # Produced by early stage of development of lsr-citool

+   echo "lsr-citool_repoName-7-54sd5fsd_${5}-${6}"

+ }

+ 

+ ##

+ # stay - decides whether log should stay or be removed (auxiliary function

+ #        used by prep_test)

+ #

+ # Parameters:

+ #   $1 - days ago divided by 10 (integer division)

+ #

+ function stay() {

+   if [[ ${1:-0} -ge 18 ]]; then

+     echo 0

+   else

+     echo 1

+   fi

+ }

+ 

+ ##

+ # stay3 - decides which group of logs should stay (see prep_test)

+ #

+ # Parameters:

+ #   $1 - group A days ago divided by 10 (integer division)

+ #   $2 - group B days ago divided by 10 (integer division)

+ #   $3 - group C days ago divided by 10 (integer division)

+ #   $4 - selector

+ #

+ function stay3() {

+   local A="$(stay "${1:-}")"

+   local B="$(stay "${2:-}")"

+   local C="$(stay "${3:-}")"

+ 

+   # It is assumed that group A is younger than group B is younger than group C

+   # (see prep_test). Thus, A should stay if all groups are deletion candidates

+   if [[ "${A}${B}${C}" = "000" ]]; then

+     A=1

+   fi

+ 

+   case "${4:-}" in

+     A) echo "${A}";;

+     B) echo "${B}";;

+     C) echo "${C}";;

+     *) error "stay3: Invalid selector (${4:-})";;

+   esac

+ }

+ 

+ ##

+ # create_log - create a log

+ #

+ # Options:

+ #   -n - do not include time stamp into log name

+ #

+ # Parameters:

+ #   $1 - log name generator

+ #   $2 - role name

+ #   $3 - pull request number

+ #   $4 - commit hash

+ #   $5 - image name

+ #   $6 - time ago (<days>_<hours>_<minutes>_<seconds>)

+ #   $7 - stay flag (1 - log should stay, 0 - log should be removed)

+ #

+ function create_log() {

+   local INCLUDE_STAMP=1

+   local LOG_STAMP=""

+   local LOG_DATE=""

+   local LOG_TIME=""

+   local LOG_NAME=""

+   local TEMP=""

+ 

+   if [[ "X${1:-}" = X-n ]]; then

+     INCLUDE_STAMP=0

+     shift

+   fi

+ 

+   if [[ "${6:-}" =~ ^([[:digit:]]+)_([[:digit:]]+)_([[:digit:]]+)_([[:digit:]]+)$ ]]; then

+     if [[ "${BASH_REMATCH[1]}" -gt 0 ]]; then

+       TEMP+="${BASH_REMATCH[1]} days ago "

+     fi

+     if [[ "${BASH_REMATCH[2]}" -gt 0 ]]; then

+       TEMP+="${BASH_REMATCH[2]} hours ago "

+     fi

+     if [[ "${BASH_REMATCH[3]}" -gt 0 ]]; then

+       TEMP+="${BASH_REMATCH[3]} minutes ago "

+     fi

+     if [[ "${BASH_REMATCH[4]}" -gt 0 ]]; then

+       TEMP+="${BASH_REMATCH[4]} seconds ago "

+     fi

+     LOG_STAMP="$(date --date="${NOW} ${TEMP}" +'%Y-%m-%d %H:%M:%S')"

+     LOG_DATE="$(date --date="${LOG_STAMP}" +%Y%m%d)"

+     LOG_TIME="$(date --date="${LOG_STAMP}" +%H%M%S)"

+   else

+     error "create_log: Invalid time ago (${6:-})"

+   fi

+ 

+   if [[ ${INCLUDE_STAMP} -eq 1 ]]; then

+     LOG_NAME="$(${1} "${2}" "${3}" "${4}" "${5}" ${LOG_DATE} ${LOG_TIME})"

+   else

+     LOG_NAME="$(${1} "${2}" "${3}" "${4}" "${5}" "" "")"

+   fi

+ 

+   mkdir -p ${TEMPDIR}/logs/${LOG_NAME}

+   echo log > ${TEMPDIR}/logs/${LOG_NAME}/log

+ 

+   touch --date="${LOG_STAMP}" ${TEMPDIR}/logs/${LOG_NAME}/log ${TEMPDIR}/logs/${LOG_NAME}

+ 

+   echo ${7} ${LOG_NAME} >> ${TEMPDIR}/assertions

+ }

+ 

+ ##

+ # prep_test - prepare a test directory

+ #

+ # Parameters:

+ #   $1 - A-group age (1 - young, 0 - old)

+ #   $2 - B-group age (1 - young, 0 - old)

+ #   $3 - C-group age (1 - young, 0 - old)

+ #   $4 - D-group age (1 - young, 0 - old)

+ #

+ function prep_test() {

+   local A=""

+   local B=""

+   local C=""

+   local D=""

+ 

+   if [[ ${1:-1} -eq 0 ]]; then

+     A=20

+   fi

+   if [[ ${2:-1} -eq 0 ]]; then

+     B=20

+   fi

+   if [[ ${3:-1} -eq 0 ]]; then

+     C=20

+   fi

+   if [[ ${4:-1} -eq 0 ]]; then

+     D=20

+   fi

+ 

+   create_log    log_1_name certificate      0 HEAD    centos-8                      ${A}5_0_0_0   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_1_name certificate      0 HEAD    rhel-8-y                      ${A}5_0_0_5   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_1_name certificate      0 HEAD    rhel-x                        ${A}5_0_0_10  $(stay3 "${A}" "${B}" "${C}" A)

+ 

+   create_log    log_1_name certificate      1 3dbef8b centos-8                      ${B}5_0_5_0   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_1_name certificate      1 3dbef8b rhel-8-y                      ${B}5_0_5_5   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_1_name certificate      1 3dbef8b rhel-x                        ${B}5_0_5_10  $(stay3 "${A}" "${B}" "${C}" B)

+ 

+   create_log    log_1_name certificate     15 0c2a97e centos-8                      ${C}5_0_10_0  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_1_name certificate     15 0c2a97e rhel-8-y                      ${C}5_0_10_5  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_1_name certificate     15 0c2a97e rhel-x                        ${C}5_0_10_10 $(stay3 "${A}" "${B}" "${C}" C)

+ 

+   create_log    log_1_name network        127 2936b72 fedora-35                     ${A}5_0_1_0   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_1_name network        127 2936b72 rhel-6                        ${A}5_0_1_5   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_1_name network        127 2936b72 rhel-x                        ${A}5_0_1_10  $(stay3 "${A}" "${B}" "${C}" A)

+ 

+   create_log    log_1_name network        136 5480c71 fedora-35                     ${B}5_0_6_0   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_1_name network        136 5480c71 rhel-6                        ${B}5_0_6_5   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_1_name network        136 5480c71 rhel-x                        ${B}5_0_6_10  $(stay3 "${A}" "${B}" "${C}" B)

+ 

+   create_log    log_1_name network        201 c09573b fedora-35                     ${C}5_0_11_0  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_1_name network        201 c09573b rhel-6                        ${C}5_0_11_5  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_1_name network        201 c09573b rhel-x                        ${C}5_0_11_10 $(stay3 "${A}" "${B}" "${C}" C)

+ 

+   create_log    log_1_name timesync        17 ec8e4f4 centos-6                      ${A}5_0_2_0   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_1_name timesync        17 ec8e4f4 fedora-34                     ${A}5_0_2_5   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_1_name timesync        17 ec8e4f4 rhel-7                        ${A}5_0_2_10  $(stay3 "${A}" "${B}" "${C}" A)

+ 

+   create_log    log_1_name timesync        22 41265d0 centos-6                      ${B}5_0_7_0   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_1_name timesync        22 41265d0 fedora-34                     ${B}5_0_7_5   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_1_name timesync        22 41265d0 rhel-7                        ${B}5_0_7_10  $(stay3 "${A}" "${B}" "${C}" B)

+ 

+   create_log    log_1_name timesync        99 f5dae11 centos-6                      ${C}5_0_12_0  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_1_name timesync        99 f5dae11 fedora-34                     ${C}5_0_12_5  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_1_name timesync        99 f5dae11 rhel-7                        ${C}5_0_12_10 $(stay3 "${A}" "${B}" "${C}" C)

+ 

+   create_log    log_2_name ad_integration   1 aeae773 ""                            ${A}5_0_3_0   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name ad_integration   1 aeae773 CentOS-7-latest               ${A}5_0_3_5   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name ad_integration   1 aeae773 CentOS-Stream-8               ${A}5_0_3_10  $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name ad_integration   1 aeae773 Fedora-37                     ${A}5_0_3_15  $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name ad_integration   1 aeae773 RHEL-6.10-updates-20201110.17 ${A}5_0_3_20  $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name ad_integration   1 aeae773 RHEL-9.2.0-20221212.0         ${A}5_0_3_25  $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name ad_integration   1 aeae773 RHEL-6.10-updates-20201210.18 ${A}5_0_3_30  $(stay "${A}")

+   create_log    log_2_name ad_integration   1 aeae773 RHEL-9.2.1-20221221.1         ${A}5_0_3_35  $(stay "${A}")

+ 

+   create_log    log_2_name ad_integration  11 f1f12e2 ""                            ${B}5_0_8_0   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name ad_integration  11 f1f12e2 CentOS-7-latest               ${B}5_0_8_5   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name ad_integration  11 f1f12e2 CentOS-Stream-8               ${B}5_0_8_10  $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name ad_integration  11 f1f12e2 Fedora-37                     ${B}5_0_8_15  $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name ad_integration  11 f1f12e2 RHEL-6.10-updates-20201110.17 ${B}5_0_8_20  $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name ad_integration  11 f1f12e2 RHEL-9.2.0-20221212.0         ${B}5_0_8_25  $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name ad_integration  11 f1f12e2 RHEL-6.10-updates-20201210.18 ${B}5_0_8_30  $(stay "${B}")

+   create_log    log_2_name ad_integration  11 f1f12e2 RHEL-9.2.1-20221221.1         ${B}5_0_8_35  $(stay "${B}")

+ 

+   create_log    log_2_name ad_integration  32 1148efe ""                            ${C}5_0_13_0  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name ad_integration  32 1148efe CentOS-7-latest               ${C}5_0_13_5  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name ad_integration  32 1148efe CentOS-Stream-8               ${C}5_0_13_10 $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name ad_integration  32 1148efe Fedora-37                     ${C}5_0_13_15 $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name ad_integration  32 1148efe RHEL-6.10-updates-20201110.17 ${C}5_0_13_20 $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name ad_integration  32 1148efe RHEL-9.2.0-20221212.0         ${C}5_0_13_25 $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name ad_integration  32 1148efe RHEL-6.10-updates-20201210.18 ${C}5_0_13_30 $(stay "${C}")

+   create_log    log_2_name ad_integration  32 1148efe RHEL-9.2.1-20221221.1         ${C}5_0_13_35 $(stay "${C}")

+ 

+   create_log    log_2_name bootloader       2 264569a ""                            ${A}5_0_4_0   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name bootloader       2 264569a CentOS-Stream-8               ${A}5_0_4_5   $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name bootloader       2 264569a Fedora-36                     ${A}5_0_4_10  $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name bootloader       2 264569a RHEL-8.8.0-20221211.0         ${A}5_0_4_15  $(stay3 "${A}" "${B}" "${C}" A)

+   create_log -n log_2_name bootloader       2 264569a RHEL-9.2.0-20221212.0         ${A}5_0_4_20  $(stay3 "${A}" "${B}" "${C}" A)

+   create_log    log_2_name bootloader       2 264569a RHEL-8.8.1-20221221.5         ${A}5_0_4_25  $(stay "${A}")

+   create_log -n log_2_name bootloader       2 264569a RHEL-9.2.2-20221222.11        ${A}5_0_4_30  $(stay "${A}")

+ 

+   create_log    log_2_name bootloader      21 d1f589b ""                            ${B}5_0_9_0   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name bootloader      21 d1f589b CentOS-Stream-8               ${B}5_0_9_5   $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name bootloader      21 d1f589b Fedora-36                     ${B}5_0_9_10  $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name bootloader      21 d1f589b RHEL-8.8.0-20221211.0         ${B}5_0_9_15  $(stay3 "${A}" "${B}" "${C}" B)

+   create_log -n log_2_name bootloader      21 d1f589b RHEL-9.2.0-20221212.0         ${B}5_0_9_20  $(stay3 "${A}" "${B}" "${C}" B)

+   create_log    log_2_name bootloader      21 d1f589b RHEL-8.8.1-20221221.5         ${B}5_0_9_25  $(stay "${B}")

+   create_log -n log_2_name bootloader      21 d1f589b RHEL-9.2.2-20221222.11        ${B}5_0_9_30  $(stay "${B}")

+ 

+   create_log    log_2_name bootloader      82 bfa1e33 ""                            ${C}5_0_14_0  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name bootloader      82 bfa1e33 CentOS-Stream-8               ${C}5_0_14_5  $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name bootloader      82 bfa1e33 Fedora-36                     ${C}5_0_14_10 $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name bootloader      82 bfa1e33 RHEL-8.8.0-20221211.0         ${C}5_0_14_15 $(stay3 "${A}" "${B}" "${C}" C)

+   create_log -n log_2_name bootloader      82 bfa1e33 RHEL-9.2.0-20221212.0         ${C}5_0_14_20 $(stay3 "${A}" "${B}" "${C}" C)

+   create_log    log_2_name bootloader      82 bfa1e33 RHEL-8.8.1-20221221.5         ${C}5_0_14_25 $(stay "${C}")

+   create_log -n log_2_name bootloader      82 bfa1e33 RHEL-9.2.2-20221222.11        ${C}5_0_14_30 $(stay "${C}")

+ 

+   create_log -n log_3_name ""              "" ""      ""                            ${D}5_0_30_0  $(stay ${D})

+   create_log    log_4_name ""              "" ""      ""                            ${D}5_0_30_5  $(stay ${D})

+ }

+ 

+ ##

+ # verify_results - verify the test results

+ #

+ function verify_results() {

+   while read PRESENT LOG_NAME; do

+     if [[ "${PRESENT}" -eq 0 && -d ${TEMPDIR}/logs/${LOG_NAME} ]]; then

+       failure "[FAIL] Log ${LOG_NAME} should be removed."

+     elif [[ "${PRESENT}" -eq 1 && ! -d ${TEMPDIR}/logs/${LOG_NAME} ]]; then

+       failure "[FAIL] Log ${LOG_NAME} should be present."

+     fi

+   done < ${TEMPDIR}/assertions

+ }

+ 

+ ##

+ # clean_test - remove test assets

+ #

+ function clean_test() {

+   rm -rf ${TEMPDIR}/logs ${TEMPDIR}/assertions

+ }

+ 

+ ##

+ # test_log_pruning - test log pruning script

+ #

+ # Parameters:

+ #   see prep_test

+ #

+ function test_log_pruning() {

+   prep_test "$@"

+   ${HERE}/linuxsystemroles-logs-clean ${TEMPDIR}/logs >/dev/null

+   verify_results

+   clean_test

+ }

+ 

+ test_log_pruning 0 0 0 0

+ test_log_pruning 0 0 0 1

+ test_log_pruning 0 0 1 0

+ test_log_pruning 0 0 1 1

+ 

+ test_log_pruning 0 1 0 0

+ test_log_pruning 0 1 0 1

+ test_log_pruning 0 1 1 0

+ test_log_pruning 0 1 1 1

+ 

+ test_log_pruning 1 0 0 0

+ test_log_pruning 1 0 0 1

+ test_log_pruning 1 0 1 0

+ test_log_pruning 1 0 1 1

+ 

+ test_log_pruning 1 1 0 0

+ test_log_pruning 1 1 0 1

+ test_log_pruning 1 1 1 0

+ test_log_pruning 1 1 1 1

+ 

+ [[ ${STATUS} -eq 0 ]]

@@ -68,10 +68,22 @@ 

    - name: add cron script to update fullfiletimelist

      copy: src="{{ files }}/scripts/update-fullfiletimelist" dest=/usr/local/bin/update-fullfiletimelist mode=0755

  

+   - name: add cron script to prune old logs at /srv/pub/alt/linuxsystemroles/logs

+     copy: src="{{ files }}/scripts/linuxsystemroles-logs-clean" dest=/usr/local/bin/linuxsystemroles-logs-clean mode=0755

+ 

    - name: Update fullfiletimelist job

      cron: name="update-fullfiletimelist" hour="*/2" minute="55" user="root"

          job="/usr/local/bin/lock-wrapper update-fullfiletimelist '/usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /srv/pub alt'"

          cron_file=update-fullfiletimelist

  

+   - name: Prune old logs at /srv/pub/alt/linuxsystemroles/logs

+     cron: name="linuxsystemroles-logs-clean" hour="0" minute="15" user="root"

+         job="/usr/local/bin/linuxsystemroles-logs-clean /srv/pub/alt/linuxsystemroles/logs >/dev/null"

+         cron_file=linuxsystemroles-logs-clean

+ 

+   - name: Set MAILTO for the previous cron job

+     cron: env=true name=MAILTO value="systemroles-owner@lists.fedorahosted.org"

+         cron_file=linuxsystemroles-logs-clean

+ 

    handlers:

    - import_tasks: "{{ handlers_path }}/restart_services.yml"

Add cron task that prune /srv/pub/alt/linuxsystemroles/logs.

Related: RHELPLAN-104620

Sign-off-by: Jiri Kucera jkucera@redhat.com

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

rebased onto b6baaaf2c8bbeab93871e8100951e409568d9fb2

a year ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

Looks good. The test script is cool. :)

One question: Does this cron produce any output when run normally? How about when there are errors?

We typically don't want cron jobs to mail on normal operation, but do want them to on error/failures.

The CI failures seem to be our playbook that needs fixing.

This will result in something like this:

certificate RHEL-9.1.0-20220814.1 20220818 223408 ./lsr-citool_certificate-132-212741b_RHEL-9.1.0-20220814.1_20220818-223408

which will "work" but it will not clean up as much as we would like. The problem is that, currently, baseos ci uses image names like RHEL-9.1.0-20220814.1 which can
change very frequently. So this means we will have logs from RHEL-9.1.0-20220814.1 and RHEL-9.1.0-20220815.1 etc.
Ideally, we would normalize the image name so that anything matching RHEL-9.1.\d-\d+ gets normalized to RHEL-9.1 - that way we would clean up all of the RHEL-9.1 logs, no matter
the date on the image.

Looks good. The test script is cool. :)

Thanks :) I am glad you like it.

One question: Does this cron produce any output when run normally? How about when there are errors?

It prints pushd/popd output and what files are kept to stdout.

We typically don't want cron jobs to mail on normal operation, but do want them to on error/failures.

Should I then add >/dev/null to the job command? I found nothing in the official Ansible cron task documentation[1] about customizing sending emails, so I suppose cron sends email whenever something appears on the output (stdout or stderr).

The CI failures seem to be our playbook that needs fixing.

[1] https://docs.ansible.com/ansible/latest/collections/ansible/builtin/cron_module.html

Yeah, I guess sending stdout to /dev/null would be fine.

If there's errors hopefully they would go to stderr, which yeah, goes to sysadmins.

Another alternative would be to set env=true and add a "MAILTO=youremail" but then you would get bugged by it all the time.

And thinking of which, could you add a small header saying who to ping if this cron goes off the rails/etc... ?

And thinking of which, could you add a small header saying who to ping if this cron goes off the rails/etc... ?

Email: systemroles-owner@lists.fedorahosted.org
IRC/matrix: #systemroles - @all - irc.libera.chat

Thats great, can you add that to the top of the script in comments? :)

2 new commits added

  • lsr-logs-clean: Add MAILTO to cron file
  • lsr-logs-clean: Normalize image names
a year ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

Thats great, can you add that to the top of the script in comments? :)

Done. I also add MAILTO=systemroles-owner@lists.fedorahosted.org to linuxsystemroles-logs-clean cron file. Hope that will work as expected (it should in Vixie's cron implementation; also something like

MAILTO=email1
job1
MAILTO=email2
job2

should be possible; have no idea what version of cron the server uses). stdout is redirected to /dev/null, hope the cron will not send emails with empty bodies.

Ideally, we would normalize the image name ...

This should be also implemented and covered with test.

Does nested BASH_REMATCH work? That is, if BASH_REMATCH is a global variable, then calling normalize_image_name will reset it?

rebased onto daa4f5b34710558dfae98bb2a89fb909719668f9

a year ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

Does nested BASH_REMATCH work? That is, if BASH_REMATCH is a global variable, then calling normalize_image_name will reset it?

It works, but only because it runs in a subshell:

$(normalize_image_name "${BASH_REMATCH[3]}")

rebased onto dc8c275

a year ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

ok, lets give it a go! Thanks everyone!

Pull-Request has been merged by kevin

a year ago