#882 run all sanity beaker tests by one command
Closed 4 months ago by praiskup. Opened 4 months ago by praiskup.
copr/ praiskup/copr all-in-tmux  into  master

@@ -15,11 +15,14 @@ 

  VOLUME ["/sys/fs/cgroup"]

  

  RUN dnf -y install dnf-plugins-core && dnf -y copr enable @copr/copr

- RUN dnf -y install htop wget net-tools iputils vim mlocate git sudo openssh-server psmisc python-jedi procps-ng findutils

+ RUN dnf -y install htop wget net-tools iputils vim mlocate git sudo openssh-server psmisc python-jedi procps-ng findutils tmux expect

  

  # Sanity tests

  RUN wget https://beaker-project.org/yum/beaker-client-Fedora.repo -P /etc/yum.repos.d

  

+ # Hack, beakerlib is not installable on F30, bug 1719226

+ RUN sed -i 's/$releasever/29/g' /etc/yum.repos.d/beaker-client-Fedora.repo

Again? Or this was unintentional?

yeah :-( see the commit message

I'd actually swear that it was already working ... but it is not working now :-(

+ 

  RUN dnf -y install rhts-test-env beakerlib

  

  RUN echo "LANG=en_US.UTF-8" >> /etc/locale

@@ -43,8 +43,8 @@ 

          mkdir -p ~/.config || :

          rlRun "dnf config-manager --set-enabled fedora --save"

          rlRun "dnf install -y python3-copr copr-cli"

-         rlRun "dnf upgrade python3-copr copr-cli"

-         rlRun "dnf install rpm-build"

+         rlRun "dnf upgrade -y python3-copr copr-cli"

+         rlRun "dnf install -y rpm-build jq"

  

          cat > /etc/dnf/plugins/copr.d/tested-copr.conf <<EOF

  [tested-copr]

@@ -0,0 +1,33 @@ 

+ #! /bin/bash

+ 

+ our_dir=$(readlink -f "$(dirname "$0")")

+ export PATH=$PATH:$our_dir

+ 

+ exp_wrapper=bash-interactive-initialized.exp

+ 

+ cmd=(tmux)

+ first=true

+ 

+ if test $# -eq 0; then

+     # default set of tests

+     set -- `ls *.sh`

+ fi

+ 

+ 

+ for arg; do

+     case $arg in

+     *all-in-tmux.sh|*runtest-production.sh|*upload_authentication.sh)

+         continue

+         ;;

+     esac

+ 

+     quoted=$(printf %q "$arg")

+     if $first; then

+         first=false

+         cmd+=( new-session -n "$arg" "$exp_wrapper $quoted" )

+     else

+         cmd+=( ';' new-window -n "$arg" "$exp_wrapper $quoted" )

+     fi

+ done

+ 

+ "${cmd[@]}"

@@ -0,0 +1,13 @@ 

+ #! /usr/bin/expect

+ 

+ set bash_prompt {\$ $}

+ set env(PS1) {$ }

+ 

+ spawn bash --norc

+ 

+ foreach arg $argv {

+     expect -re $bash_prompt {send -- "$arg\r"}

+ }

+ 

+ interact

+ puts "exiting subshell"

@@ -51,7 +51,6 @@ 

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

          sed -i "s+https://copr.fedoraproject.org+$FRONTEND_URL+g" \

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

-         dnf -y install jq

      rlPhaseEnd

  

      rlPhaseStartTest

@@ -33,6 +33,7 @@ 

  HERE=$SCRIPTPATH

  FILES=$HERE/files

  

+ : ${COPR_CLEANUP=true}

So, the logic won't be as we discussed earlier, that if the tests succeeds, everything will be cleaned, if failed, projects and builds will be preserved, right?

By default we will clean up, but if I want to, I can set the variable to false and avoid it.

There's no magic trying to guess whether something failed or not, yet. You need to decide in advance (by default everything is cleaned, always).
We can make this more clever though.

  

  # Debug

  echo "USER = $USER"

@@ -13,3 +13,29 @@ 

     test -n "$id" || return 1

     export BUILD_ID=$id

  }

+ 

+ quote_args()

+ {

+     quote_args_result=

+     __qa_space=

+     for __qa_arg; do

+         quote_args_result+=$__qa_space$(printf %q "$__qa_arg")

+         __qa_space=' '

+     done

+ }

+ 

+ cleanAction()

+ {

+     quote_args "$@"

+ 

+     if $COPR_CLEANUP; then

+         rlRun "$quote_args_result"

+     else

+         rlLogInfo "Cleanup disabled, run manually: $quote_args_result"

+     fi

+ }

+ 

+ cleanProject()

+ {

+     cleanAction copr-cli delete "$1"

+ }

@@ -51,7 +51,6 @@ 

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

          sed -i "s+https://copr.fedoraproject.org+$FRONTEND_URL+g" \

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

-         dnf -y install jq

      rlPhaseEnd

  

      rlPhaseStartTest

@@ -30,11 +30,11 @@ 

          rlRun "dnf --disablerepo='*' \

              --enablerepo='copr:${URL}:group_copr:${NAME_VAR}Createrepo' \

              list available 2>&1 | grep 'Failed to synchronize'" 1

- 

-         rlRun "copr-cli delete ${NAME_PREFIX}Createrepo"

      rlPhaseEnd

  

      rlPhaseStartCleanup

+         rlRun "copr-cli delete ${NAME_PREFIX}Createrepo"

+         rlRun "dnf -y copr remove ${URL}/${NAME_PREFIX}Createrepo"

      rlPhaseEnd

  rlJournalPrintText

  rlJournalEnd

@@ -32,6 +32,7 @@ 

  # Load config settings

  HERE=$(dirname "$(realpath "$0")")

  source "$HERE/config"

+ source "$HERE/helpers"

  

  

  function wait_for_finished_module()

@@ -87,7 +88,6 @@ 

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

          sed -i "s+https://copr.fedoraproject.org+$FRONTEND_URL+g" \

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

-         dnf -y install jq

      rlPhaseEnd

  

      rlPhaseStartTest

@@ -204,11 +204,11 @@ 

      rlPhaseEnd

  

      rlPhaseStartCleanup

-         rlRun "copr-cli delete module-testmodule-beakertest-$DATE"

-         rlRun "copr-cli delete module-testmoduleurl-beakertest-$DATE"

-         rlRun "copr-cli delete module-test-macros-module-beakertest-$DATE"

-         rlRun "copr-cli delete $OWNER/TestModule$DATE$SUFFIX"

-         rlRun "copr-cli delete module-coprtestmodule-beakertest-$DATE"

+         cleanProject "module-testmodule-beakertest-$DATE"

+         cleanProject "module-testmoduleurl-beakertest-$DATE"

+         cleanProject "module-test-macros-module-beakertest-$DATE"

+         cleanProject "$OWNER/TestModule$DATE$SUFFIX"

+         cleanProject "module-coprtestmodule-beakertest-$DATE"

      rlPhaseEnd

  rlJournalPrintText

  rlJournalEnd

@@ -47,7 +47,6 @@ 

          rlRun "copr-cli list"

          # and install... things

          yum -y install dnf dnf-plugins-core

-         dnf -y install jq

      rlPhaseEnd

  

      rlPhaseStartTest

@@ -545,52 +544,51 @@ 

          rlRun "copr-cli add-package-scm ${NAME_PREFIX}CoprDirTest --name example --clone-url $COPR_HELLO_GIT" 0

          rlRun "copr-cli buildscm ${NAME_PREFIX}CoprDirTest:example --clone-url $COPR_HELLO_GIT" 0

  

-         ### ---- DELETING PROJECTS ------- ###

          # delete - wrong project name

          rlRun "copr-cli delete ${NAME_PREFIX}wrong-name" 1

-         # delete the projects

-         rlRun "copr-cli delete ${NAME_PREFIX}Project1"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project2"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project3"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project4"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project5"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project6"

-         rlRun "copr-cli delete ${NAME_PREFIX}DisableCreaterepoFalse"

-         rlRun "copr-cli delete ${NAME_PREFIX}DisableCreaterepoTrue"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project7"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project8"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project9"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project10"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project10Fork"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project11"

-         rlRun "copr-cli delete ${NAME_PREFIX}Project12"

-         rlRun "copr-cli delete ${NAME_PREFIX}DownloadMockCfgs"

-         rlRun "copr-cli delete ${NAME_PREFIX}TestBug1370704"

-         rlRun "copr-cli delete ${NAME_PREFIX}ProjectDistGitBuilds"

-         rlRun "copr-cli delete ${NAME_PREFIX}TestBug1393361-1"

-         rlRun "copr-cli delete ${NAME_PREFIX}TestBug1393361-2"

-         rlRun "copr-cli delete ${NAME_PREFIX}ModifyProjectChroots"

-         rlRun "copr-cli delete ${NAME_PREFIX}EditChrootProject"

-         rlRun "copr-cli delete ${NAME_PREFIX}TestDeleteGroupBuild"

-         rlRun "copr-cli delete ${NAME_PREFIX}MockConfig"

-         rlRun "copr-cli delete ${NAME_PREFIX}MockConfigParent"

-         rlRun "copr-cli delete ${NAME_PREFIX}TestBug1444804"

-         rlRun "copr-cli delete ${NAME_PREFIX}BootstrapProject"

-         rlRun "copr-cli delete ${NAME_PREFIX}CoprDirTest"

+     rlPhaseEnd

+ 

+     rlPhaseStartCleanup

+         cleanProject "${NAME_PREFIX}Project1"

+         cleanProject "${NAME_PREFIX}Project2"

+         cleanProject "${NAME_PREFIX}Project3"

+         cleanProject "${NAME_PREFIX}Project4"

+         cleanProject "${NAME_PREFIX}Project5"

+         cleanProject "${NAME_PREFIX}Project6"

+         cleanProject "${NAME_PREFIX}DisableCreaterepoFalse"

+         cleanProject "${NAME_PREFIX}DisableCreaterepoTrue"

+         cleanProject "${NAME_PREFIX}Project7"

+         cleanProject "${NAME_PREFIX}Project8"

+         cleanProject "${NAME_PREFIX}Project9"

+         cleanProject "${NAME_PREFIX}Project10"

+         cleanProject "${NAME_PREFIX}Project10Fork"

+         cleanProject "${NAME_PREFIX}Project11"

+         cleanProject "${NAME_PREFIX}Project12"

+         cleanProject "${NAME_PREFIX}DownloadMockCfgs"

+         cleanProject "${NAME_PREFIX}TestBug1370704"

+         cleanProject "${NAME_PREFIX}ProjectDistGitBuilds"

+         cleanProject "${NAME_PREFIX}TestBug1393361-1"

+         cleanProject "${NAME_PREFIX}TestBug1393361-2"

+         cleanProject "${NAME_PREFIX}ModifyProjectChroots"

+         cleanProject "${NAME_PREFIX}EditChrootProject"

+         cleanProject "${NAME_PREFIX}TestDeleteGroupBuild"

+         cleanProject "${NAME_PREFIX}MockConfig"

+         cleanProject "${NAME_PREFIX}MockConfigParent"

+         cleanProject "${NAME_PREFIX}TestBug1444804"

+         cleanProject "${NAME_PREFIX}BootstrapProject"

+         cleanProject "${NAME_PREFIX}CoprDirTest"

  

          # and make sure we haven't left any mess

          rlRun "copr-cli list | grep $NAME_PREFIX" 1

-         ### left after this section: hello installed

-     rlPhaseEnd

  

-     rlPhaseStartCleanup

-         rm $TMP/TestDeleteGroupBuild_example_build_id.txt

-         rm $TMP/failed_example_build_id

-         rm $TMP/hello_p3.id

-         rm $TMP/hello_p3.out

-         rm $TMP/pubkey_fork.gpg

-         rm $TMP/pubkey_source.gpg

-         rm $TMP/succeeded_example_build_id

+         ### left after this section: hello installed

+         cleanAction rm "$TMP"/TestDeleteGroupBuild_example_build_id.txt

+         cleanAction rm "$TMP"/failed_example_build_id

+         cleanAction rm "$TMP"/hello_p3.id

+         cleanAction rm "$TMP"/hello_p3.out

+         cleanAction rm "$TMP"/pubkey_fork.gpg

+         cleanAction rm "$TMP"/pubkey_source.gpg

+         cleanAction rm "$TMP"/succeeded_example_build_id

      rlPhaseEnd

  rlJournalPrintText

  rlJournalEnd

@@ -51,7 +51,6 @@ 

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

          sed -i "s+https://copr.fedoraproject.org+$FRONTEND_URL+g" \

          /usr/lib/python3.4/site-packages/dnf-plugins/copr.py

-         dnf -y install jq

      rlPhaseEnd

  

      rlPhaseStartTest

file modified
+8 -3

@@ -59,10 +59,15 @@ 

  ::

  

      [root@test-env ~] cd ~/copr/beaker-tests/Sanity/copr-cli-basic-operations/

-     [root@test-env copr-cli-basic-operations] ./build-spec.sh

-     [root@test-env copr-cli-basic-operations] ./runtest.sh

-     [root@test-env copr-cli-basic-operations] ./runtest-modules.sh

  

      # To see all the test scripts

      [root@test-env copr-cli-basic-operations] ls *.sh

  

+     # To execute all the sanity tests in parallel

+     [root@test-env copr-cli-basic-operations] ./all-in-tmux.sh

+ 

+     # To execute subset of sanity tests in parallel

+     [root@test-env copr-cli-basic-operations] ./all-in-tmux.sh runtest.sh runtest-modules.sh

+ 

+     # To execute individual test

+     [root@test-env copr-cli-basic-operations] ./build-spec.sh

no initial comment

I'd actually swear that it was already working ... but it is not working now :-(

I have a feeling, that everybody will forget to update this list. Wouldn't it be better to run all executable files in the directory? Or all runtest-*.sh files?

Anyway, this is a long wanted feature, so I am very happy about it. Thank you @praiskup!

I've originally intended to run the tests one by one, which was the easier way, but running them in parallel in tmux is uber cool.

rebased onto bac5e04620c57632392d8f4d09bc079aa886e89b

4 months ago

Ok, updated to execute all *.sh files. Requires some caution when adding new *.sh files, but we should have some automation soon.

Ok, updated to execute all *.sh files.

Thank you

One more thing though. Can you please mention the usage somewhere?
I would put it on the end of
https://docs.pagure.org/copr.copr/beaker_tests.html

rebased onto 9f6ef3cc0ae463b5b4d20db403ea84b6c3ec84b8

4 months ago

One more thing though. Can you please mention the usage somewhere?

I've mentioned in the sanity_tests.rst file where we described how to execute the tests.

rebased onto 28343db

4 months ago

1 new commit added

  • beaker-tests: fix cleanup tasks in createrepo test
4 months ago

1 new commit added

  • beaker-tests: make the cleanup actions opt-out
4 months ago

So, the logic won't be as we discussed earlier, that if the tests succeeds, everything will be cleaned, if failed, projects and builds will be preserved, right?

By default we will clean up, but if I want to, I can set the variable to false and avoid it.

There's no magic trying to guess whether something failed or not, yet. You need to decide in advance (by default everything is cleaned, always).
We can make this more clever though.

Pull-Request has been closed by praiskup

4 months ago