From bb08872187a084c4c37e957aaac700427f2c2260 Mon Sep 17 00:00:00 2001
From: Jakub Kadlcik <frostyx@email.cz>
Date: Apr 30 2022 07:02:00 +0000
Subject: backend: attempt to sign multiple times


I don't really know what caused #2133, we assumed it's a glitch caused
by overloaded backend/keygen, and that simply running the /bin/sign a
second time would fix the issue.

We added `time.sleep` between failed attempts, so we need to mock some
unrelated tests to save some time.

Fixes: #2133
Merges: #2177

---

diff --git a/backend/copr_backend/sign.py b/backend/copr_backend/sign.py
index 5c5b761..ec30db7 100644
--- a/backend/copr_backend/sign.py
+++ b/backend/copr_backend/sign.py
@@ -6,6 +6,7 @@ Wrapper for /bin/sign from obs-sign package
 
 from subprocess import Popen, PIPE, SubprocessError
 import os
+import time
 
 from packaging import version
 
@@ -42,8 +43,12 @@ def call_sign_bin(cmd, log):
             new_err = CoprSignError("Failed to invoke '{}'".format(cmd_pretty))
             raise new_err from err
 
-        if handle.returncode != 0 and "Connection timed out" in stderr:
-            log.warning("Timeout on %s, re-trying", cmd_pretty)
+        if handle.returncode != 0:
+            log.warning("Command '%s' failed with: %s",
+                        cmd_pretty, stderr.rstrip())
+            sleeptime = 20
+            log.warning("Going to sleep %ss and re-try.", sleeptime)
+            time.sleep(sleeptime)
             continue
         break
     return handle.returncode, stdout, stderr
diff --git a/backend/tests/test_sign.py b/backend/tests/test_sign.py
index 1cd27c5..1bf6cc5 100644
--- a/backend/tests/test_sign.py
+++ b/backend/tests/test_sign.py
@@ -12,6 +12,7 @@ from copr_backend.exceptions import CoprSignError, CoprSignNoKeyError, CoprKeyge
 from copr_backend.sign import (
     get_pubkey, _sign_one, sign_rpms_in_dir, create_user_keys,
     gpg_hashtype_for_chroot,
+    call_sign_bin,
 )
 
 STDOUT = "stdout"
@@ -19,6 +20,7 @@ STDERR = "stderr"
 
 
 class TestSign(object):
+    # pylint: disable=too-many-public-methods
 
     def setup_method(self, method):
         self.username = "foo"
@@ -70,8 +72,9 @@ class TestSign(object):
             get_pubkey(self.username, self.projectname, MagicMock())
 
 
+    @mock.patch("copr_backend.sign.time.sleep")
     @mock.patch("copr_backend.sign.Popen")
-    def test_get_pubkey_unknown_key(self, mc_popen):
+    def test_get_pubkey_unknown_key(self, mc_popen, _sleep):
         mc_handle = MagicMock()
         mc_handle.communicate.return_value = (STDOUT, "unknown key: foobar")
         mc_handle.returncode = 1
@@ -82,8 +85,9 @@ class TestSign(object):
 
         assert "There are no gpg keys for user foo in keyring" in str(err)
 
+    @mock.patch("copr_backend.sign.time.sleep")
     @mock.patch("copr_backend.sign.Popen")
-    def test_get_pubkey_unknown_error(self, mc_popen):
+    def test_get_pubkey_unknown_error(self, mc_popen, _sleep):
         mc_handle = MagicMock()
         mc_handle.communicate.return_value = (STDOUT, STDERR)
         mc_handle.returncode = 1
@@ -134,8 +138,9 @@ class TestSign(object):
         with pytest.raises(CoprSignError):
             _sign_one(fake_path, self.usermail, "sha256", MagicMock())
 
+    @mock.patch("copr_backend.sign.time.sleep")
     @mock.patch("copr_backend.sign.Popen")
-    def test_sign_one_cmd_erro(self, mc_popen):
+    def test_sign_one_cmd_erro(self, mc_popen, _sleep):
         mc_handle = MagicMock()
         mc_handle.communicate.return_value = (STDOUT, STDERR)
         mc_handle.returncode = 1
@@ -145,6 +150,21 @@ class TestSign(object):
         with pytest.raises(CoprSignError):
             _sign_one(fake_path, self.usermail, "sha256", MagicMock())
 
+    @staticmethod
+    @mock.patch("copr_backend.sign.time.sleep")
+    @mock.patch("copr_backend.sign.Popen")
+    def test_call_sign_bin_repeatedly(mc_popen, _sleep):
+        """
+        Test that we attempt to run /bin/sign multiple times if it returns
+        non-zero exit status
+        """
+        mc_handle = MagicMock()
+        mc_handle.communicate.return_value = (STDOUT, STDERR)
+        mc_handle.returncode = 1
+        mc_popen.return_value = mc_handle
+        call_sign_bin(cmd=[], log=MagicMock())
+        assert mc_popen.call_count == 3
+
     @mock.patch("copr_backend.sign.SafeRequest.send")
     def test_create_user_keys(self, mc_request):
         mc_request.return_value.status_code = 200