+         except FrontendClientException:

+ from backend.exceptions import FrontendClientException

+         self.frontend_client = FrontendClient(self.opts, self.log)

+             raw_actions = self.frontend_client.get('pending-actions').json()

+         except (FrontendClientException, ValueError) as error:

+ from ..exceptions import CoprBackendError, FrontendClientException

+             self.frontend_client.reschedule_all_running()

+         except FrontendClientException as err:

+ from ..exceptions import (DispatchBuildError, NoVmAvailable,

+                           FrontendClientException)

+                 tasks = self.frontend_client.get('pending-jobs').json()

+             except (FrontendClientException, ValueError) as error:

+         except (FrontendClientException, ValueError) as error:

+         self.frontend_client.try_indefinitely = True

+ 

+ 

+ class FrontendClientException(Exception):

+     pass

+ import logging

+ from requests import get, post, put, RequestException

+ 

+ from backend.exceptions import FrontendClientException

+ 

+ # prolong the sleep time before asking frontend again

+ SLEEP_INCREMENT_TIME = 5

+ # reasonable timeout for requests that block backend daemon

+ BACKEND_TIMEOUT = 2*60

+ 

+ class FrontendClientRetryError(Exception):

+     pass

+     # do we block the main daemon process?

+     try_indefinitely = False

+ 

+         self.logger = logger

+ 

+     @property

+     def log(self):

+         'return configured logger object, or no-op logger'

+         if not self.logger:

+             self.logger = logging.getLogger(__name__)

+             self.logger.addHandler(logging.NullHandler())

+         return self.logger

+ 

+     def _frontend_request(self, url_path, data=None, authenticate=True,

+                           method='post'):

+         auth = ("user", self.frontend_auth) if authenticate else None

+             kwargs = {

+                 'auth': auth,

+                 'headers': headers,

+             }

+             method = method.lower()

+             if method in ['post', 'put']:

+                 kwargs['data'] = json.dumps(data)

+                 method = post if method == 'post' else put

+             else:

+                 method = get

+             response = method(url, **kwargs)

+         except RequestException as ex:

+             raise FrontendClientRetryError(

+                 "Requests error on {}: {}".format(url, str(ex)))

+ 

+         if response.status_code >= 500:

+             # Server error.  Hopefully this is only temporary problem, we wan't

+             # to re-try, and wait till the server works again.

+             raise FrontendClientRetryError(

+                 "Request server error on {}: {} {}".format(

+                     url, response.status_code, response.reason))

+ 

+         if response.status_code >= 400:

+             # Client error.  The mistake is on our side, it doesn't make sense

+             # to continue with retries.

+             raise FrontendClientException(

+                 "Request client error on {}: {} {}".format(

+                     url, response.status_code, response.reason))

+ 

+         # TODO: Success, but tighten the redirects etc.

+     def _frontend_request_repeatedly(self, url_path, method='post', data=None,

+                                      authenticate=True):

+         Repeat the request until it succeeds, or timeout is reached.

+         sleep = SLEEP_INCREMENT_TIME

+         start = time.time()

+         stop = start + BACKEND_TIMEOUT

+         i = 0

+             i += 1

+             if not self.try_indefinitely and time.time() > stop:

+                 raise FrontendClientException(

+                     "Attempt to talk to frontend timeouted "

+                     "(we gave it {} attempts)".format(i))

+             try:

+                 return self._frontend_request(url_path, data=data,

+                                               authenticate=authenticate,

+                                               method=method)

+             except FrontendClientRetryError as ex:

+                 self.log.warning("Retry request #%s on %s: %s", i, url_path,

+                                  str(ex))

+                 time.sleep(sleep)

+                 sleep += SLEEP_INCREMENT_TIME

+     def _post_to_frontend_repeatedly(self, data, url_path):

+         Repeat the request until it succeeds, or timeout is reached.

+         return self._frontend_request_repeatedly(url_path, data=data)

+ 

+     def get(self, url_path):

+         'Issue relentless GET request to Frontend'

+         return self._frontend_request_repeatedly(url_path, method='get')

+ 

+     def post(self, url_path, data):

+         'Issue relentless POST request to Frontend'

+         return self._frontend_request_repeatedly(url_path, data=data)

+ 

+     def put(self, url_path, data):

+         'Issue relentless POST request to Frontend'

+         return self._frontend_request_repeatedly(url_path, data=data,

+                                                  method='put')

+             raise FrontendClientException("Bad response from the frontend")

+     def reschedule_all_running(self):

+         response = self._post_to_frontend_repeatedly({}, "reschedule_all_running")

+             raise FrontendClientException("Failed to reschedule builds")

+     resp = frontend_client.get('action/{}'.format(task_id))

+         response = self.frontend_client.get("chroots-prunerepo-status")

+         self.frontend_client.post(chroots_to_prune, "final-prunerepo-done")

+ from requests import RequestException, Response

+ from backend.frontend import (FrontendClient, FrontendClientRetryError,

+                               SLEEP_INCREMENT_TIME)

+ from backend.exceptions import FrontendClientException

+ @pytest.fixture(scope='function', params=['get', 'post', 'put'])

+ def f_request_method(request):

+     'mock the requests.{get,post,put} method'

+     with mock.patch("backend.frontend.{}".format(request.param)) as ctx:

+         yield (request.param, ctx)

+ 

+     def mask_frontend_request(self):

+         self.f_r = MagicMock()

+         self.fc._frontend_request = self.f_r

+     def test_post_to_frontend(self, f_request_method):

+         name, method = f_request_method

+         method.return_value.status_code = 200

+         self.fc._frontend_request(self.url_path, self.data, method=name)

+         assert method.called

+     def test_post_to_frontend_wrappers(self, f_request_method):

+         name, method = f_request_method

+         method.return_value.status_code = 200

+ 

+         call = getattr(self.fc, name)

+         if name == 'get':

+             call(self.url_path)

+         else:

+             call(self.url_path, self.data)

+ 

+         assert method.called

+         with pytest.raises(FrontendClientRetryError):

+             self.fc._frontend_request(self.url_path, self.data)

+         with pytest.raises(FrontendClientRetryError):

+             self.fc._frontend_request(self.url_path, self.data)

+     def test_post_to_frontend_repeated_first_try_ok(self, mask_frontend_request, mc_time):

+         self.f_r.return_value = response

+         mc_time.time.return_value = 0

+     def test_post_to_frontend_repeated_second_try_ok(self, f_request_method,

+             mask_frontend_request, mc_time):

+         method_name, method = f_request_method

+ 

+         self.f_r.side_effect = [

+             FrontendClientRetryError(),

+         mc_time.time.return_value = 0

+         assert self.fc._frontend_request_repeatedly(

+             self.url_path,

+             data=self.data,

+             method=method_name

+         ) == response

+     def test_post_to_frontend_err_400(self, post_req, mc_time):

+         response = Response()

+         response.status_code = 404

+         response.reason = 'NOT FOUND'

+         post_req.side_effect = [

+             FrontendClientRetryError(),

+             response,

+         ]

+         mc_time.time.return_value = 0

+         with pytest.raises(FrontendClientException):

+             assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response

+     @mock.patch('backend.frontend.BACKEND_TIMEOUT', 100)

+     def test_post_to_frontend_repeated_all_attempts_failed(self,

+             mask_frontend_request, caplog, mc_time):

+         mc_time.time.side_effect = [0, 0, 5, 5+10, 5+10+15, 5+10+15+20, 1000]

+         self.f_r.side_effect = FrontendClientRetryError()

+         with pytest.raises(FrontendClientException):

+             self.fc._post_to_frontend_repeatedly(self.data, self.url_path)

+         assert mc_time.sleep.call_args_list == [mock.call(x) for x in [5, 10, 15, 20, 25]]

+         assert len(caplog.records) == 5

+ 

+     def test_post_to_frontend_repeated_indefinitely(self,

+             mask_frontend_request, caplog, mc_time):

+         mc_time.time.return_value = 1

+         self.fc.try_indefinitely = True

+         self.f_r.side_effect = [FrontendClientRetryError() for _ in range(100)] \

+                              + [FrontendClientException()] # e.g. 501 eventually

+         with pytest.raises(FrontendClientException):

+             self.fc._post_to_frontend_repeatedly(self.data, self.url_path)

+         assert mc_time.sleep.called

+         assert len(caplog.records) == 100

+ 

+     def test_reschedule_300(self, mask_frontend_request, post_req):

+         response = Response()

+         response.status_code = 302

+         response.reason = 'whatever'

+         post_req.side_effect = response

+         with pytest.raises(FrontendClientException) as ex:

+             self.fc.reschedule_all_running()

+         assert 'Failed to reschedule builds' in str(ex)

+ 

+         with pytest.raises(FrontendClientException):

+         with pytest.raises(FrontendClientException):

+ @backend_ns.route("/chroots-prunerepo-status/")