taskotron / resultsdb_api

Clone
Source Code
GIT

#4 Retry on HTTP 500 errors by default
Merged 6 years ago by jskladan. Opened 6 years ago by jskladan.
feature/err_500_retry  into  develop

file modified
+64 -40 
@@ -17,6 +17,8 @@ 
 

  # Author: Josef Skladanka <jskladan@redhat.com>
 

  

  import requests
 

+ from requests.adapters import HTTPAdapter
 

+ from requests.packages.urllib3.util.retry import Retry
 

  import json
 

  import inspect
 

  import simplejson
 
@@ -62,18 +64,10 @@ 
 

          return repr(self.message)
 

  

  

- class ResultsDBapi(object):
 

- 

-     def __init__(self, api_url, auth_token=None):
 

-         # remove trailing slash(es), so we don't generate
 

-         # urls with a double slash which breaks werkzeug
 

-         # https://github.com/mitsuhiko/werkzeug/issues/491
 

-         self.url = api_url.rstrip('/')
 

-         self.auth_token = auth_token
 

- 

-     def __raise_on_error(self, r):
 

+ class _ResultsDBSession(requests.Session):
 

+     def __resultsdb_raise_on_error(self, r):
 

          if r.ok:
 

-             return
 

+             return r
 

  

          try:
 

              logger.warn('Received HTTP failure status code %s for request: %s',
 
@@ -81,11 +75,48 @@ 
 

              raise ResultsDBapiException(
 

                  '%s (HTTP %s)' % (r.json()['message'], r.status_code), r)
 

          except simplejson.JSONDecodeError as e:
 

-             logger.debug('Received invalid JSON data: %s\n%s', e, r.text)
 

+             logger.debug('Could not parse JSON data: %s\n%s', e, r.text)
 

              raise ResultsDBapiException(
 

-                 'Invalid JSON (HTTP %s): %s' % (r.status_code, e), r)
 

+                 '(HTTP %s)' % r.status_code, r)
 

          except KeyError:
 

-             raise ResultsDBapiException('HTTP %s Error' % r.status_code, r)
 

+             logger.debug('JSON data in unexpected format: %s\n%s', e, r.text)
 

+             raise ResultsDBapiException('Unexpected JSON data (HTTP %s): %s' % (r.status_code, r.json()), r)
 

+ 

+         raise ResultsDBapiException('Unknown Error', r)
 

+ 

+     def get(self, *args, **kwargs):
 

+         try:
 

+             r = super(_ResultsDBSession, self).get(*args, **kwargs)
 

+         except requests.exceptions.RetryError as e:
 

+             raise ResultsDBapiException('Maximum number of retries exceeded: %s' % e.message, None)
 

+         return self.__resultsdb_raise_on_error(r)
 

+ 

+     def post(self, *args, **kwargs):
 

+         try:
 

+             r = super(_ResultsDBSession, self).post(*args, **kwargs)
 

+         except requests.exceptions.RetryError as e:
 

+             raise ResultsDBapiException('Maximum number of retries exceeded: %s' % e.message, None)
 

+         return self.__resultsdb_raise_on_error(r)
 

+ 

+ class ResultsDBapi(object):
 

+ 

+     def __init__(self, api_url, auth_token=None, max_retries=3, backoff_factor=1.0):
 

+         # remove trailing slash(es), so we don't generate
 

+         # urls with a double slash which breaks werkzeug
 

+         # https://github.com/mitsuhiko/werkzeug/issues/491
 

+         self.url = api_url.rstrip('/')
 

+         self.auth_token = auth_token
 

+ 

+         self._retry = Retry(
 

+                 total=max_retries,
 

+                 backoff_factor=backoff_factor,
 

+                 status_forcelist=(500, 502, 503, 504),
 

+                 method_whitelist=False # Enables POST retries
 

+                 )
 

+         self._adapter = HTTPAdapter(max_retries=self._retry)
 

+         self.session = _ResultsDBSession()
 

+         self.session.mount("http://", self._adapter)
 

+         self.session.mount("https://", self._adapter)
 

  

      def __prepare_params(self, params_all):
 

          params = {}
 
@@ -117,8 +148,7 @@ 
 

          headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
 

          data = _fparams()
 

          data['_auth_token'] = self.auth_token
 

-         r = requests.post(url, data=json.dumps(data), headers=headers)
 

-         self.__raise_on_error(r)
 

+         r = self.session.post(url, data=json.dumps(data), headers=headers)
 

  

          return r.json()
 

  
@@ -128,28 +158,27 @@ 
 

              data['ref_url'] = ref_url
 

          if description is not _KEEP:
 

              data['description'] = description
 

-         if data:
 

-             data['uuid'] = uuid
 

+         if not data:
 

+             return self.get_group(uuid)
 

+ 

+         data['uuid'] = uuid
 

          data['_auth_token'] = self.auth_token
 

  

          url = "%s/groups" % self.url
 

          headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
 

-         r = requests.post(url, data=json.dumps(data), headers=headers)
 

-         self.__raise_on_error(r)
 

+         r = self.session.post(url, data=json.dumps(data), headers=headers)
 

  

          return r.json()
 

  

      def get_group(self, uuid):
 

          url = "%s/groups/%s" % (self.url, uuid)
 

-         r = requests.get(url)
 

-         self.__raise_on_error(r)
 

+         r = self.session.get(url)
 

  

          return r.json()
 

  

      def get_groups(self, page=None, limit=None, description=None, description_like=None, uuid=None):
 

          url = "%s/groups" % self.url
 

-         r = requests.get(url, params=self.__prepare_params(_fparams()))
 

-         self.__raise_on_error(r)
 

+         r = self.session.get(url, params=self.__prepare_params(_fparams()))
 

  

          return r.json()
 

  
@@ -158,22 +187,19 @@ 
 

          data = _fparams(expand_kwargs=False)
 

          data['_auth_token'] = self.auth_token
 

          headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
 

-         r = requests.post(url, data=json.dumps(data), headers=headers)
 

-         self.__raise_on_error(r)
 

+         r = self.session.post(url, data=json.dumps(data), headers=headers)
 

  

          return r.json()
 

  

      def get_result(self, id):
 

          url = "%s/results/%s" % (self.url, id)
 

-         r = requests.get(url)
 

-         self.__raise_on_error(r)
 

+         r = self.session.get(url)
 

  

          return r.json()
 

  

      def get_results(self, page=None, limit=None, since=None, outcome=None, groups=None, testcases=None, testcases_like=None, raw_params=None, **kwargs):
 

          url = "%s/results" % self.url
 

-         r = requests.get(url, params=self.__prepare_params(_fparams()))
 

-         self.__raise_on_error(r)
 

+         r = self.session.get(url, params=self.__prepare_params(_fparams()))
 

  

          return r.json()
 

  
@@ -182,8 +208,7 @@ 
 

          data = _fparams()
 

          data['_auth_token'] = self.auth_token
 

          headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
 

-         r = requests.post(url, data=json.dumps(data), headers=headers)
 

-         self.__raise_on_error(r)
 

+         r = self.session.post(url, data=json.dumps(data), headers=headers)
 

  

          return r.json()
 

  
@@ -191,27 +216,26 @@ 
 

          data = {}
 

          if ref_url is not _KEEP:
 

              data['ref_url'] = ref_url
 

-         if data:
 

-             data['name'] = name
 

+         if not data:
 

+             return self.get_testcase(name)
 

+ 

+         data['name'] = name
 

          data['_auth_token'] = self.auth_token
 

  

          url = "%s/testcases" % self.url
 

          headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
 

-         r = requests.post(url, data=json.dumps(data), headers=headers)
 

-         self.__raise_on_error(r)
 

+         r = self.session.post(url, data=json.dumps(data), headers=headers)
 

  

          return r.json()
 

  

      def get_testcase(self, name):
 

          url = "%s/testcases/%s" % (self.url, name)
 

-         r = requests.get(url)
 

-         self.__raise_on_error(r)
 

+         r = self.session.get(url)
 

  

          return r.json()
 

  

      def get_testcases(self, page=None, limit=None, name=None, name_like=None):
 

          url = "%s/testcases" % self.url
 

-         r = requests.get(url, params=self.__prepare_params(_fparams()))
 

-         self.__raise_on_error(r)
 

+         r = self.session.get(url, params=self.__prepare_params(_fparams()))
 

  

          return r.json()

Adds urllib3-style retries for HTTP 500 errors on any request. ResultsDB app does not return HTTP 500 on its own, anywhere in the code, but some weird network/db/apache/proxy error causes these randomly.
This might remedy the situation a bit.

Fixes #3

Searching through recent errors, I also found quite a lot of 504 errors. Personally I'd include 500, 502, 503 and 504.

I'm not familiar with the code, but I don't have any further concerns except the comment above.

rebased onto 65297a4
6 years ago

Pull-Request has been merged by jskladan
6 years ago
Metadata
None
No Tags
Changes Summary 1
+64 -40
file changed
resultsdb_api.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.