From 88f24d566adc8f10ed10b4875cae0e0140fc1560 Mon Sep 17 00:00:00 2001
From: Giulia Naponiello <gnaponie@redhat.com>
Date: Apr 19 2019 07:16:10 +0000
Subject: Check old decision before a specific time


The decision endpoint allows to pass results and waivers IDs lists to ignore
(ignore_result, ignore_waiver). These are used to compare the new decision
with older one. In case of multiple new results or waivers there could be a
race condition. This change introduces new parameters results_since and
waivers_since, used to determin the decision before these specific dates.
This solves the race conditions. ignore_result and ignore_waiver are not
used anymore to gather the old decision, but they are still parameters of the
API for backwards compatibility.

---

diff --git a/functional-tests/consumers/test_resultsdb.py b/functional-tests/consumers/test_resultsdb.py
index 16fbbb6..314e95d 100644
--- a/functional-tests/consumers/test_resultsdb.py
+++ b/functional-tests/consumers/test_resultsdb.py
@@ -3,10 +3,10 @@
 import hashlib
 import json
 import mock
-import pprint
 import time
 
 from greenwave.consumers import resultsdb
+from greenwave.utils import right_before_this_time
 
 import handlers
 
@@ -39,7 +39,8 @@ def test_consume_new_result(
                 'data': {
                     'item': [nvr],
                     'type': ['koji_build'],
-                }
+                },
+                'submit_time': result['submit_time']
             }
         }
     }
@@ -181,153 +182,15 @@ def test_consume_unchanged_result(
                 'data': {
                     'item': [nvr],
                     'type': ['koji_build'],
-                }
-            }
-        }
-    }
-    handler = create_resultdb_handler(greenwave_server)
-    handler.consume(message)
-
-    assert len(mock_fedmsg.mock_calls) == 0
-
-
-@mock.patch('greenwave.consumers.resultsdb.fedmsg.publish')
-def test_invalidate_new_result_with_mocked_cache(
-        mock_fedmsg, requests_session, greenwave_server,
-        testdatabuilder):
-    """ Consume a result, and ensure that `delete` is called. """
-    nvr = testdatabuilder.unique_nvr()
-    result = testdatabuilder.create_result(
-        item=nvr, testcase_name='dist.rpmdeplint', outcome='PASSED')
-    message = {
-        'body': {
-            'topic': 'resultsdb.result.new',
-            'msg': {
-                'id': result['id'],
-                'outcome': 'PASSED',
-                'testcase': {
-                    'name': 'dist.rpmdeplint',
                 },
-                'data': {
-                    'item': [nvr],
-                    'type': ['koji_build'],
-                }
+                'submit_time': new_result['submit_time']
             }
         }
     }
     handler = create_resultdb_handler(greenwave_server)
-    handler.cache = mock.MagicMock()
-    handler.consume(message)
-    cache_key1 = 'greenwave.resources:CachedResults|koji_build {} dist.rpmdeplint'.format(nvr)
-    cache_key2 = 'greenwave.resources:CachedResults|koji_build {} None'.format(nvr)
-    handler.cache.delete.assert_has_calls([
-        mock.call(cache_key1),
-        mock.call(cache_key2)
-    ])
-    assert handler.cache.delete.call_count == 2
-
-
-@mock.patch('greenwave.consumers.resultsdb.fedmsg.publish')
-def test_invalidate_new_result_with_real_cache(
-        mock_fedmsg, requests_session, greenwave_server,
-        testdatabuilder, cache_config):
-    nvr = testdatabuilder.unique_nvr()
-    for testcase_name in ['dist.rpmdeplint', 'dist.upgradepath', 'dist.abicheck']:
-        testdatabuilder.create_result(
-            item=nvr, testcase_name=testcase_name, outcome='PASSED')
-
-    # get first passing decision
-    query = {
-        'decision_context': 'bodhi_update_push_stable',
-        'product_version': 'fedora-26',
-        'subject': [{'item': nvr, 'type': 'koji_build'}],
-    }
-    r = requests_session.post(greenwave_server + 'api/v1.0/decision',
-                              headers={'Content-Type': 'application/json'},
-                              data=json.dumps(query))
-    assert r.status_code == 200
-    response = r.json()
-    # Ensure it is passing...
-    assert response['policies_satisfied'], pprint.pformat(response)
-
-    # Now, insert a new result and ensure that caching has made it such that
-    # even though the new result fails, our decision still passes (bad)
-    testdatabuilder.create_result(
-        item=nvr, testcase_name='dist.abicheck', outcome='FAILED')
-    r = requests_session.post(greenwave_server + 'api/v1.0/decision',
-                              headers={'Content-Type': 'application/json'},
-                              data=json.dumps(query))
-    assert r.status_code == 200
-    response = r.json()
-    # Ensure it is passing...  BUT IT SHOULDN'T BE!
-    assert response['policies_satisfied'], pprint.pformat(response)
-
-    # Now, handle a message about the new failing result
-    message = {
-        'body': {
-            'topic': 'resultsdb.result.new',
-            'msg': {
-                'id': 'whatever',
-                'outcome': 'doesn\'t matter',
-                'testcase': {
-                    'name': 'dist.abicheck'
-                },
-                'data': {
-                    'item': [nvr],
-                    'type': ['koji_build'],
-                }
-            }
-        }
-    }
-    handler = create_resultdb_handler(greenwave_server, cache_config)
     handler.consume(message)
 
-    # At this point, the invalidator should have invalidated the cache.  If we
-    # ask again, the decision should be correct now.  It should be a stone cold
-    # "no".
-    r = requests_session.post(greenwave_server + 'api/v1.0/decision',
-                              headers={'Content-Type': 'application/json'},
-                              data=json.dumps(query))
-    assert r.status_code == 200
-    response = r.json()
-    # Ensure it is failing -- as it should be.
-    assert not response['policies_satisfied'], pprint.pformat(response)
-
-
-@mock.patch('greenwave.consumers.resultsdb.fedmsg.publish')
-def test_invalidate_new_result_with_no_preexisting_cache(
-        mock_fedmsg, requests_session, greenwave_server,
-        testdatabuilder):
-    """ Ensure that invalidating an unknown value is sane. """
-    nvr = testdatabuilder.unique_nvr()
-    result = testdatabuilder.create_result(
-        item=nvr, testcase_name='dist.rpmdeplint', outcome='PASSED')
-    message = {
-        'body': {
-            'topic': 'resultsdb.result.new',
-            'msg': {
-                'id': result['id'],
-                'outcome': 'PASSED',
-                'testcase': {
-                    'name': 'dist.rpmdeplint'
-                },
-                'data': {
-                    'item': [nvr],
-                    'type': ['koji_build'],
-                }
-            }
-        }
-    }
-    handler = create_resultdb_handler(greenwave_server)
-    handler.cache.delete = mock.MagicMock()
-    handler.consume(message)
-    cache_key1 = 'greenwave.resources:CachedResults|koji_build {} dist.rpmdeplint'.format(nvr)
-    cache_key2 = 'greenwave.resources:CachedResults|koji_build {} None'.format(nvr)
-    handler.cache.delete.assert_has_calls([
-        mock.call(cache_key1),
-        mock.call(cache_key2)
-    ])
-    assert handler.cache.delete.call_count == 2
+    assert len(mock_fedmsg.mock_calls) == 0
 
 
 @mock.patch('greenwave.consumers.resultsdb.fedmsg.publish')
@@ -352,6 +215,7 @@ def test_consume_compose_id_result(
                 'data': {
                     'productmd.compose.id': [compose_id],
                 },
+                'submit_time': result['submit_time']
             }
         }
     }
@@ -363,7 +227,7 @@ def test_consume_compose_id_result(
         'decision_context': 'rawhide_compose_sync_to_mirrors',
         'product_version': 'fedora-rawhide',
         'subject': [{'productmd.compose.id': compose_id}],
-        'ignore_result': [result['id']]
+        'when': right_before_this_time(result['submit_time'])
     }
     r = requests_session.post(greenwave_server + 'api/v1.0/decision',
                               headers={'Content-Type': 'application/json'},
@@ -424,7 +288,8 @@ def test_consume_legacy_result(
                     'item': nvr,
                     'type': 'koji_build',
                     'name': 'dist.rpmdeplint'
-                }
+                },
+                'submit_time': result['submit_time']
             }
         }
     }
@@ -436,7 +301,7 @@ def test_consume_legacy_result(
         'decision_context': 'bodhi_update_push_stable',
         'product_version': 'fedora-26',
         'subject': [{'item': nvr, 'type': 'koji_build'}],
-        'ignore_result': [result['id']]
+        'when': right_before_this_time(result['submit_time']),
     }
     r = requests_session.post(greenwave_server + 'api/v1.0/decision',
                               headers={'Content-Type': 'application/json'},
@@ -497,7 +362,7 @@ def test_consume_legacy_result(
         'decision_context': 'bodhi_update_push_testing',
         'product_version': 'fedora-26',
         'subject': [{'item': nvr, 'type': 'koji_build'}],
-        'ignore_result': [result['id']]
+        'when': right_before_this_time(result['submit_time']),
     }
     r = requests_session.post(greenwave_server + 'api/v1.0/decision',
                               headers={'Content-Type': 'application/json'},
@@ -555,7 +420,8 @@ def test_no_message_for_nonapplicable_policies(
                 'data': {
                     'item': [nvr],
                     'type': ['koji_build'],
-                }
+                },
+                'submit_time': new_result['submit_time']
             }
         }
     }
@@ -612,7 +478,7 @@ def test_consume_new_result_container_image(
                 "groups": [
                     "341d4cba-ffe2-4d83-b36c-5d819181e86d"
                 ],
-                "submit_time": "2019-01-07T10:54:08.265369",
+                "submit_time": result['submit_time'],
                 "outcome": "PASSED",
                 "data": {
                     "category": [
@@ -692,7 +558,7 @@ def test_consume_new_result_container_image(
         'decision_context': 'container-image-test',
         'product_version': 'c3i',
         'subject': [{'item': nvr, 'type': 'container-image'}],
-        'ignore_result': [result['id']]
+        'when': right_before_this_time(result['submit_time']),
     }
     r = requests_session.post(greenwave_server + 'api/v1.0/decision',
                               headers={'Content-Type': 'application/json'},
diff --git a/functional-tests/consumers/test_waiverdb.py b/functional-tests/consumers/test_waiverdb.py
index 3382c25..3964453 100644
--- a/functional-tests/consumers/test_waiverdb.py
+++ b/functional-tests/consumers/test_waiverdb.py
@@ -56,7 +56,7 @@ def test_consume_new_waiver(
     message = {
         'body': {
             'topic': 'waiver.new',
-            "msg": waiver,
+            'msg': waiver,
         }
     }
     handler = create_waiverdb_handler(greenwave_server)
diff --git a/functional-tests/test_api_v1.py b/functional-tests/test_api_v1.py
index a90e0ce..b33b8d4 100644
--- a/functional-tests/test_api_v1.py
+++ b/functional-tests/test_api_v1.py
@@ -7,6 +7,7 @@ import re
 from textwrap import dedent
 
 from greenwave import __version__
+from greenwave.utils import right_before_this_time
 
 
 TASKTRON_RELEASE_CRITICAL_TASKS = [
@@ -661,14 +662,14 @@ def test_ignore_result(requests_session, greenwave_server, testdatabuilder):
     This tests that a result can be ignored when making the decision.
     """
     nvr = testdatabuilder.unique_nvr()
-    result = testdatabuilder.create_result(
-        item=nvr,
-        testcase_name=TASKTRON_RELEASE_CRITICAL_TASKS[0],
-        outcome='PASSED')
     for testcase_name in TASKTRON_RELEASE_CRITICAL_TASKS[1:]:
         testdatabuilder.create_result(item=nvr,
                                       testcase_name=testcase_name,
                                       outcome='PASSED')
+    result = testdatabuilder.create_result(
+        item=nvr,
+        testcase_name=TASKTRON_RELEASE_CRITICAL_TASKS[0],
+        outcome='PASSED')
     data = {
         'decision_context': 'bodhi_update_push_stable',
         'product_version': 'fedora-26',
@@ -703,6 +704,18 @@ def test_ignore_result(requests_session, greenwave_server, testdatabuilder):
     assert res_data['policies_satisfied'] is False
     assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements
 
+    # repeating the test for "when" parameter instead of "ignore_result"
+    # ...we should get the same behaviour.
+    del(data['ignore_result'])
+    data['when'] = right_before_this_time(result['submit_time'])
+    r = requests_session.post(greenwave_server + 'api/v1.0/decision',
+                              headers={'Content-Type': 'application/json'},
+                              data=json.dumps(data))
+    assert r.status_code == 200
+    res_data = r.json()
+    assert res_data['policies_satisfied'] is False
+    assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements
+
 
 def test_make_a_decision_on_passed_result_with_scenario(
         requests_session, greenwave_server, testdatabuilder):
@@ -787,15 +800,15 @@ def test_ignore_waiver(requests_session, greenwave_server, testdatabuilder):
     result = testdatabuilder.create_result(item=nvr,
                                            testcase_name=TASKTRON_RELEASE_CRITICAL_TASKS[0],
                                            outcome='FAILED')
-    waiver = testdatabuilder.create_waiver(nvr=nvr,
-                                           testcase_name=TASKTRON_RELEASE_CRITICAL_TASKS[0],
-                                           product_version='fedora-26',
-                                           comment='This is fine')
     # The rest passed
     for testcase_name in TASKTRON_RELEASE_CRITICAL_TASKS[1:]:
         testdatabuilder.create_result(item=nvr,
                                       testcase_name=testcase_name,
                                       outcome='PASSED')
+    waiver = testdatabuilder.create_waiver(nvr=nvr,
+                                           testcase_name=TASKTRON_RELEASE_CRITICAL_TASKS[0],
+                                           product_version='fedora-26',
+                                           comment='This is fine')
     data = {
         'decision_context': 'bodhi_update_push_stable',
         'product_version': 'fedora-26',
@@ -829,6 +842,18 @@ def test_ignore_waiver(requests_session, greenwave_server, testdatabuilder):
     assert res_data['policies_satisfied'] is False
     assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements
 
+    # repeating the test for "when" parameter instead of "ignore_waiver"
+    # ...we should get the same behaviour.
+    del(data['ignore_waiver'])
+    data['when'] = right_before_this_time(waiver['timestamp'])
+    r_ = requests_session.post(greenwave_server + 'api/v1.0/decision',
+                               headers={'Content-Type': 'application/json'},
+                               data=json.dumps(data))
+    assert r_.status_code == 200
+    res_data = r_.json()
+    assert res_data['policies_satisfied'] is False
+    assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements
+
 
 def test_distgit_server(requests_session, distgit_server, tmpdir):
     """ This test is checking if the distgit server is working.
@@ -1347,3 +1372,39 @@ def test_api_returns_not_repeated_waiver_in_verbose_info(
     assert r_.status_code == 200
     res_data = r_.json()
     assert len(res_data['waivers']) == 1
+
+
+def test_api_with_when(requests_session, greenwave_server, testdatabuilder):
+    nvr = testdatabuilder.unique_nvr()
+    results = []
+    results.append(testdatabuilder.create_result(item=nvr,
+                                                 testcase_name=TASKTRON_RELEASE_CRITICAL_TASKS[1],
+                                                 outcome='PASSED'))
+    results.append(testdatabuilder.create_result(item=nvr,
+                                                 testcase_name=TASKTRON_RELEASE_CRITICAL_TASKS[2],
+                                                 outcome='FAILED'))
+    data = {
+        'decision_context': 'bodhi_update_push_stable',
+        'product_version': 'fedora-26',
+        'subject_type': 'koji_build',
+        'subject_identifier': nvr,
+        'when': right_before_this_time(results[1]['submit_time']),
+        'verbose': True,
+    }
+    r = requests_session.post(greenwave_server + 'api/v1.0/decision',
+                              headers={'Content-Type': 'application/json'},
+                              data=json.dumps(data))
+    assert r.status_code == 200
+    res_data = r.json()
+
+    assert len(res_data['results']) == 1
+    assert res_data['results'] == [results[0]]
+
+    del data['when']
+    r = requests_session.post(greenwave_server + 'api/v1.0/decision',
+                              headers={'Content-Type': 'application/json'},
+                              data=json.dumps(data))
+    assert r.status_code == 200
+    res_data = r.json()
+
+    assert len(res_data['results']) == 2
diff --git a/greenwave/api_v1.py b/greenwave/api_v1.py
index aac01f4..af1a7c0 100644
--- a/greenwave/api_v1.py
+++ b/greenwave/api_v1.py
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0+
 
 import logging
+import datetime
 from flask import Blueprint, request, current_app, jsonify, url_for, redirect, Response
 from werkzeug.exceptions import BadRequest, NotFound, UnsupportedMediaType
 from prometheus_client import generate_latest
@@ -292,6 +293,8 @@ def make_decision():
         the decision.
     :jsonparam list ignore_waiver: A list of waiver ids that will be ignored when making
         the decision.
+    :jsonparam string when: A date (or datetime) in ISO8601 format. Greenwave will
+        take a decision considering only results and waivers from that point in time.
     :statuscode 200: A decision was made.
     :statuscode 400: Invalid data was given.
     """  # noqa: E501
@@ -318,6 +321,13 @@ def make_decision():
         raise BadRequest('Invalid verbose flag, must be a bool')
     ignore_results = data.get('ignore_result', [])
     ignore_waivers = data.get('ignore_waiver', [])
+    when = data.get('when')
+
+    if when:
+        try:
+            datetime.datetime.strptime(when, '%Y-%m-%dT%H:%M:%S.%f')
+        except ValueError:
+            raise BadRequest('Invalid "when" parameter, must be in ISO8601 format')
 
     answers = []
     verbose_results = []
@@ -326,6 +336,7 @@ def make_decision():
     results_retriever = ResultsRetriever(
         cache=current_app.cache,
         ignore_results=ignore_results,
+        when=when,
         timeout=current_app.config['REQUESTS_TIMEOUT'],
         verify=current_app.config['REQUESTS_VERIFY'],
         url=current_app.config['RESULTSDB_API_URL'])
@@ -351,8 +362,10 @@ def make_decision():
                 'Cannot find any applicable policies for %s subjects at gating point %s in %s' % (
                     subject_type, decision_context, product_version))
 
-        waivers = retrieve_waivers(product_version, subject_type, [subject_identifier])
-        waivers = [w for w in waivers if w['id'] not in ignore_waivers]
+        waivers = retrieve_waivers(
+            product_version, subject_type, [subject_identifier], when)
+        if ignore_waivers:
+            waivers = [w for w in waivers if w['id'] not in ignore_waivers]
 
         for policy in subject_policies:
             answers.extend(
@@ -363,7 +376,7 @@ def make_decision():
         if verbose:
             # Retrieve test results for all items when verbose output is requested.
             verbose_results.extend(
-                results_retriever.retrieve_latest(subject_type, subject_identifier))
+                results_retriever.retrieve(subject_type, subject_identifier))
             verbose_waivers.extend(waivers)
 
     response = {
diff --git a/greenwave/consumers/resultsdb.py b/greenwave/consumers/resultsdb.py
index 5503d87..6da6b33 100644
--- a/greenwave/consumers/resultsdb.py
+++ b/greenwave/consumers/resultsdb.py
@@ -20,6 +20,7 @@ import greenwave.resources
 from greenwave.api_v1 import subject_type_identifier_to_list
 from greenwave.monitoring import publish_decision_exceptions_result_counter
 from greenwave.policies import applicable_decision_context_product_version_pairs
+from greenwave.utils import right_before_this_time
 
 import xmlrpc.client
 
@@ -240,10 +241,7 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer):
         except KeyError:
             testcase = message['msg']['task']['name']  # Old format
 
-        try:
-            result_id = message['msg']['id']  # New format
-        except KeyError:
-            result_id = message['msg']['result']['id']  # Old format
+        submit_time = message['msg']['submit_time']
 
         with self.flask_app.app_context():
             for subject_type, subject_identifier in self.announcement_subjects(message):
@@ -251,17 +249,18 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer):
                 _invalidate_results_cache(
                     self.cache, subject_type, subject_identifier, testcase)
                 self._publish_decision_changes(subject_type, subject_identifier,
-                                               result_id, testcase)
+                                               submit_time, testcase)
 
     @publish_decision_exceptions_result_counter.count_exceptions()
-    def _publish_decision_changes(self, subject_type, subject_identifier, result_id, testcase):
+    def _publish_decision_changes(self, subject_type, subject_identifier, submit_time, testcase):
         """
         Process the given subject and publish a message if the decision is changed.
 
         Args:
-            subject (munch.Munch): A subject argument, used to query greenwave.
-            result_id (int): A result ID to ignore for comparison.
-            testcase (munch.Munch): The name of a testcase to consider.
+            subject_type (munch.Munch): subject type argument, used to query greenwave.
+            subject_identifier (munch.Munch): subject identifier argument, used to query greenwave.
+            submit_time (string): date. After this date, results will be ignored for comparison.
+            testcase (munch.Munch): the name of a testcase to consider.
         """
         product_version = _subject_product_version(
             subject_identifier, subject_type, self.koji_base_url)
@@ -291,7 +290,7 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer):
 
                 # get old decision
                 data.update({
-                    'ignore_result': [result_id],
+                    'when': right_before_this_time(submit_time),
                 })
                 old_decision = greenwave.resources.retrieve_decision(greenwave_url, data)
                 log.debug('old decision: %s', old_decision)
diff --git a/greenwave/consumers/waiverdb.py b/greenwave/consumers/waiverdb.py
index 282e5a2..9f67772 100644
--- a/greenwave/consumers/waiverdb.py
+++ b/greenwave/consumers/waiverdb.py
@@ -19,6 +19,7 @@ import greenwave.app_factory
 from greenwave.api_v1 import subject_type_identifier_to_list
 from greenwave.monitoring import publish_decision_exceptions_waiver_counter
 from greenwave.policies import applicable_decision_context_product_version_pairs
+from greenwave.utils import right_before_this_time
 
 try:
     import fedora_messaging.api
@@ -81,13 +82,14 @@ class WaiverDBHandler(fedmsg.consumers.FedmsgConsumer):
         testcase = msg['testcase']
         subject_type = msg['subject_type']
         subject_identifier = msg['subject_identifier']
+        submit_time = msg['timestamp']
 
         with self.flask_app.app_context():
-            self._publish_decision_changes(subject_type, subject_identifier, msg['id'],
+            self._publish_decision_changes(subject_type, subject_identifier, submit_time,
                                            product_version, testcase)
 
     @publish_decision_exceptions_waiver_counter.count_exceptions()
-    def _publish_decision_changes(self, subject_type, subject_identifier, waiver_id,
+    def _publish_decision_changes(self, subject_type, subject_identifier, submit_time,
                                   product_version, testcase):
         policies = self.flask_app.config['policies']
         contexts_product_versions = applicable_decision_context_product_version_pairs(
@@ -117,7 +119,7 @@ class WaiverDBHandler(fedmsg.consumers.FedmsgConsumer):
 
             # get old decision
             data.update({
-                'ignore_waiver': [waiver_id],
+                'when': right_before_this_time(submit_time),
             })
             response = requests_session.post(
                 self.greenwave_api_url + '/decision',
diff --git a/greenwave/policies.py b/greenwave/policies.py
index b3b6f4a..e528234 100644
--- a/greenwave/policies.py
+++ b/greenwave/policies.py
@@ -2,7 +2,6 @@
 
 from fnmatch import fnmatch
 import glob
-import itertools
 import logging
 import os
 import re
@@ -409,13 +408,12 @@ class PassingTestCaseRule(Rule):
             w for w in waivers if (w['testcase'] == self.test_case_name and w['waived'] is True)]
 
         if self.scenario is not None:
-            matching_results = (
+            matching_results = [
                 result for result in matching_results
-                if self.scenario in result['data']['scenario'])
+                if self.scenario in result['data']['scenario']]
 
         # Investigate the absence of result first.
-        latest_result = next(matching_results, None)
-        if not latest_result:
+        if not matching_results:
             if not matching_waivers:
                 return TestResultMissing(
                     policy.subject_type, subject_identifier, self.test_case_name, self.scenario)
@@ -424,7 +422,6 @@ class PassingTestCaseRule(Rule):
 
         # For compose make decisions based on all architectures and variants.
         if policy.subject_type == 'compose':
-            matching_results = itertools.chain([latest_result], matching_results)
             visited_arch_variants = set()
             answers = []
             for result in matching_results:
@@ -448,8 +445,11 @@ class PassingTestCaseRule(Rule):
         # If we find multiple matching results, we always use the first one which
         # will be the latest chronologically, because ResultsDB always returns
         # results ordered by `submit_time` descending.
-        return self._answer_for_result(
-            latest_result, waivers, policy.subject_type, subject_identifier)
+        answers = []
+        for result in matching_results:
+            answers.append(self._answer_for_result(
+                result, waivers, policy.subject_type, subject_identifier))
+        return answers
 
     def matches(self, policy, **attributes):
         testcase = attributes.get('testcase')
diff --git a/greenwave/resources.py b/greenwave/resources.py
index 6a14264..ec1a895 100644
--- a/greenwave/resources.py
+++ b/greenwave/resources.py
@@ -52,88 +52,57 @@ class ResultsRetriever(object):
     """
     Retrieves results from cache or ResultsDB.
     """
-    def __init__(self, cache, ignore_results, timeout, verify, url):
+    def __init__(self, cache, ignore_results, when, timeout, verify, url):
         self.cache = cache
         self.ignore_results = ignore_results
+        self.when = when
         self.timeout = timeout
         self.verify = verify
         self.url = url
 
-    def retrieve_latest(self, subject_type, subject_identifier):
-        """
-        Return generator over latest results.
-        """
-        params = {}
-        return self._retrieve_helper(params, subject_type, subject_identifier, latest=True)
-
-    def retrieve(self, subject_type, subject_identifier, testcase=None):
+    def retrieve(self, subject_type, subject_identifier, testcase=None, scenarios=None):
         """
         Return generator over results.
         """
-        for result in self._retrieve_all(subject_type, subject_identifier, testcase):
-            if result['id'] not in self.ignore_results:
-                yield result
-
-    def _retrieve_all(self, subject_type, subject_identifier, testcase):
-        cache_key = results_cache_key(
-            subject_type, subject_identifier, testcase)
-
-        cached_results = self.cache.get(cache_key)
-        if not isinstance(cached_results, CachedResults):
-            cached_results = CachedResults()
-
-        for result in cached_results.results:
-            yield result
-
-        while cached_results.can_fetch_more:
-            cached_results.last_page += 1
-            params = {
-                'limit': 1,
-                'page': cached_results.last_page,
-            }
-
-            if testcase:
-                params['testcases'] = testcase
-            results = self._retrieve_helper(params, subject_type, subject_identifier)
-            cached_results.results.extend(results)
-            cached_results.can_fetch_more = bool(results)
-            self.cache.set(cache_key, cached_results)
-            for result in results:
-                yield result
-
-    def _make_request(self, params, latest=False):
-        request_url = self.url + '/results'
-        if latest:
-            request_url += '/latest'
-            # we need to consider also the scenario
-            params['_distinct_on'] = 'scenario'
+        params = {}
+        if self.when:
+            params.update({'since': '1900-01-01T00:00:00.000000,{}'.format(self.when)})
+        if testcase:
+            params.update({'testcases': testcase})
+        if scenarios:
+            params.update({'scenario': ','.join(scenarios)})
+        return self._retrieve_helper(params, subject_type, subject_identifier)
+
+    def _make_request(self, params):
+        params['_distinct_on'] = 'scenario,system_architecture'
         response = requests_session.get(
-            request_url, params=params, verify=self.verify, timeout=self.timeout)
+            self.url + '/results/latest', params=params, verify=self.verify, timeout=self.timeout)
         response.raise_for_status()
         return response.json()['data']
 
-    def _retrieve_helper(self, params, subject_type, subject_identifier, latest=False):
+    def _retrieve_helper(self, params, subject_type, subject_identifier):
         results = []
         if subject_type == 'koji_build':
             params['type'] = subject_type
             params['item'] = subject_identifier
-            results = self._make_request(params=params, latest=latest)
+            results = self._make_request(params=params)
 
             params['type'] = 'brew-build'
-            results.extend(self._make_request(params=params, latest=latest))
+            results.extend(self._make_request(params=params))
 
             del params['type']
             del params['item']
             params['original_spec_nvr'] = subject_identifier
-            results.extend(self._make_request(params=params, latest=latest))
+            results.extend(self._make_request(params=params))
         elif subject_type == 'compose':
             params['productmd.compose.id'] = subject_identifier
-            results = self._make_request(params=params, latest=latest)
+            results = self._make_request(params=params)
         else:
             params['type'] = subject_type
             params['item'] = subject_identifier
-            results = self._make_request(params=params, latest=latest)
+            results = self._make_request(params=params)
 
+        results = [r for r in results if r['id'] not in self.ignore_results]
         return results
 
 
@@ -244,17 +213,22 @@ def _retrieve_yaml_remote_rule_git_archive(rev, pkg_name, pkg_namespace):
 
 # NOTE - not cached, for now.
 @greenwave.utils.retry(wait_on=urllib3.exceptions.NewConnectionError)
-def retrieve_waivers(product_version, subject_type, subject_identifiers):
+def retrieve_waivers(product_version, subject_type, subject_identifiers, when):
     if not subject_identifiers:
         return []
 
     timeout = current_app.config['REQUESTS_TIMEOUT']
     verify = current_app.config['REQUESTS_VERIFY']
-    filters = [{
-        'product_version': product_version,
-        'subject_type': subject_type,
-        'subject_identifier': subject_identifier,
-    } for subject_identifier in subject_identifiers]
+    filters = []
+    for subject_identifier in subject_identifiers:
+        d = {
+            'product_version': product_version,
+            'subject_type': subject_type,
+            'subject_identifier': subject_identifier
+        }
+        if when:
+            d['since']: '1900-01-01T00:00:00.000000,{}'.format(when)
+        filters.append(d)
     response = requests_session.post(
         current_app.config['WAIVERDB_API_URL'] + '/waivers/+filtered',
         headers={'Content-Type': 'application/json'},
diff --git a/greenwave/tests/test_cache.py b/greenwave/tests/test_cache.py
deleted file mode 100644
index 2e5aaa5..0000000
--- a/greenwave/tests/test_cache.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0+
-
-import mock
-
-import greenwave.resources
-
-
-def test_resultsdb_cache():
-    subject_type = 'koji_build'
-    subject_identifier = 'nethack-1.2.3-1.el9000'
-    testcase = ''
-
-    cache = mock.Mock()
-
-    with mock.patch('greenwave.resources.ResultsRetriever._make_request') as retrieve_method:
-        retrieve_method.return_value = []
-
-        results_retriever = greenwave.resources.ResultsRetriever(
-            cache=cache,
-            ignore_results=[],
-            timeout=0,
-            verify=False,
-            url='')
-
-        results = results_retriever.retrieve(subject_type, subject_identifier, testcase)
-
-        assert retrieve_method.call_count == 0
-        assert cache.set.call_count == 0
-
-        assert len(list(results)) == 0
-
-        assert retrieve_method.call_count > 0
-        assert cache.set.call_count == 1
-
-    key = greenwave.resources.results_cache_key(
-        subject_type, subject_identifier, testcase)
-    actual_key, actual_cached_results = cache.set.call_args[0]
-    assert actual_key == key
-
-    cache.get.return_value = actual_cached_results
-
-    with mock.patch('greenwave.resources.ResultsRetriever._make_request') as retrieve_method:
-        retrieve_method.return_value = []
-
-        results_retriever = greenwave.resources.ResultsRetriever(
-            cache=cache,
-            ignore_results=[],
-            timeout=0,
-            verify=False,
-            url='')
-
-        results = results_retriever.retrieve(subject_type, subject_identifier, testcase)
-
-        assert retrieve_method.call_count == 0
-        assert cache.set.call_count == 1
-
-        assert len(list(results)) == 0
-
-        assert retrieve_method.call_count == 0
-        assert cache.set.call_count == 1
diff --git a/greenwave/tests/test_policies.py b/greenwave/tests/test_policies.py
index 3c2a015..cfbddbb 100644
--- a/greenwave/tests/test_policies.py
+++ b/greenwave/tests/test_policies.py
@@ -29,6 +29,7 @@ class DummyResultsRetriever(ResultsRetriever):
         super(DummyResultsRetriever, self).__init__(
             cache=mock.Mock(),
             ignore_results=[],
+            when='',
             timeout=0,
             verify=False,
             url='')
@@ -37,7 +38,7 @@ class DummyResultsRetriever(ResultsRetriever):
         self.testcase = testcase
         self.outcome = outcome
 
-    def _make_request(self, params, latest=False):
+    def _make_request(self, params):
         if (params.get('item') == self.subject_identifier and
                 params.get('type') == self.subject_type and
                 params.get('testcases') == self.testcase):
diff --git a/greenwave/tests/test_resultsdb_consumer.py b/greenwave/tests/test_resultsdb_consumer.py
index 4f2b022..85fca67 100644
--- a/greenwave/tests/test_resultsdb_consumer.py
+++ b/greenwave/tests/test_resultsdb_consumer.py
@@ -160,12 +160,13 @@ def test_remote_rule_decision_change(
                 'testcase': {'name': 'dist.rpmdeplint'},
                 'outcome': 'PASSED',
                 'data': {'item': nvr, 'type': 'koji_build'},
+                'submit_time': '2019-03-25T16:34:41.882620'
             }
             mock_retrieve_results.return_value = [result]
 
             def retrieve_decision(url, data):
                 #pylint: disable=unused-argument
-                if 'ignore_result' in data:
+                if 'when' in data:
                     return None
                 return {}
             mock_retrieve_decision.side_effect = retrieve_decision
@@ -184,7 +185,8 @@ def test_remote_rule_decision_change(
                         'data': {
                             'item': [nvr],
                             'type': ['koji_build'],
-                        }
+                        },
+                        'submit_time': '2019-03-25T16:34:41.882620'
                     }
                 }
             }
@@ -266,12 +268,13 @@ def test_remote_rule_decision_change_not_matching(
                 'testcase': {'name': 'dist.rpmdeplint'},
                 'outcome': 'PASSED',
                 'data': {'item': nvr, 'type': 'koji_build'},
+                'submit_time': '2019-03-25T16:34:41.882620'
             }
             mock_retrieve_results.return_value = [result]
 
             def retrieve_decision(url, data):
                 #pylint: disable=unused-argument
-                if 'ignore_result' in data:
+                if 'when' in data:
                     return None
                 return {}
             mock_retrieve_decision.side_effect = retrieve_decision
@@ -290,7 +293,8 @@ def test_remote_rule_decision_change_not_matching(
                         'data': {
                             'item': [nvr],
                             'type': ['koji_build'],
-                        }
+                        },
+                        'submit_time': '2019-03-25T16:34:41.882620'
                     }
                 }
             }
@@ -374,12 +378,13 @@ def test_decision_change_for_modules(
                 'testcase': {'name': 'baseos-ci.redhat-module.tier1.functional'},
                 'outcome': 'PASSED',
                 'data': {'item': nsvc, 'type': 'redhat-module'},
+                'submit_time': '2019-03-25T16:34:41.882620'
             }
             mock_retrieve_results.return_value = [result]
 
             def retrieve_decision(url, data):
                 #pylint: disable=unused-argument
-                if 'ignore_result' in data:
+                if 'when' in data:
                     return None
                 return {}
             mock_retrieve_decision.side_effect = retrieve_decision
@@ -398,7 +403,8 @@ def test_decision_change_for_modules(
                         'data': {
                             'item': [nsvc],
                             'type': ['redhat-module'],
-                        }
+                        },
+                        'submit_time': '2019-03-25T16:34:41.882620'
                     }
                 }
             }
diff --git a/greenwave/utils.py b/greenwave/utils.py
index 7c499c0..ff0301c 100644
--- a/greenwave/utils.py
+++ b/greenwave/utils.py
@@ -5,6 +5,7 @@ import logging
 import os
 import time
 import hashlib
+import datetime
 
 from flask import jsonify, current_app, request
 from flask.config import Config
@@ -147,3 +148,17 @@ def sha1_mangle_key(key):
     to hashlib.sha1()).
     """
     return hashlib.sha1(key.encode('utf-8')).hexdigest()
+
+
+def right_before_this_time(timestamp):
+    """
+    A utility function that takes a timestamp with format %Y-%m-%dT%H:%M:%S.%f
+    and returns the microsecond before that timestamp. It returns always a
+    %Y-%m-%dT%H:%M:%S.%f format. It is useful to detect at which moment we should
+    ask for a decision before a specific timestamp (example: the creation of a
+    result).
+    """
+    date_format = '%Y-%m-%dT%H:%M:%S.%f'
+    return datetime.datetime.strftime(
+        datetime.datetime.strptime(timestamp, date_format) -
+        datetime.timedelta(microseconds=1), date_format)