#8552 Random failure in test_ipahealthcheck.py::TestIpaHealthCheck::test_ipa_healthcheck_expiring
Opened a month ago by frenaud. Modified 10 days ago

Issue

The nightly test test_ipahealthcheck.py::TestIpaHealthCheck::test_ipa_healthcheck_expiring can randomly fail, see for instance PR #488 with the followings report:

self = <ipatests.test_integration.test_ipahealthcheck.TestIpaHealthCheck object at 0x7f393ab5ed90>
restart_service = <function restart_service.<locals>._stop_service at 0x7f393af99790>

    def test_ipa_healthcheck_expiring(self, restart_service):
        """
        There are two overlapping tests for expiring certs, check both.
        """

        def execute_expiring_check(check):
            """
            Test that certmonger will report warnings if expiration is near
            """

            returncode, data = run_healthcheck(
                self.master,
                "ipahealthcheck.ipa.certs",
                check,
            )

            assert returncode == 1
            assert len(data) == 12  # KRA is 12 tracked certs

            for check in data:
                if check["result"] == "SUCCESS":
                    # The CA is not expired
                    request = self.master.run_command(
                        ["getcert", "list", "-i", check["kw"]["key"]]
                    )
                    assert "caSigningCert cert-pki-ca" in request.stdout_text
                else:
                    assert check["result"] == "WARNING"
                    if check["kw"]["days"] == 21:
                        # the httpd, 389-ds and KDC renewal dates are later
                        certs = (paths.HTTPD_CERT_FILE, paths.KDC_CERT,
                                 '/etc/dirsrv/slapd-',)
                        request = self.master.run_command(
                            ["getcert", "list", "-i", check["kw"]["key"]]
                        )
                        assert any(cert in request.stdout_text
                                   for cert in certs)
                    else:
                        assert check["kw"]["days"] == 10

        # Store the current date to restore at the end of the test
        now = datetime.utcnow()
        now_str = datetime.strftime(now, "%Y-%m-%d %H:%M:%S Z")

        # Pick a cert to find the upcoming expiration
        certfile = self.master.get_file_contents(paths.RA_AGENT_PEM)
        cert = x509.load_certificate_list(certfile)
        cert_expiry = cert[0].not_valid_after

        for service in ('chronyd', 'pki_tomcatd',):
            restart_service(self.master, service)

        try:
            # move date to the grace period
            grace_date = cert_expiry - timedelta(days=10)
            grace_date = datetime.strftime(grace_date, "%Y-%m-%d 00:00:01 Z")
            self.master.run_command(['date', '-s', grace_date])

            for check in ("IPACertmongerExpirationCheck",
                          "IPACertfileExpirationCheck",):
>               execute_expiring_check(check)

test_integration/test_ipahealthcheck.py:963: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

check = {'check': 'IPACertmongerExpirationCheck', 'duration': '1.386648', 'kw': {'days': 11, 'expiration_date': '2022101200164...s. certmonger should renew this automatically. Watch the status withgetcert list -i {key}.'}, 'result': 'WARNING', ...}

    def execute_expiring_check(check):
        """
        Test that certmonger will report warnings if expiration is near
        """

        returncode, data = run_healthcheck(
            self.master,
            "ipahealthcheck.ipa.certs",
            check,
        )

        assert returncode == 1
        assert len(data) == 12  # KRA is 12 tracked certs

        for check in data:
            if check["result"] == "SUCCESS":
                # The CA is not expired
                request = self.master.run_command(
                    ["getcert", "list", "-i", check["kw"]["key"]]
                )
                assert "caSigningCert cert-pki-ca" in request.stdout_text
            else:
                assert check["result"] == "WARNING"
                if check["kw"]["days"] == 21:
                    # the httpd, 389-ds and KDC renewal dates are later
                    certs = (paths.HTTPD_CERT_FILE, paths.KDC_CERT,
                             '/etc/dirsrv/slapd-',)
                    request = self.master.run_command(
                        ["getcert", "list", "-i", check["kw"]["key"]]
                    )
                    assert any(cert in request.stdout_text
                               for cert in certs)
                else:
>                   assert check["kw"]["days"] == 10
E                   assert 11 == 10
E                     +11
E                     -10

test_integration/test_ipahealthcheck.py:941: AssertionError

The issue happens if the test is launched around midnight:
- server install happens on day X just before midnight, pki server certs will expire on day Y
- KRA install happens on day X+1 just after midnight, KRA certs will expire on day Y+1
- the date is moved to 10 days before server cert expiration, which is 11 days before KRA certs expiration
- the test checks that ipa-healthcheck detects a correct delay before KRA cert expiration but fails


Metadata Update from @frenaud:
- Issue priority set to: minor

10 days ago

Login to comment on this ticket.

Metadata