#8256 Nightly test failure in test_integration/test_installation.py::TestInstallCA::test_certmonger_reads_token_HSM
Closed: worksforme 3 years ago by frenaud. Opened 4 years ago by frenaud.

The nightly test test_integration/test_installation.py::TestInstallCA::test_certmonger_reads_token_HSM failed in [testing_master_latest] Nightly PR #4491. The logs show that certmonger resubmit request was stuck on SUBMITTING:

self = <ipatests.test_integration.test_installation.TestInstallCA object at 0x7f4fb991c390>

    def test_certmonger_reads_token_HSM(self):
        """Test if certmonger reads the token in HSM

        This is to ensure added HSM support for FreeIPA. This test adds
        certificate with sofhsm token and checks if certmonger is tracking
        it.

        related : https://pagure.io/certmonger/issue/125
        """
        test_service = 'test/%s' % self.master.hostname
        pkcs_passwd = 'Secret123'
        pin = '123456'
        noisefile = '/tmp/noisefile'
        self.master.put_file_contents(noisefile, os.urandom(64))

        tasks.kinit_admin(self.master)
        tasks.install_dns(self.master)
        self.master.run_command(['ipa', 'service-add', test_service])

        # create a csr
        cmd_args = ['certutil', '-d', paths.NSS_DB_DIR, '-R', '-a',
                    '-o', '/root/ipa.csr',
                    '-s', "CN=%s" % self.master.hostname,
                    '-z', noisefile]
        self.master.run_command(cmd_args)

        # request certificate
        cmd_args = ['ipa', 'cert-request', '--principal', test_service,
                    '--certificate-out', '/root/test.pem', '/root/ipa.csr']
        self.master.run_command(cmd_args)

        # adding trust flag
        cmd_args = ['certutil', '-A', '-d', paths.NSS_DB_DIR, '-n',
                    'test', '-a', '-i', '/root/test.pem', '-t', 'u,u,u']
        self.master.run_command(cmd_args)

        # export pkcs12 file
        cmd_args = ['pk12util', '-o', '/root/test.p12',
                    '-d', paths.NSS_DB_DIR, '-n', 'test', '-W', pkcs_passwd]
        self.master.run_command(cmd_args)

        # add softhsm lib
        cmd_args = ['modutil', '-dbdir', paths.NSS_DB_DIR, '-add',
                    'softhsm', '-libfile', '/usr/lib64/softhsm/libsofthsm.so']
        self.master.run_command(cmd_args, stdin_text="\n\n")

        # create a token
        cmd_args = ['softhsm2-util', '--init-token', '--label', 'test',
                    '--pin', pin, '--so-pin', pin, '--free']
        self.master.run_command(cmd_args)

        self.master.run_command(['softhsm2-util', '--show-slots'])

        cmd_args = ['certutil', '-F', '-d', paths.NSS_DB_DIR, '-n', 'test']
        self.master.run_command(cmd_args)

        cmd_args = ['pk12util', '-i', '/root/test.p12',
                    '-d', paths.NSS_DB_DIR, '-h', 'test',
                    '-W', pkcs_passwd, '-K', pin]
        self.master.run_command(cmd_args)

        cmd_args = ['certutil', '-A', '-d', paths.NSS_DB_DIR, '-n', 'IPA CA',
                    '-t', 'CT,,', '-a', '-i', paths.IPA_CA_CRT]
        self.master.run_command(cmd_args)

        # validate the certificate
        self.master.put_file_contents('/root/pinfile', pin)
        cmd_args = ['certutil', '-V', '-u', 'V', '-e', '-d', paths.NSS_DB_DIR,
                    '-h', 'test', '-n', 'test:test', '-f', '/root/pinfile']
        result = self.master.run_command(cmd_args)
        assert 'certificate is valid' in result.stdout_text

        # add certificate tracking to certmonger
        cmd_args = ['ipa-getcert', 'start-tracking', '-d', paths.NSS_DB_DIR,
                    '-n', 'test', '-t', 'test', '-P', pin,
                    '-K', test_service]
        result = self.master.run_command(cmd_args)
        request_id = re.findall(r'\d+', result.stdout_text)

        # check if certificate is tracked by certmonger
        status = tasks.wait_for_request(self.master, request_id[0], 300)
        assert status == "MONITORING"

        # ensure if key and token are re-usable
        cmd_args = ['getcert', 'resubmit', '-i', request_id[0]]
        self.master.run_command(cmd_args)

>       status = tasks.wait_for_request(self.master, request_id[0], 300)

test_integration/test_installation.py:346: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

host = <ipatests.pytest_ipa.integration.host.Host master.ipa.test (master)>
request_id = '20200404001852', timeout = 300

    def wait_for_request(host, request_id, timeout=120):
        for _i in range(0, timeout, 5):
            result = host.run_command(
                "getcert list -i %s | grep status: | awk '{ print $2 }'" %
                request_id
            )

            state = result.stdout_text.strip()
            logger.info("certmonger request is in state %s", state)
            if state in ('CA_REJECTED', 'CA_UNREACHABLE', 'CA_UNCONFIGURED',
                         'NEED_GUIDANCE', 'NEED_CA', 'MONITORING'):
                break
            time.sleep(5)
        else:
>           raise RuntimeError("request timed out")
E           RuntimeError: request timed out

pytest_ipa/integration/tasks.py:2145: RuntimeError

Also happened in PR 4515 [testing_master_latest]

Metadata Update from @frenaud:
- Issue tagged with: test-failure, tests

4 years ago

Also happened in PR 4531 [testing_master_previous] logs

Another instance: testing_master_testing PR 4555
logs

Also happened in PR 280 testing_master_previous, logs

This test had 10+ green runs. The issue was probably fixed by the certmonger fix https://pagure.io/certmonger/c/00e948049acf0ca1b61ed9c2b8579b06b4bcb46a?branch=master which made it into certmonger 0.79.12.
The issue can be closed.

Metadata Update from @frenaud:
- Issue close_status updated to: worksforme
- Issue status updated to: Closed (was: Open)

3 years ago

Login to comment on this ticket.

Metadata