From 1f9f84a66d6cf9391b91ee4a13ac0f1119212578 Mon Sep 17 00:00:00 2001
From: Simo Sorce <simo@redhat.com>
Date: May 03 2017 14:37:21 +0000
Subject: Make sure remote hosts have our keys


In complex replication setups a replica may try to obtain CA keys from a
host that is not the master we initially create the keys against.
In this case race conditions may happen due to replication. So we need
to make sure the server we are contacting to get the CA keys has our
keys in LDAP. We do this by waiting to positively fetch our encryption
public key (the last one we create) from the target host LDAP server.

Fixes: https://pagure.io/freeipa/issue/6838

Signed-off-by: Simo Sorce <simo@redhat.com>
Reviewed-By: Stanislav Laznicka <slaznick@redhat.com>
Reviewed-By: Christian Heimes <cheimes@redhat.com>

---

diff --git a/ipaserver/install/custodiainstance.py b/ipaserver/install/custodiainstance.py
index 6a61392..390576b 100644
--- a/ipaserver/install/custodiainstance.py
+++ b/ipaserver/install/custodiainstance.py
@@ -1,6 +1,6 @@
 # Copyright (C) 2015 FreeIPa Project Contributors, see 'COPYING' for license.
 
-from ipaserver.secrets.kem import IPAKEMKeys
+from ipaserver.secrets.kem import IPAKEMKeys, KEMLdap
 from ipaserver.secrets.client import CustodiaClient
 from ipaplatform.paths import paths
 from ipaplatform.constants import constants
@@ -18,6 +18,7 @@ import shutil
 import os
 import stat
 import tempfile
+import time
 import pwd
 
 
@@ -122,6 +123,27 @@ class CustodiaInstance(SimpleServiceInstance):
         cli = self.__CustodiaClient(server=master_host_name)
         cli.fetch_key('dm/DMHash')
 
+    def __wait_keys(self, host, timeout=300):
+        ldap_uri = 'ldap://%s' % host
+        deadline = int(time.time()) + timeout
+        root_logger.info("Waiting up to {} seconds to see our keys "
+                         "appear on host: {}".format(timeout, host))
+
+        konn = KEMLdap(ldap_uri)
+        saved_e = None
+        while True:
+            try:
+                return konn.check_host_keys(self.fqdn)
+            except Exception as e:
+                # log only once for the same error
+                if not isinstance(e, type(saved_e)):
+                    root_logger.debug(
+                        "Transient error getting keys: '{err}'".format(err=e))
+                    saved_e = e
+                if int(time.time()) > deadline:
+                    raise RuntimeError("Timed out trying to obtain keys.")
+                time.sleep(1)
+
     def __get_keys(self, ca_host, cacerts_file, cacerts_pwd, data):
         # Fecth all needed certs one by one, then combine them in a single
         # p12 file
@@ -129,6 +151,10 @@ class CustodiaInstance(SimpleServiceInstance):
         prefix = data['prefix']
         certlist = data['list']
 
+        # Before we attempt to fetch keys from this host, make sure our public
+        # keys have been replicated there.
+        self.__wait_keys(ca_host)
+
         cli = self.__CustodiaClient(server=ca_host)
 
         # Temporary nssdb
diff --git a/ipaserver/secrets/kem.py b/ipaserver/secrets/kem.py
index 28fb4d3..c1991c6 100644
--- a/ipaserver/secrets/kem.py
+++ b/ipaserver/secrets/kem.py
@@ -24,6 +24,7 @@ import ldap
 
 IPA_REL_BASE_DN = 'cn=custodia,cn=ipa,cn=etc'
 IPA_KEYS_QUERY = '(&(ipaKeyUsage={usage:s})(memberPrincipal={princ:s}))'
+IPA_CHECK_QUERY = '(cn=enc/{host:s})'
 RFC5280_USAGE_MAP = {KEY_USAGE_SIG: 'digitalSignature',
                      KEY_USAGE_ENC: 'dataEncipherment'}
 
@@ -78,6 +79,17 @@ class KEMLdap(iSecLdap):
         jwk['use'] = KEY_USAGE_MAP[usage]
         return json_encode(jwk)
 
+    def check_host_keys(self, host):
+        conn = self.connect()
+        scope = ldap.SCOPE_SUBTREE
+
+        ldap_filter = self.build_filter(IPA_CHECK_QUERY, {'host': host})
+        r = conn.search_s(self.keysbase, scope, ldap_filter)
+        if len(r) != 1:
+            raise ValueError("Incorrect number of results (%d) searching for"
+                             "public key for %s" % (len(r), host))
+        return True
+
     def _format_public_key(self, key):
         if isinstance(key, str):
             jwkey = json_decode(key)