From 7b3c401a8fa914e8436e17dd46e8b7866759240b Mon Sep 17 00:00:00 2001 From: Simon Pichugin Date: Jul 30 2018 12:22:23 +0000 Subject: Issue 49761 - Fix replication test suite issues Description - the issues: cleanallruv - add 'certify' to the abort tasks certify and adjust a timeout; encryption_cl5 - add the encryption attributes to the schema (we had used extensibleObject before but now we use nsChangelogConfig); tombstone - fix how Tombstone(DSLdapObject) handles the searches. https://pagure.io/389-ds-base/issue/49761 Reviewed by: vashirov, mreynolds (Thanks!) --- diff --git a/dirsrvtests/tests/suites/replication/cleanallruv_test.py b/dirsrvtests/tests/suites/replication/cleanallruv_test.py index 4486c2f..74eae91 100644 --- a/dirsrvtests/tests/suites/replication/cleanallruv_test.py +++ b/dirsrvtests/tests/suites/replication/cleanallruv_test.py @@ -61,6 +61,7 @@ class AddUsers(threading.Thread): pass conn.close() + def remove_master4_agmts(msg, topology_m4): """Remove all the repl agmts to master4. """ @@ -71,6 +72,7 @@ def remove_master4_agmts(msg, topology_m4): repl.remove_master(topology_m4.ms["master4"], [topology_m4.ms["master1"], topology_m4.ms["master2"], topology_m4.ms["master3"]]) + def check_ruvs(msg, topology_m4, m4rid): """Check masters 1- 3 for master 4's rid.""" for inst in (topology_m4.ms["master1"], topology_m4.ms["master2"], topology_m4.ms["master3"]): @@ -90,6 +92,7 @@ def check_ruvs(msg, topology_m4, m4rid): raise Exception("Master %s was not cleaned in time." % inst.serverid) return True + def task_done(topology_m4, task_dn, timeout=60): """Check if the task is complete""" @@ -136,8 +139,11 @@ def restore_master4(topology_m4): log.info('Master 4 has been successfully restored.') + @pytest.fixture() def m4rid(request, topology_m4): + log.debug("Wait a bit before the reset - it is required fot the slow machines") + time.sleep(5) log.debug("-------------- BEGIN RESET of m4 -----------------") repl = ReplicationManager(DEFAULT_SUFFIX) repl.test_replication_topology(topology_m4.ms.values()) @@ -146,10 +152,15 @@ def m4rid(request, topology_m4): def fin(): try: + # Restart the masters and rerun cleanallruv + for inst in topology_m4.ms.values(): + inst.restart() + cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no', }) cruv_task.wait() except ldap.UNWILLING_TO_PERFORM: @@ -162,6 +173,7 @@ def m4rid(request, topology_m4): log.debug("-------------- FINISH RESET of m4 -----------------") return m4rid + def test_clean(topology_m4, m4rid): """Check that cleanallruv task works properly @@ -192,7 +204,8 @@ def test_clean(topology_m4, m4rid): cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no' }) cruv_task.wait() @@ -247,7 +260,9 @@ def test_clean_restart(topology_m4, m4rid): cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no', + 'replica-certify-all': 'yes' }) # Sleep a bit, then stop master 1 @@ -272,6 +287,7 @@ def test_clean_restart(topology_m4, m4rid): log.info('test_clean_restart PASSED, restoring master 4...') + def test_clean_force(topology_m4, m4rid): """Check that multiple tasks with a 'force' option work properly @@ -362,7 +378,9 @@ def test_abort(topology_m4, m4rid): cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no', + 'replica-certify-all': 'yes' }) # Wait a bit time.sleep(2) @@ -382,6 +400,7 @@ def test_abort(topology_m4, m4rid): log.info('test_abort PASSED, restoring master 4...') + def test_abort_restart(topology_m4, m4rid): """Test the abort task can handle a restart, and then resume @@ -424,13 +443,15 @@ def test_abort_restart(topology_m4, m4rid): cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no', + 'replica-certify-all': 'yes' }) # Wait a bit time.sleep(2) # Abort the task - abort_task = cruv_task.abort() + cruv_task.abort(certify=True) # Check master 1 does not have the clean task running log.info('test_abort_abort: check master 1 no longer has a cleanAllRUV task...') @@ -488,7 +509,9 @@ def test_abort_certify(topology_m4, m4rid): cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no', + 'replica-certify-all': 'yes' }) # Wait a bit time.sleep(2) @@ -498,10 +521,9 @@ def test_abort_certify(topology_m4, m4rid): abort_task = cruv_task.abort(certify=True) # Wait a while and make sure the abort task is still running - log.info('test_abort_certify: sleep for 5 seconds') - time.sleep(5) + log.info('test_abort_certify...') - if task_done(topology_m4, abort_task.dn, 60): + if task_done(topology_m4, abort_task.dn, 10): log.fatal('test_abort_certify: abort task incorrectly finished') assert False @@ -510,7 +532,7 @@ def test_abort_certify(topology_m4, m4rid): topology_m4.ms["master2"].start() # Wait for the abort task to stop - if not task_done(topology_m4, abort_task.dn, 60): + if not task_done(topology_m4, abort_task.dn, 90): log.fatal('test_abort_certify: The abort CleanAllRUV task was not aborted') assert False @@ -577,7 +599,8 @@ def test_stress_clean(topology_m4, m4rid): cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no' }) cruv_task.wait() @@ -657,18 +680,21 @@ def test_multiple_tasks_with_force(topology_m4, m4rid): cruv_task.create(properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX, - 'replica-force-cleaning': 'yes' + 'replica-force-cleaning': 'yes', + 'replica-certify-all': 'no' }) log.info('test_multiple_tasks_with_force: run the cleanAllRUV task with "force" off...') # NOTE: This must be try not py.test raises, because the above may or may - # not have completed yet .... + # not have completed yet .... try: cruv_task_fail = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task_fail.create(properties={ 'replica-id': m4rid, - 'replica-base-dn': DEFAULT_SUFFIX + 'replica-base-dn': DEFAULT_SUFFIX, + 'replica-force-cleaning': 'no', + 'replica-certify-all': 'no' }) cruv_task_fail.wait() except ldap.UNWILLING_TO_PERFORM: diff --git a/dirsrvtests/tests/suites/replication/replica_config_test.py b/dirsrvtests/tests/suites/replication/replica_config_test.py index c93c58b..a8f9d63 100644 --- a/dirsrvtests/tests/suites/replication/replica_config_test.py +++ b/dirsrvtests/tests/suites/replication/replica_config_test.py @@ -178,7 +178,7 @@ def test_replica_num_modify(topo, attr, too_small, too_big, overflow, notnum, va replica.replace(attr, valid) -@pytest.mark.skip(reason="Agreement validation current does not work.") +@pytest.mark.xfail(reason="Agreement validation current does not work.") @pytest.mark.parametrize("attr, too_small, too_big, overflow, notnum, valid", agmt_attrs) def test_agmt_num_add(topo, attr, too_small, too_big, overflow, notnum, valid): """Test all the number values you can set for a replica config entry @@ -198,6 +198,7 @@ def test_agmt_num_add(topo, attr, too_small, too_big, overflow, notnum, valid): 4. Add is rejected 5. Add is allowed """ + agmt_reset(topo) replica = replica_setup(topo) @@ -217,7 +218,7 @@ def test_agmt_num_add(topo, attr, too_small, too_big, overflow, notnum, valid): agmts.create(properties=my_agmt) -@pytest.mark.skip(reason="Agreement validation current does not work.") +@pytest.mark.xfail(reason="Agreement validation current does not work.") @pytest.mark.parametrize("attr, too_small, too_big, overflow, notnum, valid", agmt_attrs) def test_agmt_num_modify(topo, attr, too_small, too_big, overflow, notnum, valid): """Test all the number values you can set for a replica config entry diff --git a/dirsrvtests/tests/suites/replication/ruvstore_test.py b/dirsrvtests/tests/suites/replication/ruvstore_test.py index 096670b..6fadfb0 100755 --- a/dirsrvtests/tests/suites/replication/ruvstore_test.py +++ b/dirsrvtests/tests/suites/replication/ruvstore_test.py @@ -113,7 +113,8 @@ def test_ruv_entry_backup(topo): parser = MyLDIF(ldif_file) parser.parse() -@pytest.mark.skip(reason="No method to safety access DB ruv currenty exists online.") + +@pytest.mark.xfail(reason="No method to safety access DB ruv currently exists online.") def test_memoryruv_sync_with_databaseruv(topo): """Check if memory ruv and database ruv are synced diff --git a/ldap/schema/01core389.ldif b/ldap/schema/01core389.ldif index 67f5469..a595b76 100644 --- a/ldap/schema/01core389.ldif +++ b/ldap/schema/01core389.ldif @@ -310,6 +310,8 @@ attributeTypes: ( 2.16.840.1.113730.3.1.2339 NAME 'nsslapd-changelogdir' DESC 'T attributeTypes: ( 2.16.840.1.113730.3.1.2340 NAME 'nsslapd-changelogmaxage' DESC 'The changelog5 time where an entry will be retained' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' ) attributeTypes: ( 2.16.840.1.113730.3.1.2341 NAME 'nsslapd-changelogmaxentries' DESC 'The changelog5 max entries limit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' ) attributeTypes: ( 2.16.840.1.113730.3.1.2344 NAME 'nsslapd-tls-check-crl' DESC 'Check CRL when opening outbound TLS connections. Valid options are none, peer, all.' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' ) +attributeTypes: ( 2.16.840.1.113730.3.1.2353 NAME 'nsslapd-encryptionalgorithm' DESC 'The encryption algorithm used to encrypt the changelog' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' ) +attributeTypes: ( 2.16.840.1.113730.3.1.2084 NAME 'nsSymmetricKey' DESC 'A symmetric key - currently used by attribute encryption' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 SINGLE-VALUE X-ORIGIN 'attribute encryption' ) # # objectclasses # @@ -329,4 +331,4 @@ objectClasses: ( nsEncryptionConfig-oid NAME 'nsEncryptionConfig' DESC 'Netscape objectClasses: ( nsEncryptionModule-oid NAME 'nsEncryptionModule' DESC 'Netscape defined objectclass' SUP top MUST ( cn ) MAY ( nsSSLToken $ nsSSLPersonalityssl $ nsSSLActivation $ ServerKeyExtractFile $ ServerCertExtractFile ) X-ORIGIN 'Netscape' ) objectClasses: ( 2.16.840.1.113730.3.2.327 NAME 'rootDNPluginConfig' DESC 'Netscape defined objectclass' SUP top MUST ( cn ) MAY ( rootdn-open-time $ rootdn-close-time $ rootdn-days-allowed $ rootdn-allow-host $ rootdn-deny-host $ rootdn-allow-ip $ rootdn-deny-ip ) X-ORIGIN 'Netscape' ) objectClasses: ( 2.16.840.1.113730.3.2.328 NAME 'nsSchemaPolicy' DESC 'Netscape defined objectclass' SUP top MAY ( cn $ schemaUpdateObjectclassAccept $ schemaUpdateObjectclassReject $ schemaUpdateAttributeAccept $ schemaUpdateAttributeReject) X-ORIGIN 'Netscape Directory Server' ) -objectClasses: ( 2.16.840.1.113730.3.2.332 NAME 'nsChangelogConfig' DESC 'Configuration of the changelog5 object' SUP top MUST ( cn $ nsslapd-changelogdir ) MAY ( nsslapd-changelogmaxage $ nsslapd-changelogtrim-interval $ nsslapd-changelogmaxentries $ nsslapd-changelogsuffix $ nsslapd-changelogcompactdb-interval ) X-ORIGIN '389 Directory Server' ) +objectClasses: ( 2.16.840.1.113730.3.2.332 NAME 'nsChangelogConfig' DESC 'Configuration of the changelog5 object' SUP top MUST ( cn $ nsslapd-changelogdir ) MAY ( nsslapd-changelogmaxage $ nsslapd-changelogtrim-interval $ nsslapd-changelogmaxentries $ nsslapd-changelogsuffix $ nsslapd-changelogcompactdb-interval $ nsslapd-encryptionalgorithm $ nsSymmetricKey ) X-ORIGIN '389 Directory Server' ) diff --git a/ldap/schema/50ns-directory.ldif b/ldap/schema/50ns-directory.ldif index 1f85398..298d607 100644 --- a/ldap/schema/50ns-directory.ldif +++ b/ldap/schema/50ns-directory.ldif @@ -82,7 +82,6 @@ attributeTypes: ( 2.16.840.1.113730.3.1.54 NAME 'replicaUseSSL' DESC 'Netscape d attributeTypes: ( 2.16.840.1.113730.3.1.57 NAME 'replicaRoot' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-ORIGIN 'Netscape Directory Server' ) attributeTypes: ( 2.16.840.1.113730.3.1.58 NAME 'replicaBindDn' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-ORIGIN 'Netscape Directory Server' ) attributeTypes: ( 2.16.840.1.113730.3.1.69 NAME 'subtreeACI' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 X-ORIGIN 'Netscape Directory Server 1.0' ) -attributeTypes: ( 2.16.840.1.113730.3.1.2084 NAME 'nsSymmetricKey' DESC 'A symmetric key - currently used by attribute encryption' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 SINGLE-VALUE X-ORIGIN 'attribute encryption' ) objectClasses: ( 2.16.840.1.113730.3.2.23 NAME 'netscapeDirectoryServer' DESC 'Netscape defined objectclass' SUP top MUST ( objectclass ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( nsDirectoryServer-oid NAME 'nsDirectoryServer' DESC 'Netscape defined objectclass' SUP top MUST ( objectclass $ nsServerID ) MAY ( serverHostName $ nsServerPort $ nsSecureServerPort $ nsBindPassword $ nsBindDN $ nsBaseDN ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( 2.16.840.1.113730.3.2.8 NAME 'ntUser' DESC 'Netscape defined objectclass' SUP top MUST ( ntUserDomainId ) MAY ( description $ l $ ou $ seeAlso $ ntUserPriv $ ntUserHomeDir $ ntUserComment $ ntUserFlags $ ntUserScriptPath $ ntUserAuthFlags $ ntUserUsrComment $ ntUserParms $ ntUserWorkstations $ ntUserLastLogon $ ntUserLastLogoff $ ntUserAcctExpires $ ntUserMaxStorage $ ntUserUnitsPerWeek $ ntUserLogonHours $ ntUserBadPwCount $ ntUserNumLogons $ ntUserLogonServer $ ntUserCountryCode $ ntUserCodePage $ ntUserUniqueId $ ntUserPrimaryGroupId $ ntUserProfile $ ntUserHomeDirDrive $ ntUserPasswordExpired $ ntUserCreateNewAccount $ ntUserDeleteAccount $ ntUniqueId $ ntUserNtPassword ) X-ORIGIN 'Netscape NT Synchronization' ) diff --git a/src/lib389/lib389/tombstone.py b/src/lib389/lib389/tombstone.py index 154fa1a..839fc3e 100644 --- a/src/lib389/lib389/tombstone.py +++ b/src/lib389/lib389/tombstone.py @@ -10,8 +10,10 @@ import ldap from lib389._entry import Entry - +from lib389._constants import DIRSRV_STATE_ONLINE from lib389._mapped_object import DSLdapObject, DSLdapObjects, _gen_and, _gen_filter, _term_gen +from lib389.utils import ensure_bytes + class Tombstone(DSLdapObject): """A tombstone is created during a conflict or a delete in a @@ -28,16 +30,139 @@ class Tombstone(DSLdapObject): self._rdn_attribute = 'nsUniqueId' self._create_objectclasses = ['nsTombStone'] self._protected = True + self._entry_rdn = self._dn.split(',')[1] + self._grandparent_dn = self._dn.split(',', 2)[-1] # We need to always add this filter, else we won't see the ts - self._object_filter = '(objectclass=nsTombStone)' + self._object_filter = '(&(objectclass=nsTombStone)({}))'.format(self._entry_rdn) + + def raw_entry(self): + """Get an Entry object + + :returns: Entry object + """ + + return self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=["*"], + serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + + def exists(self): + """Check if the entry exists + + :returns: True if it exists + """ + + try: + self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrsonly=1, + serverctrls=self._server_controls, clientctrls=self._client_controls) + except ldap.NO_SUCH_OBJECT: + return False + + return True + + def display(self): + """Get an entry but represent it as a string LDIF + + :returns: LDIF formatted string + """ + + e = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=["*"], + serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + return e.__repr__() + + def present(self, attr, value=None): + """Assert that some attr, or some attr / value exist on the entry. + + :param attr: an attribute name + :type attr: str + :param value: an attribute value + :type value: str + + :returns: True if attr is present + """ + + if self._instance.state != DIRSRV_STATE_ONLINE: + raise ValueError("Invalid state. Cannot get presence on instance that is not ONLINE") + self._log.debug("%s present(%r) %s", self._dn, attr, value) + + self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=[attr, ], + serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + values = self.get_attr_vals_bytes(attr) + self._log.debug("%s contains %s", self._dn, values) + + if value is None: + # We are just checking if SOMETHING is present .... + return len(values) > 0 + else: + # Check if a value really does exist. + return ensure_bytes(value).lower() in [x.lower() for x in values] + + def get_all_attrs(self, use_json=False): + """Get a dictionary having all the attributes of the entry + + :returns: Dict with real attributes and operational attributes + """ + + self._log.debug("%s get_all_attrs", self._dn) + if self._instance.state != DIRSRV_STATE_ONLINE: + raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE") + else: + # retrieving real(*) and operational attributes(+) + attrs_entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=["*", "+"], + serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + # getting dict from 'entry' object + attrs_dict = attrs_entry.data + return attrs_dict + + def get_attrs_vals(self, keys, use_json=False): + self._log.debug("%s get_attrs_vals(%r)", self._dn, keys) + if self._instance.state != DIRSRV_STATE_ONLINE: + raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE") + else: + entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=keys, + serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + return entry.getValuesSet(keys) + + def get_attr_vals(self, key, use_json=False): + self._log.debug("%s get_attr_vals(%r)", self._dn, key) + # We might need to add a state check for NONE dn. + if self._instance.state != DIRSRV_STATE_ONLINE: + raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE") + # In the future, I plan to add a mode where if local == true, we + # can use get on dse.ldif to get values offline. + else: + # It would be good to prevent the entry code intercepting this .... + # We have to do this in this method, because else we ignore the scope base. + entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=[key], + serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + + vals = entry.getValues(key) + if use_json: + result = {key: []} + for val in vals: + result[key].append(val) + return result + else: + return vals + + def get_attr_val(self, key, use_json=False): + self._log.debug("%s getVal(%r)", self._dn, key) + # We might need to add a state check for NONE dn. + if self._instance.state != DIRSRV_STATE_ONLINE: + raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE") + # In the future, I plan to add a mode where if local == true, we + # can use get on dse.ldif to get values offline. + else: + entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=[key], + serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + return entry.getValue(key) def revive(self): """Revive this object within the tree. This duplicates "as much as possible", excluding some internal attributes. """ + orig_dn = self.get_attr_val_utf8('nscpEntryDN') - self._log.info("Reviving %s -> %s" % (self.dn, orig_dn)) + self._log.info("Reviving %s -> %s", self.dn, orig_dn) # Get all our attributes properties = self.get_all_attrs() properties.pop('nsuniqueid', None) @@ -57,6 +182,7 @@ class Tombstone(DSLdapObject): e.update(properties) self._instance.add_ext_s(e, serverctrls=self._server_controls, clientctrls=self._client_controls) + class Tombstones(DSLdapObjects): """Represents the set of tombstone objects that may exist on this replica. Tombstones are locally generated, so they are