#49846 Issue 49761 - Fix replication test suite issues
Closed 3 years ago by spichugi. Opened 5 years ago by spichugi.
spichugi/389-ds-base repl_ts_fix  into  master

@@ -61,6 +61,7 @@ 

                  pass

          conn.close()

  

+ 

  def remove_master4_agmts(msg, topology_m4):

      """Remove all the repl agmts to master4. """

  
@@ -71,6 +72,7 @@ 

      repl.remove_master(topology_m4.ms["master4"],

          [topology_m4.ms["master1"], topology_m4.ms["master2"], topology_m4.ms["master3"]])

  

+ 

  def check_ruvs(msg, topology_m4, m4rid):

      """Check masters 1- 3 for master 4's rid."""

      for inst in (topology_m4.ms["master1"], topology_m4.ms["master2"], topology_m4.ms["master3"]):
@@ -90,6 +92,7 @@ 

              raise Exception("Master %s was not cleaned in time." % inst.serverid)

      return True

  

+ 

  def task_done(topology_m4, task_dn, timeout=60):

      """Check if the task is complete"""

  
@@ -136,8 +139,11 @@ 

  

      log.info('Master 4 has been successfully restored.')

  

+ 

  @pytest.fixture()

  def m4rid(request, topology_m4):

+     log.debug("Wait a bit before the reset - it is required fot the slow machines")

+     time.sleep(5)

      log.debug("-------------- BEGIN RESET of m4 -----------------")

      repl = ReplicationManager(DEFAULT_SUFFIX)

      repl.test_replication_topology(topology_m4.ms.values())
@@ -146,10 +152,15 @@ 

  

      def fin():

          try:

+             # Restart the masters and rerun cleanallruv

+             for inst in topology_m4.ms.values():

+                 inst.restart()

+ 

              cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])

              cruv_task.create(properties={

                  'replica-id': m4rid,

-                 'replica-base-dn': DEFAULT_SUFFIX

+                 'replica-base-dn': DEFAULT_SUFFIX,

+                 'replica-force-cleaning': 'no',

                  })

              cruv_task.wait()

          except ldap.UNWILLING_TO_PERFORM:
@@ -162,6 +173,7 @@ 

      log.debug("-------------- FINISH RESET of m4 -----------------")

      return m4rid

  

+ 

  def test_clean(topology_m4, m4rid):

      """Check that cleanallruv task works properly

  
@@ -192,7 +204,8 @@ 

      cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])

      cruv_task.create(properties={

          'replica-id': m4rid,

-         'replica-base-dn': DEFAULT_SUFFIX

+         'replica-base-dn': DEFAULT_SUFFIX,

+         'replica-force-cleaning': 'no'

          })

      cruv_task.wait()

  
@@ -247,7 +260,9 @@ 

      cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])

      cruv_task.create(properties={

          'replica-id': m4rid,

-         'replica-base-dn': DEFAULT_SUFFIX

+         'replica-base-dn': DEFAULT_SUFFIX,

+         'replica-force-cleaning': 'no',

+         'replica-certify-all': 'yes'

          })

  

      # Sleep a bit, then stop master 1
@@ -272,6 +287,7 @@ 

  

      log.info('test_clean_restart PASSED, restoring master 4...')

  

+ 

  def test_clean_force(topology_m4, m4rid):

      """Check that multiple tasks with a 'force' option work properly

  
@@ -362,7 +378,9 @@ 

      cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])

      cruv_task.create(properties={

          'replica-id': m4rid,

-         'replica-base-dn': DEFAULT_SUFFIX

+         'replica-base-dn': DEFAULT_SUFFIX,

+         'replica-force-cleaning': 'no',

+         'replica-certify-all': 'yes'

          })

      # Wait a bit

      time.sleep(2)
@@ -382,6 +400,7 @@ 

  

      log.info('test_abort PASSED, restoring master 4...')

  

+ 

  def test_abort_restart(topology_m4, m4rid):

      """Test the abort task can handle a restart, and then resume

  
@@ -424,13 +443,15 @@ 

      cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])

      cruv_task.create(properties={

          'replica-id': m4rid,

-         'replica-base-dn': DEFAULT_SUFFIX

+         'replica-base-dn': DEFAULT_SUFFIX,

+         'replica-force-cleaning': 'no',

+         'replica-certify-all': 'yes'

          })

      # Wait a bit

      time.sleep(2)

  

      # Abort the task

-     abort_task = cruv_task.abort()

+     cruv_task.abort(certify=True)

  

      # Check master 1 does not have the clean task running

      log.info('test_abort_abort: check master 1 no longer has a cleanAllRUV task...')
@@ -488,7 +509,9 @@ 

      cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])

      cruv_task.create(properties={

          'replica-id': m4rid,

-         'replica-base-dn': DEFAULT_SUFFIX

+         'replica-base-dn': DEFAULT_SUFFIX,

+         'replica-force-cleaning': 'no',

+         'replica-certify-all': 'yes'

          })

      # Wait a bit

      time.sleep(2)
@@ -498,10 +521,9 @@ 

      abort_task = cruv_task.abort(certify=True)

  

      # Wait a while and make sure the abort task is still running

-     log.info('test_abort_certify: sleep for 5 seconds')

-     time.sleep(5)

+     log.info('test_abort_certify...')

  

-     if task_done(topology_m4, abort_task.dn, 60):

+     if task_done(topology_m4, abort_task.dn, 10):

          log.fatal('test_abort_certify: abort task incorrectly finished')

          assert False

  
@@ -510,7 +532,7 @@ 

      topology_m4.ms["master2"].start()

  

      # Wait for the abort task to stop

-     if not task_done(topology_m4, abort_task.dn, 60):

+     if not task_done(topology_m4, abort_task.dn, 90):

          log.fatal('test_abort_certify: The abort CleanAllRUV task was not aborted')

          assert False

  
@@ -577,7 +599,8 @@ 

      cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])

      cruv_task.create(properties={

          'replica-id': m4rid,

-         'replica-base-dn': DEFAULT_SUFFIX

+         'replica-base-dn': DEFAULT_SUFFIX,

+         'replica-force-cleaning': 'no'

          })

      cruv_task.wait()

  
@@ -657,18 +680,21 @@ 

      cruv_task.create(properties={

          'replica-id': m4rid,

          'replica-base-dn': DEFAULT_SUFFIX,

-         'replica-force-cleaning': 'yes'

+         'replica-force-cleaning': 'yes',

+         'replica-certify-all': 'no'

          })

  

      log.info('test_multiple_tasks_with_force: run the cleanAllRUV task with "force" off...')

  

      # NOTE: This must be try not py.test raises, because the above may or may

-     # not have completed yet .... 

+     # not have completed yet ....

      try:

          cruv_task_fail = CleanAllRUVTask(topology_m4.ms["master1"])

          cruv_task_fail.create(properties={

              'replica-id': m4rid,

-             'replica-base-dn': DEFAULT_SUFFIX

+             'replica-base-dn': DEFAULT_SUFFIX,

+             'replica-force-cleaning': 'no',

+             'replica-certify-all': 'no'

              })

          cruv_task_fail.wait()

      except ldap.UNWILLING_TO_PERFORM:

@@ -178,7 +178,7 @@ 

      replica.replace(attr, valid)

  

  

- @pytest.mark.skip(reason="Agreement validation current does not work.")

+ @pytest.mark.xfail(reason="Agreement validation current does not work.")

  @pytest.mark.parametrize("attr, too_small, too_big, overflow, notnum, valid", agmt_attrs)

  def test_agmt_num_add(topo, attr, too_small, too_big, overflow, notnum, valid):

      """Test all the number values you can set for a replica config entry
@@ -198,6 +198,7 @@ 

          4. Add is rejected

          5. Add is allowed

      """

+ 

      agmt_reset(topo)

      replica = replica_setup(topo)

  
@@ -217,7 +218,7 @@ 

      agmts.create(properties=my_agmt)

  

  

- @pytest.mark.skip(reason="Agreement validation current does not work.")

+ @pytest.mark.xfail(reason="Agreement validation current does not work.")

  @pytest.mark.parametrize("attr, too_small, too_big, overflow, notnum, valid", agmt_attrs)

  def test_agmt_num_modify(topo, attr, too_small, too_big, overflow, notnum, valid):

      """Test all the number values you can set for a replica config entry

@@ -113,7 +113,8 @@ 

          parser = MyLDIF(ldif_file)

          parser.parse()

  

- @pytest.mark.skip(reason="No method to safety access DB ruv currenty exists online.")

+ 

+ @pytest.mark.xfail(reason="No method to safety access DB ruv currently exists online.")

  def test_memoryruv_sync_with_databaseruv(topo):

      """Check if memory ruv and database ruv are synced

  

file modified
+3 -1
@@ -310,6 +310,8 @@ 

  attributeTypes: ( 2.16.840.1.113730.3.1.2340 NAME 'nsslapd-changelogmaxage' DESC 'The changelog5 time where an entry will be retained' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' )

  attributeTypes: ( 2.16.840.1.113730.3.1.2341 NAME 'nsslapd-changelogmaxentries' DESC 'The changelog5 max entries limit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' )

  attributeTypes: ( 2.16.840.1.113730.3.1.2344 NAME 'nsslapd-tls-check-crl' DESC 'Check CRL when opening outbound TLS connections. Valid options are none, peer, all.' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' )

+ attributeTypes: ( 2.16.840.1.113730.3.1.2353 NAME 'nsslapd-encryptionalgorithm' DESC 'The encryption algorithm used to encrypt the changelog' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN '389 Directory Server' )

+ attributeTypes: ( 2.16.840.1.113730.3.1.2084 NAME 'nsSymmetricKey' DESC 'A symmetric key - currently used by attribute encryption' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 SINGLE-VALUE X-ORIGIN 'attribute encryption' )

  #

  # objectclasses

  #
@@ -329,4 +331,4 @@ 

  objectClasses: ( nsEncryptionModule-oid NAME 'nsEncryptionModule' DESC 'Netscape defined objectclass' SUP top MUST ( cn ) MAY ( nsSSLToken $ nsSSLPersonalityssl $ nsSSLActivation $ ServerKeyExtractFile $ ServerCertExtractFile ) X-ORIGIN 'Netscape' )

  objectClasses: ( 2.16.840.1.113730.3.2.327 NAME 'rootDNPluginConfig' DESC 'Netscape defined objectclass' SUP top MUST ( cn ) MAY ( rootdn-open-time $ rootdn-close-time $ rootdn-days-allowed $ rootdn-allow-host $ rootdn-deny-host $ rootdn-allow-ip $ rootdn-deny-ip ) X-ORIGIN 'Netscape' )

  objectClasses: ( 2.16.840.1.113730.3.2.328 NAME 'nsSchemaPolicy' DESC 'Netscape defined objectclass' SUP top  MAY ( cn $ schemaUpdateObjectclassAccept $ schemaUpdateObjectclassReject $ schemaUpdateAttributeAccept $ schemaUpdateAttributeReject) X-ORIGIN 'Netscape Directory Server' )

- objectClasses: ( 2.16.840.1.113730.3.2.332 NAME 'nsChangelogConfig' DESC 'Configuration of the changelog5 object' SUP top MUST ( cn $ nsslapd-changelogdir ) MAY ( nsslapd-changelogmaxage $ nsslapd-changelogtrim-interval $ nsslapd-changelogmaxentries $ nsslapd-changelogsuffix $ nsslapd-changelogcompactdb-interval ) X-ORIGIN '389 Directory Server' )

+ objectClasses: ( 2.16.840.1.113730.3.2.332 NAME 'nsChangelogConfig' DESC 'Configuration of the changelog5 object' SUP top MUST ( cn $ nsslapd-changelogdir ) MAY ( nsslapd-changelogmaxage $ nsslapd-changelogtrim-interval $ nsslapd-changelogmaxentries $ nsslapd-changelogsuffix $ nsslapd-changelogcompactdb-interval $ nsslapd-encryptionalgorithm $ nsSymmetricKey ) X-ORIGIN '389 Directory Server' )

@@ -82,7 +82,6 @@ 

  attributeTypes: ( 2.16.840.1.113730.3.1.57 NAME 'replicaRoot' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-ORIGIN 'Netscape Directory Server' )

  attributeTypes: ( 2.16.840.1.113730.3.1.58 NAME 'replicaBindDn' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-ORIGIN 'Netscape Directory Server' )

  attributeTypes: ( 2.16.840.1.113730.3.1.69 NAME 'subtreeACI' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 X-ORIGIN 'Netscape Directory Server 1.0' )

- attributeTypes: ( 2.16.840.1.113730.3.1.2084 NAME 'nsSymmetricKey' DESC 'A symmetric key - currently used by attribute encryption' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 SINGLE-VALUE X-ORIGIN 'attribute encryption' )

  objectClasses: ( 2.16.840.1.113730.3.2.23 NAME 'netscapeDirectoryServer' DESC 'Netscape defined objectclass' SUP top MUST ( objectclass ) X-ORIGIN 'Netscape Directory Server' )

  objectClasses: ( nsDirectoryServer-oid NAME 'nsDirectoryServer' DESC 'Netscape defined objectclass' SUP top MUST ( objectclass $ nsServerID ) MAY ( serverHostName $ nsServerPort $ nsSecureServerPort $ nsBindPassword $ nsBindDN $ nsBaseDN ) X-ORIGIN 'Netscape Directory Server' )

  objectClasses: ( 2.16.840.1.113730.3.2.8 NAME 'ntUser' DESC 'Netscape defined objectclass' SUP top MUST ( ntUserDomainId ) MAY ( description $ l $ ou $ seeAlso $ ntUserPriv $ ntUserHomeDir $ ntUserComment $ ntUserFlags $ ntUserScriptPath $ ntUserAuthFlags $ ntUserUsrComment $ ntUserParms $ ntUserWorkstations $ ntUserLastLogon $ ntUserLastLogoff $ ntUserAcctExpires $ ntUserMaxStorage $ ntUserUnitsPerWeek $ ntUserLogonHours $ ntUserBadPwCount $ ntUserNumLogons $ ntUserLogonServer $ ntUserCountryCode $ ntUserCodePage $ ntUserUniqueId $ ntUserPrimaryGroupId $ ntUserProfile $ ntUserHomeDirDrive $ ntUserPasswordExpired $ ntUserCreateNewAccount $ ntUserDeleteAccount $ ntUniqueId $ ntUserNtPassword ) X-ORIGIN 'Netscape NT Synchronization' )

file modified
+129 -3
@@ -10,8 +10,10 @@ 

  import ldap

  

  from lib389._entry import Entry

- 

+ from lib389._constants import DIRSRV_STATE_ONLINE

  from lib389._mapped_object import DSLdapObject, DSLdapObjects, _gen_and, _gen_filter, _term_gen

+ from lib389.utils import ensure_bytes

+ 

  

  class Tombstone(DSLdapObject):

      """A tombstone is created during a conflict or a delete in a
@@ -28,16 +30,139 @@ 

          self._rdn_attribute = 'nsUniqueId'

          self._create_objectclasses = ['nsTombStone']

          self._protected = True

+         self._entry_rdn = self._dn.split(',')[1]

+         self._grandparent_dn = self._dn.split(',', 2)[-1]

          # We need to always add this filter, else we won't see the ts

-         self._object_filter = '(objectclass=nsTombStone)'

+         self._object_filter = '(&(objectclass=nsTombStone)({}))'.format(self._entry_rdn)

+ 

+     def raw_entry(self):

+         """Get an Entry object

+ 

+         :returns: Entry object

+         """

+ 

+         return self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=["*"],

+                                            serverctrls=self._server_controls, clientctrls=self._client_controls)[0]

+ 

+     def exists(self):

+         """Check if the entry exists

+ 

+         :returns: True if it exists

+         """

+ 

+         try:

+             self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrsonly=1,

+                                         serverctrls=self._server_controls, clientctrls=self._client_controls)

+         except ldap.NO_SUCH_OBJECT:

+             return False

+ 

+         return True

+ 

+     def display(self):

+         """Get an entry but represent it as a string LDIF

+ 

+         :returns: LDIF formatted string

+         """

+ 

+         e = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=["*"],

+                                         serverctrls=self._server_controls, clientctrls=self._client_controls)[0]

+         return e.__repr__()

+ 

+     def present(self, attr, value=None):

+         """Assert that some attr, or some attr / value exist on the entry.

+ 

+         :param attr: an attribute name

+         :type attr: str

+         :param value: an attribute value

+         :type value: str

+ 

+         :returns: True if attr is present

+         """

+ 

+         if self._instance.state != DIRSRV_STATE_ONLINE:

+             raise ValueError("Invalid state. Cannot get presence on instance that is not ONLINE")

+         self._log.debug("%s present(%r) %s", self._dn, attr, value)

+ 

+         self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=[attr, ],

+                                     serverctrls=self._server_controls, clientctrls=self._client_controls)[0]

+         values = self.get_attr_vals_bytes(attr)

+         self._log.debug("%s contains %s", self._dn, values)

+ 

+         if value is None:

+             # We are just checking if SOMETHING is present ....

+             return len(values) > 0

+         else:

+             # Check if a value really does exist.

+             return ensure_bytes(value).lower() in [x.lower() for x in values]

+ 

+     def get_all_attrs(self, use_json=False):

+         """Get a dictionary having all the attributes of the entry

+ 

+         :returns: Dict with real attributes and operational attributes

+         """

+ 

+         self._log.debug("%s get_all_attrs", self._dn)

+         if self._instance.state != DIRSRV_STATE_ONLINE:

+             raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE")

+         else:

+             # retrieving real(*) and operational attributes(+)

+             attrs_entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=["*", "+"],

+                                                       serverctrls=self._server_controls, clientctrls=self._client_controls)[0]

+             # getting dict from 'entry' object

+             attrs_dict = attrs_entry.data

+             return attrs_dict

+ 

+     def get_attrs_vals(self, keys, use_json=False):

+         self._log.debug("%s get_attrs_vals(%r)", self._dn, keys)

+         if self._instance.state != DIRSRV_STATE_ONLINE:

+             raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE")

+         else:

+             entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=keys,

+                                                 serverctrls=self._server_controls, clientctrls=self._client_controls)[0]

+             return entry.getValuesSet(keys)

+ 

+     def get_attr_vals(self, key, use_json=False):

+         self._log.debug("%s get_attr_vals(%r)", self._dn, key)

+         # We might need to add a state check for NONE dn.

+         if self._instance.state != DIRSRV_STATE_ONLINE:

+             raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE")

+             # In the future, I plan to add a mode where if local == true, we

+             # can use get on dse.ldif to get values offline.

+         else:

+             # It would be good to prevent the entry code intercepting this ....

+             # We have to do this in this method, because else we ignore the scope base.

+             entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=[key],

+                                                 serverctrls=self._server_controls, clientctrls=self._client_controls)[0]

+ 

+             vals = entry.getValues(key)

+             if use_json:

+                 result = {key: []}

+                 for val in vals:

+                     result[key].append(val)

+                 return result

+             else:

+                 return vals

+ 

+     def get_attr_val(self, key, use_json=False):

+         self._log.debug("%s getVal(%r)", self._dn, key)

+         # We might need to add a state check for NONE dn.

+         if self._instance.state != DIRSRV_STATE_ONLINE:

+             raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE")

+             # In the future, I plan to add a mode where if local == true, we

+             # can use get on dse.ldif to get values offline.

+         else:

+             entry = self._instance.search_ext_s(self._grandparent_dn, ldap.SCOPE_SUBTREE, self._object_filter, attrlist=[key],

+                                                 serverctrls=self._server_controls, clientctrls=self._client_controls)[0]

+             return entry.getValue(key)

  

      def revive(self):

          """Revive this object within the tree.

  

          This duplicates "as much as possible", excluding some internal attributes.

          """

+ 

          orig_dn = self.get_attr_val_utf8('nscpEntryDN')

-         self._log.info("Reviving %s -> %s" % (self.dn, orig_dn))

+         self._log.info("Reviving %s -> %s", self.dn, orig_dn)

          # Get all our attributes

          properties = self.get_all_attrs()

          properties.pop('nsuniqueid', None)
@@ -57,6 +182,7 @@ 

          e.update(properties)

          self._instance.add_ext_s(e, serverctrls=self._server_controls, clientctrls=self._client_controls)

  

+ 

  class Tombstones(DSLdapObjects):

      """Represents the set of tombstone objects that may exist on

      this replica. Tombstones are locally generated, so they are

Description - the issues:
cleanallruv - make abort tasks 'certify and adjust timeout;
encryption_cl5 - add the encryption attributes to the schema
(we had used extensibleObject before but now we use nsChangelogConfig);
tombstone - fix how Tombstone(DSLdapObject) handles the searches.

https://pagure.io/389-ds-base/issue/49761

Reviewed by: ?

This is an incorrect OID for nsslapd-encryptionalgorithm.

rebased onto 1eadb2fd192bb9396974e73d149828350d1a56b4

5 years ago

rebased onto 9a557bdadc434fc60205d25cfb178213b721dad3

5 years ago

The issue is fixed, please review.

rebased onto 584ae7c5f02255b7b0148d03def789a315dc0dfd

5 years ago

Changes with Mark's patch are made and tested on a fast machine. Please, review.

It still fails on slow machines but we have a separate issue for this, we can proceed there:
https://pagure.io/389-ds-base/issue/49863

rebased onto 7b3c401

5 years ago

Pull-Request has been merged by spichugi

5 years ago

389-ds-base is moving from Pagure to Github. This means that new issues and pull requests
will be accepted only in 389-ds-base's github repository.

This pull request has been cloned to Github as issue and is available here:
- https://github.com/389ds/389-ds-base/issues/2905

If you want to continue to work on the PR, please navigate to the github issue,
download the patch from the attachments and file a new pull request.

Thank you for understanding. We apologize for all inconvenience.

Pull-Request has been closed by spichugi

3 years ago