#51103 Issue 51102 - RFE - ds-replcheck - make online timeout configurable
Closed 3 years ago by spichugi. Opened 3 years ago by mreynolds.
mreynolds/389-ds-base issue51102  into  master

@@ -1,7 +1,7 @@ 

  #!/usr/bin/python3

  

  # --- BEGIN COPYRIGHT BLOCK ---

- # Copyright (C) 2018 Red Hat, Inc.

+ # Copyright (C) 2020 Red Hat, Inc.

  # All rights reserved.

  #

  # License: GPL (version 3 or any later version).
@@ -21,10 +21,9 @@ 

  import signal

  from ldif import LDIFRecordList

  from ldap.ldapobject import SimpleLDAPObject

- from ldap.cidict import cidict

  from ldap.controls import SimplePagedResultsControl

  from lib389._entry import Entry

- from lib389.utils import ensure_str, ensure_list_str, ensure_int

+ from lib389.utils import ensure_list_str, ensure_int

  

  VERSION = "2.0"

  RUV_FILTER = '(&(nsuniqueid=ffffffff-ffffffff-ffffffff-ffffffff)(objectclass=nstombstone))'
@@ -185,11 +184,11 @@ 

      report = True

  

      if 'nscpentrywsi' in entry.data:

-         found = False

          for val in entry.data['nscpentrywsi']:

              if val.lower().startswith(attr + ';'):

                  if (opts['starttime'] - extract_time(val)) <= opts['lag']:

                      report = False

+                     break

  

      return report

  
@@ -321,6 +320,9 @@ 

      count = 0

      ignore_list = ['conflictcsn', 'modifytimestamp', 'modifiersname']

      val = ""

+     attr = ""

+     state_attr = ""

+     part_dn = ""

      result['entry'] = None

      result['conflict'] = None

      result['tombstone'] = False
@@ -570,6 +572,7 @@ 

                          if val.lower().startswith(mattr + ';'):

                              if not found:

                                  diff['diff'].append("      Master:")

+                             diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))

                              diff['diff'].append("        - State Info: %s" % (val))

                              diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))

                              found = True
@@ -588,6 +591,7 @@ 

                          if val.lower().startswith(mattr + ';'):

                              if not found:

                                  diff['diff'].append("      Replica:")

+                             diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))

                              diff['diff'].append("        - State Info: %s" % (val))

                              diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))

                              found = True
@@ -654,7 +658,6 @@ 

      rconflicts = []

      rtombstones = 0

      mtombstones = 0

-     idx = 0

  

      # Open LDIF files

      try:
@@ -926,7 +929,7 @@ 

      :return - True if suffix exists, otherwise False

      """

      try:

-         master_basesuffix = ldapnode.search_s(suffix, ldap.SCOPE_BASE )

+         ldapnode.search_s(suffix, ldap.SCOPE_BASE)

      except ldap.NO_SUCH_OBJECT:

          print("Error: Failed to validate suffix in {}. {} does not exist.".format(hostname, suffix))

          return False
@@ -968,12 +971,12 @@ 

      replica = SimpleLDAPObject(ruri)

  

      # Set timeouts

-     master.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)

-     master.set_option(ldap.OPT_TIMEOUT,5.0)

-     replica.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)

-     replica.set_option(ldap.OPT_TIMEOUT,5.0)

+     master.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])

+     master.set_option(ldap.OPT_TIMEOUT, opts['timeout'])

+     replica.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])

+     replica.set_option(ldap.OPT_TIMEOUT, opts['timeout'])

  

-     # Setup Secure Conenction

+     # Setup Secure Connection

      if opts['certdir'] is not None:

          # Setup Master

          if opts['mprotocol'] != LDAPI:
@@ -1003,7 +1006,7 @@ 

      try:

          master.simple_bind_s(opts['binddn'], opts['bindpw'])

      except ldap.SERVER_DOWN as e:

-         print("Cannot connect to %r" % muri)

+         print(f"Cannot connect to {muri} ({str(e)})")

          sys.exit(1)

      except ldap.LDAPError as e:

          print("Error: Failed to authenticate to Master: ({}).  "
@@ -1014,7 +1017,7 @@ 

      try:

          replica.simple_bind_s(opts['binddn'], opts['bindpw'])

      except ldap.SERVER_DOWN as e:

-         print("Cannot connect to %r" % ruri)

+         print(f"Cannot connect to {ruri} ({str(e)})")

          sys.exit(1)

      except ldap.LDAPError as e:

          print("Error: Failed to authenticate to Replica: ({}).  "
@@ -1218,7 +1221,6 @@ 

      """

      m_done = False

      r_done = False

-     done = False

      report = {}

      report['diff'] = []

      report['m_missing'] = []
@@ -1257,15 +1259,22 @@ 

  

      # Read the results and start comparing

      while not m_done or not r_done:

-         if not m_done:

-             m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)

-         elif not r_done:

-             m_rdata = []

- 

-         if not r_done:

-             r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)

-         elif not m_done:

-             r_rdata = []

+         try:

+             if not m_done:

+                 m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)

+             elif not r_done:

+                 m_rdata = []

+         except ldap.LDAPError as e:

+             print("Error: Problem getting the results from the master: %s", str(e))

+             sys.exit(1)

+         try:

+             if not r_done:

+                 r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)

+             elif not m_done:

+                 r_rdata = []

+         except ldap.LDAPError as e:

+             print("Error: Problem getting the results from the replica: %s", str(e))

+             sys.exit(1)

  

          # Convert entries

          mresult = convert_entries(m_rdata)
@@ -1291,11 +1300,15 @@ 

                  ]

              if m_pctrls:

                  if m_pctrls[0].cookie:

-                     # Copy cookie from response control to request control

-                     req_pr_ctrl.cookie = m_pctrls[0].cookie

-                     master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,

-                         "(|(objectclass=*)(objectclass=ldapsubentry))",

-                         ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)

+                     try:

+                         # Copy cookie from response control to request control

+                         req_pr_ctrl.cookie = m_pctrls[0].cookie

+                         master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,

+                             "(|(objectclass=*)(objectclass=ldapsubentry))",

+                             ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)

+                     except ldap.LDAPError as e:

+                         print("Error: Problem searching the master: %s", str(e))

+                         sys.exit(1)

                  else:

                      m_done = True  # No more pages available

              else:
@@ -1311,11 +1324,15 @@ 

  

              if r_pctrls:

                  if r_pctrls[0].cookie:

-                     # Copy cookie from response control to request control

-                     req_pr_ctrl.cookie = r_pctrls[0].cookie

-                     replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,

-                         "(|(objectclass=*)(objectclass=ldapsubentry))",

-                         ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)

+                     try:

+                         # Copy cookie from response control to request control

+                         req_pr_ctrl.cookie = r_pctrls[0].cookie

+                         replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,

+                             "(|(objectclass=*)(objectclass=ldapsubentry))",

+                             ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)

+                     except ldap.LDAPError as e:

+                         print("Error: Problem searching the replica: %s", str(e))

+                         sys.exit(1)

                  else:

                      r_done = True  # No more pages available

              else:
@@ -1426,6 +1443,9 @@ 

          # prompt for password

          opts['bindpw'] = getpass.getpass('Enter password: ')

  

+     # lastly handle the timeout

+     opts['timeout'] = int(args.timeout)

+ 

      return opts

  

  
@@ -1553,6 +1573,8 @@ 

      state_parser.add_argument('-y', '--pass-file', help='A text file containing the clear text password for the bind dn', dest='pass_file', default=None)

      state_parser.add_argument('-Z', '--cert-dir', help='The certificate database directory for secure connections',

                                dest='certdir', default=None)

+     state_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',

+                               type=int, dest='timeout', default=-1)

  

      # Online mode

      online_parser = subparsers.add_parser('online', help="Compare two online replicas for differences")
@@ -1577,6 +1599,8 @@ 

      online_parser.add_argument('-p', '--page-size', help='The paged-search result grouping size (default 500 entries)',

                                 dest='pagesize', default=500)

      online_parser.add_argument('-o', '--out-file', help='The output file', dest='file', default=None)

+     online_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',

+                                type=int, dest='timeout', default=-1)

  

      # Offline LDIF mode

      offline_parser = subparsers.add_parser('offline', help="Compare two replication LDIF files for differences (LDIF file generated by 'db2ldif -r')")

Bug Description:

When doing an online check with replicas that are very far apart the connection can time out as the hardcoded timeout is 5 seconds.

Fix Description:

Change the default timeout to never timeout, and add an CLI option to specify a specific timeout.

Also caught all the possible LDAP exceptions so we can cleanly "fail". Fixed some python syntax issues, and improved the entry inconsistency report

relates: https://pagure.io/389-ds-base/issue/51102

Looks reasonable to me. Ack,

We, probably, need to add the argument to state_parser also because get_state also calls init_online_params(args)

rebased onto c350ddc

3 years ago

We, probably, need to add the argument to state_parser also because get_state also calls init_online_params(args)

Nice catch! Done, merging...

Pull-Request has been merged by mreynolds

3 years ago

389-ds-base is moving from Pagure to Github. This means that new issues and pull requests
will be accepted only in 389-ds-base's github repository.

This pull request has been cloned to Github as issue and is available here:
- https://github.com/389ds/389-ds-base/issues/4156

If you want to continue to work on the PR, please navigate to the github issue,
download the patch from the attachments and file a new pull request.

Thank you for understanding. We apologize for all inconvenience.

Pull-Request has been closed by spichugi

3 years ago
Metadata