389-ds-base

Clone
Source Code
GIT

#51107 Issue: 51070 - Port Import TET module to python3 part2
Closed 3 years ago by spichugi. Opened 3 years ago by aborah.
aborah/389-ds-base import2  into  master

file modified
+191 -2 
@@ -13,7 +13,7 @@ 
 

  import os
 

  import pytest
 

  import time
 

- import re
 

+ import shutil
 

  import glob
 

  import os
 

  from lib389.topologies import topology_st as topo
 
@@ -22,6 +22,7 @@ 
 

  from lib389.tasks import ImportTask
 

  from lib389.index import Indexes
 

  from lib389.monitor import Monitor
 

+ from lib389.backend import Backends
 

  from lib389.config import LDBMConfig
 

  from lib389.utils import ds_is_newer
 

  from lib389.idm.user import UserAccount, UserAccounts
 
@@ -64,7 +65,9 @@ 
 

  

          ldif_dir = topo.standalone.get_ldif_dir()
 

          import_ldif = ldif_dir + '/basic_import.ldif'
 

-         os.remove(import_ldif)
 

+         if os.path.exists(import_ldif):
 

+             os.remove(import_ldif)
 

+ 

      request.addfinalizer(finofaci)
 

  

  
@@ -116,6 +119,28 @@ 
 

      _search_for_user(topo, no_no)
 

  

  

+ def _create_bogus_ldif(topo):
 

+     """
 

+     Will create bogus ldifs
 

+     """
 

+     ldif_dir = topo.standalone.get_ldif_dir()
 

+     line1 = r'dn: cn=Eladio \"A\"\, Santabarbara\, (A\, B\, C),ou=Accounting, dc=example,dc=com'
 

+     line2 = """objectClass: top
 

+             objectClass: person
 

+             objectClass: organizationalPerson
 

+             objectClass: inetOrgPerson
 

+             cn: Eladio "A", Santabarbara, (A, B, C)
 

+             cn: Eladio Santabarbara
 

+             sn: Santabarbara
 

+             givenName: Eladio
 

+             ou: Accounting"""
 

+     with open(f'{ldif_dir}/bogus.dif', 'w') as out:
 

+         out.write(f'{line1}{line2}')
 

+     out.close()
 

+     import_ldif1 = ldif_dir + '/bogus.ldif'
 

+     return import_ldif1
 

+ 

+ 

  def test_import_with_index(topo, _import_clean):
 

      """
 

      Add an index, then import via cn=tasks
 
@@ -168,6 +193,170 @@ 
 

      _import_offline(topo, 5)
 

  

  

+ @pytest.mark.bz185477
 

+ def test_ldif2db_allows_entries_without_a_parent_to_be_imported(topo, _import_clean):
 

+     """Should reject import of entries that's missing parent suffix
 

+ 

+     :id: 27195cea-9c0e-11ea-800b-8c16451d917b
 

+     :setup: Standalone Instance
 

+     :steps:
 

+         1. Import the offending LDIF data - offline
 

+         2. Violates schema, ending line
 

+     :expected results:
 

+         1. Operation successful
 

+         2. Operation Fail
 

+     """
 

+     import_ldif1 = _create_bogus_ldif(topo)
 

+     # Import the offending LDIF data - offline
 

+     topo.standalone.stop()
 

+     topo.standalone.ldif2db('userRoot', None, None, None, import_ldif1)
 

+     # which violates schema, ending line
 

+     topo.standalone.searchErrorsLog('import_producer - import userRoot: Skipping entry '
 

+                                     '"dc=example,dc=com" which violates schema')
 

+     topo.standalone.start()
 

+ 

+ 

+ def test_issue_a_warning_if_the_cache_size_is_smaller(topo, _import_clean):
 

+     """Report during startup if nsslapd-cachememsize is too small
 

+ 

+     :id: 1aa8cbda-9c0e-11ea-9297-8c16451d917b
 

+     :setup: Standalone Instance
 

+     :steps:
 

+         1. Set nsslapd-cache-autosize to 0
 

+         2. Change cachememsize
 

+         3. Check that cachememsize is sufficiently small
 

+         4. Import some users to make id2entry.db big
 

+         5. Warning message should be there in error logs
 

+     :expected results:
 

+         1. Operation successful
 

+         2. Operation successful
 

+         3. Operation successful
 

+         4. Operation successful
 

+         5. Operation successful
 

+     """
 

+     config = LDBMConfig(topo.standalone)
 

+     backend = Backends(topo.standalone).list()[0]
 

+     # Set nsslapd-cache-autosize to 0
 

+     config.replace('nsslapd-cache-autosize', '0')
 

+     # Change cachememsize
 

+     backend.replace('nsslapd-cachememsize', '1')
 

+     # Check that cachememsize is sufficiently small
 

+     assert int(backend.get_attr_val_utf8('nsslapd-cachememsize')) < 1500000
 

+     # Import some users to make id2entry.db big
 

+     _import_offline(topo, 20)
 

+     # warning message should look like
 

+     assert topo.standalone.searchErrorsLog('INFO - ldbm_instance_config_cachememsize_set - '
 

+                                            'force a minimal value 512000')
 

+ 

+ 

+ @pytest.fixture(scope="function")
 

+ def _toggle_private_import_mem(request, topo):
 

+     config = LDBMConfig(topo.standalone)
 

+     config.replace_many(
 

+         ('nsslapd-db-private-import-mem', 'on'),
 

+         ('nsslapd-import-cache-autosize', '0'))
 

+ 

+     def finofaci():
 

+         # nsslapd-import-cache-autosize: off and
 

+         # nsslapd-db-private-import-mem: off
 

+         config.replace_many(
 

+             ('nsslapd-db-private-import-mem', 'off'))
 

+     request.addfinalizer(finofaci)
 

+ 

+ 

+ def test_fast_slow_import(topo, _toggle_private_import_mem, _import_clean):
 

+     """With nsslapd-db-private-import-mem: on is faster import.
 

+ 

+     :id: 3044331c-9c0e-11ea-ac9f-8c16451d917b
 

+     :setup: Standalone Instance
 

+     :steps:
 

+         1. Let's set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: 0
 

+         2. Measure offline import time duration total_time1
 

+         3. Now nsslapd-db-private-import-mem:off
 

+         4. Measure offline import time duration total_time2
 

+         5. total_time1 < total_time2
 

+         6. Set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: -1
 

+         7. Measure offline import time duration total_time1
 

+         8. Now nsslapd-db-private-import-mem:off
 

+         9. Measure offline import time duration total_time2
 

+         10. total_time1 < total_time2
 

+     :expected results:
 

+         1. Operation successful
 

+         2. Operation successful
 

+         3. Operation successful
 

+         4. Operation successful
 

+         5. Operation successful
 

+         6. Operation successful
 

+         7. Operation successful
 

+         8. Operation successful
 

+         9. Operation successful
 

+         10. Operation successful
 

+     """
 

+     # Let's set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: 0
 

+     config = LDBMConfig(topo.standalone)
 

+     # Measure offline import time duration total_time1
 

+     total_time1 = _import_offline(topo, 20)
 

+     # Now nsslapd-db-private-import-mem:off
 

+     config.replace('nsslapd-db-private-import-mem', 'off')
 

+     accounts = Accounts(topo.standalone, DEFAULT_SUFFIX)
 

+     for i in accounts.filter('(uid=*)'):
 

+         UserAccount(topo.standalone, i.dn).delete()
 

+     # Measure offline import time duration total_time2
 

+     total_time2 = _import_offline(topo, 20)
 

+     # total_time1 < total_time2
 

+     assert total_time1 < total_time2
 

+     # Set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: -1
 

+     config.replace_many(
 

+         ('nsslapd-db-private-import-mem', 'on'),
 

+         ('nsslapd-import-cache-autosize', '-1'))
 

+     for i in accounts.filter('(uid=*)'):
 

+         UserAccount(topo.standalone, i.dn).delete()
 

+     # Measure offline import time duration total_time1
 

+     total_time1 = _import_offline(topo, 20)
 

+     # Now nsslapd-db-private-import-mem:off
 

+     config.replace('nsslapd-db-private-import-mem', 'off')
 

+     for i in accounts.filter('(uid=*)'):
 

+         UserAccount(topo.standalone, i.dn).delete()
 

+     # Measure offline import time duration total_time2
 

+     total_time2 = _import_offline(topo, 20)
 

+     # total_time1 < total_time2
 

+     assert total_time1 < total_time2
 

+ 

+ 

+ @pytest.mark.bz175063
 

+ def test_entry_with_escaped_characters_fails_to_import_and_index(topo, _import_clean):
 

+     """If missing entry_id is found, skip it and continue reading the primary db to be re indexed.
 

+ 

+     :id: 358c938c-9c0e-11ea-adbc-8c16451d917b
 

+     :setup: Standalone Instance
 

+     :steps:
 

+         1. Import the example data from ldif.
 

+         2. Remove some of the other entries that were successfully imported.
 

+         3. Now re-index the database.
 

+         4. Should not return error.
 

+     :expected results:
 

+         1. Operation successful
 

+         2. Operation successful
 

+         3. Operation successful
 

+         4. Operation successful
 

+     """
 

+     # Import the example data from ldif
 

+     _import_offline(topo, 10)
 

+     count = 0
 

+     # Remove some of the other entries that were successfully imported
 

+     for user1 in [user for user in Accounts(topo.standalone, DEFAULT_SUFFIX).list() if user.dn.startswith('uid')]:
 

+         if count <= 2:
 

+             UserAccount(topo.standalone, user1.dn).delete()
 

+             count += 1
 

+     # Now re-index the database
 

+     topo.standalone.stop()
 

+     topo.standalone.db2index()
 

+     topo.standalone.start()
 

+     # Should not return error.
 

+     assert not topo.standalone.searchErrorsLog('error')
 

+     assert not topo.standalone.searchErrorsLog('foreman fifo error')
 

+ 

+ 

  if __name__ == '__main__':
 

      # Run isolated
 

      # -s for DEBUG mode

Bug Description: Port Import TET module to python3 part2

Fixes: https://pagure.io/389-ds-base/issue/51070

Author: aborah

Reviewed by: ???

rebased onto ba024953ea16e37c6c5f24784f882d63335da040
3 years ago

rebased onto d2fb1e1162bd79b101140be4005f7f3e2c12f1a7
3 years ago

rebased onto f0e62d6e3ed946bd831dcd2a42b34f917d3be01e
3 years ago

rebased onto daabe431a8ee7d3b47d8c3011d934ef37d5fc60e
3 years ago

Instead of silent exception we can use a check if a file exists. And if it is there, then delete it.

For a better readability let's use multiline instead of hardcoding new line symbols.

Please put this on the same line at the start of the docstring

Perhaps this should be a fixture? If the test fails in the middle, it will leave the instance with this setting on for the following tests.

rebased onto e44c1d454646743b5cc768e70af7e9dd9dc2daae
3 years ago

rebased onto b7e76dcf408a821d81d5ac5895cadf77bba173ab
3 years ago

Instead of silent exception we can use a check if a file exists. And if it is there, then delete it.

Done

For a better readability let's use multiline instead of hardcoding new line symbols.

Done

Please put this on the same line at the start of the docstring

Done

Perhaps this should be a fixture? If the test fails in the middle, it will leave the instance with this setting on for the following tests.

Done

Instead of silent exception we can use a check if a file exists. And if it is there, then delete it.

Done

I meant something like this:

if os.path.exists(import_ldif):
    os.remove(import_ldif)
Edited 3 years ago by vashirov

Typo in "bogus" here and below in the file name.

Please use triple quotes for the multiline here.

Please use more descriptive name for this fixture, for example toggle_private_import_mem.

rebased onto 52879f7342a31cca26c1eed0b81d9972a6daa377
3 years ago

rebased onto 7cad81162fe9c65a6e9221dc21187be16da4bfdc
3 years ago

rebased onto 322fb6e551959ffbfcd70bb053c2fb0d95089e35
3 years ago

Instead of silent exception we can use a check if a file exists. And if it is there, then delete it.

Done

I meant something like this:
if os.path.exists(import_ldif):
os.remove(import_ldif)

Done

Typo in "bogus" here and below in the file name.

Done

Please use triple quotes for the multiline here.

Done

Please use more descriptive name for this fixture, for example toggle_private_import_mem.

Done

rebased onto bf843907ca33b3bcca517a1d30d74733f1f3b8db
3 years ago

rebased onto 0a71c3d
3 years ago

Pull-Request has been merged by vashirov
3 years ago

389-ds-base is moving from Pagure to Github. This means that new issues and pull requests
will be accepted only in 389-ds-base's github repository.

This pull request has been cloned to Github as issue and is available here:
- https://github.com/389ds/389-ds-base/issues/4160

If you want to continue to work on the PR, please navigate to the github issue,
download the patch from the attachments and file a new pull request.

Thank you for understanding. We apologize for all inconvenience.

Pull-Request has been closed by spichugi
3 years ago
Metadata
None
No Tags
Changes Summary 1
+191 -2
file changed
dirsrvtests/tests/suites/import/import_test.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.