From 0a71c3db0f5a6a09f0dd3d736cd173aefe582613 Mon Sep 17 00:00:00 2001 From: Anuj Borah Date: Jun 24 2020 07:53:31 +0000 Subject: Issue: 51070 - Port Import TET module to python3 part2 Bug Description: Port Import TET module to python3 part2 Fixes: https://pagure.io/389-ds-base/issue/51070 Author: aborah Reviewed by: Viktor Ashirov --- diff --git a/dirsrvtests/tests/suites/import/import_test.py b/dirsrvtests/tests/suites/import/import_test.py index 1e4872d..a9dcc7e 100644 --- a/dirsrvtests/tests/suites/import/import_test.py +++ b/dirsrvtests/tests/suites/import/import_test.py @@ -13,7 +13,7 @@ Will test Import (Offline/Online) import os import pytest import time -import re +import shutil import glob import os from lib389.topologies import topology_st as topo @@ -22,6 +22,7 @@ from lib389.dbgen import dbgen_users from lib389.tasks import ImportTask from lib389.index import Indexes from lib389.monitor import Monitor +from lib389.backend import Backends from lib389.config import LDBMConfig from lib389.utils import ds_is_newer from lib389.idm.user import UserAccount, UserAccounts @@ -64,7 +65,9 @@ def _import_clean(request, topo): ldif_dir = topo.standalone.get_ldif_dir() import_ldif = ldif_dir + '/basic_import.ldif' - os.remove(import_ldif) + if os.path.exists(import_ldif): + os.remove(import_ldif) + request.addfinalizer(finofaci) @@ -116,6 +119,28 @@ def _import_online(topo, no_no): _search_for_user(topo, no_no) +def _create_bogus_ldif(topo): + """ + Will create bogus ldifs + """ + ldif_dir = topo.standalone.get_ldif_dir() + line1 = r'dn: cn=Eladio \"A\"\, Santabarbara\, (A\, B\, C),ou=Accounting, dc=example,dc=com' + line2 = """objectClass: top + objectClass: person + objectClass: organizationalPerson + objectClass: inetOrgPerson + cn: Eladio "A", Santabarbara, (A, B, C) + cn: Eladio Santabarbara + sn: Santabarbara + givenName: Eladio + ou: Accounting""" + with open(f'{ldif_dir}/bogus.dif', 'w') as out: + out.write(f'{line1}{line2}') + out.close() + import_ldif1 = ldif_dir + '/bogus.ldif' + return import_ldif1 + + def test_import_with_index(topo, _import_clean): """ Add an index, then import via cn=tasks @@ -168,6 +193,170 @@ def test_crash_on_ldif2db(topo, _import_clean): _import_offline(topo, 5) +@pytest.mark.bz185477 +def test_ldif2db_allows_entries_without_a_parent_to_be_imported(topo, _import_clean): + """Should reject import of entries that's missing parent suffix + + :id: 27195cea-9c0e-11ea-800b-8c16451d917b + :setup: Standalone Instance + :steps: + 1. Import the offending LDIF data - offline + 2. Violates schema, ending line + :expected results: + 1. Operation successful + 2. Operation Fail + """ + import_ldif1 = _create_bogus_ldif(topo) + # Import the offending LDIF data - offline + topo.standalone.stop() + topo.standalone.ldif2db('userRoot', None, None, None, import_ldif1) + # which violates schema, ending line + topo.standalone.searchErrorsLog('import_producer - import userRoot: Skipping entry ' + '"dc=example,dc=com" which violates schema') + topo.standalone.start() + + +def test_issue_a_warning_if_the_cache_size_is_smaller(topo, _import_clean): + """Report during startup if nsslapd-cachememsize is too small + + :id: 1aa8cbda-9c0e-11ea-9297-8c16451d917b + :setup: Standalone Instance + :steps: + 1. Set nsslapd-cache-autosize to 0 + 2. Change cachememsize + 3. Check that cachememsize is sufficiently small + 4. Import some users to make id2entry.db big + 5. Warning message should be there in error logs + :expected results: + 1. Operation successful + 2. Operation successful + 3. Operation successful + 4. Operation successful + 5. Operation successful + """ + config = LDBMConfig(topo.standalone) + backend = Backends(topo.standalone).list()[0] + # Set nsslapd-cache-autosize to 0 + config.replace('nsslapd-cache-autosize', '0') + # Change cachememsize + backend.replace('nsslapd-cachememsize', '1') + # Check that cachememsize is sufficiently small + assert int(backend.get_attr_val_utf8('nsslapd-cachememsize')) < 1500000 + # Import some users to make id2entry.db big + _import_offline(topo, 20) + # warning message should look like + assert topo.standalone.searchErrorsLog('INFO - ldbm_instance_config_cachememsize_set - ' + 'force a minimal value 512000') + + +@pytest.fixture(scope="function") +def _toggle_private_import_mem(request, topo): + config = LDBMConfig(topo.standalone) + config.replace_many( + ('nsslapd-db-private-import-mem', 'on'), + ('nsslapd-import-cache-autosize', '0')) + + def finofaci(): + # nsslapd-import-cache-autosize: off and + # nsslapd-db-private-import-mem: off + config.replace_many( + ('nsslapd-db-private-import-mem', 'off')) + request.addfinalizer(finofaci) + + +def test_fast_slow_import(topo, _toggle_private_import_mem, _import_clean): + """With nsslapd-db-private-import-mem: on is faster import. + + :id: 3044331c-9c0e-11ea-ac9f-8c16451d917b + :setup: Standalone Instance + :steps: + 1. Let's set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: 0 + 2. Measure offline import time duration total_time1 + 3. Now nsslapd-db-private-import-mem:off + 4. Measure offline import time duration total_time2 + 5. total_time1 < total_time2 + 6. Set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: -1 + 7. Measure offline import time duration total_time1 + 8. Now nsslapd-db-private-import-mem:off + 9. Measure offline import time duration total_time2 + 10. total_time1 < total_time2 + :expected results: + 1. Operation successful + 2. Operation successful + 3. Operation successful + 4. Operation successful + 5. Operation successful + 6. Operation successful + 7. Operation successful + 8. Operation successful + 9. Operation successful + 10. Operation successful + """ + # Let's set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: 0 + config = LDBMConfig(topo.standalone) + # Measure offline import time duration total_time1 + total_time1 = _import_offline(topo, 20) + # Now nsslapd-db-private-import-mem:off + config.replace('nsslapd-db-private-import-mem', 'off') + accounts = Accounts(topo.standalone, DEFAULT_SUFFIX) + for i in accounts.filter('(uid=*)'): + UserAccount(topo.standalone, i.dn).delete() + # Measure offline import time duration total_time2 + total_time2 = _import_offline(topo, 20) + # total_time1 < total_time2 + assert total_time1 < total_time2 + # Set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: -1 + config.replace_many( + ('nsslapd-db-private-import-mem', 'on'), + ('nsslapd-import-cache-autosize', '-1')) + for i in accounts.filter('(uid=*)'): + UserAccount(topo.standalone, i.dn).delete() + # Measure offline import time duration total_time1 + total_time1 = _import_offline(topo, 20) + # Now nsslapd-db-private-import-mem:off + config.replace('nsslapd-db-private-import-mem', 'off') + for i in accounts.filter('(uid=*)'): + UserAccount(topo.standalone, i.dn).delete() + # Measure offline import time duration total_time2 + total_time2 = _import_offline(topo, 20) + # total_time1 < total_time2 + assert total_time1 < total_time2 + + +@pytest.mark.bz175063 +def test_entry_with_escaped_characters_fails_to_import_and_index(topo, _import_clean): + """If missing entry_id is found, skip it and continue reading the primary db to be re indexed. + + :id: 358c938c-9c0e-11ea-adbc-8c16451d917b + :setup: Standalone Instance + :steps: + 1. Import the example data from ldif. + 2. Remove some of the other entries that were successfully imported. + 3. Now re-index the database. + 4. Should not return error. + :expected results: + 1. Operation successful + 2. Operation successful + 3. Operation successful + 4. Operation successful + """ + # Import the example data from ldif + _import_offline(topo, 10) + count = 0 + # Remove some of the other entries that were successfully imported + for user1 in [user for user in Accounts(topo.standalone, DEFAULT_SUFFIX).list() if user.dn.startswith('uid')]: + if count <= 2: + UserAccount(topo.standalone, user1.dn).delete() + count += 1 + # Now re-index the database + topo.standalone.stop() + topo.standalone.db2index() + topo.standalone.start() + # Should not return error. + assert not topo.standalone.searchErrorsLog('error') + assert not topo.standalone.searchErrorsLog('foreman fifo error') + + if __name__ == '__main__': # Run isolated # -s for DEBUG mode