From 0f1ab5f0531eca6af07aeda59bf4c8f97f59357f Mon Sep 17 00:00:00 2001 From: Ludwig Krispenz Date: Aug 17 2020 18:37:21 +0000 Subject: Ticket - 49562 integrate changelog database to main database Bug description: PHASE 2 of backend redesign: http://www.port389.org/docs/389ds/design/integrate-changelog-database-and-backend-database.html Mainly changelog managed its own access to the database and it uses a global config entry (cn=changelog5,cn=config) not related to the backend/replica. Fix description: The fix is described in the design. Plus: - use-after-free (remove+add replica, set) - various leaks (triggered with CI tests fixup-tombstone, cascading) - Plus some changes in the CI tests https://pagure.io/389-ds-base/issue/49562 Reviewed by: Mark Reynolds, William Brown, Thierry Bordaz --- diff --git a/dirsrvtests/tests/suites/password/regression_test.py b/dirsrvtests/tests/suites/password/regression_test.py index 81735c0..4dbdab6 100644 --- a/dirsrvtests/tests/suites/password/regression_test.py +++ b/dirsrvtests/tests/suites/password/regression_test.py @@ -8,11 +8,11 @@ import pytest import time from lib389._constants import PASSWORD, DN_DM, DEFAULT_SUFFIX -from lib389._constants import SUFFIX, PASSWORD, DN_DM, DN_CONFIG, PLUGIN_RETRO_CHANGELOG, DEFAULT_SUFFIX, DEFAULT_CHANGELOG_DB +from lib389._constants import SUFFIX, PASSWORD, DN_DM, DN_CONFIG, PLUGIN_RETRO_CHANGELOG, DEFAULT_SUFFIX, DEFAULT_CHANGELOG_DB, DEFAULT_BENAME from lib389 import Entry from lib389.topologies import topology_m1 as topo_master from lib389.idm.user import UserAccounts -from lib389.utils import ldap, os, logging, ensure_bytes, ds_is_newer +from lib389.utils import ldap, os, logging, ensure_bytes, ds_is_newer, ds_supports_new_changelog from lib389.topologies import topology_st as topo from lib389.idm.organizationalunit import OrganizationalUnits @@ -43,13 +43,17 @@ def _check_unhashed_userpw(inst, user_dn, is_present=False): """Check if unhashed#user#password attribute is present or not in the changelog""" unhashed_pwd_attribute = 'unhashed#user#password' - changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB) - for dbfile in os.listdir(changelog_dbdir): - if dbfile.endswith('.db'): - changelog_dbfile = os.path.join(changelog_dbdir, dbfile) - log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile)) - log.info('Running dbscan -f to check {} attr'.format(unhashed_pwd_attribute)) - dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile) + if ds_supports_new_changelog(): + dbscanOut = inst.dbscan(DEFAULT_BENAME, 'changelog') + else: + changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB) + for dbfile in os.listdir(changelog_dbdir): + if dbfile.endswith('.db'): + changelog_dbfile = os.path.join(changelog_dbdir, dbfile) + log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile)) + log.info('Running dbscan -f to check {} attr'.format(unhashed_pwd_attribute)) + dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile) + for entry in dbscanOut.split(b'dbid: '): if ensure_bytes('operation: modify') in entry and ensure_bytes(user_dn) in entry and ensure_bytes('userPassword') in entry: if is_present: diff --git a/dirsrvtests/tests/suites/replication/changelog_test.py b/dirsrvtests/tests/suites/replication/changelog_test.py index 9c6f15d..211d8ce 100644 --- a/dirsrvtests/tests/suites/replication/changelog_test.py +++ b/dirsrvtests/tests/suites/replication/changelog_test.py @@ -22,12 +22,16 @@ from lib389.plugins import RetroChangelogPlugin from lib389.dseldif import DSEldif from lib389.tasks import * from lib389.utils import * +from lib389.utils import ldap, os, logging, ensure_bytes, ds_is_newer, ds_supports_new_changelog pytestmark = pytest.mark.tier1 TEST_ENTRY_NAME = 'replusr' NEW_RDN_NAME = 'cl5usr' -CHANGELOG = 'cn=changelog5,cn=config' +if ds_supports_new_changelog(): + CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM) +else: + CHANGELOG = 'cn=changelog5,cn=config' RETROCHANGELOG = 'cn=Retro Changelog Plugin,cn=plugins,cn=config' MAXAGE = 'nsslapd-changelogmaxage' TRIMINTERVAL = 'nsslapd-changelogtrim-interval' @@ -73,12 +77,17 @@ def _create_changelog_dump(topo): """Dump changelog using nss5task and check if ldap operations are logged""" log.info('Dump changelog using nss5task and check if ldap operations are logged') - changelog_dir = topo.ms['master1'].get_changelog_dir() + if ds_supports_new_changelog(): + changelog_dir = topo.ms['master1'].get_ldif_dir() + changelog_end = '_cl.ldif' + else: + changelog_dir = topo.ms['master1'].get_changelog_dir() + changelog_end = '.ldif' replicas = Replicas(topo.ms["master1"]) replica = replicas.get(DEFAULT_SUFFIX) log.info('Remove ldif files, if present in: {}'.format(changelog_dir)) for files in os.listdir(changelog_dir): - if files.endswith('.ldif'): + if files.endswith(changelog_end): changelog_file = os.path.join(changelog_dir, files) try: os.remove(changelog_file) @@ -94,7 +103,7 @@ def _create_changelog_dump(topo): log.info('Check if changelog ldif file exist in: {}'.format(changelog_dir)) for files in os.listdir(changelog_dir): - if files.endswith('.ldif'): + if files.endswith(changelog_end): changelog_ldif = os.path.join(changelog_dir, files) log.info('Changelog ldif file exist: {}'.format(changelog_ldif)) return changelog_ldif @@ -129,22 +138,23 @@ def get_ldap_error_msg(e, type): @pytest.fixture(scope="module") def changelog_init(topo): - """Initialize the test environment by changing log dir and - enabling cn=Retro Changelog Plugin,cn=plugins,cn=config - """ + """ changlog dir is not configuarable, just + enable cn=Retro Changelog Plugin,cn=plugins,cn=config + """ log.info('Testing Ticket 47669 - Test duration syntax in the changelogs') # bind as directory manager topo.ms["master1"].log.info("Bind as %s" % DN_DM) topo.ms["master1"].simple_bind_s(DN_DM, PASSWORD) - try: - changelogdir = os.path.join(os.path.dirname(topo.ms["master1"].dbdir), 'changelog') - topo.ms["master1"].modify_s(CHANGELOG, [(ldap.MOD_REPLACE, 'nsslapd-changelogdir', - ensure_bytes(changelogdir))]) - except ldap.LDAPError as e: - log.error('Failed to modify ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc'))) - assert False + if not ds_supports_new_changelog(): + try: + changelogdir = os.path.join(os.path.dirname(topo.ms["master1"].dbdir), 'changelog') + topo.ms["master1"].modify_s(CHANGELOG, [(ldap.MOD_REPLACE, 'nsslapd-changelogdir', + ensure_bytes(changelogdir))]) + except ldap.LDAPError as e: + log.error('Failed to modify ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc'))) + assert False try: topo.ms["master1"].modify_s(RETROCHANGELOG, [(ldap.MOD_REPLACE, 'nsslapd-pluginEnabled', b'on')]) @@ -204,7 +214,10 @@ def remove_ldif_files_from_changelogdir(topo, extension): """ Remove existing ldif files from changelog dir """ - changelog_dir = topo.ms['master1'].get_changelog_dir() + if ds_supports_new_changelog(): + changelog_dir = topo.ms['master1'].get_ldif_dir() + else: + changelog_dir = topo.ms['master1'].get_changelog_dir() log.info('Remove %s files, if present in: %s' % (extension, changelog_dir)) for files in os.listdir(changelog_dir): @@ -220,6 +233,7 @@ def remove_ldif_files_from_changelogdir(topo, extension): @pytest.mark.xfail(ds_is_older('1.3.10.1', '1.4.3'), reason="bug bz1685059") +@pytest.mark.skip(reason="does not work for prefix builds") @pytest.mark.bz1685059 @pytest.mark.ds50498 @pytest.mark.bz1769296 @@ -351,7 +365,10 @@ def test_dsconf_dump_changelog_files_removed(topo): 10. .ldif.done generated files are present in the changelog dir """ - changelog_dir = topo.ms['master1'].get_changelog_dir() + if ds_supports_new_changelog(): + changelog_dir = topo.ms['master1'].get_ldif_dir() + else: + changelog_dir = topo.ms['master1'].get_changelog_dir() instance = topo.ms['master1'] instance_url = 'ldap://%s:%s' % (HOST_MASTER_1, PORT_MASTER_1) @@ -466,7 +483,10 @@ def test_verify_changelog_online_backup(topo): log.fatal('test_changelog5: Online backup failed') assert False - backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB) + if ds_supports_new_changelog(): + backup_checkdir = os.path.join(backup_dir, DEFAULT_BENAME, 'changelog.db') + else: + backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB) if os.path.exists(backup_checkdir): log.info('Database backup is created successfully') else: @@ -524,7 +544,10 @@ def test_verify_changelog_offline_backup(topo): assert False topo.ms['master1'].start() - backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB) + if ds_supports_new_changelog(): + backup_checkdir = os.path.join(backup_dir, DEFAULT_BENAME, 'changelog.db') + else: + backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB) if os.path.exists(backup_checkdir): log.info('Database backup is created successfully') else: @@ -603,6 +626,7 @@ def test_ticket47669_changelog_triminterval(topo, changelog_init): @pytest.mark.ds47669 +@pytest.mark.skipif(ds_supports_new_changelog(), reason="changelog compaction is done by the backend itself, with id2entry as well, nsslapd-changelogcompactdb-interval is no longer supported") def test_changelog_compactdbinterval(topo, changelog_init): """Check nsslapd-changelog compactdbinterval values diff --git a/dirsrvtests/tests/suites/replication/changelog_trimming_test.py b/dirsrvtests/tests/suites/replication/changelog_trimming_test.py index ed7b279..0a4d15b 100644 --- a/dirsrvtests/tests/suites/replication/changelog_trimming_test.py +++ b/dirsrvtests/tests/suites/replication/changelog_trimming_test.py @@ -8,6 +8,7 @@ from lib389.properties import * from lib389.topologies import topology_m1 as topo from lib389.replica import Changelog5 from lib389.idm.domain import Domain +from lib389.utils import ensure_bytes, ds_supports_new_changelog pytestmark = pytest.mark.tier1 @@ -18,6 +19,10 @@ else: logging.getLogger(__name__).setLevel(logging.INFO) log = logging.getLogger(__name__) +CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM) +MAXAGE = 'nsslapd-changelogmaxage' +MAXENTRIES = 'nsslapd-changelogmaxentries' +TRIMINTERVAL = 'nsslapd-changelogtrim-interval' def do_mods(master, num): """Perform a num of mods on the default suffix @@ -26,6 +31,16 @@ def do_mods(master, num): for i in range(num): domain.replace('description', 'change %s' % i) +def set_value(master, attr, val): + """ + Helper function to add/replace attr: val and check the added value + """ + try: + master.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, attr, ensure_bytes(val))]) + except ldap.LDAPError as e: + log.error('Failed to add ' + attr + ': ' + val + ' to ' + plugin + ': error {}'.format(get_ldap_error_msg(e,'desc'))) + assert False + @pytest.fixture(scope="module") def setup_max_entries(topo, request): """Configure logging and changelog max entries @@ -34,9 +49,12 @@ def setup_max_entries(topo, request): master.config.loglevel((ErrorLog.REPLICA,), 'error') - cl = Changelog5(master) - cl.set_max_entries('2') - cl.set_trim_interval('300') + if ds_supports_new_changelog(): + set_value(master, MAXENTRIES, '2') + set_value(master, TRIMINTERVAL, '300') + else: + cl = Changelog5(master) + cl.set_trim_interval('300') @pytest.fixture(scope="module") def setup_max_age(topo, request): @@ -45,9 +63,13 @@ def setup_max_age(topo, request): master = topo.ms["master1"] master.config.loglevel((ErrorLog.REPLICA,), 'error') - cl = Changelog5(master) - cl.set_max_age('5') - cl.set_trim_interval('300') + if ds_supports_new_changelog(): + set_value(master, MAXAGE, '5') + set_value(master, TRIMINTERVAL, '300') + else: + cl = Changelog5(master) + cl.set_max_age('5') + cl.set_trim_interval('300') def test_max_age(topo, setup_max_age): """Test changing the trimming interval works with max age @@ -68,7 +90,8 @@ def test_max_age(topo, setup_max_age): log.info("Testing changelog triming interval with max age...") master = topo.ms["master1"] - cl = Changelog5(master) + if not ds_supports_new_changelog(): + cl = Changelog5(master) # Do mods to build if cl entries do_mods(master, 10) @@ -78,7 +101,10 @@ def test_max_age(topo, setup_max_age): log.fatal('Trimming event unexpectedly occurred') assert False - cl.set_trim_interval('5') + if ds_supports_new_changelog(): + set_value(master, TRIMINTERVAL, '5') + else: + cl.set_trim_interval('5') time.sleep(6) # Trimming should have occured @@ -106,7 +132,8 @@ def test_max_entries(topo, setup_max_entries): log.info("Testing changelog triming interval with max entries...") master = topo.ms["master1"] - cl = Changelog5(master) + if not ds_supports_new_changelog(): + cl = Changelog5(master) # reset errors log master.deleteErrorLogs() @@ -118,7 +145,10 @@ def test_max_entries(topo, setup_max_entries): log.fatal('Trimming event unexpectedly occurred') assert False - cl.set_trim_interval('5') + if ds_supports_new_changelog(): + set_value(master, TRIMINTERVAL, '5') + else: + cl.set_trim_interval('5') time.sleep(6) # Trimming should have occured diff --git a/dirsrvtests/tests/suites/replication/cleanallruv_test.py b/dirsrvtests/tests/suites/replication/cleanallruv_test.py index 6bb59e8..fa1a984 100644 --- a/dirsrvtests/tests/suites/replication/cleanallruv_test.py +++ b/dirsrvtests/tests/suites/replication/cleanallruv_test.py @@ -82,6 +82,7 @@ def check_ruvs(msg, topology_m4, m4rid): clean = False replicas = Replicas(inst) replica = replicas.get(DEFAULT_SUFFIX) + log.info('check_ruvs for replica %s:%s (suffix:rid)' % (replica.get_suffix(), replica.get_rid())) count = 0 while not clean and count < 20: @@ -582,13 +583,15 @@ def test_stress_clean(topology_m4, m4rid): ldbm_config = LDBMConfig(topology_m4.ms["master4"]) # Put all the masters under load - m1_add_users = AddUsers(topology_m4.ms["master1"], 2000) + # not too high load else it takes a long time to converge and + # the test result becomes instable + m1_add_users = AddUsers(topology_m4.ms["master1"], 500) m1_add_users.start() - m2_add_users = AddUsers(topology_m4.ms["master2"], 2000) + m2_add_users = AddUsers(topology_m4.ms["master2"], 500) m2_add_users.start() - m3_add_users = AddUsers(topology_m4.ms["master3"], 2000) + m3_add_users = AddUsers(topology_m4.ms["master3"], 500) m3_add_users.start() - m4_add_users = AddUsers(topology_m4.ms["master4"], 2000) + m4_add_users = AddUsers(topology_m4.ms["master4"], 500) m4_add_users.start() # Allow sometime to get replication flowing in all directions diff --git a/dirsrvtests/tests/suites/replication/encryption_cl5_test.py b/dirsrvtests/tests/suites/replication/encryption_cl5_test.py index d6a5b9e..268442e 100644 --- a/dirsrvtests/tests/suites/replication/encryption_cl5_test.py +++ b/dirsrvtests/tests/suites/replication/encryption_cl5_test.py @@ -8,7 +8,8 @@ # import logging import pytest -from lib389.utils import ensure_bytes +import pdb +from lib389.utils import ensure_bytes, ds_supports_new_changelog from lib389.replica import ReplicationManager from lib389.dseldif import DSEldif from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES @@ -45,26 +46,34 @@ def _enable_changelog_encryption(inst, encrypt_algorithm): dse_ldif = DSEldif(inst) log.info('Configuring changelog encryption:{} for: {}'.format(inst.serverid, encrypt_algorithm)) inst.stop() - dse_ldif.replace(DN_CHANGELOG, 'nsslapd-encryptionalgorithm', encrypt_algorithm) - if dse_ldif.get(DN_CHANGELOG, 'nsSymmetricKey'): - dse_ldif.delete(DN_CHANGELOG, 'nsSymmetricKey') + if ds_supports_new_changelog(): + changelog = 'cn=changelog,{}'.format(DN_USERROOT_LDBM) + else: + changelog = DN_CHANGELOG + + dse_ldif.replace(changelog, 'nsslapd-encryptionalgorithm', encrypt_algorithm) + if dse_ldif.get(changelog, 'nsSymmetricKey'): + dse_ldif.delete(changelog, 'nsSymmetricKey') inst.start() def _check_unhashed_userpw_encrypted(inst, change_type, user_dn, user_pw, is_encrypted): """Check if unhashed#user#password attribute value is encrypted or not""" - changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB) - for dbfile in os.listdir(changelog_dbdir): - if dbfile.endswith('.db'): - changelog_dbfile = os.path.join(changelog_dbdir, dbfile) - log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile)) - log.info('Running dbscan -f to check {} attr'.format(ATTRIBUTE)) - dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile) + if ds_supports_new_changelog(): + dbscanOut = inst.dbscan(DEFAULT_BENAME, 'changelog') + else: + changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB) + for dbfile in os.listdir(changelog_dbdir): + if dbfile.endswith('.db'): + changelog_dbfile = os.path.join(changelog_dbdir, dbfile) + log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile)) + log.info('Running dbscan -f to check {} attr'.format(ATTRIBUTE)) + dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile) count = 0 for entry in dbscanOut.split(b'dbid: '): if ensure_bytes('operation: {}'.format(change_type)) in entry and\ - ensure_bytes(ATTRIBUTE) in entry and ensure_bytes(user_dn) in entry: + ensure_bytes(ATTRIBUTE) in entry and ensure_bytes(user_dn.lower()) in entry.lower(): count += 1 user_pw_attr = ensure_bytes('{}: {}'.format(ATTRIBUTE, user_pw)) if is_encrypted: @@ -112,7 +121,16 @@ def test_algorithm_unhashed(topology_with_tls, encryption): _enable_changelog_encryption(m1, encryption) for inst1, inst2 in ((m1, m2), (m2, m1)): - user_props = TEST_USER_PROPERTIES.copy() + # need to create a user specific to the encryption + # else the two runs will hit the same user + user_props={ + 'uid': 'testuser_%s' % encryption, + 'cn' : 'testuser_%s' % encryption, + 'sn' : 'user', + 'uidNumber' : '1000', + 'gidNumber' : '1000', + 'homeDirectory' : '/home/testuser_%s' % encryption + } user_props["userPassword"] = PASSWORD users = UserAccounts(inst1, DEFAULT_SUFFIX) tuser = users.create(properties=user_props) diff --git a/dirsrvtests/tests/suites/replication/regression_test.py b/dirsrvtests/tests/suites/replication/regression_test.py index d64814f..d03dffd 100644 --- a/dirsrvtests/tests/suites/replication/regression_test.py +++ b/dirsrvtests/tests/suites/replication/regression_test.py @@ -29,6 +29,7 @@ pytestmark = pytest.mark.tier1 NEW_SUFFIX_NAME = 'test_repl' NEW_SUFFIX = 'o={}'.format(NEW_SUFFIX_NAME) NEW_BACKEND = 'repl_base' +CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM) MAXAGE_ATTR = 'nsslapd-changelogmaxage' MAXAGE_STR = '30' TRIMINTERVAL_STR = '5' @@ -41,6 +42,16 @@ else: logging.getLogger(__name__).setLevel(logging.INFO) log = logging.getLogger(__name__) +@pytest.fixture(scope="module") +def set_value(master, attr, val): + """ + Helper function to add/replace attr: val and check the added value + """ + try: + master.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, attr, ensure_bytes(val))]) + except ldap.LDAPError as e: + log.error('Failed to add ' + attr + ': ' + val + ' to ' + plugin + ': error {}'.format(get_ldap_error_msg(e,'desc'))) + assert False def find_start_location(file, no): log_pattern = re.compile("slapd_daemon - slapd started.") @@ -675,13 +686,28 @@ def test_cleanallruv_repl(topo_m3): m1_m3 = M1.agreement.list(suffix=SUFFIX, consumer_host=M3.host, consumer_port=M3.port) m3_m1 = M3.agreement.list(suffix=SUFFIX, consumer_host=M1.host, consumer_port=M1.port) - log.info("Get the changelog enteries for M1 and M2") - changelog_m1 = Changelog5(M1) - changelog_m2 = Changelog5(M2) - log.info("Modify nsslapd-changelogmaxage=30 and nsslapd-changelogtrim-interval=5 for M1 and M2") - changelog_m1.set_max_age(MAXAGE_STR) - changelog_m1.set_trim_interval(TRIMINTERVAL_STR) + if ds_supports_new_changelog(): + CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM) + + #set_value(M1, MAXAGE_ATTR, MAXAGE_STR) + try: + M1.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, MAXAGE_ATTR, ensure_bytes(MAXAGE_STR))]) + except ldap.LDAPError as e: + log.error('Failed to add ' + MAXAGE_ATTR, + ': ' + MAXAGE_STR + ' to ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc'))) + assert False + + #set_value(M2, TRIMINTERVAL, TRIMINTERVAL_STR) + try: + M2.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, TRIMINTERVAL, ensure_bytes(TRIMINTERVAL_STR))]) + except ldap.LDAPError as e: + log.error('Failed to add ' + TRIMINTERVAL, + ': ' + TRIMINTERVAL_STR + ' to ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc'))) + assert False + else: + log.info("Get the changelog enteries for M1 and M2") + changelog_m1 = Changelog5(M1) + changelog_m1.set_max_age(MAXAGE_STR) + changelog_m1.set_trim_interval(TRIMINTERVAL_STR) log.info("Add test users to 3 masters") users_m1 = UserAccounts(M1, DEFAULT_SUFFIX) diff --git a/ldap/admin/src/scripts/DSCreate.pm.in b/ldap/admin/src/scripts/DSCreate.pm.in index a2f15f9..6ed2835 100644 --- a/ldap/admin/src/scripts/DSCreate.pm.in +++ b/ldap/admin/src/scripts/DSCreate.pm.in @@ -481,7 +481,7 @@ sub makeOtherConfigFiles { if ($!) { return ('error_copying_file', $src, $dest, $!); } - if (@errs = changeOwnerMode($inf, 4, $dest)) { + if (@errs = changeOwnerMode($inf, 6, $dest)) { return @errs; } } diff --git a/ldap/admin/src/scripts/db2ldif.in b/ldap/admin/src/scripts/db2ldif.in index f1b0826..169990d 100755 --- a/ldap/admin/src/scripts/db2ldif.in +++ b/ldap/admin/src/scripts/db2ldif.in @@ -25,6 +25,7 @@ usage() echo " -x - Suffix to exclude" echo " -a outputfile - Name of the exported LDIF file" echo " -r - Include replication data" + echo " -R - Include changelog data" echo " -E - Decrypt attributes" echo " -u - Do not export the nsUniqueId attribute" echo " -U - Do not wrap long lines" @@ -100,7 +101,7 @@ then exit 1 fi -while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qVc:" flag +while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qRVc:" flag do case $flag in h) usage @@ -119,6 +120,7 @@ do S) args=$args" -S";; v) args=$args" -v";; r) args=$args" -r";; + R) args=$args" -R";; C) args=$args" -C";; u) args=$args" -u";; U) args=$args" -U";; diff --git a/ldap/admin/src/scripts/ldif2db.in b/ldap/admin/src/scripts/ldif2db.in index e5ba43d..bb9925d 100755 --- a/ldap/admin/src/scripts/ldif2db.in +++ b/ldap/admin/src/scripts/ldif2db.in @@ -30,6 +30,7 @@ usage() echo " -G name - Namespace id for name based uniqueid (-g deterministic)" echo " -O - Do not index the attributes" echo " -E - Encrypt attributes" + echo " -R - Import changelog data" echo " -q - Quiet mode - suppresses output" echo " -V - Verbose output" echo " -v - Display version" @@ -54,7 +55,7 @@ handleopts() return 0 } -while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:EqV" flag +while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:ERqV" flag do case $flag in h) usage @@ -71,6 +72,7 @@ do t) args=$args" -t \"$OPTARG\"";; D) args=$args" -D \"$OPTARG\"";; E) args=$args" -E";; + R) args=$args" -R";; v) args=$args" -v";; N) args=$args" -N";; C) args=$args" -C";; diff --git a/ldap/servers/plugins/replication/cl5.h b/ldap/servers/plugins/replication/cl5.h index 2af57e3..b7e3203 100644 --- a/ldap/servers/plugins/replication/cl5.h +++ b/ldap/servers/plugins/replication/cl5.h @@ -21,17 +21,18 @@ typedef struct changelog5Config { char *dir; - /* These 2 parameters are needed for changelog trimming. Already present in 5.0 */ + /* These 3 parameters are needed for changelog trimming. */ char *maxAge; int maxEntries; - /* the changelog DB configuration parameters are defined as CL5DBConfig in cl5_api.h */ - CL5DBConfig dbconfig; - char *symmetricKey; - long compactInterval; long trimInterval; + /* configuration of changelog encryption */ + char *encryptionAlgorithm; + char *symmetricKey; } changelog5Config; -/* initializes changelog*/ +/* upgrade changelog*/ +int changelog5_upgrade(void); +/* initialize changelog*/ int changelog5_init(void); /* cleanups changelog data */ void changelog5_cleanup(void); @@ -41,6 +42,11 @@ int changelog5_config_init(void); void changelog5_config_cleanup(void); /* reads changelog configuration */ int changelog5_read_config(changelog5Config *config); +/* transforms entry to internal config */ +void changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config); +/* registeri/unregister functions to handle config changes */ +int changelog5_register_config_callbacks(const char *dn, Replica *replica); +int changelog5_remove_config_callbacks(const char *dn); /* cleanups the content of the config structure */ void changelog5_config_done(changelog5Config *config); /* frees the content and the config structure */ diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c index d7e4749..724b35a 100644 --- a/ldap/servers/plugins/replication/cl5_api.c +++ b/ldap/servers/plugins/replication/cl5_api.c @@ -68,14 +68,6 @@ #define HASH_BACKETS_COUNT 16 /* number of buckets in a hash table */ #define DEFAULT_DB_ENV_OP_FLAGS DB_AUTO_COMMIT -#define DB_OPEN(oflags, db, txnid, file, database, type, flags, mode, rval) \ - { \ - if (((oflags)&DB_INIT_TXN) && ((oflags)&DB_INIT_LOG)) { \ - (rval) = (db)->open((db), (txnid), (file), (database), (type), (flags) | DB_AUTO_COMMIT, (mode)); \ - } else { \ - (rval) = (db)->open((db), (txnid), (file), (database), (type), (flags), (mode)); \ - } \ - } #define TXN_BEGIN(env, parent_txn, tid, flags) \ (env)->txn_begin((env), (parent_txn), (tid), (flags)) @@ -98,7 +90,6 @@ #define DEFAULT_THREAD_STACKSIZE 0 #endif -#define FILE_CREATE_MODE S_IRUSR | S_IWUSR #define DIR_CREATE_MODE 0755 #define NO_DISK_SPACE 1024 @@ -117,28 +108,45 @@ typedef enum { CL5_OPEN_CLEAN_RECOVER /* remove env after recover open (upgrade) */ } CL5OpenMode; -#define DB_FILE_DELETED 0x1 -#define DB_FILE_INIT 0x2 +#define DB_FILE_ACTIVE 0x01 +#define DB_FILE_DONE 0x08 +/* changelog trimming configuration */ +typedef struct cl5config +{ + time_t maxAge; /* maximum entry age in seconds */ + int maxEntries; /* maximum number of entries across all changelog files */ + int trimInterval; /* trimming interval */ + char *encryptionAlgorithm; /* nsslapd-encryptionalgorithm */ +} CL5Config; + /* this structure represents one changelog file, Each changelog file contains changes applied to a single backend. Files are named by the database id */ -typedef struct cl5dbfile + +struct cl5DBFileHandle +/* info about the changelog file in the main database environment */ +/* usage as CL5DBFile, but for new implementation use a new struct + * can be replaced later + */ { - char *name; /* file name (with the extension) */ - char *replGen; /* replica generation of the data */ - char *replName; /* replica name */ - DB *db; /* db handle to the changelog file*/ - int entryCount; /* number of entries in the file */ - int flags; /* currently used to mark the file as deleted - * or as initialized */ + DB *db; /* db handle to the changelog file*/ + char *ident; /* identifier for changelog, used in error messages */ + int entryCount; /* number of entries in the file */ + int flags; /* currently used to mark the file or as initialized */ RUV *purgeRUV; /* ruv to which the file has been purged */ - RUV *maxRUV; /* ruv that marks the upper boundary of the data */ -} CL5DBFile; + RUV *maxRUV; /* ruv that marks the upper boundary of the data */ + CL5Config clConf; /* trimming and encryption config */ + Slapi_Counter *clThreads; /* track threads operating on the changelog */ + PRLock *clLock; /* controls access to trimming configuration and */ + /* Lock associated to clVar, used to notify threads on close */ + PRCondVar *clCvar; /* Condition Variable used to notify threads on close */ + void *clcrypt_handle; /* for cl encryption */ +}; /* structure that allows to iterate through entries to be sent to a consumer that originated on a particular supplier. */ struct cl5replayiterator { - Object *fileObj; + cldb_Handle *it_cldb; CLC_Buffer *clcache; /* changelog cache */ ReplicaId consumerRID; /* consumer's RID */ const RUV *consumerRuv; /* consumer's update vector */ @@ -148,41 +156,18 @@ struct cl5replayiterator typedef struct cl5iterator { DBC *cursor; /* current position in the db file */ - Object *file; /* handle to release db file object */ + cldb_Handle *it_cldb; /* handle to release db file object */ } CL5Iterator; -/* changelog trimming configuration */ -typedef struct cl5trim -{ - time_t maxAge; /* maximum entry age in seconds */ - int maxEntries; /* maximum number of entries across all changelog files */ - int compactInterval; /* interval to compact changelog db */ - int trimInterval; /* trimming interval */ - PRLock *lock; /* controls access to trimming configuration */ -} CL5Trim; - /* this structure defines 5.0 changelog internals */ typedef struct cl5desc { - char *dbDir; /* absolute path to changelog directory */ DB_ENV *dbEnv; /* db environment shared by all db files */ - int dbEnvOpenFlags; /* openflag used for env->open */ - Objset *dbFiles; /* ref counted set of changelog files (CL5DBFile) */ - PRLock *fileLock; /* ensures that changelog file is not added twice */ CL5OpenMode dbOpenMode; /* how we open db */ - CL5DBConfig dbConfig; /* database configuration params */ - CL5Trim dbTrim; /* trimming parameters */ CL5State dbState; /* changelog current state */ Slapi_RWLock *stLock; /* lock that controls access to the changelog state */ - PRBool dbRmOnClose; /* indicates whether changelog should be removed when - it is closed */ - PRBool fatalError; /* bad stuff happened like out of disk space; don't - write guardian file on close - UnUsed so far */ int threadCount; /* threads that globally access changelog like deadlock detection, etc. */ - PRLock *clLock; /* Lock associated to clVar, used to notify threads on close */ - PRCondVar *clCvar; /* Condition Variable used to notify threads on close */ - void *clcrypt_handle; /* for cl encryption */ } CL5Desc; typedef void (*VFP)(void *); @@ -193,87 +178,63 @@ static CL5Desc s_cl5Desc = {0}; /***** Forward Declarations *****/ /* changelog initialization and cleanup */ -static int _cl5Open(const char *dir, const CL5DBConfig *config, CL5OpenMode openMode); -static int _cl5AppInit(void); -static int _cl5DBOpen(void); -static void _cl5SetDefaultDBConfig(void); -static void _cl5SetDBConfig(const CL5DBConfig *config); -static int _cl5CheckDBVersion(void); -static int _cl5ReadDBVersion(const char *dir, char *clVersion, int buflen); -static int _cl5WriteDBVersion(void); +static int _cl5Open(CL5OpenMode openMode); +static int _cldb_CheckAndSetEnv(Slapi_Backend *be); static void _cl5Close(void); -static int _cl5Delete(const char *dir, PRBool rmDir); static void _cl5DBClose(void); /* thread management */ -static int _cl5DispatchDBThreads(void); +static int _cl5DispatchTrimThread(Replica *replica); static int _cl5AddThread(void); static void _cl5RemoveThread(void); /* functions that work with individual changelog files */ -static int _cl5NewDBFile(const char *replName, const char *replGen, CL5DBFile **dbFile); -static int _cl5DBOpenFile(Replica *replica, Object **obj, PRBool checkDups); -static int _cl5DBOpenFileByReplicaName(const char *replName, const char *replGen, Object **obj, PRBool checkDups); -static void _cl5DBCloseFile(void **data); -static void _cl5DBDeleteFile(Object *obj); -static void _cl5DBFileInitialized(Object *obj); -static int _cl5GetDBFile(Replica *replica, Object **obj); -static int _cl5GetDBFileByReplicaName(const char *replName, const char *replGen, Object **obj); -static int _cl5AddDBFile(CL5DBFile *file, Object **obj); -static int _cl5CompareDBFile(Object *el1, const void *el2); -static char *_cl5Replica2FileName(Replica *replica); -static char *_cl5MakeFileName(const char *replName, const char *replGen); -static PRBool _cl5FileName2Replica(const char *fileName, Replica **replica); -static int _cl5ExportFile(PRFileDesc *prFile, Object *obj); -static PRBool _cl5ReplicaInList(Replica *replica, Replica **replicas); +static int _cl5ExportFile(PRFileDesc *prFile, cldb_Handle *cldb); /* data storage and retrieval */ -static int _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len); -static int _cl5WriteOperation(const char *replName, const char *replGen, const slapi_operation_parameters *op, PRBool local); -static int _cl5WriteOperationTxn(const char *replName, const char *replGen, const slapi_operation_parameters *op, PRBool local, void *txn); -static int _cl5GetFirstEntry(Object *obj, CL5Entry *entry, void **iterator, DB_TXN *txnid); +static int _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len, void *clcrypt_handle); +static int _cl5WriteOperation(cldb_Handle *cldb, const slapi_operation_parameters *op); +static int _cl5WriteOperationTxn(cldb_Handle *cldb, const slapi_operation_parameters *op, void *txn); +static int _cl5GetFirstEntry(cldb_Handle *cldb, CL5Entry *entry, void **iterator, DB_TXN *txnid); static int _cl5GetNextEntry(CL5Entry *entry, void *iterator); static int _cl5CurrentDeleteEntry(void *iterator); static const char *_cl5OperationType2Str(int type); static int _cl5Str2OperationType(const char *str); static void _cl5WriteString(const char *str, char **buff); static void _cl5ReadString(char **str, char **buff); -static void _cl5WriteMods(LDAPMod **mods, char **buff); -static int _cl5WriteMod(LDAPMod *mod, char **buff); -static int _cl5ReadMods(LDAPMod ***mods, char **buff); -static int _cl5ReadMod(Slapi_Mod *mod, char **buff); +static void _cl5WriteMods(LDAPMod **mods, char **buff, void *clcrypt_handle); +static int _cl5WriteMod(LDAPMod *mod, char **buff, void *clcrypt_handle); +static int _cl5ReadMods(LDAPMod ***mods, char **buff, void *clcrypt_handle); +static int _cl5ReadMod(Slapi_Mod *mod, char **buff, void *clcrypt_handle); static int _cl5GetModsSize(LDAPMod **mods); static int _cl5GetModSize(LDAPMod *mod); static void _cl5ReadBerval(struct berval *bv, char **buff); static void _cl5WriteBerval(struct berval *bv, char **buff); static int _cl5ReadBervals(struct berval ***bv, char **buff, unsigned int size); static int _cl5WriteBervals(struct berval **bv, char **buff, u_int32_t *size); -static int32_t _cl5CheckMaxRUV(CL5DBFile *file, RUV *maxruv); +static int32_t _cl5CheckMaxRUV(cldb_Handle *cldb, RUV *maxruv); static int32_t _cl5CheckCSNinCL(const ruv_enum_data *element, void *arg); /* replay iteration */ #ifdef FOR_DEBUGGING static PRBool _cl5ValidReplayIterator(const CL5ReplayIterator *iterator); #endif -static int _cl5PositionCursorForReplay(ReplicaId consumerRID, const RUV *consumerRuv, Replica *replica, Object *fileObject, CL5ReplayIterator **iterator, int *continue_on_missing); -static int _cl5CheckMissingCSN(const CSN *minCsn, const RUV *supplierRUV, CL5DBFile *file); +static int _cl5PositionCursorForReplay(ReplicaId consumerRID, const RUV *consumerRuv, Replica *replica, CL5ReplayIterator **iterator, int *continue_on_missing); +static int _cl5CheckMissingCSN(const CSN *minCsn, const RUV *supplierRUV, cldb_Handle *cldb); /* changelog trimming */ -static int _cl5TrimInit(void); -static void _cl5TrimCleanup(void); +static int cldb_IsTrimmingEnabled(cldb_Handle *cldb); static int _cl5TrimMain(void *param); -static void _cl5DoTrimming(void); -static void _cl5CompactDBs(void); -static void _cl5PurgeRID(Object *file_obj, ReplicaId cleaned_rid); -static int _cl5PurgeGetFirstEntry(Object *file_obj, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key); +static void _cl5TrimReplica(Replica *r); +static void _cl5PurgeRID(cldb_Handle *cldb, ReplicaId cleaned_rid); +static int _cl5PurgeGetFirstEntry(cldb_Handle *cldb, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key); static int _cl5PurgeGetNextEntry(CL5Entry *entry, void *iterator, DBT *key); -static void _cl5TrimFile(Object *obj, long *numToTrim); -static PRBool _cl5CanTrim(time_t time, long *numToTrim); -static int _cl5ReadRUV(const char *replGen, Object *obj, PRBool purge); -static int _cl5WriteRUV(CL5DBFile *file, PRBool purge); -static int _cl5ConstructRUV(const char *replGen, Object *obj, PRBool purge); -static int _cl5UpdateRUV(Object *obj, CSN *csn, PRBool newReplica, PRBool purge); -static int _cl5GetRUV2Purge2(Object *fileObj, RUV **ruv); +static PRBool _cl5CanTrim(time_t time, long *numToTrim, Replica *replica, CL5Config *dbTrim); +static int _cl5ReadRUV(cldb_Handle *cldb, PRBool purge); +static int _cl5WriteRUV(cldb_Handle *cldb, PRBool purge); +static int _cl5ConstructRUV(cldb_Handle *cldb, PRBool purge); +static int _cl5UpdateRUV (cldb_Handle *cldb, CSN *csn, PRBool newReplica, PRBool purge); +static int _cl5GetRUV2Purge2(Replica *r, RUV **ruv); void trigger_cl_purging_thread(void *rid); /* bakup/recovery, import/export */ @@ -281,19 +242,14 @@ static int _cl5LDIF2Operation(char *ldifEntry, slapi_operation_parameters *op, c static int _cl5Operation2LDIF(const slapi_operation_parameters *op, const char *replGen, char **ldifEntry, PRInt32 *lenLDIF); /* entry count */ -static int _cl5GetEntryCount(CL5DBFile *file); -static int _cl5WriteEntryCount(CL5DBFile *file); +static int _cl5GetEntryCount(cldb_Handle *cldb); +static int _cl5WriteEntryCount(cldb_Handle *cldb); /* misc */ static char *_cl5GetHelperEntryKey(int type, char *csnStr); -static Replica *_cl5GetReplica(const slapi_operation_parameters *op, const char *replGen); -static int _cl5FileEndsWith(const char *filename, const char *ext); -static PRLock *cl5_diskfull_lock = NULL; -static int cl5_diskfull_flag = 0; -static void cl5_set_diskfull(void); -static void cl5_set_no_diskfull(void); +static int _cl5WriteReplicaRUV(Replica *r, void *arg); /***** Module APIs *****/ @@ -314,32 +270,14 @@ cl5Init(void) PR_GetError()); return CL5_SYSTEM_ERROR; } - if ((s_cl5Desc.clLock = PR_NewLock()) == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5Init - Failed to create on close lock; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; - } - if ((s_cl5Desc.clCvar = PR_NewCondVar(s_cl5Desc.clLock)) == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5Init - Failed to create on close cvar; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; - } - if ((clcache_init(&s_cl5Desc.dbEnv) != 0)) { + if ((clcache_init() != 0)) { return CL5_SYSTEM_ERROR; } s_cl5Desc.dbState = CL5_STATE_CLOSED; - s_cl5Desc.fatalError = PR_FALSE; - s_cl5Desc.dbRmOnClose = PR_FALSE; s_cl5Desc.threadCount = 0; - if (NULL == cl5_diskfull_lock) { - cl5_diskfull_lock = PR_NewLock(); - } - return CL5_SUCCESS; } @@ -361,20 +299,6 @@ cl5Cleanup() slapi_destroy_rwlock(s_cl5Desc.stLock); s_cl5Desc.stLock = NULL; - if (cl5_diskfull_lock) { - PR_DestroyLock(cl5_diskfull_lock); - cl5_diskfull_lock = NULL; - } - if (s_cl5Desc.clLock != NULL) { - PR_DestroyLock(s_cl5Desc.clLock); - s_cl5Desc.clLock = NULL; - } - - if (s_cl5Desc.clCvar != NULL) { - PR_DestroyCondVar(s_cl5Desc.clCvar); - s_cl5Desc.clCvar = NULL; - } - memset(&s_cl5Desc, 0, sizeof(s_cl5Desc)); } @@ -382,8 +306,6 @@ cl5Cleanup() Description: opens changelog; must be called after changelog is initialized using cl5Init. It is thread safe and the second call is ignored. - Parameters: dir - changelog dir - config - db configuration parameters; currently not used Return: CL5_SUCCESS if successfull; CL5_BAD_DATA if invalid directory is passed; CL5_BAD_STATE if changelog is not initialized; @@ -393,15 +315,10 @@ cl5Cleanup() CL5_DB_ERROR if db initialization fails. */ int -cl5Open(const char *dir, const CL5DBConfig *config) +cl5Open(void) { int rc; - if (dir == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5Open: null directory\n"); - return CL5_BAD_DATA; - } - if (s_cl5Desc.dbState == CL5_STATE_NONE) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5Open - Changelog is not initialized\n"); @@ -424,29 +341,27 @@ cl5Open(const char *dir, const CL5DBConfig *config) goto done; } - rc = _cl5Open(dir, config, CL5_OPEN_NORMAL); + /* if we are here we know that the database environment is setup + * what remains is to setup the config info for all the individual + * changelogs. + * If we fail set state back to closed. + */ + s_cl5Desc.dbState = CL5_STATE_OPEN; + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "cl5Open - setting dbState to CL5_STATE_OPEN\n"); + rc = _cl5Open(CL5_OPEN_NORMAL); if (rc != CL5_SUCCESS) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5Open - Failed to open changelog\n"); goto done; } - /* dispatch global threads like deadlock detection, trimming, etc */ - rc = _cl5DispatchDBThreads(); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5Open - Failed to start database monitoring threads\n"); - - _cl5Close(); - } else { - s_cl5Desc.dbState = CL5_STATE_OPEN; - clcache_set_config(); - - /* Set the cl encryption algorithm (if configured) */ - rc = clcrypt_init(config, &s_cl5Desc.clcrypt_handle); - } + clcache_set_config(); done: + if (rc != CL5_SUCCESS) { + s_cl5Desc.dbState = CL5_STATE_CLOSED; + } slapi_rwlock_unlock(s_cl5Desc.stLock); return rc; @@ -489,115 +404,47 @@ cl5Close() /* signal changelog closing to all threads */ s_cl5Desc.dbState = CL5_STATE_CLOSING; - - PR_Lock(s_cl5Desc.clLock); - PR_NotifyCondVar(s_cl5Desc.clCvar); - PR_Unlock(s_cl5Desc.clLock); + /* replica_enumerate_replicas(cldb_StopTrimming, NULL); */ _cl5Close(); s_cl5Desc.dbState = CL5_STATE_CLOSED; - rc = clcrypt_destroy(s_cl5Desc.clcrypt_handle); + + s_cl5Desc.dbEnv = NULL; slapi_rwlock_unlock(s_cl5Desc.stLock); return rc; } -/* Name: cl5Delete - Description: removes changelog; changelog must be in the closed state. - Parameters: dir - changelog directory - Return: CL5_SUCCESS if successful; - CL5_BAD_STATE if the changelog is not in closed state; - CL5_BAD_DATA if invalid directory supplied - CL5_SYSTEM_ERROR if NSPR call fails - */ -int -cl5Delete(const char *dir) +static int +_cldb_DeleteDB(Replica *replica) { - int rc; - - if (dir == NULL) { - slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name_cl, "cl5Delete - NULL directory\n"); - return CL5_BAD_DATA; - } - - if (s_cl5Desc.dbState == CL5_STATE_NONE) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5Delete - Changelog is not initialized\n"); - return CL5_BAD_STATE; - } + int rc = 0; + cldb_Handle *cldb; + Slapi_Backend *be; - slapi_rwlock_wrlock(s_cl5Desc.stLock); + cldb = replica_get_file_info(replica); + /* make sure that changelog stays open while operation is in progress */ - if (s_cl5Desc.dbState != CL5_STATE_CLOSED) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5Delete - Invalid state - %d\n", s_cl5Desc.dbState); - slapi_rwlock_unlock(s_cl5Desc.stLock); - return CL5_BAD_STATE; - } + slapi_counter_increment(cldb->clThreads); - rc = _cl5Delete(dir, PR_TRUE /* remove changelog dir */); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5Delete - Failed to remove changelog\n"); - } + be = slapi_be_select(replica_get_root(replica)); + + slapi_back_ctrl_info(be, BACK_INFO_DBENV_CLDB_REMOVE, (void *)(cldb->db)); + cldb->db = NULL; - slapi_rwlock_unlock(s_cl5Desc.stLock); + slapi_counter_decrement(cldb->clThreads); return rc; } - -/* Name: cl5DeleteDBSync - Description: The same as cl5DeleteDB except the function does not return - until the file is removed. -*/ int -cl5DeleteDBSync(Replica *replica) +cldb_RemoveReplicaDB(Replica *replica) { - Object *obj; - int rc; - CL5DBFile *file; - - if (replica == NULL) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5DeleteDBSync - invalid database id\n"); - return CL5_BAD_DATA; - } - - /* changelog is not initialized */ - if (s_cl5Desc.dbState == CL5_STATE_NONE) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "cl5DeleteDBSync - " - "Changelog is not initialized\n"); - return CL5_BAD_STATE; - } - - /* make sure that changelog stays open while operation is in progress */ - rc = _cl5AddThread(); - if (rc != CL5_SUCCESS) - return rc; - - rc = _cl5GetDBFile(replica, &obj); - if (rc == CL5_SUCCESS) { - char *filename = NULL; - file = (CL5DBFile *)object_get_data(obj); - PR_ASSERT(file); - /* file->name is freed in _cl5DBDeleteFile */ - filename = slapi_ch_strdup(file->name); - - _cl5DBDeleteFile(obj); - - /* wait until the file is gone */ - while (PR_Access(filename, PR_ACCESS_EXISTS) == PR_SUCCESS) { - DS_Sleep(PR_MillisecondsToInterval(100)); - } - slapi_ch_free_string(&filename); - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "cl5DeleteDBSync - " - "File for replica at (%s) not found\n", - slapi_sdn_get_dn(replica_get_root(replica))); - } + int rc =0; + cldb_Handle *cldb = replica_get_file_info(replica); - _cl5RemoveThread(); + cldb->flags |= DB_FILE_DONE; + rc = cldb_UnSetReplicaDB(replica, NULL); return rc; } @@ -615,11 +462,8 @@ cl5DeleteDBSync(Replica *replica) int cl5GetUpperBoundRUV(Replica *r, RUV **ruv) { - int rc; - Object *file_obj; - CL5DBFile *file; - const char *replName; - char *replGen; + int rc = CL5_SUCCESS; + cldb_Handle *cldb = NULL; if (r == NULL || ruv == NULL) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, @@ -634,28 +478,13 @@ cl5GetUpperBoundRUV(Replica *r, RUV **ruv) return CL5_BAD_STATE; } + cldb = replica_get_file_info(r); /* make sure that changelog stays open while operation is in progress */ - rc = _cl5AddThread(); - if (rc != CL5_SUCCESS) - return rc; - - replName = replica_get_name(r); - replGen = replica_get_generation(r); - rc = _cl5GetDBFileByReplicaName(replName, replGen, &file_obj); - slapi_ch_free_string(&replGen); - if (rc == CL5_SUCCESS) { - file = (CL5DBFile *)object_get_data(file_obj); - PR_ASSERT(file && file->maxRUV); - - *ruv = ruv_dup(file->maxRUV); - - object_release(file_obj); - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "cl5GetUpperBoundRUV - " - "Could not find DB object for replica\n"); - } - - _cl5RemoveThread(); + slapi_counter_increment(cldb->clThreads); + PR_ASSERT(cldb && cldb->maxRUV); + *ruv = ruv_dup(cldb->maxRUV); + + slapi_counter_decrement(cldb->clThreads); return rc; } @@ -674,12 +503,10 @@ cl5GetUpperBoundRUV(Replica *r, RUV **ruv) CL5_MEMORY_ERROR if memory allocation fials. */ int -cl5ExportLDIF(const char *ldifFile, Replica **replicas) +cl5ExportLDIF(const char *ldifFile, Replica *replica) { - int i; - int rc; PRFileDesc *prFile = NULL; - Object *file_obj; + int rc; if (ldifFile == NULL) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, @@ -710,28 +537,17 @@ cl5ExportLDIF(const char *ldifFile, Replica **replicas) slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name_cl, "cl5ExportLDIF: starting changelog export to (%s) ...\n", ldifFile); - if (replicas) /* export only selected files */ - { - for (i = 0; replicas[i]; i++) { - rc = _cl5GetDBFile(replicas[i], &file_obj); - if (rc == CL5_SUCCESS) { - rc = _cl5ExportFile(prFile, file_obj); - object_release(file_obj); - } else { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5ExportLDIF - " - "Failed to locate changelog file for replica at (%s)\n", - slapi_sdn_get_dn(replica_get_root(replicas[i]))); - } - } - } else /* export all files */ + if (replica) /* export only selected files */ { - for (file_obj = objset_first_obj(s_cl5Desc.dbFiles); file_obj; - file_obj = objset_next_obj(s_cl5Desc.dbFiles, file_obj)) { - rc = _cl5ExportFile(prFile, file_obj); + cldb_Handle *cldb = replica_get_file_info(replica); + rc = _cl5ExportFile (prFile, cldb); + if (rc) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5ExportLDIF - " + "failed to locate changelog file for replica at (%s)\n", + slapi_sdn_get_dn (replica_get_root (replica))); } } - rc = CL5_SUCCESS; done:; _cl5RemoveThread(); @@ -759,7 +575,7 @@ done:; CL5_MEMORY_ERROR if memory allocation fials. */ int -cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas) +cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica *replica) { LDIFFP *file = NULL; int buflen = 0; @@ -767,17 +583,13 @@ cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas) int rc; char *buff = NULL; slapi_operation_parameters op; - Replica *prim_replica = NULL; - Replica *replica = NULL; - Object *file_obj = NULL; char *replGen = NULL; - CL5DBFile *dbfile = NULL; - struct berval **purgevals = NULL; - struct berval **maxvals = NULL; int purgeidx = 0; int maxidx = 0; int maxpurgesz = 0; int maxmaxsz = 0; + struct berval **purgevals = NULL; + struct berval **maxvals = NULL; int entryCount = 0; /* validate params */ @@ -793,14 +605,7 @@ cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas) return CL5_BAD_STATE; } - if (replicas == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5ImportLDIF - null list of replicas\n"); - return CL5_BAD_DATA; - } - - prim_replica = replicas[0]; - if (NULL == prim_replica) { + if (NULL == replica) { /* Never happens for now. (see replica_execute_ldif2cl_task) */ slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5ImportLDIF - empty replica list\n"); @@ -830,21 +635,25 @@ cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas) } /* remove changelog */ + /* TBD (LK) remove and recreate cl database */ + /* rc = _cl5Delete(clDir, PR_FALSE); rc = _cl5Delete(clDir, PR_FALSE); if (rc != CL5_SUCCESS) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5ImportLDIF - Failed to remove changelog\n"); goto done; } + */ /* open changelog */ - rc = _cl5Open(clDir, NULL, CL5_OPEN_LDIF2CL); + rc = _cl5Open(CL5_OPEN_LDIF2CL); if (rc != CL5_SUCCESS) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5ImportLDIF - Failed to open changelog\n"); goto done; } s_cl5Desc.dbState = CL5_STATE_OPEN; /* force to change the state */ + cldb_Handle *cldb = replica_get_file_info(replica); /* read entries and write them to changelog */ while (ldif_read_record(file, &lineno, &buff, &buflen)) @@ -908,9 +717,8 @@ cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas) } slapi_ch_free_string(&buff); buflen = 0; - /* if we perform selective import, check if the operation should be wriiten to changelog */ - replica = _cl5GetReplica(&op, replGen); - if (replica == NULL) { + /* check if the operation should be written to changelog */ + if (0 == strcmp(replGen, cldb->ident)) { /* * changetype: delete * replgen: 4d13a124000000010000 @@ -918,8 +726,7 @@ cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas) * nsuniqueid: 00000000-00000000-00000000-00000000 * dn: cn=start iteration */ - rc = _cl5WriteOperation(replica_get_name(prim_replica), - replGen, &op, 1); + rc = _cl5WriteOperation (cldb, &op); if (rc != CL5_SUCCESS) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5ImportLDIF - " @@ -934,52 +741,23 @@ cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas) goto next; } - if (!replicas || _cl5ReplicaInList(replica, replicas)) { - /* write operation creates the file if it does not exist */ - rc = _cl5WriteOperation(replica_get_name(replica), - replGen, &op, 1); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5ImportLDIF - " - "Failed to write operation to the changelog: " - "type: %lu, dn: %s\n", - op.operation_type, REPL_GET_DN(&op.target_address)); - slapi_ch_free_string(&replGen); - operation_parameters_done(&op); - goto done; - } - entryCount++; - } next: slapi_ch_free_string(&replGen); operation_parameters_done(&op); } /* Set RUVs and entry count */ - file_obj = objset_first_obj(s_cl5Desc.dbFiles); - while (file_obj) { - dbfile = (CL5DBFile *)object_get_data(file_obj); - if (0 == strcasecmp(dbfile->replName, replica_get_name(prim_replica))) { - break; - } - dbfile = NULL; - file_obj = objset_next_obj(s_cl5Desc.dbFiles, file_obj); - } - - if (dbfile) { + if (cldb) { if (purgeidx > 0) { - ruv_destroy(&dbfile->purgeRUV); - rc = ruv_init_from_bervals(purgevals, &dbfile->purgeRUV); + ruv_destroy(&cldb->purgeRUV); + rc = ruv_init_from_bervals(purgevals, &cldb->purgeRUV); } if (maxidx > 0) { - ruv_destroy(&dbfile->maxRUV); - rc = ruv_init_from_bervals(maxvals, &dbfile->maxRUV); + ruv_destroy(&cldb->maxRUV); + rc = ruv_init_from_bervals(maxvals, &cldb->maxRUV); } - dbfile->entryCount = entryCount; - } - if (file_obj) { - object_release(file_obj); + cldb->entryCount = entryCount; } done: @@ -1018,60 +796,64 @@ cl5GetState() Description: sets changelog trimming parameters; changelog must be open. Parameters: maxEntries - maximum number of entries in the chnagelog (in all files); maxAge - maximum entry age; - compactInterval - interval to compact changelog db; trimInterval - changelog trimming interval. Return: CL5_SUCCESS if successful; CL5_BAD_STATE if changelog is not open */ int -cl5ConfigTrimming(int maxEntries, const char *maxAge, int compactInterval, int trimInterval) +cl5ConfigTrimming(Replica *replica, int maxEntries, const char *maxAge, int trimInterval) { + int isTrimmingEnabledBefore = 0; + int isTrimmingEnabledAfter = 0; + cldb_Handle *cldb = replica_get_file_info(replica); + if (s_cl5Desc.dbState == CL5_STATE_NONE) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5ConfigTrimming - Changelog is not initialized\n"); return CL5_BAD_STATE; } - /* make sure changelog is not closed while trimming configuration - is updated.*/ - if (CL5_SUCCESS != _cl5AddThread()) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5ConfigTrimming - Could not start changelog trimming thread\n"); - return CL5_BAD_STATE; - } + slapi_counter_increment(cldb->clThreads); + /* make sure changelog is not closed while trimming configuration is updated.*/ - PR_Lock(s_cl5Desc.dbTrim.lock); + PR_Lock(cldb->clLock); + + isTrimmingEnabledBefore = cldb_IsTrimmingEnabled(cldb); if (maxAge) { /* don't ignore this argument */ if (strcmp(maxAge, CL5_STR_IGNORE) != 0) { - s_cl5Desc.dbTrim.maxAge = slapi_parse_duration(maxAge); + cldb->clConf.maxAge = slapi_parse_duration(maxAge); } } else { /* unlimited */ - s_cl5Desc.dbTrim.maxAge = 0; + cldb->clConf.maxAge = 0; } if (maxEntries != CL5_NUM_IGNORE) { - s_cl5Desc.dbTrim.maxEntries = maxEntries; - } - - if (compactInterval != CL5_NUM_IGNORE) { - s_cl5Desc.dbTrim.compactInterval = compactInterval; + cldb->clConf.maxEntries = maxEntries; } if (trimInterval != CL5_NUM_IGNORE) { - s_cl5Desc.dbTrim.trimInterval = trimInterval; + cldb->clConf.trimInterval = trimInterval; } - /* The config was updated, notify the changelog trimming thread */ - PR_Lock(s_cl5Desc.clLock); - PR_NotifyCondVar(s_cl5Desc.clCvar); - PR_Unlock(s_cl5Desc.clLock); + isTrimmingEnabledAfter = cldb_IsTrimmingEnabled(cldb); - PR_Unlock(s_cl5Desc.dbTrim.lock); + if (isTrimmingEnabledAfter && !isTrimmingEnabledBefore) { + /* start trimming */ + cldb_StartTrimming(replica); + } else if (!isTrimmingEnabledAfter && isTrimmingEnabledBefore) { + /* stop trimming */ + cldb_StopTrimming(replica, NULL); + } else { + /* The config was updated, notify the changelog trimming thread */ + PR_NotifyCondVar(cldb->clCvar); + } - _cl5RemoveThread(); + PR_Unlock(cldb->clLock); + + slapi_counter_decrement(cldb->clThreads); return CL5_SUCCESS; } @@ -1093,8 +875,11 @@ cl5DestroyIterator(void *iterator) if (it->cursor) it->cursor->c_close(it->cursor); + /* NOTE (LK) locking of CL files ?*/ + /* if (it->file) object_release(it->file); + */ slapi_ch_free((void **)&it); } @@ -1107,7 +892,6 @@ cl5DestroyIterator(void *iterator) replica object since generation can change while operation is in progress (if the data is reloaded). !!! op - operation to write - local - this is a non-replicated operation txn - the transaction containing this operation Return: CL5_SUCCESS if function is successfull; CL5_BAD_DATA if invalid op is passed; @@ -1116,7 +900,7 @@ cl5DestroyIterator(void *iterator) CL5_DB_ERROR if any other db error occured; */ int -cl5WriteOperationTxn(const char *replName, const char *replGen, const slapi_operation_parameters *op, PRBool local, void *txn) +cl5WriteOperationTxn(cldb_Handle *cldb, const slapi_operation_parameters *op, void *txn) { int rc; @@ -1138,23 +922,16 @@ cl5WriteOperationTxn(const char *replName, const char *replGen, const slapi_oper } /* make sure that changelog is open while operation is in progress */ - rc = _cl5AddThread(); - if (rc != CL5_SUCCESS) - return rc; + slapi_counter_increment(cldb->clThreads); - rc = _cl5WriteOperationTxn(replName, replGen, op, local, txn); + rc = _cl5WriteOperationTxn(cldb, op, txn); /* update the upper bound ruv vector */ if (rc == CL5_SUCCESS) { - Object *file_obj = NULL; - - if (_cl5GetDBFileByReplicaName(replName, replGen, &file_obj) == CL5_SUCCESS) { - rc = _cl5UpdateRUV(file_obj, op->csn, PR_FALSE, PR_FALSE); - object_release(file_obj); - } + rc = _cl5UpdateRUV(cldb, op->csn, PR_FALSE, PR_FALSE); } - _cl5RemoveThread(); + slapi_counter_decrement(cldb->clThreads); return rc; } @@ -1167,7 +944,6 @@ cl5WriteOperationTxn(const char *replName, const char *replGen, const slapi_oper replica object since generation can change while operation is in progress (if the data is reloaded). !!! op - operation to write - local - this is a non-replicated operation Return: CL5_SUCCESS if function is successfull; CL5_BAD_DATA if invalid op is passed; CL5_BAD_STATE if db has not been initialized; @@ -1175,9 +951,9 @@ cl5WriteOperationTxn(const char *replName, const char *replGen, const slapi_oper CL5_DB_ERROR if any other db error occured; */ int -cl5WriteOperation(const char *replName, const char *replGen, const slapi_operation_parameters *op, PRBool local) +cl5WriteOperation(cldb_Handle *cldb, const slapi_operation_parameters *op) { - return cl5WriteOperationTxn(replName, replGen, op, local, NULL); + return cl5WriteOperationTxn(cldb, op, NULL); } /* Name: cl5CreateReplayIterator @@ -1218,7 +994,6 @@ cl5CreateReplayIteratorEx(Private_Repl_Protocol *prp, const RUV *consumerRuv, CL { int rc; Replica *replica; - Object *file_obj = NULL; replica = prp->replica; if (replica == NULL || consumerRuv == NULL || iterator == NULL) { @@ -1240,22 +1015,11 @@ cl5CreateReplayIteratorEx(Private_Repl_Protocol *prp, const RUV *consumerRuv, CL if (rc != CL5_SUCCESS) return rc; - - rc = _cl5GetDBFile(replica, &file_obj); - if (rc == CL5_SUCCESS && file_obj) { - /* iterate through the ruv in csn order to find first master for which - we can replay changes */ - - rc = _cl5PositionCursorForReplay(consumerRID, consumerRuv, replica, file_obj, iterator, NULL); - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5CreateReplayIteratorEx - Could not find DB object for replica\n"); - } + /* iterate through the ruv in csn order to find first master for which + we can replay changes */ + rc = _cl5PositionCursorForReplay (consumerRID, consumerRuv, replica, iterator, NULL); if (rc != CL5_SUCCESS) { - if (file_obj) { - object_release(file_obj); - } /* release the thread */ _cl5RemoveThread(); } @@ -1275,7 +1039,6 @@ cl5CreateReplayIterator(Private_Repl_Protocol *prp, const RUV *consumerRuv, CL5R int rc; Replica *replica; - Object *file_obj = NULL; replica = prp->replica; if (replica == NULL || consumerRuv == NULL || iterator == NULL) { @@ -1297,28 +1060,18 @@ cl5CreateReplayIterator(Private_Repl_Protocol *prp, const RUV *consumerRuv, CL5R if (rc != CL5_SUCCESS) return rc; - - rc = _cl5GetDBFile(replica, &file_obj); - if (rc == CL5_SUCCESS && file_obj) { - /* iterate through the ruv in csn order to find first master for which - we can replay changes */ - ReplicaId consumerRID = agmt_get_consumer_rid(prp->agmt, prp->conn); - int continue_on_missing = agmt_get_ignoremissing(prp->agmt); - int save_cont_miss = continue_on_missing; - rc = _cl5PositionCursorForReplay(consumerRID, consumerRuv, replica, file_obj, iterator, &continue_on_missing); - if (save_cont_miss == 1 && continue_on_missing == 0) { - /* the option to continue once on a missing csn was used, rest */ - agmt_set_ignoremissing(prp->agmt, 0); - } - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5CreateReplayIterator - Could not find DB object for replica\n"); + /* iterate through the ruv in csn order to find first master for which + we can replay changes */ + ReplicaId consumerRID = agmt_get_consumer_rid(prp->agmt, prp->conn); + int continue_on_missing = agmt_get_ignoremissing(prp->agmt); + int save_cont_miss = continue_on_missing; + rc = _cl5PositionCursorForReplay(consumerRID, consumerRuv, replica, iterator, &continue_on_missing); + if (save_cont_miss == 1 && continue_on_missing == 0) { + /* the option to continue once on a missing csn was used, rest */ + agmt_set_ignoremissing(prp->agmt, 0); } if (rc != CL5_SUCCESS) { - if (file_obj) - object_release(file_obj); - /* release the thread */ _cl5RemoveThread(); } @@ -1396,7 +1149,7 @@ cl5GetNextOperationToReplay(CL5ReplayIterator *iterator, CL5Entry *entry) /* there is an entry we should return */ /* Callers of this function should cl5_operation_parameters_done(op) */ - if (0 != cl5DBData2Entry(data, datalen, entry)) { + if (0 != cl5DBData2Entry(data, datalen, entry, iterator->it_cldb->clcrypt_handle)) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "cl5GetNextOperationToReplay - %s - Failed to format entry rc=%d\n", agmt_name, rc); return rc; @@ -1421,10 +1174,12 @@ cl5DestroyReplayIterator(CL5ReplayIterator **iterator) clcache_return_buffer(&(*iterator)->clcache); - if ((*iterator)->fileObj) { - object_release((*iterator)->fileObj); - (*iterator)->fileObj = NULL; + /* TBD (LK) lock/unlock cldb ? + if ((*iterator)->it_cldb) { + object_release((*iterator)->it_cldb); + (*iterator)->it_cldb = NULL; } + */ /* release supplier's ruv */ if ((*iterator)->supplierRuvObj) { @@ -1438,50 +1193,6 @@ cl5DestroyReplayIterator(CL5ReplayIterator **iterator) _cl5RemoveThread(); } -/* Name: cl5DeleteOnClose - Description: marks changelog for deletion when it is closed - Parameters: flag; if flag = 1 then delete else don't - Return: none - */ -void -cl5DeleteOnClose(PRBool rm) -{ - s_cl5Desc.dbRmOnClose = rm; -} - -/* Name: cl5GetDir - Description: returns changelog directory - Parameters: none - Return: copy of the directory; caller needs to free the string - */ -char * -cl5GetDir() -{ - if (s_cl5Desc.dbDir == NULL) { - return NULL; - } else { - return slapi_ch_strdup(s_cl5Desc.dbDir); - } -} - -/* Name: cl5Exist - Description: checks if a changelog exists in the specified directory; - We consider changelog to exist if it contains the dbversion file. - Parameters: clDir - directory to check - Return: 1 - if changelog exists; 0 - otherwise - */ -PRBool -cl5Exist(const char *clDir) -{ - char fName[MAXPATHLEN + 1]; - int rc; - - PR_snprintf(fName, MAXPATHLEN, "%s/%s", clDir, VERSION_FILE); - rc = PR_Access(fName, PR_ACCESS_EXISTS); - - return (rc == PR_SUCCESS); -} - /* Name: cl5GetOperationCount Description: returns number of entries in the changelog. The changelog must be open for the value to be meaningful. @@ -1493,8 +1204,6 @@ cl5Exist(const char *clDir) int cl5GetOperationCount(Replica *replica) { - Object *file_obj; - CL5DBFile *file; int count = 0; int rc; @@ -1511,6 +1220,7 @@ cl5GetOperationCount(Replica *replica) if (replica == NULL) /* compute total entry count */ { + /* TBD (LK) get count for all backends file_obj = objset_first_obj(s_cl5Desc.dbFiles); while (file_obj) { file = (CL5DBFile *)object_get_data(file_obj); @@ -1518,22 +1228,18 @@ cl5GetOperationCount(Replica *replica) count += file->entryCount; file_obj = objset_next_obj(s_cl5Desc.dbFiles, file_obj); } + */ + count = 0; } else /* return count for particular db */ { - /* select correct db file */ - rc = _cl5GetDBFile(replica, &file_obj); - if (rc == CL5_SUCCESS) { - file = (CL5DBFile *)object_get_data(file_obj); - PR_ASSERT(file); - - count = file->entryCount; - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5GetOperationCount - Found DB object %p\n", file_obj); - object_release(file_obj); + cldb_Handle *cldb = replica_get_file_info(replica); + if (cldb) { + count = cldb->entryCount; } else { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "cl5GetOperationCount - Could not get DB object for replica\n"); - count = 0; + /* replica is not enabled */ + count = -1; } } @@ -1545,72 +1251,20 @@ cl5GetOperationCount(Replica *replica) /* this call happens under state lock */ static int -_cl5Open(const char *dir, const CL5DBConfig *config, CL5OpenMode openMode) +_cl5Open(CL5OpenMode openMode) { - int rc; - - PR_ASSERT(dir); - - /* setup db configuration parameters */ - if (config) { - _cl5SetDBConfig(config); - } else { - _cl5SetDefaultDBConfig(); - } - - /* initialize trimming */ - rc = _cl5TrimInit(); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Open - Failed to initialize trimming\n"); - goto done; - } - - /* create the changelog directory if it does not exist */ - rc = cl5CreateDirIfNeeded(dir); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Open - Failed to create changelog directory (%s)\n", dir); - goto done; - } - - if (s_cl5Desc.dbDir) { - slapi_ch_free_string(&s_cl5Desc.dbDir); - } - s_cl5Desc.dbDir = slapi_ch_strdup(dir); - - /* check database version */ - rc = _cl5CheckDBVersion(); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5Open - Invalid db version\n"); - goto done; - } + int rc = CL5_SUCCESS; s_cl5Desc.dbOpenMode = openMode; - /* initialize db environment */ - rc = _cl5AppInit(); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Open - Failed to initialize db environment\n"); - goto done; - } - + replica_enumerate_replicas(cldb_SetReplicaDB, NULL); + /* init the clcache */ - if ((clcache_init(&s_cl5Desc.dbEnv) != 0)) { + if ((clcache_init() != 0)) { rc = CL5_SYSTEM_ERROR; goto done; } - /* open database files */ - rc = _cl5DBOpen(); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Open - Failed to open changelog database\n"); - - goto done; - } - done:; if (rc != CL5_SUCCESS) { @@ -1621,176 +1275,216 @@ done:; } int -cl5CreateDirIfNeeded(const char *dirName) +cldb_UnSetReplicaDB(Replica *replica, void *arg) { - int rc; - char buff[MAXPATHLEN + 1]; - char *t; + int rc = 0; + cldb_Handle *cldb = replica_get_file_info(replica); + Slapi_Backend *be = slapi_be_select(replica_get_root(replica)); + + if (cldb == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cldb_UnSetReplicaDB: cldb is NULL\n"); + return -1; + } - PR_ASSERT(dirName); + cldb->flags &= ~DB_FILE_ACTIVE; - rc = PR_Access(dirName, PR_ACCESS_EXISTS); - if (rc == PR_SUCCESS) { - return CL5_SUCCESS; + /* cleanup trimming */ + cldb_StopThreads(replica, NULL); + + /* write or cleanup changelog ruvs */ + if (s_cl5Desc.dbState == CL5_STATE_CLOSING) { + _cl5WriteReplicaRUV(replica, NULL); + } else { + ruv_destroy(&cldb->maxRUV); + ruv_destroy(&cldb->purgeRUV); } - /* directory does not exist - try to create */ - PL_strncpyz(buff, dirName, sizeof(buff) - 1); - t = strchr(buff, '/'); + if (cldb->clLock != NULL) { + PR_DestroyLock(cldb->clLock); + cldb->clLock = NULL; + } + if (cldb->clCvar != NULL) { + PR_DestroyCondVar(cldb->clCvar); + cldb->clCvar = NULL; + } + /* Clear the cl encryption data (if configured) */ + rc = clcrypt_destroy(cldb->clcrypt_handle, be); - /* skip first slash */ - if (t) { - t = strchr(t + 1, '/'); + if (cldb->flags & ~DB_FILE_DONE) { + _cldb_DeleteDB(replica); } - while (t) { - *t = '\0'; - if (PR_Access(buff, PR_ACCESS_EXISTS) != PR_SUCCESS) { - rc = PR_MkDir(buff, DIR_CREATE_MODE); - if (rc != PR_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5CreateDirIfNeeded - Failed to create dir (%s); NSPR error - %d\n", - dirName, PR_GetError()); - return CL5_SYSTEM_ERROR; - } - } + slapi_counter_destroy(&cldb->clThreads); - *t++ = FILE_PATHSEP; + rc = replica_set_file_info(replica, NULL); + slapi_ch_free_string(&cldb->ident); + slapi_ch_free((void **)&cldb); - t = strchr(t, '/'); - } + return rc; +} - /* last piece */ - rc = PR_MkDir(buff, DIR_CREATE_MODE); - if (rc != PR_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5CreateDirIfNeeded - Failed to create dir; NSPR error - %d\n", +int +cldb_SetReplicaDB(Replica *replica, void *arg) +{ + int rc = -1; + DB *pDB = NULL; + cldb_Handle *cldb = NULL; + + if (!replica_is_flag_set(replica, REPLICA_LOG_CHANGES)) { + /* replica does not have a changelog */ + return 0; + } + cldb = replica_get_file_info(replica); + if (cldb) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cldb_SetReplicaDB - DB already set to replica\n"); + return 0; + } + + Slapi_Backend *be = slapi_be_select(replica_get_root(replica)); + Object *ruv_obj = replica_get_ruv(replica); + + rc = _cldb_CheckAndSetEnv(be); + + rc = slapi_back_get_info(be, BACK_INFO_DBENV_CLDB, (void **)&pDB); + if (rc == 0) { + cldb = (cldb_Handle *)slapi_ch_calloc(1, sizeof(cldb_Handle)); + cldb->db = pDB; + cldb->ident = ruv_get_replica_generation ((RUV*)object_get_data (ruv_obj)); + _cl5ReadRUV(cldb, PR_TRUE); + _cl5ReadRUV(cldb, PR_FALSE); + _cl5GetEntryCount(cldb); + } + object_release(ruv_obj); + + cldb->clThreads = slapi_counter_new(); + + cldb->flags = DB_FILE_ACTIVE; + rc = replica_set_file_info(replica, cldb); + + if ((cldb->clLock = PR_NewLock()) == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cldb_SetReplicaDB - Failed to create on close lock; NSPR error - %d\n", + PR_GetError()); + return CL5_SYSTEM_ERROR; + } + if ((cldb->clCvar = PR_NewCondVar(cldb->clLock)) == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cldb_SetReplicaDB - Failed to create on close cvar; NSPR error - %d\n", PR_GetError()); return CL5_SYSTEM_ERROR; } - return CL5_SUCCESS; + /* get cl configuration for backend */ + back_info_config_entry config_entry = {0}; + config_entry.dn = "cn=changelog"; + changelog5Config config = {}; + rc = slapi_back_ctrl_info(be, BACK_INFO_CLDB_GET_CONFIG, (void *)&config_entry); + if (rc !=0 || config_entry.ce == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cldb_SetReplicaDB - failed to read config for changelog\n"); + return CL5_BAD_DATA; + } + + changelog5_extract_config(config_entry.ce, &config); + changelog5_register_config_callbacks(slapi_entry_get_dn_const(config_entry.ce), replica); + slapi_entry_free(config_entry.ce); + + /* set trimming parameters */ + rc = cl5ConfigTrimming(replica, config.maxEntries, config.maxAge, config.trimInterval); + if (rc != CL5_SUCCESS) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cldb_SetReplicaDB - failed to configure changelog trimming\n"); + return CL5_BAD_DATA; + } + + /* Set the cl encryption algorithm (if configured) */ + if (config.encryptionAlgorithm) { + cldb->clConf.encryptionAlgorithm = config.encryptionAlgorithm; + cldb->clcrypt_handle = clcrypt_init(config.encryptionAlgorithm, be); + } + changelog5_config_done(&config); + + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "cldb_SetReplicaDB: cldb is set\n"); + + return rc; } static int -_cl5AppInit(void) +cldb_IsTrimmingEnabled(cldb_Handle *cldb) { - int rc = -1; /* initialize to failure */ - DB_ENV *dbEnv = NULL; - uint32_t pagesize = 0; - int openflags = 0; - char *cookie = NULL; - Slapi_Backend *be = slapi_get_first_backend(&cookie); - while (be) { - rc = slapi_back_get_info(be, BACK_INFO_DBENV, (void **)&dbEnv); - if ((LDAP_SUCCESS == rc) && dbEnv) { - rc = slapi_back_get_info(be, - BACK_INFO_INDEXPAGESIZE, (void **)&pagesize); - if ((LDAP_SUCCESS == rc) && pagesize) { - rc = slapi_back_get_info(be, - BACK_INFO_DBENV_OPENFLAGS, (void **)&openflags); - if (LDAP_SUCCESS == rc) { - break; /* Successfully fetched */ - } - } - } - be = slapi_get_next_backend(cookie); - } - slapi_ch_free((void **)&cookie); - - if (rc == 0 && dbEnv && pagesize) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5AppInit - Fetched backend dbEnv (%p)\n", dbEnv); - s_cl5Desc.dbEnv = dbEnv; - s_cl5Desc.dbEnvOpenFlags = openflags; - s_cl5Desc.dbConfig.pageSize = pagesize; - return CL5_SUCCESS; + if ((cldb->clConf.maxAge != 0 || cldb->clConf.maxEntries != 0) && cldb->clConf.trimInterval > 0) { + return 1; } else { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5AppInit - Failed to fetch backend dbenv (%p) and/or " - "index page size (%lu)\n", - dbEnv, (long unsigned int)pagesize); - return CL5_DB_ERROR; + return 0; } } -static int -_cl5DBOpen() +int +cldb_StartTrimming(Replica *replica) { - PRBool dbFile; - PRDir *dir; - PRDirEntry *entry = NULL; - int rc = -1; /* initialize to failure */ - Replica *replica; - int count = 0; + return _cl5DispatchTrimThread(replica); +} +int +cldb_StopTrimming(Replica *replica, void *arg) +{ + cldb_Handle *cldb = replica_get_file_info(replica); + + /* we need to stop the changelog threads - trimming or purging */ + PR_Lock(cldb->clLock); + PR_NotifyCondVar(cldb->clCvar); + PR_Unlock(cldb->clLock); + + return 0; +} - /* create lock that guarantees that each file is only added once to the list */ - s_cl5Desc.fileLock = PR_NewLock(); +int +cldb_StopThreads(Replica *replica, void *arg) +{ + cldb_Handle *cldb = replica_get_file_info(replica); + PRIntervalTime interval; + uint64_t threads; + + /* we need to stop the changelog threads - trimming or purging */ + PR_Lock(cldb->clLock); + PR_NotifyCondVar(cldb->clCvar); + PR_Unlock(cldb->clLock); - /* loop over all db files and open them; file name format is cl5_. */ - dir = PR_OpenDir(s_cl5Desc.dbDir); - if (dir == NULL) { + interval = PR_MillisecondsToInterval(100); + while ((threads = slapi_counter_get_value(cldb->clThreads)) > 0) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBOpen - Failed to open changelog dir; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; + "cldb_StopThreads -Waiting for threads to exit: %lu thread(s) still active\n", + threads); + DS_Sleep(interval); } + return 0; +} - /* initialize set of db file objects */ - s_cl5Desc.dbFiles = objset_new(NULL); - while (NULL != (entry = PR_ReadDir(dir, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { - if (NULL == entry->name) { - break; - } +static int +_cldb_CheckAndSetEnv(Slapi_Backend *be) +{ + int rc = -1; /* initialize to failure */ + DB_ENV *dbEnv = NULL; - dbFile = _cl5FileName2Replica(entry->name, &replica); - if (dbFile) /* this is db file, not a log or dbversion; those are just skipped */ - { - /* we only open files for existing replicas */ - if (replica) { - rc = _cl5DBOpenFile(replica, NULL /* file object */, - PR_FALSE /* check for duplicates */); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBOpen - " - "Error opening file %s\n", - entry->name); - return rc; - } - count++; - } else /* there is no matching replica for the file - remove */ - { - char fullpathname[MAXPATHLEN]; - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBOpen - " - "File %s has no matching replica; removing\n", - entry->name); - - PR_snprintf(fullpathname, MAXPATHLEN, "%s/%s", s_cl5Desc.dbDir, entry->name); - rc = s_cl5Desc.dbEnv->dbremove(s_cl5Desc.dbEnv, - 0, fullpathname, 0, - DEFAULT_DB_ENV_OP_FLAGS); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBOpen - Failed to remove (%s) file; " - "libdb error - %d (%s)\n", - fullpathname, rc, db_strerror(rc)); - if (PR_Delete(fullpathname) != PR_SUCCESS) { - PRErrorCode prerr = PR_GetError(); - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5DBOpen - Failed to remove (%s) file; " - "nspr error - %d (%s)\n", - fullpathname, prerr, slapd_pr_strerror(prerr)); - } - } - } - } + if (s_cl5Desc.dbEnv) { + /* dbEnv already set */ + return CL5_SUCCESS; } - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBOpen - " - "Opened %d existing databases in %s\n", - count, s_cl5Desc.dbDir); - PR_CloseDir(dir); + rc = slapi_back_get_info(be, BACK_INFO_DBENV, (void **)&dbEnv); - return CL5_SUCCESS; + if (rc == 0 && dbEnv) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "_cldb_CheckAndSetEnv - Fetched backend dbEnv (%p)\n", dbEnv); + s_cl5Desc.dbEnv = dbEnv; + return CL5_SUCCESS; + } else { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cldb_CheckAndSetEnv - Failed to fetch backend dbenv\n"); + return CL5_DB_ERROR; + } } /* this function assumes that the entry was validated @@ -1808,7 +1502,7 @@ _cl5DBOpen() <4 byte value size><4 byte value size> */ static int -_cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len) +_cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len, void *clcrypt_handle) { int size = 1 /* version */ + 1 /* operation type */ + sizeof(time_t); char *pos; @@ -1835,7 +1529,7 @@ _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len) slapi_entry2mods(op->p.p_add.target_entry, &rawDN /* dn */, &add_mods); size += strlen(rawDN) + 1; /* Need larger buffer for the encrypted changelog */ - if (s_cl5Desc.clcrypt_handle) { + if (clcrypt_handle) { size += (_cl5GetModsSize(add_mods) * (1 + BACK_CRYPT_OUTBUFF_EXTLEN)); } else { size += _cl5GetModsSize(add_mods); @@ -1845,7 +1539,7 @@ _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len) case SLAPI_OPERATION_MODIFY: size += REPL_GET_DN_LEN(&op->target_address) + 1; /* Need larger buffer for the encrypted changelog */ - if (s_cl5Desc.clcrypt_handle) { + if (clcrypt_handle) { size += (_cl5GetModsSize(op->p.p_modify.modify_mods) * (1 + BACK_CRYPT_OUTBUFF_EXTLEN)); } else { size += _cl5GetModsSize(op->p.p_modify.modify_mods); @@ -1865,7 +1559,7 @@ _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len) else size++; /* for NULL char */ /* Need larger buffer for the encrypted changelog */ - if (s_cl5Desc.clcrypt_handle) { + if (clcrypt_handle) { size += (_cl5GetModsSize(op->p.p_modrdn.modrdn_mods) * (1 + BACK_CRYPT_OUTBUFF_EXTLEN)); } else { size += _cl5GetModsSize(op->p.p_modrdn.modrdn_mods); @@ -1907,14 +1601,14 @@ _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len) case SLAPI_OPERATION_ADD: _cl5WriteString(op->p.p_add.parentuniqueid, &pos); _cl5WriteString(rawDN, &pos); - _cl5WriteMods(add_mods, &pos); + _cl5WriteMods(add_mods, &pos, clcrypt_handle); slapi_ch_free((void **)&rawDN); ldap_mods_free(add_mods, 1); break; case SLAPI_OPERATION_MODIFY: _cl5WriteString(REPL_GET_DN(&op->target_address), &pos); - _cl5WriteMods(op->p.p_modify.modify_mods, &pos); + _cl5WriteMods(op->p.p_modify.modify_mods, &pos, clcrypt_handle); break; case SLAPI_OPERATION_MODRDN: @@ -1924,7 +1618,7 @@ _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len) pos++; _cl5WriteString(REPL_GET_DN(&op->p.p_modrdn.modrdn_newsuperior_address), &pos); _cl5WriteString(op->p.p_modrdn.modrdn_newsuperior_address.uniqueid, &pos); - _cl5WriteMods(op->p.p_modrdn.modrdn_mods, &pos); + _cl5WriteMods(op->p.p_modrdn.modrdn_mods, &pos, clcrypt_handle); break; case SLAPI_OPERATION_DELETE: @@ -1960,7 +1654,7 @@ _cl5Entry2DBData(const CL5Entry *entry, char **data, PRUint32 *len) int -cl5DBData2Entry(const char *data, PRUint32 len __attribute__((unused)), CL5Entry *entry) +cl5DBData2Entry(const char *data, PRUint32 len __attribute__((unused)), CL5Entry *entry, void *clcrypt_handle) { int rc; PRUint8 version; @@ -2016,7 +1710,7 @@ cl5DBData2Entry(const char *data, PRUint32 len __attribute__((unused)), CL5Entry _cl5ReadString(&rawDN, &pos); op->target_address.sdn = slapi_sdn_new_dn_passin(rawDN); /* convert mods to entry */ - rc = _cl5ReadMods(&add_mods, &pos); + rc = _cl5ReadMods(&add_mods, &pos, clcrypt_handle); slapi_mods2entry(&(op->p.p_add.target_entry), rawDN, add_mods); ldap_mods_free(add_mods, 1); break; @@ -2024,7 +1718,7 @@ cl5DBData2Entry(const char *data, PRUint32 len __attribute__((unused)), CL5Entry case SLAPI_OPERATION_MODIFY: _cl5ReadString(&rawDN, &pos); op->target_address.sdn = slapi_sdn_new_dn_passin(rawDN); - rc = _cl5ReadMods(&op->p.p_modify.modify_mods, &pos); + rc = _cl5ReadMods(&op->p.p_modify.modify_mods, &pos, clcrypt_handle); break; case SLAPI_OPERATION_MODRDN: @@ -2036,7 +1730,7 @@ cl5DBData2Entry(const char *data, PRUint32 len __attribute__((unused)), CL5Entry _cl5ReadString(&rawDN, &pos); op->p.p_modrdn.modrdn_newsuperior_address.sdn = slapi_sdn_new_dn_passin(rawDN); _cl5ReadString(&op->p.p_modrdn.modrdn_newsuperior_address.uniqueid, &pos); - rc = _cl5ReadMods(&op->p.p_modrdn.modrdn_mods, &pos); + rc = _cl5ReadMods(&op->p.p_modrdn.modrdn_mods, &pos, clcrypt_handle); break; case SLAPI_OPERATION_DELETE: @@ -2057,17 +1751,17 @@ cl5DBData2Entry(const char *data, PRUint32 len __attribute__((unused)), CL5Entry /* thread management functions */ static int -_cl5DispatchDBThreads(void) +_cl5DispatchTrimThread(Replica *replica) { PRThread *pth = NULL; pth = PR_CreateThread(PR_USER_THREAD, (VFP)(void *)_cl5TrimMain, - NULL, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + (void *)replica, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_UNJOINABLE_THREAD, DEFAULT_THREAD_STACKSIZE); if (NULL == pth) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DispatchDBThreads - Failed to create trimming thread" - "; NSPR error - %d\n", + "_cl5DispatchTrimThread - Failed to create trimming thread for %s" + "; NSPR error - %d\n", replica_get_name(replica), PR_GetError()); return CL5_SYSTEM_ERROR; } @@ -2151,7 +1845,7 @@ _cl5ReadString(char **str, char **buff) <4 byte size><4 byte size>... */ static void -_cl5WriteMods(LDAPMod **mods, char **buff) +_cl5WriteMods(LDAPMod **mods, char **buff, void *clcrypt_handle) { PRInt32 i; char *mod_start; @@ -2165,7 +1859,7 @@ _cl5WriteMods(LDAPMod **mods, char **buff) /* write mods*/ for (i = 0; mods[i]; i++) { - if (0 <= _cl5WriteMod(mods[i], &mod_start)) { + if (0 <= _cl5WriteMod(mods[i], &mod_start, clcrypt_handle)) { count++; } } @@ -2183,7 +1877,7 @@ _cl5WriteMods(LDAPMod **mods, char **buff) * netative: failed to encrypt && no write to the changelog */ static int -_cl5WriteMod(LDAPMod *mod, char **buff) +_cl5WriteMod(LDAPMod *mod, char **buff, void *clcrypt_handle) { char *orig_pos; char *pos; @@ -2226,7 +1920,7 @@ _cl5WriteMod(LDAPMod *mod, char **buff) bv = slapi_mod_get_first_value(&smod); while (bv) { encbv = NULL; - rc = clcrypt_encrypt_value(s_cl5Desc.clcrypt_handle, + rc = clcrypt_encrypt_value(clcrypt_handle, bv, &encbv); if (rc > 0) { /* no encryption needed. use the original bv */ @@ -2271,7 +1965,7 @@ _cl5WriteMod(LDAPMod *mod, char **buff) */ static int -_cl5ReadMods(LDAPMod ***mods, char **buff) +_cl5ReadMods(LDAPMod ***mods, char **buff, void *clcrypt_handle) { char *pos = *buff; int i; @@ -2289,7 +1983,7 @@ _cl5ReadMods(LDAPMod ***mods, char **buff) slapi_mods_init(&smods, mod_count); for (i = 0; i < mod_count; i++) { - rc = _cl5ReadMod(&smod, &pos); + rc = _cl5ReadMod(&smod, &pos, clcrypt_handle); if (rc != CL5_SUCCESS) { slapi_mods_done(&smods); return rc; @@ -2307,7 +2001,7 @@ _cl5ReadMods(LDAPMod ***mods, char **buff) } static int -_cl5ReadMod(Slapi_Mod *smod, char **buff) +_cl5ReadMod(Slapi_Mod *smod, char **buff, void *clcrypt_handle) { char *pos = *buff; int i; @@ -2338,7 +2032,7 @@ _cl5ReadMod(Slapi_Mod *smod, char **buff) _cl5ReadBerval(&bv, &pos); decbv = NULL; rc = 0; - rc = clcrypt_decrypt_value(s_cl5Desc.clcrypt_handle, + rc = clcrypt_decrypt_value(clcrypt_handle, &bv, &decbv); if (rc > 0) { /* not encrypted. use the original bv */ @@ -2351,11 +2045,11 @@ _cl5ReadMod(Slapi_Mod *smod, char **buff) char *encend = encstr + 128; char *ptr; int i; - for (i = 0, ptr = encstr; (i < bv.bv_len) && (ptr < encend - 4); + for (i = 0, ptr = encstr; (i < bv.bv_len) && (ptr < encend - 6); i++, ptr += 3) { sprintf(ptr, "%x", 0xff & bv.bv_val[i]); } - if (ptr >= encend - 4) { + if (ptr >= encend - 6) { sprintf(ptr, "..."); ptr += 3; } @@ -2544,7 +2238,7 @@ _cl5WriteBervals(struct berval **bv, char **buff, u_int32_t *size) static int32_t _cl5CheckCSNinCL(const ruv_enum_data *element, void *arg) { - CL5DBFile *file = (CL5DBFile *)arg; + cldb_Handle *cldb = (cldb_Handle *)arg; int rc = 0; DBT key = {0}, data = {0}; @@ -2556,903 +2250,313 @@ _cl5CheckCSNinCL(const ruv_enum_data *element, void *arg) data.flags = DB_DBT_MALLOC; - rc = file->db->get(file->db, NULL /*txn*/, &key, &data, 0); + rc = cldb->db->get(cldb->db, NULL /*txn*/, &key, &data, 0); slapi_ch_free(&(data.data)); return rc; } static int32_t -_cl5CheckMaxRUV(CL5DBFile *file, RUV *maxruv) +_cl5CheckMaxRUV(cldb_Handle *cldb, RUV *maxruv) { int rc = 0; - rc = ruv_enumerate_elements(maxruv, _cl5CheckCSNinCL, (void *)file); + rc = ruv_enumerate_elements(maxruv, _cl5CheckCSNinCL, (void *)cldb); return rc; } -/* upgrade from db33 to db41 - * 1. Run recovery on the database environment using the DB_ENV->open method - * 2. Remove any Berkeley DB environment using the DB_ENV->remove method - * 3. Remove any Berkeley DB transaction log files - * 4. extention .db3 -> .db4 - */ -static int -_cl5UpgradeMajor(char *fromVersion, char *toVersion) + +/* must be called under the state lock */ +static void +_cl5Close(void) { - PRDir *dir = NULL; - PRDirEntry *entry = NULL; - DB *thisdb = NULL; - CL5OpenMode backup; - int rc = 0; + + if (s_cl5Desc.dbState != CL5_STATE_CLOSED) /* Don't try to close twice */ + { + + /* cl5Close() set the state flag to CL5_STATE_CLOSING, which should + trigger all of the db housekeeping threads to exit, and which will + eventually cause no new update threads to start - so we wait here + for those other threads to finish before we proceed */ + /* Stopping and waiting for threads to complet is now done for each + * replica in cldb_UnSetReplicaDB + */ - backup = s_cl5Desc.dbOpenMode; - s_cl5Desc.dbOpenMode = CL5_OPEN_CLEAN_RECOVER; - /* CL5_OPEN_CLEAN_RECOVER does 1 and 2 */ - rc = _cl5AppInit(); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5UpgradeMajor - Failed to open the db env\n"); - s_cl5Desc.dbOpenMode = backup; - return rc; + replica_enumerate_replicas(cldb_UnSetReplicaDB, NULL); + + /* There should now be no threads accessing any of the changelog databases - + it is safe to remove those databases */ + _cl5DBClose(); + } - s_cl5Desc.dbOpenMode = backup; +} - dir = PR_OpenDir(s_cl5Desc.dbDir); - if (dir == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5UpgradeMajor - Failed to open changelog dir %s; NSPR error - %d\n", - s_cl5Desc.dbDir, PR_GetError()); - goto out; - } +static void +_cl5DBClose(void) +{ + replica_enumerate_replicas(_cl5WriteReplicaRUV, NULL); +} - while (NULL != (entry = PR_ReadDir(dir, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { - if (NULL == entry->name) { - break; - } - if (_cl5FileEndsWith(entry->name, DB_EXTENSION_DB3) || - _cl5FileEndsWith(entry->name, DB_EXTENSION_DB4)) { - char oName[MAXPATHLEN + 1]; - char nName[MAXPATHLEN + 1]; - char *p = NULL; - char c; - int baselen = 0; - PR_snprintf(oName, MAXPATHLEN, "%s/%s", s_cl5Desc.dbDir, entry->name); - p = strstr(oName, DB_EXTENSION_DB3); - if (NULL == p) { - p = strstr(oName, DB_EXTENSION_DB4); - if (NULL == p) { - continue; - } - } +static int +_cl5TrimMain(void *param) +{ + time_t timePrev = slapi_current_utc_time(); + time_t timeNow; + Replica *replica = (Replica *)param; + cldb_Handle *cldb = replica_get_file_info(replica); - /* db->rename closes DB; need to create every time */ - rc = db_create(&thisdb, s_cl5Desc.dbEnv, 0); - if (0 != rc) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5UpgradeMajor - Failed to get db handle\n"); - goto out; - } + slapi_counter_increment(cldb->clThreads); - baselen = p - oName; - c = *p; - *p = '\0'; - PR_snprintf(nName, MAXPATHLEN + 1, "%s", oName); - PR_snprintf(nName + baselen, MAXPATHLEN + 1 - baselen, "%s", DB_EXTENSION); - *p = c; - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5UpgradeMajor - Renaming %s to %s\n", oName, nName); - rc = thisdb->rename(thisdb, (const char *)oName, NULL /* subdb */, - (const char *)nName, 0); - if (rc != PR_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5UpgradeMajor - Failed to rename file (%s -> %s); " - "db error - %d %s\n", - oName, nName, rc, db_strerror(rc)); - break; - } + while ((s_cl5Desc.dbState != CL5_STATE_CLOSING) && + (cldb->flags & DB_FILE_ACTIVE)){ + timeNow = slapi_current_utc_time(); + if (timeNow - timePrev >= cldb->clConf.trimInterval) { + /* time to trim */ + timePrev = timeNow; + _cl5TrimReplica(replica); } + if (NULL == cldb->clLock) { + /* most likely, emergency */ + break; + } + + PR_Lock(cldb->clLock); + PR_WaitCondVar(cldb->clCvar, PR_SecondsToInterval(cldb->clConf.trimInterval)); + PR_Unlock(cldb->clLock); } - /* update the version file */ - _cl5WriteDBVersion(); - slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name_cl, - "_cl5UpgradeMajor - Upgrading from %s to %s is successfully done (%s)\n", - fromVersion, toVersion, s_cl5Desc.dbDir); -out: - if (NULL != dir) { - PR_CloseDir(dir); - } - return rc; + + slapi_counter_decrement(cldb->clThreads); + + return 0; } -/* upgrade from db41 -> db42 -> db43 -> db44 -> db45 - * 1. Run recovery on the database environment using the DB_ENV->open method - * 2. Remove any Berkeley DB environment using the DB_ENV->remove method - * 3. Remove any Berkeley DB transaction log files +/* + * We remove an entry if it has been replayed to all consumers and the number + * of entries in the changelog is larger than maxEntries or age of the entry + * is larger than maxAge. Also we can't purge entries which correspond to max + * csns in the supplier's ruv. Here is a example where we can get into trouble: + * + * The server is setup with time based trimming and no consumer's + * At some point all the entries are trimmed from the changelog. + * At a later point a consumer is added and initialized online. + * Then a change is made on the supplier. + * To update the consumer, the supplier would attempt to locate the last + * change sent to the consumer in the changelog and will fail because the + * change was removed. */ -static int -_cl5UpgradeMinor(char *fromVersion, char *toVersion) +/* + * We are purging a changelog after a cleanAllRUV task. Find the specific + * changelog for the backend that is being cleaned, and purge all the records + * with the cleaned rid. + */ +static void +_cl5DoPurging(cleanruv_purge_data *purge_data) { - CL5OpenMode backup; - int rc = 0; - - backup = s_cl5Desc.dbOpenMode; - s_cl5Desc.dbOpenMode = CL5_OPEN_CLEAN_RECOVER; - /* CL5_OPEN_CLEAN_RECOVER does 1 and 2 */ - rc = _cl5AppInit(); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5UpgradeMinor - Failed to open the db env\n"); - return rc; - } - s_cl5Desc.dbOpenMode = backup; - - /* update the version file */ - _cl5WriteDBVersion(); - slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name_cl, - "_cl5UpgradeMinor - Upgrading from %s to %s is successfully done (%s)\n", - fromVersion, toVersion, s_cl5Desc.dbDir); + ReplicaId rid = purge_data->cleaned_rid; + const Slapi_DN *suffix_sdn = purge_data->suffix_sdn; + cldb_Handle *cldb = replica_get_file_info(purge_data->replica); - return rc; + PR_Lock (cldb->clLock); + _cl5PurgeRID (cldb, rid); + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "_cl5DoPurging - Purged rid (%d) from suffix (%s)\n", + rid, slapi_sdn_get_dn(suffix_sdn)); + PR_Unlock (cldb->clLock); + return; } +/* + * If the rid is not set it is the very first iteration of the changelog. + * If the rid is set, we are doing another pass, and we have a key as our + * starting point. + */ static int -_cl5CheckDBVersion(void) +_cl5PurgeGetFirstEntry(cldb_Handle *cldb, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key) { - char clVersion[VERSION_SIZE + 1]; - char dbVersion[VERSION_SIZE + 1]; + DBC *cursor = NULL; + DBT data = {0}; + CL5Iterator *it; int rc; - if (!cl5Exist(s_cl5Desc.dbDir)) { - /* this is new changelog - write DB version and guardian file */ - rc = _cl5WriteDBVersion(); - } else { - char *versionp = NULL; - char *versionendp = NULL; - char *dotp = NULL; - int dbmajor = 0; - int dbminor = 0; - - PR_snprintf(clVersion, VERSION_SIZE, "%s/%d.%d/%s", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, BDB_REPLPLUGIN); - - rc = _cl5ReadDBVersion(s_cl5Desc.dbDir, dbVersion, sizeof(dbVersion)); + /* create cursor */ + rc = cldb->db->cursor(cldb->db, txnid, &cursor, 0); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5PurgeGetFirstEntry - Failed to create cursor; db error - %d %s\n", rc, db_strerror(rc)); + rc = CL5_DB_ERROR; + goto done; + } - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CheckDBVersion - Invalid dbversion\n"); - rc = CL5_BAD_DBVERSION; - goto bailout; - } - versionendp = dbVersion + strlen(dbVersion); - /* get the version number */ - /* old DBVERSION string: CL5_TYPE/REPL_PLUGIN_NAME/#.# */ - if (PL_strncmp(dbVersion, CL5_TYPE, strlen(CL5_TYPE)) == 0) { - versionp = strrchr(dbVersion, '/'); - } - /* new DBVERSION string: bdb/#.#/libreplication-plugin */ - else if (PL_strncmp(dbVersion, BDB_IMPL, strlen(BDB_IMPL)) == 0) { - versionp = strchr(dbVersion, '/'); - } - if (NULL == versionp || versionp == versionendp) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CheckDBVersion - Invalid dbversion: %s\n", dbVersion); - rc = CL5_BAD_DBVERSION; - goto bailout; - } - dotp = strchr(++versionp, '.'); - if (NULL != dotp) { - *dotp = '\0'; - dbmajor = strtol(versionp, (char **)NULL, 10); - dbminor = strtol(dotp + 1, (char **)NULL, 10); - *dotp = '.'; - } else { - dbmajor = strtol(versionp, (char **)NULL, 10); + key->flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + while ((rc = cursor->c_get(cursor, key, &data, rid ? DB_SET : DB_NEXT)) == 0) { + /* skip service entries on the first pass (rid == 0)*/ + if (!rid && cl5HelperEntry((char *)key->data, NULL)) { + slapi_ch_free(&key->data); + slapi_ch_free(&(data.data)); + continue; } - if (dbmajor < DB_VERSION_MAJOR) { - /* upgrade */ - rc = _cl5UpgradeMajor(dbVersion, clVersion); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CheckDBVersion - Upgrade %s -> %s failed\n", - dbVersion, clVersion); - rc = CL5_BAD_DBVERSION; - } - } else if (dbminor < DB_VERSION_MINOR) { - /* minor upgrade */ - rc = _cl5UpgradeMinor(dbVersion, clVersion); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CheckDBVersion - Upgrade %s -> %s failed\n", - dbVersion, clVersion); - rc = CL5_BAD_DBVERSION; - } + /* format entry */ + rc = cl5DBData2Entry(data.data, data.size, entry, cldb->clcrypt_handle); + slapi_ch_free(&(data.data)); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "_cl5PurgeGetFirstEntry - Failed to format entry: %d\n", rc); + goto done; } - } -bailout: - return rc; -} -static int -_cl5ReadDBVersion(const char *dir, char *clVersion, int buflen) -{ - int rc; - PRFileDesc *file; - char fName[MAXPATHLEN + 1]; - char buff[BUFSIZ]; - PRInt32 size; - char *tok; - char *iter = NULL; + it = (CL5Iterator *)slapi_ch_malloc(sizeof(CL5Iterator)); + it->cursor = cursor; + /* TBD do we need to lock the file in the iterator ?? */ + /* object_acquire (obj); */ + it->it_cldb = cldb; + *(CL5Iterator **)iterator = it; - if (clVersion) { - clVersion[0] = '\0'; + return CL5_SUCCESS; } - PR_snprintf(fName, MAXPATHLEN, "%s/%s", dir, VERSION_FILE); + slapi_ch_free(&key->data); + slapi_ch_free(&(data.data)); - file = PR_Open(fName, PR_RDONLY, 777); - if (file == NULL) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5ReadDBVersion - Failed to open DBVERSION; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; + /* walked of the end of the file */ + if (rc == DB_NOTFOUND) { + rc = CL5_NOTFOUND; + goto done; } - size = slapi_read_buffer(file, buff, BUFSIZ); - if (size < 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5ReadDBVersion - Failed to read DBVERSION; NSPR error - %d\n", - PR_GetError()); - PR_Close(file); - return CL5_SYSTEM_ERROR; - } + /* db error occured while iterating */ + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5PurgeGetFirstEntry - Failed to get entry; db error - %d %s\n", + rc, db_strerror(rc)); + rc = CL5_DB_ERROR; - /* parse the data */ - buff[size] = '\0'; - tok = ldap_utf8strtok_r(buff, "\n", &iter); - if (tok) { - if (clVersion) { - PL_strncpyz(clVersion, tok, buflen); - } - } - - rc = PR_Close(file); - if (rc != PR_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5ReadDBVersion - Failed to close DBVERSION; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; - } - - return CL5_SUCCESS; -} - -static int -_cl5WriteDBVersion(void) -{ - int rc; - PRFileDesc *file; - char fName[MAXPATHLEN + 1]; - char clVersion[VERSION_SIZE + 1]; - PRInt32 len, size; - - PR_snprintf(fName, MAXPATHLEN, "%s/%s", s_cl5Desc.dbDir, VERSION_FILE); - - file = PR_Open(fName, PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE, - s_cl5Desc.dbConfig.fileMode); - if (file == NULL) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5WriteDBVersion - Failed to open DBVERSION; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; - } - - /* write changelog version */ - PR_snprintf(clVersion, VERSION_SIZE, "%s/%d.%d/%s\n", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, BDB_REPLPLUGIN); - - len = strlen(clVersion); - size = slapi_write_buffer(file, clVersion, len); - if (size != len) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5WriteDBVersion - Failed to write DBVERSION; NSPR error - %d\n", - PR_GetError()); - PR_Close(file); - return CL5_SYSTEM_ERROR; - } - - rc = PR_Close(file); - if (rc != PR_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5WriteDBVersion - Failed to close DBVERSION; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; - } - - return CL5_SUCCESS; -} - -/* must be called under the state lock */ -static void -_cl5Close(void) -{ - PRIntervalTime interval; - - if (s_cl5Desc.dbState != CL5_STATE_CLOSED) /* Don't try to close twice */ - { - - /* cl5Close() set the state flag to CL5_STATE_CLOSING, which should - trigger all of the db housekeeping threads to exit, and which will - eventually cause no new update threads to start - so we wait here - for those other threads to finish before we proceed */ - interval = PR_MillisecondsToInterval(100); - while (s_cl5Desc.threadCount > 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Close -Waiting for threads to exit: %d thread(s) still active\n", - s_cl5Desc.threadCount); - DS_Sleep(interval); - } - - /* There should now be no threads accessing any of the changelog databases - - it is safe to remove those databases */ - _cl5DBClose(); - - /* cleanup trimming */ - _cl5TrimCleanup(); - - /* remove changelog if requested */ - if (s_cl5Desc.dbRmOnClose) { - - if (_cl5Delete(s_cl5Desc.dbDir, 1) != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5Close - Failed to remove changelog\n"); - } - s_cl5Desc.dbRmOnClose = PR_FALSE; - } - - slapi_ch_free((void **)&s_cl5Desc.dbDir); - memset(&s_cl5Desc.dbConfig, 0, sizeof(s_cl5Desc.dbConfig)); - s_cl5Desc.fatalError = PR_FALSE; - s_cl5Desc.threadCount = 0; - s_cl5Desc.dbOpenMode = CL5_OPEN_NONE; - } -} - -static void -_cl5DBClose(void) -{ - if (NULL != s_cl5Desc.dbFiles) { - Object *obj; - for (obj = objset_first_obj(s_cl5Desc.dbFiles); obj; - obj = objset_next_obj(s_cl5Desc.dbFiles, obj)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBClose - Deleting DB object %p\n", obj); - } - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBClose - Closing databases in %s\n", s_cl5Desc.dbDir); - objset_delete(&s_cl5Desc.dbFiles); - } - if (NULL != s_cl5Desc.fileLock) { - PR_DestroyLock(s_cl5Desc.fileLock); - s_cl5Desc.fileLock = NULL; - } -} - -/* see if the given file is a changelog db file */ -static int -_cl5IsDbFile(const char *fname) -{ - if (!fname || !*fname) { - return 0; - } - - if (!strcmp(fname, VERSION_FILE)) { - return 1; - } - - if (_cl5FileEndsWith(fname, DB_EXTENSION)) { - return 1; - } +done: + /* + * We didn't success in assigning this cursor to the iterator, + * so we need to free the cursor here. + */ + if (cursor) + cursor->c_close(cursor); - return 0; /* not a filename we recognize as being associated with the db */ + return rc; } -/* state lock must be locked */ +/* + * Get the next entry. If we get a lock error we will restart the process + * starting at the current key. + */ static int -_cl5Delete(const char *clDir, int rmDir) +_cl5PurgeGetNextEntry(CL5Entry *entry, void *iterator, DBT *key) { - PRDir *dir; - char filename[MAXPATHLEN + 1]; - PRDirEntry *entry = NULL; + CL5Iterator *it; + DBT data = {0}; int rc; - int dirisempty = 1; - /* remove all files in the directory and the directory */ - dir = PR_OpenDir(clDir); - if (dir == NULL) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Delete - Failed to open changelog dir; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; - } + it = (CL5Iterator *)iterator; - while (NULL != (entry = PR_ReadDir(dir, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { - if (NULL == entry->name) { - break; - } - if (!_cl5IsDbFile(entry->name)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Delete - Skipping file [%s/%s] because it is not a changelogdb file.\n", - clDir, entry->name); - dirisempty = 0; /* skipped at least one file - dir not empty */ + key->flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + while ((rc = it->cursor->c_get(it->cursor, key, &data, DB_NEXT)) == 0) { + if (cl5HelperEntry((char *)key->data, NULL)) { + slapi_ch_free(&key->data); + slapi_ch_free(&(data.data)); continue; } - PR_snprintf(filename, MAXPATHLEN, "%s/%s", clDir, entry->name); - /* _cl5Delete deletes the whole changelog directory with all the files - * underneath. Thus, we can just remove them physically. */ - if (0 == strcmp(entry->name, VERSION_FILE)) { - /* DBVERSION */ - rc = PR_Delete(filename) != PR_SUCCESS; - if (PR_SUCCESS != rc) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5Delete - Failed to remove \"%s\"; NSPR error - %d\n", - filename, PR_GetError()); - } - } else { - /* DB files */ - rc = s_cl5Desc.dbEnv->dbremove(s_cl5Desc.dbEnv, 0, filename, 0, - DEFAULT_DB_ENV_OP_FLAGS); - if (rc) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5Delete - Failed to remove \"%s\"; " - "libdb error - %d (%s)\n", - filename, rc, db_strerror(rc)); - } - } - } - - rc = PR_CloseDir(dir); - if (rc != PR_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Delete - Failed to close changelog dir (%s); NSPR error - %d\n", - clDir, PR_GetError()); - return CL5_SYSTEM_ERROR; - } - if (rmDir && dirisempty) { - rc = PR_RmDir(clDir); + /* format entry */ + rc = cl5DBData2Entry(data.data, data.size, entry, it->it_cldb->clcrypt_handle); + slapi_ch_free(&(data.data)); if (rc != 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Delete - Failed to remove changelog dir (%s); errno = %d\n", - clDir, errno); - return CL5_SYSTEM_ERROR; + if (rc != CL5_DB_LOCK_ERROR) { + /* Not a lock error, free the key */ + slapi_ch_free(&key->data); + } + slapi_log_err(rc == CL5_DB_LOCK_ERROR ? SLAPI_LOG_REPL : SLAPI_LOG_ERR, + repl_plugin_name_cl, + "_cl5PurgeGetNextEntry - Failed to format entry: %d\n", + rc); } - } else if (rmDir && !dirisempty) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5Delete - Changelog dir (%s) is not empty - cannot remove\n", - clDir); - } - - /* invalidate the clcache */ - clcache_destroy(); - - return CL5_SUCCESS; -} - -static void -_cl5SetDefaultDBConfig(void) -{ - s_cl5Desc.dbConfig.fileMode = FILE_CREATE_MODE; -} - -static void -_cl5SetDBConfig(const CL5DBConfig *config) -{ - /* s_cl5Desc.dbConfig.pageSize is retrieved from backend */ - /* Some other configuration parameters are hardcoded... */ - s_cl5Desc.dbConfig.fileMode = FILE_CREATE_MODE; -} - -/* Trimming helper functions */ -static int -_cl5TrimInit(void) -{ - /* just create the lock while we are singlethreaded */ - s_cl5Desc.dbTrim.lock = PR_NewLock(); - if (s_cl5Desc.dbTrim.lock == NULL) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5InitTrimming - Failed to create lock; NSPR error - %d\n", - PR_GetError()); - return CL5_SYSTEM_ERROR; - } else { - return CL5_SUCCESS; + return rc; } -} - -static void -_cl5TrimCleanup(void) -{ - if (s_cl5Desc.dbTrim.lock) - PR_DestroyLock(s_cl5Desc.dbTrim.lock); - - memset(&s_cl5Desc.dbTrim, 0, sizeof(s_cl5Desc.dbTrim)); -} - -static int -_cl5TrimMain(void *param __attribute__((unused))) -{ - time_t timePrev = slapi_current_utc_time(); - time_t timeCompactPrev = slapi_current_utc_time(); - time_t timeNow; - - PR_AtomicIncrement(&s_cl5Desc.threadCount); - - while (s_cl5Desc.dbState != CL5_STATE_CLOSING) { - timeNow = slapi_current_utc_time(); - if (timeNow - timePrev >= s_cl5Desc.dbTrim.trimInterval) { - /* time to trim */ - timePrev = timeNow; - _cl5DoTrimming(); - } - if ((s_cl5Desc.dbTrim.compactInterval > 0) && - (timeNow - timeCompactPrev >= s_cl5Desc.dbTrim.compactInterval)) { - /* time to trim */ - timeCompactPrev = timeNow; - _cl5CompactDBs(); - } - if (NULL == s_cl5Desc.clLock) { - /* most likely, emergency */ - break; - } + slapi_ch_free(&(data.data)); - PR_Lock(s_cl5Desc.clLock); - PR_WaitCondVar(s_cl5Desc.clCvar, PR_SecondsToInterval(s_cl5Desc.dbTrim.trimInterval)); - PR_Unlock(s_cl5Desc.clLock); + /* walked of the end of the file or entry is out of range */ + if (rc == 0 || rc == DB_NOTFOUND) { + slapi_ch_free(&key->data); + return CL5_NOTFOUND; + } + if (rc != CL5_DB_LOCK_ERROR) { + /* Not a lock error, free the key */ + slapi_ch_free(&key->data); } - PR_AtomicDecrement(&s_cl5Desc.threadCount); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5TrimMain - Exiting\n"); + /* cursor operation failed */ + slapi_log_err(rc == CL5_DB_LOCK_ERROR ? SLAPI_LOG_REPL : SLAPI_LOG_ERR, + repl_plugin_name_cl, + "_cl5PurgeGetNextEntry - Failed to get entry; db error - %d %s\n", + rc, db_strerror(rc)); - return 0; + return rc; } +#define MAX_RETRIES 10 /* - * We remove an entry if it has been replayed to all consumers and the number - * of entries in the changelog is larger than maxEntries or age of the entry - * is larger than maxAge. Also we can't purge entries which correspond to max - * csns in the supplier's ruv. Here is a example where we can get into trouble: + * _cl5PurgeRID(Object *obj, ReplicaId cleaned_rid) * - * The server is setup with time based trimming and no consumer's - * At some point all the entries are trimmed from the changelog. - * At a later point a consumer is added and initialized online. - * Then a change is made on the supplier. - * To update the consumer, the supplier would attempt to locate the last - * change sent to the consumer in the changelog and will fail because the - * change was removed. + * Clean the entire changelog of updates from the "cleaned rid" via CLEANALLRUV + * Delete entries in batches so we don't consume too many db locks, and we don't + * lockup the changelog during the entire purging process using one transaction. + * We save the key from the last iteration so we don't have to start from the + * beginning for each new iteration. */ static void -_cl5DoTrimming(void) +_cl5PurgeRID(cldb_Handle *cldb, ReplicaId cleaned_rid) { - Object *file_obj; - long numToTrim; + slapi_operation_parameters op = {0}; + ReplicaId csn_rid; + CL5Entry entry; + DB_TXN *txnid = NULL; + DBT key = {0}; + void *iterator = NULL; + long totalTrimmed = 0; + long trimmed = 0; + char *starting_key = NULL; + int batch_count = 0; + int db_lock_retry_count = 0; + int first_pass = 1; + int finished = 0; + int rc = 0; - PR_Lock(s_cl5Desc.dbTrim.lock); + entry.op = &op; /* - * We are trimming all the changelogs. We trim file by file which - * means that some files will be trimmed more often than other. We - * might have to fix that by, for example, randomizing the starting - * point. + * Keep processing the changelog until we are done, shutting down, or we + * maxed out on the db lock retries. */ - file_obj = objset_first_obj(s_cl5Desc.dbFiles); - while (file_obj && _cl5CanTrim((time_t)0, &numToTrim)) { - _cl5TrimFile(file_obj, &numToTrim); - file_obj = objset_next_obj(s_cl5Desc.dbFiles, file_obj); - } - - if (file_obj) - object_release(file_obj); + while (!finished && db_lock_retry_count < MAX_RETRIES && !slapi_is_shutting_down()) { + trimmed = 0; - PR_Unlock(s_cl5Desc.dbTrim.lock); + /* + * Sleep a bit to allow others to use the changelog - we can't hog the + * changelog for the entire purge. + */ + DS_Sleep(PR_MillisecondsToInterval(100)); - return; -} - -/* - * We are purging a changelog after a cleanAllRUV task. Find the specific - * changelog for the backend that is being cleaned, and purge all the records - * with the cleaned rid. - */ -static void -_cl5DoPurging(cleanruv_purge_data *purge_data) -{ - ReplicaId rid = purge_data->cleaned_rid; - const Slapi_DN *suffix_sdn = purge_data->suffix_sdn; - const char *replName = purge_data->replName; - char *replGen = purge_data->replGen; - char *fileName; - Object *file_obj; - - PR_Lock(s_cl5Desc.dbTrim.lock); - fileName = _cl5MakeFileName(replName, replGen); - file_obj = objset_find(s_cl5Desc.dbFiles, _cl5CompareDBFile, fileName); - if (file_obj) { - /* We found our changelog, now purge it */ - _cl5PurgeRID(file_obj, rid); - object_release(file_obj); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DoPurging - Purged rid (%d) from suffix (%s)\n", - rid, slapi_sdn_get_dn(suffix_sdn)); - } else { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5DoPurging - Purge rid (%d) failed to find changelog file (%s) for suffix (%s)\n", - rid, fileName, slapi_sdn_get_dn(suffix_sdn)); - } - PR_Unlock(s_cl5Desc.dbTrim.lock); - - return; -} - -/* clear free page files to reduce changelog */ -static void -_cl5CompactDBs(void) -{ - int rc; - Object *fileObj = NULL; - CL5DBFile *dbFile = NULL; - DB *db = NULL; - DB_TXN *txnid = NULL; - DB_COMPACT c_data = {0}; - - PR_Lock(s_cl5Desc.dbTrim.lock); - rc = TXN_BEGIN(s_cl5Desc.dbEnv, NULL, &txnid, 0); - if (rc) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CompactDBs - Failed to begin transaction; db error - %d %s\n", - rc, db_strerror(rc)); - goto bail; - } - for (fileObj = objset_first_obj(s_cl5Desc.dbFiles); - fileObj; - fileObj = objset_next_obj(s_cl5Desc.dbFiles, fileObj)) { - dbFile = (CL5DBFile *)object_get_data(fileObj); - if (!dbFile) { - continue; - } - db = dbFile->db; - rc = db->compact(db, txnid, NULL /*start*/, NULL /*stop*/, - &c_data, DB_FREE_SPACE, NULL /*end*/); - if (rc) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CompactDBs - Failed to compact %s; db error - %d %s\n", - dbFile->replName, rc, db_strerror(rc)); - goto bail; - } - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5CompactDBs - %s - %d pages freed\n", - dbFile->replName, c_data.compact_pages_free); - } -bail: - if (fileObj) { - object_release(fileObj); - } - if (rc) { - rc = TXN_ABORT(txnid); - if (rc) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CompactDBs - Failed to abort transaction; db error - %d %s\n", - rc, db_strerror(rc)); - } - } else { - rc = TXN_COMMIT(txnid); - if (rc) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CompactDBs - Failed to commit transaction; db error - %d %s\n", - rc, db_strerror(rc)); - } - } - PR_Unlock(s_cl5Desc.dbTrim.lock); - - return; -} - -/* - * If the rid is not set it is the very first iteration of the changelog. - * If the rid is set, we are doing another pass, and we have a key as our - * starting point. - */ -static int -_cl5PurgeGetFirstEntry(Object *file_obj, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key) -{ - DBC *cursor = NULL; - DBT data = {0}; - CL5Iterator *it; - CL5DBFile *file; - int rc; - - file = (CL5DBFile *)object_get_data(file_obj); - - /* create cursor */ - rc = file->db->cursor(file->db, txnid, &cursor, 0); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5PurgeGetFirstEntry - Failed to create cursor; db error - %d %s\n", rc, db_strerror(rc)); - rc = CL5_DB_ERROR; - goto done; - } - - key->flags = DB_DBT_MALLOC; - data.flags = DB_DBT_MALLOC; - while ((rc = cursor->c_get(cursor, key, &data, rid ? DB_SET : DB_NEXT)) == 0) { - /* skip service entries on the first pass (rid == 0)*/ - if (!rid && cl5HelperEntry((char *)key->data, NULL)) { - slapi_ch_free(&key->data); - slapi_ch_free(&(data.data)); - continue; - } - - /* format entry */ - rc = cl5DBData2Entry(data.data, data.size, entry); - slapi_ch_free(&(data.data)); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5PurgeGetFirstEntry - Failed to format entry: %d\n", rc); - goto done; - } - - it = (CL5Iterator *)slapi_ch_malloc(sizeof(CL5Iterator)); - it->cursor = cursor; - object_acquire(file_obj); - it->file = file_obj; - *(CL5Iterator **)iterator = it; - - return CL5_SUCCESS; - } - - slapi_ch_free(&key->data); - slapi_ch_free(&(data.data)); - - /* walked of the end of the file */ - if (rc == DB_NOTFOUND) { - rc = CL5_NOTFOUND; - goto done; - } - - /* db error occured while iterating */ - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5PurgeGetFirstEntry - Failed to get entry; db error - %d %s\n", - rc, db_strerror(rc)); - rc = CL5_DB_ERROR; - -done: - /* - * We didn't success in assigning this cursor to the iterator, - * so we need to free the cursor here. - */ - if (cursor) - cursor->c_close(cursor); - - return rc; -} - -/* - * Get the next entry. If we get a lock error we will restart the process - * starting at the current key. - */ -static int -_cl5PurgeGetNextEntry(CL5Entry *entry, void *iterator, DBT *key) -{ - CL5Iterator *it; - DBT data = {0}; - int rc; - - it = (CL5Iterator *)iterator; - - key->flags = DB_DBT_MALLOC; - data.flags = DB_DBT_MALLOC; - while ((rc = it->cursor->c_get(it->cursor, key, &data, DB_NEXT)) == 0) { - if (cl5HelperEntry((char *)key->data, NULL)) { - slapi_ch_free(&key->data); - slapi_ch_free(&(data.data)); - continue; - } - - /* format entry */ - rc = cl5DBData2Entry(data.data, data.size, entry); - slapi_ch_free(&(data.data)); - if (rc != 0) { - if (rc != CL5_DB_LOCK_ERROR) { - /* Not a lock error, free the key */ - slapi_ch_free(&key->data); - } - slapi_log_err(rc == CL5_DB_LOCK_ERROR ? SLAPI_LOG_REPL : SLAPI_LOG_ERR, - repl_plugin_name_cl, - "_cl5PurgeGetNextEntry - Failed to format entry: %d\n", - rc); - } - - return rc; - } - slapi_ch_free(&(data.data)); - - /* walked of the end of the file or entry is out of range */ - if (rc == 0 || rc == DB_NOTFOUND) { - slapi_ch_free(&key->data); - return CL5_NOTFOUND; - } - if (rc != CL5_DB_LOCK_ERROR) { - /* Not a lock error, free the key */ - slapi_ch_free(&key->data); - } - - /* cursor operation failed */ - slapi_log_err(rc == CL5_DB_LOCK_ERROR ? SLAPI_LOG_REPL : SLAPI_LOG_ERR, - repl_plugin_name_cl, - "_cl5PurgeGetNextEntry - Failed to get entry; db error - %d %s\n", - rc, db_strerror(rc)); - - return rc; -} - -#define MAX_RETRIES 10 -/* - * _cl5PurgeRID(Object *obj, ReplicaId cleaned_rid) - * - * Clean the entire changelog of updates from the "cleaned rid" via CLEANALLRUV - * Delete entries in batches so we don't consume too many db locks, and we don't - * lockup the changelog during the entire purging process using one transaction. - * We save the key from the last iteration so we don't have to start from the - * beginning for each new iteration. - */ -static void -_cl5PurgeRID(Object *file_obj, ReplicaId cleaned_rid) -{ - slapi_operation_parameters op = {0}; - ReplicaId csn_rid; - CL5Entry entry; - DB_TXN *txnid = NULL; - DBT key = {0}; - void *iterator = NULL; - long totalTrimmed = 0; - long trimmed = 0; - char *starting_key = NULL; - int batch_count = 0; - int db_lock_retry_count = 0; - int first_pass = 1; - int finished = 0; - int rc = 0; - - PR_ASSERT(file_obj); - entry.op = &op; - - /* - * Keep processing the changelog until we are done, shutting down, or we - * maxed out on the db lock retries. - */ - while (!finished && db_lock_retry_count < MAX_RETRIES && !slapi_is_shutting_down()) { - trimmed = 0; - - /* - * Sleep a bit to allow others to use the changelog - we can't hog the - * changelog for the entire purge. - */ - DS_Sleep(PR_MillisecondsToInterval(100)); - - rc = TXN_BEGIN(s_cl5Desc.dbEnv, NULL, &txnid, 0); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5PurgeRID - Failed to begin transaction; db error - %d %s. " - "Changelog was not purged of rid(%d)\n", - rc, db_strerror(rc), cleaned_rid); - return; - } + rc = TXN_BEGIN(s_cl5Desc.dbEnv, NULL, &txnid, 0); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5PurgeRID - Failed to begin transaction; db error - %d %s. " + "Changelog was not purged of rid(%d)\n", + rc, db_strerror(rc), cleaned_rid); + return; + } /* * Check every changelog entry for the cleaned rid */ - rc = _cl5PurgeGetFirstEntry(file_obj, &entry, &iterator, txnid, first_pass ? 0 : cleaned_rid, &key); + rc = _cl5PurgeGetFirstEntry(cldb, &entry, &iterator, txnid, first_pass?0:cleaned_rid, &key); first_pass = 0; while (rc == CL5_SUCCESS && !slapi_is_shutting_down()) { /* @@ -3594,7 +2698,7 @@ _cl5PurgeRID(Object *file_obj, ReplicaId cleaned_rid) #define CL5_TRIM_MAX_PER_TRANSACTION 10 static void -_cl5TrimFile(Object *obj, long *numToTrim) +_cl5TrimReplica(Replica *r) { DB_TXN *txnid; RUV *ruv = NULL; @@ -3606,11 +2710,16 @@ _cl5TrimFile(Object *obj, long *numToTrim) PRBool abort; char strCSN[CSN_STRSIZE]; int rc; + long numToTrim; - PR_ASSERT(obj); + cldb_Handle *cldb = replica_get_file_info(r); + + if (!_cl5CanTrim ((time_t)0, &numToTrim, r, &cldb->clConf) ) { + return; + } /* construct the ruv up to which we can purge */ - rc = _cl5GetRUV2Purge2(obj, &ruv); + rc = _cl5GetRUV2Purge2(r, &ruv); if (rc != CL5_SUCCESS || ruv == NULL) { return; } @@ -3627,20 +2736,20 @@ _cl5TrimFile(Object *obj, long *numToTrim) rc = TXN_BEGIN(s_cl5Desc.dbEnv, NULL, &txnid, 0); if (rc != 0) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5TrimFile - Failed to begin transaction; db error - %d %s\n", + "_cl5TrimReplica - Failed to begin transaction; db error - %d %s\n", rc, db_strerror(rc)); finished = PR_TRUE; break; } - finished = _cl5GetFirstEntry(obj, &entry, &it, txnid); + finished = _cl5GetFirstEntry(cldb, &entry, &it, txnid); while (!finished && !slapi_is_shutting_down()) { /* * This change can be trimmed if it exceeds purge * parameters and has been seen by all consumers. */ if (op.csn == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5TrimFile - " + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5TrimReplica - " "Operation missing csn, moving on to next entry.\n"); cl5_operation_parameters_done(&op); finished = _cl5GetNextEntry(&entry, it); @@ -3648,15 +2757,15 @@ _cl5TrimFile(Object *obj, long *numToTrim) } csn_rid = csn_get_replicaid(op.csn); - if ((*numToTrim > 0 || _cl5CanTrim(entry.time, numToTrim)) && + if ((numToTrim > 0 || _cl5CanTrim(entry.time, &numToTrim, r, &cldb->clConf)) && ruv_covers_csn_strict(ruv, op.csn)) { rc = _cl5CurrentDeleteEntry(it); if (rc == CL5_SUCCESS) { - rc = _cl5UpdateRUV(obj, op.csn, PR_FALSE, PR_TRUE); + rc = _cl5UpdateRUV(cldb, op.csn, PR_FALSE, PR_TRUE); } if (rc == CL5_SUCCESS) { - if (*numToTrim > 0) - (*numToTrim)--; + if (numToTrim > 0) + (numToTrim)--; count++; } else { /* The above two functions have logged the error */ @@ -3679,7 +2788,7 @@ _cl5TrimFile(Object *obj, long *numToTrim) } else { if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5TrimFile - Changelog purge skipped anchor csn %s\n", + "_cl5TrimReplica - Changelog purge skipped anchor csn %s\n", csn_as_string(maxcsn, PR_FALSE, strCSN)); } @@ -3712,7 +2821,7 @@ _cl5TrimFile(Object *obj, long *numToTrim) rc = TXN_ABORT(txnid); if (rc != 0) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5TrimFile - Failed to abort transaction; db error - %d %s\n", + "_cl5TrimReplica - Failed to abort transaction; db error - %d %s\n", rc, db_strerror(rc)); } } else { @@ -3720,7 +2829,7 @@ _cl5TrimFile(Object *obj, long *numToTrim) if (rc != 0) { finished = 1; slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5TrimFile - Failed to commit transaction; db error - %d %s\n", + "_cl5TrimReplica - Failed to commit transaction; db error - %d %s\n", rc, db_strerror(rc)); } else { totalTrimmed += count; @@ -3733,52 +2842,46 @@ _cl5TrimFile(Object *obj, long *numToTrim) ruv_destroy(&ruv); if (totalTrimmed) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5TrimFile - Trimmed %d changes from the changelog\n", + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5TrimReplica - Trimmed %d changes from the changelog\n", totalTrimmed); } } static PRBool -_cl5CanTrim(time_t time, long *numToTrim) +_cl5CanTrim(time_t time, long *numToTrim, Replica *replica, CL5Config *dbTrim) { *numToTrim = 0; - if (s_cl5Desc.dbTrim.maxAge == 0 && s_cl5Desc.dbTrim.maxEntries == 0) { + if (dbTrim->maxAge == 0 && dbTrim->maxEntries == 0) { return PR_FALSE; } - if (s_cl5Desc.dbTrim.maxAge == 0) { - *numToTrim = cl5GetOperationCount(NULL) - s_cl5Desc.dbTrim.maxEntries; + if (dbTrim->maxAge == 0) { + *numToTrim = cl5GetOperationCount(replica) - dbTrim->maxEntries; return (*numToTrim > 0); } - if (s_cl5Desc.dbTrim.maxEntries > 0 && - (*numToTrim = cl5GetOperationCount(NULL) - s_cl5Desc.dbTrim.maxEntries) > 0) { + if (dbTrim->maxEntries > 0 && + (*numToTrim = cl5GetOperationCount(replica) - dbTrim->maxEntries) > 0) { return PR_TRUE; } if (time) { - return (slapi_current_utc_time() - time > s_cl5Desc.dbTrim.maxAge); + return (slapi_current_utc_time() - time > dbTrim->maxAge); } else { return PR_TRUE; } } static int -_cl5ReadRUV(const char *replGen, Object *obj, PRBool purge) +_cl5ReadRUV (cldb_Handle *cldb, PRBool purge) { int rc; char csnStr[CSN_STRSIZE]; DBT key = {0}, data = {0}; struct berval **vals = NULL; - CL5DBFile *file; char *pos; char *agmt_name; - PR_ASSERT(replGen && obj); - - file = (CL5DBFile *)object_get_data(obj); - PR_ASSERT(file); - agmt_name = get_thread_private_agmtname(); if (purge) { /* read purge vector entry */ @@ -3789,7 +2892,7 @@ _cl5ReadRUV(const char *replGen, Object *obj, PRBool purge) key.size = CSN_STRSIZE; data.flags = DB_DBT_MALLOC; - rc = file->db->get(file->db, NULL /*txn*/, &key, &data, 0); + rc = cldb->db->get(cldb->db, NULL /*txn*/, &key, &data, 0); switch (rc) { case 0: pos = data.data; @@ -3799,9 +2902,9 @@ _cl5ReadRUV(const char *replGen, Object *obj, PRBool purge) goto done; if (purge) { - rc = ruv_init_from_bervals(vals, &file->purgeRUV); + rc = ruv_init_from_bervals(vals, &cldb->purgeRUV); } else { - rc = ruv_init_from_bervals(vals, &file->maxRUV); + rc = ruv_init_from_bervals(vals, &cldb->maxRUV); } if (rc != RUV_SUCCESS) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, @@ -3815,13 +2918,13 @@ _cl5ReadRUV(const char *replGen, Object *obj, PRBool purge) /* delete the entry; it is re-added when file is successfully closed */ - file->db->del(file->db, NULL, &key, 0); + cldb->db->del(cldb->db, NULL, &key, 0); rc = CL5_SUCCESS; goto done; case DB_NOTFOUND: /* RUV is lost - need to construct */ - rc = _cl5ConstructRUV(replGen, obj, purge); + rc = _cl5ConstructRUV(cldb, purge); goto done; default: @@ -3839,7 +2942,7 @@ done: } static int -_cl5WriteRUV(CL5DBFile *file, PRBool purge) +_cl5WriteRUV (cldb_Handle *cldb, PRBool purge) { int rc; DBT key = {0}, data = {0}; @@ -3848,7 +2951,7 @@ _cl5WriteRUV(CL5DBFile *file, PRBool purge) DB_TXN *txnid = NULL; char *buff; - if ((purge && file->purgeRUV == NULL) || (!purge && file->maxRUV == NULL)) + if ((purge && cldb->purgeRUV == NULL) || (!purge && cldb->maxRUV == NULL)) return CL5_SUCCESS; if (purge) { @@ -3858,18 +2961,18 @@ _cl5WriteRUV(CL5DBFile *file, PRBool purge) * matter, but it needs to be set to something so that it can be * flushed to changelog at shutdown and parsed at startup with the * regular string-to-RUV parsing routines. */ - ruv_insert_dummy_min_csn(file->purgeRUV); + ruv_insert_dummy_min_csn(cldb->purgeRUV); key.data = _cl5GetHelperEntryKey(PURGE_RUV_TIME, csnStr); - rc = ruv_to_bervals(file->purgeRUV, &vals); + rc = ruv_to_bervals(cldb->purgeRUV, &vals); } else { key.data = _cl5GetHelperEntryKey(MAX_RUV_TIME, csnStr); - rc = ruv_to_bervals(file->maxRUV, &vals); + rc = ruv_to_bervals(cldb->maxRUV, &vals); } - if (!purge && _cl5CheckMaxRUV(file, file->maxRUV)) { + if (!purge && _cl5CheckMaxRUV(cldb, cldb->maxRUV)) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5WriteRUV - changelog maxRUV not found in changelog for file %s\n", - file->name); + cldb->ident); ber_bvecfree(vals); return CL5_DB_ERROR; } @@ -3883,7 +2986,7 @@ _cl5WriteRUV(CL5DBFile *file, PRBool purge) return rc; } - rc = file->db->put(file->db, txnid, &key, &data, 0); + rc = cldb->db->put(cldb->db, txnid, &key, &data, 0); slapi_ch_free(&(data.data)); if (rc == 0) { @@ -3891,12 +2994,8 @@ _cl5WriteRUV(CL5DBFile *file, PRBool purge) } else { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5WriteRUV - Failed to write %s RUV for file %s; db error - %d (%s)\n", - purge ? "purge" : "upper bound", file->name, rc, db_strerror(rc)); + purge ? "purge" : "upper bound", cldb->ident, rc, db_strerror(rc)); - if (CL5_OS_ERR_IS_DISKFULL(rc)) { - cl5_set_diskfull(); - return CL5_DB_ERROR; - } return CL5_DB_ERROR; } } @@ -3904,29 +3003,23 @@ _cl5WriteRUV(CL5DBFile *file, PRBool purge) /* This is a very slow process since we have to read every changelog entry. Hopefully, this function is not called too often */ static int -_cl5ConstructRUV(const char *replGen, Object *obj, PRBool purge) +_cl5ConstructRUV (cldb_Handle *cldb, PRBool purge) { int rc; CL5Entry entry; void *iterator = NULL; slapi_operation_parameters op = {0}; - CL5DBFile *file; ReplicaId rid; - PR_ASSERT(replGen && obj); - - file = (CL5DBFile *)object_get_data(obj); - PR_ASSERT(file); - /* construct the RUV */ if (purge) - rc = ruv_init_new(replGen, 0, NULL, &file->purgeRUV); + rc = ruv_init_new(cldb->ident, 0, NULL, &cldb->purgeRUV); else - rc = ruv_init_new(replGen, 0, NULL, &file->maxRUV); + rc = ruv_init_new(cldb->ident, 0, NULL, &cldb->maxRUV); if (rc != RUV_SUCCESS) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5ConstructRUV - " "Failed to initialize %s RUV for file %s; ruv error - %d\n", - purge ? "purge" : "upper bound", file->name, rc); + purge ? "purge" : "upper bound", cldb->ident, rc); return CL5_RUV_ERROR; } @@ -3935,7 +3028,7 @@ _cl5ConstructRUV(const char *replGen, Object *obj, PRBool purge) "this may take several minutes...\n"); entry.op = &op; - rc = _cl5GetFirstEntry(obj, &entry, &iterator, NULL); + rc = _cl5GetFirstEntry(cldb, &entry, &iterator, NULL); while (rc == CL5_SUCCESS) { if (op.csn) { rid = csn_get_replicaid(op.csn); @@ -3956,15 +3049,15 @@ _cl5ConstructRUV(const char *replGen, Object *obj, PRBool purge) continue; } if (purge) - rc = ruv_set_csns_keep_smallest(file->purgeRUV, op.csn); + rc = ruv_set_csns_keep_smallest(cldb->purgeRUV, op.csn); else - rc = ruv_set_csns(file->maxRUV, op.csn, NULL); + rc = ruv_set_csns(cldb->maxRUV, op.csn, NULL); cl5_operation_parameters_done(&op); if (rc != RUV_SUCCESS) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5ConstructRUV - " "Failed to update %s RUV for file %s; ruv error - %d\n", - purge ? "purge" : "upper bound", file->name, rc); + purge ? "purge" : "upper bound", cldb->ident, rc); rc = CL5_RUV_ERROR; continue; } @@ -3981,9 +3074,9 @@ _cl5ConstructRUV(const char *replGen, Object *obj, PRBool purge) rc = CL5_SUCCESS; } else { if (purge) - ruv_destroy(&file->purgeRUV); + ruv_destroy(&cldb->purgeRUV); else - ruv_destroy(&file->maxRUV); + ruv_destroy(&cldb->maxRUV); } slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl, @@ -3994,43 +3087,40 @@ _cl5ConstructRUV(const char *replGen, Object *obj, PRBool purge) } static int -_cl5UpdateRUV(Object *obj, CSN *csn, PRBool newReplica, PRBool purge) +_cl5UpdateRUV (cldb_Handle *cldb, CSN *csn, PRBool newReplica, PRBool purge) { ReplicaId rid; int rc = RUV_SUCCESS; /* initialize rc to avoid erroneous logs */ - CL5DBFile *file; - PR_ASSERT(obj && csn); - - file = (CL5DBFile *)object_get_data(obj); + PR_ASSERT(csn); /* - * if purge is TRUE, file->purgeRUV must be set; + * if purge is TRUE, cldb->purgeRUV must be set; * if purge is FALSE, maxRUV must be set */ - PR_ASSERT(file && ((purge && file->purgeRUV) || (!purge && file->maxRUV))); + PR_ASSERT(cldb && ((purge && cldb->purgeRUV) || (!purge && cldb->maxRUV))); rid = csn_get_replicaid(csn); /* update vector only if this replica is not yet part of RUV */ if (purge && newReplica) { - if (ruv_contains_replica(file->purgeRUV, rid)) { + if (ruv_contains_replica(cldb->purgeRUV, rid)) { return CL5_SUCCESS; } else { /* if the replica is not part of the purgeRUV yet, add it unless it's from a cleaned rid */ - ruv_add_replica(file->purgeRUV, rid, multimaster_get_local_purl()); + ruv_add_replica(cldb->purgeRUV, rid, multimaster_get_local_purl()); } } else { if (purge) { - rc = ruv_set_csns(file->purgeRUV, csn, NULL); + rc = ruv_set_csns(cldb->purgeRUV, csn, NULL); } else { - rc = ruv_set_csns(file->maxRUV, csn, NULL); + rc = ruv_set_csns(cldb->maxRUV, csn, NULL); } } if (rc != RUV_SUCCESS) { slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5UpdatePurgeRUV - " "Failed to update %s RUV for file %s; ruv error - %d\n", - purge ? "purge" : "upper bound", file->name, rc); + purge ? "purge" : "upper bound", cldb->ident, rc); return CL5_RUV_ERROR; } @@ -4060,36 +3150,22 @@ _cl5EnumConsumerRUV(const ruv_enum_data *element, void *arg) } static int -_cl5GetRUV2Purge2(Object *fileObj, RUV **ruv) +_cl5GetRUV2Purge2(Replica *replica, RUV **ruv) { int rc = CL5_SUCCESS; - CL5DBFile *dbFile; - Replica *r = NULL; Object *agmtObj = NULL; Repl_Agmt *agmt; Object *consRUVObj, *supRUVObj; RUV *consRUV, *supRUV; CSN *csn; - PR_ASSERT(fileObj && ruv); - if (!ruv) { rc = CL5_UNKNOWN_ERROR; goto done; } - dbFile = (CL5DBFile *)object_get_data(fileObj); - PR_ASSERT(dbFile); - - r = replica_get_by_name(dbFile->replName); - - if (!r) { - rc = CL5_NOTFOUND; - goto done; - } - - /* We start with this replica's RUV. See note in _cl5DoTrimming */ - supRUVObj = replica_get_ruv(r); + /* We start with this replica's RUV. */ + supRUVObj = replica_get_ruv(replica); PR_ASSERT(supRUVObj); supRUV = (RUV *)object_get_data(supRUVObj); @@ -4099,7 +3175,7 @@ _cl5GetRUV2Purge2(Object *fileObj, RUV **ruv) object_release(supRUVObj); - agmtObj = agmtlist_get_first_agreement_for_replica(r); + agmtObj = agmtlist_get_first_agreement_for_replica(replica); while (agmtObj) { agmt = (Repl_Agmt *)object_get_data(agmtObj); PR_ASSERT(agmt); @@ -4124,7 +3200,7 @@ _cl5GetRUV2Purge2(Object *fileObj, RUV **ruv) object_release(consRUVObj); } - agmtObj = agmtlist_get_next_agreement_for_replica(r, agmtObj); + agmtObj = agmtlist_get_next_agreement_for_replica(replica, agmtObj); } /* check if there is any data in the constructed ruv - otherwise get rid of it */ @@ -4140,41 +3216,62 @@ done: return rc; } -static int -_cl5GetEntryCount(CL5DBFile *file) +int +cl5NotifyRUVChange(Replica *replica) { - int rc; - char csnStr[CSN_STRSIZE]; - DBT key = {0}, data = {0}; - DB_BTREE_STAT *stats = NULL; + int rc = 0; + cldb_Handle *cldb = replica_get_file_info(replica); + Object *ruv_obj = replica_get_ruv(replica); - PR_ASSERT(file); + PR_Lock(cldb->clLock); - /* read entry count. if the entry is there - the file was successfully closed + slapi_ch_free_string(&cldb->ident); + ruv_destroy(&cldb->maxRUV); + ruv_destroy(&cldb->purgeRUV); + + cldb->ident = ruv_get_replica_generation ((RUV*)object_get_data (ruv_obj)); + _cl5ReadRUV(cldb, PR_TRUE); + _cl5ReadRUV(cldb, PR_FALSE); + _cl5GetEntryCount(cldb); + + PR_Unlock(cldb->clLock); + object_release(ruv_obj); + return rc; +} + +static int +_cl5GetEntryCount(cldb_Handle *cldb) +{ + int rc; + char csnStr[CSN_STRSIZE]; + DBT key = {0}, data = {0}; + DB_BTREE_STAT *stats = NULL; + + /* read entry count. if the entry is there - the file was successfully closed last time it was used */ key.data = _cl5GetHelperEntryKey(ENTRY_COUNT_TIME, csnStr); key.size = CSN_STRSIZE; data.flags = DB_DBT_MALLOC; - rc = file->db->get(file->db, NULL /*txn*/, &key, &data, 0); + rc = cldb->db->get(cldb->db, NULL /*txn*/, &key, &data, 0); switch (rc) { case 0: - file->entryCount = *(int *)data.data; + cldb->entryCount = *(int *)data.data; slapi_ch_free(&(data.data)); /* delete the entry. the entry is re-added when file is successfully closed */ - file->db->del(file->db, NULL, &key, 0); + cldb->db->del(cldb->db, NULL, &key, 0); slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5GetEntryCount - %d changes for replica %s\n", - file->entryCount, file->replName); + cldb->entryCount, cldb->ident); return CL5_SUCCESS; case DB_NOTFOUND: - file->entryCount = 0; + cldb->entryCount = 0; - rc = file->db->stat(file->db, NULL, (void *)&stats, 0); + rc = cldb->db->stat(cldb->db, NULL, (void *)&stats, 0); if (rc != 0) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5GetEntryCount - Failed to get changelog statistics; " @@ -4183,14 +3280,10 @@ _cl5GetEntryCount(CL5DBFile *file) return CL5_DB_ERROR; } -#ifdef DB30 - file->entryCount = stats->bt_nrecs; -#else /* DB31 */ - file->entryCount = stats->bt_ndata; -#endif + cldb->entryCount = stats->bt_ndata; slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5GetEntryCount - %d changes for replica %s\n", - file->entryCount, file->replName); + cldb->entryCount, cldb->ident); slapi_ch_free((void **)&stats); return CL5_SUCCESS; @@ -4205,7 +3298,7 @@ _cl5GetEntryCount(CL5DBFile *file) } static int -_cl5WriteEntryCount(CL5DBFile *file) +_cl5WriteEntryCount(cldb_Handle *cldb) { int rc; DBT key = {0}, data = {0}; @@ -4214,21 +3307,17 @@ _cl5WriteEntryCount(CL5DBFile *file) key.data = _cl5GetHelperEntryKey(ENTRY_COUNT_TIME, csnStr); key.size = CSN_STRSIZE; - data.data = (void *)&file->entryCount; - data.size = sizeof(file->entryCount); + data.data = (void *)&cldb->entryCount; + data.size = sizeof(cldb->entryCount); - rc = file->db->put(file->db, txnid, &key, &data, 0); + rc = cldb->db->put(cldb->db, txnid, &key, &data, 0); if (rc == 0) { return CL5_SUCCESS; } else { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5WriteEntryCount - " "Failed to write count entry for file %s; db error - %d %s\n", - file->name, rc, db_strerror(rc)); - if (CL5_OS_ERR_IS_DISKFULL(rc)) { - cl5_set_diskfull(); - return CL5_DB_ERROR; - } + cldb->ident, rc, db_strerror(rc)); return CL5_DB_ERROR; } } @@ -4555,7 +3644,7 @@ _cl5LDIF2Operation(char *ldifEntry, slapi_operation_parameters *op, char **replG } static int -_cl5WriteOperationTxn(const char *replName, const char *replGen, const slapi_operation_parameters *op, PRBool local __attribute__((unused)), void *txn) +_cl5WriteOperationTxn(cldb_Handle *cldb, const slapi_operation_parameters *op, void *txn) { int rc; int cnt; @@ -4564,1256 +3653,719 @@ _cl5WriteOperationTxn(const char *replName, const char *replGen, const slapi_ope char csnStr[CSN_STRSIZE]; PRIntervalTime interval; CL5Entry entry; - CL5DBFile *file = NULL; - Object *file_obj = NULL; DB_TXN *txnid = NULL; DB_TXN *parent_txnid = (DB_TXN *)txn; - rc = _cl5GetDBFileByReplicaName(replName, replGen, &file_obj); - if (rc == CL5_NOTFOUND) { - rc = _cl5DBOpenFileByReplicaName(replName, replGen, &file_obj, - PR_TRUE /* check for duplicates */); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to find or open DB object for replica %s\n", replName); - return rc; - } - } else if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to get db file for target dn (%s)", - REPL_GET_DN(&op->target_address)); - return CL5_OBJSET_ERROR; - } - - /* assign entry time - used for trimming */ - entry.time = slapi_current_utc_time(); - entry.op = (slapi_operation_parameters *)op; - - /* construct the key */ - key.data = csn_as_string(op->csn, PR_FALSE, csnStr); - key.size = CSN_STRSIZE; - - /* construct the data */ - data = (DBT *)slapi_ch_calloc(1, sizeof(DBT)); - rc = _cl5Entry2DBData(&entry, (char **)&data->data, &data->size); - if (rc != CL5_SUCCESS) { - char s[CSN_STRSIZE]; - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to convert entry with csn (%s) " - "to db format\n", - csn_as_string(op->csn, PR_FALSE, s)); - goto done; - } - - file = (CL5DBFile *)object_get_data(file_obj); - PR_ASSERT(file); - - /* if this is part of ldif2cl - just write the entry without transaction */ - if (s_cl5Desc.dbOpenMode == CL5_OPEN_LDIF2CL) { - rc = file->db->put(file->db, NULL, &key, data, 0); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to write entry; db error - %d %s\n", - rc, db_strerror(rc)); - if (CL5_OS_ERR_IS_DISKFULL(rc)) { - cl5_set_diskfull(); - } - rc = CL5_DB_ERROR; - } - goto done; - } - - /* write the entry */ - rc = EAGAIN; - cnt = 0; - - while ((rc == EAGAIN || rc == DB_LOCK_DEADLOCK) && cnt < MAX_TRIALS) { - if (cnt != 0) { - /* abort previous transaction */ - rc = TXN_ABORT(txnid); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to abort transaction; db error - %d %s\n", - rc, db_strerror(rc)); - rc = CL5_DB_ERROR; - goto done; - } - /* back off */ - interval = PR_MillisecondsToInterval(slapi_rand() % 100); - DS_Sleep(interval); - } - /* begin transaction */ - rc = TXN_BEGIN(s_cl5Desc.dbEnv, parent_txnid, &txnid, 0); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to start transaction; db error - %d %s\n", - rc, db_strerror(rc)); - rc = CL5_DB_ERROR; - goto done; - } - - rc = file->db->put(file->db, txnid, &key, data, 0); - if (CL5_OS_ERR_IS_DISKFULL(rc)) { - slapi_log_err(SLAPI_LOG_CRIT, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Changelog (%s) DISK FULL; db error - %d %s\n", - s_cl5Desc.dbDir, rc, db_strerror(rc)); - cl5_set_diskfull(); - rc = CL5_DB_ERROR; - goto done; - } - if (cnt != 0) { - if (rc == 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5WriteOperationTxn - " - "retry (%d) the transaction (csn=%s) succeeded\n", - cnt, (char *)key.data); - } else if ((cnt + 1) >= MAX_TRIALS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5WriteOperationTxn - " - "retry (%d) the transaction (csn=%s) failed (rc=%d (%s))\n", - cnt, (char *)key.data, rc, db_strerror(rc)); - } - } - cnt++; - } - - if (rc == 0) /* we successfully added entry */ - { - rc = TXN_COMMIT(txnid); - } else { - char s[CSN_STRSIZE]; - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to write entry with csn (%s); " - "db error - %d %s\n", - csn_as_string(op->csn, PR_FALSE, s), - rc, db_strerror(rc)); - rc = TXN_ABORT(txnid); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5WriteOperationTxn - Failed to abort transaction; db error - %d %s\n", - rc, db_strerror(rc)); - } - rc = CL5_DB_ERROR; - goto done; - } - - /* update entry count - we assume that all entries are new */ - PR_AtomicIncrement(&file->entryCount); - - /* update purge vector if we have not seen any changes from this replica before */ - _cl5UpdateRUV(file_obj, op->csn, PR_TRUE, PR_TRUE); - - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5WriteOperationTxn - Successfully written entry with csn (%s)\n", csnStr); - rc = CL5_SUCCESS; -done: - if (data->data) - slapi_ch_free(&(data->data)); - slapi_ch_free((void **)&data); - - if (file_obj) - object_release(file_obj); - - return rc; -} - -static int -_cl5WriteOperation(const char *replName, const char *replGen, const slapi_operation_parameters *op, PRBool local) -{ - return _cl5WriteOperationTxn(replName, replGen, op, local, NULL); -} - -static int -_cl5GetFirstEntry(Object *file_obj, CL5Entry *entry, void **iterator, DB_TXN *txnid) -{ - int rc; - DBC *cursor = NULL; - DBT key = {0}, data = {0}; - CL5Iterator *it; - CL5DBFile *file; - - PR_ASSERT(file_obj && entry && iterator); - - file = (CL5DBFile *)object_get_data(file_obj); - PR_ASSERT(file); - /* create cursor */ - rc = file->db->cursor(file->db, txnid, &cursor, 0); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5GetFirstEntry - Failed to create cursor; db error - %d %s\n", rc, db_strerror(rc)); - rc = CL5_DB_ERROR; - goto done; - } - - key.flags = DB_DBT_MALLOC; - data.flags = DB_DBT_MALLOC; - while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { - /* skip service entries */ - if (cl5HelperEntry((char *)key.data, NULL)) { - slapi_ch_free(&(key.data)); - slapi_ch_free(&(data.data)); - continue; - } - - /* format entry */ - slapi_ch_free(&(key.data)); - rc = cl5DBData2Entry(data.data, data.size, entry); - slapi_ch_free(&(data.data)); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5GetFirstOperation - Failed to format entry: %d\n", rc); - goto done; - } - - it = (CL5Iterator *)slapi_ch_malloc(sizeof(CL5Iterator)); - it->cursor = cursor; - object_acquire(file_obj); - it->file = file_obj; - *(CL5Iterator **)iterator = it; - - return CL5_SUCCESS; - } - /* - * Bug 430172 - memory leaks after db "get" deadlocks, e.g. in CL5 trim - * Even when db->c_get() does not return success, memory may have been - * allocated in the DBT. This seems to happen when DB_DBT_MALLOC was set, - * the data being retrieved is larger than the page size, and we got - * DB_LOCK_DEADLOCK. libdb allocates the memory and then finds itself - * deadlocked trying to go through the overflow page list. It returns - * DB_LOCK_DEADLOCK which we've assumed meant that no memory was allocated - * for the DBT. - * - * The following slapi_ch_free frees the memory only when the value is - * non NULL, which is true if the situation described above occurs. - */ - slapi_ch_free((void **)&key.data); - slapi_ch_free((void **)&data.data); - - /* walked of the end of the file */ - if (rc == DB_NOTFOUND) { - rc = CL5_NOTFOUND; - goto done; - } - - /* db error occured while iterating */ - /* On this path, the condition "rc != 0" cannot be false */ - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5GetFirstEntry - Failed to get entry; db error - %d %s\n", - rc, db_strerror(rc)); - rc = CL5_DB_ERROR; - -done: - /* error occured */ - /* We didn't success in assigning this cursor to the iterator, - * so we need to free the cursor here */ - if (cursor) - cursor->c_close(cursor); - - return rc; -} - -static int -_cl5GetNextEntry(CL5Entry *entry, void *iterator) -{ - int rc; - CL5Iterator *it; - DBT key = {0}, data = {0}; - - PR_ASSERT(entry && iterator); - - it = (CL5Iterator *)iterator; - - key.flags = DB_DBT_MALLOC; - data.flags = DB_DBT_MALLOC; - while ((rc = it->cursor->c_get(it->cursor, &key, &data, DB_NEXT)) == 0) { - if (cl5HelperEntry((char *)key.data, NULL)) { - slapi_ch_free(&(key.data)); - slapi_ch_free(&(data.data)); - continue; - } - - slapi_ch_free(&(key.data)); - /* format entry */ - rc = cl5DBData2Entry(data.data, data.size, entry); - slapi_ch_free(&(data.data)); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5GetNextEntry - Failed to format entry: %d\n", rc); - } - - return rc; - } - /* - * Bug 430172 - memory leaks after db "get" deadlocks, e.g. in CL5 trim - * Even when db->c_get() does not return success, memory may have been - * allocated in the DBT. This seems to happen when DB_DBT_MALLOC was set, - * the data being retrieved is larger than the page size, and we got - * DB_LOCK_DEADLOCK. libdb allocates the memory and then finds itself - * deadlocked trying to go through the overflow page list. It returns - * DB_LOCK_DEADLOCK which we've assumed meant that no memory was allocated - * for the DBT. - * - * The following slapi_ch_free frees the memory only when the value is - * non NULL, which is true if the situation described above occurs. - */ - slapi_ch_free((void **)&key.data); - slapi_ch_free((void **)&data.data); - - /* walked of the end of the file or entry is out of range */ - if (rc == 0 || rc == DB_NOTFOUND) { - return CL5_NOTFOUND; - } - - /* cursor operation failed */ - slapi_log_err(rc == CL5_DB_LOCK_ERROR ? SLAPI_LOG_REPL : SLAPI_LOG_ERR, - repl_plugin_name_cl, - "_cl5GetNextEntry - Failed to get entry; db error - %d %s\n", - rc, db_strerror(rc)); - - return rc; -} - -static int -_cl5CurrentDeleteEntry(void *iterator) -{ - int rc; - CL5Iterator *it; - CL5DBFile *file; - - PR_ASSERT(iterator); - - it = (CL5Iterator *)iterator; - - rc = it->cursor->c_del(it->cursor, 0); - - if (rc == 0) { - /* decrement entry count */ - file = (CL5DBFile *)object_get_data(it->file); - PR_AtomicDecrement(&file->entryCount); - return CL5_SUCCESS; - } else { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5CurrentDeleteEntry - Failed, err=%d %s\n", - rc, db_strerror(rc)); - /* - * We don't free(close) the cursor here, as the caller will free it by - * a call to cl5DestroyIterator. Freeing it here is a potential bug, - * as the cursor can't be referenced later once freed. - */ - return rc; - } -} - -PRBool -cl5HelperEntry(const char *csnstr, CSN *csnp) -{ - CSN *csn; - time_t csnTime; - PRBool retval = PR_FALSE; - - if (csnp) { - csn = csnp; - } else { - csn = csn_new_by_string(csnstr); - } - if (csn == NULL) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5HelperEntry - Failed to get csn time; csn error\n"); - return PR_FALSE; - } - csnTime = csn_get_time(csn); - - if (csnTime == ENTRY_COUNT_TIME || csnTime == PURGE_RUV_TIME) { - retval = PR_TRUE; - } - - if (NULL == csnp) - csn_free(&csn); - return retval; -} - -#ifdef FOR_DEBUGGING -/* Replay iteration helper functions */ -static PRBool -_cl5ValidReplayIterator(const CL5ReplayIterator *iterator) -{ - if (iterator == NULL || - iterator->consumerRuv == NULL || iterator->supplierRuvObj == NULL || - iterator->fileObj == NULL) - return PR_FALSE; - - return PR_TRUE; -} -#endif - -/* Algorithm: ONREPL!!! - */ -struct replica_hash_entry -{ - ReplicaId rid; /* replica id */ - PRBool sendChanges; /* indicates whether changes should be sent for this replica */ -}; - - -static int -_cl5PositionCursorForReplay(ReplicaId consumerRID, const RUV *consumerRuv, Replica *replica, Object *fileObj, CL5ReplayIterator **iterator, int *continue_on_missing) -{ - CLC_Buffer *clcache = NULL; - CL5DBFile *file; - CSN *startCSN = NULL; - char csnStr[CSN_STRSIZE]; - int rc = CL5_SUCCESS; - Object *supplierRuvObj = NULL; - RUV *supplierRuv = NULL; - PRBool haveChanges = PR_FALSE; - char *agmt_name; - - PR_ASSERT(consumerRuv && replica && fileObj && iterator); - csnStr[0] = '\0'; - - file = (CL5DBFile *)object_get_data(fileObj); - - /* get supplier's RUV */ - supplierRuvObj = replica_get_ruv(replica); - PR_ASSERT(supplierRuvObj); - - if (!supplierRuvObj) { - rc = CL5_UNKNOWN_ERROR; - goto done; - } - - supplierRuv = (RUV *)object_get_data(supplierRuvObj); - PR_ASSERT(supplierRuv); - - agmt_name = get_thread_private_agmtname(); - - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5PositionCursorForReplay - (%s): Consumer RUV:\n", agmt_name); - ruv_dump(consumerRuv, agmt_name, NULL); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5PositionCursorForReplay - (%s): Supplier RUV:\n", agmt_name); - ruv_dump(supplierRuv, agmt_name, NULL); - } - + /* assign entry time - used for trimming */ + entry.time = slapi_current_utc_time(); + entry.op = (slapi_operation_parameters *)op; - /* initialize the changelog buffer and do the initial load */ + /* construct the key */ + key.data = csn_as_string(op->csn, PR_FALSE, csnStr); + key.size = CSN_STRSIZE; - rc = clcache_get_buffer(&clcache, file->db, consumerRID, consumerRuv, supplierRuv); - if (rc != 0) + /* construct the data */ + data = (DBT *)slapi_ch_calloc(1, sizeof(DBT)); + rc = _cl5Entry2DBData(&entry, (char **)&data->data, &data->size, cldb->clcrypt_handle); + if (rc != CL5_SUCCESS) { + char s[CSN_STRSIZE]; + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "_cl5WriteOperationTxn - Failed to convert entry with csn (%s) " + "to db format\n", + csn_as_string(op->csn, PR_FALSE, s)); goto done; + } - rc = clcache_load_buffer(clcache, &startCSN, continue_on_missing); + /* if this is part of ldif2cl - just write the entry without transaction */ + if (s_cl5Desc.dbOpenMode == CL5_OPEN_LDIF2CL) { + rc = cldb->db->put(cldb->db, NULL, &key, data, 0); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5WriteOperationTxn - Failed to write entry; db error - %d %s\n", + rc, db_strerror(rc)); + rc = CL5_DB_ERROR; + } + goto done; + } - if (rc == 0) { - haveChanges = PR_TRUE; - rc = CL5_SUCCESS; - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - csn_as_string(startCSN, PR_FALSE, csnStr); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "%s: CSN %s found, position set for replay\n", agmt_name, csnStr); + /* write the entry */ + rc = EAGAIN; + cnt = 0; + + while ((rc == EAGAIN || rc == DB_LOCK_DEADLOCK) && cnt < MAX_TRIALS) { + if (cnt != 0) { + /* abort previous transaction */ + rc = TXN_ABORT(txnid); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5WriteOperationTxn - Failed to abort transaction; db error - %d %s\n", + rc, db_strerror(rc)); + rc = CL5_DB_ERROR; + goto done; + } + /* back off */ + interval = PR_MillisecondsToInterval(slapi_rand() % 100); + DS_Sleep(interval); } - } else if (rc == DB_NOTFOUND) { - /* buffer not loaded. - * either because no changes have to be sent ==> startCSN is NULL - * or the calculated startCSN cannot be found in the changelog - */ - if (startCSN == NULL) { - rc = CL5_NOTFOUND; + /* begin transaction */ + rc = TXN_BEGIN(s_cl5Desc.dbEnv, parent_txnid, &txnid, 0); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5WriteOperationTxn - Failed to start transaction; db error - %d %s\n", + rc, db_strerror(rc)); + rc = CL5_DB_ERROR; goto done; } - /* check whether this csn should be present */ - rc = _cl5CheckMissingCSN(startCSN, supplierRuv, file); - if (rc == CL5_MISSING_DATA) /* we should have had the change but we don't */ - { - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - csn_as_string(startCSN, PR_FALSE, csnStr); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "repl_plugin_name_cl - %s: CSN %s not found, seems to be missing\n", agmt_name, csnStr); + + rc = cldb->db->put(cldb->db, txnid, &key, data, 0); + if (CL5_OS_ERR_IS_DISKFULL(rc)) { + slapi_log_err(SLAPI_LOG_CRIT, repl_plugin_name_cl, + "_cl5WriteOperationTxn - Changelog DISK FULL; db error - %d %s\n", + rc, db_strerror(rc)); + rc = CL5_DB_ERROR; + goto done; + } + if (cnt != 0) { + if (rc == 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5WriteOperationTxn - " + "retry (%d) the transaction (csn=%s) succeeded\n", + cnt, (char *)key.data); + } else if ((cnt + 1) >= MAX_TRIALS) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5WriteOperationTxn - " + "retry (%d) the transaction (csn=%s) failed (rc=%d (%s))\n", + cnt, (char *)key.data, rc, db_strerror(rc)); } - } else /* we are not as up to date or we purged */ - { - csn_as_string(startCSN, PR_FALSE, csnStr); - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "repl_plugin_name_cl - %s: CSN %s not found, we aren't as up to date, or we purged\n", - agmt_name, csnStr); } + cnt++; + } + + if (rc == 0) /* we successfully added entry */ + { + rc = TXN_COMMIT(txnid); } else { - csn_as_string(startCSN, PR_FALSE, csnStr); - /* db error */ + char s[CSN_STRSIZE]; slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "repl_plugin_name_cl - %s: Failed to retrieve change with CSN %s; db error - %d %s\n", - agmt_name, csnStr, rc, db_strerror(rc)); - + "_cl5WriteOperationTxn - Failed to write entry with csn (%s); " + "db error - %d %s\n", + csn_as_string(op->csn, PR_FALSE, s), + rc, db_strerror(rc)); + rc = TXN_ABORT(txnid); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5WriteOperationTxn - Failed to abort transaction; db error - %d %s\n", + rc, db_strerror(rc)); + } rc = CL5_DB_ERROR; + goto done; } + /* update entry count - we assume that all entries are new */ + PR_AtomicIncrement(&cldb->entryCount); - /* setup the iterator */ - if (haveChanges) { - *iterator = (CL5ReplayIterator *)slapi_ch_calloc(1, sizeof(CL5ReplayIterator)); - - if (*iterator == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5PositionCursorForReplay - %s - Failed to allocate iterator\n", agmt_name); - rc = CL5_MEMORY_ERROR; - goto done; - } - - /* ONREPL - should we make a copy of both RUVs here ?*/ - (*iterator)->fileObj = fileObj; - (*iterator)->clcache = clcache; - clcache = NULL; - (*iterator)->consumerRID = consumerRID; - (*iterator)->consumerRuv = consumerRuv; - (*iterator)->supplierRuvObj = supplierRuvObj; - } else if (rc == CL5_SUCCESS) { - /* we have no changes to send */ - rc = CL5_NOTFOUND; - } + /* update purge vector if we have not seen any changes from this replica before */ + _cl5UpdateRUV(cldb, op->csn, PR_TRUE, PR_TRUE); + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "cl5WriteOperationTxn - Successfully written entry with csn (%s)\n", csnStr); + rc = CL5_SUCCESS; done: - if (clcache) - clcache_return_buffer(&clcache); - - if (rc != CL5_SUCCESS) { - if (supplierRuvObj) - object_release(supplierRuvObj); - } + if (data->data) + slapi_ch_free(&(data->data)); + slapi_ch_free((void **)&data); return rc; } -struct ruv_it -{ - CSN **csns; /* csn list */ - int alloc; /* allocated size */ - int pos; /* position in the list */ -}; - static int -ruv_consumer_iterator(const ruv_enum_data *enum_data, void *arg) +_cl5WriteOperation(cldb_Handle *cldb, const slapi_operation_parameters *op) { - struct ruv_it *data = (struct ruv_it *)arg; - - PR_ASSERT(data); - - /* check if we have space for one more element */ - if (data->pos >= data->alloc - 2) { - data->alloc += 4; - data->csns = (CSN **)slapi_ch_realloc((void *)data->csns, data->alloc * sizeof(CSN *)); - } - - data->csns[data->pos] = csn_dup(enum_data->csn); - data->pos++; - - return 0; + return _cl5WriteOperationTxn(cldb, op, NULL); } - static int -ruv_supplier_iterator(const ruv_enum_data *enum_data, void *arg) +_cl5GetFirstEntry(cldb_Handle *cldb, CL5Entry *entry, void **iterator, DB_TXN *txnid) { - int i; - PRBool found = PR_FALSE; - ReplicaId rid; - struct ruv_it *data = (struct ruv_it *)arg; - - PR_ASSERT(data); + int rc; + DBC *cursor = NULL; + DBT key = {0}, data = {0}; + CL5Iterator *it; - rid = csn_get_replicaid(enum_data->min_csn); - /* check if the replica that generated the csn is already in the list */ - for (i = 0; i < data->pos; i++) { - if (rid == csn_get_replicaid(data->csns[i])) { - found = PR_TRUE; + PR_ASSERT(entry && iterator); - /* remove datacsn[i] if it is greater or equal to the supplier's maxcsn */ - if (csn_compare(data->csns[i], enum_data->csn) >= 0) { - int j; + /* create cursor */ + rc = cldb->db->cursor(cldb->db, txnid, &cursor, 0); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5GetFirstEntry - Failed to create cursor; db error - %d %s\n", rc, db_strerror(rc)); + rc = CL5_DB_ERROR; + goto done; + } - csn_free(&data->csns[i]); - for (j = i + 1; j < data->pos; j++) { - data->csns[j - 1] = data->csns[j]; - } - data->pos--; - } - break; + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { + /* skip service entries */ + if (cl5HelperEntry((char *)key.data, NULL)) { + slapi_ch_free(&(key.data)); + slapi_ch_free(&(data.data)); + continue; } - } - if (!found) { - /* check if we have space for one more element */ - if (data->pos >= data->alloc - 2) { - data->alloc += 4; - data->csns = (CSN **)slapi_ch_realloc((void *)data->csns, - data->alloc * sizeof(CSN *)); + /* format entry */ + slapi_ch_free(&(key.data)); + rc = cl5DBData2Entry(data.data, data.size, entry, cldb->clcrypt_handle); + slapi_ch_free(&(data.data)); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "_cl5GetFirstOperation - Failed to format entry: %d\n", rc); + goto done; } - data->csns[data->pos] = csn_dup(enum_data->min_csn); - data->pos++; + it = (CL5Iterator *)slapi_ch_malloc(sizeof(CL5Iterator)); + it->cursor = cursor; + it->it_cldb = cldb; + *(CL5Iterator **)iterator = it; + + return CL5_SUCCESS; + } + /* + * Bug 430172 - memory leaks after db "get" deadlocks, e.g. in CL5 trim + * Even when db->c_get() does not return success, memory may have been + * allocated in the DBT. This seems to happen when DB_DBT_MALLOC was set, + * the data being retrieved is larger than the page size, and we got + * DB_LOCK_DEADLOCK. libdb allocates the memory and then finds itself + * deadlocked trying to go through the overflow page list. It returns + * DB_LOCK_DEADLOCK which we've assumed meant that no memory was allocated + * for the DBT. + * + * The following slapi_ch_free frees the memory only when the value is + * non NULL, which is true if the situation described above occurs. + */ + slapi_ch_free((void **)&key.data); + slapi_ch_free((void **)&data.data); + + /* walked of the end of the file */ + if (rc == DB_NOTFOUND) { + rc = CL5_NOTFOUND; + goto done; } - return 0; -} + /* db error occured while iterating */ + /* On this path, the condition "rc != 0" cannot be false */ + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5GetFirstEntry - Failed to get entry; db error - %d %s\n", + rc, db_strerror(rc)); + rc = CL5_DB_ERROR; -static int -my_csn_compare(const void *arg1, const void *arg2) -{ - return (csn_compare(*((CSN **)arg1), *((CSN **)arg2))); -} +done: + /* error occured */ + /* We didn't success in assigning this cursor to the iterator, + * so we need to free the cursor here */ + if (cursor) + cursor->c_close(cursor); + return rc; +} -/* builds CSN ordered list of all csns in the RUV */ -CSN ** -cl5BuildCSNList(const RUV *consRuv, const RUV *supRuv) +static int +_cl5GetNextEntry(CL5Entry *entry, void *iterator) { - struct ruv_it data; - int count, rc; - CSN **csns; + int rc; + CL5Iterator *it; + DBT key = {0}, data = {0}; - PR_ASSERT(consRuv); + PR_ASSERT(entry && iterator); - count = ruv_replica_count(consRuv); - csns = (CSN **)slapi_ch_calloc(count + 1, sizeof(CSN *)); + it = (CL5Iterator *)iterator; - data.csns = csns; - data.alloc = count + 1; - data.pos = 0; + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + while ((rc = it->cursor->c_get(it->cursor, &key, &data, DB_NEXT)) == 0) { + if (cl5HelperEntry((char *)key.data, NULL)) { + slapi_ch_free(&(key.data)); + slapi_ch_free(&(data.data)); + continue; + } - /* add consumer elements to the list */ - rc = ruv_enumerate_elements(consRuv, ruv_consumer_iterator, &data); - if (rc == 0 && supRuv) { - /* add supplier elements to the list */ - rc = ruv_enumerate_elements(supRuv, ruv_supplier_iterator, &data); + slapi_ch_free(&(key.data)); + /* format entry */ + rc = cl5DBData2Entry(data.data, data.size, entry, it->it_cldb->clcrypt_handle); + slapi_ch_free(&(data.data)); + if (rc != 0) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5GetNextEntry - Failed to format entry: %d\n", rc); + } + + return rc; } + /* + * Bug 430172 - memory leaks after db "get" deadlocks, e.g. in CL5 trim + * Even when db->c_get() does not return success, memory may have been + * allocated in the DBT. This seems to happen when DB_DBT_MALLOC was set, + * the data being retrieved is larger than the page size, and we got + * DB_LOCK_DEADLOCK. libdb allocates the memory and then finds itself + * deadlocked trying to go through the overflow page list. It returns + * DB_LOCK_DEADLOCK which we've assumed meant that no memory was allocated + * for the DBT. + * + * The following slapi_ch_free frees the memory only when the value is + * non NULL, which is true if the situation described above occurs. + */ + slapi_ch_free((void **)&key.data); + slapi_ch_free((void **)&data.data); - /* we have no csns */ - if (data.csns[0] == NULL) { - /* csns might have been realloced in ruv_supplier_iterator() */ - slapi_ch_free((void **)&data.csns); - csns = NULL; - } else { - csns = data.csns; - data.csns[data.pos] = NULL; - if (rc == 0) { - qsort(csns, data.pos, sizeof(CSN *), my_csn_compare); - } else { - cl5DestroyCSNList(&csns); - } + /* walked of the end of the file or entry is out of range */ + if (rc == 0 || rc == DB_NOTFOUND) { + return CL5_NOTFOUND; } - return csns; + /* cursor operation failed */ + slapi_log_err(rc == CL5_DB_LOCK_ERROR ? SLAPI_LOG_REPL : SLAPI_LOG_ERR, + repl_plugin_name_cl, + "_cl5GetNextEntry - Failed to get entry; db error - %d %s\n", + rc, db_strerror(rc)); + + return rc; } -void -cl5DestroyCSNList(CSN ***csns) +static int +_cl5CurrentDeleteEntry(void *iterator) { - if (csns && *csns) { - int i; + int rc; + CL5Iterator *it; + cldb_Handle *cldb; - for (i = 0; (*csns)[i]; i++) { - csn_free(&(*csns)[i]); - } + PR_ASSERT(iterator); - slapi_ch_free((void **)csns); + it = (CL5Iterator *)iterator; + + rc = it->cursor->c_del(it->cursor, 0); + + if (rc == 0) { + /* decrement entry count */ + cldb = it->it_cldb; + PR_AtomicDecrement(&cldb->entryCount); + return CL5_SUCCESS; + } else { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5CurrentDeleteEntry - Failed, err=%d %s\n", + rc, db_strerror(rc)); + /* + * We don't free(close) the cursor here, as the caller will free it by + * a call to cl5DestroyIterator. Freeing it here is a potential bug, + * as the cursor can't be referenced later once freed. + */ + return rc; } } -/* A csn should be in the changelog if it is larger than purge vector csn for the same - replica and is smaller than the csn in supplier's ruv for the same replica. - The functions returns - CL5_PURGED if data was purged from the changelog or was never logged - because it was loaded as part of replica initialization - CL5_MISSING if the data erouneously missing - CL5_SUCCESS if that has not and should not been seen by the server - */ -static int -_cl5CheckMissingCSN(const CSN *csn, const RUV *supplierRuv, CL5DBFile *file) +PRBool +cl5HelperEntry(const char *csnstr, CSN *csnp) { - ReplicaId rid; - CSN *supplierCsn = NULL; - CSN *purgeCsn = NULL; - int rc = CL5_SUCCESS; - char csnStr[CSN_STRSIZE]; - - PR_ASSERT(csn && supplierRuv && file); + CSN *csn; + time_t csnTime; + PRBool retval = PR_FALSE; - rid = csn_get_replicaid(csn); - ruv_get_largest_csn_for_replica(supplierRuv, rid, &supplierCsn); - if (supplierCsn == NULL) { - /* we have not seen any changes from this replica so it is - ok not to have this csn */ - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " - "can't locate %s csn: we have not seen any changes for replica %d\n", - csn_as_string(csn, PR_FALSE, csnStr), rid); - } - return CL5_SUCCESS; + if (csnp) { + csn = csnp; + } else { + csn = csn_new_by_string(csnstr); } - - ruv_get_largest_csn_for_replica(file->purgeRUV, rid, &purgeCsn); - if (purgeCsn == NULL) { - /* changelog never contained any changes for this replica */ - if (csn_compare(csn, supplierCsn) <= 0) { - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " - "the change with %s csn was never logged because it was imported " - "during replica initialization\n", - csn_as_string(csn, PR_FALSE, csnStr)); - } - rc = CL5_PURGED_DATA; /* XXXggood is that the correct return value? */ - } else { - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " - "change with %s csn has not yet been seen by this server; " - " last csn seen from that replica is %s\n", - csn_as_string(csn, PR_FALSE, csnStr), - csn_as_string(supplierCsn, PR_FALSE, csnStr)); - } - rc = CL5_SUCCESS; - } - } else /* we have both purge and supplier csn */ - { - if (csn_compare(csn, purgeCsn) < 0) /* the csn is below the purge point */ - { - rc = CL5_PURGED_DATA; - } else { - if (csn_compare(csn, supplierCsn) <= 0) /* we should have the data but we don't */ - { - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " - "change with %s csn has been purged by this server; " - "the current purge point for that replica is %s\n", - csn_as_string(csn, PR_FALSE, csnStr), - csn_as_string(purgeCsn, PR_FALSE, csnStr)); - } - rc = CL5_MISSING_DATA; - } else { - if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " - "change with %s csn has not yet been seen by this server; " - " last csn seen from that replica is %s\n", - csn_as_string(csn, PR_FALSE, csnStr), - csn_as_string(supplierCsn, PR_FALSE, csnStr)); - } - rc = CL5_SUCCESS; - } - } + if (csn == NULL) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, + "cl5HelperEntry - Failed to get csn time; csn error\n"); + return PR_FALSE; } + csnTime = csn_get_time(csn); - if (supplierCsn) - csn_free(&supplierCsn); - - if (purgeCsn) - csn_free(&purgeCsn); + if (csnTime == ENTRY_COUNT_TIME || csnTime == PURGE_RUV_TIME) { + retval = PR_TRUE; + } - return rc; + if (NULL == csnp) + csn_free(&csn); + return retval; } -/* Helper functions that work with individual changelog files */ - -/* file name format : _db{2,3,...} */ +#ifdef FOR_DEBUGGING +/* Replay iteration helper functions */ static PRBool -_cl5FileName2Replica(const char *file_name, Replica **replica) +_cl5ValidReplayIterator(const CL5ReplayIterator *iterator) { - char *repl_name, *file_gen, *repl_gen; - int len; - - PR_ASSERT(file_name && replica); - - *replica = NULL; - - /* this is database file */ - if (_cl5FileEndsWith(file_name, DB_EXTENSION) || - _cl5FileEndsWith(file_name, DB_EXTENSION_DB4) || - _cl5FileEndsWith(file_name, DB_EXTENSION_DB3)) { - repl_name = slapi_ch_strdup(file_name); - file_gen = strstr(repl_name, FILE_SEP); - if (file_gen) { - int extlen = strlen(DB_EXTENSION); - *file_gen = '\0'; - file_gen += strlen(FILE_SEP); - len = strlen(file_gen); - if (len <= extlen + 1) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5FileName2Replica - " - "Invalid file name (%s)\n", - file_name); - } else { - /* get rid of the file extension */ - file_gen[len - extlen - 1] = '\0'; - *replica = replica_get_by_name(repl_name); - if (*replica) { - repl_gen = replica_get_generation(*replica); - PR_ASSERT(repl_gen); - if (strcmp(file_gen, repl_gen) != 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5FileName2Replica - " - "Replica generation mismatch for replica at (%s), " - "file generation %s, new replica generation %s\n", - slapi_sdn_get_dn(replica_get_root(*replica)), file_gen, repl_gen); - - *replica = NULL; - } - slapi_ch_free((void **)&repl_gen); - } - } - slapi_ch_free((void **)&repl_name); - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5FileName2Replica - " - "Malformed file name - %s\n", - file_name); - } - - return PR_TRUE; - } else + if (iterator == NULL || + iterator->consumerRuv == NULL || iterator->supplierRuvObj == NULL || + iterator->it_cldb == NULL) return PR_FALSE; + + return PR_TRUE; } +#endif -/* file name format : _db{2,3} */ -static char * -_cl5Replica2FileName(Replica *r) +/* Algorithm: ONREPL!!! + */ +struct replica_hash_entry { - const char *replName; - char *replGen, *fileName; - - PR_ASSERT(r); + ReplicaId rid; /* replica id */ + PRBool sendChanges; /* indicates whether changes should be sent for this replica */ +}; - replName = replica_get_name(r); - replGen = replica_get_generation(r); - fileName = _cl5MakeFileName(replName, replGen); +static int +_cl5PositionCursorForReplay(ReplicaId consumerRID, const RUV *consumerRuv, Replica *replica, CL5ReplayIterator **iterator, int *continue_on_missing) +{ + CLC_Buffer *clcache = NULL; + CSN *startCSN = NULL; + char csnStr[CSN_STRSIZE]; + int rc = CL5_SUCCESS; + Object *supplierRuvObj = NULL; + RUV *supplierRuv = NULL; + PRBool haveChanges = PR_FALSE; + char *agmt_name; - slapi_ch_free((void **)&replGen); + cldb_Handle *cldb = replica_get_file_info(replica); + PR_ASSERT (consumerRuv && replica && iterator); + + csnStr[0] = '\0'; - return fileName; -} + /* get supplier's RUV */ + supplierRuvObj = replica_get_ruv(replica); + PR_ASSERT(supplierRuvObj); -static char * -_cl5MakeFileName(const char *replName, const char *replGen) -{ - char *fileName = slapi_ch_smprintf("%s/%s%s%s.%s", - s_cl5Desc.dbDir, replName, - FILE_SEP, replGen, DB_EXTENSION); + if (!supplierRuvObj) { + rc = CL5_UNKNOWN_ERROR; + goto done; + } - return fileName; -} + supplierRuv = (RUV *)object_get_data(supplierRuvObj); + PR_ASSERT(supplierRuv); -/* open file that corresponds to a particular database */ -static int -_cl5DBOpenFile(Replica *replica, Object **obj, PRBool checkDups) -{ - int rc; - const char *replName; - char *replGen; + agmt_name = get_thread_private_agmtname(); - PR_ASSERT(replica); + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5PositionCursorForReplay - (%s): Consumer RUV:\n", agmt_name); + ruv_dump(consumerRuv, agmt_name, NULL); + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5PositionCursorForReplay - (%s): Supplier RUV:\n", agmt_name); + ruv_dump(supplierRuv, agmt_name, NULL); + } - replName = replica_get_name(replica); - PR_ASSERT(replName); - replGen = replica_get_generation(replica); - PR_ASSERT(replGen); - rc = _cl5DBOpenFileByReplicaName(replName, replGen, obj, checkDups); + /* initialize the changelog buffer and do the initial load */ - slapi_ch_free((void **)&replGen); + rc = clcache_get_buffer(&clcache, cldb->db, consumerRID, consumerRuv, supplierRuv); + if (rc != 0) + goto done; - return rc; -} + rc = clcache_load_buffer(clcache, &startCSN, continue_on_missing); -static int -_cl5DBOpenFileByReplicaName(const char *replName, const char *replGen, Object **obj, PRBool checkDups) -{ - int rc = CL5_SUCCESS; - Object *tmpObj; - CL5DBFile *file; - char *file_name; - - PR_ASSERT(replName && replGen); - - if (checkDups) { - PR_Lock(s_cl5Desc.fileLock); - file_name = _cl5MakeFileName(replName, replGen); - tmpObj = objset_find(s_cl5Desc.dbFiles, _cl5CompareDBFile, file_name); - slapi_ch_free((void **)&file_name); - if (tmpObj) /* this file already exist */ - { + if (rc == 0) { + haveChanges = PR_TRUE; + rc = CL5_SUCCESS; + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + csn_as_string(startCSN, PR_FALSE, csnStr); slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBOpenFileByReplicaName - Found DB object %p for replica %s\n", tmpObj, replName); - /* if we were asked for file handle - keep the handle */ - if (obj) { - *obj = tmpObj; - } else { - object_release(tmpObj); - } - - rc = CL5_SUCCESS; + "%s: CSN %s found, position set for replay\n", agmt_name, csnStr); + } + } else if (rc == DB_NOTFOUND) { + /* buffer not loaded. + * either because no changes have to be sent ==> startCSN is NULL + * or the calculated startCSN cannot be found in the changelog + */ + if (startCSN == NULL) { + rc = CL5_NOTFOUND; goto done; } - } - - rc = _cl5NewDBFile(replName, replGen, &file); - if (rc == CL5_SUCCESS) { - /* This creates the file but doesn't set the init flag - * The flag is set later when the purge and max ruvs are set. - * This is to prevent some thread to get file access before the - * structure is fully initialized */ - rc = _cl5AddDBFile(file, &tmpObj); - if (rc == CL5_SUCCESS) { - /* read purge RUV - done here because it needs file object rather than file pointer */ - rc = _cl5ReadRUV(replGen, tmpObj, PR_TRUE); - if (rc != CL5_SUCCESS) { + /* check whether this csn should be present */ + rc = _cl5CheckMissingCSN(startCSN, supplierRuv, cldb); + if (rc == CL5_MISSING_DATA) /* we should have had the change but we don't */ + { + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + csn_as_string(startCSN, PR_FALSE, csnStr); slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBOpenFileByReplicaName - Failed to get purge RUV\n"); - goto done; + "repl_plugin_name_cl - %s: CSN %s not found, seems to be missing\n", agmt_name, csnStr); } + } else /* we are not as up to date or we purged */ + { + csn_as_string(startCSN, PR_FALSE, csnStr); + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "repl_plugin_name_cl - %s: CSN %s not found, we aren't as up to date, or we purged\n", + agmt_name, csnStr); + } + } else { + csn_as_string(startCSN, PR_FALSE, csnStr); + /* db error */ + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "repl_plugin_name_cl - %s: Failed to retrieve change with CSN %s; db error - %d %s\n", + agmt_name, csnStr, rc, db_strerror(rc)); - /* read ruv that represents the upper bound of the changes stored in the file */ - rc = _cl5ReadRUV(replGen, tmpObj, PR_FALSE); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBOpenFileByReplicaName - Failed to get upper bound RUV\n"); - goto done; - } + rc = CL5_DB_ERROR; + } - /* Mark the DB File initialize */ - _cl5DBFileInitialized(tmpObj); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBOpenFileByReplicaName - Created new DB object %p\n", tmpObj); - if (obj) { - *obj = tmpObj; - } else { - object_release(tmpObj); - } + /* setup the iterator */ + if (haveChanges) { + *iterator = (CL5ReplayIterator *)slapi_ch_calloc(1, sizeof(CL5ReplayIterator)); + + if (*iterator == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "_cl5PositionCursorForReplay - %s - Failed to allocate iterator\n", agmt_name); + rc = CL5_MEMORY_ERROR; + goto done; } - } -done:; - if (rc != CL5_SUCCESS) { - if (file) - _cl5DBCloseFile((void **)&file); + /* ONREPL - should we make a copy of both RUVs here ?*/ + (*iterator)->it_cldb = cldb; + (*iterator)->clcache = clcache; + clcache = NULL; + (*iterator)->consumerRID = consumerRID; + (*iterator)->consumerRuv = consumerRuv; + (*iterator)->supplierRuvObj = supplierRuvObj; + } else if (rc == CL5_SUCCESS) { + /* we have no changes to send */ + rc = CL5_NOTFOUND; } - if (checkDups) { - PR_Unlock(s_cl5Desc.fileLock); +done: + if (clcache) + clcache_return_buffer(&clcache); + + if (rc != CL5_SUCCESS) { + if (supplierRuvObj) + object_release(supplierRuvObj); } return rc; } -/* adds file to the db file list */ +struct ruv_it +{ + CSN **csns; /* csn list */ + int alloc; /* allocated size */ + int pos; /* position in the list */ +}; + static int -_cl5AddDBFile(CL5DBFile *file, Object **obj) +ruv_consumer_iterator(const ruv_enum_data *enum_data, void *arg) { - int rc; - Object *tmpObj; + struct ruv_it *data = (struct ruv_it *)arg; - PR_ASSERT(file); + PR_ASSERT(data); - tmpObj = object_new(file, _cl5DBCloseFile); - rc = objset_add_obj(s_cl5Desc.dbFiles, tmpObj); - if (rc != OBJSET_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5AddDBFile - Failed to add db file to the list; " - "repl_objset error - %d\n", - rc); - object_release(tmpObj); - return CL5_OBJSET_ERROR; - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5AddDBFile - Added new DB object %p\n", tmpObj); + /* check if we have space for one more element */ + if (data->pos >= data->alloc - 2) { + data->alloc += 4; + data->csns = (CSN **)slapi_ch_realloc((void *)data->csns, data->alloc * sizeof(CSN *)); } - if (obj) { - *obj = tmpObj; - } else - object_release(tmpObj); + data->csns[data->pos] = csn_dup(enum_data->csn); + data->pos++; - return CL5_SUCCESS; + return 0; } + static int -_cl5NewDBFile(const char *replName, const char *replGen, CL5DBFile **dbFile) +ruv_supplier_iterator(const ruv_enum_data *enum_data, void *arg) { - int rc; - DB *db = NULL; - char *name; -#ifdef HPUX - char cwd[PATH_MAX + 1]; -#endif - - PR_ASSERT(replName && replGen && dbFile); - - if (!dbFile) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5NewDBFile - NULL dbFile\n"); - return CL5_UNKNOWN_ERROR; - } - - (*dbFile) = (CL5DBFile *)slapi_ch_calloc(1, sizeof(CL5DBFile)); - if (*dbFile == NULL) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5NewDBFile - memory allocation failed\n"); - return CL5_MEMORY_ERROR; - } + int i; + PRBool found = PR_FALSE; + ReplicaId rid; + struct ruv_it *data = (struct ruv_it *)arg; - name = _cl5MakeFileName(replName, replGen); - { - /* The subname argument allows applications to have - * subdatabases, i.e., multiple databases inside of a single - * physical file. This is useful when the logical databases - * are both numerous and reasonably small, in order to - * avoid creating a large number of underlying files. - */ - char *subname = NULL; - DB_ENV *dbEnv = s_cl5Desc.dbEnv; + PR_ASSERT(data); - rc = db_create(&db, dbEnv, 0); - if (0 != rc) { - goto out; - } + rid = csn_get_replicaid(enum_data->min_csn); + /* check if the replica that generated the csn is already in the list */ + for (i = 0; i < data->pos; i++) { + if (rid == csn_get_replicaid(data->csns[i])) { + found = PR_TRUE; - rc = db->set_pagesize( - db, - s_cl5Desc.dbConfig.pageSize); + /* remove datacsn[i] if it is greater or equal to the supplier's maxcsn */ + if (csn_compare(data->csns[i], enum_data->csn) >= 0) { + int j; - if (0 != rc) { - goto out; + csn_free(&data->csns[i]); + for (j = i + 1; j < data->pos; j++) { + data->csns[j - 1] = data->csns[j]; + } + data->pos--; + } + break; } - - DB_OPEN(s_cl5Desc.dbEnvOpenFlags, - db, NULL /* txnid */, name, subname, DB_BTREE, - DB_CREATE | DB_THREAD, s_cl5Desc.dbConfig.fileMode, rc); - } -out: - if (rc != 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "_cl5NewDBFile - db_open failed; db error - %d %s\n", - rc, db_strerror(rc)); - rc = CL5_DB_ERROR; - goto done; } - (*dbFile)->db = db; - (*dbFile)->name = name; - name = NULL; /* transfer ownership to dbFile struct */ - (*dbFile)->replName = slapi_ch_strdup(replName); - (*dbFile)->replGen = slapi_ch_strdup(replGen); - - - /* compute number of entries in the file */ - /* ONREPL - to improve performance, we keep entry count in memory - and write it down during shutdown. Problem: this will not - work with multiple processes. Do we have to worry about that? - */ - if (s_cl5Desc.dbOpenMode == CL5_OPEN_NORMAL) { - rc = _cl5GetEntryCount(*dbFile); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5NewDBFile - Failed to get entry count\n"); - goto done; + if (!found) { + /* check if we have space for one more element */ + if (data->pos >= data->alloc - 2) { + data->alloc += 4; + data->csns = (CSN **)slapi_ch_realloc((void *)data->csns, + data->alloc * sizeof(CSN *)); } - } - -done: - if (rc != CL5_SUCCESS) { - _cl5DBCloseFile((void **)dbFile); - /* slapi_ch_free accepts NULL pointer */ - slapi_ch_free((void **)&name); - slapi_ch_free((void **)dbFile); + data->csns[data->pos] = csn_dup(enum_data->min_csn); + data->pos++; } - - return rc; + return 0; } -static void -_cl5DBCloseFile(void **data) + +static int +my_csn_compare(const void *arg1, const void *arg2) { - CL5DBFile *file; - int rc = 0; + return (csn_compare(*((CSN **)arg1), *((CSN **)arg2))); +} - PR_ASSERT(data); - file = *(CL5DBFile **)data; +/* builds CSN ordered list of all csns in the RUV */ +CSN ** +cl5BuildCSNList(const RUV *consRuv, const RUV *supRuv) +{ + struct ruv_it data; + int count, rc; + CSN **csns; + + PR_ASSERT(consRuv); - PR_ASSERT(file); + count = ruv_replica_count(consRuv); + csns = (CSN **)slapi_ch_calloc(count + 1, sizeof(CSN *)); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBCloseFile - " - "Closing database %s\n", - file->name); + data.csns = csns; + data.alloc = count + 1; + data.pos = 0; - /* close the file */ - /* if this is normal close or close after import, update entry count */ - if ((s_cl5Desc.dbOpenMode == CL5_OPEN_NORMAL && s_cl5Desc.dbState == CL5_STATE_CLOSING) || - s_cl5Desc.dbOpenMode == CL5_OPEN_LDIF2CL) { - _cl5WriteEntryCount(file); - _cl5WriteRUV(file, PR_TRUE); - _cl5WriteRUV(file, PR_FALSE); + /* add consumer elements to the list */ + rc = ruv_enumerate_elements(consRuv, ruv_consumer_iterator, &data); + if (rc == 0 && supRuv) { + /* add supplier elements to the list */ + rc = ruv_enumerate_elements(supRuv, ruv_supplier_iterator, &data); } - /* close the db */ - if (file->db) { - rc = file->db->close(file->db, 0); - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "_cl5DBCloseFile - " - "Closed the changelog database handle for %s " - "(rc: %d)\n", - file->name, rc); - file->db = NULL; - } - - if (file->flags & DB_FILE_DELETED) { - /* We need to use the libdb API to delete the files, otherwise we'll - * run into problems when we try to checkpoint transactions later. */ - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBCloseFile - " - "removing the changelog %s (flag %d)\n", - file->name, DEFAULT_DB_ENV_OP_FLAGS); - rc = s_cl5Desc.dbEnv->dbremove(s_cl5Desc.dbEnv, 0, file->name, 0, - DEFAULT_DB_ENV_OP_FLAGS); - if (rc != 0) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBCloseFile - " - "failed to remove (%s) file; libdb error - %d (%s)\n", - file->name, rc, db_strerror(rc)); + /* we have no csns */ + if (data.csns[0] == NULL) { + /* csns might have been realloced in ruv_supplier_iterator() */ + slapi_ch_free((void **)&data.csns); + csns = NULL; + } else { + csns = data.csns; + data.csns[data.pos] = NULL; + if (rc == 0) { + qsort(csns, data.pos, sizeof(CSN *), my_csn_compare); } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBCloseFile - " - "Deleted the changelog database file %s\n", - file->name); + cl5DestroyCSNList(&csns); } } - /* slapi_ch_free accepts NULL pointer */ - slapi_ch_free((void **)&file->name); - slapi_ch_free((void **)&file->replName); - slapi_ch_free((void **)&file->replGen); - ruv_destroy(&file->maxRUV); - ruv_destroy(&file->purgeRUV); - file->db = NULL; - - slapi_ch_free(data); + return csns; } -static int -_cl5GetDBFile(Replica *replica, Object **obj) +void +cl5DestroyCSNList(CSN ***csns) { - char *fileName; - - PR_ASSERT(replica && obj); + if (csns && *csns) { + int i; - fileName = _cl5Replica2FileName(replica); + for (i = 0; (*csns)[i]; i++) { + csn_free(&(*csns)[i]); + } - *obj = objset_find(s_cl5Desc.dbFiles, _cl5CompareDBFile, fileName); - if (*obj) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5GetDBFile - " - "found DB object %p for database %s\n", - *obj, fileName); - slapi_ch_free_string(&fileName); - return CL5_SUCCESS; - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5GetDBFile - " - "no DB object found for database %s\n", - fileName); - slapi_ch_free_string(&fileName); - return CL5_NOTFOUND; + slapi_ch_free((void **)csns); } } +/* A csn should be in the changelog if it is larger than purge vector csn for the same + replica and is smaller than the csn in supplier's ruv for the same replica. + The functions returns + CL5_PURGED if data was purged from the changelog or was never logged + because it was loaded as part of replica initialization + CL5_MISSING if the data erouneously missing + CL5_SUCCESS if that has not and should not been seen by the server + */ static int -_cl5GetDBFileByReplicaName(const char *replName, const char *replGen, Object **obj) +_cl5CheckMissingCSN(const CSN *csn, const RUV *supplierRuv, cldb_Handle *cldb) { - char *fileName; - - PR_ASSERT(replName && replGen && obj); + ReplicaId rid; + CSN *supplierCsn = NULL; + CSN *purgeCsn = NULL; + int rc = CL5_SUCCESS; + char csnStr[CSN_STRSIZE]; - fileName = _cl5MakeFileName(replName, replGen); + PR_ASSERT(csn && supplierRuv && cldb); - *obj = objset_find(s_cl5Desc.dbFiles, _cl5CompareDBFile, fileName); - if (*obj) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5GetDBFileByReplicaName - " - "found DB object %p for database %s\n", - *obj, fileName); - slapi_ch_free_string(&fileName); + rid = csn_get_replicaid(csn); + ruv_get_largest_csn_for_replica(supplierRuv, rid, &supplierCsn); + if (supplierCsn == NULL) { + /* we have not seen any changes from this replica so it is + ok not to have this csn */ + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " + "can't locate %s csn: we have not seen any changes for replica %d\n", + csn_as_string(csn, PR_FALSE, csnStr), rid); + } return CL5_SUCCESS; - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5GetDBFileByReplicaName - " - "no DB object found for database %s\n", - fileName); - slapi_ch_free_string(&fileName); - return CL5_NOTFOUND; } -} - -static void -_cl5DBDeleteFile(Object *file_obj) -{ - CL5DBFile *file; - int rc = 0; - PR_ASSERT(file_obj); - - file = (CL5DBFile *)object_get_data(file_obj); - PR_ASSERT(file); - file->flags |= DB_FILE_DELETED; - rc = objset_remove_obj(s_cl5Desc.dbFiles, file_obj); - if (rc != OBJSET_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBDeleteFile - " - "could not find DB object %p\n", - file_obj); - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5DBDeleteFile - " - "removed DB object %p\n", - file_obj); + ruv_get_largest_csn_for_replica(cldb->purgeRUV, rid, &purgeCsn); + if (purgeCsn == NULL) { + /* changelog never contained any changes for this replica */ + if (csn_compare(csn, supplierCsn) <= 0) { + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " + "the change with %s csn was never logged because it was imported " + "during replica initialization\n", + csn_as_string(csn, PR_FALSE, csnStr)); + } + rc = CL5_PURGED_DATA; /* XXXggood is that the correct return value? */ + } else { + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " + "change with %s csn has not yet been seen by this server; " + " last csn seen from that replica is %s\n", + csn_as_string(csn, PR_FALSE, csnStr), + csn_as_string(supplierCsn, PR_FALSE, csnStr)); + } + rc = CL5_SUCCESS; + } + } else /* we have both purge and supplier csn */ + { + if (csn_compare(csn, purgeCsn) < 0) /* the csn is below the purge point */ + { + rc = CL5_PURGED_DATA; + } else { + if (csn_compare(csn, supplierCsn) <= 0) /* we should have the data but we don't */ + { + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " + "change with %s csn has been purged by this server; " + "the current purge point for that replica is %s\n", + csn_as_string(csn, PR_FALSE, csnStr), + csn_as_string(purgeCsn, PR_FALSE, csnStr)); + } + rc = CL5_MISSING_DATA; + } else { + if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { + slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5CheckMissingCSN - " + "change with %s csn has not yet been seen by this server; " + " last csn seen from that replica is %s\n", + csn_as_string(csn, PR_FALSE, csnStr), + csn_as_string(supplierCsn, PR_FALSE, csnStr)); + } + rc = CL5_SUCCESS; + } + } } - object_release(file_obj); -} - -static void -_cl5DBFileInitialized(Object *file_obj) -{ - CL5DBFile *file; - - PR_ASSERT(file_obj); - - file = (CL5DBFile *)object_get_data(file_obj); - PR_ASSERT(file); - file->flags |= DB_FILE_INIT; -} -static int -_cl5CompareDBFile(Object *el1, const void *el2) -{ - CL5DBFile *file; - const char *name; + if (supplierCsn) + csn_free(&supplierCsn); - PR_ASSERT(el1 && el2); + if (purgeCsn) + csn_free(&purgeCsn); - file = (CL5DBFile *)object_get_data(el1); - name = (const char *)el2; - return ((file->flags & DB_FILE_INIT) ? strcmp(file->name, name) : 1); + return rc; } -/* - * return 1: true (the "filename" ends with "ext") - * return 0: false - */ -static int -_cl5FileEndsWith(const char *filename, const char *ext) -{ - char *p = NULL; - int flen = strlen(filename); - int elen = strlen(ext); - if (0 == flen || 0 == elen) { - return 0; - } - p = PL_strrstr(filename, ext); - if (NULL == p) { - return 0; - } - if (p - filename + elen == flen) { - return 1; - } - return 0; -} +/* Helper functions that work with individual changelog files */ static int -_cl5ExportFile(PRFileDesc *prFile, Object *obj) +_cl5ExportFile(PRFileDesc *prFile, cldb_Handle *cldb) { int rc; void *iterator = NULL; @@ -5821,23 +4373,19 @@ _cl5ExportFile(PRFileDesc *prFile, Object *obj) char *buff; PRInt32 len, wlen; CL5Entry entry; - CL5DBFile *file; - PR_ASSERT(prFile && obj); - - file = (CL5DBFile *)object_get_data(obj); - PR_ASSERT(file); + PR_ASSERT(prFile && cldb); if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) { - ruv_dump(file->purgeRUV, "clpurgeruv", prFile); - ruv_dump(file->maxRUV, "clmaxruv", prFile); + ruv_dump(cldb->purgeRUV, "clpurgeruv", prFile); + ruv_dump(cldb->maxRUV, "clmaxruv", prFile); } slapi_write_buffer(prFile, "\n", strlen("\n")); entry.op = &op; - rc = _cl5GetFirstEntry(obj, &entry, &iterator, NULL); + rc = _cl5GetFirstEntry(cldb, &entry, &iterator, NULL); while (rc == CL5_SUCCESS) { - rc = _cl5Operation2LDIF(&op, file->replGen, &buff, &len); + rc = _cl5Operation2LDIF(&op, cldb->ident, &buff, &len); if (rc != CL5_SUCCESS) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "_cl5ExportFile - Failed to convert operation to ldif\n"); @@ -5875,24 +4423,6 @@ _cl5ExportFile(PRFileDesc *prFile, Object *obj) return rc; } -static PRBool -_cl5ReplicaInList(Replica *replica, Replica **replicas) -{ - int i; - - PR_ASSERT(replica && replicas); - - /* ONREPL I think it should be sufficient to just compare replica pointers */ - /* LK not sure about this, but it is only used for changlog import for - * multiple changelogs, shozld probably be deprecated anyway */ - for (i = 0; replicas[i]; i++) { - if (replica == replicas[i]) - return PR_TRUE; - } - - return PR_FALSE; -} - static char * _cl5GetHelperEntryKey(int type, char *csnStr) { @@ -5908,338 +4438,106 @@ _cl5GetHelperEntryKey(int type, char *csnStr) return rt; } -static Replica * -_cl5GetReplica(const slapi_operation_parameters *op, const char *replGen) +/* + * Write RUVs into the changelog; + * implemented for backup to make sure the backed up changelog contains RUVs + * Return values: 0 -- success + * 1 -- failure + */ +static int +_cl5WriteReplicaRUV(Replica *r, void *arg) { - Slapi_DN *sdn; - Replica *replica; - char *newGen; - - PR_ASSERT(op && replGen); - - sdn = op->target_address.sdn; - - replica = replica_get_replica_from_dn(sdn); - if (replica) { - /* check to see if replica generation has not change */ - newGen = replica_get_generation(replica); - PR_ASSERT(newGen); - if (strcmp(replGen, newGen) != 0) { - replica = NULL; - } - - slapi_ch_free((void **)&newGen); + int rc = 0; + cldb_Handle *cldb = replica_get_file_info(r); + + if (NULL == cldb) { + /* TBD should this really happen, do we need an error msg */ + return rc; } - return replica; -} + _cl5WriteEntryCount(cldb); + rc = _cl5WriteRUV(cldb, PR_TRUE); + rc = _cl5WriteRUV(cldb, PR_FALSE); + ruv_destroy(&cldb->maxRUV); + ruv_destroy(&cldb->purgeRUV); -int -cl5_is_diskfull() -{ - int rc; - PR_Lock(cl5_diskfull_lock); - rc = cl5_diskfull_flag; - PR_Unlock(cl5_diskfull_lock); return rc; } -static void -cl5_set_diskfull(void) -{ - PR_Lock(cl5_diskfull_lock); - cl5_diskfull_flag = 1; - PR_Unlock(cl5_diskfull_lock); -} - -static void -cl5_set_no_diskfull(void) -{ - PR_Lock(cl5_diskfull_lock); - cl5_diskfull_flag = 0; - PR_Unlock(cl5_diskfull_lock); -} - -int -cl5_diskspace_is_available() +static char * +_cl5LdifFileName(char *instance_ldif) { - int rval = 1; + char *cl_ldif = NULL; + char *p =strstr(instance_ldif, ".ldif"); -#if defined(OS_solaris) || defined(hpux) - struct statvfs fsbuf; - if (statvfs(s_cl5Desc.dbDir, &fsbuf) < 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5_diskspace_is_available - Cannot get file system info\n"); - rval = 0; - } else { - unsigned long fsiz = fsbuf.f_bavail * fsbuf.f_frsize; - if (fsiz < NO_DISK_SPACE) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5_diskspace_is_available - No enough diskspace for changelog: (%u bytes free)\n", fsiz); - rval = 0; - } else if (fsiz > MIN_DISK_SPACE) { - /* assume recovered */ - cl5_set_no_diskfull(); - } - } -#endif -#if defined(linux) - struct statfs fsbuf; - if (statfs(s_cl5Desc.dbDir, &fsbuf) < 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5_diskspace_is_available - Cannot get file system info\n"); - rval = 0; + if (p) { + *p = '\0'; + cl_ldif = slapi_ch_smprintf("%s_cl.ldif", instance_ldif); } else { - unsigned long fsiz = fsbuf.f_bavail * fsbuf.f_bsize; - if (fsiz < NO_DISK_SPACE) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5_diskspace_is_available - No enough diskspace for changelog: (%lu bytes free)\n", fsiz); - rval = 0; - } else if (fsiz > MIN_DISK_SPACE) { - /* assume recovered */ - cl5_set_no_diskfull(); - } + cl_ldif = slapi_ch_smprintf("%s_cl", instance_ldif); } -#endif - return rval; -} - -int -cl5DbDirIsEmpty(const char *dir) -{ - PRDir *prDir; - PRDirEntry *prDirEntry; - int isempty = 1; - if (!dir || !*dir) { - return isempty; - } - /* assume failure means it does not exist - other failure - cases will be handled by code which attempts to create the - db in this directory */ - if (PR_Access(dir, PR_ACCESS_EXISTS)) { - return isempty; - } - prDir = PR_OpenDir(dir); - if (prDir == NULL) { - return isempty; /* assume failure means does not exist */ - } - while (NULL != (prDirEntry = PR_ReadDir(prDir, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { - if (NULL == prDirEntry->name) { /* NSPR doesn't behave like the docs say it should */ - break; - } - isempty = 0; /* found at least one "real" file */ - break; - } - PR_CloseDir(prDir); + return cl_ldif; - return isempty; } -/* - * Write RUVs into the changelog; - * implemented for backup to make sure the backed up changelog contains RUVs - * Return values: 0 -- success - * 1 -- failure - */ int -cl5WriteRUV() +cl5Import(Slapi_PBlock *pb) { - int rc = 0; - Object *file_obj = NULL; - CL5DBFile *dbfile = NULL; - int closeit = 0; - int slapd_pid = 0; - - changelog5Config config; - - /* read changelog configuration */ - changelog5_read_config(&config); - if (config.dir == NULL) { - /* Changelog is not configured; Replication is not enabled. - * we don't have to update RUVs. - * bail out - return success */ - goto bail; - } - - slapd_pid = is_slapd_running(); - if (slapd_pid <= 0) { - /* I'm not a server, rather a utility. - * And the server is NOT running. - * RUVs should be in the changelog. - * we don't have to update RUVs. - * bail out - return success */ - goto bail; - } - - if (getpid() != slapd_pid) { - /* I'm not a server, rather a utility. - * And the server IS running. - * RUVs are not in the changelog and no easy way to retrieve them. - * bail out - return failure */ - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5WriteRUV - server (pid %d) is already running; bail.\n", - slapd_pid); - rc = 1; - goto bail; - } - - /* file is stored in the changelog directory and is named - * .ldif */ - if (CL5_STATE_OPEN != s_cl5Desc.dbState) { - rc = _cl5Open(config.dir, &config.dbconfig, CL5_OPEN_NORMAL); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5WriteRUV - Failed to open changelog\n"); - goto bail; - } - s_cl5Desc.dbState = CL5_STATE_OPEN; /* force to change the state */ - closeit = 1; /* It had not been opened; close it */ - } + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cl5Export - Importing changelog\n"); - file_obj = objset_first_obj(s_cl5Desc.dbFiles); - while (file_obj) { - dbfile = (CL5DBFile *)object_get_data(file_obj); - if (dbfile) { - _cl5WriteEntryCount(dbfile); - _cl5WriteRUV(dbfile, PR_TRUE); - _cl5WriteRUV(dbfile, PR_FALSE); - } - file_obj = objset_next_obj(s_cl5Desc.dbFiles, file_obj); - } -bail: - if (closeit && (CL5_STATE_OPEN == s_cl5Desc.dbState)) { - _cl5Close(); - s_cl5Desc.dbState = CL5_STATE_CLOSED; /* force to change the state */ - } - changelog5_config_done(&config); - return rc; + /* TBD + * as in cl5Export + * get ldif dir from pblock + * generate cl ldif name + * call clImportLDIF + */ + return 0; } -/* - * Delete RUVs from the changelog; - * implemented for backup to clean up RUVs - * Return values: 0 -- success - * 1 -- failure - */ int -cl5DeleteRUV() +cl5Export(Slapi_PBlock *pb) { - int rc = 0; - Object *file_obj = NULL; - CL5DBFile *dbfile = NULL; - int slapd_pid = 0; - int closeit = 0; - - changelog5Config config; - - /* read changelog configuration */ - changelog5_read_config(&config); - if (config.dir == NULL) { - /* Changelog is not configured; Replication is not enabled. - * we don't have to update RUVs. - * bail out - return success */ - goto bail; - } - - slapd_pid = is_slapd_running(); - if (slapd_pid <= 0) { - /* I'm not a server, rather a utility. - * And the server is NOT running. - * RUVs should be in the changelog. - * we don't have to update RUVs. - * bail out - return success */ - goto bail; - } - - if (getpid() != slapd_pid) { - /* I'm not a server, rather a utility. - * And the server IS running. - * RUVs are not in the changelog. - * bail out - return success */ - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5DeleteRUV - server (pid %d) is already running; bail.\n", - slapd_pid); - goto bail; - } + char *instance_name; + char *instance_ldif; + char *instance_cl_ldif; + Slapi_Backend *be; + Replica *replica = NULL; + int rc; - /* file is stored in the changelog directory and is named - * .ldif */ - if (CL5_STATE_OPEN != s_cl5Desc.dbState) { - rc = _cl5Open(config.dir, &config.dbconfig, CL5_OPEN_NORMAL); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5DeleteRUV - Failed to open changelog\n"); - goto bail; - } - s_cl5Desc.dbState = CL5_STATE_OPEN; /* force to change the state */ - closeit = 1; /* It had been opened; no need to close */ + slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); + slapi_pblock_get(pb, SLAPI_DB2LDIF_FILE, &instance_ldif); + slapi_pblock_get(pb, SLAPI_BACKEND, &be); + replica = replica_get_replica_from_dn(slapi_be_getsuffix(be, 0)); + if (replica == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cl5Export - No replica defined for instance %s\n", instance_name); + return 0; } - file_obj = objset_first_obj(s_cl5Desc.dbFiles); - while (file_obj) { - dbfile = (CL5DBFile *)object_get_data(file_obj); + instance_cl_ldif = _cl5LdifFileName(instance_ldif); + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cl5Export - Exporting changelog for instance %s to file %s\n", + instance_name, instance_cl_ldif); + rc = cl5ExportLDIF(instance_cl_ldif, replica); - /* _cl5GetEntryCount deletes entry count after reading it */ - rc = _cl5GetEntryCount(dbfile); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "cl5DeleteRUV - Failed to get/delete entry count\n"); - goto bail; - } - /* _cl5ReadRUV deletes RUV after reading it */ - rc = _cl5ReadRUV(dbfile->replGen, file_obj, PR_TRUE); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5DeleteRUV - Failed to read/delete purge RUV\n"); - goto bail; - } - rc = _cl5ReadRUV(dbfile->replGen, file_obj, PR_FALSE); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "cl5DeleteRUV - Failed to read/delete upper bound RUV\n"); - goto bail; - } - file_obj = objset_next_obj(s_cl5Desc.dbFiles, file_obj); - } -bail: - if (file_obj) { - object_release(file_obj); - } - if (closeit && (CL5_STATE_OPEN == s_cl5Desc.dbState)) { - _cl5Close(); - s_cl5Desc.dbState = CL5_STATE_CLOSED; /* force to change the state */ - } - changelog5_config_done(&config); return rc; } - /* * Clean the in memory RUV, at shutdown we will write the update to the db */ void -cl5CleanRUV(ReplicaId rid) +cl5CleanRUV(ReplicaId rid, Replica *replica) { - CL5DBFile *file; - Object *obj = NULL; - - slapi_rwlock_wrlock(s_cl5Desc.stLock); - - obj = objset_first_obj(s_cl5Desc.dbFiles); - while (obj) { - file = (CL5DBFile *)object_get_data(obj); - ruv_delete_replica(file->purgeRUV, rid); - ruv_delete_replica(file->maxRUV, rid); - obj = objset_next_obj(s_cl5Desc.dbFiles, obj); - } - - slapi_rwlock_unlock(s_cl5Desc.stLock); + cldb_Handle *cldb = replica_get_file_info(replica); + ruv_delete_replica(cldb->purgeRUV, rid); + ruv_delete_replica(cldb->maxRUV, rid); } static void free_purge_data(cleanruv_purge_data *purge_data) { - slapi_ch_free_string(&purge_data->replGen); slapi_ch_free((void **)&purge_data); } @@ -6300,3 +4598,18 @@ trigger_cl_purging_thread(void *arg) free_and_return: free_purge_data(purge_data); } + +char * +cl5GetLdifDir(Slapi_Backend *be) +{ + char *dir = NULL; + char *dbdir = NULL; + + if (NULL == be) { + dir = slapi_ch_strdup("/tmp"); + } else { + slapi_back_get_info(be, BACK_INFO_DIRECTORY, (void **)&dbdir); + dir = slapi_ch_smprintf("%s/../ldif",dbdir); + } + return dir; +} diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h index 302af97..85927a7 100644 --- a/ldap/servers/plugins/replication/cl5_api.h +++ b/ldap/servers/plugins/replication/cl5_api.h @@ -36,15 +36,6 @@ /***** Data Structures *****/ -/* changelog configuration structure */ -typedef struct cl5dbconfig -{ - uint32_t pageSize; /* page size in bytes */ - PRInt32 fileMode; /* file mode */ - char *encryptionAlgorithm; /* nsslapd-encryptionalgorithm */ - char *symmetricKey; -} CL5DBConfig; - /* changelog entry format */ typedef struct cl5entry { @@ -81,6 +72,9 @@ typedef struct cl5entry /* data structure that allows iteration through changelog */ typedef struct cl5replayiterator CL5ReplayIterator; +/* database information for th echangelog */ +typedef struct cl5DBFileHandle cldb_Handle; + /* changelog state */ typedef enum { CL5_STATE_NONE, /* changelog has not been initialized */ @@ -137,9 +131,6 @@ void cl5Cleanup(void); Description: opens changelog ; must be called after changelog is initialized using cl5Init. It is thread safe and the second call is ignored. - Parameters: dir - changelog dir - config - db configuration parameters; currently not used - openMode - open mode Return: CL5_SUCCESS if successful; CL5_BAD_DATA if invalid directory is passed; CL5_BAD_DBVERSION if dbversion file is missing or has unexpected data @@ -147,7 +138,7 @@ void cl5Cleanup(void); CL5_MEMORY_ERROR if memory allocation fails; CL5_DB_ERROR if db initialization or open fails. */ -int cl5Open(const char *dir, const CL5DBConfig *config); +int cl5Open(void); /* Name: cl5Close Description: closes changelog and cleanups changelog module; waits until @@ -159,21 +150,11 @@ int cl5Open(const char *dir, const CL5DBConfig *config); */ int cl5Close(void); -/* Name: cl5Delete - Description: removes changelog - Parameters: dir - changelog directory - Return: CL5_SUCCESS if successful; - CL5_BAD_STATE if the changelog is not in closed state; - CL5_BAD_DATA if invalid directory supplied - CL5_SYSTEM_ERROR if NSPR call fails - */ -int cl5Delete(const char *dir); - -/* Name: cl5DeleteDBSync - Description: The same as cl5DeleteDB except the function does not return - until the file is removed. +/* Name: cldb_RemoveReplicaDB + Description: Clear the cldb information from the replica + and delete the database file */ -int cl5DeleteDBSync(Replica *replica); +int cldb_RemoveReplicaDB(Replica *replica); /* Name: cl5GetUpperBoundRUV Description: retrieves vector that represent the upper bound of changes @@ -202,7 +183,7 @@ int cl5GetUpperBoundRUV(Replica *r, RUV **ruv); CL5_SYSTEM_ERROR if NSPR call fails; CL5_MEMORY_ERROR if memory allocation fails. */ -int cl5ExportLDIF(const char *ldifFile, Replica **replicas); +int cl5ExportLDIF(const char *ldifFile, Replica *replica); /* Name: cl5ImportLDIF Description: imports ldif file into changelog; changelog must be in the closed state @@ -217,7 +198,7 @@ int cl5ExportLDIF(const char *ldifFile, Replica **replicas); CL5_SYSTEM_ERROR if NSPR call fails; CL5_MEMORY_ERROR if memory allocation fails. */ -int cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas); +int cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica *replica); /* Name: cl5GetState Description: returns database state @@ -231,12 +212,11 @@ int cl5GetState(void); Description: sets changelog trimming parameters Parameters: maxEntries - maximum number of entries in the log; maxAge - maximum entry age; - compactInterval - interval to compact changelog db; trimInterval - interval for changelog trimming. Return: CL5_SUCCESS if successful; CL5_BAD_STATE if changelog has not been open */ -int cl5ConfigTrimming(int maxEntries, const char *maxAge, int compactInterval, int trimInterval); +int cl5ConfigTrimming(Replica *replica, int maxEntries, const char *maxAge, int trimInterval); void cl5DestroyIterator(void *iterator); @@ -248,7 +228,6 @@ void cl5DestroyIterator(void *iterator); replica object since generation can change while operation is in progress (if the data is reloaded). !!! op - operation to write - local - this is a non-replicated operation txn - the containing transaction Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid op is passed; @@ -256,7 +235,7 @@ void cl5DestroyIterator(void *iterator); CL5_MEMORY_ERROR if memory allocation failed; CL5_DB_ERROR if any other db error occurred; */ -int cl5WriteOperationTxn(const char *repl_name, const char *repl_gen, const slapi_operation_parameters *op, PRBool local, void *txn); +int cl5WriteOperationTxn(cldb_Handle *cldb, const slapi_operation_parameters *op, void *txn); /* Name: cl5WriteOperation Description: writes operation to changelog @@ -266,14 +245,13 @@ int cl5WriteOperationTxn(const char *repl_name, const char *repl_gen, const slap replica object since generation can change while operation is in progress (if the data is reloaded). !!! op - operation to write - local - this is a non-replicated operation Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid op is passed; CL5_BAD_STATE if db has not been initialized; CL5_MEMORY_ERROR if memory allocation failed; CL5_DB_ERROR if any other db error occurred; */ -int cl5WriteOperation(const char *repl_name, const char *repl_gen, const slapi_operation_parameters *op, PRBool local); +int cl5WriteOperation(cldb_Handle *cldb, const slapi_operation_parameters *op); /* Name: cl5CreateReplayIterator Description: creates an iterator that allows to retrieve changes that should @@ -321,29 +299,13 @@ int cl5GetNextOperationToReplay(CL5ReplayIterator *iterator, */ void cl5DestroyReplayIterator(CL5ReplayIterator **iterator); -/* Name: cl5DeleteOnClose - Description: marks changelog for deletion when it is closed - Parameters: flag; if flag = 1 then delete else don't - Return: none - */ - -void cl5DeleteOnClose(PRBool rm); - -/* Name: cl5GetDir - Description: returns changelog directory; must be freed by the caller; - Parameters: none +/* Name: cl5GetLdifDir + Description: returns the default ldif directory; must be freed by the caller; + Parameters: backend used for export/import Return: copy of the directory; caller needs to free the string */ -char *cl5GetDir(void); - -/* Name: cl5Exist - Description: checks if a changelog exists in the specified directory - Parameters: clDir - directory to check; - Return: 1 - if changelog exists; 0 - otherwise - */ - -PRBool cl5Exist(const char *clDir); +char *cl5GetLdifDir(Slapi_Backend *be); /* Name: cl5GetOperationCount Description: returns number of entries in the changelog. The changelog must be @@ -371,39 +333,24 @@ void cl5_operation_parameters_done(struct slapi_operation_parameters *sop); */ int cl5CreateDirIfNeeded(const char *dir); -int cl5DBData2Entry(const char *data, PRUint32 len, CL5Entry *entry); +int cl5DBData2Entry(const char *data, PRUint32 len, CL5Entry *entry, void *clcrypt_handle); PRBool cl5HelperEntry(const char *csnstr, CSN *csn); CSN **cl5BuildCSNList(const RUV *consRuv, const RUV *supRuv); void cl5DestroyCSNList(CSN ***csns); -int cl5_is_diskfull(void); -int cl5_diskspace_is_available(void); +int cl5Export(Slapi_PBlock *pb); +int cl5Import(Slapi_PBlock *pb); -/* Name: cl5DbDirIsEmpty - Description: See if the given cldb directory is empty or doesn't yet exist. - Parameters: dir - Contains the name of the directory. - Return: TRUE - directory does not exist or is empty, is NULL, or is - an empty string - FALSE - otherwise -*/ -int cl5DbDirIsEmpty(const char *dir); - -/* Name: cl5WriteRUV - Description: Write RUVs into changelog db's. Called before backup. - Parameters: none - Return: TRUE -*/ -int cl5WriteRUV(void); +int cl5NotifyRUVChange(Replica *replica); -/* Name: cl5DeleteRUV - Description: Read and delete RUVs from changelog db's. Called after backup. - Parameters: none - Return: TRUE -*/ -int cl5DeleteRUV(void); -void cl5CleanRUV(ReplicaId rid); +void cl5CleanRUV(ReplicaId rid, Replica *replica); void cl5NotifyCleanup(int rid); void trigger_cl_purging(cleanruv_purge_data *purge_data); +int cldb_SetReplicaDB(Replica *replica, void *arg); +int cldb_UnSetReplicaDB(Replica *replica, void *arg); +int cldb_StartTrimming(Replica *replica); +int cldb_StopTrimming(Replica *replica, void *arg); +int cldb_StopThreads(Replica *replica, void *arg); #endif diff --git a/ldap/servers/plugins/replication/cl5_clcache.c b/ldap/servers/plugins/replication/cl5_clcache.c index e4ae739..1a1b735 100644 --- a/ldap/servers/plugins/replication/cl5_clcache.c +++ b/ldap/servers/plugins/replication/cl5_clcache.c @@ -129,7 +129,6 @@ struct clc_busy_list struct clc_pool { Slapi_RWLock *pl_lock; /* cl writer and agreements */ - DB_ENV **pl_dbenv; /* pointer to DB_ENV for all the changelog files */ CLC_Busy_List *pl_busy_lists; /* busy buffer lists, one list per changelog file */ int pl_buffer_cnt_now; /* total number of buffers */ int pl_buffer_cnt_min; /* free a newly returned buffer if _now > _min */ @@ -163,16 +162,12 @@ static void csn_dup_or_init_by_csn(CSN **csn1, CSN *csn2); * once and only once when process starts. */ int -clcache_init(DB_ENV **dbenv) +clcache_init(void) { if (_pool) { return 0; /* already initialized */ } - if (NULL == dbenv) { - return -1; - } _pool = (struct clc_pool *)slapi_ch_calloc(1, sizeof(struct clc_pool)); - _pool->pl_dbenv = dbenv; _pool->pl_buffer_cnt_min = DEFAULT_CLC_BUFFER_COUNT_MIN; _pool->pl_buffer_cnt_max = DEFAULT_CLC_BUFFER_COUNT_MAX; _pool->pl_buffer_default_pages = DEFAULT_CLC_BUFFER_COUNT_MAX; @@ -1140,7 +1135,6 @@ clcache_destroy() bl = next; } _pool->pl_busy_lists = NULL; - _pool->pl_dbenv = NULL; if (_pool->pl_lock) { slapi_rwlock_unlock(_pool->pl_lock); slapi_destroy_rwlock(_pool->pl_lock); diff --git a/ldap/servers/plugins/replication/cl5_clcache.h b/ldap/servers/plugins/replication/cl5_clcache.h index 73eb415..b5b0170 100644 --- a/ldap/servers/plugins/replication/cl5_clcache.h +++ b/ldap/servers/plugins/replication/cl5_clcache.h @@ -20,7 +20,7 @@ typedef struct clc_buffer CLC_Buffer; -int clcache_init(DB_ENV **dbenv); +int clcache_init(void); void clcache_set_config(void); int clcache_get_buffer(CLC_Buffer **buf, DB *db, ReplicaId consumer_rid, const RUV *consumer_ruv, const RUV *local_ruv); int clcache_load_buffer(CLC_Buffer *buf, CSN **anchorCSN, int *continue_on_miss); diff --git a/ldap/servers/plugins/replication/cl5_config.c b/ldap/servers/plugins/replication/cl5_config.c index e0530be..344eb79 100644 --- a/ldap/servers/plugins/replication/cl5_config.c +++ b/ldap/servers/plugins/replication/cl5_config.c @@ -29,6 +29,10 @@ #define CONFIG_BASE "cn=changelog5,cn=config" /*"cn=changelog,cn=supplier,cn=replication5.0,cn=replication,cn=config"*/ #define CONFIG_FILTER "(objectclass=*)" +/* the changelog config is now separate for each backend in "cn=changelog,,cn=ldbm database,cn=plugins,cn=config" */ +#define CL_CONFIG_BASE "cn=ldbm database,cn=plugins,cn=config" +#define CL_CONFIG_FILTER "cn=changelog" + static Slapi_RWLock *s_configLock; /* guarantees that only on thread at a time modifies changelog configuration */ @@ -37,12 +41,12 @@ static int changelog5_config_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry * static int changelog5_config_modify(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); static int changelog5_config_delete(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); static int dont_allow_that(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry *e, int *returncode, char *returntext, void *arg); - -static void changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config); +static int cldb_config_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); +static int cldb_config_modify(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); +static int cldb_config_delete(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); static changelog5Config *changelog5_dup_config(changelog5Config *config); + static void replace_bslash(char *dir); -static int notify_replica(Replica *r, void *arg); -static int _is_absolutepath(char *dir); int changelog5_config_init() @@ -61,6 +65,7 @@ changelog5_config_init() return 1; } + /* callbacks to handle attempts to modify the old cn=changelog5 config */ slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, CONFIG_BASE, LDAP_SCOPE_BASE, CONFIG_FILTER, changelog5_config_add, NULL); slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, CONFIG_BASE, LDAP_SCOPE_BASE, @@ -70,6 +75,7 @@ changelog5_config_init() slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, CONFIG_BASE, LDAP_SCOPE_BASE, CONFIG_FILTER, changelog5_config_delete, NULL); + return 0; } @@ -111,11 +117,11 @@ changelog5_read_config(changelog5Config *config) changelog5_extract_config(entries[0], config); } else { memset(config, 0, sizeof(*config)); - rc = LDAP_SUCCESS; + rc = LDAP_NO_SUCH_OBJECT; } } else { memset(config, 0, sizeof(*config)); - rc = LDAP_SUCCESS; + rc = LDAP_NO_SUCH_OBJECT; } slapi_free_search_results_internal(pb); @@ -124,6 +130,21 @@ changelog5_read_config(changelog5Config *config) return rc; } +static changelog5Config * +changelog5_dup_config(changelog5Config *config) +{ + changelog5Config *dup = (changelog5Config *)slapi_ch_calloc(1, sizeof(changelog5Config)); + + if (config->maxAge) + dup->maxAge = slapi_ch_strdup(config->maxAge); + + dup->maxEntries = config->maxEntries; + dup->trimInterval = config->trimInterval; + + return dup; +} + + void changelog5_config_done(changelog5Config *config) { @@ -132,8 +153,7 @@ changelog5_config_done(changelog5Config *config) slapi_ch_free_string(&config->maxAge); slapi_ch_free_string(&config->dir); slapi_ch_free_string(&config->symmetricKey); - slapi_ch_free_string(&config->dbconfig.encryptionAlgorithm); - slapi_ch_free_string(&config->dbconfig.symmetricKey); + slapi_ch_free_string(&config->encryptionAlgorithm); } } @@ -152,95 +172,15 @@ changelog5_config_add(Slapi_PBlock *pb __attribute__((unused)), char *returntext, void *arg __attribute__((unused))) { - int rc; - changelog5Config config; - - *returncode = LDAP_SUCCESS; - - slapi_rwlock_wrlock(s_configLock); - - /* we already have a configured changelog - don't need to do anything - since add operation will fail */ - if (cl5GetState() == CL5_STATE_OPEN) { - *returncode = 1; - if (returntext) { - strcpy(returntext, "attempt to add changelog when it already exists"); - } - - slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl, - "changelog5_config_add - Changelog already exist; " - "request ignored\n"); - goto done; - } - - changelog5_extract_config(e, &config); - if (config.dir == NULL) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "NULL changelog directory"); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_add - NULL changelog directory\n"); - goto done; - } - - if (!cl5DbDirIsEmpty(config.dir)) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, - "The changelog directory [%s] already exists and is not empty. " - "Please choose a directory that does not exist or is empty.\n", - config.dir); - } - - goto done; - } - - /* start the changelog */ - rc = cl5Open(config.dir, &config.dbconfig); - if (rc != CL5_SUCCESS) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Failed to start changelog; error - %d", rc); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_add - Failed to start changelog\n"); - goto done; - } - - /* set trimming parameters */ - rc = cl5ConfigTrimming(config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval); - if (rc != CL5_SUCCESS) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to configure changelog trimming; error - %d", rc); - } - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_add - Failed to configure changelog trimming\n"); - goto done; + /* we no longer support a separate changelog configuration */ + slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl, + "changelog5_config_add - Separate changelog no longer supported; " + "use cn=changelog, instead\n"); + + if (returntext) { + PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Changelog configuration is part of the backend configuration"); } - - /* notify all the replicas that the changelog is configured - so that the can log dummy changes if necessary. */ - replica_enumerate_replicas(notify_replica, NULL); - -#ifdef TEST_CL5 - testChangelog(TEST_ITERATION); -#endif - -done:; - slapi_rwlock_unlock(s_configLock); - changelog5_config_done(&config); - if (*returncode == LDAP_SUCCESS) { - if (returntext) { - returntext[0] = '\0'; - } - - return SLAPI_DSE_CALLBACK_OK; - } - + *returncode = LDAP_UNWILLING_TO_PERFORM; return SLAPI_DSE_CALLBACK_ERROR; } @@ -252,60 +192,65 @@ changelog5_config_modify(Slapi_PBlock *pb, char *returntext, void *arg __attribute__((unused))) { - int rc = 0; - LDAPMod **mods; - int i; - changelog5Config config; - changelog5Config *originalConfig = NULL; - char *currentDir = NULL; - + /* we no longer support a separate changelog configuration */ + /* the entry does not exist and the client will be notified + */ + slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl, + "changelog5_config_modify - Separate changelog no longer supported; " + "request ignored\n"); + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; +} - /* changelog must be open before its parameters can be modified */ - if (cl5GetState() != CL5_STATE_OPEN) { - if (returntext) { - strcpy(returntext, "changelog is not configured"); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_modify - Changelog is not configured\n"); - return SLAPI_DSE_CALLBACK_ERROR; - } +static int +changelog5_config_delete(Slapi_PBlock *pb __attribute__((unused)), + Slapi_Entry *e __attribute__((unused)), + Slapi_Entry *entryAfter __attribute__((unused)), + int *returncode, + char *returntext, + void *arg __attribute__((unused))) +{ + /* we no longer support a separate changelog configuration */ + /* the entry does not exist and the client will be notified + */ + slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl, + "changelog5_config_delete - Separate changelog no longer supported; " + "request ignored\n"); + + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; +} - slapi_rwlock_wrlock(s_configLock); +static int +cldb_config_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + return SLAPI_DSE_CALLBACK_OK; +} - /* changelog must be open before its parameters can be modified */ - if (cl5GetState() != CL5_STATE_OPEN) { - *returncode = 1; - if (returntext) { - strcpy(returntext, "changelog is not configured"); - } +static int +cldb_config_modify(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + int rc = 0; + LDAPMod **mods; + *returncode = LDAP_SUCCESS; + changelog5Config config; + changelog5Config *originalConfig = NULL; + Replica *replica = (Replica *)arg; - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_modify - Changelog is not configured\n"); - goto done; - } - /* - * Extract all the original configuration: This is needed to ensure that the configuration - * is trully reloaded. This was not needed before 091401 because the changelog configuration - * was always hardcoded (NULL was being passed to cl5Open). Now we need to ensure we pass to - * cl5Open the proper configuration... - */ changelog5_extract_config(e, &config); originalConfig = changelog5_dup_config(&config); /* Reset all the attributes that have been potentially modified by the current MODIFY operation */ - slapi_ch_free_string(&config.dir); - config.dir = NULL; config.maxEntries = CL5_NUM_IGNORE; - config.compactInterval = CL5_NUM_IGNORE; slapi_ch_free_string(&config.maxAge); config.maxAge = slapi_ch_strdup(CL5_STR_IGNORE); config.trimInterval = CL5_NUM_IGNORE; + slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); - for (i = 0; mods && mods[i] != NULL; i++) { + for (size_t i = 0; mods && mods[i] != NULL; i++) { if (mods[i]->mod_op & LDAP_MOD_DELETE) { /* We don't support deleting changelog attributes */ } else if (mods[i]->mod_values == NULL) { @@ -328,19 +273,7 @@ changelog5_config_modify(Slapi_PBlock *pb, } /* replace existing value */ - if (strcasecmp(config_attr, CONFIG_CHANGELOG_DIR_ATTRIBUTE) == 0) { - if (config_attr_value && config_attr_value[0] != '\0') { - slapi_ch_free_string(&config.dir); - config.dir = slapi_ch_strdup(config_attr_value); - replace_bslash(config.dir); - } else { - *returncode = 1; - if (returntext) { - strcpy(returntext, "null changelog directory"); - } - goto done; - } - } else if (strcasecmp(config_attr, CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE) == 0) { + if (strcasecmp(config_attr, CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE) == 0) { if (config_attr_value && config_attr_value[0] != '\0') { config.maxEntries = atoi(config_attr_value); } else { @@ -361,20 +294,6 @@ changelog5_config_modify(Slapi_PBlock *pb, *returncode = LDAP_UNWILLING_TO_PERFORM; goto done; } - } else if (strcasecmp(config_attr, CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE) == 0) { - if (slapi_is_duration_valid(config_attr_value)) { - config.compactInterval = (long)slapi_parse_duration(config_attr_value); - } else { - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, - "%s: invalid value \"%s\", %s must range from 0 to %lld or digit[sSmMhHdD]", - CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE, config_attr_value, - CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE, - (long long int)LONG_MAX); - } - *returncode = LDAP_UNWILLING_TO_PERFORM; - goto done; - } } else if (strcasecmp(config_attr, CONFIG_CHANGELOG_TRIM_ATTRIBUTE) == 0) { if (slapi_is_duration_valid(config_attr_value)) { config.trimInterval = (long)slapi_parse_duration(config_attr_value); @@ -410,8 +329,6 @@ changelog5_config_modify(Slapi_PBlock *pb, * except config.dir */ if (config.maxEntries == CL5_NUM_IGNORE) config.maxEntries = originalConfig->maxEntries; - if (config.compactInterval == CL5_NUM_IGNORE) - config.compactInterval = originalConfig->compactInterval; if (config.trimInterval == CL5_NUM_IGNORE) config.trimInterval = originalConfig->trimInterval; if (strcmp(config.maxAge, CL5_STR_IGNORE) == 0) { @@ -420,106 +337,11 @@ changelog5_config_modify(Slapi_PBlock *pb, config.maxAge = slapi_ch_strdup(originalConfig->maxAge); } - /* attempt to change chagelog dir */ - if (config.dir) { - currentDir = cl5GetDir(); - if (currentDir == NULL) { - /* something is wrong: we should never be here */ - *returncode = 1; - if (returntext) { - strcpy(returntext, "internal failure"); - } - - goto done; - } - - if (strcmp(currentDir, config.dir) != 0) { - if (!cl5DbDirIsEmpty(config.dir)) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, - "The changelog directory [%s] already exists and is not empty. " - "Please choose a directory that does not exist or is empty.\n", - config.dir); - } - - goto done; - } - - if (!_is_absolutepath(config.dir) || (CL5_SUCCESS != cl5CreateDirIfNeeded(config.dir))) { - *returncode = 1; - if (returntext) { - PL_strncpyz(returntext, "invalid changelog directory or insufficient access", SLAPI_DSE_RETURNTEXT_SIZE); - } - - goto done; - } - - /* changelog directory changed - need to remove the - previous changelog and create new one */ - - slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name_cl, - "changelog5_config_modify - Changelog directory changed; " - "old dir - %s, new dir - %s; recreating changelog.\n", - currentDir, config.dir); - - rc = cl5Close(); - if (rc != CL5_SUCCESS) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Failed to close changelog; error - %d", rc); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_modify - Failed to close changelog\n"); - goto done; - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "changelog5_config_modif - Closed the changelog\n"); - } - - rc = cl5Delete(currentDir); - if (rc != CL5_SUCCESS) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to remove changelog; error - %d", rc); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_modify - Failed to remove changelog\n"); - goto done; - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "changelog5_config_modify - Deleted the changelog at %s\n", currentDir); - } - - rc = cl5Open(config.dir, &config.dbconfig); - if (rc != CL5_SUCCESS) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Failed to restart changelog; error - %d", rc); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_modify - Failed to restart changelog\n"); - /* before finishing, let's try to do some error recovery */ - if (CL5_SUCCESS != cl5Open(currentDir, &config.dbconfig)) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_modify - Failed to restore previous changelog\n"); - } - goto done; - } else { - slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl, - "changelog5_config_modify - Opened the changelog at %s\n", config.dir); - } - } - } - /* one of the changelog parameters is modified */ if (config.maxEntries != CL5_NUM_IGNORE || config.trimInterval != CL5_NUM_IGNORE || strcmp(config.maxAge, CL5_STR_IGNORE) != 0) { - rc = cl5ConfigTrimming(config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval); + rc = cl5ConfigTrimming(replica, config.maxEntries, config.maxAge, config.trimInterval); if (rc != CL5_SUCCESS) { *returncode = 1; if (returntext) { @@ -538,9 +360,6 @@ done:; changelog5_config_done(&config); changelog5_config_free(&originalConfig); - /* slapi_ch_free accepts NULL pointer */ - slapi_ch_free((void **)¤tDir); - if (*returncode == LDAP_SUCCESS) { if (returntext) { @@ -554,100 +373,12 @@ done:; } static int -changelog5_config_delete(Slapi_PBlock *pb __attribute__((unused)), - Slapi_Entry *e __attribute__((unused)), - Slapi_Entry *entryAfter __attribute__((unused)), - int *returncode, - char *returntext, - void *arg __attribute__((unused))) +cldb_config_delete(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) { - int rc; - char *currentDir = NULL; - *returncode = LDAP_SUCCESS; - - /* changelog must be open before it can be deleted */ - if (cl5GetState() != CL5_STATE_OPEN) { - *returncode = 1; - if (returntext) { - PL_strncpyz(returntext, "changelog is not configured", SLAPI_DSE_RETURNTEXT_SIZE); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_delete - Chagelog is not configured\n"); - return SLAPI_DSE_CALLBACK_ERROR; - } - - slapi_rwlock_wrlock(s_configLock); - - /* changelog must be open before it can be deleted */ - if (cl5GetState() != CL5_STATE_OPEN) { - *returncode = 1; - if (returntext) { - PL_strncpyz(returntext, "changelog is not configured", SLAPI_DSE_RETURNTEXT_SIZE); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_delete - Changelog is not configured\n"); - goto done; - } - - currentDir = cl5GetDir(); - - if (currentDir == NULL) { - /* something is wrong: we should never be here */ - *returncode = 1; - if (returntext) { - PL_strncpyz(returntext, "internal failure", SLAPI_DSE_RETURNTEXT_SIZE); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_delete - NULL directory\n"); - goto done; - } - - /* this call will block until all threads using changelog - release changelog by calling cl5RemoveThread () */ - rc = cl5Close(); - if (rc != CL5_SUCCESS) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to close changelog; error - %d", rc); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_delete - Failed to close changelog\n"); - goto done; - } - - rc = cl5Delete(currentDir); - if (rc != CL5_SUCCESS) { - *returncode = 1; - if (returntext) { - PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to remove changelog; error - %d", rc); - } - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_config_delete - Failed to remove changelog\n"); - goto done; - } - -done:; - slapi_rwlock_unlock(s_configLock); - - /* slapi_ch_free accepts NULL pointer */ - slapi_ch_free((void **)¤tDir); - - if (*returncode == LDAP_SUCCESS) { - if (returntext) { - returntext[0] = '\0'; - } - - return SLAPI_DSE_CALLBACK_OK; - } - - return SLAPI_DSE_CALLBACK_ERROR; + return SLAPI_DSE_CALLBACK_OK; } + static int dont_allow_that(Slapi_PBlock *pb __attribute__((unused)), Slapi_Entry *entryBefore __attribute__((unused)), @@ -660,31 +391,10 @@ dont_allow_that(Slapi_PBlock *pb __attribute__((unused)), return SLAPI_DSE_CALLBACK_ERROR; } -static changelog5Config * -changelog5_dup_config(changelog5Config *config) -{ - changelog5Config *dup = (changelog5Config *)slapi_ch_calloc(1, sizeof(changelog5Config)); - - if (config->dir) - dup->dir = slapi_ch_strdup(config->dir); - if (config->maxAge) - dup->maxAge = slapi_ch_strdup(config->maxAge); - - dup->maxEntries = config->maxEntries; - dup->compactInterval = config->compactInterval; - dup->trimInterval = config->trimInterval; - - dup->dbconfig.pageSize = config->dbconfig.pageSize; - dup->dbconfig.fileMode = config->dbconfig.fileMode; - - return dup; -} - - /* * Given the changelog configuration entry, extract the configuration directives. */ -static void +void changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config) { const char *arg; @@ -698,18 +408,6 @@ changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config) if (arg) { config->maxEntries = atoi(arg); } - arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE); - if (arg) { - if (slapi_is_duration_valid(arg)) { - config->compactInterval = (long)slapi_parse_duration(arg); - } else { - slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl, - "changelog5_extract_config - %s: invalid value \"%s\", ignoring the change.\n", - CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE, arg); - } - } else { - config->compactInterval = CHANGELOGDB_COMPACT_INTERVAL; - } arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_TRIM_ATTRIBUTE); if (arg) { @@ -745,21 +443,56 @@ changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config) */ arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM); if (arg) { - config->dbconfig.encryptionAlgorithm = slapi_ch_strdup(arg); + config->encryptionAlgorithm = slapi_ch_strdup(arg); } else { - config->dbconfig.encryptionAlgorithm = NULL; /* no encryption */ + config->encryptionAlgorithm = NULL; /* no encryption */ } /* * symmetric key */ arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_SYMMETRIC_KEY); if (arg) { - config->dbconfig.symmetricKey = slapi_ch_strdup(arg); + config->symmetricKey = slapi_ch_strdup(arg); } else { - config->dbconfig.symmetricKey = NULL; /* no symmetric key */ + config->symmetricKey = NULL; /* no symmetric key */ } } +/* register functions handling attempted operations on the changelog config entries */ +int +changelog5_register_config_callbacks(const char *dn, Replica *replica) +{ + int rc = 0; + /* callbacks to handle changes to the new changelog configuration in the main database */ + rc =slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, cldb_config_add, replica); + rc |= slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, cldb_config_modify, replica); + rc |= slapi_config_register_callback(SLAPI_OPERATION_MODRDN, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, dont_allow_that, replica); + rc |= slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, cldb_config_delete, replica); + + return rc; +} + +int +changelog5_remove_config_callbacks(const char *dn) +{ + int rc = 0; + + rc =slapi_config_remove_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, cldb_config_add); + rc |= slapi_config_remove_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, cldb_config_modify); + rc |= slapi_config_remove_callback(SLAPI_OPERATION_MODRDN, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, dont_allow_that); + rc |= slapi_config_remove_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE, + CL_CONFIG_FILTER, cldb_config_delete); + + return rc; +} + static void replace_bslash(char *dir) { @@ -774,18 +507,3 @@ replace_bslash(char *dir) bslash = strchr(bslash, '\\'); } } - -static int -notify_replica(Replica *r, void *arg __attribute__((unused))) -{ - return replica_log_ruv_elements(r); -} - -static int -_is_absolutepath(char *dir) -{ - if (dir[0] == '/') - return 1; - - return 0; -} diff --git a/ldap/servers/plugins/replication/cl5_init.c b/ldap/servers/plugins/replication/cl5_init.c index 112c4ec..318677e 100644 --- a/ldap/servers/plugins/replication/cl5_init.c +++ b/ldap/servers/plugins/replication/cl5_init.c @@ -20,12 +20,47 @@ #include "cl5.h" #include "repl5.h" +static int _cl5_upgrade_replica(Replica *replica, void *arg); +static int _cl5_upgrade_replica_config(Replica *replica, changelog5Config *config); +static int _cl5_upgrade_removedir(char *path); +static int _cl5_upgrade_removeconfig(void); + +/* upgrade changelog*/ +/* the changelog5 configuration maintained all changlog files in a separate directory + * now the changlog is part of the instance database. + * If this is the first startup with the new version we willi + * - try to move the changelog files for each replica to the instance database. + * - create a config entry for trimming and encryption in each backend. + */ +int +changelog5_upgrade(void) +{ + int rc = 0; + changelog5Config config = {}; + + changelog5_read_config(&config); + + if (config.dir == NULL) { + /* we do not have a valid legacy config, nothing to upgrade */ + return rc; + } + + replica_enumerate_replicas(_cl5_upgrade_replica, (void *)&config); + + rc = _cl5_upgrade_removedir(config.dir); + + rc = _cl5_upgrade_removeconfig(); + + changelog5_config_done(&config); + + return rc; +} + /* initializes changelog*/ int -changelog5_init() +changelog5_init(void) { int rc; - changelog5Config config; rc = cl5Init(); if (rc != CL5_SUCCESS) { @@ -34,41 +69,22 @@ changelog5_init() return 1; } - /* read changelog configuration */ + /* setup callbacks for operations on changelog */ changelog5_config_init(); - changelog5_read_config(&config); - - if (config.dir == NULL) { - /* changelog is not configured - bail out */ - /* Note: but still changelog needs to be initialized to allow it - * to configure after this point. (don't call cl5Cleanup) */ - rc = 0; /* OK */ - goto done; - } /* start changelog */ - rc = cl5Open(config.dir, &config.dbconfig); + rc = cl5Open(); if (rc != CL5_SUCCESS) { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_init: failed to start changelog at %s\n", - config.dir); + "changelog5_init: failed to start changelog\n"); rc = 1; goto done; } - /* set trimming parameters */ - rc = cl5ConfigTrimming(config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval); - if (rc != CL5_SUCCESS) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "changelog5_init: failed to configure changelog trimming\n"); - rc = 1; - goto done; - } rc = 0; done: - changelog5_config_done(&config); return rc; } @@ -83,3 +99,115 @@ changelog5_cleanup() /* cleanup config */ changelog5_config_cleanup(); } +static int +_cl5_upgrade_replica_config(Replica *replica, changelog5Config *config) +{ + int rc = 0; + Slapi_Backend *be = slapi_be_select(replica_get_root(replica)); + + Slapi_Entry *config_entry = slapi_entry_alloc(); + slapi_entry_init(config_entry, slapi_ch_strdup("cn=changelog"), NULL); + slapi_entry_add_string(config_entry, "objectclass", "top"); + slapi_entry_add_string(config_entry, "objectclass", "extensibleObject"); + + /* keep the changelog trimming config */ + if (config->maxEntries) { + char *maxEnt = slapi_ch_smprintf("%d", config->maxEntries); + slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE, maxEnt); + } + if (strcmp(config->maxAge, CL5_STR_IGNORE)) { + slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_MAXAGE_ATTRIBUTE, config->maxAge); + } + if (config->trimInterval != CHANGELOGDB_TRIM_INTERVAL) { + /* char *interval = slapi_ch_smprintf("%ld", config->trimInterval); */ + slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_TRIM_ATTRIBUTE, gen_duration(config->trimInterval)); + } + + /* if changelog encryption is enabled then in the upgrade mode all backends will have + * an encrypted changelog, store the encryption attrs */ + + if (config->encryptionAlgorithm) { + slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM, config->encryptionAlgorithm); + slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_SYMMETRIC_KEY, config->symmetricKey); + } + rc = slapi_back_ctrl_info(be, BACK_INFO_CLDB_SET_CONFIG, (void *)config_entry); + + return rc; +} +static int +_cl5_upgrade_replica(Replica *replica, void *arg) +{ + int rc = 0; + changelog5Config *config = (changelog5Config *)arg; + + /* Move existing database file to backend */ + char *replGen = replica_get_generation (replica); + const char *replName = replica_get_name (replica); + char *oldFile = slapi_ch_smprintf("%s/%s_%s.db", + config->dir, replName, replGen); + slapi_ch_free_string(&replGen); + if (PR_Access(oldFile, PR_ACCESS_EXISTS) == PR_SUCCESS) { + Slapi_Backend *be = slapi_be_select(replica_get_root(replica)); + char *instancedir; + slapi_back_get_info(be, BACK_INFO_INSTANCE_DIR, (void **)&instancedir); + char *newFile = slapi_ch_smprintf("%s/changelog.db", instancedir); + + rc = slapi_back_ctrl_info(be, BACK_INFO_DBENV_CLDB_UPGRADE, oldFile); + slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name_cl, + "_cl5_upgrade_replica: moving file (%s) to (%s) %s\n", + oldFile, newFile, rc?"failed":"succeeded"); + slapi_ch_free_string(&instancedir); + } + + /* Move changelog config to backend config */ + rc = _cl5_upgrade_replica_config(replica, config); + + return rc; +} +static int +_cl5_upgrade_removedir(char *path) +{ + /* this is duplicated from ldbm_delete_dirs, we unfortunately + * cannot access the backend functions + */ + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + char fullpath[MAXPATHLEN]; + int rval = 0; + PRFileInfo64 info; + + dirhandle = PR_OpenDir(path); + if (!dirhandle) { + PR_Delete(path); + return 0; + } + + while (NULL != (direntry = + PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { + if (!direntry->name) + break; + + PR_snprintf(fullpath, MAXPATHLEN, "%s/%s", path, direntry->name); + rval = PR_GetFileInfo64(fullpath, &info); + if (PR_SUCCESS == rval) { + PR_Delete(fullpath); + } + } + PR_CloseDir(dirhandle); + /* remove the directory itself too */ + rval += PR_RmDir(path); + return rval; +} +static int +_cl5_upgrade_removeconfig(void) +{ + int rc = LDAP_SUCCESS; + + Slapi_PBlock *pb = slapi_pblock_new(); + slapi_delete_internal_set_pb(pb, "cn=changelog5,cn=config", NULL, NULL, + repl_get_plugin_identity(PLUGIN_MULTIMASTER_REPLICATION), 0); + slapi_delete_internal_pb(pb); + slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); + slapi_pblock_destroy(pb); + return rc; +} diff --git a/ldap/servers/plugins/replication/cl5_test.c b/ldap/servers/plugins/replication/cl5_test.c index 0e2894a..84686e2 100644 --- a/ldap/servers/plugins/replication/cl5_test.c +++ b/ldap/servers/plugins/replication/cl5_test.c @@ -29,7 +29,6 @@ static void testBasic(); static void testBackupRestore(); static void testIteration(); -static void testTrimming(); static void testPerformance(); static void testPerformanceMT(); static void testLDIF(); @@ -57,9 +56,6 @@ testChangelog(TestType type) case TEST_ITERATION: testIteration(); break; - case TEST_TRIMMING: - testTrimming(); - break; case TEST_PERFORMANCE: testPerformance(); break; @@ -115,7 +111,7 @@ testBackupRestore() slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "Starting backup and recovery test ...\n"); - dir = cl5GetDir(); + dir = cl5GetLdifDir(NULL); if (dir) { baseDir = getBaseDir(dir); @@ -127,7 +123,7 @@ testBackupRestore() cl5Close(); rc = cl5Restore(dir, bkDir, NULL); if (rc == CL5_SUCCESS) - rc = cl5Open(dir, NULL); + rc = cl5Open(); /* PR_RmDir (bkDir);*/ } @@ -264,45 +260,6 @@ testIteration() } static void -testTrimming() -{ - PRIntervalTime interval; - int count; - int rc; - - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "Starting trimming test ...\n"); - - rc = populateChangelog(200, NULL); - - if (rc == 0) { - interval = PR_SecondsToInterval(2); - DS_Sleep(interval); - - rc = populateChangelog(300, NULL); - - if (rc == 0) - rc = cl5ConfigTrimming(300, "1d", CHANGELOGDB_COMPACT_INTERVAL, CHANGELOGDB_TRIM_INTERVAL); - - interval = PR_SecondsToInterval(300); /* 5 min is default trimming interval */ - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "Trimming test: sleeping for 5 minutes until trimming kicks in\n"); - DS_Sleep(interval); - - count = cl5GetOperationCount(NULL); - } - - if (rc == 0 && count == 300) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "Trimming test completed successfully: changelog contains 300 entries\n"); - } else if (rc == 0) { - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, - "Trimming test failed: changelog contains %d entries; expected - 300\n", - count); - } else /* general failure */ - slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "Trimming test failed\n"); -} - -static void testPerformance() { PRTime starttime, endtime, totaltime; @@ -393,7 +350,7 @@ testPerformanceMT() static void testLDIF() { - char *clDir = cl5GetDir(); + char *clDir = cl5GetLdifDir(NULL); int rc; char *baseDir; char ldifFile[MAXPATHLEN]; @@ -411,7 +368,7 @@ testLDIF() cl5Close(); rc = cl5ImportLDIF(clDir, ldifFile, NULL); if (rc == CL5_SUCCESS) - cl5Open(clDir, NULL); + cl5Open(); } } diff --git a/ldap/servers/plugins/replication/cl_crypt.c b/ldap/servers/plugins/replication/cl_crypt.c index 8385534..340e3e3 100644 --- a/ldap/servers/plugins/replication/cl_crypt.c +++ b/ldap/servers/plugins/replication/cl_crypt.c @@ -29,44 +29,33 @@ /* * BACK_INFO_CRYPT_INIT */ -int -clcrypt_init(const CL5DBConfig *config, void **clcrypt_handle) +void * +clcrypt_init(char *encryptionAlgorithm, Slapi_Backend *be) { int rc = 0; - char *cookie = NULL; - Slapi_Backend *be = NULL; back_info_crypt_init crypt_init = {0}; + void *crypt_handle = NULL; slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name, "-> clcrypt_init\n"); - /* Encryption is not specified */ - if (!config->encryptionAlgorithm || !clcrypt_handle) { + + if (!encryptionAlgorithm) { + /* Encryption is not specified */ goto bail; } - crypt_init.dn = "cn=changelog5,cn=config"; - crypt_init.encryptionAlgorithm = config->encryptionAlgorithm; + crypt_init.dn = "cn=changelog"; + crypt_init.encryptionAlgorithm = encryptionAlgorithm; + crypt_init.be = be; - be = slapi_get_first_backend(&cookie); - while (be) { - crypt_init.be = be; - rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_INIT, + rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_INIT, (void *)&crypt_init); - if (LDAP_SUCCESS == rc) { - break; /* Successfully fetched */ - } - be = slapi_get_next_backend(cookie); - } - slapi_ch_free((void **)&cookie); if (LDAP_SUCCESS == rc && crypt_init.state_priv) { - *clcrypt_handle = crypt_init.state_priv; - rc = 0; - } else { - rc = 1; + crypt_handle = crypt_init.state_priv; } bail: slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name, "<- clcrypt_init : %d\n", rc); - return rc; + return crypt_handle; } /* @@ -77,38 +66,26 @@ bail: * : NULL - failure */ int -clcrypt_destroy(void *clcrypt_handle) +clcrypt_destroy(void *clcrypt_handle, Slapi_Backend *be) { int rc = -1; - char *cookie = NULL; - Slapi_Backend *be = NULL; back_info_crypt_destroy crypt_destroy = {0}; slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name, "-> clcrypt_destroy\n"); if (NULL == clcrypt_handle) { /* Nothing to free */ - rc = 0; - goto bail; + return 0; } crypt_destroy.state_priv = clcrypt_handle; - be = slapi_get_first_backend(&cookie); - while (be) { - rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_DESTROY, - (void *)&crypt_destroy); - if (LDAP_SUCCESS == rc) { - break; /* Successfully freed */ - } - be = slapi_get_next_backend(cookie); - } - slapi_ch_free((void **)&cookie); + rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_DESTROY, + (void *)&crypt_destroy); if (LDAP_SUCCESS == rc) { rc = 0; } else { rc = -1; } -bail: slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name, "<- clcrypt_destroy (returning %d)\n", rc); return rc; diff --git a/ldap/servers/plugins/replication/cl_crypt.h b/ldap/servers/plugins/replication/cl_crypt.h index 9b69052..6632544 100644 --- a/ldap/servers/plugins/replication/cl_crypt.h +++ b/ldap/servers/plugins/replication/cl_crypt.h @@ -18,8 +18,8 @@ #include "nss.h" #include "cert.h" -int clcrypt_init(const CL5DBConfig *config, void **clcrypt_handle); -int clcrypt_destroy(void *clcrypt_handle); +void *clcrypt_init(char *encryptionAlgorithm, Slapi_Backend *be); +int clcrypt_destroy(void *clcrypt_handle, Slapi_Backend *be); int clcrypt_encrypt_value(void *clcrypt_handle, struct berval *in, struct berval **out); int clcrypt_decrypt_value(void *state_priv, struct berval *in, struct berval **out); #endif /* _CLCRYPT_H_ */ diff --git a/ldap/servers/plugins/replication/repl5.h b/ldap/servers/plugins/replication/repl5.h index 059dc46..4215f5a 100644 --- a/ldap/servers/plugins/replication/repl5.h +++ b/ldap/servers/plugins/replication/repl5.h @@ -752,6 +752,8 @@ PRBool ignore_error_and_keep_going(int error); void replica_check_release_timeout(Replica *r, Slapi_PBlock *pb); void replica_lock_replica(Replica *r); void replica_unlock_replica(Replica *r); +void *replica_get_file_info(Replica *r); +int replica_set_file_info(Replica *r, void *cl); /* The functions below handles the state flag */ /* Current internal state flags */ @@ -807,8 +809,7 @@ typedef struct _cleanruv_purge_data { int cleaned_rid; const Slapi_DN *suffix_sdn; - char *replName; - char *replGen; + Replica *replica; } cleanruv_purge_data; typedef struct _csngen_test_data diff --git a/ldap/servers/plugins/replication/repl5_init.c b/ldap/servers/plugins/replication/repl5_init.c index 5a748e3..5a6ae99 100644 --- a/ldap/servers/plugins/replication/repl5_init.c +++ b/ldap/servers/plugins/replication/repl5_init.c @@ -338,8 +338,8 @@ multimaster_bepreop_init(Slapi_PBlock *pb) if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) != 0 || slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepreopdesc) != 0 || - slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_CLOSE_FN, (void *)cl5Cleanup) != 0 || - slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_BACKUP_FN, (void *)cl5WriteRUV) != 0) { + /* slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_CLOSE_FN, (void *)cl5Cleanup) != 0) { */ + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_CLOSE_FN, (void *)cl5Close) != 0) { slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_bepreop_init - Failed\n"); rc = -1; } @@ -385,10 +385,9 @@ multimaster_bepostop_init(Slapi_PBlock *pb) if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) != 0 || slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepostopdesc) != 0 || + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)cl5Open) != 0 || slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_MODRDN_FN, (void *)multimaster_bepostop_modrdn) != 0 || - slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_DELETE_FN, (void *)multimaster_bepostop_delete) != 0 || - slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)changelog5_init) != 0 || - slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN, (void *)cl5DeleteRUV) != 0) { + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_DELETE_FN, (void *)multimaster_bepostop_delete) != 0) { slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_bepostop_init - Failed\n"); rc = -1; } @@ -408,8 +407,9 @@ multimaster_betxn_bepostop_init(Slapi_PBlock *pb) if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) || slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepostopdesc) || - slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)changelog5_init) || - slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN, (void *)cl5DeleteRUV)) { + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)cl5Open) != 0 || + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN, (void *)cl5Export) || + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_IMPORT_FN, (void *)cl5Import)) { slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_betxn_bepostop_init - Failed\n"); rc = -1; } @@ -797,17 +797,16 @@ multimaster_start(Slapi_PBlock *pb) /* create replicas */ multimaster_mtnode_construct_replicas(); - /* Initialise the 5.0 Changelog */ + /* Upgrade the 5.0 Changelog if it still exists */ + rc = changelog5_upgrade(); + if (rc != 0) + goto out; + + /* perform initial changelog setup */ rc = changelog5_init(); if (rc != 0) goto out; - /* Initialize the replication agreements, unless we're dumping LDIF */ - if (!is_ldif_dump) { - rc = agmtlist_config_init(); - if (rc != 0) - goto out; - } rc = create_repl_schema_policy(); if (rc != 0) goto out; @@ -815,9 +814,14 @@ multimaster_start(Slapi_PBlock *pb) /* check if the replica's data was reloaded offline and we need to reinitialize replica's changelog. This should be done after the changelog is initialized */ - replica_enumerate_replicas(replica_check_for_data_reload, NULL); + /* Initialize the replication agreements, unless we're dumping LDIF */ + if (!is_ldif_dump) { + rc = agmtlist_config_init(); + if (rc != 0) + goto out; + } /* register to be notified when backend state changes */ slapi_register_backend_state_change((void *)multimaster_be_state_change, multimaster_be_state_change); diff --git a/ldap/servers/plugins/replication/repl5_plugins.c b/ldap/servers/plugins/replication/repl5_plugins.c index caa99d1..694dc62 100644 --- a/ldap/servers/plugins/replication/repl5_plugins.c +++ b/ldap/servers/plugins/replication/repl5_plugins.c @@ -987,19 +987,15 @@ write_changelog_and_ruv(Slapi_PBlock *pb) replica_check_release_timeout(r, pb); - if (replica_is_flag_set(r, REPLICA_LOG_CHANGES) && - (cl5GetState() == CL5_STATE_OPEN)) { + if (replica_is_flag_set(r, REPLICA_LOG_CHANGES)) { supplier_operation_extension *opext = NULL; - const char *repl_name; - char *repl_gen; + cldb_Handle *cldb = NULL; opext = (supplier_operation_extension *)repl_sup_get_ext(REPL_SUP_EXT_OP, op); PR_ASSERT(opext); - /* get replica generation and replica name to pass to the write function */ - repl_name = replica_get_name(r); - repl_gen = opext->repl_gen; - PR_ASSERT(repl_name && repl_gen); + /* changelog database information to pass to the write function */ + cldb = replica_get_file_info(r); /* for replicated operations, we log the original, non-urp data which is saved in the operation extension */ @@ -1042,6 +1038,9 @@ write_changelog_and_ruv(Slapi_PBlock *pb) if (op_params->csn && is_cleaned_rid(csn_get_replicaid(op_params->csn))) { /* this RID has been cleaned */ + if (!operation_is_flag_set(op, OP_FLAG_REPLICATED)) { + slapi_ch_free((void **)&op_params->target_address.uniqueid); + } goto common_return; } @@ -1052,6 +1051,9 @@ write_changelog_and_ruv(Slapi_PBlock *pb) { slapi_log_err(SLAPI_LOG_REPL, "write_changelog_and_ruv", "Skipping internal operation on read-only replica\n"); + if (!operation_is_flag_set(op, OP_FLAG_REPLICATED)) { + slapi_ch_free((void **)&op_params->target_address.uniqueid); + } goto common_return; } @@ -1061,11 +1063,6 @@ write_changelog_and_ruv(Slapi_PBlock *pb) op_params->p.p_modify.modify_mods != NULL) { void *txn = NULL; char csn_str[CSN_STRSIZE]; - if (cl5_is_diskfull() && !cl5_diskspace_is_available()) { - slapi_log_err(SLAPI_LOG_CRIT, repl_plugin_name, - "write_changelog_and_ruv - Skipped due to DISKFULL\n"); - goto common_return; - } slapi_pblock_get(pb, SLAPI_TXN, &txn); slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name, "write_changelog_and_ruv - Writing change for " @@ -1074,8 +1071,7 @@ write_changelog_and_ruv(Slapi_PBlock *pb) op_params->target_address.uniqueid, op_params->operation_type, csn_as_string(op_params->csn, PR_FALSE, csn_str)); - rc = cl5WriteOperationTxn(repl_name, repl_gen, op_params, - !operation_is_flag_set(op, OP_FLAG_REPLICATED), txn); + rc = cl5WriteOperationTxn(cldb, op_params, txn); if (rc != CL5_SUCCESS) { /* ONREPL - log error */ slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name, diff --git a/ldap/servers/plugins/replication/repl5_replica.c b/ldap/servers/plugins/replication/repl5_replica.c index f017823..1f8a970 100644 --- a/ldap/servers/plugins/replication/repl5_replica.c +++ b/ldap/servers/plugins/replication/repl5_replica.c @@ -66,6 +66,7 @@ struct replica uint64_t agmt_count; /* Number of agmts */ Slapi_Counter *release_timeout; /* The amount of time to wait before releasing active replica */ uint64_t abort_session; /* Abort the current replica session */ + cldb_Handle *cldb; /* database info for the changelog */ }; @@ -84,6 +85,7 @@ static Slapi_Entry *_replica_get_config_entry(const Slapi_DN *root, const char * static int _replica_check_validity(const Replica *r); static int _replica_init_from_config(Replica *r, Slapi_Entry *e, char *errortext); static int _replica_update_entry(Replica *r, Slapi_Entry *e, char *errortext); +static int _replica_config_changelog(Replica *r); static int _replica_configure_ruv(Replica *r, PRBool isLocked); static char *_replica_get_config_dn(const Slapi_DN *root); static char *_replica_type_as_string(const Replica *r); @@ -205,6 +207,7 @@ replica_new_from_entry(Slapi_Entry *e, char *errortext, PRBool is_add_operation, rc = LDAP_SUCCESS; } + /* If smallest csn exists in RUV for our local replica, it's ok to begin iteration */ PR_ASSERT(object_get_data(r->repl_ruv)); @@ -215,6 +218,16 @@ replica_new_from_entry(Slapi_Entry *e, char *errortext, PRBool is_add_operation, * during replica initialization */ rc = _replica_update_entry(r, e, errortext); + /* add changelog config entry to config + * this is only needed for replicas logging changes, + * but for now let it exist for all replicas. Makes handling + * of changing replica flags easier + */ + _replica_config_changelog(r); + if (r->repl_flags & REPLICA_LOG_CHANGES) { + /* Init changelog db file */ + cldb_SetReplicaDB(r, NULL); + } } else { /* * Entry is already in dse.ldif - update it on the disk @@ -795,6 +808,10 @@ replica_set_ruv(Replica *r, RUV *ruv) r->repl_ruv = object_new((void *)ruv, (FNFree)ruv_destroy); + if (r->repl_flags & REPLICA_LOG_CHANGES) { + cl5NotifyRUVChange(r); + } + replica_unlock(r->repl_lock); } @@ -1553,12 +1570,20 @@ replica_reload_ruv(Replica *r) " Recreating the changelog file. This could affect replication with replica's " " consumers in which case the consumers should be reinitialized.\n", slapi_sdn_get_dn(r->repl_root)); - rc = cl5DeleteDBSync(r); + + /* need to reset changelog db */ + rc = cldb_RemoveReplicaDB(r); + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "replica_reload_ruv: reset cldb for replica\n"); /* reinstate new ruv */ replica_lock(r->repl_lock); r->repl_ruv = new_ruv_obj; + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "replica_reload_ruv: set cldb for replica\n"); + + cldb_SetReplicaDB(r, NULL); if (rc == CL5_SUCCESS) { /* log changes to mark starting point for replication */ @@ -1685,7 +1710,10 @@ replica_check_for_data_reload(Replica *r, void *arg __attribute__((unused))) "consumers should be reinitialized.\n", slapi_sdn_get_dn(r->repl_root)); - rc = cl5DeleteDBSync(r); + + /* need to reset changelog db */ + rc = cldb_RemoveReplicaDB(r); + cldb_SetReplicaDB(r, NULL); if (rc == CL5_SUCCESS) { /* log changes to mark starting point for replication */ @@ -2409,6 +2437,26 @@ _replica_get_config_dn(const Slapi_DN *root) REPLICA_RDN, slapi_sdn_get_dn(root), mp_base); return dn; } +/* when a replica is added the changelog config entry is created + * it will only the container entry, specifications for trimming + * or encyrption need to be added separately + */ +static int +_replica_config_changelog(Replica *replica) +{ + int rc = 0; + + Slapi_Backend *be = slapi_be_select(replica_get_root(replica)); + + Slapi_Entry *config_entry = slapi_entry_alloc(); + slapi_entry_init(config_entry, slapi_ch_strdup("cn=changelog"), NULL); + slapi_entry_add_string(config_entry, "objectclass", "top"); + slapi_entry_add_string(config_entry, "objectclass", "extensibleObject"); + + rc = slapi_back_ctrl_info(be, BACK_INFO_CLDB_SET_CONFIG, (void *)config_entry); + + return rc; +} /* This function retrieves RUV from the root of the replicated tree. * The attribute can be missing if @@ -3499,18 +3547,13 @@ replica_get_referrals_nolock(const Replica *r, char ***referrals) } } -typedef struct replinfo -{ - char *repl_gen; - char *repl_name; -} replinfo; - static int replica_log_start_iteration(const ruv_enum_data *rid_data, void *data) { int rc = 0; - replinfo *r_info = (replinfo *)data; slapi_operation_parameters op_params; + Replica *replica = (Replica *)data; + cldb_Handle *cldb = NULL; if (rid_data->csn == NULL) return 0; @@ -3520,7 +3563,8 @@ replica_log_start_iteration(const ruv_enum_data *rid_data, void *data) op_params.target_address.sdn = slapi_sdn_new_ndn_byval(START_ITERATION_ENTRY_DN); op_params.target_address.uniqueid = START_ITERATION_ENTRY_UNIQUEID; op_params.csn = csn_dup(rid_data->csn); - rc = cl5WriteOperation(r_info->repl_name, r_info->repl_gen, &op_params, PR_FALSE); + cldb = replica_get_file_info(replica); + rc = cl5WriteOperation(cldb, &op_params); if (rc == CL5_SUCCESS) rc = 0; else @@ -3537,8 +3581,6 @@ replica_log_ruv_elements_nolock(const Replica *r) { int rc = 0; RUV *ruv; - char *repl_gen; - replinfo r_info; ruv = (RUV *)object_get_data(r->repl_ruv); PR_ASSERT(ruv); @@ -3546,15 +3588,7 @@ replica_log_ruv_elements_nolock(const Replica *r) /* we log it as a delete operation to have the least number of fields to set. the entry can be identified by a special target uniqueid and special target dn */ - repl_gen = ruv_get_replica_generation(ruv); - - r_info.repl_name = r->repl_name; - r_info.repl_gen = repl_gen; - - rc = ruv_enumerate_elements(ruv, replica_log_start_iteration, &r_info); - - slapi_ch_free((void **)&repl_gen); - + rc = ruv_enumerate_elements(ruv, replica_log_start_iteration, (void *)r); return rc; } @@ -3777,6 +3811,9 @@ replica_enable_replication(Replica *r) /* prevent creation of new agreements until the replica is enabled */ PR_Lock(r->agmt_lock); + if (r->repl_flags & REPLICA_LOG_CHANGES) { + cldb_SetReplicaDB(r, NULL); + } /* retrieve new ruv */ rc = replica_reload_ruv(r); @@ -3863,6 +3900,12 @@ replica_disable_replication(Replica *r) slapi_ch_free_string(&locking_purl); replica_set_state_flag(r, REPLICA_AGREEMENTS_DISABLED, PR_FALSE); PR_Unlock(r->agmt_lock); + /* no thread will access the changelog for this replica + * remove reference from replica object + */ + if (r->repl_flags & REPLICA_LOG_CHANGES) { + cldb_UnSetReplicaDB(r, NULL); + } slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name, "replica_disable_replication - " "replica %s is acquired\n", @@ -4147,3 +4190,12 @@ replica_unlock_replica(Replica *r) { replica_unlock(r->repl_lock); } +void* replica_get_file_info(Replica *r) +{ + return r->cldb; +} +int replica_set_file_info(Replica *r, void *cl) +{ + r->cldb = (cldb_Handle *)cl; + return 0; +} diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c index ed6b29e..72247a1 100644 --- a/ldap/servers/plugins/replication/repl5_replica_config.c +++ b/ldap/servers/plugins/replication/repl5_replica_config.c @@ -375,6 +375,10 @@ replica_config_modify(Slapi_PBlock *pb, */ char *new_repl_id = NULL; char *new_repl_type = NULL; + /* we also need to handle the change of repl_flags and enable or disable + * the changelog + */ + char *new_repl_flag = NULL; if (*returncode != LDAP_SUCCESS) break; @@ -619,6 +623,10 @@ replica_config_modify(Slapi_PBlock *pb, slapi_ch_free_string(&new_repl_type); agmtlist_notify_all(pb); } + if (new_repl_flag) { + *returncode = replica_config_change_flags(r, new_repl_flag, errortext, apply_mods); + slapi_ch_free_string(&new_repl_flag); + } } done: @@ -760,6 +768,8 @@ replica_config_delete(Slapi_PBlock *pb __attribute__((unused)), { multimaster_mtnode_extension *mtnode_ext; Replica *r; + int rc; + Slapi_Backend *be; PR_Lock(s_configLock); @@ -768,7 +778,22 @@ replica_config_delete(Slapi_PBlock *pb __attribute__((unused)), if (mtnode_ext->replica) { /* remove object from the hash */ + Object *r_obj = mtnode_ext->replica; + back_info_config_entry config_entry = {0}; + r = (Replica *)object_get_data(mtnode_ext->replica); + + /* retrieves changelog DN and keep it in config_entry.ce */ + be = slapi_be_select(replica_get_root(r)); + config_entry.dn = "cn=changelog"; + rc = slapi_back_ctrl_info(be, BACK_INFO_CLDB_GET_CONFIG, (void *)&config_entry); + if (rc !=0 || config_entry.ce == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "replica_config_delete - failed to read config for changelog\n"); + PR_Unlock(s_configLock); + *returncode = LDAP_OPERATIONS_ERROR; + return SLAPI_DSE_CALLBACK_ERROR; + } mtnode_ext->replica = NULL; /* moving it before deleting the CL because * deletion can take some time giving the opportunity * to an operation to start while CL is deleted @@ -779,8 +804,18 @@ replica_config_delete(Slapi_PBlock *pb __attribute__((unused)), "The changelog for replica %s is no longer valid since " "the replica config is being deleted. Removing the changelog.\n", slapi_sdn_get_dn(replica_get_root(r))); - cl5DeleteDBSync(r); + + /* As we are removing a replica, all references to it need to be cleared + * There are many of them: + * - all replicas are stored in a hash table (s_hash): replicat_delete_by_name + * - callbacks have been registered with the replica as argument: changelog5_register_config_callbacks + * - replica is stored in mapping tree extension mtnode_ext: object_release + */ + cldb_RemoveReplicaDB(r); replica_delete_by_name(replica_get_name(r)); + changelog5_remove_config_callbacks(slapi_entry_get_dn_const(config_entry.ce)); + slapi_entry_free(config_entry.ce); + object_release(r_obj); } PR_Unlock(s_configLock); @@ -973,7 +1008,7 @@ replica_config_change_type_and_id(Replica *r, const char *new_type, const char * replica_reset_csn_pl(r); } ruv_delete_replica(ruv, oldrid); - cl5CleanRUV(oldrid); + cl5CleanRUV(oldrid, r); replica_set_csn_assigned(r); } object_release(ruv_obj); @@ -1034,6 +1069,13 @@ replica_config_change_flags(Replica *r, const char *new_flags, char *returntext flags = atol(new_flags); + if (replica_is_flag_set(r, REPLICA_LOG_CHANGES) && !(flags&REPLICA_LOG_CHANGES)) { + /* the replica no longer maintains a changelog, reset */ + cldb_UnSetReplicaDB(r, NULL); + } else if (!replica_is_flag_set(r, REPLICA_LOG_CHANGES) && (flags&REPLICA_LOG_CHANGES)) { + /* the replica starts to maintains a changelog, set */ + cldb_SetReplicaDB(r, NULL); + } replica_replace_flags(r, flags); } @@ -1153,7 +1195,6 @@ static int replica_execute_cl2ldif_task(Replica *replica, char *returntext) { int rc; - Replica *rlist[2]; char fName[MAXPATHLEN]; char *clDir = NULL; @@ -1167,7 +1208,8 @@ replica_execute_cl2ldif_task(Replica *replica, char *returntext) /* file is stored in the changelog directory and is named .ldif */ - clDir = cl5GetDir(); + Slapi_Backend *be = slapi_be_select(replica_get_root(replica)); + clDir = cl5GetLdifDir(be); if (NULL == clDir) { rc = LDAP_OPERATIONS_ERROR; goto bail; @@ -1177,15 +1219,12 @@ replica_execute_cl2ldif_task(Replica *replica, char *returntext) rc = LDAP_OPERATIONS_ERROR; goto bail; } - rlist[0] = replica; - rlist[1] = NULL; - - PR_snprintf(fName, MAXPATHLEN, "%s/%s.ldif", clDir, replica_get_name(replica)); + PR_snprintf(fName, MAXPATHLEN, "%s/%s_cl.ldif", clDir, replica_get_name(replica)); slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name, "replica_execute_cl2ldif_task - Beginning changelog export of replica \"%s\"\n", replica_get_name(replica)); - rc = cl5ExportLDIF(fName, rlist); + rc = cl5ExportLDIF(fName, replica); if (rc == CL5_SUCCESS) { slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name, "replica_execute_cl2ldif_task - Finished changelog export of replica \"%s\"\n", @@ -1210,10 +1249,8 @@ static int replica_execute_ldif2cl_task(Replica *replica, char *returntext) { int rc, imprc = 0; - Replica *rlist[2]; char fName[MAXPATHLEN]; char *clDir = NULL; - changelog5Config config; if (cl5GetState() != CL5_STATE_OPEN) { PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "changelog is not open"); @@ -1225,7 +1262,8 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext) /* file is stored in the changelog directory and is named .ldif */ - clDir = cl5GetDir(); + Slapi_Backend *be = slapi_be_select(replica_get_root(replica)); + clDir = cl5GetLdifDir(be); if (NULL == clDir) { rc = LDAP_OPERATIONS_ERROR; goto bail; @@ -1235,8 +1273,6 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext) rc = LDAP_OPERATIONS_ERROR; goto bail; } - rlist[0] = replica; - rlist[1] = NULL; PR_snprintf(fName, MAXPATHLEN, "%s/%s.ldif", clDir, replica_get_name(replica)); @@ -1254,7 +1290,7 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext) slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name, "replica_execute_ldif2cl_task - Beginning changelog import of replica \"%s\"\n", replica_get_name(replica)); - imprc = cl5ImportLDIF(clDir, fName, rlist); + imprc = cl5ImportLDIF(clDir, fName, replica); if (CL5_SUCCESS == imprc) { slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name, "replica_execute_ldif2cl_task - Finished changelog import of replica \"%s\"\n", @@ -1268,21 +1304,18 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext) "replica_execute_ldif2cl_task - %s\n", returntext); imprc = LDAP_OPERATIONS_ERROR; } - changelog5_read_config(&config); /* restart changelog */ - rc = cl5Open(config.dir, &config.dbconfig); + rc = cl5Open(); if (CL5_SUCCESS == rc) { rc = LDAP_SUCCESS; } else { slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name, - "replica_execute_ldif2cl_task - Failed to start changelog at %s\n", - config.dir ? config.dir : "null config dir"); + "replica_execute_ldif2cl_task - Failed to start changelog after import\n"); rc = LDAP_OPERATIONS_ERROR; } bail: slapi_ch_free_string(&clDir); - changelog5_config_done(&config); /* if cl5ImportLDIF returned an error, report it first. */ return imprc ? imprc : rc; @@ -1363,7 +1396,7 @@ replica_execute_cleanruv_task(Replica *replica, ReplicaId rid, char *returntext /* * Clean the changelog RUV's */ - cl5CleanRUV(rid); + cl5CleanRUV(rid, replica); /* * Now purge the changelog. The purging thread will free the purge_data @@ -1371,8 +1404,7 @@ replica_execute_cleanruv_task(Replica *replica, ReplicaId rid, char *returntext purge_data = (cleanruv_purge_data *)slapi_ch_calloc(1, sizeof(cleanruv_purge_data)); purge_data->cleaned_rid = rid; purge_data->suffix_sdn = replica_get_root(replica); - purge_data->replName = (char *)replica_get_name(replica); - purge_data->replGen = replica_get_generation(replica); + purge_data->replica = replica; trigger_cl_purging(purge_data); if (rc != RUV_SUCCESS) { diff --git a/ldap/servers/plugins/replication/repl_extop.c b/ldap/servers/plugins/replication/repl_extop.c index 14c8e0b..3f94951 100644 --- a/ldap/servers/plugins/replication/repl_extop.c +++ b/ldap/servers/plugins/replication/repl_extop.c @@ -1159,11 +1159,13 @@ multimaster_extop_EndNSDS50ReplicationRequest(Slapi_PBlock *pb) The best solution I think, would be to "fake" on the supplier an entry that corresponds to the ruv sent to the consumer and then send it as part of the data */ - +/* if (cl5GetState() == CL5_STATE_OPEN) { cl5DeleteDBSync(connext->replica_acquired); } - + no longer needed, the cl was recreated when replication was reenabled + at the end of bulk import +*/ replica_set_ruv(r, connext->supplier_ruv); connext->supplier_ruv = NULL; @@ -1171,7 +1173,8 @@ multimaster_extop_EndNSDS50ReplicationRequest(Slapi_PBlock *pb) smallest csn in the new ruv, so that this replica ca supply other servers. */ - if (cl5GetState() == CL5_STATE_OPEN) { + if (replica_is_flag_set(r, REPLICA_LOG_CHANGES) && + cl5GetState() == CL5_STATE_OPEN) { replica_log_ruv_elements(r); } diff --git a/ldap/servers/slapd/back-ldbm/archive.c b/ldap/servers/slapd/back-ldbm/archive.c index 177c3f3..e51a563 100644 --- a/ldap/servers/slapd/back-ldbm/archive.c +++ b/ldap/servers/slapd/back-ldbm/archive.c @@ -412,19 +412,6 @@ ldbm_back_ldbm2archive(Slapi_PBlock *pb) } } - return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_PRE_BACKUP_FN); - if (return_value) { - slapi_log_err(SLAPI_LOG_BACKLDBM, - "ldbm_back_ldbm2archive", "pre-backup-plugin failed (%d).\n", return_value); - if (is_slapd_running() && run_from_cmdline) { - slapi_log_err(SLAPI_LOG_ERR, - "ldbm_back_ldbm2archive", "Standalone db2bak is not supported when a " - "multimaster replication enabled server is " - "coexisting.\nPlease use db2bak.pl, instead.\n"); - goto err; - } - } - /* tell it to archive */ return_value = dblayer_backup(li, directory, task); if (return_value) { @@ -433,12 +420,6 @@ ldbm_back_ldbm2archive(Slapi_PBlock *pb) goto err; } - return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN); - if (return_value) { - slapi_log_err(SLAPI_LOG_BACKLDBM, - "ldbm_back_ldbm2archive", "post-backup-plugin failed (%d).\n", return_value); - } - if (!run_from_cmdline) { ldbm_instance *inst; Object *inst_obj; diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h index 2636d4d..94ab66e 100644 --- a/ldap/servers/slapd/back-ldbm/back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h @@ -734,6 +734,7 @@ typedef struct ldbm_instance PRLock *inst_handle_list_mutex; DB *inst_id2entry; /* id2entry for this instance. */ + DB *inst_changelog; /* changelog for this instance. */ perfctrs_private inst_perf_private; /* Private data for the performance counters specific to this instance */ attrcrypt_state_private *inst_attrcrypt_state_private; diff --git a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c index 464f89f..44e2467 100644 --- a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c +++ b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c @@ -74,7 +74,8 @@ static int commit_good_database(bdb_config *priv, int mode); static int read_metadata(struct ldbminfo *li); static int count_dbfiles_in_dir(char *directory, int *count, int recurse); static int dblayer_override_libdb_functions(void); -static int dblayer_force_checkpoint(struct ldbminfo *li); +static int bdb_force_checkpoint(struct ldbminfo *li); +static int bdb_force_logrenewal(struct ldbminfo *li); static int log_flush_threadmain(void *param); static int dblayer_delete_transaction_logs(const char *log_dir); static int dblayer_is_logfilename(const char *path); @@ -95,6 +96,7 @@ static PRLock *sync_txn_log_flush = NULL; static PRCondVar *sync_txn_log_flush_done = NULL; static PRCondVar *sync_txn_log_do_flush = NULL; static int bdb_db_remove_ex(bdb_db_env *env, char const path[], char const dbName[], PRBool use_lock); +static int bdb_db_compact_one_db(DB *db, ldbm_instance *inst); static int bdb_restore_file_check(struct ldbminfo *li); #define MEGABYTE (1024 * 1024) @@ -2345,9 +2347,12 @@ bdb_get_db(backend *be, char *indexname, int open_flag, struct attrinfo *ai, DB goto out; dbp = *ppDB; - return_value = _dblayer_set_db_callbacks(conf, dbp, ai); - if (return_value) - goto out; + if (ai) { + return_value = _dblayer_set_db_callbacks(conf, dbp, ai); + if (return_value) { + goto out; + } + } /* The subname argument allows applications to have * subdatabases, i.e., multiple databases inside of a single @@ -2379,9 +2384,12 @@ bdb_get_db(backend *be, char *indexname, int open_flag, struct attrinfo *ai, DB goto out; } dbp = *ppDB; - return_value = _dblayer_set_db_callbacks(conf, dbp, ai); - if (return_value) - goto out; + if (ai) { + return_value = _dblayer_set_db_callbacks(conf, dbp, ai); + if (return_value) { + goto out; + } + } slapi_ch_free_string(&abs_file_name); } @@ -2457,6 +2465,62 @@ bdb_db_remove(bdb_db_env *env, char const path[], char const dbName[]) return (bdb_db_remove_ex(env, path, dbName, PR_TRUE)); } +static int +bdb_db_compact_one_db(DB *db, ldbm_instance *inst) +{ + DBTYPE type; + int rc = 0; + back_txn txn; + DB_COMPACT c_data = {0}; + + rc = db->get_type(db, &type); + if (rc) { + slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", + "compactdb: failed to determine db type for %s: db error - %d %s\n", + inst->inst_name, rc, db_strerror(rc)); + return rc; + } + + rc = dblayer_txn_begin(inst->inst_be, NULL, &txn); + if (rc) { + slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", "compactdb: transaction begin failed: %d\n", rc); + return rc; + } + /* + * https://docs.oracle.com/cd/E17275_01/html/api_reference/C/BDB-C_APIReference.pdf + * "DB_FREELIST_ONLY + * Do no page compaction, only returning pages to the filesystem that are already free and at the end + * of the file. This flag must be set if the database is a Hash access method database." + * + */ + + uint32_t compact_flags = DB_FREE_SPACE; + if (type == DB_HASH) { + compact_flags |= DB_FREELIST_ONLY; + } + rc = db->compact(db, txn.back_txn_txn, NULL /*start*/, NULL /*stop*/, + &c_data, compact_flags, NULL /*end*/); + if (rc) { + slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", + "compactdb: failed to compact %s; db error - %d %s\n", + inst->inst_name, rc, db_strerror(rc)); + if ((rc = dblayer_txn_abort(inst->inst_be, &txn))) { + slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", "compactdb: failed to abort txn (%s) db error - %d %s\n", + inst->inst_name, rc, db_strerror(rc)); + } + } else { + slapi_log_err(SLAPI_LOG_NOTICE, "bdb_db_compact_one_db", + "compactdb: compact %s - %d pages freed\n", + inst->inst_name, c_data.compact_pages_free); + if ((rc = dblayer_txn_commit(inst->inst_be, &txn))) { + slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", "compactdb: failed to commit txn (%s) db error - %d %s\n", + inst->inst_name, rc, db_strerror(rc)); + } + } + + return rc; +} + #define DBLAYER_CACHE_DELAY PR_MillisecondsToInterval(250) int bdb_rm_db_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_checkpoint) @@ -2497,7 +2561,7 @@ bdb_rm_db_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_ch Force a checkpoint here to break deadlock. */ if (0 == no_force_checkpoint) { - dblayer_force_checkpoint(li); + bdb_force_checkpoint(li); } if (0 == dblayer_get_index_file(be, a, &db, 0 /* Don't create an index file @@ -3554,7 +3618,7 @@ bdb_start_checkpoint_thread(struct ldbminfo *li) } /* - * checkpoint thread -- borrow the timing for compacting id2entry, as well. + * checkpoint thread -- borrow the timing for compacting id2entry, and eventually changelog, as well. */ static int checkpoint_threadmain(void *param) @@ -3573,7 +3637,6 @@ checkpoint_threadmain(void *param) time_t checkpoint_interval_update = 0; time_t compactdb_interval = 0; time_t checkpoint_interval = 0; - back_txn txn; PR_ASSERT(NULL != param); li = (struct ldbminfo *)param; @@ -3593,7 +3656,7 @@ checkpoint_threadmain(void *param) } /* work around a problem with newly created environments */ - dblayer_force_checkpoint(li); + bdb_force_checkpoint(li); PR_Lock(li->li_config_mutex); checkpoint_interval = (time_t)BDB_CONFIG(li)->bdb_checkpoint_interval; @@ -3602,7 +3665,7 @@ checkpoint_threadmain(void *param) debug_checkpointing = BDB_CONFIG(li)->bdb_debug_checkpointing; PR_Unlock(li->li_config_mutex); - /* assumes dblayer_force_checkpoint worked */ + /* assumes bdb_force_checkpoint worked */ /* * Importantly, the use of this api is not affected by backwards time steps * and the like. Because this use relative system time, rather than utc, @@ -3706,7 +3769,6 @@ checkpoint_threadmain(void *param) Object *inst_obj; ldbm_instance *inst; DB *db = NULL; - DB_COMPACT c_data = {0}; for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; @@ -3719,58 +3781,26 @@ checkpoint_threadmain(void *param) slapi_log_err(SLAPI_LOG_NOTICE, "checkpoint_threadmain", "Compacting DB start: %s\n", inst->inst_name); - /* - * It's possible for this to heap us after free because when we access db - * *just* as the server shut's down, we don't know it. So we should probably - * do something like wrapping access to the db var in a rwlock, and have "read" - * to access, and take writes to change the state. This would prevent the issue. - */ - DBTYPE type; - rc = db->get_type(db, &type); + rc = bdb_db_compact_one_db(db, inst); if (rc) { slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", - "compactdb: failed to determine db type for %s: db error - %d %s\n", - inst->inst_name, rc, db_strerror(rc)); - continue; - } - - rc = dblayer_txn_begin(inst->inst_be, NULL, &txn); - if (rc) { - slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", "compactdb: transaction begin failed: %d\n", rc); + "compactdb: failed to compact id2entry for %s; db error - %d %s\n", + inst->inst_name, rc, db_strerror(rc)); break; } - /* - * https://docs.oracle.com/cd/E17275_01/html/api_reference/C/BDB-C_APIReference.pdf - * "DB_FREELIST_ONLY - * Do no page compaction, only returning pages to the filesystem that are already free and at the end - * of the file. This flag must be set if the database is a Hash access method database." - * + + /* compact changelog db */ + /* NOTE (LK) this is now done along regular compaction, + * if it should be configurable add a switch to changelog config */ + dblayer_get_changelog(inst->inst_be, &db, 0); - uint32_t compact_flags = DB_FREE_SPACE; - if (type == DB_HASH) { - compact_flags |= DB_FREELIST_ONLY; - } - rc = db->compact(db, txn.back_txn_txn, NULL /*start*/, NULL /*stop*/, - &c_data, compact_flags, NULL /*end*/); + rc = bdb_db_compact_one_db(db, inst); if (rc) { slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", - "compactdb: failed to compact %s; db error - %d %s\n", - inst->inst_name, rc, db_strerror(rc)); - if ((rc = dblayer_txn_abort(inst->inst_be, &txn))) { - slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", "compactdb: failed to abort txn (%s) db error - %d %s\n", - inst->inst_name, rc, db_strerror(rc)); - break; - } - } else { - slapi_log_err(SLAPI_LOG_NOTICE, "checkpoint_threadmain", - "compactdb: compact %s - %d pages freed\n", - inst->inst_name, c_data.compact_pages_free); - if ((rc = dblayer_txn_commit(inst->inst_be, &txn))) { - slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", "compactdb: failed to commit txn (%s) db error - %d %s\n", - inst->inst_name, rc, db_strerror(rc)); - break; - } + "compactdb: failed to compact changelog for %s; db error - %d %s\n", + inst->inst_name, rc, db_strerror(rc)); + break; } } compactdb_interval = compactdb_interval_update; @@ -3778,7 +3808,7 @@ checkpoint_threadmain(void *param) } } slapi_log_err(SLAPI_LOG_TRACE, "checkpoint_threadmain", "Check point before leaving\n"); - rval = dblayer_force_checkpoint(li); + rval = bdb_force_checkpoint(li); error_return: DECR_THREAD_COUNT(pEnv); @@ -4036,7 +4066,7 @@ read_metadata(struct ldbminfo *li) /* handy routine for checkpointing the db */ static int -dblayer_force_checkpoint(struct ldbminfo *li) +bdb_force_checkpoint(struct ldbminfo *li) { int ret = 0, i; dblayer_private *priv = li->li_dblayer_private; @@ -4051,7 +4081,7 @@ dblayer_force_checkpoint(struct ldbminfo *li) if (BDB_CONFIG(li)->bdb_enable_transactions) { - slapi_log_err(SLAPI_LOG_TRACE, "dblayer_force_checkpoint", "Checkpointing database ...\n"); + slapi_log_err(SLAPI_LOG_TRACE, "bdb_force_checkpoint", "Checkpointing database ...\n"); /* * DB workaround. Newly created environments do not know what the @@ -4063,7 +4093,7 @@ dblayer_force_checkpoint(struct ldbminfo *li) for (i = 0; i < 2; i++) { ret = dblayer_txn_checkpoint(li, pEnv, PR_FALSE, PR_TRUE); if (ret != 0) { - slapi_log_err(SLAPI_LOG_ERR, "dblayer_force_checkpoint", "Checkpoint FAILED, error %s (%d)\n", + slapi_log_err(SLAPI_LOG_ERR, "bdb_force_checkpoint", "Checkpoint FAILED, error %s (%d)\n", dblayer_strerror(ret), ret); break; } @@ -4073,6 +4103,34 @@ dblayer_force_checkpoint(struct ldbminfo *li) return ret; } +/* routine to force all existing transaction logs to be cleared + * This is necessary if the transaction logs can contain references + * to no longer existing files, but would be processed in a fatal + * recovery (like in backup/restore). + * There is no straight forward way to do this, but the following + * scenario should work: + * + * 1. check for no longer needed transaction logs by + * calling log_archive() + * 2. delete these logs (1and2 similar to checkpointing + * 3. force a checkpoint + * 4. use log_printf() to write a "comment" to the current txn log + * force a checkpoint + * this could be done by writing once about 10MB or + * by writing smaller chunks in a loop + * 5. force a checkpoint and check again + * if a txn log to remove exists remove it and we are done + * else repeat step 4 + * + * NOTE: double check if force_checkpoint also does remove txn files + * then the check would have to be modified + */ +static int +bdb_force_logrenewal(struct ldbminfo *li) +{ + return 0; +} + static int _dblayer_delete_aux_dir(struct ldbminfo *li, char *path) { @@ -4206,6 +4264,12 @@ _dblayer_delete_instance_dir(ldbm_instance *inst, int startdb) if (pEnv && /* PL_strcmp takes NULL arg */ (PL_strcmp(LDBM_FILENAME_SUFFIX, strrchr(direntry->name, '.')) == 0)) { + if (strcmp(direntry->name, "changelog.db") == 0) { + /* do not delete the changelog, if it no longer + * matches the database it will be recreated later + */ + continue; + } rval = bdb_db_remove_ex(pEnv, filename, 0, PR_TRUE); } else { rval = ldbm_delete_dirs(filename); @@ -4221,8 +4285,10 @@ _dblayer_delete_instance_dir(ldbm_instance *inst, int startdb) } done: /* remove the directory itself too */ + /* no if (0 == rval) PR_RmDir(inst_dirp); + */ if (inst_dirp != inst_dir) slapi_ch_free_string(&inst_dirp); return rval; @@ -4236,7 +4302,7 @@ int dblayer_delete_instance_dir(backend *be) { struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; - int ret = dblayer_force_checkpoint(li); + int ret = bdb_force_checkpoint(li); if (ret != 0) { return ret; @@ -4783,84 +4849,6 @@ out: return return_value; } -/* - * Get changelogdir from cn=changelog5,cn=config - * The value does not have trailing spaces nor slashes. - * The changelogdir value must be a fullpath. - */ -static int -_dblayer_get_changelogdir(struct ldbminfo *li, char **changelogdir) -{ - Slapi_PBlock *pb = NULL; - Slapi_Entry **entries = NULL; - Slapi_Attr *attr = NULL; - Slapi_Value *v = NULL; - const char *s = NULL; - char *attrs[2]; - int rc = -1; - - if (NULL == li || NULL == changelogdir) { - slapi_log_err(SLAPI_LOG_ERR, - "_dblayer_get_changelogdir", "Invalid arg: " - "li: 0x%p, changelogdir: 0x%p\n", - li, changelogdir); - return rc; - } - *changelogdir = NULL; - - pb = slapi_pblock_new(); - attrs[0] = CHANGELOGDIRATTR; - attrs[1] = NULL; - slapi_search_internal_set_pb(pb, CHANGELOGENTRY, - LDAP_SCOPE_BASE, "cn=*", attrs, 0, NULL, NULL, - li->li_identity, 0); - slapi_search_internal_pb(pb); - slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); - - if (LDAP_NO_SUCH_OBJECT == rc) { - /* No changelog; Most likely standalone or not a master. */ - rc = LDAP_SUCCESS; - goto bail; - } - if (LDAP_SUCCESS != rc) { - slapi_log_err(SLAPI_LOG_ERR, - "_dblayer_get_changelogdir", "Failed to search \"%s\"\n", CHANGELOGENTRY); - goto bail; - } - /* rc == LDAP_SUCCESS */ - slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); - if (NULL == entries) { - /* No changelog */ - goto bail; - } - /* There should be only one entry. */ - rc = slapi_entry_attr_find(entries[0], CHANGELOGDIRATTR, &attr); - if (rc || NULL == attr) { - /* No changelog dir */ - rc = LDAP_SUCCESS; - goto bail; - } - rc = slapi_attr_first_value(attr, &v); - if (rc || NULL == v) { - /* No changelog dir */ - rc = LDAP_SUCCESS; - goto bail; - } - rc = LDAP_SUCCESS; - s = slapi_value_get_string(v); - if (NULL == s) { - /* No changelog dir */ - goto bail; - } - *changelogdir = slapi_ch_strdup(s); - /* Remove trailing spaces and '/' if any */ - normalize_dir(*changelogdir); -bail: - slapi_free_search_results_internal(pb); - slapi_pblock_destroy(pb); - return rc; -} - /* Destination Directory is an absolute pathname */ int bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task) @@ -4869,6 +4857,7 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task) bdb_config *conf = NULL; char **listA = NULL, **listB = NULL, **listi, **listj, *prefix; char *home_dir = NULL; + char *db_dir = NULL; int return_value = -1; char *pathname1; char *pathname2; @@ -4883,6 +4872,9 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task) conf = (bdb_config *)li->li_dblayer_config; priv = li->li_dblayer_private; PR_ASSERT(NULL != priv); + + db_dir = bdb_get_db_dir(li); + home_dir = bdb_get_home_dir(li, NULL); if (NULL == home_dir || '\0' == *home_dir) { slapi_log_err(SLAPI_LOG_ERR, @@ -4919,7 +4911,7 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task) */ /* do a quick checkpoint */ - dblayer_force_checkpoint(li); + bdb_force_checkpoint(li); dblayer_txn_init(li, &txn); return_value = dblayer_txn_begin_all(li, NULL, &txn); if (return_value) { @@ -4998,46 +4990,6 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task) if (inst_dirp != inst_dir) slapi_ch_free_string(&inst_dirp); } - /* Get changelogdir, if any */ - _dblayer_get_changelogdir(li, &changelogdir); - if (changelogdir) { - /* dest dir for changelog: dest_dir/repl_changelog_backup */ - char *changelog_destdir = slapi_ch_smprintf("%s/%s", - dest_dir, CHANGELOG_BACKUPDIR); - return_value = bdb_copy_directory(li, task, changelogdir, - changelog_destdir, - 0 /* backup */, - &cnt, 0, 1); - if (return_value) { - slapi_log_err(SLAPI_LOG_ERR, - "dblayer_backup", "Error in copying directory " - "(%s -> %s): err=%d\n", - changelogdir, changelog_destdir, return_value); - if (task) { - slapi_task_log_notice(task, - "Backup: error in copying directory " - "(%s -> %s): err=%d\n", - changelogdir, changelog_destdir, return_value); - } - slapi_ch_free_string(&changelog_destdir); - goto bail; - } - /* Copy DBVERSION */ - pathname1 = slapi_ch_smprintf("%s/%s", - changelogdir, DBVERSION_FILENAME); - pathname2 = slapi_ch_smprintf("%s/%s", - changelog_destdir, DBVERSION_FILENAME); - return_value = dblayer_copyfile(pathname1, pathname2, - 0, priv->dblayer_file_mode); - slapi_ch_free_string(&pathname2); - slapi_ch_free_string(&changelog_destdir); - if (0 > return_value) { - slapi_log_err(SLAPI_LOG_ERR, "dblayer_backup", "Failed to copy file %s\n", pathname1); - slapi_ch_free_string(&pathname1); - goto bail; - } - slapi_ch_free_string(&pathname1); - } if (conf->bdb_enable_transactions) { /* now, get the list of logfiles that still exist */ return_value = LOG_ARCHIVE(((bdb_db_env *)priv->dblayer_env)->bdb_DB_ENV, @@ -5088,7 +5040,7 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task) (0 != strlen(conf->bdb_log_directory))) { prefix = conf->bdb_log_directory; } else { - prefix = home_dir; + prefix = db_dir; } /* log files have the same filename len(100 is a safety net:) */ p1len = strlen(prefix) + strlen(*listB) + 100; @@ -5368,13 +5320,6 @@ bdb_restore(struct ldbminfo *li, char *src_dir, Slapi_Task *task) { tmp_rval = PR_GetFileInfo64(filename1, &info); if (tmp_rval == PR_SUCCESS && PR_FILE_DIRECTORY == info.type) { - /* Is it CHANGELOG_BACKUPDIR? */ - if (0 == strcmp(CHANGELOG_BACKUPDIR, direntry->name)) { - /* Yes, this is a changelog backup. */ - /* Get the changelog path */ - _dblayer_get_changelogdir(li, &changelogdir); - continue; - } inst = ldbm_instance_find_by_name(li, (char *)direntry->name); if (inst == NULL) { slapi_log_err(SLAPI_LOG_ERR, @@ -6019,6 +5964,14 @@ bdb_get_info(Slapi_Backend *be, int cmd, void **info) } break; } + case BACK_INFO_INSTANCE_DIR: { + if (li) { + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; + *(char **)info = dblayer_get_full_inst_dir(li, inst, NULL, 0); + rc = 0; + } + break; + } case BACK_INFO_LOG_DIRECTORY: { if (li) { *(char **)info = bdb_config_db_logdirectory_get_ext((void *)li); @@ -6034,6 +5987,21 @@ bdb_get_info(Slapi_Backend *be, int cmd, void **info) rc = get_suffix_key(be, (struct _back_info_index_key *)info); break; } + case BACK_INFO_DBENV_CLDB: { + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + if (inst->inst_changelog) { + rc = 0; + } else { + DB *db; + rc = dblayer_get_changelog(be, &db,DB_CREATE); + } + if (rc == 0) { + *(DB **)info = inst->inst_changelog; + } else { + *(DB **)info = NULL; + } + break; + } default: break; } @@ -6069,7 +6037,13 @@ bdb_back_ctrl(Slapi_Backend *be, int cmd, void *info) switch (cmd) { case BACK_INFO_CRYPT_INIT: { back_info_crypt_init *crypt_init = (back_info_crypt_init *)info; - rc = back_crypt_init(crypt_init->be, crypt_init->dn, + Slapi_DN configdn; + slapi_sdn_init(&configdn); + be_getbasedn(be, &configdn); + char *crypt_dn = slapi_ch_smprintf("%s,%s", + crypt_init->dn, + slapi_sdn_get_dn(&configdn)); + rc = back_crypt_init(crypt_init->be, crypt_dn, crypt_init->encryptionAlgorithm, &(crypt_init->state_priv)); break; @@ -6091,6 +6065,106 @@ bdb_back_ctrl(Slapi_Backend *be, int cmd, void *info) &(crypt_value->out)); break; } + case BACK_INFO_DBENV_CLDB_REMOVE: { + DB *db = (DB *)info; + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + if (li) { + dblayer_private *priv = (dblayer_private *)li->li_dblayer_private; + if (priv && priv->dblayer_env) { + char *instancedir; + slapi_back_get_info(be, BACK_INFO_INSTANCE_DIR, (void **)&instancedir); + char *path = slapi_ch_smprintf("%s/changelog.db", instancedir); + db->close(db, 0); + rc = bdb_db_remove_ex((bdb_db_env *)priv->dblayer_env, path, NULL, PR_TRUE); + inst->inst_changelog = NULL; + slapi_ch_free_string(&instancedir); + } + } + break; + } + case BACK_INFO_DBENV_CLDB_UPGRADE: { + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + char *oldFile = (char *)info; + if (li) { + dblayer_private *priv = (dblayer_private *)li->li_dblayer_private; + if (priv && priv->dblayer_env) { + DB_ENV *pEnv = ((bdb_db_env *)priv->dblayer_env)->bdb_DB_ENV; + if (pEnv) { + char *instancedir; + slapi_back_get_info(be, BACK_INFO_INSTANCE_DIR, (void **)&instancedir); + char *newFile = slapi_ch_smprintf("%s/changelog.db", instancedir); + rc = pEnv->dbrename(pEnv, 0, oldFile, 0, newFile, 0); + slapi_ch_free_string(&instancedir); + bdb_force_logrenewal(li); + } + } + } + break; + } + case BACK_INFO_CLDB_GET_CONFIG: { + /* get a config entry relative to the + * backend config entry + * Caller must free the returned entry (config->ce) + * If it fails config->ce is left unchanged + */ + back_info_config_entry *config = (back_info_config_entry *)info; + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + Slapi_DN configdn; + slapi_sdn_init(&configdn); + be_getbasedn(be, &configdn); + char *config_dn = slapi_ch_smprintf("%s,%s", + config->dn, + slapi_sdn_get_dn(&configdn)); + Slapi_PBlock *search_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(search_pb, config_dn, LDAP_SCOPE_BASE, "objectclass=*", + NULL, 0, NULL, NULL, li->li_identity, 0); + slapi_search_internal_pb(search_pb); + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); + if (LDAP_SUCCESS == rc ) { + Slapi_Entry **entries; + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); + if (entries && entries[0]) { + config->ce = slapi_entry_dup(entries[0]); + } else { + rc = -1; + } + } + slapi_free_search_results_internal(search_pb); + slapi_pblock_destroy(search_pb); + slapi_ch_free_string(&config_dn); + break; + } + case BACK_INFO_CLDB_SET_CONFIG: { + /* This control option allows a plugin to set a backend configuration + * entry without knowing the location of the backend config. + * It passes an entry with a relative dn and this dn is expanded by the + * backend config dn. + */ + Slapi_DN fulldn; + Slapi_DN configdn; + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + Slapi_Entry *config_entry = (Slapi_Entry *)info; + + slapi_sdn_init(&configdn); + be_getbasedn(be, &configdn); + char *newdn = slapi_ch_smprintf("%s,%s", + slapi_entry_get_dn_const(config_entry), + slapi_sdn_get_dn(&configdn)); + slapi_sdn_init(&fulldn); + slapi_sdn_init_dn_byref(&fulldn, newdn); + slapi_entry_set_sdn(config_entry, &fulldn); + slapi_ch_free_string(&newdn); + + Slapi_PBlock *pb = slapi_pblock_new(); + slapi_pblock_init(pb); + slapi_add_entry_internal_set_pb(pb, config_entry, NULL, + li->li_identity, 0); + slapi_add_internal_pb(pb); + slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_RESULT, &rc); + slapi_pblock_destroy(pb); + break; + } default: break; } diff --git a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_ldif2db.c b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_ldif2db.c index 84dfb7d..4ce2269 100644 --- a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_ldif2db.c +++ b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_ldif2db.c @@ -706,6 +706,7 @@ bdb_db2ldif(Slapi_PBlock *pb) int decrypt = 0; int32_t dump_replica = 0; int dump_uniqueid = 1; + int dump_changelog = 0; int fd = STDOUT_FILENO; IDList *idl = NULL; /* optimization for -s include lists */ int cnt = 0, lastcnt = 0; @@ -816,6 +817,7 @@ bdb_db2ldif(Slapi_PBlock *pb) slapi_pblock_get(pb, SLAPI_DB2LDIF_FILE, &fname); slapi_pblock_get(pb, SLAPI_DB2LDIF_PRINTKEY, &printkey); slapi_pblock_get(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &dump_uniqueid); + slapi_pblock_get(pb, SLAPI_LDIF_CHANGELOG, &dump_changelog); /* tsk, overloading printkey. shame on me. */ ok_index = !(printkey & EXPORT_ID2ENTRY_ONLY); @@ -1286,6 +1288,12 @@ bdb_db2ldif(Slapi_PBlock *pb) "export %s: Processed %d entries (100%%).\n", inst->inst_name, cnt); } + if (run_from_cmdline && dump_changelog) { + return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN); + slapi_log_err(SLAPI_LOG_INFO, "ldbm_back_ldbm2ldif", + "export changelog for %s.\n", inst->inst_name); + } + bye: if (idl) { idl_free(&idl); diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index 05cc5b8..452405f 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -92,6 +92,7 @@ #define NEWDIR_MODE 0755 #define DB_REGION_PREFIX "__db." +#define BE_CHANGELOG_FILE "changelog" static int dblayer_post_restore = 0; @@ -396,6 +397,35 @@ dblayer_release_id2entry(backend *be __attribute__((unused)), DB *pDB __attribut } int +dblayer_close_changelog(backend *be) +{ + ldbm_instance *inst; + DB *pDB = NULL; + int return_value = 0; + + PR_ASSERT(NULL != be); + inst = (ldbm_instance *) be->be_instance_info; + PR_ASSERT(NULL != inst); + + pDB = inst->inst_changelog; + if (pDB) { + return_value = pDB->close(pDB,0); + inst->inst_changelog = NULL; + } + return return_value; +} + +int +dblayer_erase_changelog_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_chkpt) +{ + if ((NULL == be) || (NULL == be->be_database)) { + return 0; + } + /* TBD (LK) */ + return 0; +} + +int dblayer_close_indexes(backend *be) { ldbm_instance *inst; @@ -464,6 +494,7 @@ dblayer_instance_close(backend *be) } return_value = dblayer_close_indexes(be); + return_value |= dblayer_close_changelog(be); /* Now close id2entry if it's open */ pDB = inst->inst_id2entry; @@ -604,6 +635,52 @@ dblayer_get_index_file(backend *be, struct attrinfo *a, DB **ppDB, int open_flag return return_value; } +int dblayer_get_changelog(backend *be, DB** ppDB, int open_flags) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int return_value = -1; + DB *pDB = NULL; + + *ppDB = NULL; + + if (inst->inst_changelog) { + /* This means that the pointer is valid, so we should return it. */ + *ppDB = inst->inst_changelog; + return 0; + } + + /* only one thread should open the chgangelog, we can use the mutex + * for opening the index files. + */ + PR_Lock(inst->inst_handle_list_mutex); + if (inst->inst_changelog) { + /* another thread set the handle while we were waiting on the lock */ + *ppDB = inst->inst_changelog; + PR_Unlock(inst->inst_handle_list_mutex); + return 0; + } + + /* attrinfo handle is still blank, and we have the mutex: open the + * index file and stuff it in the attrinfo. + */ + return_value = dblayer_open_file(be, BE_CHANGELOG_FILE, open_flags, + NULL, &pDB); + if (0 == return_value) { + /* Opened it OK */ + inst->inst_changelog = pDB; + /* And, most importantly, return something to the caller!*/ + *ppDB = pDB; + } else { + /* Did not open it OK ! */ + /* Do nothing, because return value and fact that we didn't + * store a DB* in the attrinfo is enough + */ + } + PR_Unlock(inst->inst_handle_list_mutex); + + return return_value; +} + /* * Unlock the db lib mutex here if we need to. */ diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h index 8f842d9..854e814 100644 --- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h @@ -82,6 +82,7 @@ int dblayer_get_index_file(backend *be, struct attrinfo *a, DB **ppDB, int creat int dblayer_release_index_file(backend *be, struct attrinfo *a, DB *pDB); int dblayer_erase_index_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_chkpt); int dblayer_get_id2entry(backend *be, DB **ppDB); +int dblayer_get_changelog(backend *be, DB** ppDB, int create); int dblayer_release_id2entry(backend *be, DB *pDB); int dblayer_txn_init(struct ldbminfo *li, back_txn *txn); int dblayer_txn_begin(backend *be, back_txnid parent_txn, back_txn *txn); diff --git a/ldap/servers/slapd/backend.c b/ldap/servers/slapd/backend.c index 3e9edc8..85db5b3 100644 --- a/ldap/servers/slapd/backend.c +++ b/ldap/servers/slapd/backend.c @@ -271,6 +271,17 @@ slapi_be_gettype(Slapi_Backend *be) } Slapi_DN * +be_getbasedn(Slapi_Backend *be, Slapi_DN *dn) +{ + if (be->be_state == BE_STATE_DELETED) { + slapi_sdn_set_ndn_byval(dn, NULL); + } else { + slapi_sdn_set_ndn_byref(dn, be->be_basedn); + } + return dn; +} + +Slapi_DN * be_getconfigdn(Slapi_Backend *be, Slapi_DN *dn) { if (be->be_state == BE_STATE_DELETED) { diff --git a/ldap/servers/slapd/main.c b/ldap/servers/slapd/main.c index 694375b..ec4d863 100644 --- a/ldap/servers/slapd/main.c +++ b/ldap/servers/slapd/main.c @@ -89,6 +89,7 @@ struct main_config char *archive_name; int db2ldif_dump_replica; int db2ldif_dump_uniqueid; + int ldif_include_changelog; int ldif2db_generate_uniqueid; char *ldif2db_namespaceid; int importexport_encrypt; @@ -517,6 +518,7 @@ main(int argc, char **argv) /* Set a number of defaults */ mcfg.slapd_exemode = SLAPD_EXEMODE_UNKNOWN; mcfg.ldif_printkey = EXPORT_PRINTKEY | EXPORT_APPENDMODE; + mcfg.ldif_include_changelog = 0; mcfg.db2ldif_dump_uniqueid = 1; mcfg.ldif2db_generate_uniqueid = SLAPI_UNIQUEID_GENERATE_TIME_BASED; mcfg.ldif2db_removedupvals = 1; @@ -1181,7 +1183,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg) * */ - char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1qV"; + char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1qRV"; struct opt_ext long_options_db2ldif[] = { {"version", ArgNone, 'v'}, {"debug", ArgRequired, 'd'}, @@ -1195,6 +1197,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg) {"noUniqueIds", ArgNone, 'u'}, {"configDir", ArgRequired, 'D'}, {"encrypt", ArgOptional, 'E'}, + {"includechangelog", ArgNone, 'R'}, {"nowrap", ArgNone, 'U'}, {"minimalEncode", ArgNone, 'm'}, {"oneOutputFile", ArgNone, 'o'}, @@ -1600,6 +1603,20 @@ process_command_line(int argc, char **argv, struct main_config *mcfg) break; + case 'R': /* db2ldif and ldif2db only */ + if (mcfg->slapd_exemode != SLAPD_EXEMODE_DB2LDIF && + mcfg->slapd_exemode != SLAPD_EXEMODE_LDIF2DB) { + usage(mcfg->myname, mcfg->extraname, mcfg->slapd_exemode); + exit(1); + } + + /* + * import/export should handle changelog. + */ + mcfg->ldif_include_changelog = 1; + + break; + case 'C': if (mcfg->slapd_exemode == SLAPD_EXEMODE_LDIF2DB) { /* used to mean "Cool new import" (which is now @@ -2177,6 +2194,7 @@ slapd_exemode_db2ldif(int argc, char **argv, struct main_config *mcfg) slapi_pblock_set(pb, SLAPI_BACKEND_INSTANCE_NAME, *instp); slapi_pblock_set_ldif_dump_replica(pb, mcfg->db2ldif_dump_replica); slapi_pblock_set(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &(mcfg->db2ldif_dump_uniqueid)); + slapi_pblock_set(pb, SLAPI_LDIF_CHANGELOG, &(mcfg->ldif_include_changelog)); int32_t task_flags = SLAPI_TASK_RUNNING_FROM_COMMANDLINE; slapi_pblock_set(pb, SLAPI_TASK_FLAGS, &task_flags); int32_t is_running = 0; diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c index 2b6b4fd..d342427 100644 --- a/ldap/servers/slapd/pblock.c +++ b/ldap/servers/slapd/pblock.c @@ -1221,12 +1221,6 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value) } (*(IFP *)value) = pblock->pb_plugin->plg_bepreclose; break; - case SLAPI_PLUGIN_BE_PRE_BACKUP_FN: - if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPREOPERATION) { - return (-1); - } - (*(IFP *)value) = pblock->pb_plugin->plg_beprebackup; - break; /* backend postoperation plugin */ case SLAPI_PLUGIN_BE_POST_MODIFY_FN: @@ -1259,11 +1253,18 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value) } (*(IFP *)value) = pblock->pb_plugin->plg_bepostopen; break; - case SLAPI_PLUGIN_BE_POST_BACKUP_FN: + + case SLAPI_PLUGIN_BE_POST_EXPORT_FN: if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { return (-1); } - (*(IFP *)value) = pblock->pb_plugin->plg_bepostbackup; + (*(IFP *)value) = pblock->pb_plugin->plg_bepostexport; + break; + case SLAPI_PLUGIN_BE_POST_IMPORT_FN: + if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { + return (-1); + } + (*(IFP *)value) = pblock->pb_plugin->plg_bepostimport; break; /* internal preoperation plugin */ @@ -2079,6 +2080,14 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value) (*(int *)value) = 0; } break; + case SLAPI_LDIF_CHANGELOG: + if (pblock->pb_task != NULL) { + (*(int *)value) = pblock->pb_task->ldif_include_changelog; + } else { + (*(int *)value) = 0; + } + break; + /* dbverify */ case SLAPI_DBVERIFY_DBDIR: if (pblock->pb_task != NULL) { @@ -3110,12 +3119,6 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value) } pblock->pb_plugin->plg_bepreclose = (IFP)value; break; - case SLAPI_PLUGIN_BE_PRE_BACKUP_FN: - if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPREOPERATION) { - return (-1); - } - pblock->pb_plugin->plg_beprebackup = (IFP)value; - break; /* backend postoperation plugin */ case SLAPI_PLUGIN_BE_POST_MODIFY_FN: @@ -3148,11 +3151,17 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value) } pblock->pb_plugin->plg_bepostopen = (IFP)value; break; - case SLAPI_PLUGIN_BE_POST_BACKUP_FN: + case SLAPI_PLUGIN_BE_POST_EXPORT_FN: if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { return (-1); } - pblock->pb_plugin->plg_bepostbackup = (IFP)value; + pblock->pb_plugin->plg_bepostexport = (IFP)value; + break; + case SLAPI_PLUGIN_BE_POST_IMPORT_FN: + if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { + return (-1); + } + pblock->pb_plugin->plg_bepostimport = (IFP)value; break; /* internal preoperation plugin */ @@ -3867,6 +3876,11 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value) pblock->pb_task->import_state = *((int *)value); break; + case SLAPI_LDIF_CHANGELOG: + _pblock_assert_pb_task(pblock); + pblock->pb_task->ldif_include_changelog = *((int *)value); + break; + case SLAPI_LDIF2DB_ENCRYPT: case SLAPI_DB2LDIF_DECRYPT: _pblock_assert_pb_task(pblock); diff --git a/ldap/servers/slapd/pblock_v3.h b/ldap/servers/slapd/pblock_v3.h index 90498c0..2f28a2d 100644 --- a/ldap/servers/slapd/pblock_v3.h +++ b/ldap/servers/slapd/pblock_v3.h @@ -60,6 +60,7 @@ typedef struct _slapi_pblock_task char *ldif_namespaceid; /* used for name based uniqueid generation */ int ldif_dump_replica; int ldif_dump_uniqueid; /* dump uniqueid during db2ldif */ + int ldif_include_changelog; /* include changelog for import/export */ int ldif_generate_uniqueid; /* generate uniqueid during db2ldif */ int ldif_encrypt; /* used to enable encrypt/decrypt on import and export */ int seq_type; diff --git a/ldap/servers/slapd/plugin.c b/ldap/servers/slapd/plugin.c index c4ad6f1..539b1e7 100644 --- a/ldap/servers/slapd/plugin.c +++ b/ldap/servers/slapd/plugin.c @@ -350,7 +350,6 @@ plugin_call_plugins(Slapi_PBlock *pb, int whichfunction) case SLAPI_PLUGIN_BE_PRE_ADD_FN: case SLAPI_PLUGIN_BE_PRE_DELETE_FN: case SLAPI_PLUGIN_BE_PRE_CLOSE_FN: - case SLAPI_PLUGIN_BE_PRE_BACKUP_FN: plugin_list_number = PLUGIN_LIST_BEPREOPERATION; do_op = 1; /* always allow backend callbacks (even during startup) */ break; @@ -359,7 +358,8 @@ plugin_call_plugins(Slapi_PBlock *pb, int whichfunction) case SLAPI_PLUGIN_BE_POST_ADD_FN: case SLAPI_PLUGIN_BE_POST_DELETE_FN: case SLAPI_PLUGIN_BE_POST_OPEN_FN: - case SLAPI_PLUGIN_BE_POST_BACKUP_FN: + case SLAPI_PLUGIN_BE_POST_EXPORT_FN: + case SLAPI_PLUGIN_BE_POST_IMPORT_FN: plugin_list_number = PLUGIN_LIST_BEPOSTOPERATION; do_op = 1; /* always allow backend callbacks (even during startup) */ break; @@ -3565,8 +3565,8 @@ plugin_invoke_plugin_pb(struct slapdplugin *plugin, int operation, Slapi_PBlock operation == SLAPI_PLUGIN_CLEANUP_FN || operation == SLAPI_PLUGIN_BE_PRE_CLOSE_FN || operation == SLAPI_PLUGIN_BE_POST_OPEN_FN || - operation == SLAPI_PLUGIN_BE_PRE_BACKUP_FN || - operation == SLAPI_PLUGIN_BE_POST_BACKUP_FN) + operation == SLAPI_PLUGIN_BE_POST_EXPORT_FN || + operation == SLAPI_PLUGIN_BE_POST_IMPORT_FN) return PR_TRUE; slapi_pblock_get(pb, SLAPI_OPERATION, &pb_op); diff --git a/ldap/servers/slapd/proto-slap.h b/ldap/servers/slapd/proto-slap.h index 2356738..254115f 100644 --- a/ldap/servers/slapd/proto-slap.h +++ b/ldap/servers/slapd/proto-slap.h @@ -180,6 +180,7 @@ void be_done(Slapi_Backend *be); void be_addsuffix(Slapi_Backend *be, const Slapi_DN *suffix); Slapi_DN *be_getconfigdn(Slapi_Backend *be, Slapi_DN *dn); Slapi_DN *be_getmonitordn(Slapi_Backend *be, Slapi_DN *dn); +Slapi_DN *be_getbasedn(Slapi_Backend *be, Slapi_DN *dn); int be_writeconfig(Slapi_Backend *be); void global_backend_lock_init(void); int global_backend_lock_requested(void); diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h index 894efd2..7e5dfb9 100644 --- a/ldap/servers/slapd/slap.h +++ b/ldap/servers/slapd/slap.h @@ -1179,14 +1179,12 @@ struct slapdplugin IFP plg_un_bepre_delete; /* delete */ IFP plg_un_bepre_delete_tombstone; /* tombstone creation */ IFP plg_un_bepre_close; /* close */ - IFP plg_un_bepre_backup; /* backup */ } plg_un_bepre; #define plg_bepremodify plg_un.plg_un_bepre.plg_un_bepre_modify #define plg_bepremodrdn plg_un.plg_un_bepre.plg_un_bepre_modrdn #define plg_bepreadd plg_un.plg_un_bepre.plg_un_bepre_add #define plg_bepredelete plg_un.plg_un_bepre.plg_un_bepre_delete #define plg_bepreclose plg_un.plg_un_bepre.plg_un_bepre_close -#define plg_beprebackup plg_un.plg_un_bepre.plg_un_bepre_backup /* backend post-operation plugin structure */ struct plg_un_bepost_operation @@ -1196,14 +1194,16 @@ struct slapdplugin IFP plg_un_bepost_add; /* add */ IFP plg_un_bepost_delete; /* delete */ IFP plg_un_bepost_open; /* open */ - IFP plg_un_bepost_backup; /* backup */ + IFP plg_un_bepost_import; /* import */ + IFP plg_un_bepost_export; /* export */ } plg_un_bepost; #define plg_bepostmodify plg_un.plg_un_bepost.plg_un_bepost_modify #define plg_bepostmodrdn plg_un.plg_un_bepost.plg_un_bepost_modrdn #define plg_bepostadd plg_un.plg_un_bepost.plg_un_bepost_add #define plg_bepostdelete plg_un.plg_un_bepost.plg_un_bepost_delete #define plg_bepostopen plg_un.plg_un_bepost.plg_un_bepost_open -#define plg_bepostbackup plg_un.plg_un_bepost.plg_un_bepost_backup +#define plg_bepostimport plg_un.plg_un_bepost.plg_un_bepost_import +#define plg_bepostexport plg_un.plg_un_bepost.plg_un_bepost_export /* internal pre-operation plugin structure */ struct plg_un_internal_pre_operation diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h index 3aadf38..713aaee 100644 --- a/ldap/servers/slapd/slapi-plugin.h +++ b/ldap/servers/slapd/slapi-plugin.h @@ -7075,7 +7075,6 @@ typedef struct slapi_plugindesc #define SLAPI_PLUGIN_BE_PRE_MODRDN_FN 452 #define SLAPI_PLUGIN_BE_PRE_DELETE_FN 453 #define SLAPI_PLUGIN_BE_PRE_CLOSE_FN 454 -#define SLAPI_PLUGIN_BE_PRE_BACKUP_FN 455 /* preoperation plugin to the backend - just after transaction creation */ #define SLAPI_PLUGIN_BE_TXN_PRE_ADD_FN 460 @@ -7112,7 +7111,8 @@ typedef struct slapi_plugindesc #define SLAPI_PLUGIN_BE_POST_MODRDN_FN 552 #define SLAPI_PLUGIN_BE_POST_DELETE_FN 553 #define SLAPI_PLUGIN_BE_POST_OPEN_FN 554 -#define SLAPI_PLUGIN_BE_POST_BACKUP_FN 555 +#define SLAPI_PLUGIN_BE_POST_EXPORT_FN 556 +#define SLAPI_PLUGIN_BE_POST_IMPORT_FN 557 /* postoperation plugin to the backend - just before transaction commit */ #define SLAPI_PLUGIN_BE_TXN_POST_ADD_FN 560 @@ -7413,6 +7413,7 @@ typedef enum _slapi_op_note_t { #define SLAPI_DB2LDIF_FILE 184 /* dump uniqueid */ #define SLAPI_DB2LDIF_DUMP_UNIQUEID 176 +#define SLAPI_LDIF_CHANGELOG 1761 #define SLAPI_DB2LDIF_SERVER_RUNNING 197 /* db2ldif/ldif2db/bak2db/db2bak arguments */ @@ -7726,6 +7727,7 @@ int slapi_check_account_lock(Slapi_PBlock *pb, Slapi_Entry *bind_target_entry, i * * \note Implemented cmd: * BACK_INFO_DBENV - Get the dbenv + * BACK_INFO_DBENV_CLBD - Get the changelog db for the backend * BACK_INFO_DBENV_OPENFLAGS - Get the dbenv openflags * BACK_INFO_INDEXPAGESIZE - Get the index page size */ @@ -7764,6 +7766,12 @@ int slapi_back_ctrl_info(Slapi_Backend *be, int cmd, void *info); enum { BACK_INFO_DBENV, /* Get the dbenv */ + BACK_INFO_DBENV_CLDB, /* Get the changelog */ + BACK_INFO_DBENV_CLDB_REMOVE, /* Remove the changelog */ + BACK_INFO_DBENV_CLDB_RESET, /* Recreate the changelog */ + BACK_INFO_DBENV_CLDB_UPGRADE, /* Move an old cl file to the instance database */ + BACK_INFO_CLDB_SET_CONFIG, /* Set the CL configuration for a backend database */ + BACK_INFO_CLDB_GET_CONFIG, /* Get the CL configuration for a backend database */ BACK_INFO_DB_PAGESIZE, /* Get the db page size */ BACK_INFO_INDEXPAGESIZE, /* Get the index page size */ BACK_INFO_DBENV_OPENFLAGS, /* Get the dbenv openflags */ @@ -7771,7 +7779,8 @@ enum BACK_INFO_CRYPT_DESTROY, /* Ctrl: clcrypt_destroy */ BACK_INFO_CRYPT_ENCRYPT_VALUE, /* Ctrl: clcrypt_encrypt_value */ BACK_INFO_CRYPT_DECRYPT_VALUE, /* Ctrl: clcrypt_decrypt_value */ - BACK_INFO_DIRECTORY, /* Get the directory path */ + BACK_INFO_DIRECTORY, /* Get the db directory path */ + BACK_INFO_INSTANCE_DIR, /* Get the path to an instance */ BACK_INFO_LOG_DIRECTORY, /* Get the txn log directory */ BACK_INFO_INDEX_KEY, /* Get the status of a key in an index */ BACK_INFO_DB_DIRECTORY, /* Get the db directory */ @@ -7809,6 +7818,13 @@ struct _back_info_crypt_value }; typedef struct _back_info_crypt_value back_info_crypt_value; +struct _back_info_config_entry +{ + char *dn; /* input -- part of dn below backend config entry */ + Slapi_Entry *ce; /* output -- requested config entry */ +}; +typedef struct _back_info_config_entry back_info_config_entry; + #define BACK_CRYPT_OUTBUFF_EXTLEN 16 /** diff --git a/ldap/servers/slapd/task.c b/ldap/servers/slapd/task.c index 0c17f90..2616488 100644 --- a/ldap/servers/slapd/task.c +++ b/ldap/servers/slapd/task.c @@ -2514,6 +2514,9 @@ task_fixup_tombstones_add(Slapi_PBlock *pb, task_data->base = base; task_data->task = task; + /* Stash a pointer to our data in the task */ + slapi_task_set_data(task, task_data); + if ((stripcsn = slapi_entry_attr_get_ref(e, TASK_TOMBSTONE_FIXUP_STRIPCSN))) { if (strcasecmp(stripcsn, "yes") == 0 || strcasecmp(stripcsn, "on") == 0) { task_data->stripcsn = 1; diff --git a/ldap/servers/slapd/time.c b/ldap/servers/slapd/time.c index 5455384..9edf901 100644 --- a/ldap/servers/slapd/time.c +++ b/ldap/servers/slapd/time.c @@ -688,8 +688,8 @@ gen_duration(long duration) char *duration_str = NULL; long remainder = 0; long devided = duration; - int devider[] = {60, 60, 24, 0}; - char *unit[] = {"", "M", "H", "D", NULL}; + int devider[] = {60, 60, 24, 7, 0}; + char *unit[] = {"", "m", "h", "d", "w", NULL}; int i = 0; if (0 > duration) { diff --git a/ldap/servers/slapd/tools/dbscan.c b/ldap/servers/slapd/tools/dbscan.c index 32eea07..4e7795b 100644 --- a/ldap/servers/slapd/tools/dbscan.c +++ b/ldap/servers/slapd/tools/dbscan.c @@ -1047,6 +1047,9 @@ is_changelog(char *filename) } else { ptr++; } + + if (0 == strcmp(ptr,"changelog.db")) return 1; + for (; ptr && *ptr; ptr++) { if ('.' == *ptr) { if (0 == strncmp(ptr, ".db", 3)) { diff --git a/src/lib389/lib389/__init__.py b/src/lib389/lib389/__init__.py index 99ea9cc..31ce74e 100644 --- a/src/lib389/lib389/__init__.py +++ b/src/lib389/lib389/__init__.py @@ -182,6 +182,8 @@ def wrapper(f, name): def pid_exists(pid): + if not pid: + return False if pid <= 0: return False try: diff --git a/src/lib389/lib389/agreement.py b/src/lib389/lib389/agreement.py index efcd507..ecbe25b 100644 --- a/src/lib389/lib389/agreement.py +++ b/src/lib389/lib389/agreement.py @@ -446,6 +446,13 @@ class Agreement(DSLdapObject): """ return self.get_attr_val_utf8('nsDS5ReplicaWaitForAsyncResults') + def set_flowcontrolwindow(self, value): + """Set nsds5ReplicaFlowControlWindow to value. + + :param value: During total update Number of entries to send without waiting ack + :type value: str + """ + self.replace('nsds5ReplicaFlowControlWindow', value) class WinsyncAgreement(Agreement): """A replication agreement from this server instance to diff --git a/src/lib389/lib389/config.py b/src/lib389/lib389/config.py index aa4c92b..33dd101 100644 --- a/src/lib389/lib389/config.py +++ b/src/lib389/lib389/config.py @@ -473,8 +473,12 @@ class CertmapLegacy(object): output += "%s:%s %s\n" % (name, v, certmap[v]) # Now write it out certmap = os.path.join(self._instance.get_config_dir(), 'certmap.conf') + if not os.access(certmap, os.W_OK): + os.chmod(certmap, 0o660) with open(certmap, 'w') as f: f.write(output) + if os.access(certmap, os.W_OK): + os.chmod(certmap, 0o440) class LDBMConfig(DSLdapObject): diff --git a/src/lib389/lib389/replica.py b/src/lib389/lib389/replica.py index f575e58..56cccfa 100644 --- a/src/lib389/lib389/replica.py +++ b/src/lib389/lib389/replica.py @@ -1685,8 +1685,8 @@ class Replicas(DSLdapObjects): repl_roots = [] try: - cl = Changelog5(self._instance) - cl_dir = cl.get_attr_val_utf8_l("nsslapd-changelogdir") + # Changelog is now dumped in the ldif directory + cl_dir = self._instance.get_ldif_dir() except ldap.NO_SUCH_OBJECT: raise ValueError("Changelog entry was not found. Probably, the replication is not enabled on this instance") @@ -1878,7 +1878,8 @@ class ReplicationManager(object): # So this can wrap it and make it easy. self._log.debug("Creating first master on %s" % instance.ldapuri) - self._ensure_changelog(instance) + # With changelog now integrated with the main database + # The config cn=changelog5,cn=config entry is no longer needed rgroup_dn = self._create_service_account(instance, instance) @@ -1991,6 +1992,7 @@ class ReplicationManager(object): 'nsDS5ReplicaHost': to_instance.host, 'nsDS5ReplicaPort': str(to_instance.port), 'nsDS5ReplicaCredentials': repl_manager_password, + 'nsds5ReplicaFlowControlWindow': '100', }) # Do a replica refresh. temp_agmt.begin_reinit() @@ -2030,7 +2032,7 @@ class ReplicationManager(object): from_r = from_replicas.get(self._suffix) # Ensure we have a cl - self._ensure_changelog(to_instance) + # self._ensure_changelog(to_instance) # Create our credentials repl_dn = self._create_service_account(from_instance, to_instance) @@ -2101,7 +2103,7 @@ class ReplicationManager(object): from_r = from_replicas.get(self._suffix) # Ensure we have a changelog - self._ensure_changelog(to_instance) + # self._ensure_changelog(to_instance) # Create replica on to_instance, with bootstrap details. to_r = to_replicas.create(properties={ @@ -2417,7 +2419,9 @@ class ReplicationManager(object): if change == desc: self._log.info("SUCCESS: Replication from %s to %s is working" % (from_instance.ldapuri, to_instance.ldapuri)) return True + self._log.info("Retry: Replication from %s to %s is NOT working (expect %s / got description=%s)" % (from_instance.ldapuri, to_instance.ldapuri, change, desc)) time.sleep(1) + self._log.info("FAIL: Replication from %s to %s is NOT working (expect %s / got description=%s)" % (from_instance.ldapuri, to_instance.ldapuri, change, desc)) raise Exception("Replication did not sync in time!") diff --git a/src/lib389/lib389/topologies.py b/src/lib389/lib389/topologies.py index 19e1628..d402d88 100644 --- a/src/lib389/lib389/topologies.py +++ b/src/lib389/lib389/topologies.py @@ -108,6 +108,10 @@ def _create_instances(topo_dict, suffix): if role == ReplicaRole.HUB: hs[instance.serverid] = instance instances.update(hs) + if DEBUGGING: + instance.config.set('nsslapd-accesslog-logbuffering','off') + instance.config.set('nsslapd-errorlog-level','8192') + instance.config.set('nsslapd-auditlog-logging-enabled','on') log.info("Instance with parameters {} was created.".format(args_instance)) if "standalone1" in instances and len(instances) == 1: diff --git a/src/lib389/lib389/utils.py b/src/lib389/lib389/utils.py index 9a9260b..dc1bc5c 100644 --- a/src/lib389/lib389/utils.py +++ b/src/lib389/lib389/utils.py @@ -1108,6 +1108,11 @@ def ds_is_newer(*ver, instance=None): """ return ds_is_related('newer', *ver, instance=instance) +def ds_supports_new_changelog(): + """ + Return True if the current version of ns-slapd supports changelogs under cn=changelog,cn=,cn=ldbm.. + """ + return ds_is_newer('1.4.4.3') def gentime_to_datetime(gentime): """Convert Generalized time to datetime object diff --git a/test/libslapd/pblock/pblock_accessors.txt b/test/libslapd/pblock/pblock_accessors.txt index 93e4864..facb3cb 100644 --- a/test/libslapd/pblock/pblock_accessors.txt +++ b/test/libslapd/pblock/pblock_accessors.txt @@ -109,13 +109,11 @@ SLAPI_PLUGIN_ACL_SYNTAX_CHECK SLAPI_PLUGIN_ARGC SLAPI_PLUGIN_ARGV SLAPI_PLUGIN_BE_POST_ADD_FN -SLAPI_PLUGIN_BE_POST_BACKUP_FN SLAPI_PLUGIN_BE_POST_DELETE_FN SLAPI_PLUGIN_BE_POST_MODIFY_FN SLAPI_PLUGIN_BE_POST_MODRDN_FN SLAPI_PLUGIN_BE_POST_OPEN_FN SLAPI_PLUGIN_BE_PRE_ADD_FN -SLAPI_PLUGIN_BE_PRE_BACKUP_FN SLAPI_PLUGIN_BE_PRE_CLOSE_FN SLAPI_PLUGIN_BE_PRE_DELETE_FN SLAPI_PLUGIN_BE_PRE_MODIFY_FN diff --git a/test/libslapd/pblock/pblock_accessors_freq.txt b/test/libslapd/pblock/pblock_accessors_freq.txt index e332d9b..6b405c2 100644 --- a/test/libslapd/pblock/pblock_accessors_freq.txt +++ b/test/libslapd/pblock/pblock_accessors_freq.txt @@ -220,8 +220,6 @@ SLAPI_PLUGIN_ARGV 21 SLAPI_PLUGIN_BE_POST_ADD_FN 12 -SLAPI_PLUGIN_BE_POST_BACKUP_FN -12 SLAPI_PLUGIN_BE_POST_DELETE_FN 13 SLAPI_PLUGIN_BE_POST_MODIFY_FN @@ -232,8 +230,6 @@ SLAPI_PLUGIN_BE_POST_OPEN_FN 12 SLAPI_PLUGIN_BE_PRE_ADD_FN 14 -SLAPI_PLUGIN_BE_PRE_BACKUP_FN -11 SLAPI_PLUGIN_BE_PRE_CLOSE_FN 11 SLAPI_PLUGIN_BE_PRE_DELETE_FN