From d01155c2b511e4cce89ca9e06d29632b774dbd68 Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Aug 22 2013 16:53:48 +0000 Subject: Ticket #197 - BDB backend - clear free page files to reduce changelog size Bug description: Even if entries in the database and changes in the changelog database are deleted/trimmed, the unused pages of the data- bases were not returned to the filesystem. Fix description: This patch calls the compact API that Berkeley DB provides, which compacts the database. 2 config parameters are introduced to specify the interval of the compact calls. Primary DBs (id2entry): dn: cn=config,cn=ldbm database,cn=plugins,cn=config nsslapd-db-compactdb-interval: Changelog DBs: dn: cn=changelog5,cn=config nsslapd-changelogcompactdb-interval: By default, 2592000 seconds (30 days) Reviewed by Rich (Thank you!!) https://fedorahosted.org/389/ticket/197 --- diff --git a/ldap/servers/plugins/replication/cl5.h b/ldap/servers/plugins/replication/cl5.h index 33f8140..9939d5b 100644 --- a/ldap/servers/plugins/replication/cl5.h +++ b/ldap/servers/plugins/replication/cl5.h @@ -56,6 +56,7 @@ typedef struct changelog5Config /* the changelog DB configuration parameters are defined as CL5DBConfig in cl5_api.h */ CL5DBConfig dbconfig; char *symmetricKey; + int compactInterval; }changelog5Config; /* initializes changelog*/ diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c index 0a70d6b..6bf4217 100644 --- a/ldap/servers/plugins/replication/cl5_api.c +++ b/ldap/servers/plugins/replication/cl5_api.c @@ -240,6 +240,7 @@ typedef struct cl5trim { time_t maxAge; /* maximum entry age in seconds */ int maxEntries; /* maximum number of entries across all changelog files */ + int compactInterval; /* interval to compact changelog db */ PRLock* lock; /* controls access to trimming configuration */ } CL5Trim; @@ -350,6 +351,7 @@ static int _cl5TrimInit (); static void _cl5TrimCleanup (); static int _cl5TrimMain (void *param); static void _cl5DoTrimming (ReplicaId rid); +static void _cl5CompactDBs(); static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid); static PRBool _cl5CanTrim (time_t time, long *numToTrim); static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge); @@ -1175,10 +1177,12 @@ int cl5GetState () Description: sets changelog trimming parameters; changelog must be open. Parameters: maxEntries - maximum number of entries in the chnagelog (in all files); maxAge - maximum entry age; + compactInterval - interval to compact changelog db Return: CL5_SUCCESS if successful; CL5_BAD_STATE if changelog is not open */ -int cl5ConfigTrimming (int maxEntries, const char *maxAge) +int +cl5ConfigTrimming (int maxEntries, const char *maxAge, int compactInterval) { if (s_cl5Desc.dbState == CL5_STATE_NONE) { @@ -1216,6 +1220,11 @@ int cl5ConfigTrimming (int maxEntries, const char *maxAge) s_cl5Desc.dbTrim.maxEntries = maxEntries; } + if (compactInterval != CL5_NUM_IGNORE) + { + s_cl5Desc.dbTrim.compactInterval = compactInterval; + } + PR_Unlock (s_cl5Desc.dbTrim.lock); _cl5RemoveThread (); @@ -3420,6 +3429,7 @@ static int _cl5TrimMain (void *param) { PRIntervalTime interval; time_t timePrev = current_time (); + time_t timeCompactPrev = current_time (); time_t timeNow; PR_AtomicIncrement (&s_cl5Desc.threadCount); @@ -3434,6 +3444,13 @@ static int _cl5TrimMain (void *param) timePrev = timeNow; _cl5DoTrimming (0 /* there's no cleaned rid */); } + if ((s_cl5Desc.dbTrim.compactInterval > 0) && + (timeNow - timeCompactPrev >= s_cl5Desc.dbTrim.compactInterval)) + { + /* time to trim */ + timeCompactPrev = timeNow; + _cl5CompactDBs(); + } if (NULL == s_cl5Desc.clLock) { /* most likely, emergency */ @@ -3478,14 +3495,77 @@ static void _cl5DoTrimming (ReplicaId rid) example, randomizing starting point */ obj = objset_first_obj (s_cl5Desc.dbFiles); while (obj && _cl5CanTrim ((time_t)0, &numToTrim)) - { + { _cl5TrimFile (obj, &numToTrim, rid); - obj = objset_next_obj (s_cl5Desc.dbFiles, obj); + obj = objset_next_obj (s_cl5Desc.dbFiles, obj); } - if (obj) - object_release (obj); + if (obj) + object_release (obj); + + PR_Unlock (s_cl5Desc.dbTrim.lock); + + return; +} +/* clear free page files to reduce changelog */ +static void +_cl5CompactDBs() +{ + int rc; + Object *fileObj = NULL; + CL5DBFile *dbFile = NULL; + DB *db = NULL; + DB_TXN *txnid = NULL; + DB_COMPACT c_data = {0}; + + PR_Lock (s_cl5Desc.dbTrim.lock); + rc = TXN_BEGIN(s_cl5Desc.dbEnv, NULL, &txnid, 0); + if (rc) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5CompactDBs: failed to begin transaction; db error - %d %s\n", + rc, db_strerror(rc)); + goto bail; + } + for (fileObj = objset_first_obj(s_cl5Desc.dbFiles); + fileObj; + fileObj = objset_next_obj(s_cl5Desc.dbFiles, fileObj)) { + dbFile = (CL5DBFile*)object_get_data(fileObj); + if (!dbFile) { + continue; + } + db = dbFile->db; + rc = db->compact(db, txnid, NULL/*start*/, NULL/*stop*/, + &c_data, DB_FREE_SPACE, NULL/*end*/); + if (rc) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5CompactDBs: failed to compact %s; db error - %d %s\n", + dbFile->replName, rc, db_strerror(rc)); + goto bail; + } + slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, + "_cl5CompactDBs: %s - %d pages freed\n", + dbFile->replName, c_data.compact_pages_free); + } +bail: + if (fileObj) { + object_release(fileObj); + } + if (rc) { + rc = TXN_ABORT (txnid); + if (rc) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5CompactDBs: failed to abort transaction; db error - %d %s\n", + rc, db_strerror(rc)); + } + } else { + rc = TXN_COMMIT (txnid, 0); + if (rc) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5CompactDBs: failed to commit transaction; db error - %d %s\n", + rc, db_strerror(rc)); + } + } PR_Unlock (s_cl5Desc.dbTrim.lock); return; diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h index 9b285ca..59d3bf2 100644 --- a/ldap/servers/plugins/replication/cl5_api.h +++ b/ldap/servers/plugins/replication/cl5_api.h @@ -269,10 +269,11 @@ int cl5GetState (); Description: sets changelog trimming parameters Parameters: maxEntries - maximum number of entries in the log; maxAge - maximum entry age; + compactInterval - interval to compact changelog db Return: CL5_SUCCESS if successful; CL5_BAD_STATE if changelog has not been open */ -int cl5ConfigTrimming (int maxEntries, const char *maxAge); +int cl5ConfigTrimming (int maxEntries, const char *maxAge, int compactInterval); /* Name: cl5GetOperation Description: retireves operation specified by its csn and databaseid diff --git a/ldap/servers/plugins/replication/cl5_config.c b/ldap/servers/plugins/replication/cl5_config.c index 980cb7f..a523a80 100644 --- a/ldap/servers/plugins/replication/cl5_config.c +++ b/ldap/servers/plugins/replication/cl5_config.c @@ -244,7 +244,7 @@ changelog5_config_add (Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter } /* set trimming parameters */ - rc = cl5ConfigTrimming (config.maxEntries, config.maxAge); + rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval); if (rc != CL5_SUCCESS) { *returncode = 1; @@ -336,6 +336,7 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr slapi_ch_free_string(&config.dir); config.dir = NULL; config.maxEntries = CL5_NUM_IGNORE; + config.compactInterval = CL5_NUM_IGNORE; slapi_ch_free_string(&config.maxAge); config.maxAge = slapi_ch_strdup(CL5_STR_IGNORE); @@ -399,6 +400,17 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr slapi_ch_free_string(&config.maxAge); config.maxAge = slapi_ch_strdup(config_attr_value); } + else if ( strcasecmp ( config_attr, CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE ) == 0 ) + { + if (config_attr_value && config_attr_value[0] != '\0') + { + config.compactInterval = atoi (config_attr_value); + } + else + { + config.compactInterval = 0; + } + } else if ( strcasecmp ( config_attr, CONFIG_CHANGELOG_SYMMETRIC_KEY ) == 0 ) { slapi_ch_free_string(&config.symmetricKey); @@ -424,6 +436,8 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr * except config.dir */ if (config.maxEntries == CL5_NUM_IGNORE) config.maxEntries = originalConfig->maxEntries; + if (config.compactInterval == CL5_NUM_IGNORE) + config.compactInterval = originalConfig->compactInterval; if (strcmp (config.maxAge, CL5_STR_IGNORE) == 0) { slapi_ch_free_string(&config.maxAge); if (originalConfig->maxAge) @@ -548,7 +562,7 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr if (config.maxEntries != CL5_NUM_IGNORE || strcmp (config.maxAge, CL5_STR_IGNORE) != 0) { - rc = cl5ConfigTrimming (config.maxEntries, config.maxAge); + rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval); if (rc != CL5_SUCCESS) { *returncode = 1; @@ -706,6 +720,7 @@ static changelog5Config * changelog5_dup_config(changelog5Config *config) dup->maxAge = slapi_ch_strdup(config->maxAge); dup->maxEntries = config->maxEntries; + dup->compactInterval = config->compactInterval; dup->dbconfig.pageSize = config->dbconfig.pageSize; dup->dbconfig.fileMode = config->dbconfig.fileMode; @@ -732,6 +747,16 @@ static void changelog5_extract_config(Slapi_Entry* entry, changelog5Config *conf config->maxEntries = atoi (arg); slapi_ch_free_string(&arg); } + arg= slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE); + if (arg) + { + config->compactInterval = atoi (arg); + slapi_ch_free_string(&arg); + } + else + { + config->compactInterval = CHANGELOGDB_COMPACT_INTERVAL; + } config->maxAge = slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_MAXAGE_ATTRIBUTE); diff --git a/ldap/servers/plugins/replication/cl5_init.c b/ldap/servers/plugins/replication/cl5_init.c index 8ee725d..60e538d 100644 --- a/ldap/servers/plugins/replication/cl5_init.c +++ b/ldap/servers/plugins/replication/cl5_init.c @@ -88,7 +88,7 @@ int changelog5_init() } /* set trimming parameters */ - rc = cl5ConfigTrimming (config.maxEntries, config.maxAge); + rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval); if (rc != CL5_SUCCESS) { slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, diff --git a/ldap/servers/plugins/replication/repl_shared.h b/ldap/servers/plugins/replication/repl_shared.h index 99f785e..f79a657 100644 --- a/ldap/servers/plugins/replication/repl_shared.h +++ b/ldap/servers/plugins/replication/repl_shared.h @@ -58,10 +58,12 @@ #endif #define CHANGELOGDB_TRIM_INTERVAL 300 /* 5 minutes */ +#define CHANGELOGDB_COMPACT_INTERVAL 2592000 /* 30 days */ #define CONFIG_CHANGELOG_DIR_ATTRIBUTE "nsslapd-changelogdir" #define CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE "nsslapd-changelogmaxentries" #define CONFIG_CHANGELOG_MAXAGE_ATTRIBUTE "nsslapd-changelogmaxage" +#define CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE "nsslapd-changelogcompactdb-interval" /* Changelog Internal Configuration Parameters -> Changelog Cache related */ #define CONFIG_CHANGELOG_MAX_CONCURRENT_WRITES "nsslapd-changelogmaxconcurrentwrites" #define CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM "nsslapd-encryptionalgorithm" diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index 5b981b0..013d1c0 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -4603,10 +4603,8 @@ static int log_flush_threadmain(void *param) PR_NotifyAllCondVar(sync_txn_log_flush_done); } /* wait until flushing conditions are met */ - while ( trans_batch_count == 0 || - ( trans_batch_count < trans_batch_limit && - trans_batch_count < txn_in_progress_count)) - { + while ((trans_batch_count == 0) || + (trans_batch_count < trans_batch_limit && trans_batch_count < txn_in_progress_count)) { if (priv->dblayer_stop_threads) break; if (PR_IntervalNow() - last_flush > interval_flush) { @@ -4617,7 +4615,7 @@ static int log_flush_threadmain(void *param) } PR_Unlock(sync_txn_log_flush); LDAPDebug(LDAP_DEBUG_BACKLDBM, "log_flush_threadmain (wakeup): batchcount: %d, " - "txn_in_progress: %d\n", trans_batch_count, txn_in_progress_count, 0); + "txn_in_progress: %d\n", trans_batch_count, txn_in_progress_count, 0); } else { DS_Sleep(interval_def); } @@ -4654,6 +4652,9 @@ dblayer_start_checkpoint_thread(struct ldbminfo *li) return return_value; } +/* + * checkpoint thread -- borrow the timing for compacting id2entry, as well. + */ static int checkpoint_threadmain(void *param) { time_t time_of_last_checkpoint_completion = 0; /* seconds since epoch */ @@ -4667,6 +4668,9 @@ static int checkpoint_threadmain(void *param) char **list = NULL; char **listp = NULL; struct dblayer_private_env *penv = NULL; + time_t time_of_last_comapctdb_completion = current_time(); /* seconds since epoch */ + int compactdb_interval = 0; + back_txn txn; PR_ASSERT(NULL != param); li = (struct ldbminfo*)param; @@ -4676,6 +4680,7 @@ static int checkpoint_threadmain(void *param) INCR_THREAD_COUNT(priv); + compactdb_interval = priv->dblayer_compactdb_interval; interval = PR_MillisecondsToInterval(DBLAYER_SLEEP_INTERVAL); home_dir = dblayer_get_home_dir(li, NULL); if (NULL == home_dir || '\0' == *home_dir) @@ -4774,6 +4779,49 @@ static int checkpoint_threadmain(void *param) } /* find out which log files don't contain active txns */ DB_CHECKPOINT_LOCK(PR_TRUE, penv->dblayer_env_lock); + /* Compacting DB borrowing the timing of the log flush */ + if ((compactdb_interval > 0) && + (current_time() - time_of_last_comapctdb_completion > compactdb_interval)) { + int rc = 0; + Object *inst_obj; + ldbm_instance *inst; + DB *db = NULL; + DB_COMPACT c_data = {0}; + + for (inst_obj = objset_first_obj(li->li_instance_set); + inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + rc = dblayer_get_id2entry(inst->inst_be, &db); + if (!db) { + continue; + } + LDAPDebug1Arg(LDAP_DEBUG_BACKLDBM, "compactdb: Compacting DB start: %s\n", + inst->inst_name); + rc = dblayer_txn_begin(inst->inst_be, NULL, &txn); + if (rc) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "compactdb: transaction begin failed: %d\n", + rc); + break; + } + rc = db->compact(db, txn.back_txn_txn, NULL/*start*/, NULL/*stop*/, + &c_data, DB_FREE_SPACE, NULL/*end*/); + if (rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "compactdb: failed to compact %s; db error - %d %s\n", + inst->inst_name, rc, db_strerror(rc)); + rc = dblayer_txn_abort(inst->inst_be, &txn); + } else { + LDAPDebug2Args(LDAP_DEBUG_BACKLDBM, + "compactdb: compact %s - %d pages freed\n", + inst->inst_name, c_data.compact_pages_free); + rc = dblayer_txn_commit(inst->inst_be, &txn); + } + } + time_of_last_comapctdb_completion = current_time(); /* seconds since epoch */ + compactdb_interval = priv->dblayer_compactdb_interval; + } rval = LOG_ARCHIVE(penv->dblayer_DB_ENV, &list, DB_ARCH_ABS, (void *)slapi_ch_malloc); DB_CHECKPOINT_UNLOCK(PR_TRUE, penv->dblayer_env_lock); @@ -4794,10 +4842,10 @@ static int checkpoint_threadmain(void *param) checkpoint_debug_message(debug_checkpointing, "Renaming %s -> %s\n",*listp, new_filename, 0); if(rename(*listp, new_filename) != 0){ - LDAPDebug(LDAP_DEBUG_ANY, "checkpoint_threadmain: failed to rename log (%s) to (%s)\n", - *listp, new_filename, 0); - rval = -1; - goto error_return; + LDAPDebug(LDAP_DEBUG_ANY, "checkpoint_threadmain: failed to rename log (%s) to (%s)\n", + *listp, new_filename, 0); + rval = -1; + goto error_return; } } } diff --git a/ldap/servers/slapd/back-ldbm/dblayer.h b/ldap/servers/slapd/back-ldbm/dblayer.h index 7f3200c..c382a98 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.h +++ b/ldap/servers/slapd/back-ldbm/dblayer.h @@ -179,6 +179,7 @@ struct dblayer_private int dblayer_lockdown; /* use DB_LOCKDOWN */ int dblayer_lock_config; u_int32_t dblayer_deadlock_policy; /* i.e. the atype to DB_ENV->lock_detect in deadlock_threadmain */ + int dblayer_compactdb_interval; /* interval to execute compact id2entry dbs */ }; #if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 4300 diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.c b/ldap/servers/slapd/back-ldbm/ldbm_config.c index 4eb53e5..d53e9b3 100644 --- a/ldap/servers/slapd/back-ldbm/ldbm_config.c +++ b/ldap/servers/slapd/back-ldbm/ldbm_config.c @@ -679,6 +679,26 @@ static int ldbm_config_db_checkpoint_interval_set(void *arg, void *value, char * return retval; } +static void *ldbm_config_db_compactdb_interval_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) ((uintptr_t)li->li_dblayer_private->dblayer_compactdb_interval); +} + +static int ldbm_config_db_compactdb_interval_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) ((uintptr_t)value); + + if (apply) { + li->li_dblayer_private->dblayer_compactdb_interval = val; + } + + return retval; +} + static void *ldbm_config_db_page_size_get(void *arg) { struct ldbminfo *li = (struct ldbminfo *) arg; @@ -1447,6 +1467,7 @@ static config_info ldbm_config[] = { {CONFIG_DB_CIRCULAR_LOGGING, CONFIG_TYPE_ONOFF, "on", &ldbm_config_db_circular_logging_get, &ldbm_config_db_circular_logging_set, 0}, {CONFIG_DB_TRANSACTION_LOGGING, CONFIG_TYPE_ONOFF, "on", &ldbm_config_db_transaction_logging_get, &ldbm_config_db_transaction_logging_set, 0}, {CONFIG_DB_CHECKPOINT_INTERVAL, CONFIG_TYPE_INT, "60", &ldbm_config_db_checkpoint_interval_get, &ldbm_config_db_checkpoint_interval_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DB_COMPACTDB_INTERVAL, CONFIG_TYPE_INT, "2592000"/*30days*/, &ldbm_config_db_compactdb_interval_get, &ldbm_config_db_compactdb_interval_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, {CONFIG_DB_TRANSACTION_BATCH, CONFIG_TYPE_INT, "0", &dblayer_get_batch_transactions, &dblayer_set_batch_transactions, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, {CONFIG_DB_TRANSACTION_BATCH_MIN_SLEEP, CONFIG_TYPE_INT, "50", &dblayer_get_batch_txn_min_sleep, &dblayer_set_batch_txn_min_sleep, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, {CONFIG_DB_TRANSACTION_BATCH_MAX_SLEEP, CONFIG_TYPE_INT, "50", &dblayer_get_batch_txn_max_sleep, &dblayer_set_batch_txn_max_sleep, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.h b/ldap/servers/slapd/back-ldbm/ldbm_config.h index f420707..cb9d3c7 100644 --- a/ldap/servers/slapd/back-ldbm/ldbm_config.h +++ b/ldap/servers/slapd/back-ldbm/ldbm_config.h @@ -110,6 +110,7 @@ struct config_info { #define CONFIG_DB_CIRCULAR_LOGGING "nsslapd-db-circular-logging" #define CONFIG_DB_TRANSACTION_LOGGING "nsslapd-db-transaction-logging" #define CONFIG_DB_CHECKPOINT_INTERVAL "nsslapd-db-checkpoint-interval" +#define CONFIG_DB_COMPACTDB_INTERVAL "nsslapd-db-compactdb-interval" #define CONFIG_DB_TRANSACTION_BATCH "nsslapd-db-transaction-batch-val" #define CONFIG_DB_TRANSACTION_BATCH_MIN_SLEEP "nsslapd-db-transaction-batch-min-wait" #define CONFIG_DB_TRANSACTION_BATCH_MAX_SLEEP "nsslapd-db-transaction-batch-max-wait"