From 3606b78bacce984ab2226755c5921dffac9552c2 Mon Sep 17 00:00:00 2001 From: Noriko Hosoi Date: Jun 09 2016 00:00:43 +0000 Subject: Ticket #48755 - moving an entry could make the online init fail Bug Description: Online init (aka Total update, bulk import) scans the primary id2entry db in the order of ID. If Entry A is moved under a new superior Entry B which was generated after Entry A, when Entry A is sent to a consumer using online init, its parent entry does not exist on the consumer and the online init fails. Fix Description: - Added a command BACK_INFO_IS_ENTRYRDN to slapi_back_get_info, which returns the status of entryrdn switch maintained in the backend. - If slapi_backend_get_info(BACK_INFO_IS_ENTRYRDN) returns true for the replicated backend, repl5_tot_run searches the entry with the filter: (|(parentid>=1)(objectclass=ldapsubentry)(objectclass=nstombstone)) instead of: (|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))". - In addition, idl_new_range_fetch had to be modified so that ... * A range search for parentid ignores nsslapd-idlistscanlimit by setting SLAPI_OP_RANGE_NO_ALLIDS as well as it skips sorting the IDlist by ID by setting SLAPI_OP_RANGE_NO_IDL_SORT. * In case SLAPI_OP_RANGE_NO_IDL_SORT is set, idl_new_range_fetch checks whether the key (in this case parentid) is in the IDlist. If it exists, the ID is appended. If it does not, the ID is in the leftover list and appended when the parent ID is found in the IDlist. - Increased the version of rdn-format-# in DBVERSION to 3. - Upgrade script 91reindex.pl.in is added which reindex the parentid index file in the integer order if the version of rdn-format-# in DBVERSION is less than 3. https://fedorahosted.org/389/ticket/48755 Reviewed by wibrown@redhat.com and lkrispen@redhat.com (Thanks, William and Ludwig!) --- diff --git a/Makefile.am b/Makefile.am index 824b745..cad1b61 100644 --- a/Makefile.am +++ b/Makefile.am @@ -839,7 +839,8 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \ ldap/admin/src/scripts/50AES-pbe-plugin.ldif\ ldap/admin/src/scripts/50updateconfig.ldif \ ldap/admin/src/scripts/52updateAESplugin.pl \ - ldap/admin/src/scripts/dnaplugindepends.ldif + ldap/admin/src/scripts/dnaplugindepends.ldif \ + ldap/admin/src/scripts/91reindex.pl update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh diff --git a/Makefile.in b/Makefile.in index 0752ff1..a45a0d6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -2240,7 +2240,8 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \ ldap/admin/src/scripts/50AES-pbe-plugin.ldif\ ldap/admin/src/scripts/50updateconfig.ldif \ ldap/admin/src/scripts/52updateAESplugin.pl \ - ldap/admin/src/scripts/dnaplugindepends.ldif + ldap/admin/src/scripts/dnaplugindepends.ldif \ + ldap/admin/src/scripts/91reindex.pl update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh diff --git a/ldap/admin/src/scripts/91reindex.pl.in b/ldap/admin/src/scripts/91reindex.pl.in new file mode 100644 index 0000000..c861f64 --- /dev/null +++ b/ldap/admin/src/scripts/91reindex.pl.in @@ -0,0 +1,103 @@ +use Mozilla::LDAP::Conn; +use Mozilla::LDAP::Utils qw(normalizeDN); +use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn); +use DSUpdate qw(isOffline); + +sub runinst { + my ($inf, $inst, $dseldif, $conn) = @_; + my $rc, @errs; + + # List of index to be reindexed + my @toreindex = qw(parentid); + # rdn-format value. See $rdn_format set below. + # If equal to or greater than this value, no need to reindex. + # If it needs to be unconditionally reindexed, set 0. + my @rdnconditions = (4) + + my $config = $conn->search("cn=config", "base", "(objectclass=*)"); + if (!$config) { + push @errs, ['error_finding_config_entry', 'cn=config', + $conn->getErrorString()]; + return @errs; + } + + ($rc, @errs) = isOffline($inf, $inst, $conn); + if (!$rc) { + return @errs; + } + + my $reindex = "@sbindir@/db2index -Z $inst"; + my @errs; + my $instconf = $conn->search("cn=ldbm database,cn=plugins,cn=config", "onelevel", "(objectclass=*)"); + if (!$instconf) { + push @errs, ['error_finding_config_entry', 'cn=*,cn=ldbm database,cn=plugins,cn=config', $conn->getErrorString()]; + return @errs; + } + + my $dbconf = $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config", "base", "(objectclass=*)"); + if (!$dbconf) { + push @errs, ['error_finding_config_entry', + 'cn=config,cn=ldbm database,cn=plugins,cn=config', + $conn->getErrorString()]; + return @errs; + } + + # Get the value of nsslapd-subtree-rename-switch. + my $switch = $dbconf->getValues('nsslapd-subtree-rename-switch'); + if ("" eq $switch) { + return (); # subtree-rename-switch does not exist; do nothing. + } elsif ("off" eq $switch || "OFF" eq $switch) { + return (); # subtree-rename-switch is OFF; do nothing. + } + + my $dbdir = $dbconf->getValues('nsslapd-directory'); + my $dbversion0 = $dbdir . "/DBVERSION"; + my $rdn_format = 0; + my $dbversionstr = ""; + if (!open(DBVERSION, "$dbversion0")) { + push @errs, ['error_opening_file', $dbversion0, $!]; + return @errs; + } else { + while () { + if ($_ =~ /rdn-format/) { + $rdn_format = 1; + $dbversionstr = $_; + if ($_ =~ /rdn-format-1/) { + $rdn_format = 2; + } elsif ($_ =~ /rdn-format-2/) { + $rdn_format = 3; + } elsif ($_ =~ /rdn-format-3/) { + $rdn_format = 4; + } elsif ($_ =~ /rdn-format-4/) { + $rdn_format = 5; + } elsif ($_ =~ /rdn-format-5/) { + $rdn_format = 6; + } elsif ($_ =~ /rdn-format-/) { + # assume greater than -5 + $rdn_format = 7; + } + } + } + close DBVERSION; + } + + while ($instconf) { + my $backend= $instconf->getValues('cn'); + if (($backend eq "config") || ($backend eq "monitor")) { + goto NEXT; + } + + for (my $idx = 0; $ <= $#toreindex; $idx++) { + if (0 == $rdnconditions[$idx] || $rdnconditions[$idx] > $rdn_format) { + my $rc = system("$reindex -n $backend -t $idx"); + if ($rc) { + push @errs, ["error_reindexng", $idx, $backend, $rc]; + } + } + } +NEXT: + $instconf = $conn->nextEntry(); + } + + return @errs; +} diff --git a/ldap/admin/src/scripts/91subtreereindex.pl b/ldap/admin/src/scripts/91subtreereindex.pl index a031cc1..c4b40a3 100644 --- a/ldap/admin/src/scripts/91subtreereindex.pl +++ b/ldap/admin/src/scripts/91subtreereindex.pl @@ -51,14 +51,18 @@ sub runinst { if ($_ =~ /rdn-format-1/) { $is_rdn_format = 2; } - if ($_ =~ /rdn-format-2/) { + elsif ($_ =~ /rdn-format-2/) { $is_rdn_format = 3; } + elsif ($_ =~ /rdn-format-/) { + # assume greater than -2 + $is_rdn_format = 4; + } } } close DBVERSION; - if (3 == $is_rdn_format) { + if (3 <= $is_rdn_format) { return (); # DB already has the new rdn format. } diff --git a/ldap/admin/src/scripts/setup-ds.res.in b/ldap/admin/src/scripts/setup-ds.res.in index fa37567..e46b858 100644 --- a/ldap/admin/src/scripts/setup-ds.res.in +++ b/ldap/admin/src/scripts/setup-ds.res.in @@ -209,3 +209,4 @@ error_opening_file = Opening file '%s' failed. Error: %s\n error_format_error = '%s' has invalid format.\n error_update_not_offline = Error: offline mode selected but the server [%s] is still running.\n error_update_all = Failed to update all the Directory Server instances.\n +error_reindexing = Failed to reindex '%s' in backend '%s'. Error: %s\n diff --git a/ldap/ldif/template-dse.ldif.in b/ldap/ldif/template-dse.ldif.in index f18c082..46b416b 100644 --- a/ldap/ldif/template-dse.ldif.in +++ b/ldap/ldif/template-dse.ldif.in @@ -930,6 +930,7 @@ objectclass: nsIndex cn: parentid nssystemindex: true nsindextype: eq +nsmatchingrule: integerOrderingMatch dn: cn=seeAlso,cn=default indexes, cn=config,cn=ldbm database,cn=plugins,cn=config objectclass: top diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c index 14020b7..3330a62 100644 --- a/ldap/servers/plugins/replication/repl5_replica_config.c +++ b/ldap/servers/plugins/replication/repl5_replica_config.c @@ -405,7 +405,7 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* { if (apply_mods) replica_set_precise_purging(r, 0); - } + } else { *returncode = LDAP_UNWILLING_TO_PERFORM; @@ -567,8 +567,7 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* { if (apply_mods) { - if (apply_mods && config_attr_value[0]) - { + if (config_attr_value[0]) { PRUint64 on_off = 0; if (strcasecmp(config_attr_value, "on") == 0){ @@ -587,7 +586,7 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* break; } replica_set_precise_purging(r, on_off); - } else if (apply_mods) { + } else { replica_set_precise_purging(r, 0); } } diff --git a/ldap/servers/plugins/replication/repl5_tot_protocol.c b/ldap/servers/plugins/replication/repl5_tot_protocol.c index d0c4402..03d0c3e 100644 --- a/ldap/servers/plugins/replication/repl5_tot_protocol.c +++ b/ldap/servers/plugins/replication/repl5_tot_protocol.c @@ -323,6 +323,10 @@ repl5_tot_run(Private_Repl_Protocol *prp) int init_retry = 0; Replica *replica; ReplicaId rid = 0; /* Used to create the replica keep alive subentry */ + Slapi_Entry *suffix = NULL; + char **instances = NULL; + Slapi_Backend *be = NULL; + int is_entryrdn = 0; PR_ASSERT(NULL != prp); @@ -354,21 +358,21 @@ retry: */ if (rc != ACQUIRE_SUCCESS) { - int optype, ldaprc, wait_retry; - conn_get_error(prp->conn, &optype, &ldaprc); - if (rc == ACQUIRE_TRANSIENT_ERROR && INIT_RETRY_MAX > init_retry++) { - wait_retry = init_retry * INIT_RETRY_INT; - slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to " - "acquire replica for total update, error: %d," + int optype, ldaprc, wait_retry; + conn_get_error(prp->conn, &optype, &ldaprc); + if (rc == ACQUIRE_TRANSIENT_ERROR && INIT_RETRY_MAX > init_retry++) { + wait_retry = init_retry * INIT_RETRY_INT; + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to " + "acquire replica for total update, error: %d," " retrying in %d seconds.\n", - ldaprc, wait_retry); - DS_Sleep(PR_SecondsToInterval(wait_retry)); - goto retry; - } else { - agmt_set_last_init_status(prp->agmt, ldaprc, - prp->last_acquire_response_code, 0, NULL); - goto done; - } + ldaprc, wait_retry); + DS_Sleep(PR_SecondsToInterval(wait_retry)); + goto retry; + } else { + agmt_set_last_init_status(prp->agmt, ldaprc, + prp->last_acquire_response_code, 0, NULL); + goto done; + } } else if (prp->terminate) { @@ -405,48 +409,121 @@ retry: and that the order implies that perent entry is always ahead of the child entry in the list. Otherwise, the consumer would not be properly updated because bulk import at the moment skips orphand entries. */ - /* XXXggood above assumption may not be valid if orphaned entry moved???? */ + /* XXXggood above assumption may not be valid if orphaned entry moved???? */ agmt_set_last_init_status(prp->agmt, 0, 0, 0, "Total update in progress"); slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Beginning total update of replica " - "\"%s\".\n", agmt_get_long_name(prp->agmt)); + "\"%s\".\n", agmt_get_long_name(prp->agmt)); /* RMREPL - need to send schema here */ pb = slapi_pblock_new (); - /* we need to provide managedsait control so that referral entries can - be replicated */ - ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *)); - ctrls[0] = create_managedsait_control (); - ctrls[1] = create_backend_control(area_sdn); + replica = (Replica*) object_get_data(prp->replica_object); + /* + * Get the info about the entryrdn vs. entrydn from the backend. + * If NOT is_entryrdn, its ancestor entries are always found prior to an entry. + */ + rc = slapi_lookup_instance_name_by_suffix((char *)slapi_sdn_get_dn(area_sdn), NULL, &instances, 1); + if (rc || !instances) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to " + "get the instance name for the suffix \"%s\".\n", slapi_sdn_get_dn(area_sdn)); + goto done; + } + be = slapi_be_select_by_instance_name(instances[0]); + if (!be) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to " + "get the instance for the suffix \"%s\".\n", slapi_sdn_get_dn(area_sdn)); + goto done; + } + rc = slapi_back_get_info(be, BACK_INFO_IS_ENTRYRDN, (void **)&is_entryrdn); + if (is_entryrdn) { + /* + * Supporting entries out of order -- parent could have a larger id than its children. + * Entires are retireved sorted by parentid without the allid threshold. + */ + /* Get suffix */ + rc = slapi_search_internal_get_entry(area_sdn, NULL, &suffix, repl_get_plugin_identity(PLUGIN_MULTIMASTER_REPLICATION)); + if (rc) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to " + "get the suffix entry \"%s\".\n", slapi_sdn_get_dn(area_sdn)); + goto done; + } - /* Time to make sure it exists a keep alive subentry for that replica */ - replica = (Replica*) object_get_data(prp->replica_object); - if (replica) - { - rid = replica_get_rid(replica); - } - replica_subentry_check(area_sdn, rid); - - slapi_search_internal_set_pb (pb, slapi_sdn_get_dn (area_sdn), - LDAP_SCOPE_SUBTREE, "(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))", NULL, 0, ctrls, NULL, - repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), 0); - - cb_data.prp = prp; - cb_data.rc = 0; - cb_data.num_entries = 0UL; - cb_data.sleep_on_busy = 0UL; - cb_data.last_busy = current_time (); - cb_data.flowcontrol_detection = 0; - cb_data.lock = PR_NewLock(); - - /* This allows during perform_operation to check the callback data - * especially to do flow contol on delta send msgid / recv msgid - */ - conn_set_tot_update_cb(prp->conn, (void *) &cb_data); + cb_data.prp = prp; + cb_data.rc = 0; + cb_data.num_entries = 1UL; + cb_data.sleep_on_busy = 0UL; + cb_data.last_busy = current_time (); + cb_data.flowcontrol_detection = 0; + cb_data.lock = PR_NewLock(); + + /* This allows during perform_operation to check the callback data + * especially to do flow contol on delta send msgid / recv msgid + */ + conn_set_tot_update_cb(prp->conn, (void *) &cb_data); + + /* Send suffix first. */ + rc = send_entry(suffix, (void *)&cb_data); + if (rc) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to " + "send the suffix entry \"%s\" to the consumer.\n", slapi_sdn_get_dn(area_sdn)); + goto done; + } + + /* we need to provide managedsait control so that referral entries can + be replicated */ + ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *)); + ctrls[0] = create_managedsait_control (); + ctrls[1] = create_backend_control(area_sdn); + + /* Time to make sure it exists a keep alive subentry for that replica */ + if (replica) + { + rid = replica_get_rid(replica); + } + replica_subentry_check(area_sdn, rid); + /* Send the subtree of the suffix in the order of parentid index plus ldapsubentry and nstombstone. */ + slapi_search_internal_set_pb(pb, slapi_sdn_get_dn (area_sdn), + LDAP_SCOPE_SUBTREE, "(|(parentid>=1)(objectclass=ldapsubentry)(objectclass=nstombstone))", NULL, 0, ctrls, NULL, + repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), 0); + cb_data.num_entries = 0UL; + } else { + /* Original total update */ + /* we need to provide managedsait control so that referral entries can + be replicated */ + ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *)); + ctrls[0] = create_managedsait_control (); + ctrls[1] = create_backend_control(area_sdn); + + /* Time to make sure it exists a keep alive subentry for that replica */ + replica = (Replica*) object_get_data(prp->replica_object); + if (replica) + { + rid = replica_get_rid(replica); + } + replica_subentry_check(area_sdn, rid); + + slapi_search_internal_set_pb (pb, slapi_sdn_get_dn (area_sdn), + LDAP_SCOPE_SUBTREE, "(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))", NULL, 0, ctrls, NULL, + repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), 0); + + cb_data.prp = prp; + cb_data.rc = 0; + cb_data.num_entries = 0UL; + cb_data.sleep_on_busy = 0UL; + cb_data.last_busy = current_time (); + cb_data.flowcontrol_detection = 0; + cb_data.lock = PR_NewLock(); + + /* This allows during perform_operation to check the callback data + * especially to do flow contol on delta send msgid / recv msgid + */ + conn_set_tot_update_cb(prp->conn, (void *) &cb_data); + } + /* Before we get started on sending entries to the replica, we need to * setup things for async propagation: * 1. Create a thread that will read the LDAP results from the connection. @@ -470,7 +547,7 @@ retry: slapi_search_internal_callback_pb (pb, &cb_data /* callback data */, get_result /* result callback */, send_entry /* entry callback */, - NULL /* referral callback*/); + NULL /* referral callback*/); /* * After completing the sending operation (or optionally failing), we need to clean up diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h index 9499292..2d77a8a 100644 --- a/ldap/servers/slapd/back-ldbm/back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h @@ -132,7 +132,7 @@ typedef unsigned short u_int16_t; #define BDB_BACKEND "libback-ldbm" /* This backend plugin */ #define BDB_NEWIDL "newidl" /* new idl format */ #define BDB_RDNFORMAT "rdn-format" /* Subtree rename enabled */ -#define BDB_RDNFORMAT_VERSION "2" /* rdn-format version (by default, 0) */ +#define BDB_RDNFORMAT_VERSION "3" /* rdn-format version (by default, 0) */ #define BDB_DNFORMAT "dn-4514" /* DN format RFC 4514 compliant */ #define BDB_DNFORMAT_VERSION "1" /* DN format version */ @@ -808,11 +808,11 @@ typedef struct _back_search_result_set /* #define LDBM_ENTRYRDN_OID "2.16.840.1.113730.3.1.2097" */ #define LDBM_ANCESTORID_STR "ancestorid" -#define LDBM_ENTRYDN_STR "entrydn" +#define LDBM_ENTRYDN_STR SLAPI_ATTR_ENTRYDN #define LDBM_ENTRYRDN_STR "entryrdn" #define LDBM_NUMSUBORDINATES_STR "numsubordinates" #define LDBM_TOMBSTONE_NUMSUBORDINATES_STR "tombstonenumsubordinates" -#define LDBM_PARENTID_STR "parentid" +#define LDBM_PARENTID_STR SLAPI_ATTR_PARENTID /* Name of psuedo attribute used to track default indexes */ #define LDBM_PSEUDO_ATTR_DEFAULT ".default" diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index bc290cb..93d42be 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -7315,6 +7315,11 @@ ldbm_back_get_info(Slapi_Backend *be, int cmd, void **info) } break; } + case BACK_INFO_IS_ENTRYRDN: + { + *(int *)info = entryrdn_get_switch(); + break; + } default: break; } diff --git a/ldap/servers/slapd/back-ldbm/filterindex.c b/ldap/servers/slapd/back-ldbm/filterindex.c index 9c14de4..9a7e7be 100644 --- a/ldap/servers/slapd/back-ldbm/filterindex.c +++ b/ldap/servers/slapd/back-ldbm/filterindex.c @@ -552,6 +552,7 @@ range_candidates( struct berval *low = NULL, *high = NULL; struct berval **lows = NULL, **highs = NULL; back_txn txn = {NULL}; + int operator = 0; LDAPDebug(LDAP_DEBUG_TRACE, "=> range_candidates attr=%s\n", type, 0, 0); @@ -578,18 +579,21 @@ range_candidates( } high = attr_value_lowest(highs, slapi_berval_cmp); } - + if (entryrdn_get_switch() && !strcasecmp(type, LDBM_PARENTID_STR)) { + /* parentid is treated specially that is needed for the bulk import. (See #48755) */ + operator = SLAPI_OP_RANGE_NO_IDL_SORT|SLAPI_OP_RANGE_NO_ALLIDS; + } if (low == NULL) { - idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, - SLAPI_OP_LESS_OR_EQUAL, + operator |= SLAPI_OP_LESS_OR_EQUAL; + idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator, high, NULL, 0, &txn, err, allidslimit); } else if (high == NULL) { - idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, - SLAPI_OP_GREATER_OR_EQUAL, + operator |= SLAPI_OP_GREATER_OR_EQUAL; + idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator, low, NULL, 0, &txn, err, allidslimit); } else { - idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, - SLAPI_OP_LESS_OR_EQUAL, + operator |= SLAPI_OP_LESS_OR_EQUAL; + idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator, low, high, 1, &txn, err, allidslimit); } diff --git a/ldap/servers/slapd/back-ldbm/idl_new.c b/ldap/servers/slapd/back-ldbm/idl_new.c index 25b3bfa..6ca6c96 100644 --- a/ldap/servers/slapd/back-ldbm/idl_new.c +++ b/ldap/servers/slapd/back-ldbm/idl_new.c @@ -350,17 +350,31 @@ error: return idl; } +typedef struct _range_id_pair { + ID key; + ID id; +} idl_range_id_pair; /* * Perform the range search in the idl layer instead of the index layer * to improve the performance. */ +/* + * NOTE: + * In the total update (bulk import), an entry requires its ancestors already added. + * To guarantee it, the range search with parentid is used with setting the flag + * SLAPI_OP_RANGE_NO_IDL_SORT in operator. + * + * If the flag is set, + * 1. the IDList is not sorted by the ID. + * 2. holding to add an ID to the IDList unless the key is found in the IDList. + */ IDList * idl_new_range_fetch( - backend *be, - DB* db, - DBT *lowerkey, + backend *be, + DB* db, + DBT *lowerkey, DBT *upperkey, - DB_TXN *txn, + DB_TXN *txn, struct attrinfo *ai, int *flag_err, int allidslimit, @@ -380,7 +394,7 @@ idl_new_range_fetch( size_t count = 0; #ifdef DB_USE_BULK_FETCH /* beware that a large buffer on the stack might cause a stack overflow on some platforms */ - char buffer[BULK_FETCH_BUFFER_SIZE]; + char buffer[BULK_FETCH_BUFFER_SIZE]; void *ptr; DBT dataret; #endif @@ -388,15 +402,21 @@ idl_new_range_fetch( struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; time_t curtime; void *saved_key = NULL; + int coreop = operator & SLAPI_OP_RANGE; + ID key; + ID suffix; + idl_range_id_pair *leftover = NULL; + size_t leftoverlen = 32; + int leftovercnt = 0; if (NULL == flag_err) { return NULL; } - *flag_err = 0; if (NEW_IDL_NOOP == *flag_err) { return NULL; } + dblayer_txn_init(li, &s_txn); if (txn) { dblayer_read_txn_begin(be, txn, &s_txn); @@ -460,7 +480,7 @@ idl_new_range_fetch( #ifdef DB_USE_BULK_FETCH while (cur_key.data && (upperkey && upperkey->data ? - ((operator == SLAPI_OP_LESS) ? + ((coreop == SLAPI_OP_LESS) ? DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) < 0 : DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) <= 0) : PR_TRUE /* e.g., (x > a) */)) { @@ -496,6 +516,9 @@ idl_new_range_fetch( goto error; } } + if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) { + key = (ID)strtol((char *)cur_key.data+1 , (char **)NULL, 10); + } while (PR_TRUE) { DB_MULTIPLE_NEXT(ptr, &data, dataret.data, dataret.size); if (dataret.data == NULL) break; @@ -524,7 +547,29 @@ idl_new_range_fetch( /* note the last id read to check for dups */ lastid = id; /* we got another ID, add it to our IDL */ - idl_rc = idl_append_extend(&idl, id); + if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) { + if (!idl) { + /* First time. Keep the suffix ID. */ + suffix = key; + idl_rc = idl_append_extend(&idl, id); + } else if ((key == suffix) || idl_id_is_in_idlist(idl, key)) { + /* the parent is the suffix or already in idl. */ + idl_rc = idl_append_extend(&idl, id); + } else { + /* Otherwise, keep the {key,id} in leftover array */ + if (!leftover) { + leftover = (idl_range_id_pair *)slapi_ch_calloc(leftoverlen, sizeof(idl_range_id_pair)); + } else if (leftovercnt == leftoverlen) { + leftover = (idl_range_id_pair *)slapi_ch_realloc((char *)leftover, 2 * leftoverlen * sizeof(idl_range_id_pair)); + memset(leftover + leftovercnt, 0, leftoverlen); + leftoverlen *= 2; + } + leftover[leftovercnt].key = key; + leftover[leftovercnt++].id = id; + } + } else { + idl_rc = idl_append_extend(&idl, id); + } if (idl_rc) { LDAPDebug1Arg(LDAP_DEBUG_ANY, "unable to extend id list (err=%d)\n", idl_rc); @@ -581,7 +626,7 @@ idl_new_range_fetch( } #else while (upperkey && upperkey->data ? - ((operator == SLAPI_OP_LESS) ? + ((coreop == SLAPI_OP_LESS) ? DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) < 0 : DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) <= 0) : PR_TRUE /* e.g., (x > a) */) { @@ -698,9 +743,27 @@ error: *flag_err = ret; /* sort idl */ - if (idl && !ALLIDS(idl)) { - qsort((void *)&idl->b_ids[0], idl->b_nids, - (size_t)sizeof(ID), idl_sort_cmp); + if (idl && !ALLIDS(idl) && !(operator & SLAPI_OP_RANGE_NO_IDL_SORT)) { + qsort((void *)&idl->b_ids[0], idl->b_nids, (size_t)sizeof(ID), idl_sort_cmp); + } + if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) { + int i; + int left = leftovercnt; + while (left) { + for (i = 0; i < leftovercnt; i++) { + if (leftover[i].key && idl_id_is_in_idlist(idl, leftover[i].key)) { + idl_rc = idl_append_extend(&idl, leftover[i].id); + if (idl_rc) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, "unable to extend id list (err=%d)\n", idl_rc); + idl_free(&idl); + return NULL; + } + leftover[i].key = 0; + left--; + } + } + } + slapi_ch_free((void **)&leftover); } return idl; } diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c index 00e78a7..81d6621 100644 --- a/ldap/servers/slapd/back-ldbm/index.c +++ b/ldap/servers/slapd/back-ldbm/index.c @@ -1232,6 +1232,7 @@ index_range_read_ext( int timelimit = -1; back_search_result_set *sr = NULL; int isroot = 0; + int coreop = operator & SLAPI_OP_RANGE; if (!pb) { LDAPDebug(LDAP_DEBUG_ANY, "index_range_read: NULL pblock\n", @@ -1278,7 +1279,7 @@ index_range_read_ext( LDAPDebug1Arg(LDAP_DEBUG_TRACE, "index_range_read lookthrough_limit=%d\n", lookthrough_limit); - switch( operator ) { + switch( coreop ) { case SLAPI_OP_LESS: case SLAPI_OP_LESS_OR_EQUAL: case SLAPI_OP_GREATER_OR_EQUAL: @@ -1287,7 +1288,7 @@ index_range_read_ext( default: LDAPDebug( LDAP_DEBUG_ANY, "<= index_range_read(%s,%s) NULL (operator %i)\n", - type, prefix, operator ); + type, prefix, coreop ); index_free_prefix(prefix); return( NULL ); } @@ -1343,7 +1344,7 @@ index_range_read_ext( if (range != 1) { /* open range search */ char *tmpbuf = NULL; /* this is a search with only one boundary value */ - switch( operator ) { + switch( coreop ) { case SLAPI_OP_LESS: case SLAPI_OP_LESS_OR_EQUAL: lowerkey.dptr = slapi_ch_strdup(prefix); @@ -1451,8 +1452,17 @@ index_range_read_ext( cur_key.data = lowerkey.data; cur_key.size = lowerkey.size; lowerkey.data = NULL; /* Don't need this any more, since the memory will be freed from cur_key */ - if (operator == SLAPI_OP_GREATER) { - *err = index_range_next_key(db,&cur_key,db_txn); + *err = 0; + if (coreop == SLAPI_OP_GREATER) { + *err = index_range_next_key(db, &cur_key, db_txn); + if (*err) { + LDAPDebug(LDAP_DEBUG_ANY, "<= index_range_read(%s,%s) op==GREATER, no next key: %i)\n", + type, prefix, *err ); + goto error; + } + } + if (operator & SLAPI_OP_RANGE_NO_ALLIDS) { + *err = NEW_IDL_NO_ALLID; } if (idl_get_idl_new()) { /* new idl */ idl = idl_new_range_fetch(be, db, &cur_key, &upperkey, db_txn, @@ -1462,7 +1472,7 @@ index_range_read_ext( int retry_count = 0; while (*err == 0 && (upperkey.data && - (operator == SLAPI_OP_LESS) ? + (coreop == SLAPI_OP_LESS) ? DBTcmp(&cur_key, &upperkey, ai->ai_key_cmp_fn) < 0 : DBTcmp(&cur_key, &upperkey, ai->ai_key_cmp_fn) <= 0)) { /* exit the loop when we either run off the end of the table, diff --git a/ldap/servers/slapd/back-ldbm/init.c b/ldap/servers/slapd/back-ldbm/init.c index a531abb..04cc936 100644 --- a/ldap/servers/slapd/back-ldbm/init.c +++ b/ldap/servers/slapd/back-ldbm/init.c @@ -36,7 +36,7 @@ ldbm_back_add_schema( Slapi_PBlock *pb ) SLAPI_ATTR_FLAG_NOUSERMOD ); rc |= slapi_add_internal_attr_syntax( LDBM_PARENTID_STR, - LDBM_PARENTID_OID, DIRSTRING_SYNTAX_OID, CASEIGNOREMATCH_NAME, + LDBM_PARENTID_OID, INTEGER_SYNTAX_OID, INTEGERMATCH_NAME, SLAPI_ATTR_FLAG_SINGLE|SLAPI_ATTR_FLAG_NOUSERMOD ); rc |= slapi_add_internal_attr_syntax( "entryid", diff --git a/ldap/servers/slapd/back-ldbm/misc.c b/ldap/servers/slapd/back-ldbm/misc.c index fe3d01b..77c1e70 100644 --- a/ldap/servers/slapd/back-ldbm/misc.c +++ b/ldap/servers/slapd/back-ldbm/misc.c @@ -79,6 +79,7 @@ static const char *systemIndexes[] = { SLAPI_ATTR_NSCP_ENTRYDN, ATTR_NSDS5_REPLCONFLICT, SLAPI_ATTR_ENTRYUSN, + SLAPI_ATTR_PARENTID, NULL }; diff --git a/ldap/servers/slapd/entry.c b/ldap/servers/slapd/entry.c index 3124ff6..d38f970 100644 --- a/ldap/servers/slapd/entry.c +++ b/ldap/servers/slapd/entry.c @@ -3429,8 +3429,8 @@ slapi_entry_rename(Slapi_Entry *e, const char *newrdn, int deleteoldrdn, Slapi_D /* We remove the parentid and entrydn since the backend will change these. * We don't want to give the caller an inconsistent entry. */ - slapi_entry_attr_delete(e, "parentid"); - slapi_entry_attr_delete(e, "entrydn"); + slapi_entry_attr_delete(e, SLAPI_ATTR_PARENTID); + slapi_entry_attr_delete(e, SLAPI_ATTR_ENTRYDN); /* Build new DN. If newsuperior is set, just use "newrdn,newsuperior". If * newsuperior is not set, need to add newrdn to old superior. */ diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h index a594a29..ea834f1 100644 --- a/ldap/servers/slapd/slap.h +++ b/ldap/servers/slapd/slap.h @@ -536,12 +536,16 @@ typedef int (*SyntaxEnumFunc)(char **names, Slapi_PluginDesc *plugindesc, /* OIDs for some commonly used matching rules */ #define DNMATCH_OID "2.5.13.1" /* distinguishedNameMatch */ #define CASEIGNOREMATCH_OID "2.5.13.2" /* caseIgnoreMatch */ +#define INTEGERMATCH_OID "2.5.13.14" /* integerMatch */ +#define INTEGERORDERINGMATCH_OID "2.5.13.15" /* integerOrderingMatch */ #define INTFIRSTCOMPMATCH_OID "2.5.13.29" /* integerFirstComponentMatch */ #define OIDFIRSTCOMPMATCH_OID "2.5.13.30" /* objectIdentifierFirstComponentMatch */ /* Names for some commonly used matching rules */ #define DNMATCH_NAME "distinguishedNameMatch" #define CASEIGNOREMATCH_NAME "caseIgnoreMatch" +#define INTEGERMATCH_NAME "integerMatch" +#define INTEGERORDERINGMATCH_NAME "integerOrderingMatch" #define INTFIRSTCOMPMATCH_NAME "integerFirstComponentMatch" #define OIDFIRSTCOMPMATCH_NAME "objectIdentifierFirstComponentMatch" diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h index 294f2c3..856432e 100644 --- a/ldap/servers/slapd/slapi-plugin.h +++ b/ldap/servers/slapd/slapi-plugin.h @@ -483,6 +483,7 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...) #define SLAPI_ATTR_ENTRYDN "entrydn" #define SLAPI_ATTR_DN "dn" #define SLAPI_ATTR_RDN "rdn" +#define SLAPI_ATTR_PARENTID "parentid" #define SLAPI_ATTR_UNIQUEID_LENGTH 10 #define SLAPI_ATTR_OBJECTCLASS_LENGTH 11 #define SLAPI_ATTR_VALUE_TOMBSTONE_LENGTH 11 @@ -494,6 +495,7 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...) #define SLAPI_ATTR_ENTRYDN_LENGTH 7 #define SLAPI_ATTR_DN_LENGTH 2 #define SLAPI_ATTR_RDN_LENGTH 3 +#define SLAPI_ATTR_PARENTID_LENGTH 8 /* plugin shared config area */ #define SLAPI_PLUGIN_SHARED_CONFIG_AREA "nsslapd-pluginConfigArea" @@ -7002,6 +7004,9 @@ typedef struct slapi_plugindesc { #define SLAPI_OP_GREATER_OR_EQUAL 4 #define SLAPI_OP_GREATER 5 #define SLAPI_OP_SUBSTRING 6 +#define SLAPI_OP_RANGE 0xff +#define SLAPI_OP_RANGE_NO_IDL_SORT 0x100 +#define SLAPI_OP_RANGE_NO_ALLIDS 0x200 /* Defined values of SLAPI_PLUGIN_MR_USAGE: */ #define SLAPI_PLUGIN_MR_USAGE_INDEX 0 @@ -7585,7 +7590,8 @@ enum BACK_INFO_CRYPT_ENCRYPT_VALUE, /* Ctrl: clcrypt_encrypt_value */ BACK_INFO_CRYPT_DECRYPT_VALUE, /* Ctrl: clcrypt_decrypt_value */ BACK_INFO_DIRECTORY, /* Get the directory path */ - BACK_INFO_LOG_DIRECTORY /* Get the txn log directory */ + BACK_INFO_LOG_DIRECTORY, /* Get the txn log directory */ + BACK_INFO_IS_ENTRYRDN /* Get the flag for entryrdn */ }; struct _back_info_crypt_init {