From 8f3887e08cb77c0d023a572462832b7e042b01ed Mon Sep 17 00:00:00 2001 From: William Brown Date: Jun 22 2020 04:16:31 +0000 Subject: Ticket 50544 - OpenLDAP syncrepl compatability Bug Description: Some customers have asked for the ability to sync openldap fro 389-ds in a read only mode. OpenLDAP's syncrepl functionality is slightly different to what our module expected, requiring changes to be made. Fix Description: This fixes a number of syncrepl issues within our plugin, works around a number of deviations from OpenLDAP's syncrepl client, adds tests, and the needed schema to allow OpenLDAP to sync from 389-ds. Outstanding issue is that when the EntryUUID plugin is enabled, it can confuse OpenLDAP, so a subsequent PR will address that issue. Note, the provided tests require a fix to python-ldap, so you may not be able to run these tests yet. See: https://github.com/python-ldap/python-ldap/pull/351 https://pagure.io/389-ds-base/issue/50544 Author: William Brown Review by: mreynolds (Thanks!) --- diff --git a/LICENSE.openldap b/LICENSE.openldap new file mode 100644 index 0000000..05ad757 --- /dev/null +++ b/LICENSE.openldap @@ -0,0 +1,47 @@ +The OpenLDAP Public License + Version 2.8, 17 August 2003 + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions in source form must retain copyright statements + and notices, + +2. Redistributions in binary form must reproduce applicable copyright + statements and notices, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution, and + +3. Redistributions must contain a verbatim copy of this document. + +The OpenLDAP Foundation may revise this license from time to time. +Each revision is distinguished by a version number. You may use +this Software under terms of this license revision or under the +terms of any subsequent revision of the license. + +THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS +CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) +OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +The names of the authors and copyright holders must not be used in +advertising or otherwise to promote the sale, use or other dealing +in this Software without specific, written prior permission. Title +to copyright in this Software shall at all times remain with copyright +holders. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, +California, USA. All Rights Reserved. Permission to copy and +distribute verbatim copies of this document is granted. diff --git a/Makefile.am b/Makefile.am index 826efb8..3f21645 100644 --- a/Makefile.am +++ b/Makefile.am @@ -725,6 +725,7 @@ sampledata_DATA = ldap/admin/src/scripts/DSSharedLib \ $(srcdir)/ldap/schema/60rfc4876.ldif \ $(srcdir)/ldap/schema/60samba.ldif \ $(srcdir)/ldap/schema/60sendmail.ldif \ + $(srcdir)/ldap/schema/dsee.schema \ $(LIBPRESENCE_SCHEMA) systemschema_DATA = $(srcdir)/ldap/schema/00core.ldif \ diff --git a/dirsrvtests/tests/suites/syncrepl_plugin/__init__.py b/dirsrvtests/tests/suites/syncrepl_plugin/__init__.py new file mode 100644 index 0000000..d51f52f --- /dev/null +++ b/dirsrvtests/tests/suites/syncrepl_plugin/__init__.py @@ -0,0 +1,163 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2020 William Brown +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +import logging +import ldap +import time +from ldap.syncrepl import SyncreplConsumer +import pytest +from lib389 import DirSrv +from lib389.idm.user import nsUserAccounts, UserAccounts +from lib389.topologies import topology_st as topology +from lib389.paths import Paths +from lib389.utils import ds_is_older +from lib389.plugins import RetroChangelogPlugin, ContentSyncPlugin +from lib389._constants import * + +log = logging.getLogger(__name__) + +class ISyncRepl(DirSrv, SyncreplConsumer): + """ + This implements a test harness for checking syncrepl, and allowing us to check various actions or + behaviours. During a "run" it stores the results in it's instance, so that they can be inspected + later to ensure that syncrepl worked as expected. + """ + def __init__(self, inst, openldap=False): + self.inst = inst + self.msgid = None + + self.last_cookie = None + self.next_cookie = None + self.cookie = None + self.openldap = openldap + if self.openldap: + # In openldap mode, our initial cookie needs to be a rid. + self.cookie = "rid=123" + self.delete = [] + self.present = [] + self.entries = {} + + super().__init__() + + def result4(self, *args, **kwargs): + return self.inst.result4(*args, **kwargs, escapehatch='i am sure') + + def search_ext(self, *args, **kwargs): + return self.inst.search_ext(*args, **kwargs, escapehatch='i am sure') + + def syncrepl_search(self, base=DEFAULT_SUFFIX, scope=ldap.SCOPE_SUBTREE, mode='refreshOnly', cookie=None, **search_args): + # Wipe the last result set. + self.delete = [] + self.present = [] + self.entries = {} + self.next_cookie = None + # Start the sync + # If cookie is none, will call "get_cookie" we have. + self.msgid = super().syncrepl_search(base, scope, mode, cookie, **search_args) + log.debug(f'syncrepl_search -> {self.msgid}') + assert self.msgid is not None + + def syncrepl_complete(self): + log.debug(f'syncrepl_complete -> {self.msgid}') + assert self.msgid is not None + # Loop until the operation is complete. + while super().syncrepl_poll(msgid=self.msgid) is True: + pass + assert self.next_cookie is not None + self.last_cookie = self.cookie + self.cookie = self.next_cookie + + def check_cookie(self): + assert self.last_cookie != self.cookie + + def syncrepl_set_cookie(self, cookie): + log.debug(f'set_cookie -> {cookie}') + if self.openldap: + assert self.cookie.startswith("rid=123") + self.next_cookie = cookie + + def syncrepl_get_cookie(self): + log.debug('get_cookie -> %s' % self.cookie) + if self.openldap: + assert self.cookie.startswith("rid=123") + return self.cookie + + def syncrepl_present(self, uuids, refreshDeletes=False): + log.debug(f'=====> refdel -> {refreshDeletes} uuids -> {uuids}') + if uuids is not None: + self.present = self.present + uuids + + def syncrepl_delete(self, uuids): + log.debug(f'delete -> {uuids}') + self.delete = uuids + + def syncrepl_entry(self, dn, attrs, uuid): + log.debug(f'entry -> {dn}') + self.entries[dn] = (uuid, attrs) + + def syncrepl_refreshdone(self): + log.debug('refreshdone') + +def syncstate_assert(st, sync): + # How many entries do we have? + r = st.search_ext_s( + base=DEFAULT_SUFFIX, + scope=ldap.SCOPE_SUBTREE, + filterstr='(objectClass=*)', + attrsonly=1, + escapehatch='i am sure' + ) + + # Initial sync + log.debug("*test* initial") + sync.syncrepl_search() + sync.syncrepl_complete() + # check we caught them all + assert len(r) == len(sync.entries.keys()) + assert len(r) == len(sync.present) + assert 0 == len(sync.delete) + + # Add a new entry + + account = nsUserAccounts(st, DEFAULT_SUFFIX).create_test_user() + # Check + log.debug("*test* add") + sync.syncrepl_search() + sync.syncrepl_complete() + sync.check_cookie() + assert 1 == len(sync.entries.keys()) + assert 1 == len(sync.present) + assert 0 == len(sync.delete) + + # Mod + account.replace('description', 'change') + # Check + log.debug("*test* mod") + sync.syncrepl_search() + sync.syncrepl_complete() + sync.check_cookie() + assert 1 == len(sync.entries.keys()) + assert 1 == len(sync.present) + assert 0 == len(sync.delete) + + ## Delete + account.delete() + + # Check + log.debug("*test* del") + sync.syncrepl_search() + sync.syncrepl_complete() + # In a delete, the cookie isn't updated (?) + sync.check_cookie() + log.debug(f'{sync.entries.keys()}') + log.debug(f'{sync.present}') + log.debug(f'{sync.delete}') + assert 0 == len(sync.entries.keys()) + assert 0 == len(sync.present) + assert 1 == len(sync.delete) + diff --git a/dirsrvtests/tests/suites/syncrepl_plugin/basic_test.py b/dirsrvtests/tests/suites/syncrepl_plugin/basic_test.py new file mode 100644 index 0000000..5928d25 --- /dev/null +++ b/dirsrvtests/tests/suites/syncrepl_plugin/basic_test.py @@ -0,0 +1,60 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2020 William Brown +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +import logging +import ldap +import time +from ldap.syncrepl import SyncreplConsumer +import pytest +from lib389 import DirSrv +from lib389.idm.user import nsUserAccounts, UserAccounts +from lib389.topologies import topology_st as topology +from lib389.paths import Paths +from lib389.utils import ds_is_older +from lib389.plugins import RetroChangelogPlugin, ContentSyncPlugin +from lib389._constants import * + +from . import ISyncRepl, syncstate_assert + +default_paths = Paths() +pytestmark = pytest.mark.tier1 + +log = logging.getLogger(__name__) + +def test_syncrepl_basic(topology): + """ Test basic functionality of the SyncRepl interface + + :id: f9fea826-8ae2-412a-8e88-b8e0ba939b06 + + :setup: Standalone instance + + :steps: + 1. Enable Retro Changelog + 2. Enable Syncrepl + 3. Run the syncstate test to check refresh, add, delete, mod. + + :expectedresults: + 1. Success + 1. Success + 1. Success + """ + st = topology.standalone + # Enable RetroChangelog. + rcl = RetroChangelogPlugin(st) + rcl.enable() + # Set the default targetid + rcl.replace('nsslapd-attribute', 'nsuniqueid:targetUniqueId') + # Enable sync repl + csp = ContentSyncPlugin(st) + csp.enable() + # Restart DS + st.restart() + # Setup the syncer + sync = ISyncRepl(st) + # Run the checks + syncstate_assert(st, sync) diff --git a/dirsrvtests/tests/suites/syncrepl_plugin/openldap_test.py b/dirsrvtests/tests/suites/syncrepl_plugin/openldap_test.py new file mode 100644 index 0000000..bc4ecb1 --- /dev/null +++ b/dirsrvtests/tests/suites/syncrepl_plugin/openldap_test.py @@ -0,0 +1,64 @@ +# Copyright (C) 2020 William Brown +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +import logging +import ldap +import time +from ldap.syncrepl import SyncreplConsumer +import pytest +from lib389 import DirSrv +from lib389.idm.user import nsUserAccounts, UserAccounts +from lib389.topologies import topology_st as topology +from lib389.paths import Paths +from lib389.utils import ds_is_older +from lib389.plugins import RetroChangelogPlugin, ContentSyncPlugin +from lib389._constants import * + +from . import ISyncRepl, syncstate_assert + +default_paths = Paths() +pytestmark = pytest.mark.tier1 + +log = logging.getLogger(__name__) + +@pytest.mark.skipif(ds_is_older('1.4.4.0'), reason="Sync repl does not support openldap compat in older versions") +def test_syncrepl_openldap(topology): + """ Test basic functionality of the openldap syncrepl + compatability handler. + + :id: 03039178-2cc6-40bd-b32c-7d6de108828b + + :setup: Standalone instance + + :steps: + 1. Enable Retro Changelog + 2. Enable Syncrepl + 3. Run the syncstate test to check refresh, add, delete, mod. + + :expectedresults: + 1. Success + 1. Success + 1. Success + """ + st = topology.standalone + # Enable RetroChangelog. + rcl = RetroChangelogPlugin(st) + rcl.enable() + # Set the default targetid + rcl.replace('nsslapd-attribute', 'nsuniqueid:targetUniqueId') + # Enable sync repl + csp = ContentSyncPlugin(st) + csp.enable() + # Restart DS + st.restart() + # log.error("+++++++++++") + # time.sleep(60) + # Setup the syncer + sync = ISyncRepl(st, openldap=True) + # Run the checks + syncstate_assert(st, sync) + diff --git a/ldap/schema/dsee.schema b/ldap/schema/dsee.schema new file mode 100644 index 0000000..a469292 --- /dev/null +++ b/ldap/schema/dsee.schema @@ -0,0 +1,208 @@ +# $OpenLDAP$ +## This work is part of OpenLDAP Software . +## +## Copyright 2019-2020 The OpenLDAP Foundation. +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted only as authorized by the OpenLDAP +## Public License. +## +## A copy of this license is available in the file LICENSE in the +## top-level directory of the distribution or, alternatively, at +## . + +# This file is provided for informational purposes only. + +# These definitions are from Sun DSEE 7's cn=schema subentry. +# None of the attributes had matching rules defined; we've +# inserted usable ones as needed. + +# Some of these attributes are defined with NO-USER-MODIFICATION, +# but slapd won't load such definitions from user-modifiable schema +# files. So that designation has been removed, and commented accordingly. + +objectidentifier NetscapeRoot 2.16.840.1.113730 +objectidentifier NetscapeDS NetscapeRoot:3 +objectidentifier NSDSat NetscapeDS:1 +objectidentifier NSDSoc NetscapeDS:2 +objectidentifier SunRoot 1.3.6.1.4.1.42 +objectidentifier SunDS SunRoot:2.27 + +attributetype ( 1.2.840.113556.1.2.102 + NAME 'memberOf' + DESC 'Group that the entry belongs to' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + X-ORIGIN 'Netscape Delegated Administrator' ) + +attributetype ( NSDSat:9999 + NAME 'entryId' + DESC 'Supplier Internal Id' + EQUALITY integerMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( 1.3.1.1.4.1.453.16.2.103 + NAME 'numSubordinates' + DESC 'Number of Subordinate Entries from Supplier' + EQUALITY integerMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:55 + NAME 'aci' + DESC 'NSDS ACI' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:9998 + NAME 'parentId' + DESC 'Supplier Internal Id' + EQUALITY integerMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:610 + NAME 'nsAccountLock' + DESC 'Operational attribute for Account Inactivation' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + X-ORIGIN '389 Directory Server Project' ) + +attributetype ( NSDSat:2343 + name 'legalName' + DESC 'An individuals legalName' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + X-ORIGIN '389 Directory Server Project' ) + +attributetype ( NSDSat:2337 + NAME 'nsCertSubjectDN' + DESC 'An x509 DN from a certificate used to map during a TLS bind process' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + X-ORIGIN '389 Directory Server Project' ) + +attributetype ( NSDSat:2111 + NAME 'tombstoneNumSubordinates' + DESC 'count of immediate subordinates for tombstone entries' + EQUALITY integerMatch + ORDERING integerOrderingMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 + SINGLE-VALUE + X-ORIGIN '389 directory server' ) + +attributetype ( NSDSat:2342 + NAME 'nsSshPublicKey' + DESC 'An nsSshPublicKey record' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + X-ORIGIN '389 Directory Server Project' ) + +attributetype ( NSDSat:5 + NAME 'changeNumber' + DESC 'Changelog attribute type' + EQUALITY integerMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:6 + NAME 'targetDn' + DESC 'Changelog attribute type' + EQUALITY distinguishedNameMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:7 + NAME 'changeType' + DESC 'Changelog attribute type' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + X-ORIGIN 'Changelog Internet Draft' ) + +# They claim Binary syntax but it's really octetString +attributetype ( NSDSat:8 + NAME 'changes' + DESC 'Changelog attribute type' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:9 + NAME 'newRdn' + DESC 'Changelog attribute type' + EQUALITY distinguishedNameMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:10 + NAME 'deleteOldRdn' + DESC 'Changelog attribute type' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 + X-ORIGIN 'Changelog Internet Draft' ) + +attributetype ( NSDSat:11 + NAME 'newSuperior' + DESC 'Changelog attribute type' + EQUALITY distinguishedNameMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + X-ORIGIN 'Changelog Internet Draft' ) + +# should be generalizedTime, but they used directoryString instead... +attributeType ( NSDSat:77 + NAME 'changeTime' + DESC 'Sun ONE defined attribute type' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + X-ORIGIN 'Sun ONE Directory Server' ) + +# These are UUIDs, but (of course) hyphenated differently than ours. +# NO-USER-MODIFICATION +attributetype ( NSDSat:542 + NAME 'nsUniqueId' + DESC 'Sun ONE defined attribute type' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + X-ORIGIN 'Sun ONE Directory Server' ) + +# NO-USER-MODIFICATION +attributeype ( SunDS:9.1.596 + NAME 'targetUniqueId' + DESC 'RetroChangelog attribute type' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + X-ORIGIN 'Sun Directory Server' ) + +objectclass ( NSDSoc:1 + NAME 'changeLogEntry' + DESC 'LDAP changelog objectclass' + SUP top STRUCTURAL + MUST ( targetDn $ changeTime $ changeNumber $ changeType ) + MAY ( changes $ newRdn $ deleteOldRdn $ newSuperior ) + X-ORIGIN 'Changelog Internet Draft' ) + +objectclass ( NSDSoc:333 + NAME 'nsPerson' + DESC 'A representation of a person in a directory server' + SUP top STRUCTURAL + MUST ( displayName $ cn ) + MAY ( userPassword $ seeAlso $ description $ legalName $ mail $ preferredLanguage ) + X-ORIGIN '389 Directory Server Project' ) + +objectclass ( NSDSoc:331 + NAME 'nsAccount' + DESC 'A representation of a binding user in a directory server' + SUP top AUXILIARY + MAY ( userCertificate $ nsCertSubjectDN $ nsSshPublicKey $ userPassword $ nsAccountLock ) + X-ORIGIN '389 Directory Server Project' ) + +objectclass ( NSDSoc:334 + NAME 'nsOrgPerson' + DESC 'A representation of an org person in directory server. See also inetOrgPerson.' + SUP top AUXILIARY + MAY ( businessCategory $ carLicense $ departmentNumber $ employeeNumber $ employeeType $ homePhone $ homePostalAddress $ initials $ jpegPhoto $ labeledURI $ manager $ mobile $ o $ pager $ photo $ roomNumber $ uid $ userCertificate $ telephoneNumber $ x500uniqueIdentifier $ userSMIMECertificate $ userPKCS12 ) + X-ORIGIN '389 Directory Server Project' ) + +objectclass ( NSDSoc:329 + NAME 'nsMemberOf' + DESC 'Allow memberOf assignment on groups for nesting and users' + SUP top AUXILIARY + MAY ( memberOf ) + X-ORIGIN '389 Directory Server Project' ) diff --git a/ldap/servers/plugins/sync/sync.h b/ldap/servers/plugins/sync/sync.h index a6b54a6..7dd5f9c 100644 --- a/ldap/servers/plugins/sync/sync.h +++ b/ldap/servers/plugins/sync/sync.h @@ -16,6 +16,7 @@ #include #include +#include #include "slapi-plugin.h" #include "slapi-private.h" @@ -44,6 +45,7 @@ typedef struct sync_cookie char *cookie_client_signature; char *cookie_server_signature; unsigned long cookie_change_info; + bool openldap_compat; } Sync_Cookie; typedef struct sync_update @@ -83,9 +85,9 @@ int sync_intermediate_msg(Slapi_PBlock *pb, int tag, Sync_Cookie *cookie, char * int sync_result_msg(Slapi_PBlock *pb, Sync_Cookie *cookie); int sync_result_err(Slapi_PBlock *pb, int rc, char *msg); -Sync_Cookie *sync_cookie_create(Slapi_PBlock *pb); +Sync_Cookie *sync_cookie_create(Slapi_PBlock *pb, Sync_Cookie *client_cookie); void sync_cookie_update(Sync_Cookie *cookie, Slapi_Entry *ec); -Sync_Cookie *sync_cookie_parse(char *cookie); +Sync_Cookie *sync_cookie_parse(char *cookie, bool *cookie_refresh); int sync_cookie_isvalid(Sync_Cookie *testcookie, Sync_Cookie *refcookie); void sync_cookie_free(Sync_Cookie **freecookie); char *sync_cookie2str(Sync_Cookie *cookie); diff --git a/ldap/servers/plugins/sync/sync_refresh.c b/ldap/servers/plugins/sync/sync_refresh.c index 646ff76..34ac958 100644 --- a/ldap/servers/plugins/sync/sync_refresh.c +++ b/ldap/servers/plugins/sync/sync_refresh.c @@ -66,8 +66,9 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb) slapi_pblock_get(pb, SLAPI_REQCONTROLS, &requestcontrols); if (slapi_control_present(requestcontrols, LDAP_CONTROL_SYNC, &psbvp, NULL)) { char *cookie = NULL; - int mode = 1; - int refresh = 0; + int32_t mode = 1; + int32_t refresh = 0; + bool cookie_refresh = 0; if (sync_parse_control_value(psbvp, &mode, &refresh, &cookie) != LDAP_SUCCESS) { @@ -83,12 +84,22 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb) } if (mode == 1 || mode == 3) { - - /* we need to return a cookie in the result message + /* + * OpenLDAP violates rfc4533 by sending a "rid=" in it's initial cookie sync, even + * when using their changelog mode. As a result, we parse the cookie to handle this + * shenangians to determine if this is valid. + */ + client_cookie = sync_cookie_parse(cookie, &cookie_refresh); + /* + * we need to return a cookie in the result message * indicating a state to be used in future sessions - * as starting point - create it now + * as starting point - create it now. We need to provide + * the client_cookie so we understand if we are in + * openldap mode or not, and to get the 'rid' of the + * consumer. */ - session_cookie = sync_cookie_create(pb); + session_cookie = sync_cookie_create(pb, client_cookie); + PR_ASSERT(session_cookie); /* * if mode is persist we need to setup the persit handler * to catch the mods while the refresh is done @@ -104,7 +115,7 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb) } } /* - * now handl the refresh request + * now handle the refresh request * there are two scenarios * 1. no cookie is provided this means send all entries matching the search request * 2. a cookie is provided: send all entries changed since the cookie was issued @@ -112,31 +123,34 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb) * -- return e-syncRefreshRequired if the data referenced in the cookie are no * longer in the history */ - if (cookie) { - if ((client_cookie = sync_cookie_parse(cookie)) && - sync_cookie_isvalid(client_cookie, session_cookie)) { + if (!cookie_refresh) { + if (sync_cookie_isvalid(client_cookie, session_cookie)) { rc = sync_refresh_update_content(pb, client_cookie, session_cookie); - if (rc == 0) + if (rc == 0) { entries_sent = 1; - if (sync_persist) + } + if (sync_persist) { rc = sync_intermediate_msg(pb, LDAP_TAG_SYNC_REFRESH_DELETE, session_cookie, NULL); - else + } else { rc = sync_result_msg(pb, session_cookie); + } } else { rc = E_SYNC_REFRESH_REQUIRED; sync_result_err(pb, rc, "Invalid session cookie"); } } else { rc = sync_refresh_initial_content(pb, sync_persist, tid, session_cookie); - if (rc == 0 && !sync_persist) + if (rc == 0 && !sync_persist) { /* maintained in postop code */ session_cookie = NULL; + } /* if persis it will be handed over to persist code */ } if (rc) { - if (sync_persist) + if (sync_persist) { sync_persist_terminate(tid); + } goto error_return; } else if (sync_persist) { Slapi_Operation *operation; @@ -194,7 +208,38 @@ sync_srch_refresh_post_search(Slapi_PBlock *pb) if (info->send_flag & SYNC_FLAG_ADD_DONE_CTRL) { LDAPControl **ctrl = (LDAPControl **)slapi_ch_calloc(2, sizeof(LDAPControl *)); char *cookiestr = sync_cookie2str(info->cookie); - sync_create_sync_done_control(&ctrl[0], 0, cookiestr); + /* + * RFC4533 + * If refreshDeletes of syncDoneValue is FALSE, the new copy includes + * all changed entries returned by the reissued Sync Operation, as well + * as all unchanged entries identified as being present by the reissued + * Sync Operation, but whose content is provided by the previous Sync + * Operation. The unchanged entries not identified as being present are + * deleted from the client content. They had been either deleted, + * moved, or otherwise scoped-out from the content. + * + * If refreshDeletes of syncDoneValue is TRUE, the new copy includes all + * changed entries returned by the reissued Sync Operation, as well as + * all other entries of the previous copy except for those that are + * identified as having been deleted from the content. + * + * Confused yet? Don't worry so am I. I have no idea what this means or + * what it will do. The best I can see from wireshark is that if refDel is + * false, then anything *not* present will be purged from the change that + * was supplied. Which probably says a lot about how confusing syncrepl is + * that we've hardcoded this to false for literally years and no one has + * complained, probably because every client is broken in their own ways + * as no one can actually interpret that dense statement above. + * + * Point is, if we set refresh to true for openldap mode, it works, and if + * it's false, the moment we send a single intermediate delete message, we + * delete literally everything 🔥. + */ + if (info->cookie->openldap_compat) { + sync_create_sync_done_control(&ctrl[0], 1, cookiestr); + } else { + sync_create_sync_done_control(&ctrl[0], 0, cookiestr); + } slapi_pblock_set(pb, SLAPI_RESCONTROLS, ctrl); slapi_ch_free((void **)&cookiestr); } @@ -254,9 +299,21 @@ sync_refresh_update_content(Slapi_PBlock *pb, Sync_Cookie *client_cookie, Sync_C Slapi_PBlock *seq_pb; char *filter; Sync_CallBackData cb_data; - int rc; - int chg_count = server_cookie->cookie_change_info - - client_cookie->cookie_change_info + 1; + int rc = LDAP_SUCCESS; + PR_ASSERT(client_cookie); + + /* + * We have nothing to send, move along. + * Should be caught by cookie is valid though if the server < client, but if + * they are equal, we return. + */ + PR_ASSERT(server_cookie->cookie_change_info >= client_cookie->cookie_change_info); + if (server_cookie->cookie_change_info == client_cookie->cookie_change_info) { + return rc; + } + + int chg_count = (server_cookie->cookie_change_info - client_cookie->cookie_change_info) + 1; + PR_ASSERT(chg_count > 0); cb_data.cb_updates = (Sync_UpdateNode *)slapi_ch_calloc(chg_count, sizeof(Sync_UpdateNode)); @@ -581,21 +638,21 @@ sync_read_entry_from_changelog(Slapi_Entry *cl_entry, void *cb_data) void sync_send_deleted_entries(Slapi_PBlock *pb, Sync_UpdateNode *upd, int chg_count, Sync_Cookie *cookie) { - char *syncUUIDs[SYNC_MAX_DELETED_UUID_BATCH + 1]; - int uuid_index = 0; - int index, i; + char *syncUUIDs[SYNC_MAX_DELETED_UUID_BATCH + 1] = {0}; + size_t uuid_index = 0; syncUUIDs[0] = NULL; - for (index = 0; index < chg_count; index++) { + for (size_t index = 0; index < chg_count; index++) { if (upd[index].upd_chgtype == LDAP_REQ_DELETE && upd[index].upd_uuid) { if (uuid_index < SYNC_MAX_DELETED_UUID_BATCH) { - syncUUIDs[uuid_index++] = sync_nsuniqueid2uuid(upd[index].upd_uuid); + syncUUIDs[uuid_index] = sync_nsuniqueid2uuid(upd[index].upd_uuid); + uuid_index++; } else { /* max number of uuids to be sent in one sync info message */ syncUUIDs[uuid_index] = NULL; - sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, &syncUUIDs[0]); - for (i = 0; i < uuid_index; i++) { + sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, (char **)&syncUUIDs); + for (size_t i = 0; i < uuid_index; i++) { slapi_ch_free((void **)&syncUUIDs[i]); syncUUIDs[i] = NULL; } @@ -607,8 +664,8 @@ sync_send_deleted_entries(Slapi_PBlock *pb, Sync_UpdateNode *upd, int chg_count, if (uuid_index > 0 && syncUUIDs[uuid_index - 1]) { /* more entries to send */ syncUUIDs[uuid_index] = NULL; - sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, &syncUUIDs[0]); - for (i = 0; i < uuid_index; i++) { + sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, (char **)&syncUUIDs); + for (size_t i = 0; i < uuid_index; i++) { slapi_ch_free((void **)&syncUUIDs[i]); syncUUIDs[i] = NULL; } diff --git a/ldap/servers/plugins/sync/sync_util.c b/ldap/servers/plugins/sync/sync_util.c index 0ed7d3c..847a20d 100644 --- a/ldap/servers/plugins/sync/sync_util.c +++ b/ldap/servers/plugins/sync/sync_util.c @@ -12,6 +12,11 @@ static struct berval *create_syncinfo_value(int type, const char *cookie, const static char *sync_cookie_get_server_info(Slapi_PBlock *pb); static char *sync_cookie_get_client_info(Slapi_PBlock *pb); +static void sync_ulong2olcsn(unsigned long chgnr, char *buf); +static unsigned long sync_olcsn2ulong(char *csn); + +#define CSN_OFFSET 4102448461 + /* * Parse the value from an LDAPv3 sync request control. They look * like this: @@ -191,14 +196,14 @@ sync_create_sync_done_control(LDAPControl **ctrlp, int refresh, char *cookie) if (cookie) { if ((rc = ber_printf(ber, "{s", cookie)) != -1) { if (refresh) { - rc = ber_printf(ber, "e}", refresh); + rc = ber_printf(ber, "b}", refresh); } else { rc = ber_printf(ber, "}"); } } } else { if (refresh) { - rc = ber_printf(ber, "{e}", refresh); + rc = ber_printf(ber, "{b}", refresh); } else { rc = ber_printf(ber, "{}"); } @@ -229,10 +234,18 @@ sync_cookie2str(Sync_Cookie *cookie) char *cookiestr = NULL; if (cookie) { - cookiestr = slapi_ch_smprintf("%s#%s#%lu", - cookie->cookie_server_signature, - cookie->cookie_client_signature, - cookie->cookie_change_info); + if (cookie->openldap_compat) { + char buf[16] = {0}; + sync_ulong2olcsn(cookie->cookie_change_info, buf); + cookiestr = slapi_ch_smprintf("%s,csn=%s.000000Z#000000#000#000000", + cookie->cookie_client_signature, + buf); + } else { + cookiestr = slapi_ch_smprintf("%s#%s#%lu", + cookie->cookie_server_signature, + cookie->cookie_client_signature, + cookie->cookie_change_info); + } } return (cookiestr); } @@ -260,7 +273,12 @@ sync_result_msg(Slapi_PBlock *pb, Sync_Cookie *cookie) char *cookiestr = sync_cookie2str(cookie); LDAPControl **ctrl = (LDAPControl **)slapi_ch_calloc(2, sizeof(LDAPControl *)); - sync_create_sync_done_control(&ctrl[0], 0, cookiestr); + + if (cookie->openldap_compat) { + sync_create_sync_done_control(&ctrl[0], 1, cookiestr); + } else { + sync_create_sync_done_control(&ctrl[0], 0, cookiestr); + } slapi_pblock_set(pb, SLAPI_RESCONTROLS, ctrl); slapi_send_ldap_result(pb, 0, NULL, NULL, 0, NULL); @@ -288,24 +306,39 @@ create_syncinfo_value(int type, const char *cookie, const char **uuids) return (NULL); } + /* + * ber_tag_t is an unsigned integer of at least 32 bits + * used to represent a BER tag. It is commonly equivalent + * to a unsigned long. + * ... + * ber_printf(...) + * t + * Tag of the next element. A pointer to a ber_tag_t should be supplied. + */ + + ber_tag_t btag = (ber_tag_t)type; + switch (type) { case LDAP_TAG_SYNC_NEW_COOKIE: - ber_printf(ber, "to", type, cookie); + ber_printf(ber, "to", btag, cookie); break; case LDAP_TAG_SYNC_REFRESH_DELETE: case LDAP_TAG_SYNC_REFRESH_PRESENT: - ber_printf(ber, "t{", type); - if (cookie) + ber_printf(ber, "t{", btag); + if (cookie) { ber_printf(ber, "s", cookie); + } /* ber_printf(ber, "b",1); */ ber_printf(ber, "}"); break; case LDAP_TAG_SYNC_ID_SET: - ber_printf(ber, "t{", type); - if (cookie) + ber_printf(ber, "t{", btag); + if (cookie) { ber_printf(ber, "s", cookie); - if (uuids) + } + if (uuids) { ber_printf(ber, "b[v]", 1, uuids); + } ber_printf(ber, "}"); break; default: @@ -471,19 +504,27 @@ sync_cookie_get_change_info(Sync_CallBackData *scbd) } Sync_Cookie * -sync_cookie_create(Slapi_PBlock *pb) +sync_cookie_create(Slapi_PBlock *pb, Sync_Cookie *client_cookie) { - - Sync_CallBackData scbd; - int rc; + Sync_CallBackData scbd = {0}; + int rc = 0; Sync_Cookie *sc = (Sync_Cookie *)slapi_ch_calloc(1, sizeof(Sync_Cookie)); scbd.cb_err = SYNC_CALLBACK_PREINIT; rc = sync_cookie_get_change_info(&scbd); if (rc == 0) { - sc->cookie_server_signature = sync_cookie_get_server_info(pb); - sc->cookie_client_signature = sync_cookie_get_client_info(pb); + /* If the client is in openldap compat, we need to generate the same. */ + if (client_cookie && client_cookie->openldap_compat) { + sc->openldap_compat = client_cookie->openldap_compat; + sc->cookie_client_signature = slapi_ch_strdup(client_cookie->cookie_client_signature); + sc->cookie_server_signature = NULL; + } else { + sc->openldap_compat = false; + sc->cookie_server_signature = sync_cookie_get_server_info(pb); + sc->cookie_client_signature = sync_cookie_get_client_info(pb); + } + if (scbd.cb_err == SYNC_CALLBACK_PREINIT) { /* changenr is not initialized. */ sc->cookie_change_info = 0; @@ -513,36 +554,110 @@ sync_cookie_update(Sync_Cookie *sc, Slapi_Entry *ec) } Sync_Cookie * -sync_cookie_parse(char *cookie) +sync_cookie_parse(char *cookie, bool *cookie_refresh) { - char *p, *q; + char *p = NULL; + char *q = NULL; Sync_Cookie *sc = NULL; + /* This is an rfc compliant initial refresh request */ if (cookie == NULL || *cookie == '\0') { + *cookie_refresh = 1; return NULL; } - /* - * Format of cookie: server_signature#client_signature#change_info_number - * If the cookie is malformed, NULL is returned. - */ + /* get ready to parse. */ p = q = cookie; - p = strchr(q, '#'); - if (p) { - *p = '\0'; - sc = (Sync_Cookie *)slapi_ch_calloc(1, sizeof(Sync_Cookie)); - sc->cookie_server_signature = slapi_ch_strdup(q); - q = p + 1; + + sc = (Sync_Cookie *)slapi_ch_calloc(1, sizeof(Sync_Cookie)); + if (strncmp(cookie, "rid=", 4) == 0) { + /* + * We are in openldap mode. + * The cookies are: + * rid=123,csn=20200525051329.534174Z#000000#000#000000 + */ + sc->openldap_compat = true; + p = strchr(q, ','); + if (p == NULL) { + /* No CSN following the rid, must be an init request. */ + *cookie_refresh = 1; + /* We need to keep the client rid though */ + sc->cookie_client_signature = slapi_ch_strdup(q); + /* server sig and change info do not need to be set. */ + sc->cookie_server_signature = NULL; + sc->cookie_change_info = 0; + } else { + /* Ensure that this really is a csn= */ + if (strncmp(p, ",csn=", 5) != 0) { + /* Yeah nahhhhhhh */ + goto error_return; + } + /* We dont care about the remainder after the . */ + if (strlen(p) < 20) { + /* Probably a corrupt CSN. We need at least 20 chars. */ + goto error_return; + } + /* + * Replace the , with a '\0' This makes q -> p a str of the rid. + * rid=123,csn=19700101001640.000000Z#000000#000#000000 + * ^ ^ + * q p + * rid=123\0csn=19700101001640.000000Z#000000#000#000000 + */ + PR_ASSERT(p[0] == ','); + p[0] = '\0'; + /* + * Now terminate the ulong which is our change num so we can parse it. + * rid=123\0csn=19700101001640.000000Z#000000#000#000000 + * ^ ^ ^ + * q p[0] p[19] + * rid=123\0csn=19700101001640\0... + */ + PR_ASSERT(p[19] == '.'); + p[19] = '\0'; + /* + * And move the pointer up to the start of the int we need to parse. + * rid=123\0csn=19700101001640\0... + * ^ ^ + * q p +5 --> + * rid=123\0csn=19700101001640\0... + * ^ ^ + * q p + */ + p = p + 5; + PR_ASSERT(strlen(p) == 14); + /* We are now ready to parse the csn and create a cookie! */ + sc->cookie_client_signature = slapi_ch_strdup(q); + sc->cookie_server_signature = NULL; + /* Get the change number from the string */ + sc->cookie_change_info = sync_olcsn2ulong(p); + if (SYNC_INVALID_CHANGENUM == sc->cookie_change_info) { + /* Sad trombone */ + goto error_return; + } + /* Done! 🎉 */ + } + } else { + /* + * Format of the 389 cookie: server_signature#client_signature#change_info_number + * If the cookie is malformed, NULL is returned. + */ p = strchr(q, '#'); if (p) { *p = '\0'; - sc->cookie_client_signature = slapi_ch_strdup(q); - sc->cookie_change_info = sync_number2ulong(p + 1); - if (SYNC_INVALID_CHANGENUM == sc->cookie_change_info) { + sc->cookie_server_signature = slapi_ch_strdup(q); + q = p + 1; + p = strchr(q, '#'); + if (p) { + *p = '\0'; + sc->cookie_client_signature = slapi_ch_strdup(q); + sc->cookie_change_info = sync_number2ulong(p + 1); + if (SYNC_INVALID_CHANGENUM == sc->cookie_change_info) { + goto error_return; + } + } else { goto error_return; } - } else { - goto error_return; } } return (sc); @@ -557,17 +672,30 @@ int sync_cookie_isvalid(Sync_Cookie *testcookie, Sync_Cookie *refcookie) { /* client and server info must match */ - if ((testcookie && refcookie) && - (strcmp(testcookie->cookie_client_signature, refcookie->cookie_client_signature) || - strcmp(testcookie->cookie_server_signature, refcookie->cookie_server_signature) || + if (testcookie == NULL || refcookie == NULL) { + return 0; + } + if ((testcookie->openldap_compat != refcookie->openldap_compat || + strcmp(testcookie->cookie_client_signature, refcookie->cookie_client_signature) || testcookie->cookie_change_info == -1 || testcookie->cookie_change_info > refcookie->cookie_change_info)) { - return (0); + return 0; + } + + if (refcookie->openldap_compat) { + if (testcookie->cookie_server_signature != NULL || + refcookie->cookie_server_signature != NULL) { + return 0; + } + } else { + if (strcmp(testcookie->cookie_server_signature, refcookie->cookie_server_signature)) { + return 0; + } } /* could add an additional check if the requested state in client cookie is still * available. Accept any state request for now. */ - return (1); + return 1; } void @@ -701,3 +829,40 @@ sync_number2ulong(char *chgnrstr) return SYNC_INVALID_CHANGENUM; } } + +/* + * Why is there a CSN offset? + * + * CSN offset is to bump our csn date to a future time so that + * we always beat openldap in conflicts. I can only hope that + * in 100 years this code is dead, buried, for no one to see + * again. If you are reading this in 2100, William of 2020 + * says "I'm so very sorry". + */ + +static unsigned long +sync_olcsn2ulong(char *csn) { + struct tm pt = {0}; + char *ret = strptime(csn, "%Y%m%d%H%M%S", &pt); + PR_ASSERT(ret); + if (ret == NULL) { + return SYNC_INVALID_CHANGENUM; + } + time_t pepoch = mktime(&pt); + unsigned long px = (unsigned long)pepoch; + PR_ASSERT(px >= CSN_OFFSET); + if (px < CSN_OFFSET) { + return SYNC_INVALID_CHANGENUM; + } + return px - CSN_OFFSET; +} + +static void +sync_ulong2olcsn(unsigned long chgnr, char *buf) { + PR_ASSERT(buf); + unsigned long x = chgnr + CSN_OFFSET; + time_t epoch = x; + struct tm t = {0}; + localtime_r(&epoch, &t); + strftime(buf, 15, "%Y%m%d%H%M%S", &t); +} diff --git a/src/lib389/lib389/cli_conf/plugin.py b/src/lib389/lib389/cli_conf/plugin.py index b50837c..3dc5225 100644 --- a/src/lib389/lib389/cli_conf/plugin.py +++ b/src/lib389/lib389/cli_conf/plugin.py @@ -27,6 +27,7 @@ from lib389.cli_conf.plugins import passthroughauth as cli_passthroughauth from lib389.cli_conf.plugins import retrochangelog as cli_retrochangelog from lib389.cli_conf.plugins import automember as cli_automember from lib389.cli_conf.plugins import posix_winsync as cli_posix_winsync +from lib389.cli_conf.plugins import contentsync as cli_contentsync SINGULAR = Plugin MANY = Plugins @@ -113,6 +114,7 @@ def create_parser(subparsers): cli_passthroughauth.create_parser(subcommands) cli_retrochangelog.create_parser(subcommands) cli_posix_winsync.create_parser(subcommands) + cli_contentsync.create_parser(subcommands) list_parser = subcommands.add_parser('list', help="List current configured (enabled and disabled) plugins") list_parser.set_defaults(func=plugin_list) diff --git a/src/lib389/lib389/cli_conf/plugins/contentsync.py b/src/lib389/lib389/cli_conf/plugins/contentsync.py new file mode 100644 index 0000000..86698e3 --- /dev/null +++ b/src/lib389/lib389/cli_conf/plugins/contentsync.py @@ -0,0 +1,15 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2020 William Brown +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +from lib389.plugins import ContentSyncPlugin +from lib389.cli_conf import add_generic_plugin_parsers + +def create_parser(subparsers): + contentsync_parser = subparsers.add_parser('contentsync', help='Manage and configure Content Sync Plugin (aka syncrepl)') + subcommands = contentsync_parser.add_subparsers(help='action') + add_generic_plugin_parsers(subcommands, ContentSyncPlugin) diff --git a/src/lib389/lib389/plugins.py b/src/lib389/lib389/plugins.py index c01a5c0..c702a96 100644 --- a/src/lib389/lib389/plugins.py +++ b/src/lib389/lib389/plugins.py @@ -2260,3 +2260,16 @@ class EntryUUIDPlugin(Plugin): task.create(properties=task_properties) return task + +class ContentSyncPlugin(Plugin): + """A single instance of Content Sync (aka syncrepl) plugin entry + + :param instance: An instance + :type instance: lib389.DirSrv + :param dn: Entry DN + :type dn: str + """ + + def __init__(self, instance, dn="cn=Content Synchronization,cn=plugins,cn=config"): + super(ContentSyncPlugin, self).__init__(instance, dn) +