From bf6e48666d8759a5365a54fd1b979457fe2b4d8b Mon Sep 17 00:00:00 2001 From: Thierry Bordaz Date: Aug 18 2020 15:23:41 +0000 Subject: Ticket - 51189 integrate changelog in main database - update CLI Bug description: PHASE 2 of backend redesign: http://www.port389.org/docs/389ds/design/integrate-changelog-database-and-backend-database.html With https://pagure.io/389-ds-base/issue/49562, the changelog uses the main database. Changelog configuration was managed with a global config entry (cn=changelog5,cn=config) Now it is managed via a per backend config entry (cn=changelog,cn=). Some config parameters are now in the backend specific changelog entry nsslapd-changelogmaxage nsslapd-changelogmaxentries nsslapd-changelogtrim-interval Some config paramters are simply abandonned (see design): nsslapd-changelogdir nsslapd-changelogcompactdb-interval Fix description: This fix (PR) is to be applied on top of 49562 It suppressed the ablity to create/delete changelog as the changelog entry is now created/suppressed when a backend becomes a replica or not. subcommands to set/get changelog attributes requires a suffix (aka backend). dsconf replication set-changelog --suffix --trim-interval dsconf replication set-changelog --suffix --max-age dsconf replication set-changelog --suffix --max-entries dsconf replication get-changelog --suffix This patch removes the ability to restore a changelog (restore-changelog) It implements a new 'class Changelog' to set/get the configuration attribute of a per backend changelog https://pagure.io/389-ds-base/issue/51189 Reviewed by: Simon Pichugin, Mark Reynolds (Big thanks) --- diff --git a/src/lib389/lib389/cli_conf/replication.py b/src/lib389/lib389/cli_conf/replication.py index b9bc3d2..f6cfc10 100644 --- a/src/lib389/lib389/cli_conf/replication.py +++ b/src/lib389/lib389/cli_conf/replication.py @@ -14,10 +14,11 @@ import ldap import stat from shutil import copyfile from getpass import getpass -from lib389._constants import ReplicaRole, DSRC_HOME +from lib389._constants import ReplicaRole, DSRC_HOME, DEFAULT_BENAME from lib389.cli_base.dsrc import dsrc_to_repl_monitor -from lib389.utils import is_a_dn, copy_with_permissions -from lib389.replica import Replicas, ReplicationMonitor, BootstrapReplicationManager, Changelog5, ChangelogLDIF +from lib389.utils import is_a_dn, copy_with_permissions, ds_supports_new_changelog, ensure_str +from lib389.backend import Backends +from lib389.replica import Replicas, ReplicationMonitor, BootstrapReplicationManager, Changelog5, ChangelogLDIF, Changelog from lib389.tasks import CleanAllRUVTask, AbortCleanAllRUVTask from lib389._mapped_object import DSLdapObjects @@ -181,14 +182,15 @@ def enable_replication(inst, basedn, log, args): repl_properties['nsDS5ReplicaBindDN'] = args.bind_dn # First create the changelog - cl = Changelog5(inst) - try: - cl.create(properties={ - 'cn': 'changelog5', - 'nsslapd-changelogdir': inst.get_changelog_dir() - }) - except ldap.ALREADY_EXISTS: - pass + if not ds_supports_new_changelog(): + cl = Changelog5(inst) + try: + cl.create(properties={ + 'cn': 'changelog5', + 'nsslapd-changelogdir': inst.get_changelog_dir() + }) + except ldap.ALREADY_EXISTS: + pass # Finally enable replication replicas = Replicas(inst) @@ -463,7 +465,7 @@ def get_repl_monitor_info(inst, basedn, log, args): if args.json: log.info(json.dumps({"type": "list", "items": report_items}, indent=4)) - +# This subcommand is available when 'not ds_supports_new_changelog' def create_cl(inst, basedn, log, args): cl = Changelog5(inst) try: @@ -476,6 +478,7 @@ def create_cl(inst, basedn, log, args): log.info("Successfully created replication changelog") +# This subcommand is available when 'not ds_supports_new_changelog' def delete_cl(inst, basedn, log, args): cl = Changelog5(inst) try: @@ -485,6 +488,7 @@ def delete_cl(inst, basedn, log, args): log.info("Successfully deleted replication changelog") +# This subcommand is available when 'not ds_supports_new_changelog' def set_cl(inst, basedn, log, args): cl = Changelog5(inst) attrs = _args_to_attrs(args) @@ -504,6 +508,7 @@ def set_cl(inst, basedn, log, args): log.info("Successfully updated replication changelog") +# This subcommand is available when 'not ds_supports_new_changelog' def get_cl(inst, basedn, log, args): cl = Changelog5(inst) if args and args.json: @@ -511,6 +516,37 @@ def get_cl(inst, basedn, log, args): else: log.info(cl.display()) +# This subcommand is available when 'ds_supports_new_changelog' +# that means there is a changelog config entry per backend (aka suffix) +def set_per_backend_cl(inst, basedn, log, args): + suffix = args.suffix + cl = Changelog(inst, suffix) + attrs = _args_to_attrs(args) + replace_list = [] + did_something = False + for attr, value in attrs.items(): + if value == "": + cl.remove_all(attr) + did_something = True + else: + replace_list.append((attr, value)) + if len(replace_list) > 0: + cl.replace_many(*replace_list) + elif not did_something: + raise ValueError("There are no changes to set for the replication changelog") + + log.info("Successfully updated replication changelog") + +# This subcommand is available when 'ds_supports_new_changelog' +# that means there is a changelog config entry per backend (aka suffix) +def get_per_backend_cl(inst, basedn, log, args): + suffix = args.suffix + cl = Changelog(inst, suffix) + if args and args.json: + log.info(cl.get_all_attrs_json()) + else: + log.info(cl.display()) + def create_repl_manager(inst, basedn, log, args): manager_name = "replication manager" @@ -1166,22 +1202,16 @@ def create_parser(subparsers): repl_get_parser.set_defaults(func=get_repl_config) repl_get_parser.add_argument('--suffix', required=True, help='Get the replication configuration for this suffix DN') - repl_create_cl = repl_subcommands.add_parser('create-changelog', help='Create the replication changelog') - repl_create_cl.set_defaults(func=create_cl) - - repl_delete_cl = repl_subcommands.add_parser('delete-changelog', help='Delete the replication changelog. This will invalidate any existing replication agreements') - repl_delete_cl.set_defaults(func=delete_cl) + repl_set_per_backend_cl = repl_subcommands.add_parser('set-changelog', help='Set replication changelog attributes.') + repl_set_per_backend_cl.set_defaults(func=set_per_backend_cl) + repl_set_per_backend_cl.add_argument('--suffix', required=True, help='The suffix that uses the changelog') + repl_set_per_backend_cl.add_argument('--max-entries', help="The maximum number of entries to get in the replication changelog") + repl_set_per_backend_cl.add_argument('--max-age', help="The maximum age of a replication changelog entry") + repl_set_per_backend_cl.add_argument('--trim-interval', help="The interval to check if the replication changelog can be trimmed") - repl_set_cl = repl_subcommands.add_parser('set-changelog', help='Set replication changelog attributes.') - repl_set_cl.set_defaults(func=set_cl) - repl_set_cl.add_argument('--cl-dir', help="The replication changelog location on the filesystem") - repl_set_cl.add_argument('--max-entries', help="The maximum number of entries to get in the replication changelog") - repl_set_cl.add_argument('--max-age', help="The maximum age of a replication changelog entry") - repl_set_cl.add_argument('--compact-interval', help="The replication changelog compaction interval") - repl_set_cl.add_argument('--trim-interval', help="The interval to check if the replication changelog can be trimmed") - - repl_get_cl = repl_subcommands.add_parser('get-changelog', help='Display replication changelog attributes.') - repl_get_cl.set_defaults(func=get_cl) + repl_get_per_backend_cl = repl_subcommands.add_parser('get-changelog', help='Display replication changelog attributes.') + repl_get_per_backend_cl.set_defaults(func=get_per_backend_cl) + repl_get_per_backend_cl.add_argument('--suffix', required=True, help='The suffix that uses the changelog') repl_dump_cl = repl_subcommands.add_parser('dump-changelog', help='Decode Directory Server replication change log and dump it to an LDIF') repl_dump_cl.set_defaults(func=dump_cl) @@ -1197,22 +1227,6 @@ def create_parser(subparsers): help="Specify replica roots whose changelog you want to dump. The replica " "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.") - repl_restore_cl = repl_subcommands.add_parser('restore-changelog', - help='Restore Directory Server replication change log from LDIF file or change log directory') - restore_subcommands = repl_restore_cl.add_subparsers(help='Restore Replication Changelog') - restore_ldif = restore_subcommands.add_parser('from-ldif', help='Restore a single LDIF file.') - restore_ldif.set_defaults(func=restore_cl_ldif) - restore_ldif.add_argument('LDIF_PATH', nargs=1, help='The path of changelog LDIF file.') - restore_ldif.add_argument('-r', '--replica-root', nargs=1, required=True, - help="Specify one replica root whose changelog you want to restore. " - "The replica root will be consumed from the LDIF file name if the option is omitted.") - - restore_changelogdir = restore_subcommands.add_parser('from-changelogdir', help='Restore LDIF files from changelogdir.') - restore_changelogdir.set_defaults(func=restore_cl_dir) - restore_changelogdir.add_argument('REPLICA_ROOTS', nargs="+", - help="Specify replica roots whose changelog you want to restore. The replica " - "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.") - repl_set_parser = repl_subcommands.add_parser('set', help='Set an attribute in the replication configuration') repl_set_parser.set_defaults(func=set_repl_config) repl_set_parser.add_argument('--suffix', required=True, help='The DN of the replication suffix') diff --git a/src/lib389/lib389/replica.py b/src/lib389/lib389/replica.py index 56cccfa..b9edaee 100644 --- a/src/lib389/lib389/replica.py +++ b/src/lib389/lib389/replica.py @@ -21,7 +21,8 @@ from itertools import permutations from lib389._constants import * from lib389.properties import * from lib389.utils import (normalizeDN, escapeDNValue, ensure_bytes, ensure_str, - ensure_list_str, ds_is_older, copy_with_permissions) + ensure_list_str, ds_is_older, copy_with_permissions, + ds_supports_new_changelog) from lib389 import DirSrv, Entry, NoSuchEntryError, InvalidArgumentError from lib389._mapped_object import DSLdapObjects, DSLdapObject from lib389.passwd import password_generate @@ -1027,6 +1028,65 @@ class ChangelogLDIF(object): encoded_str += line +class Changelog(DSLdapObject): + """Represents the Directory Server changelog of a specific backend. This is used for + replication. + + :param instance: An instance + :type instance: lib389.DirSrv + """ + def __init__(self, instance, suffix=None, dn=None): + from lib389.backend import Backends + super(Changelog, self).__init__(instance, dn) + self._rdn_attribute = 'cn' + self._create_objectclasses = [ + 'top', + 'extensibleobject', + ] + if not ds_supports_new_changelog(): + raise ValueError('changelog (integrated to main database) is not supported in that version of the server') + if not suffix: + raise ValueError('A changelog is specific to a suffix and the suffix value is missing') + + # retrieve the backend associated to the provided suffix + be_insts = Backends(instance).list() + found_suffix = False + for be in be_insts: + be_suffix = be.get_attr_val_utf8_l('nsslapd-suffix') + if suffix == be_suffix: + found_suffix = True + break + if not found_suffix: + raise ValueError(f'No backend associated with nsslapd-suffix "{suffix}"') + + # changelog is a child of the backend + self._dn = 'cn=changelog,' + be.dn + + def set_max_entries(self, value): + """Configure the max entries the changelog can hold. + + :param value: the number of entries. + :type value: str + """ + self.replace('nsslapd-changelogmaxentries', value) + + def set_trim_interval(self, value): + """The time between changelog trims in seconds. + + :param value: The time in seconds + :type value: str + """ + self.replace('nsslapd-changelogtrim-interval', value) + + def set_max_age(self, value): + """The maximum age of entries in the changelog. + + :param value: The age with a time modifier of s, m, h, d, w. + :type value: str + """ + + self.replace('nsslapd-changelogmaxage', value) + class Changelog5(DSLdapObject): """Represents the Directory Server changelog. This is used for replication. Only one changelog is needed for every server.