#51195 Ticket - 51189 integrate changelog in main database - update CLI
Closed 3 years ago by spichugi. Opened 3 years ago by tbordaz.
tbordaz/389-ds-base ticket_51189  into  master

@@ -14,10 +14,11 @@ 

  import stat

  from shutil import copyfile

  from getpass import getpass

- from lib389._constants import ReplicaRole, DSRC_HOME

+ from lib389._constants import ReplicaRole, DSRC_HOME, DEFAULT_BENAME

  from lib389.cli_base.dsrc import dsrc_to_repl_monitor

- from lib389.utils import is_a_dn, copy_with_permissions

- from lib389.replica import Replicas, ReplicationMonitor, BootstrapReplicationManager, Changelog5, ChangelogLDIF

+ from lib389.utils import is_a_dn, copy_with_permissions, ds_supports_new_changelog, ensure_str

+ from lib389.backend import Backends

+ from lib389.replica import Replicas, ReplicationMonitor, BootstrapReplicationManager, Changelog5, ChangelogLDIF, Changelog

  from lib389.tasks import CleanAllRUVTask, AbortCleanAllRUVTask

  from lib389._mapped_object import DSLdapObjects

  
@@ -181,14 +182,15 @@ 

          repl_properties['nsDS5ReplicaBindDN'] = args.bind_dn

  

      # First create the changelog

-     cl = Changelog5(inst)

-     try:

-         cl.create(properties={

-             'cn': 'changelog5',

-             'nsslapd-changelogdir': inst.get_changelog_dir()

-         })

-     except ldap.ALREADY_EXISTS:

-         pass

+     if not ds_supports_new_changelog():

+         cl = Changelog5(inst)

+         try:

+             cl.create(properties={

+                 'cn': 'changelog5',

+                 'nsslapd-changelogdir': inst.get_changelog_dir()

+             })

+         except ldap.ALREADY_EXISTS:

+             pass

  

      # Finally enable replication

      replicas = Replicas(inst)
@@ -463,7 +465,7 @@ 

      if args.json:

          log.info(json.dumps({"type": "list", "items": report_items}, indent=4))

  

- 

+ # This subcommand is available when 'not ds_supports_new_changelog'

  def create_cl(inst, basedn, log, args):

      cl = Changelog5(inst)

      try:
@@ -476,6 +478,7 @@ 

      log.info("Successfully created replication changelog")

  

  

+ # This subcommand is available when 'not ds_supports_new_changelog'

  def delete_cl(inst, basedn, log, args):

      cl = Changelog5(inst)

      try:
@@ -485,6 +488,7 @@ 

      log.info("Successfully deleted replication changelog")

  

  

+ # This subcommand is available when 'not ds_supports_new_changelog'

  def set_cl(inst, basedn, log, args):

      cl = Changelog5(inst)

      attrs = _args_to_attrs(args)
@@ -504,6 +508,7 @@ 

      log.info("Successfully updated replication changelog")

  

  

+ # This subcommand is available when 'not ds_supports_new_changelog'

  def get_cl(inst, basedn, log, args):

      cl = Changelog5(inst)

      if args and args.json:
@@ -511,6 +516,37 @@ 

      else:

          log.info(cl.display())

  

+ # This subcommand is available when 'ds_supports_new_changelog'

+ # that means there is a changelog config entry per backend (aka suffix)

+ def set_per_backend_cl(inst, basedn, log, args):

+     suffix = args.suffix

+     cl = Changelog(inst, suffix)

+     attrs = _args_to_attrs(args)

+     replace_list = []

+     did_something = False

+     for attr, value in attrs.items():

+         if value == "":

+             cl.remove_all(attr)

+             did_something = True

+         else:

+             replace_list.append((attr, value))

+     if len(replace_list) > 0:

+         cl.replace_many(*replace_list)

+     elif not did_something:

+         raise ValueError("There are no changes to set for the replication changelog")

+ 

+     log.info("Successfully updated replication changelog")

+ 

+ # This subcommand is available when 'ds_supports_new_changelog'

+ # that means there is a changelog config entry per backend (aka suffix)

+ def get_per_backend_cl(inst, basedn, log, args):

+     suffix = args.suffix

+     cl = Changelog(inst, suffix)

+     if args and args.json:

+         log.info(cl.get_all_attrs_json())

+     else:

+         log.info(cl.display())

+ 

  

  def create_repl_manager(inst, basedn, log, args):

      manager_name = "replication manager"
@@ -1166,22 +1202,16 @@ 

      repl_get_parser.set_defaults(func=get_repl_config)

      repl_get_parser.add_argument('--suffix', required=True, help='Get the replication configuration for this suffix DN')

  

-     repl_create_cl = repl_subcommands.add_parser('create-changelog', help='Create the replication changelog')

-     repl_create_cl.set_defaults(func=create_cl)

- 

-     repl_delete_cl = repl_subcommands.add_parser('delete-changelog', help='Delete the replication changelog.  This will invalidate any existing replication agreements')

-     repl_delete_cl.set_defaults(func=delete_cl)

+     repl_set_per_backend_cl = repl_subcommands.add_parser('set-changelog', help='Set replication changelog attributes.')

+     repl_set_per_backend_cl.set_defaults(func=set_per_backend_cl)

+     repl_set_per_backend_cl.add_argument('--suffix', required=True, help='The suffix that uses the changelog')

+     repl_set_per_backend_cl.add_argument('--max-entries', help="The maximum number of entries to get in the replication changelog")

+     repl_set_per_backend_cl.add_argument('--max-age', help="The maximum age of a replication changelog entry")

+     repl_set_per_backend_cl.add_argument('--trim-interval', help="The interval to check if the replication changelog can be trimmed")

  

-     repl_set_cl = repl_subcommands.add_parser('set-changelog', help='Set replication changelog attributes.')

-     repl_set_cl.set_defaults(func=set_cl)

-     repl_set_cl.add_argument('--cl-dir', help="The replication changelog location on the filesystem")

-     repl_set_cl.add_argument('--max-entries', help="The maximum number of entries to get in the replication changelog")

-     repl_set_cl.add_argument('--max-age', help="The maximum age of a replication changelog entry")

-     repl_set_cl.add_argument('--compact-interval', help="The replication changelog compaction interval")

-     repl_set_cl.add_argument('--trim-interval', help="The interval to check if the replication changelog can be trimmed")

- 

-     repl_get_cl = repl_subcommands.add_parser('get-changelog', help='Display replication changelog attributes.')

-     repl_get_cl.set_defaults(func=get_cl)

+     repl_get_per_backend_cl = repl_subcommands.add_parser('get-changelog', help='Display replication changelog attributes.')

+     repl_get_per_backend_cl.set_defaults(func=get_per_backend_cl)

+     repl_get_per_backend_cl.add_argument('--suffix', required=True, help='The suffix that uses the changelog')

  

      repl_dump_cl = repl_subcommands.add_parser('dump-changelog', help='Decode Directory Server replication change log and dump it to an LDIF')

      repl_dump_cl.set_defaults(func=dump_cl)
@@ -1197,22 +1227,6 @@ 

                                help="Specify replica roots whose changelog you want to dump. The replica "

                                     "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.")

  

-     repl_restore_cl = repl_subcommands.add_parser('restore-changelog',

-                                                   help='Restore Directory Server replication change log from LDIF file or change log directory')

-     restore_subcommands = repl_restore_cl.add_subparsers(help='Restore Replication Changelog')

-     restore_ldif = restore_subcommands.add_parser('from-ldif', help='Restore a single LDIF file.')

-     restore_ldif.set_defaults(func=restore_cl_ldif)

-     restore_ldif.add_argument('LDIF_PATH', nargs=1, help='The path of changelog LDIF file.')

-     restore_ldif.add_argument('-r', '--replica-root', nargs=1, required=True,

-                               help="Specify one replica root whose changelog you want to restore. "

-                                    "The replica root will be consumed from the LDIF file name if the option is omitted.")

- 

-     restore_changelogdir = restore_subcommands.add_parser('from-changelogdir', help='Restore LDIF files from changelogdir.')

-     restore_changelogdir.set_defaults(func=restore_cl_dir)

-     restore_changelogdir.add_argument('REPLICA_ROOTS', nargs="+",

-                                       help="Specify replica roots whose changelog you want to restore. The replica "

-                                            "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.")

- 

      repl_set_parser = repl_subcommands.add_parser('set', help='Set an attribute in the replication configuration')

      repl_set_parser.set_defaults(func=set_repl_config)

      repl_set_parser.add_argument('--suffix', required=True, help='The DN of the replication suffix')

file modified
+61 -1
@@ -21,7 +21,8 @@ 

  from lib389._constants import *

  from lib389.properties import *

  from lib389.utils import (normalizeDN, escapeDNValue, ensure_bytes, ensure_str,

-                           ensure_list_str, ds_is_older, copy_with_permissions)

+                           ensure_list_str, ds_is_older, copy_with_permissions,

+                           ds_supports_new_changelog)

  from lib389 import DirSrv, Entry, NoSuchEntryError, InvalidArgumentError

  from lib389._mapped_object import DSLdapObjects, DSLdapObject

  from lib389.passwd import password_generate
@@ -1027,6 +1028,65 @@ 

                  encoded_str += line

  

  

+ class Changelog(DSLdapObject):

+     """Represents the Directory Server changelog of a specific backend. This is used for

+     replication.

+ 

+     :param instance: An instance

+     :type instance: lib389.DirSrv

+     """

+     def __init__(self, instance, suffix=None, dn=None):

+         from lib389.backend import Backends

+         super(Changelog, self).__init__(instance, dn)

+         self._rdn_attribute = 'cn'

+         self._create_objectclasses = [

+             'top',

+             'extensibleobject',

+         ]

+         if not ds_supports_new_changelog():

+             raise ValueError('changelog (integrated to main database) is not supported in that version of the server')

+         if not suffix:

+             raise ValueError('A changelog is specific to a suffix and the suffix value is missing')

+ 

+         # retrieve the backend associated to the provided suffix

+         be_insts = Backends(instance).list()

+         found_suffix = False

+         for be in be_insts:

+             be_suffix = be.get_attr_val_utf8_l('nsslapd-suffix')

+             if suffix == be_suffix:

+                 found_suffix = True

+                 break

+         if not found_suffix:

+             raise ValueError(f'No backend associated with nsslapd-suffix "{suffix}"')

+ 

+         # changelog is a child of the backend

+         self._dn = 'cn=changelog,' + be.dn

+ 

+     def set_max_entries(self, value):

+         """Configure the max entries the changelog can hold.

+ 

+         :param value: the number of entries.

+         :type value: str

+         """

+         self.replace('nsslapd-changelogmaxentries', value)

+ 

+     def set_trim_interval(self, value):

+         """The time between changelog trims in seconds.

+ 

+         :param value: The time in seconds

+         :type value: str

+         """

+         self.replace('nsslapd-changelogtrim-interval', value)

+ 

+     def set_max_age(self, value):

+         """The maximum age of entries in the changelog.

+ 

+         :param value: The age with a time modifier of s, m, h, d, w.

+         :type value: str

+         """

+ 

+         self.replace('nsslapd-changelogmaxage', value)

+ 

  class Changelog5(DSLdapObject):

      """Represents the Directory Server changelog. This is used for

      replication. Only one changelog is needed for every server.

Bug description:
PHASE 2 of backend redesign:
http://www.port389.org/docs/389ds/design/integrate-changelog-database-and-backend-database.html
With https://pagure.io/389-ds-base/issue/49562, the changelog uses the main database.
Changelog configuration was managed with a global config entry (cn=changelog5,cn=config)
Now it is managed via a per backend config entry (cn=changelog,cn=<backend_entry>).
Some config parameters are now in the backend specific changelog entry
nsslapd-changelogmaxage
nsslapd-changelogmaxentries
nsslapd-changelogtrim-interval
Some config paramters are simply abandonned (see design):
nsslapd-changelogdir
nsslapd-changelogcompactdb-interval

Fix description:
This fix (PR) is to be applied on top of 49562
It suppressed the ablity to create/delete changelog as the changelog entry
is now created/suppressed when a backend becomes a replica or not.

subcommands to set/get changelog attributes requires a suffix (aka backend).
dsconf <inst> replication set-changelog --suffix <suffix>  --trim-interval <val>
dsconf <inst> replication set-changelog --suffix <suffix>  --max-age <val>
dsconf <inst> replication set-changelog --suffix <suffix>  --max-entries <val>
dsconf <inst> replication get-changelog --suffix <suffix>

This patch removes the ability to restore a changelog (restore-changelog)
It implements a new 'class Changelog' to set/get the configuration attribute
of a per backend changelog

https://pagure.io/389-ds-base/issue/51189

Reviewed by: ?

These errors are printed to the user so it should be more user friendly specifying exactly what they did wrong.
Also, you've already set args.suffix as "required" so this check is not needed.

Maybe, we can name it set_db_cl or something like this? It is new now but it won't be later

The_l suffix here does mean lower already.

The same here, get_new_cl is confusing for both - CLI user and the one who will use Changelog() object in their scripts

Also, I, probably, miss the point here... It looks to me that be will always exist if we have some backend (even not related to the suffix).
You probably want to have something like this:

found_suffix = False
for be in be_insts:
    be_suffix = be.get_attr_val_utf8_l('nsslapd-suffix')
    if suffix == be_suffix:
        found_suffix = True
        break
if not found_suffix:
    raise ValueError()

rebased onto d9aa580

3 years ago

@spichugi thanks for your review. I updated the patch according to your remarks.

I've tried applying both #51181 and this one but dsconf localhost -v -D "cn=directory manager" -w password replication get-changelog --suffix dc=example,dc=com fails for me because of No such object error (it's not able to find cn=changelog,cn=userroot,cn=ldbmdatabase,cn=plugins,cn=config)

Can you suggest what can be wrong?

@spichugi sorry I missed your question. :(

Did you enabled replication (master or hub) before doing get-changelog ?
(like dsconf -D "cn=directory manager" -w password standalone1 replication enable --suffix "dc=example,dc=com" --role master --replica-id 1)

Right... I had an issue with the topology. Now I see it works.
Thanks! LGTM!

Thanks @spichugi for the review.
Need to push it on top of https://pagure.io/389-ds-base/pull-request/51181 and bump a new version. So waiting for the other review

indentation is off - causes flake8 errors. There are 5 spaces instead of 4 starting at "be_suffix = be.get_attr_val_utf8_l('nsslapd-suffix')"

rebased onto 85291b6d3fe5162e94305d10bffa800035d2ada8

3 years ago

@mreynolds patch updated. Waiting for last IPA tests results

rebased onto b9ad60912941c33ded80c73e56c8a2096d7e22c3

3 years ago

rebased onto bf6e486

3 years ago

IPA tests were successful. Pushing upstream that patch

Pull-Request has been merged by tbordaz

3 years ago

389-ds-base is moving from Pagure to Github. This means that new issues and pull requests
will be accepted only in 389-ds-base's github repository.

This pull request has been cloned to Github as issue and is available here:
- https://github.com/389ds/389-ds-base/issues/4248

If you want to continue to work on the PR, please navigate to the github issue,
download the patch from the attachments and file a new pull request.

Thank you for understanding. We apologize for all inconvenience.

Pull-Request has been closed by spichugi

3 years ago