From 8350315740e3b232a69e228f7b37072660128a11 Mon Sep 17 00:00:00 2001 From: Simon Pichugin Date: Jan 07 2020 08:27:25 +0000 Subject: Issue 50754 - Add Restore Change Log option to CLI Description: dsconf can export the changelog but there is no feature to import a changelog dump. Add the feature. Fix replication CLI parsers. Add 'copy_with_permissions' function to lib389.utils. usage: dsconf instance replication restore-changelog [-h] {from-ldif,from-changelogdir} positional arguments: {from-ldif,from-changelogdir} Replication Configuration from-ldif Restore a single LDIF file. from-changelogdir Restore LDIF files from changelogdir. https://pagure.io/389-ds-base/issue/50754 Reviewed by: mreynolds (Thanks!) --- diff --git a/src/lib389/lib389/cli_conf/replication.py b/src/lib389/lib389/cli_conf/replication.py index 47f530a..ede2529 100644 --- a/src/lib389/lib389/cli_conf/replication.py +++ b/src/lib389/lib389/cli_conf/replication.py @@ -11,10 +11,12 @@ import logging import os import json import ldap +import stat +from shutil import copyfile from getpass import getpass from lib389._constants import ReplicaRole, DSRC_HOME from lib389.cli_base.dsrc import dsrc_to_repl_monitor -from lib389.utils import is_a_dn +from lib389.utils import is_a_dn, copy_with_permissions from lib389.replica import Replicas, ReplicationMonitor, BootstrapReplicationManager, Changelog5, ChangelogLDIF from lib389.tasks import CleanAllRUVTask, AbortCleanAllRUVTask from lib389._mapped_object import DSLdapObjects @@ -1014,6 +1016,39 @@ def dump_cl(inst, basedn, log, args): cl_ldif.decode() +def restore_cl_ldif(inst, basedn, log, args): + user_ldif = os.path.abspath(args.LDIF_PATH[0]) + try: + assert os.path.exists(user_ldif) + except AssertionError: + raise FileNotFoundError(f"File {args.LDIF_PATH[0]} was not found") + + replicas = Replicas(inst) + replica = replicas.get(args.replica_root[0]) + replica_name = replica.get_attr_val_utf8_l("nsDS5ReplicaName") + target_ldif = f'{replica_name}.ldif' + target_ldif_exists = os.path.exists(target_ldif) + cl = Changelog5(inst) + cl_dir = cl.get_attr_val_utf8_l("nsslapd-changelogdir") + cl_dir_file = [i.lower() for i in os.listdir(cl_dir) if i.lower().startswith(replica_name)][0] + cl_dir_stat = os.stat(os.path.join(cl_dir, cl_dir_file)) + # Make sure we don't remove existing files + if target_ldif_exists: + copy_with_permissions(target_ldif, f'{target_ldif}.backup') + copyfile(user_ldif, target_ldif) + os.chown(target_ldif, cl_dir_stat[stat.ST_UID], cl_dir_stat[stat.ST_GID]) + os.chmod(target_ldif, cl_dir_stat[stat.ST_MODE]) + replicas.restore_changelog(replica_roots=args.replica_root, log=log) + os.remove(target_ldif) + if target_ldif_exists: + os.rename(f'{target_ldif}.backup', target_ldif) + + +def restore_cl_dir(inst, basedn, log, args): + replicas = Replicas(inst) + replicas.restore_changelog(replica_roots=args.REPLICA_ROOTS, log=log) + + def create_parser(subparsers): ############################################ @@ -1104,19 +1139,35 @@ def create_parser(subparsers): repl_get_cl = repl_subcommands.add_parser('get-changelog', help='Display replication changelog attributes.') repl_get_cl.set_defaults(func=get_cl) - repl_set_cl = repl_subcommands.add_parser('dump-changelog', help='Decode Directory Server replication change log and dump it to an LDIF') - repl_set_cl.set_defaults(func=dump_cl) - repl_set_cl.add_argument('-c', '--csn-only', action='store_true', - help="Dump and interpret CSN only. This option can be used with or without -i option.") - repl_set_cl.add_argument('-l', '--preserve-ldif-done', action='store_true', - help="Preserve generated ldif.done files from changelogdir.") - repl_set_cl.add_argument('-i', '--changelog-ldif', - help="If you already have a ldif-like changelog, but the changes in that file are encoded," - " you may use this option to decode that ldif-like changelog. It should be base64 encoded.") - repl_set_cl.add_argument('-o', '--output-file', help="Path name for the final result. Default to STDOUT if omitted.") - repl_set_cl.add_argument('-r', '--replica-roots', nargs="+", - help="Specify replica roots whose changelog you want to dump. The replica " - "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.") + repl_dump_cl = repl_subcommands.add_parser('dump-changelog', help='Decode Directory Server replication change log and dump it to an LDIF') + repl_dump_cl.set_defaults(func=dump_cl) + repl_dump_cl.add_argument('-c', '--csn-only', action='store_true', + help="Dump and interpret CSN only. This option can be used with or without -i option.") + repl_dump_cl.add_argument('-l', '--preserve-ldif-done', action='store_true', + help="Preserve generated ldif.done files from changelogdir.") + repl_dump_cl.add_argument('-i', '--changelog-ldif', + help="If you already have a ldif-like changelog, but the changes in that file are encoded," + " you may use this option to decode that ldif-like changelog. It should be base64 encoded.") + repl_dump_cl.add_argument('-o', '--output-file', help="Path name for the final result. Default to STDOUT if omitted.") + repl_dump_cl.add_argument('-r', '--replica-roots', nargs="+", + help="Specify replica roots whose changelog you want to dump. The replica " + "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.") + + repl_restore_cl = repl_subcommands.add_parser('restore-changelog', + help='Restore Directory Server replication change log from LDIF file or change log directory') + restore_subcommands = repl_restore_cl.add_subparsers(help='Restore Replication Changelog') + restore_ldif = restore_subcommands.add_parser('from-ldif', help='Restore a single LDIF file.') + restore_ldif.set_defaults(func=restore_cl_ldif) + restore_ldif.add_argument('LDIF_PATH', nargs=1, help='The path of changelog LDIF file.') + restore_ldif.add_argument('-r', '--replica-root', nargs=1, required=True, + help="Specify one replica root whose changelog you want to restore. " + "The replica root will be consumed from the LDIF file name if the option is omitted.") + + restore_changelogdir = restore_subcommands.add_parser('from-changelogdir', help='Restore LDIF files from changelogdir.') + restore_changelogdir.set_defaults(func=restore_cl_dir) + restore_changelogdir.add_argument('REPLICA_ROOTS', nargs="+", + help="Specify replica roots whose changelog you want to restore. The replica " + "roots may be seperated by comma. All the replica roots would be dumped if the option is omitted.") repl_set_parser = repl_subcommands.add_parser('set', help='Set an attribute in the replication configuration') repl_set_parser.set_defaults(func=set_repl_config) diff --git a/src/lib389/lib389/replica.py b/src/lib389/lib389/replica.py index 79ebd98..c026ff7 100644 --- a/src/lib389/lib389/replica.py +++ b/src/lib389/lib389/replica.py @@ -16,11 +16,13 @@ import logging import uuid import json import copy +from shutil import copyfile from operator import itemgetter from itertools import permutations from lib389._constants import * from lib389.properties import * -from lib389.utils import normalizeDN, escapeDNValue, ensure_bytes, ensure_str, ensure_list_str, ds_is_older +from lib389.utils import (normalizeDN, escapeDNValue, ensure_bytes, ensure_str, + ensure_list_str, ds_is_older, copy_with_permissions) from lib389 import DirSrv, Entry, NoSuchEntryError, InvalidArgumentError from lib389._mapped_object import DSLdapObjects, DSLdapObject from lib389.passwd import password_generate @@ -1594,6 +1596,11 @@ class Replica(DSLdapObject): """ self.replace('nsds5task', 'cl2ldif') + def begin_task_ldif2cl(self): + """Begin ldif to changelog task + """ + self.replace('nsds5task', 'ldif2cl') + def get_suffix(self): """Return the suffix """ @@ -1656,12 +1663,16 @@ class Replicas(DSLdapObjects): return replica def process_and_dump_changelog(self, replica_roots=[], csn_only=False, preserve_ldif_done=False, log=None): - """Dump and decode Directory Server replication change log + """Dump and decode Directory Server replication changelog :param replica_roots: Replica suffixes that need to be processed :type replica_roots: list of str :param csn_only: Grep only the CSNs from the file :type csn_only: bool + :param preserve_ldif_done: Preserve the result LDIF and rename it to [old_name].done + :type preserve_ldif_done: bool + :param log: The logger object + :type log: logger """ if log is None: @@ -1688,7 +1699,7 @@ class Replicas(DSLdapObjects): current_time = time.time() replica = self.get(repl_root) log.info(f"# Replica Root: {repl_root}") - replica.replace("nsDS5Task", 'CL2LDIF') + replica.begin_task_cl2ldif() # Decode the dumped changelog for file in [i for i in os.listdir(cl_dir) if i.endswith('.ldif')]: @@ -1712,6 +1723,53 @@ class Replicas(DSLdapObjects): if not got_ldif: log.info("LDIF file: Not found") + def restore_changelog(self, replica_roots, log=None): + """Restore Directory Server replication changelog from '.ldif' or '.ldif.done' file + + :param replica_roots: Replica suffixes that need to be processed (and optional LDIF file path) + :type replica_roots: list of str + :param log: The logger object + :type log: logger + """ + + if log is None: + log = self._log + + try: + cl = Changelog5(self._instance) + cl_dir = cl.get_attr_val_utf8_l("nsslapd-changelogdir") + except ldap.NO_SUCH_OBJECT: + raise ValueError("Changelog entry was not found. Probably, the replication is not enabled on this instance") + + # Get all the replicas on the server if --replica-roots option is not specified + if not replica_roots: + raise ValueError("List of replication roots should be supplied") + + # Dump the changelog for the replica + for repl_root in replica_roots: + replica = self.get(repl_root) + replica_name = replica.get_attr_val_utf8_l("nsDS5ReplicaName") + cl_dir_content = os.listdir(cl_dir) + changelog_ldif = [i.lower() for i in cl_dir_content if i.lower() == f"{replica_name}.ldif"] + changelog_ldif_done = [i.lower() for i in cl_dir_content if i.lower() == f"{replica_name}.ldif.done"] + + if changelog_ldif: + replica.begin_task_cl2ldif() + elif changelog_ldif_done: + ldif_done_file = os.path.join(cl_dir, changelog_ldif_done[0]) + ldif_file = os.path.join(cl_dir, f"{replica_name}.ldif") + ldif_file_exists = os.path.exists(ldif_file) + if ldif_file_exists: + copy_with_permissions(ldif_file, f'{ldif_file}.backup') + copy_with_permissions(ldif_done_file, ldif_file) + replica.begin_task_cl2ldif() + os.remove(ldif_file) + if ldif_file_exists: + os.rename(f'{ldif_file}.backup', ldif_file) + else: + log.error(f"Changelog LDIF for '{repl_root}' was not found") + continue + class BootstrapReplicationManager(DSLdapObject): """A Replication Manager credential for bootstrapping the repl process. diff --git a/src/lib389/lib389/utils.py b/src/lib389/lib389/utils.py index f71b1af..a2e57c1 100644 --- a/src/lib389/lib389/utils.py +++ b/src/lib389/lib389/utils.py @@ -31,6 +31,7 @@ import shutil import ldap import socket import time +import stat from datetime import datetime import sys import filecmp @@ -1367,3 +1368,12 @@ def print_nice_time(seconds): return f'{m:d} minute{m_plural}, {s:d} second{s_plural}' else: return f'{s:d} second{s_plural}' + + +def copy_with_permissions(source, target): + """Copy a file while preserving all permissions""" + + shutil.copy2(source, target) + st = os.stat(source) + os.chown(target, st[stat.ST_UID], st[stat.ST_GID]) + os.chmod(target, st[stat.ST_MODE])