From 326be2c9eb6411a3b08987dc07f064983b8a3415 Mon Sep 17 00:00:00 2001 From: Mark Reynolds Date: Apr 24 2020 14:12:56 +0000 Subject: Issue 50545 - Port dbgen.pl to dsctl Description: Ported the main features to lib389 and added some other useful features: Now there are several LDIFs that can be created: - User LDIFs (different types) - Group LDIFs - COS LDIFs - Role LDIFs - Modification LDIFs - Nested LDIFs Design Doc: https://www.port389.org/docs/389ds/design/dbgen-design.html fixes: https://pagure.io/389-ds-base/issue/50545 Reviewed by: firstyear & spichugi(Thanks!!) Fix various issue and improve ldif file validation Add summary of settings to output, and set the default location of user/nested LDIF to be in the server's LDIF directory --- diff --git a/dirsrvtests/tests/stress/search/simple.py b/dirsrvtests/tests/stress/search/simple.py index 8222cb7..d745ff4 100644 --- a/dirsrvtests/tests/stress/search/simple.py +++ b/dirsrvtests/tests/stress/search/simple.py @@ -1,5 +1,6 @@ # --- BEGIN COPYRIGHT BLOCK --- # Copyright (C) 2019 William Brown +# Copyright (C) 2020 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). @@ -8,16 +9,15 @@ # from lib389.topologies import topology_st -from lib389.dbgen import dbgen +from lib389.dbgen import dbgen_users from lib389.ldclt import Ldclt from lib389.tasks import ImportTask - from lib389._constants import DEFAULT_SUFFIX def test_stress_search_simple(topology_st): """Test a simple stress test of searches on the directory server. - + :id: 3786d01c-ea03-4655-a4f9-450693c75863 :setup: Standalone Instance :steps: @@ -31,7 +31,6 @@ def test_stress_search_simple(topology_st): """ inst = topology_st.standalone - inst.config.set("nsslapd-verify-filter-schema", "off") # Bump idllimit to test OR worst cases. from lib389.config import LDBMConfig @@ -39,10 +38,9 @@ def test_stress_search_simple(topology_st): # lconfig.set("nsslapd-idlistscanlimit", '20000') # lconfig.set("nsslapd-lookthroughlimit", '20000') - ldif_dir = inst.get_ldif_dir() import_ldif = ldif_dir + '/basic_import.ldif' - dbgen(inst, 10000, import_ldif, DEFAULT_SUFFIX) + dbgen_users(inst, 10000, import_ldif, DEFAULT_SUFFIX) r = ImportTask(inst) r.import_suffix_from_ldif(ldiffile=import_ldif, suffix=DEFAULT_SUFFIX) @@ -50,10 +48,8 @@ def test_stress_search_simple(topology_st): # Run a small to warm up the server's caches ... l = Ldclt(inst) - l.search_loadtest(DEFAULT_SUFFIX, "(mail=XXXX@example.com)", rounds=1) # Now do it for realsies! # l.search_loadtest(DEFAULT_SUFFIX, "(|(mail=XXXX@example.com)(nonexist=foo))", rounds=10) l.search_loadtest(DEFAULT_SUFFIX, "(mail=XXXX@example.com)", rounds=10) - diff --git a/dirsrvtests/tests/suites/basic/basic_test.py b/dirsrvtests/tests/suites/basic/basic_test.py index 40f95a4..1202073 100644 --- a/dirsrvtests/tests/suites/basic/basic_test.py +++ b/dirsrvtests/tests/suites/basic/basic_test.py @@ -1,5 +1,5 @@ # --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2016 Red Hat, Inc. +# Copyright (C) 2020 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). @@ -18,10 +18,9 @@ import pytest from lib389.tasks import * from lib389.utils import * from lib389.topologies import topology_st -from lib389.dbgen import dbgen +from lib389.dbgen import dbgen_users from lib389.idm.organizationalunit import OrganizationalUnits from lib389._constants import DN_DM, PASSWORD, PW_DM -from lib389.topologies import topology_st from lib389.paths import Paths from lib389.idm.directorymanager import DirectoryManager from lib389.config import LDBMConfig @@ -266,7 +265,7 @@ def test_basic_import_export(topology_st, import_example_ldif): log.info("Generating LDIF...") ldif_dir = topology_st.standalone.get_ldif_dir() import_ldif = ldif_dir + '/basic_import.ldif' - dbgen(topology_st.standalone, 50000, import_ldif, DEFAULT_SUFFIX) + dbgen_users(topology_st.standalone, 50000, import_ldif, DEFAULT_SUFFIX) # Online log.info("Importing LDIF online...") diff --git a/dirsrvtests/tests/suites/import/regression_test.py b/dirsrvtests/tests/suites/import/regression_test.py index 7be9e39..e78d338 100644 --- a/dirsrvtests/tests/suites/import/regression_test.py +++ b/dirsrvtests/tests/suites/import/regression_test.py @@ -1,22 +1,23 @@ -# Copyright (C) 2017 Red Hat, Inc. +# Copyright (C) 2020 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). # See LICENSE for details. # --- END COPYRIGHT BLOCK --- # +import ldap +import logging +import os import pytest +import threading +import time from lib389.backend import Backends from lib389.properties import TASK_WAIT -from lib389.utils import time, ldap, os, logging from lib389.topologies import topology_st as topo -from lib389.dbgen import dbgen +from lib389.dbgen import dbgen_users from lib389._constants import DEFAULT_SUFFIX from lib389.tasks import * from lib389.idm.user import UserAccounts -import threading -import time - from lib389.idm.directorymanager import DirectoryManager pytestmark = pytest.mark.tier1 @@ -145,7 +146,7 @@ def test_import_be_default(topo): log.info('Create LDIF file and import it...') ldif_dir = topo.standalone.get_ldif_dir() ldif_file = os.path.join(ldif_dir, 'default.ldif') - dbgen(topo.standalone, 5, ldif_file, TEST_DEFAULT_SUFFIX) + dbgen_users(topo.standalone, 5, ldif_file, TEST_DEFAULT_SUFFIX) log.info('Stopping the server and running offline import...') topo.standalone.stop() @@ -185,7 +186,7 @@ def test_del_suffix_import(topo): ldif_dir = topo.standalone.get_ldif_dir() ldif_file = os.path.join(ldif_dir, 'suffix_del1.ldif') - dbgen(topo.standalone, 10, ldif_file, TEST_SUFFIX1) + dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX1) log.info('Stopping the server and running offline import') topo.standalone.stop() @@ -223,7 +224,7 @@ def test_del_suffix_backend(topo): ldif_dir = topo.standalone.get_ldif_dir() ldif_file = os.path.join(ldif_dir, 'suffix_del2.ldif') - dbgen(topo.standalone, 10, ldif_file, TEST_SUFFIX2) + dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX2) topo.standalone.tasks.importLDIF(suffix=TEST_SUFFIX2, input_file=ldif_file, args={TASK_WAIT: True}) diff --git a/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py b/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py index 730969a..9f7e3e3 100644 --- a/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py +++ b/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py @@ -1,5 +1,5 @@ # --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2017 Red Hat, Inc. +# Copyright (C) 2020 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). @@ -9,12 +9,10 @@ import ldap import pytest from lib389.topologies import topology_m2 -from lib389._constants import (DEFAULT_SUFFIX, HOST_MASTER_2, PORT_MASTER_2, TASK_WAIT) +from lib389._constants import (DEFAULT_SUFFIX) from lib389.agreement import Agreements - from lib389.idm.user import (TEST_USER_PROPERTIES, UserAccounts) - -from lib389.dbgen import dbgen +from lib389.dbgen import dbgen_users from lib389.utils import ds_is_older pytestmark = pytest.mark.tier1 @@ -26,17 +24,15 @@ def test_referral_during_tot(topology_m2): master2 = topology_m2.ms["master2"] users = UserAccounts(master2, DEFAULT_SUFFIX) - u = users.create(properties=TEST_USER_PROPERTIES) u.set('userPassword', 'password') - binddn = u.dn bindpw = 'password' # Create a bunch of entries on master1 ldif_dir = master1.get_ldif_dir() import_ldif = ldif_dir + '/ref_during_tot_import.ldif' - dbgen(master1, 10000, import_ldif, DEFAULT_SUFFIX) + dbgen_users(master1, 10000, import_ldif, DEFAULT_SUFFIX) master1.stop() master1.ldif2db(bename=None, excludeSuffixes=None, encrypt=False, suffixes=[DEFAULT_SUFFIX], import_file=import_ldif) @@ -61,9 +57,7 @@ def test_referral_during_tot(topology_m2): except ldap.REFERRAL: referred = True break - # Means we never go a referral, should not happen! + # Means we never go a referral, should not happen! assert referred # Done. - - diff --git a/dirsrvtests/tests/suites/syntax/mr_test.py b/dirsrvtests/tests/suites/syntax/mr_test.py index f622b75..a7fa5a9 100644 --- a/dirsrvtests/tests/suites/syntax/mr_test.py +++ b/dirsrvtests/tests/suites/syntax/mr_test.py @@ -1,8 +1,17 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2020 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- +# + import logging import pytest import os import ldap -from lib389.dbgen import dbgen +from lib389.dbgen import dbgen_users from lib389._constants import * from lib389.topologies import topology_st as topo from lib389._controls import SSSRequestControl @@ -33,7 +42,7 @@ def test_sss_mr(topo): log.info("Creating LDIF...") ldif_dir = topo.standalone.get_ldif_dir() ldif_file = os.path.join(ldif_dir, 'mr-crash.ldif') - dbgen(topo.standalone, 5, ldif_file, DEFAULT_SUFFIX) + dbgen_users(topo.standalone, 5, ldif_file, DEFAULT_SUFFIX) log.info("Importing LDIF...") topo.standalone.stop() diff --git a/src/lib389/cli/dsctl b/src/lib389/cli/dsctl index f3bbabc..fd9bd87 100755 --- a/src/lib389/cli/dsctl +++ b/src/lib389/cli/dsctl @@ -12,7 +12,6 @@ import json import argparse, argcomplete -import logging import sys import signal import os @@ -23,9 +22,9 @@ from lib389.cli_ctl import dbtasks as cli_dbtasks from lib389.cli_ctl import tls as cli_tls from lib389.cli_ctl import health as cli_health from lib389.cli_ctl import nsstate as cli_nsstate +from lib389.cli_ctl import dbgen as cli_dbgen from lib389.cli_ctl.instance import instance_remove_all from lib389.cli_base import ( - _get_arg, disconnect_instance, setup_script_logger, format_error_to_dict) @@ -61,6 +60,7 @@ cli_dbtasks.create_parser(subparsers) cli_tls.create_parser(subparsers) cli_health.create_parser(subparsers) cli_nsstate.create_parser(subparsers) +cli_dbgen.create_parser(subparsers) argcomplete.autocomplete(parser) diff --git a/src/lib389/lib389/cli_conf/chaining.py b/src/lib389/lib389/cli_conf/chaining.py index 5125898..380ec16 100644 --- a/src/lib389/lib389/cli_conf/chaining.py +++ b/src/lib389/lib389/cli_conf/chaining.py @@ -8,7 +8,7 @@ import json from lib389.chaining import ( - ChainingLink, ChainingLinks, ChainingConfig, ChainingDefault) + ChainingLinks, ChainingConfig, ChainingDefault) from lib389.cli_base import ( _generic_list, _generic_get, diff --git a/src/lib389/lib389/cli_ctl/dbgen.py b/src/lib389/lib389/cli_ctl/dbgen.py new file mode 100644 index 0000000..7bc3892 --- /dev/null +++ b/src/lib389/lib389/cli_ctl/dbgen.py @@ -0,0 +1,587 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2020 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +from lib389.dbgen import ( + dbgen_users, + dbgen_groups, + dbgen_cos_def, + dbgen_cos_template, + dbgen_role, + dbgen_mod_load, + dbgen_nested_ldif, +) +from lib389.utils import is_a_dn + +DEFAULT_LDIF = "/tmp/ldifgen.ldif" +USERS_LDIF_NAME = "/users.ldif" +ignore_args = [ + "ldif_file", + "func", + "verbose", + "list", + "instance", + "json", + "remove_all", +] + +def get_ldif_dir(instance): + """ + Get the server's LDIF directory. This is only used for user & nested LDIFs + """ + server_dir = instance.get_ldif_dir() + if server_dir is not None: + return server_dir + return DEFAULT_LDIF + + +def adjust_ldif_name(instance, ldif_name): + """ + If just a name is provided append it to the server's LDIF directory + """ + if ldif_name[0] == '.' or ldif_name[0] == '/': + # Name appears to already be an absolute path + return ldif_name + else: + # Its just a name, add the server's ldif directory + return instance.get_ldif_dir() + "/" + ldif_name + + +def display_args(log, args): + # Display all the options that are being used to generate the ldif file + log.info(f"\nGenerating LDIF with the following options:") + for k, v in vars(args).items(): + if k in ignore_args or v is None: + continue + k = k.replace("_", "-") # Restore arg's original name + log.info(f" - {k}={v}") + log.info(f" - ldif-file={args.ldif_file}") + log.info("\nWriting LDIF ...") + + +def validate_ldif_file(ldif_file, log=None): + """ + Check if the LDIF file exists. If interactive then return some error + text to the caller, otherwise raise an error. + """ + try: + f = open(ldif_file, 'w') + f.close() + return True + except PermissionError: + # File might already exist + msg = f"The LDIF file ({ldif_file}) exists and can not be overwritten. Please choose a different file name." + if log is not None: + log.info(msg) + else: + raise ValueError(msg) + except FileNotFoundError: + msg = f"The LDIF file ({ldif_file}) location does not exist. Please choose a different location." + if log is not None: + log.info(msg) + else: + raise ValueError(msg) + except Exception as e: + msg = f"The LDIF file ({ldif_file}) can not be written: {str(e)}" + if log is not None: + log.info(msg) + else: + raise ValueError(msg) + + return False + + +def get_ldif_file_input(log, default_name=DEFAULT_LDIF): + valid = False + while not valid: + file_name = get_input(log, "Enter the new LDIF file name", default_name) + valid = validate_ldif_file(file_name, log=log) + return file_name + + +def get_input(log, msg, default, type="", options=None): + # Interactive prompt + display_default = default + if isinstance(default, bool): + if default: + display_default = "yes" + else: + display_default = "no" + while 1: + if display_default != "": + val = input(f'\n{msg} [{display_default}]: ') + else: + val = input(f'\n{msg}: ') + if val != '': + if type == "dn": + if is_a_dn(val, allow_anon=False): + return val + else: + log.info(f"\n ---> The value you entered \"{val}\" is not a valid DN") + continue + elif type == "int": + if val.isdigit(): + if int(val) < 1: + log.info("\n ---> You must enter number greater than 0") + continue + return int(val) + else: + log.info(f"\n ---> The number you entered \"{val}\" is not a number") + continue + elif type == "bool": + if val.lower() == "y" or val.lower() == "yes": + return True + elif val.lower() == "n" or val.lower() == "no": + return False + else: + log.info(f"\n ---> Invalid value ({val}), please enter \"yes\" or \"no\".") + continue + else: + # Just a string, nothing to validate + if options is not None: + if val.lower() not in options: + opt_str = ', '.join(options) + log.info(f"\n ---> Invalid value ({val}), please enter one of the following types: {opt_str}") + continue + return val + else: + # User selected the default value + return default + + +def dbgen_create_users(inst, log, args): + """ + Create a LDIF of user entries + """ + if args.number is None or args.suffix is None: + """ + Interactively get all the info ... + """ + log.info("Missing required parameters, switching to Interactive mode ...") + + # Get the suffix + args.suffix = get_input(log, "Enter the suffix", "dc=example,dc=com", "dn") + + # Get the parent + args.parent = get_input(log, "Enter the parent entry for the users", "ou=people,dc=example,dc=com", "dn") + + # Get the number of users to create + args.number = get_input(log, "Enter the number of users to create", 100000, "int") + + # Confirm the RDN attribute + args.rdn_cn = get_input(log, "Do you want to use \"cn\" instead of \"uid\" for the entry RDN attribute (yes/no)", False, "bool") + + # Create generic entries + args.generic = get_input(log, "Create generic entries that can be used with \"ldclt\" (yes/no)", False, "bool") + + # get offset size + if args.generic: + args.start_idx = get_input(log, "Choose the starting index for the generic user entries", 0, "int") + + # localize the data + args.localize = get_input(log, "Do you want to localize the LDIF data (yes/no)", False, "bool") + + # Get the output LDIF file name + args.ldif_file = get_ldif_file_input(log, default_name=get_ldif_dir(inst) + USERS_LDIF_NAME) + else: + args.ldif_file = adjust_ldif_name(inst, args.ldif_file) + validate_ldif_file(args.ldif_file) + + display_args(log, args) + dbgen_users(inst, args.number, args.ldif_file, args.suffix, generic=args.generic, parent=args.parent, startIdx=args.start_idx, rdnCN=False, pseudol10n=args.localize) + log.info(f"Successfully created LDIF file: {args.ldif_file}") + + +def dbgen_create_groups(inst, log, args): + """ + Create static groups and their members + """ + + if args.number is None or args.suffix is None: + """ + Interactively get all the info ... + """ + log.info("Missing required parameters, switching to Interactive mode ...") + + # Get the number of users to create + args.number = get_input(log, "Enter the number of groups to create", 1, "int") + + # Get the suffix + args.suffix = get_input(log, "Enter the suffix", "dc=example,dc=com", "dn") + + # Get the parent + args.parent = get_input(log, "Enter the parent entry to add the groups under", args.suffix, "dn") + + # Get the membership attr + args.member_attr = get_input(log, "Enter the attribute to use for the group membership", "uniquemember") + + # Number of members + args.num_members = get_input(log, "Enter the number of members to add to the group", 10000, "int") + + # Create member entries + args.create_members = get_input(log, "Do you want to create the member entries (yes/no)", True, "bool") + + # member entries parent + if args.create_members: + args.member_parent = get_input(log, "Enter the parent entry to add the users under", args.suffix, "dn") + + # Get the output LDIF file name + args.ldif_file = get_ldif_file_input(log) + else: + validate_ldif_file(args.ldif_file) + + props = { + "name": args.NAME, + "parent": args.parent, + "suffix": args.suffix, + "number": args.number, + "numMembers": args.num_members, + "createMembers": args.create_members, + "memberParent": args.member_parent, + "membershipAttr": args.member_attr, + } + + display_args(log, args) + dbgen_groups(inst, args.ldif_file, props) + log.info(f"Successfully created LDIF file: {args.ldif_file}") + + +def dbgen_create_cos_def(inst, log, args): + """ + Create a COS definition + """ + if args.type is None or args.parent is None or len(args.cos_attr) == 0 or \ + ((args.type == "classic" or args.type == "indirect") and args.cos_specifier is None) \ + or ((args.type == "classic" or args.type == "pointer") and args.cos_template is None): + log.info("Missing required parameters, switching to Interactive mode ...") + + # Get the number of users to create + args.type = get_input(log, "Type of COS definition: \"classic\", \"pointer\", or \"indirect\"", + "classic", options=["classic", "pointer", "indirect"]) + + # Get the parent + args.parent = get_input(log, "Enter the parent entry to add the COS definition under", "dc=example,dc=com", "dn") + + # Create parent + args.create_members = get_input(log, "Do you want to create the parent entry (yes/no)", True, "bool") + + # COS specifier + if args.type == "classic" or args.type == "indirect": + args.cos_specifier = get_input(log, "Enter the COS specifier attribute", "description") + + # COS template DN + if args.type == "classic" or args.type == "pointer": + args.cos_template = get_input(log, "Enter the COS Template DN", "cn=COS Template Entry,dc=example,dc=com", "dn") + + # Gather the COS attributes + while True: + val = get_input(log, "Enter COS attributes, press Enter when finished", "") + if val == "" and len(args.cos_attr) > 0: + break + args.cos_attr.append(val) + + # Get the output LDIF file name + args.ldif_file = get_ldif_file_input(log) + else: + validate_ldif_file(args.ldif_file) + + props = { + "cosType": args.type, + "defName": args.NAME, + "defParent": args.parent, + "defCreateParent": args.create_parent, + "cosSpecifier": args.cos_specifier, + "cosAttrs": args.cos_attr, + "tmpName": args.cos_template + } + + display_args(log, args) + dbgen_cos_def(inst, args.ldif_file, props) + log.info(f"Successfully created LDIF file: {args.ldif_file}") + + +def dbgen_create_cos_tmp(inst, log, args): + """ + Create a COS template entry + """ + if args.parent is None or args.cos_priority is None or args.cos_attr_val is None: + log.info("Missing required parameters, switching to Interactive mode ...") + + # Get the parent + args.parent = get_input(log, "Enter the parent entry to add the COS template under", "dc=example,dc=com", "dn") + + # Create parent + args.create_parent = get_input(log, "Do you want to create the parent entry (yes/no)", True, "bool") + + # Get the COS priority + args.cos_priority = get_input(log, "Enter the COS priority for this template", "0", "int") + + # Get the attribute value pair + args.cos_attr_val = get_input(log, "Enter the attribute and value pair. Use this format: \"ATTRIBUTE:VALUE\"", "postalcode:19605") + + # Get the output LDIF file name + args.ldif_file = get_ldif_file_input(log) + else: + validate_ldif_file(args.ldif_file) + + props = { + "tmpName": args.NAME, + "tmpParent": args.parent, + "tmpCreateParent": args.create_parent, + "cosPriority": args.cos_priority, + "cosTmpAttrVal": args.cos_attr_val, + } + + display_args(log, args) + dbgen_cos_template(inst, args.ldif_file, props) + log.info(f"Successfully created LDIF file: {args.ldif_file}") + + +def dbgen_create_role(inst, log, args): + """ + Create a Role + """ + if args.type is None or args.parent is None or \ + (args.type == "filtered" and args.filter is None) or \ + (args.type == "nested" and len(args.role_dn) == 0): + log.info("Missing required parameters, switching to Interactive mode ...") + + # Get the number of users to create + args.type = get_input(log, "Type of Role: \"managed\", \"filtered\", or \"nested\"", + "managed", options=["managed", "filtered", "nested"]) + + # Get the parent + args.parent = get_input(log, "Enter the parent entry to add the Role under", "dc=example,dc=com", "dn") + + # Create parent + args.create_parent = get_input(log, "Do you want to create the parent entry (yes/no)", True, "bool") + + # Role filter + if args.type == "filtered": + args.filter = get_input(log, "Enter the Role filter", "cn=some_value") + + # Role DN (nested only) + if args.type == "nested": + while True: + val = get_input(log, "Enter the Role DN", "cn=some other role,dc=example,dc=com", "dn") + if val == "" and len(args.role_dn) > 0: + break + args.role_dn.append(val) + + # Get the output LDIF file name + args.ldif_file = get_ldif_file_input(log) + else: + validate_ldif_file(args.ldif_file) + + props = { + "role_type": args.type, + "role_name": args.NAME, + "parent": args.parent, + "createParent": args.create_parent, + "filter": args.filter, + "role_list": args.role_dn + } + + display_args(log, args) + dbgen_role(inst, args.ldif_file, props) + log.info(f"Successfully created LDIF file: {args.ldif_file}") + + +def dbgen_create_mods(inst, log, args): + """ + Create a LDIF file of update operations that can be consumed by ldapmodify + + There are a lot of options here for creating different types of modification + LDIFs. One technique/option is that you can work with existing users. Use + dbgen to generate a large generic user ldif, import it, then you can create + a modification LDIF that will work on that database. It's a lot faster than + using ldapmodify to add 1 million first then modify those entries. + + The other nice option is that you can randomize the operations, but this does + introduce a potential for some of the operations to fail. You could delete + an entry before you modify it, etc... + """ + + if args.num_users is None or args.parent is None: + log.info("Missing required parameters, switching to Interactive mode ...") + + # Create users + args.create_users = get_input(log, "Do you want to create the user entries (yes/no)", True, "bool") + + # Delete users + args.delete_users = get_input(log, "Do you want to delete all the user entries at the end (yes/no)", True, "bool") + + # Get the number of entries + args.num_users = get_input(log, "Enter the number of user entries that can be modified", "100000", "int") + + # Get the parent entry + args.parent = get_input(log, "The DN of the parent entry where the user entries are located", "ou=people,dc=example,dc=com") + + # Create parent + args.create_parent = get_input(log, "Create the parent entry (yes/no)", True, "bool") + + # Add users + args.add_users = get_input(log, "The number of users to add during the load", 0, "int") + + # Delete users + args.del_users = get_input(log, "The number of users to delete during the load", 0, "int") + + # modrdn users + args.modrdn_users = get_input(log, "The number of users to modrdn during the load", 0, "int") + + # modify users + args.mod_users = get_input(log, "The number of users to modify during the load", 0, "int") + + # Modification attributes + args.mod_attrs = get_input(log, "List of attributes that will be randomly chosen from when modifying an entry", "description cn") + args.mod_attrs = args.mod_attrs.split(' ') + + # Randomize the load + args.randomize = get_input(log, "Randomly perform the specified add, mod, delete, and modrdn operations (yes/no)", True, "bool") + + # Get the output LDIF file name + args.ldif_file = get_ldif_file_input(log) + else: + validate_ldif_file(args.ldif_file) + + props = { + "createUsers": args.create_users, + "deleteUsers": args.delete_users, + "numUsers": args.num_users, + "parent": args.parent, + "createParent": args.create_parent, + "addUsers": args.add_users, + "delUsers": args.del_users, + "modrdnUsers": args.modrdn_users, + "modUsers": args.mod_users, + "random": args.randomize, + "modAttrs": args.mod_attrs + } + + display_args(log, args) + dbgen_mod_load(args.ldif_file, props) + log.info(f"Successfully created LDIF file: {args.ldif_file}") + + +def dbgen_create_nested(inst, log, args): + """ + Create a cascading/fractal tree. Every node splits in half and keeps + branching out in that fashion until all the entries are used up. + """ + + if args.num_users is None or args.node_limit is None or args.suffix is None: + # Num users + args.num_users = get_input(log, "The number of users to add during the load", 1000000, "int") + + # Node limit + args.node_limit = get_input(log, "The total number of user entries to create under each node/subtree", 500, "int") + + # Suffix + args.suffix = get_input(log, "Enter the suffix", "dc=example,dc=com", "dn") + + # Get the output LDIF file name + args.ldif_file = get_ldif_file_input(log, default_name=get_ldif_dir(inst) + USERS_LDIF_NAME) + else: + args.ldif_file = adjust_ldif_name(inst, args.ldif_file) + validate_ldif_file(args.ldif_file) + + props = { + "numUsers": int(args.num_users), + "nodeLimit": int(args.node_limit), + "suffix": args.suffix, + } + + display_args(log, args) + node_count = dbgen_nested_ldif(inst, args.ldif_file, props) + log.info(f"Successfully created nested LDIF file ({args.ldif_file}) containing {node_count} nodes/subtrees") + + +def create_parser(subparsers): + db_gen_parser = subparsers.add_parser('ldifgen', help="LDIF generator to make sample LDIF files for testing") + subcommands = db_gen_parser.add_subparsers(help="action") + + # Create just users + dbgen_users_parser = subcommands.add_parser('users', help='Generate a LDIF containing user entries') + dbgen_users_parser.set_defaults(func=dbgen_create_users) + dbgen_users_parser.add_argument('--number', help="The number of users to create.") + dbgen_users_parser.add_argument('--suffix', help="The database suffix where the entries will be created.") + dbgen_users_parser.add_argument('--parent', help="The parent entry that the user entries should be created under. If not specified, the entries are stored under random Organizational Units.") + dbgen_users_parser.add_argument('--generic', action='store_true', help="Create generic entries in the format of \"uid=user####\". These entries are also compatible with ldclt.") + dbgen_users_parser.add_argument('--start-idx', default=0, help="For generic LDIF's you can choose the starting index for the user entries. The default is \"0\".") + dbgen_users_parser.add_argument('--rdn-cn', action='store_true', help="Use the attribute \"cn\" as the RDN attribute in the DN instead of \"uid\"") + dbgen_users_parser.add_argument('--localize', action='store_true', help="Localize the LDIF data") + dbgen_users_parser.add_argument('--ldif-file', default="users.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'users.ldif'") + + # Create static groups + dbgen_groups_parser = subcommands.add_parser('groups', help='Generate a LDIF containing groups and members') + dbgen_groups_parser.set_defaults(func=dbgen_create_groups) + dbgen_groups_parser.add_argument('NAME', help="The group name.") + dbgen_groups_parser.add_argument('--number', default=1, help="The number of groups to create.") + dbgen_groups_parser.add_argument('--suffix', help="The database suffix where the groups will be created.") + dbgen_groups_parser.add_argument('--parent', help="The parent entry that the group entries should be created under. If not specified the groups are stored under the suffix.") + dbgen_groups_parser.add_argument('--num-members', default="10000", help="The number of members in the group. Default is 10000") + dbgen_groups_parser.add_argument('--create-members', action='store_true', help="Create the member user entries.") + dbgen_groups_parser.add_argument('--member-parent', help="The entry DN that the members should be created under. The default is the suffix entry.") + dbgen_groups_parser.add_argument('--member-attr', default="uniquemember", help="The membership attribute to use in the group. Default is \"uniquemember\".") + dbgen_groups_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"") + + # Create a COS definition + dbgen_cos_def_parser = subcommands.add_parser('cos-def', help='Generate a LDIF containing a COS definition (classic, pointer, or indirect)') + dbgen_cos_def_parser.set_defaults(func=dbgen_create_cos_def) + dbgen_cos_def_parser.add_argument('NAME', help="The COS definition name.") + dbgen_cos_def_parser.add_argument('--type', help="The COS definition type: \"classic\", \"pointer\", or \"indirect\".") + dbgen_cos_def_parser.add_argument('--parent', help="The parent entry that the COS definition should be created under.") + dbgen_cos_def_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry") + dbgen_cos_def_parser.add_argument('--cos-specifier', help="Used in a classic COS definition, this attribute located in the user entry is used to select which COS template to use.") + dbgen_cos_def_parser.add_argument('--cos-template', help="The DN of the COS template entry, only used for \"classic\" and \"pointer\" COS definitions.") + dbgen_cos_def_parser.add_argument('--cos-attr', nargs='*', default=[], help="A list of attributes which defines which attribute the COS generates values for.") + dbgen_cos_def_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"") + + # Create a COS Template + dbgen_cos_tmp_parser = subcommands.add_parser('cos-template', help='Generate a LDIF containing a COS template') + dbgen_cos_tmp_parser.set_defaults(func=dbgen_create_cos_tmp) + dbgen_cos_tmp_parser.add_argument('NAME', help="The COS template name.") + dbgen_cos_tmp_parser.add_argument('--parent', help="The DN of the entry to store the COS template entry under.") + dbgen_cos_tmp_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry") + dbgen_cos_tmp_parser.add_argument('--cos-priority', type=int, help="Sets the priority of this conflicting/competing COS templates.") + dbgen_cos_tmp_parser.add_argument('--cos-attr-val', help="defines the attribute and value that the template provides.") + dbgen_cos_tmp_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"") + + # Create Role entries + dbgen_roles_parser = subcommands.add_parser('roles', help='Generate a LDIF containing a role entry (managed, filtered, or indirect)') + dbgen_roles_parser.set_defaults(func=dbgen_create_role) + dbgen_roles_parser.add_argument('NAME', help="The Role name.") + dbgen_roles_parser.add_argument('--type', help="The Role type: \"managed\", \"filtered\", or \"nested\".") + dbgen_roles_parser.add_argument('--parent', help="The DN of the entry to store the Role entry under") + dbgen_roles_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry") + dbgen_roles_parser.add_argument('--filter', help="A search filter for gathering Role members. Required for a \"filtered\" role.") + dbgen_roles_parser.add_argument('--role-dn', nargs='*', default=[], help="A DN of a role entry that should be included in this role. Used for \"nested\" roles only.") + dbgen_roles_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"") + + # Create a modification LDIF + dbgen_mod_load_parser = subcommands.add_parser('mod-load', help='Generate a LDIF containing modify operations. This is intended to be consumed by ldapmodify.') + dbgen_mod_load_parser.set_defaults(func=dbgen_create_mods) + dbgen_mod_load_parser.add_argument('--create-users', action='store_true', help="Create the entries that will be modified or deleted. By default the script assumes the user entries already exist.") + dbgen_mod_load_parser.add_argument('--delete-users', action='store_true', help="Delete all the user entries at the end of the LDIF.") + dbgen_mod_load_parser.add_argument('--num-users', type=int, help="The number of user entries that will be modified or deleted") + dbgen_mod_load_parser.add_argument('--parent', help="The DN of the parent entry where the user entries are located.") + dbgen_mod_load_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry") + dbgen_mod_load_parser.add_argument('--add-users', default=100, help="The number of additional entries to add during the load.") + dbgen_mod_load_parser.add_argument('--del-users', default=100, help="The number of entries to delete during the load.") + dbgen_mod_load_parser.add_argument('--modrdn-users', default=100, help="The number of entries to perform a modrdn operation on.") + dbgen_mod_load_parser.add_argument('--mod-users', default=100, help="The number of entries to modify.") + dbgen_mod_load_parser.add_argument('--mod-attrs', nargs="*", default=['description'], help="List of attributes the script will randomly choose from when modifying an entry. The default is \"description\".") + dbgen_mod_load_parser.add_argument('--randomize', action='store_true', help="Randomly perform the specified add, mod, delete, and modrdn operations") + dbgen_mod_load_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"") + + # Create a heavily nested LDIF + dbgen_nested_parser = subcommands.add_parser('nested', help='Generate a heavily nested database LDIF in a cascading/fractal tree design') + dbgen_nested_parser.set_defaults(func=dbgen_create_nested) + dbgen_nested_parser.add_argument('--num-users', help="The total number of user entries to create in the entire LDIF (does not include the container entries).") + dbgen_nested_parser.add_argument('--node-limit', help="The total number of user entries to create under each node/subtree") + dbgen_nested_parser.add_argument('--suffix', help="The suffix DN for the LDIF") + dbgen_nested_parser.add_argument('--ldif-file', default="nested-users.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'users.ldif'") diff --git a/src/lib389/lib389/cli_ctl/health.py b/src/lib389/lib389/cli_ctl/health.py index 10eaa44..3d15ad8 100644 --- a/src/lib389/lib389/cli_ctl/health.py +++ b/src/lib389/lib389/cli_ctl/health.py @@ -7,10 +7,9 @@ # --- END COPYRIGHT BLOCK --- import json -from getpass import getpass -from lib389.cli_base import connect_instance, disconnect_instance, format_error_to_dict +from lib389.cli_base import connect_instance, disconnect_instance from lib389.cli_base.dsrc import dsrc_to_ldap, dsrc_arg_concat -from lib389.backend import Backend, Backends +from lib389.backend import Backends from lib389.config import Encryption, Config from lib389.monitor import MonitorDiskSpace from lib389.replica import Replica, Changelog5 diff --git a/src/lib389/lib389/cli_ctl/nsstate.py b/src/lib389/lib389/cli_ctl/nsstate.py index 20855a0..28acd99 100644 --- a/src/lib389/lib389/cli_ctl/nsstate.py +++ b/src/lib389/lib389/cli_ctl/nsstate.py @@ -42,7 +42,7 @@ def create_parser(subparsers): repl_get_nsstate = subparsers.add_parser('get-nsstate', help="""Get the replication nsState in a human readable format Replica DN: The DN of the replication configuration entry -Replica SUffix: The replicated suffix +Replica Suffix: The replicated suffix Replica ID: The Replica identifier Gen Time The time the CSN generator was created Gen Time String: The time string of generator @@ -61,4 +61,4 @@ Endian: Little/Big Endian """) repl_get_nsstate.add_argument('--suffix', default=False, help='The DN of the replication suffix to read the state from') repl_get_nsstate.add_argument('--flip', default=False, help='Flip between Little/Big Endian, this might be required for certain architectures') - repl_get_nsstate.set_defaults(func=get_nsstate) + repl_get_nsstate.set_defaults(func=get_nsstate) diff --git a/src/lib389/lib389/dbgen.py b/src/lib389/lib389/dbgen.py index f104ce8..6273781 100644 --- a/src/lib389/lib389/dbgen.py +++ b/src/lib389/lib389/dbgen.py @@ -1,5 +1,5 @@ # --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2017 Red Hat, Inc. +# Copyright (C) 2020 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). @@ -8,12 +8,14 @@ # Replacement of the dbgen.pl utility -from lib389.utils import pseudolocalize +from lib389.utils import (ensure_str, pseudolocalize) import random import os import pwd import grp +global node_count + DBGEN_POSITIONS = [ "Accountant", "Admin", @@ -66,21 +68,22 @@ DBGEN_LOCATIONS = [ ] DBGEN_OUS = [ -"Accounting", -"Product Development", -"Product Testing", -"Human Resources", -"Payroll", -"People", -"Groups", +"accounting", +"product development", +"product testing", +"human resources", +"payroll", +"people", +"groups", ] -DBGEN_TEMPLATE = """dn: {DN} +DBGEN_TEMPLATE = """dn: {DN}{CHANGETYPE} objectClass: top objectClass: person objectClass: organizationalPerson objectClass: inetOrgPerson -cn: {FIRST} {LAST} +objectclass: inetUser +cn: {CN} sn: {LAST} uid: {UID} givenName: {FIRST} @@ -100,7 +103,7 @@ l: {LOCATION} ou: {OU} mail: {UID}@example.com mail: {UIDNUMBER}@example.com -postalAddress: 518, Dept #851, Room#{OU} +postalAddress: 518, Dept #851, Room#{OU} title: {TITLE} usercertificate;binary:: MIIBvjCCASegAwIBAgIBAjANBgkqhkiG9w0BAQQFADAnMQ8wDQYD VQQDEwZjb25maWcxFDASBgNVBAMTC01NUiBDQSBDZXJ0MB4XDTAxMDQwNTE1NTEwNloXDTExMDcw @@ -114,25 +117,109 @@ usercertificate;binary:: MIIBvjCCASegAwIBAgIBAjANBgkqhkiG9w0BAQQFADAnMQ8wDQYD """ -DBGEN_HEADER = """dn: {SUFFIX} +DBGEN_OU_TEMPLATE = """dn: ou={OU},{SUFFIX} objectClass: top -objectClass: domain -dc: {RDN} -aci: (target=ldap:///{SUFFIX})(targetattr=*)(version 3.0; acl "acl1"; allow(write) userdn = "ldap:///self";) -aci: (target=ldap:///{SUFFIX})(targetattr=*)(version 3.0; acl "acl2"; allow(write) groupdn = "ldap:///cn=Directory Administrators, {SUFFIX}";) -aci: (target=ldap:///{SUFFIX})(targetattr=*)(version 3.0; acl "acl3"; allow(read, search, compare) userdn = "ldap:///anyone";) +objectClass: organizationalUnit +ou: {OU} """ -DBGEN_OU_TEMPLATE = """dn: ou={OU},{SUFFIX} +RANDOM_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqurstuvwxyz0123456789_#@%&()?~$^`~*-=+{}|"\'.,<>' + + +def finalize_ldif_file(instance, ldif_file): + # Make the file owned by dirsrv + os.chmod(ldif_file, 0o644) + if os.getuid() == 0: + # root user - chown the ldif to the server user + userid = ensure_str(instance.userid) + uid = pwd.getpwnam(userid).pw_uid + gid = grp.getgrnam(userid).gr_gid + os.chown(ldif_file, uid, gid) + + +def get_index(idx, numUsers): + # Get ldclt style entry "0" padded index number + zeroLen = len(str(numUsers)) - len(str(idx)) + index = '0' * zeroLen + index = index + str(idx) + return index + + +def get_node(suffix): + # Build a node/container entry based on the suffix DN + rdn_attr = suffix.split('=')[0].lower() + rdn_attr_val = suffix.split('=')[1].split(',')[0] + if rdn_attr == 'c': + oc = 'country' + elif rdn_attr == 'cn': + oc = 'nscontainer' + elif rdn_attr == 'dc': + oc = 'domain' + elif rdn_attr == 'o': + oc = 'organization' + elif rdn_attr == 'ou': + oc = 'organizationalunit' + else: + # Unsupported rdn + raise ValueError("Suffix RDN '{}' in '{}' is not supported. Supported RDN's are: 'c', 'cn', 'dc', 'o', and 'ou'".format(rdn_attr, suffix)) + + return f"""dn: {suffix} objectClass: top -objectClass: organizationalUnit -ou: {OU} +objectClass: {oc} +{rdn_attr}: {rdn_attr_val} +aci: (target=ldap:///{suffix})(targetattr=*)(version 3.0; acl "Self Write"; allow(write) userdn = "ldap:///self";) +aci: (target=ldap:///{suffix})(targetattr=*)(version 3.0; acl "Directory Admin Group"; allow(write) groupdn = "ldap:///cn=Directory Administrators,ou=Groups,{suffix}";) +aci: (target=ldap:///{suffix})(targetattr=*)(version 3.0; acl "Anonymous Access"; allow(read, search, compare) userdn = "ldap:///anyone";) """ -def dbgen(instance, number, ldif_file, suffix, pseudol10n=False): +def randomPick(values): + # Return a randomly selected value from the provided list of values + val_count = len(values) + val_count -= 1 + idx = random.randint(0, val_count) + return values[idx].lstrip() + + +def write_generic_user(LDIF, index, number, parent, name="user", changetype="", pseudol10n=False): + uid_val = name + get_index(index, number) + ou = random.choice(DBGEN_OUS) + first = uid_val + last = uid_val[::-1] # reverse name + cn = f"{first} {last}" + initials = "%s. %s" % (first[0], last[0]) + l = random.choice(DBGEN_LOCATIONS) + title = "%s %s" % (random.choice(DBGEN_TITLE_LEVELS), random.choice(DBGEN_POSITIONS)) + if pseudol10n: + ou = pseudolocalize(ou) + first = pseudolocalize(first) + last = pseudolocalize(last) + initials = pseudolocalize(initials) + l = pseudolocalize(l) + title = pseudolocalize(title) + dn = f"uid={uid_val},{parent}" + + LDIF.write(DBGEN_TEMPLATE.format( + DN=dn, + CHANGETYPE=changetype, + UID=uid_val, + UIDNUMBER=index, + FIRST=first, + LAST=last, + CN=cn, + INITIALS=initials, + OU=ou, + LOCATION=l, + TITLE=title, + )) + return dn + +def dbgen_users(instance, number, ldif_file, suffix, generic=False, entry_name="user", parent=None, startIdx=0, rdnCN=False, pseudol10n=False): + """ + Generate an LDIF of randomly named entries + """ familyname_file = os.path.join(instance.ds_paths.data_dir, 'dirsrv/data/dbgen-FamilyNames') givename_file = os.path.join(instance.ds_paths.data_dir, 'dirsrv/data/dbgen-GivenNames') familynames = [] @@ -142,20 +229,34 @@ def dbgen(instance, number, ldif_file, suffix, pseudol10n=False): with open(givename_file, 'r') as f: givennames = [n.strip() for n in f] - with open(ldif_file, 'w') as output: - rdn = suffix.split(",", 1)[0].split("=", 1)[1] - output.write(DBGEN_HEADER.format(SUFFIX=suffix, RDN=rdn)) + with open(ldif_file, 'w') as LDIF: + LDIF.write(get_node(suffix)) for ou in DBGEN_OUS: ou = pseudolocalize(ou) if pseudol10n else ou - output.write(DBGEN_OU_TEMPLATE.format(SUFFIX=suffix, OU=ou)) - for i in range(0, number): + LDIF.write(DBGEN_OU_TEMPLATE.format(SUFFIX=suffix, OU=ou)) + + if parent is not None: + parent_rdn = parent.split(',')[0].split('=')[1] + if parent_rdn.lower() not in DBGEN_OUS: + LDIF.write(get_node(parent)) + + for i in range(1, int(number) + 1): # Pick a random ou ou = random.choice(DBGEN_OUS) first = random.choice(givennames) last = random.choice(familynames) + if generic: + i += startIdx + name = entry_name + get_index(i, number) + uid = name + cn = name + else: + first = random.choice(givennames) + last = random.choice(familynames) + uid = "%s%s%s" % (first[0], last, i) + cn = f"{first} {last}" # How do we subscript from a generator? initials = "%s. %s" % (first[0], last[0]) - uid = "%s%s%s" % (first[0], last, i) l = random.choice(DBGEN_LOCATIONS) title = "%s %s" % (random.choice(DBGEN_TITLE_LEVELS), random.choice(DBGEN_POSITIONS)) if pseudol10n: @@ -165,24 +266,468 @@ def dbgen(instance, number, ldif_file, suffix, pseudol10n=False): initials = pseudolocalize(initials) l = pseudolocalize(l) title = pseudolocalize(title) - dn = "uid=%s,ou=%s,%s" % (uid, ou, suffix) - output.write(DBGEN_TEMPLATE.format( + + if parent is None: + parent = f"ou={ou},{suffix}" + + if rdnCN: + # Not using "uid" so use "cn" instead + dn = f"cn={cn},{parent}" + else: + dn = f"uid={uid},{parent}" + + LDIF.write(DBGEN_TEMPLATE.format( DN=dn, + CHANGETYPE="", UID=uid, UIDNUMBER=i, FIRST=first, LAST=last, + CN=cn, INITIALS=initials, OU=ou, LOCATION=l, TITLE=title, - SUFFIX=suffix )) - # Make the file owned by dirsrv - os.chmod(ldif_file, 0o644) - if os.getuid() == 0: - # root user - chown the ldif to the server user - uid = pwd.getpwnam(instance.userid).pw_uid - gid = grp.getgrnam(instance.userid).gr_gid - os.chown(ldif_file, uid, gid) + finalize_ldif_file(instance, ldif_file) + + +def dbgen_groups(instance, ldif_file, props): + """ + Create static group(s) and the member entries + + props = { + "name": STRING, + "parent": DN, + "suffix": DN, + "number": ###, --> number of groups to create + "numMembers": ###, + "createMembers": True/False --> Create the member entries (default is True) + "memberParent": DN + "membershipAttr": ATTR + } + """ + with open(ldif_file, 'w') as LDIF: + # Create the top node + LDIF.write(get_node(props['suffix'])) + if props['parent'] is not None: + if props['parent'] != props['suffix']: + # Create the group container + LDIF.write(get_node(props['parent'])) + else: + props['parent'] = props['suffix'] + + if props['memberParent'] is not None: + if props['memberParent'] != props['suffix'] and props['memberParent'] != props['parent']: + # Create the member/user container + LDIF.write(get_node(props['memberParent'])) + else: + props['memberParent'] = props['suffix'] + + for idx in range(1, int(props['number']) + 1): + # Build the member list and create the member entries + group_member_list = [] + if props['createMembers']: + for user_idx in range(1, int(props['numMembers']) + 1): + dn = write_generic_user(LDIF, user_idx, props['numMembers'], props['memberParent'], name=f"group_entry{idx}-") + group_member_list.append(dn) + else: + # Not creating the member entries, just build the DN list of members + for user_idx in range(1, int(props['numMembers']) + 1): + name = "user" + get_index(user_idx, props['numMembers']) + dn = f"uid={name},{props['memberParent']}" + group_member_list.append(dn) + + if props['number'] == 0: + # Only creating one group, do not add the idx to DN + group_dn = f"dn: cn={props['name']},{props['parent']}\n" + cn = f"cn={props['name']},{props['parent']}" + else: + group_dn = f"dn: cn={props['name']}-{idx},{props['parent']}\n" + cn = f"cn={props['name']}-{idx},{props['parent']}" + + LDIF.write(group_dn) + LDIF.write('objectclass: top\n') + LDIF.write('objectclass: groupOfUniqueNames\n') + LDIF.write('objectclass: groupOfNames\n') + LDIF.write('objectclass: inetAdmin\n') + LDIF.write(f'cn: {cn}\n') + for dn in group_member_list: + LDIF.write(f"{props['membershipAttr']}: {dn}\n") + LDIF.write('\n') + + finalize_ldif_file(instance, ldif_file) + + +def dbgen_cos_def(instance, ldif_file, props): + """ + Create a COS definition + + props = { + "cosType": "classic", "pointer", "indirect", + "defName": VAL, + "defParent": VAL, + "defCreateParent": True/False, + "cosSpecifier": can be used for cosIndirectSpecifier + "cosAttrs": [], + "tmpName": DN (need to classic and pointer COS defs) + } + """ + + if props['cosType'] == 'pointer': + objectclass = 'objectclass: cosPointerDefinition\n' + if props['cosType'] == 'indirect': + objectclass = 'objectclass: cosIndirectDefinition\n' + if props['cosType'] == 'classic': + objectclass = 'objectclass: cosClassicDefinition\n' + + with open(ldif_file, 'w') as LDIF: + # Create parent + if props['defCreateParent']: + LDIF.write(get_node(props['defParent'])) + + # + # Create definition + # + dn = (f"dn: cn={props['defName']},{props['defParent']}\n") + LDIF.write(dn) + LDIF.write('objectclass: top\n') + LDIF.write('objectclass: cosSuperDefinition\n') + LDIF.write(objectclass) + LDIF.write('cn: ' + props['defName'] + "\n") + + if props['cosType'] == 'pointer' or props['cosType'] == 'classic': + LDIF.write(f"cosTemplateDN: {props['tmpName']}\n") + if props['cosType'] == 'indirect': + LDIF.write(f"cosIndirectSpecifier: {props['cosSpecifier']}\n") + elif props['cosType'] == "classic": + LDIF.write(f"cosSpecifier: {props['cosSpecifier']}\n") + for attr in props['cosAttrs']: + # There can be multiple COS attributes + LDIF.write(f"cosAttribute: {attr}\n") + + finalize_ldif_file(instance, ldif_file) + + +def dbgen_cos_template(instance, ldif_file, props): + """ + Create a COS Template + + props = { + "tmpName": VAL, + "tmpParent": VAL, + "tmpCreateParent": True/False, + "cosPriority": #### + "cosTmpAttrVal": Attr/val + } + """ + + with open(ldif_file, 'w') as LDIF: + # Create parent + if props['tmpCreateParent']: + LDIF.write(get_node(props['tmpParent'])) + + # Create template + dn = f"dn: cn={props['tmpName']},{props['tmpParent']}\n" + LDIF.write(dn) + LDIF.write('objectclass: top\n') + LDIF.write('objectclass: extensibleObject\n') + LDIF.write('objectclass: cosTemplate\n') + LDIF.write(f"cn: {props['tmpName']}\n") + if props['cosPriority'] is not None: + LDIF.write(f"cosPriority: {props['cosPriority']}\n") + pair = props['cosTmpAttrVal'].split(':') + LDIF.write(f"{pair[0]}: {pair[1]}\n") + + finalize_ldif_file(instance, ldif_file) + + +def dbgen_role(instance, ldif_file, props): + """ + Create a Role + + props = { + role_type: "managed", "filter", pr "nested", + role_name: NAME, + parent: DN of parent, + createParent: True/False, + filter: FILTER, + role_list: [DN, DN, ...] # For nested role only + } + """ + + if props['role_type'].lower() == 'managed': + objectclasses = ('objectclass: nsSimpleRoleDefinition\n' + + 'objectclass: nsManagedRoleDefinition\n') + elif props['role_type'].lower() == 'filtered': + objectclasses = ('objectclass: nsComplexRoleDefinition\n' + + 'objectclass: nsFilteredRoleDefinition\n') + elif props['role_type'].lower() == 'nested': + objectclasses = ('objectclass: nsComplexRoleDefinition\n' + + 'objectclass: nsNestedRoleDefinition\n') + + with open(ldif_file, 'w') as LDIF: + # Create parent entry + if props['createParent']: + LDIF.write(get_node(props['parent'])) + + dn = f"dn: cn={props['role_name']},{props['parent']}\n" + LDIF.write(dn) + LDIF.write('objectclass: top\n') + LDIF.write('objectclass: LdapSubEntry\n') + LDIF.write('objectclass: nsRoleDefinition\n') + LDIF.write(objectclasses) + LDIF.write(f"cn: {props['role_name']}\n") + if props['role_type'] == 'nested': + # Write out each role DN + for value in props['role_list']: + LDIF.write(f'nsRoleDN: {value}\n') + elif props['role_type'] == 'filtered': + LDIF.write(f"nsRoleFilter: {props['filter']}\n") + + finalize_ldif_file(instance, ldif_file) + + +def dbgen_mod_load(ldif_file, props): + """ + Generate a "load" LDIF file that can be consumed by ldapmodify + + props = { + "createUsers": True/False, + "deleteUsers": True/False, + "numUsers": ###, + "parent": DN, --> ou=people,dc=example,dc=com + "createParent": True/False, + "addUsers": ###, + "delUsers": ###, + 'modrdnUsers": ###, + "modUsers": ###, --> number of entries to modify + "random": True/False, + "modAttrs": [ATTR, ATTR, ...] + } + """ + + entry_dn_list = [] # List used to delete entries at the end of the LDIF + if props['modAttrs'] is None: + props['modAttrs'] = ['description', 'title'] + + with open(ldif_file, 'w') as LDIF: + if props['createParent']: + # Create the container entry that the users will be add to + LDIF.write(get_node(props['parent'])) + + # Create entries + for user_idx in range(1, props['numUsers'] + 1): + if props['createUsers']: + dn = write_generic_user( + LDIF, user_idx, props['numUsers'], props['parent'], + changetype="\nchangetype: add") + entry_dn_list.append(dn) + else: + dn = f"uid=user{get_index(user_idx, props['numUsers'])},{props['parent']}" + entry_dn_list.append(dn) + + # Set the types of operations and how many of them to perform + addc = int(props['addUsers']) + delc = int(props['delUsers']) + modc = int(props['modUsers']) + mrdnc = int(props['modrdnUsers']) + total_ops = addc + delc + modc + mrdnc + + if props['random']: + # Mix up the selected operations + operations = ['add', 'mod', 'modrdn', 'delete'] + while total_ops != 0: + op = randomPick(operations) + if op == 'add': + if addc == 0: + # no more adds to do + operations.remove('add') + continue + dn = write_generic_user( + LDIF, addc, props['addUsers'], props['parent'], + name="addUser", changetype="\nchangetype: add") + entry_dn_list.append(dn) + addc -= 1 + elif op == 'mod': + if modc == 0: + # no more mods to do + operations.remove('mod') + continue + attr = randomPick(props['modAttrs']) + val = (''.join((random.choice(RANDOM_CHARS) for i in range(0, random.randint(10, 30))))) + LDIF.write(f"dn: uid=user{get_index(modc, props['numUsers'])},{props['parent']}\n") + LDIF.write("changetype: modify\n") + LDIF.write(f"replace: {attr}\n") + LDIF.write(f"{attr}: {val}\n") + LDIF.write("\n") + modc -= 1 + elif op == 'delete': + if delc == 0: + # no more deletes to do + operations.remove('delete') + continue + + dn_val = f"uid=user{get_index(delc, props['numUsers'])},{props['parent']}" + LDIF.write(f"dn: {dn_val}\n") + LDIF.write("changetype: delete\n") + LDIF.write(" \n") + delc -= 1 + if dn_val in entry_dn_list: + entry_dn_list.remove(dn_val) + elif op == 'modrdn': + if mrdnc == 0: + # no more modrdns to do + operations.remove('modrdn') + continue + dn_val = f"uid=user{get_index(mrdnc, props['numUsers'])},{props['parent']}" + new_dn_val = f"cn=user{get_index(mrdnc, props['numUsers'])},{props['parent']}" + LDIF.write(f"dn: {dn_val}\n") + LDIF.write("changetype: modrdn\n") + LDIF.write(f"newrdn: {new_dn_val}\n") + LDIF.write("deleteoldrdn: 1\n") + LDIF.write("\n") + mrdnc -= 1 + # Revise the DN list: add the new DN, and remove the old DN + entry_dn_list.append(new_dn_val) + if dn_val in entry_dn_list: + entry_dn_list.remove(dn_val) + + # Update the total count + total_ops -= 1 + else: + # Sequentially do each type of operation + + # Do the Adds + addc = int(props['addUsers']) + while addc != 0: + dn = write_generic_user( + LDIF, addc, props['addUsers'], props['parent'], + name="addUser", changetype="\nchangetype: add") + addc -= 1 + entry_dn_list.append(dn) + + # Mods + while modc != 0: + attr = randomPick(props['modAttrs']) + val = (''.join((random.choice(RANDOM_CHARS) for i in range(0, random.randint(10, 30))))) + LDIF.write(f"dn: uid=user{get_index(modc, props['numUsers'])},{props['parent']}\n") + LDIF.write("changetype: modify\n") + LDIF.write(f"replace: {attr}\n") + LDIF.write(f"{attr}: {val}\n") + LDIF.write("\n") + modc -= 1 + + # Modrdns + while mrdnc != 0: + dn_val = f"uid=user{get_index(mrdnc, props['numUsers'])},{props['parent']}" + new_dn_val = f"cn=user{get_index(mrdnc, props['numUsers'])},{props['parent']}" + LDIF.write(f"dn: {dn_val}\n") + LDIF.write("changetype: modrdn\n") + LDIF.write(f"newrdn: {new_dn_val}\n") + LDIF.write("deleteoldrdn: 1\n") + LDIF.write("\n") + mrdnc -= 1 + # Revise the DN list: add the new DN, and remove the old DN + entry_dn_list.append(new_dn_val) + if dn_val in entry_dn_list: + entry_dn_list.remove(dn_val) + + # Deletes + while delc != 0: + dn_val = f"uid=user{get_index(delc, props['numUsers'])},{props['parent']}" + LDIF.write(f"dn: {dn_val}\n") + LDIF.write("changetype: delete\n") + LDIF.write(" \n") + delc -= 1 + if dn_val in entry_dn_list: + entry_dn_list.remove(dn_val) + + # Cleanup - delete all known entries + if props['deleteUsers']: + for dn in entry_dn_list: + LDIF.write(f"dn: {dn}\n") + LDIF.write("changetype: delete\n") + LDIF.write(" \n") + + +def build_recursive_nodes(LDIF, dn, node_limit, max_entries): + """ + Recursively create two nodes under each node, continue until there are no + more max_entries. + """ + global node_count + wrote_node1 = False + wrote_node2 = False + + # Create containers for DN1 and DN2 + dn1 = "ou=1," + dn + LDIF.write(f'dn: {dn1}\n') + LDIF.write('objectclass: top\n') + LDIF.write('objectclass: organizationalUnit\n') + LDIF.write('ou: ou=1\n\n') + + dn2 = "ou=2," + dn + LDIF.write(f'dn: {dn2}\n') + LDIF.write('objectclass: top\n') + LDIF.write('objectclass: organizationalUnit\n') + LDIF.write('ou: ou=2\n\n') + + # Add entries under each node + for entry_idx in range(1, node_limit + 1): + write_generic_user(LDIF, entry_idx, node_limit, dn1) + max_entries -= 1 + if not wrote_node1: + wrote_node1 = True + node_count += 1 + if max_entries == 0: + break + + write_generic_user(LDIF, entry_idx, node_limit, dn2) + max_entries -= 1 + if not wrote_node2: + wrote_node2 = True + node_count += 1 + if max_entries == 0: + break + + # Are we out of entries + if max_entries == 0: + return + + # Get the remaining entries to be split between two nodes + new_node_max = max_entries // 2 + if (max_entries % 2) > 0: + remainder = 1 + else: + remainder = 0 + + # Recursively build child nodes + build_recursive_nodes(LDIF, dn1, node_limit, new_node_max) + build_recursive_nodes(LDIF, dn2, node_limit, new_node_max + remainder) + + +def dbgen_nested_ldif(instance, ldif_file, props): + """ + Create a deeply nested LDIF + + props = { + "numUsers": #### --> Total number of user entries to create + 'nodeLimit': #### --> max number of entries to put into each node + "suffix": DN + } + """ + + global node_count + node_count = 0 + with open(ldif_file, 'w') as LDIF: + # Create the top suffix + LDIF.write(get_node(props['suffix'])) + + # Create all the nodes + build_recursive_nodes(LDIF, props['suffix'], props['nodeLimit'], props['numUsers']) + + finalize_ldif_file(instance, ldif_file) + + return node_count