From 56e064ef7d7414861081f36b035646b3c7954c3d Mon Sep 17 00:00:00 2001 From: Simon Pichugin Date: Mar 20 2018 07:22:00 +0000 Subject: Issue 49606 - Improve lib389 documentation Description: Refactor existing chapters and add more docstrings to existing modules. Also, make sure that the contribution guide is up to date with the new pull-request initiative. https://pagure.io/389-ds-base/issue/49606 Reviewed by: wibrown (Thanks!) --- diff --git a/src/lib389/doc/source/group.rst b/src/lib389/doc/source/group.rst index a916748..aaf231d 100644 --- a/src/lib389/doc/source/group.rst +++ b/src/lib389/doc/source/group.rst @@ -37,7 +37,8 @@ Module documentation .. autoclass:: lib389.idm.group.Groups :members: + :inherited-members: .. autoclass:: lib389.idm.group.Group :members: - + :inherited-members: diff --git a/src/lib389/doc/source/guidelines.rst b/src/lib389/doc/source/guidelines.rst index 0d67218..828766d 100644 --- a/src/lib389/doc/source/guidelines.rst +++ b/src/lib389/doc/source/guidelines.rst @@ -10,49 +10,36 @@ with master1 , etc. Basic workflow ============== -1. Clone ds repo: - + git clone ssh://git@pagure.io/389-ds-base.git - + One can change the old repos like this in .git/config files - + For ds (dirsrv) use link - https://pagure.io/389-ds-base.git - + For lib389 use link - https://pagure.io/lib389.git +1. Clone 389-ds-base repo + + Clone the origin repo + + Go to https://pagure.io/389-ds-base and fork the repo + + Add the forked repo link as a remote to your local git repo + + :: + + git clone ssh://git@pagure.io/389-ds-base.git + git remote add myfork ssh://git@pagure.io/forks/USERNAME/389-ds-base.git 2. Go to the cloned directory 3. Create a new branch for your work: :: git checkout -b new_test_suite + 4. Check out PEP8 cheat sheet: + https://gist.github.com/RichardBronosky/454964087739a449da04 + https://www.python.org/dev/peps/pep-0008/ + It is not fully mandatory in our project, but let's make our code a bit cleaner for other's sake -5. Use ./dirsrvtests/create_test.py tool to generate new test.py file. - + Usage: +5. If you develop a new test, use ./dirsrvtests/create_test.py tool to generate new test.py file. :: - create_ticket.py -t|--ticket -s|--suite - [ i|--instances [ -m|--masters - -h|--hubs -c|--consumers ] -o|--outputfile] - - Create a test suite script using "-s|--suite" instead of using - "-t|–ticket". One day, all 'tickets' will be transferred to 'suites', - so try to avoid the 'tickets' and try to find the place in 'suites' - for you case. Ask around is you have doubts. - Option "-i" can add multiple standalone instances. However, you can - not mix "-i" with the replication options(-m, -h ,-c). - - - + For example: - - :: - create_test.py -s basic -m 2 -o ./dirsrvtests/tests/suites/basic/basic_test.py # It will create basic_test.py with two masters set up and put the file to right dir - - + + + If you are creating a test suite, the script will add one test case for you with generated ID in the docstring (and it will check it for uniqueness) @@ -63,6 +50,24 @@ Basic workflow python -c 'import uuid; print(uuid.uuid4())' + + We need to have proper docstring for every test case. We need them because it makes easier to understand + what is happening in the test. Also we have an inside metric system with test plans there. + We use them for our overall testing process improvement. You can take it as an example: + + :: + + """Test if member is automatically added to the group + + :id: 38621a51-03bc-4fba-93ef-7e525df87c5d + :setup: Standalone instance, enabled Auto Membership Plugin + :steps: + 1. Create a user + 2. Assert that the user is member of the group + :expectedresults: + 1. Should be success + 2. Should be success + """ + 6. Add some fixture(s), if needed. The purpose of test fixtures is to provide a fixed baseline upon which tests can reliably and repeatedly @@ -98,23 +103,54 @@ Basic workflow 8. Write some good code with encapsulations, assertions etc. 9. Commit and push your code to your repo: - :: + + Commands: - git add ./dirsrvtests/tests/suites/basic/basic_test.py - git commit - git push $(whoami) + :: + git add ./dirsrvtests/tests/suites/basic/basic_test.py + git commit + git push myfork + + + + Basic guidelines for the commit message format + + + Separate subject from body with a blank line + + Limit the subject line to 50 characters + + Capitalizethesubject line + + Do not end the subject line with a period + + Use the imperative mood in the subject line + + Wrap the body at 72 characters + + Use the body to explain *what* and *why* vs. *how* + + In the end, put a link to the ticket + + Add "Reviewed by: ?" line. Example: + + :: + + Issue 48085 - Expand the repl acceptance test suite + + Description: Add 6 more test cases to the replication + test suite as a part of the TET to + pytest porting initiative. + Increase the number of seconds we wait before the results check. + + https://pagure.io/389-ds-base/issue/48085 + + Reviewed by: ? 10. Test your script: + If you want to have the instances not to be deleted after the test case execution, + you need to set DEBUGGING environment variable to something. + :: + export DEBUGGING=yes py.test -v -s /mnt/testarea/test/ds/dirsrvtests/suites/basic -11. If everything is alright, then create a patch file for a review: +11. If everything is alright, then create a pull-request: - + Go back to ds or lib389 dir (depends on where you want to send the patch) and do: + + Go back to 389-ds-base dir (depends on where you want to send the patch) and do: :: @@ -122,47 +158,50 @@ Basic workflow git pull git checkout new_test_suite git rebase master - git format-patch -1 - + Basic guidelines for the commit message format + + Go to https://pagure.io/389-ds-base/pull-requests + + Press "File Pull Request" button and choose your branch + + Check that all fields have a right information and press 'Create' button - + Separate subject from body with a blank line - + Limit the subject line to 50 characters - + Capitalizethesubject line - + Do not end the subject line with a period - + Use the imperative mood in the subject line - + Wrap the body at 72 characters - + Use the body to explain *what* and *why* vs. *how* - + In the end, put a link to the ticket - + Add "Reviewed by: ?" line. Example: + + Alternatively you can create and upload a patch file (it was an old way before the pull-requests) + + After the first step in this section you can run the next command to generate a patch file from the last commit: - Issue 48085 - Expand the repl acceptance test suite - Description: Add 6 more test cases to the replication - test suite as a part of the TET to - pytest porting initiative. - Increase the number of seconds we wait before the results check. + :: - https://pagure.io/389-ds-base/issue/48085 - - Reviewed by: ? + git format-patch -1 + + + Attach the patch to the Pagure issue + + Set reviewstatus field to 'review' in the Pagure issue + + Send an email to 389-devel@lists.fedoraproject.org with + + :: + + Subject: Please review: YOUR COMMIT FIRST LINE + + Body: The links to the upstream ticket and the pull-request (or the patch). 12. Fixing Review Issues - + If there are issues with your patch, git allows you to fix your + + If there are issues with your change, git allows you to fix your commits. - + If you're not already in that branch - + git checkout new_test_suite - + Make changes to some file - + Add changes to your commit and fix the commit message if necessary :: + # If you're not already in that branch + git checkout new_test_suite + # Make changes to some file + # Add changes to your commit and fix the commit message if necessary git commit -a --amend + You can also use “ git rebase -i ” to “squash” or combine several commits into one commit. + + After the changes are commited, push tht commit to your fork branch (or upload a new patch) + + :: + + git push myfork --force Fixtures @@ -354,11 +393,21 @@ So we should use lib389 functions as much as possible because they take care of If you still must use 'modify_s', 'add_s' or other python-ldap functions, you should consider defining the attribute as 'byte'. You can do this like this, with b'' symbol: -:: + :: # Modify an entry standalone.modify_s(USER_DN, [(ldap.MOD_REPLACE, 'cn', b'Mark Reynolds')]) +Or if you have a complex string or variable that you want to convert, you can use 'ensure_*' functions for that: + + :: + + from lib389.utils import (ensure_bytes, ensure_str, ensure_int, ensure_list_bytes, + ensure_list_str, ensure_list_int) + + standalone.modify_s(USER_DN, [(ldap.MOD_REPLACE, 'jpegPhoto', ensure_bytes(var_with_content)]) + + Constants ========== @@ -366,7 +415,7 @@ Constants Basic constants ~~~~~~~~~~~~~~~ -:: + :: DEFAULT_SUFFIX = “dc=example,dc=com” DN_DM = "cn=Directory Manager" @@ -390,7 +439,7 @@ https://pagure.io/lib389/blob/master/f/lib389/_constants.py . If you need a constant, use this kind of import. If you need a lot of constants, import with * -:: + :: from lib389._constants import CONSTANT_YOU_NEED from lib389._constants import * @@ -402,13 +451,12 @@ Add, Modify, and Delete Operations Please, use these methods for the operations that can't be performed by DSLdapObjects. -:: + :: # Add an entry USER_DN = 'cn=mreynolds,{}'.format(DEFAULT_SUFFIX) standalone.add_s(Entry((USER_DN, { - 'objectclass': b'top', - 'objectclass': b'person', + 'objectclass': (b'top', b'person'), 'cn': b'mreynolds', 'sn': b'reynolds', 'userpassword': b'password' @@ -428,7 +476,7 @@ Search and Bind Operations authenticated as the Root DN(Directory Manager). + So you can just start searching without having to “bind” -:: + :: # Search entries = standalone.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(cn=*)', ['cn']) @@ -455,33 +503,14 @@ Search and Bind Operations Basic instance operations =================================== -:: + :: - # First, create a new “instance” of a “DirSrv” object - standalone = DirSrv(verbose=False) -   - # Set up the instance arguments (note - args_instance is a global dictionary - # in lib389, it contains other default values) - args_instance[SER_HOST] = HOST_STANDALONE - args_instance[SER_PORT] = PORT_STANDALONE - args_instance[SER_SERVERID_PROP] = SERVERID_STANDALONE - args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX - args_standalone = args_instance.copy() - # Allocate the instance - initialize the “DirSrv” object with our arguments - standalone.allocate(args_standalone) - # Check if the instance with the args exists - assert not standalone.exists() - # Create the instance - this runs setup-ds.pl and starts the server - standalone.create() - - # Open the instance - create a connection to the instance, - # and authenticates as the Root DN (cn=directory manager) - standalone.open() - # Done, you can start using the new instance # While working with DirSrv object, you can set 'verbose' parameter to True in any moment standalone.verbose = True + # To remove an instance, simply use: standalone.delete() + # Start, Stop, and Restart the Server standalone.start(timeout=10) standalone.stop(timeout=10) @@ -494,47 +523,78 @@ Basic instance operations Setting up SSL/TLS =================================== -:: +You need only one line to enable SSL/TLS on the instance. - from lib389._constants import DEFAULT_SUFFIX, SECUREPORT_STANDALONE1 - - standalone.stop() -   - # Re-init (create) the nss db - # pin.txt is created here and the password randomly generated - assert(standalone.nss_ssl.reinit() is True) -   - # Create a self signed CA - # noise.txt is created here - assert(standalone.nss_ssl.create_rsa_ca() is True) -   - # Create a key and a cert that is signed by the self signed ca - # This will use the hostname from the DS instance, and takes a list of extra names to take. - assert(standalone.nss_ssl.create_rsa_key_and_cert() is True) - - standalone.start() - - # Create "cn=RSA,cn=encryption,cn=config" with next properties: - # {'cn': 'RSA', 'nsSSLPersonalitySSL': 'Server-Cert', 'nsSSLActivation': 'on', 'nsSSLToken': 'internal (software)'} - standalone.rsa.create() - # Set the secure port and nsslapd-security - standalone.config.set('nsslapd-secureport', str(SECUREPORT_STANDALONE1)) - standalone.config.set('nsslapd-security', 'on') - standalone.sslport = SECUREPORT_STANDALONE1 - - # Restart to allow certmaps to be re-read: Note, we CAN NOT use post_open - standalone.restart(post_open=False) + :: + + standalone.enable_tls() Certification-based authentication =================================== -You need to setup and turn on SSL first (use the previous chapter). +For the SSLCLIENTAUTH setup, you need: -:: + :: + from lib389.idm.services import ServiceAccounts from lib389.config import CertmapLegacy - + from lib389.replica import ReplicationManager, Replicas + + # Create the certmap before we restart for enable_tls + cm_m1 = CertmapLegacy(m1) + cm_m2 = CertmapLegacy(m2) + + # We need to configure the same maps for both + certmaps = cm_m1.list() + certmaps['default']['DNComps'] = None + certmaps['default']['CmapLdapAttr'] = 'nsCertSubjectDN' + + cm_m1.set(certmaps) + cm_m2.set(certmaps) + + [i.enable_tls() for i in topo_m2] + + # Create the replication dns + services = ServiceAccounts(m1, DEFAULT_SUFFIX) + repl_m1 = services.get('%s:%s' % (m1.host, m1.sslport)) + repl_m1.set('nsCertSubjectDN', m1.get_server_tls_subject()) + + repl_m2 = services.get('%s:%s' % (m2.host, m2.sslport)) + repl_m2.set('nsCertSubjectDN', m2.get_server_tls_subject()) + + # Check the replication is "done". + repl = ReplicationManager(DEFAULT_SUFFIX) + repl.wait_for_replication(m1, m2) + + # Now change the auth type + replica_m1 = Replicas(m1).get(DEFAULT_SUFFIX) + agmt_m1 = replica_m1.get_agreements().list()[0] + + agmt_m1.replace_many( + ('nsDS5ReplicaBindMethod', 'SSLCLIENTAUTH'), + ('nsDS5ReplicaTransportInfo', 'SSL'), + ('nsDS5ReplicaPort', '%s' % m2.sslport), + ) + agmt_m1.remove_all('nsDS5ReplicaBindDN') + + replica_m2 = Replicas(m2).get(DEFAULT_SUFFIX) + agmt_m2 = replica_m2.get_agreements().list()[0] + + agmt_m2.replace_many( + ('nsDS5ReplicaBindMethod', 'SSLCLIENTAUTH'), + ('nsDS5ReplicaTransportInfo', 'SSL'), + ('nsDS5ReplicaPort', '%s' % m1.sslport), + ) + agmt_m2.remove_all('nsDS5ReplicaBindDN') + + repl.test_replication_topology(topo_m2) + + +And if you want just TLS authentication on a single instance: + + :: + standalone.stop()   # Create a user @@ -573,40 +633,3 @@ You need to setup and turn on SSL first (use the previous chapter). assert(conn.whoami_s() == "dn: uid=testuser,ou=People,dc=example,dc=com") - -Replication -=================================== - -Basic configuration - -+ After the instance is created, you can enable it for replication and - set up a replication agreement. - -:: - - from lib389.replica import Replicas -   - # Enable replication - replicas = Replicas(standalone) - replica = replicas.enable(suffix=DEFAULT_SUFFIX, - role=REPLICAROLE_MASTER, - replicaID=REPLICAID_MASTER_1) - # Set up replication agreement properties - properties = {RA_NAME: r'meTo_{}:{}'.format(master2.host, port=master2.port), - RA_BINDDN: defaultProperties[REPLICATION_BIND_DN], - RA_BINDPW: defaultProperties[REPLICATION_BIND_PW], - RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD], - RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]} - - # Create the agreement - repl_agreement = standalone.agreement.create(suffix=DEFAULT_SUFFIX, - host=master2.host, - port=master2.port, - properties=properties) - # “master2” refers to another, already created, DirSrv instance(like “standalone”) - # “repl_agreement” is the “DN” of the newly created agreement - this DN is needed later to do certain tasks - - # Initialize the agreement, wait for it complete, and test that replication is really working - standalone.agreement.init(DEFAULT_SUFFIX, master2.host, master2.port) - replica.start_and_wait(repl_agreement) - assert replicas.test(master2) diff --git a/src/lib389/doc/source/index.rst b/src/lib389/doc/source/index.rst index 2d77459..95d646b 100644 --- a/src/lib389/doc/source/index.rst +++ b/src/lib389/doc/source/index.rst @@ -24,6 +24,8 @@ Contents Configuring Databases Access Control Identity Management + DSE ldif + Indexes Work in progress ----------------- diff --git a/src/lib389/doc/source/indexes.rst b/src/lib389/doc/source/indexes.rst index 3980028..713fed3 100644 --- a/src/lib389/doc/source/indexes.rst +++ b/src/lib389/doc/source/indexes.rst @@ -9,12 +9,21 @@ Usage example indexes = Indexes(standalone) - # create and delete a default index. + # Create a default index index = indexes.create(properties={ 'cn': 'modifytimestamp', 'nsSystemIndex': 'false', 'nsIndexType': 'eq' }) + + # Get an index by DN + index = indexes.get(dn=YOUR_INDEX_DN) + + # Set index types + index.replace('nsIndexType', ['eq', 'sub', 'pres']) + + # Set matching rules (matching_rules - variable with matching rules) + index.replace('caseIgnoreOrderingMatch', matching_rules) default_index_list = indexes.list() found = False @@ -37,6 +46,8 @@ Module documentation .. autoclass:: lib389.index.Index :members: + :inherited-members: .. autoclass:: lib389.index.Indexes :members: + :inherited-members: diff --git a/src/lib389/doc/source/need_to_be_triaged.rst b/src/lib389/doc/source/need_to_be_triaged.rst index fbbed03..0b9558f 100644 --- a/src/lib389/doc/source/need_to_be_triaged.rst +++ b/src/lib389/doc/source/need_to_be_triaged.rst @@ -6,9 +6,7 @@ Need to be triaged backend.rst config.rst dirsrv_log.rst - dseldif.rst paths.rst - indexes.rst ldclt.rst mappingtree.rst monitor.rst diff --git a/src/lib389/doc/source/replica.rst b/src/lib389/doc/source/replica.rst index 3d6c4d4..01589c2 100644 --- a/src/lib389/doc/source/replica.rst +++ b/src/lib389/doc/source/replica.rst @@ -1,12 +1,28 @@ Replica ========== + Usage example -------------- -:: + +Basically, when you want a simple replica configuration without any hubs, you can use create_topology function. +In more complex cases you have to use our Replica API to build your own topology exactly the way you want it. +Still, it is better if you'll use the 'create_topology' method for basic initial setup and then you can continue to expand it. + + :: + + from lib389.topologies import create_topology + + topology = create_topology({ReplicaRole.MASTER: 2, + ReplicaRole.CONSUMER: 2}) + + +For basic Replica operations (the rest in the docs bellow): + + :: from lib389.replica import Replicas - + replicas = Replicas(standalone) # Enable replication # - changelog will be created @@ -54,6 +70,14 @@ Usage example Module documentation ----------------------- +.. autoclass:: lib389.replica.ReplicationManager + :members: + :inherited-members: + +.. autoclass:: lib389.replica.RUV + :members: + :inherited-members: + .. autoclass:: lib389.replica.Replicas :members: :inherited-members: diff --git a/src/lib389/lib389/dseldif.py b/src/lib389/lib389/dseldif.py index 7e5f6ab..e6bc57e 100644 --- a/src/lib389/lib389/dseldif.py +++ b/src/lib389/lib389/dseldif.py @@ -11,7 +11,11 @@ from lib389.paths import Paths class DSEldif(object): - """A class for working with dse.ldif file""" + """A class for working with dse.ldif file + + :param instance: An instance + :type instance: lib389.DirSrv + """ def __init__(self, instance): self._instance = instance @@ -59,7 +63,13 @@ class DSEldif(object): return entry_dn_i, attr_data def get(self, entry_dn, attr): - """Return attribute values under a given entry""" + """Return attribute values under a given entry + + :param entry_dn: a DN of entry we want to get attribute from + :type entry_dn: str + :param attr: an attribute name + :type attr: str + """ try: _, attr_data = self._find_attr(entry_dn, attr) @@ -69,14 +79,30 @@ class DSEldif(object): return attr_data.values() def add(self, entry_dn, attr, value): - """Add an attribute under a given entry""" + """Add an attribute under a given entry + + :param entry_dn: a DN of entry we want to edit + :type entry_dn: str + :param attr: an attribute name + :type attr: str + :param value: an attribute value + :type value: str + """ entry_dn_i = self._contents.index("dn: {}\n".format(entry_dn)) self._contents.insert(entry_dn_i+1, "{}: {}\n".format(attr, value)) self._update() def delete(self, entry_dn, attr, value=None): - """Delete attributes under a given entry""" + """Delete attributes under a given entry + + :param entry_dn: a DN of entry we want to edit + :type entry_dn: str + :param attr: an attribute name + :type attr: str + :param value: an attribute value + :type value: str + """ entry_dn_i, attr_data = self._find_attr(entry_dn, attr) @@ -90,7 +116,15 @@ class DSEldif(object): self._update() def replace(self, entry_dn, attr, value): - """Replace attribute values with a new one under a given entry""" + """Replace attribute values with a new one under a given entry + + :param entry_dn: a DN of entry we want to edit + :type entry_dn: str + :param attr: an attribute name + :type attr: str + :param value: an attribute value + :type value: str + """ try: self.delete(entry_dn, attr) diff --git a/src/lib389/lib389/index.py b/src/lib389/lib389/index.py index 7f55d8b..54c7590 100644 --- a/src/lib389/lib389/index.py +++ b/src/lib389/lib389/index.py @@ -23,7 +23,18 @@ if MAJOR >= 3 or (MAJOR == 2 and MINOR >= 7): DEFAULT_INDEX_DN = "cn=default indexes,%s" % DN_CONFIG_LDBM + class Index(DSLdapObject): + """Index DSLdapObject with: + - must attributes = ['cn', 'nsSystemIndex', 'nsIndexType'] + - RDN attribute is 'cn' + + :param instance: An instance + :type instance: lib389.DirSrv + :param dn: Index DN + :type dn: str + """ + def __init__(self, instance, dn=None): super(Index, self).__init__(instance, dn) self._rdn_attribute = 'cn' @@ -32,7 +43,16 @@ class Index(DSLdapObject): self._protected = False self._lint_functions = [] + class Indexes(DSLdapObjects): + """DSLdapObjects that represents Index + + :param instance: An instance + :type instance: lib389.DirSrv + :param basedn: DN of suffix container. + :type basedn: str + """ + def __init__(self, instance, basedn=DEFAULT_INDEX_DN): super(Indexes, self).__init__(instance=instance) self._objectclasses = ['nsIndex'] @@ -40,6 +60,7 @@ class Indexes(DSLdapObjects): self._childobject = Index self._basedn = basedn + class IndexLegacy(object): def __init__(self, conn): diff --git a/src/lib389/lib389/mappingTree.py b/src/lib389/lib389/mappingTree.py index a248f02..a567d3d 100644 --- a/src/lib389/lib389/mappingTree.py +++ b/src/lib389/lib389/mappingTree.py @@ -414,7 +414,7 @@ class MappingTree(DSLdapObject): class MappingTrees(DSLdapObjects): - """DSLdapObjects that presents Mapping trees + """DSLdapObjects that represents Mapping trees :param instance: An instance :type instance: lib389.DirSrv diff --git a/src/lib389/lib389/topologies.py b/src/lib389/lib389/topologies.py index 0c08c96..690c7f6 100644 --- a/src/lib389/lib389/topologies.py +++ b/src/lib389/lib389/topologies.py @@ -37,9 +37,13 @@ log = logging.getLogger(__name__) def create_topology(topo_dict, suffix=DEFAULT_SUFFIX): """Create a requested topology. Cascading replication scenario isn't supported - @param topo_dict - dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.MASTER: num, + :param topo_dict: a dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.MASTER: num, ReplicaRole.CONSUMER: num} - @return - TopologyMain object + :type topo_dict: dict + :param suffix: a suffix for the replication + :type suffix: str + + :return - TopologyMain object """ if not topo_dict: