#50561 Add new test suite to test migration between RHDS versions
Closed 3 years ago by spichugi. Opened 4 years ago by bsmejkal.
bsmejkal/389-ds-base migration  into  master

@@ -0,0 +1,3 @@ 

+ """

+    :Requirement: 389-ds-base: DataBase Import

+ """ 

\ No newline at end of file

@@ -0,0 +1,82 @@ 

+ # --- BEGIN COPYRIGHT BLOCK ---

+ # Copyright (C) 2019 Red Hat, Inc.

+ # All rights reserved.

+ #

+ # License: GPL (version 3 or any later version).

+ # See LICENSE for details.

+ # --- END COPYRIGHT BLOCK ---

+ #

+ 

+ import logging

+ import pytest

+ import os

+ 

+ from lib389._constants import *

+ from lib389.topologies import topology_st

+ from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES

+ 

+ pytestmark = pytest.mark.tier3

+ 

+ logging.getLogger(__name__).setLevel(logging.DEBUG)

+ log = logging.getLogger(__name__)

+ 

+ 

+ @pytest.mark.skipif(os.getenv('MIGRATION') is None, reason="This test is meant to execute in specific test environment")

+ def test_export_data_from_source_host(topology_st):

+     """Prepare export file for migration using a single instance of Directory Server

+ 

+     :id: 47f97d87-60f7-4f80-a72b-e7daa1de0061

+     :setup: Standalone

+     :steps:

+         1. Add a test user with employeeNumber and telephoneNumber

+         2. Add a test user with escaped DN

+         3. Create export file

+         4. Check if values of searched attributes are present in exported file

+     :expectedresults:

+         1. Success

+         2. Success

+         3. Success

+         4. Success

+     """

+ 

+     standalone = topology_st.standalone

+     output_file = os.path.join(topology_st.standalone.ds_paths.ldif_dir, "migration_export.ldif")

+ 

+     log.info("Add a test user")

+     users = UserAccounts(standalone, DEFAULT_SUFFIX)

+     test_user = users.create(properties=TEST_USER_PROPERTIES)

+     test_user.add('employeeNumber', '1000')

+     test_user.add('telephoneNumber', '1234567890')

+ 

+     assert test_user.present('employeeNumber', value='1000')

+     assert test_user.present('telephoneNumber', value='1234567890')

+ 

+     log.info("Creating user with escaped DN")

+     users.create(properties={

+         'uid': '\\#\\,\\+"\\\\>:\\=\\<\\<\\>\\;/',

+         'cn': 'tuser2',

+         'sn': 'user',

+         'uidNumber': '1000',

+         'gidNumber': '2000',

+         'homeDirectory': '/home/tuser2',

+     })

+ 

+     log.info("Exporting LDIF offline...")

+     standalone.stop()

+     standalone.db2ldif(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX],

+                        excludeSuffixes=None, encrypt=None, repl_data=None, outputfile=output_file)

+     standalone.start()

+ 

+     log.info("Check that value of attribute is present in the exported file")

+     with open(output_file, 'r') as ldif_file:

+         ldif = ldif_file.read()

+         assert 'employeeNumber: 1000' in ldif

+         assert 'telephoneNumber: 1234567890' in ldif

+         assert 'uid: \\#\\,\\+"\\\\>:\\=\\<\\<\\>\\;/' in ldif

+ 

+ 

+ if __name__ == '__main__':

+     # Run isolated

+     # -s for DEBUG mode

+     CURRENT_FILE = os.path.realpath(__file__)

+     pytest.main(["-s", CURRENT_FILE])

@@ -0,0 +1,70 @@ 

+ # --- BEGIN COPYRIGHT BLOCK ---

+ # Copyright (C) 2019 Red Hat, Inc.

+ # All rights reserved.

+ #

+ # License: GPL (version 3 or any later version).

+ # See LICENSE for details.

+ # --- END COPYRIGHT BLOCK ---

+ #

+ 

+ import logging

+ import pytest

+ import os

+ 

+ from lib389._constants import *

+ from lib389.topologies import topology_st

+ from lib389.idm.user import UserAccounts

+ 

+ pytestmark = pytest.mark.tier3

+ 

+ logging.getLogger(__name__).setLevel(logging.DEBUG)

+ log = logging.getLogger(__name__)

+ 

+ 

+ @pytest.mark.skipif(os.getenv('MIGRATION') is None, reason="This test is meant to execute in specific test environment")

+ def test_import_data_to_target_host(topology_st):

+     """Import file created in export_data_test.py using a single instance of Directory Server

+ 

+     :id: 7e896b0c-6838-49c7-8e1d-5e8114f5fb02

+     :setup: Standalone

+     :steps:

+         1. Check that attribute values are present in input file

+         2. Import input file

+         3. Check imported user data

+     :expectedresults:

+         1. Success

+         2. Success

+         3. Success

+     """

+ 

+     standalone = topology_st.standalone

+     input_file = os.path.join(topology_st.standalone.ds_paths.ldif_dir, "migration_export.ldif")

+ 

+     log.info("Check that value of attribute is present in the exported file")

+     with open(input_file, 'r') as ldif_file:

+         ldif = ldif_file.read()

+         assert 'employeeNumber: 1000' in ldif

+         assert 'telephoneNumber: 1234567890' in ldif

+         assert 'uid: \\#\\,\\+"\\\\>:\\=\\<\\<\\>\\;/' in ldif

+ 

+     log.info('Stopping the server and running offline import...')

+     standalone.stop()

+     assert standalone.ldif2db(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX], encrypt=None, excludeSuffixes=None,

+                                      import_file=input_file)

+     standalone.start()

+ 

+     log.info("Check imported user data")

+     users = UserAccounts(standalone, DEFAULT_SUFFIX)

+     test_user = users.get('testuser')

+     assert test_user.present('employeeNumber', value='1000')

+     assert test_user.present('telephoneNumber', value='1234567890')

+     test_user = users.get('\\#\\,\\+"\\\\>:\\=\\<\\<\\>\\;/')

+     assert test_user.present('cn', value='tuser2')

+     assert test_user.present('uid', value='\\#\\,\\+"\\\\>:\\=\\<\\<\\>\\;/')

+ 

+ 

+ if __name__ == '__main__':

+     # Run isolated

+     # -s for DEBUG mode

+     CURRENT_FILE = os.path.realpath(__file__)

+     pytest.main(["-s", CURRENT_FILE])

Description:
Created migration test suite. This suite will be used with ansible for testing import/export and replication method between RHDS 10 and RHDS 11. These tests can be executed only in specific test environment and therefore will be skipped in normal test run.

Reviewed by: vashirov (Thanks!)

@vashirov
Hi, could you please review this when you have time?
Thanks!

Is there a better location than /tmp we could use? This could be locked down in some cases, or have other permission faults present that could influence the test? Could we instead use the instances ldif directory?

Uhhhh you can't do this. You need to have test_import_data_to_target_host be in the same file as test_export, and you probably need thedata create + export as a fixture. Imagine I ran just "test_import_data_to_target_host", this would fail in a clean env as /tmp/standalone.ldif doesn't exist.

@firstyear
I used /tmp directory because our ansible role will copy the file between hosts and it was shortest path. But I could probably move it to ldif directory because instance should be already created on target host when copying.

As for the separate files:
Both of these tests are meant to be executed with a help of ansible role, that is why we need it in separate files, because they won't be run in our "standard environment".
That's what we have the skipif for, because MIGRATION constant is meant to be created only in that environment with help of ansible.

I agree, let's change ldif file location to the default dirsrv ldif directory.

@firstyear

Uhhhh you can't do this. You need to have test_import_data_to_target_host be in the same file as test_export, and you probably need thedata create + export as a fixture. Imagine I ran just "test_import_data_to_target_host", this would fail in a clean env as /tmp/standalone.ldif doesn't exist.

To add what @bsmejkal said, think of RHDS 10 -> RHDS 11 migration. There is no easy way to do multihost tests with pytest in our case (different python versions on RHEL 7 and 8, different additional setup is required to prepare system under test, rpms vs modules, etc). So we offload all of that to ansible (which is good at it) and run only the actual migration steps in pytest.

@firstyear
I used /tmp directory because our ansible role will copy the file between hosts and it was shortest path. But I could probably move it to ldif directory because instance should be already created on target host when copying.
As for the separate files:
Both of these tests are meant to be executed with a help of ansible role, that is why we need it in separate files, because they won't be run in our "standard environment".
That's what we have the skipif for, because MIGRATION constant is meant to be created only in that environment with help of ansible.

You can't assume people have this ansible role in our upstream tests. Perhaps you should be shipping with an ldif in a data dir from exports of a specific version or something then? We have other examples of this I think.

You can't assume people have this ansible role in our upstream tests. Perhaps you should be shipping with an ldif in a data dir from exports of a specific version or something then? We have other examples of this I think.

And we don't assume! That's why these test cases are skipped by default. You won't run them by accident in your environment.
And please keep in mind, this is a first test in a series, we need to streamline our testing workflow for this.

I wonder if there is a better self contained approach here - lib389's tests are great because you can run them with very little extra setup. Given the only data needed is "an ldif" here, I still think ansible here is overkill for the purpose of testing the upgrades. Maybe there is just something about the test you have in mind that I'm not seeing.

I wonder if there is a better self contained approach here - lib389's tests are great because you can run them with very little extra setup.

That's not entirely true. Try to run them on RHEL7 or RHEL6. Only recently python3 package has appeared in RHEL7.7. Before that the best option was to use SCL. Other lib389 dependencies are also outdated on these releases, so you would need gcc and -devel packages to install python-ldap or python-nss from pypi. So it requires some non-trivial amount of efforts to make an old RHEL system ready to run pytest.

Given the only data needed is "an ldif" here, I still think ansible here is overkill for the purpose of testing the upgrades. Maybe there is just something about the test you have in mind that I'm not seeing.

It's for testing migration, not upgrade. So it involves 2 different systems. We're automating scenarios covered in https://access.redhat.com/documentation/en-us/red_hat_directory_server/10/html/installation_guide/migrating_from_rhds_9_to_rhds_10

rebased onto a0aefc9e553f86df1e59bd7886c0b8f920139a32

4 years ago

@vashirov @firstyear
Changed exported ldif file location to the default dirsrv ldif directory. Please review.
Thanks!

rebased onto c95f6cf

4 years ago

Pull-Request has been merged by vashirov

4 years ago

389-ds-base is moving from Pagure to Github. This means that new issues and pull requests
will be accepted only in 389-ds-base's github repository.

This pull request has been cloned to Github as issue and is available here:
- https://github.com/389ds/389-ds-base/issues/3617

If you want to continue to work on the PR, please navigate to the github issue,
download the patch from the attachments and file a new pull request.

Thank you for understanding. We apologize for all inconvenience.

Pull-Request has been closed by spichugi

3 years ago