From 63747bc0c041382536e2cad121bb591e42df3a2f Mon Sep 17 00:00:00 2001 From: Stanislav Levin Date: Apr 28 2020 15:50:10 +0000 Subject: ipatests: Collect all logs on all Unix hosts Each integration test entity sets up its own list of logfiles. This is made by calling the callback of host's 'collect_log', which knows nothing about the context of execution: whether it's the test class scope or the test method one. Of course, in this case one-time collection of test method log is not supported because the logs tracker collects only test class logs. In the meantime, almost all the entities (except 'client') collect identical logs. Besides, due to the IPA roles transformation an each IPA host can become master, replica or client, all of these, in turn, can have subroles. So, the most common case is the collection of all the possible logs from all the IPA (Unix) hosts. However, the customization of a logfiles collection is possible. The collection is performed with the help of 'integration_logs' fixture. For example, to add a logfile to list of logs on a test completion one should add the dependency on this fixture and call its 'collect_method_log' method. ``` class TestFoo(IntegrationTest): def test_foo(self): pass def test_bar(self, integration_logs): integration_logs.collect_method_log(self.master, '/logfile') ``` Collected logs: 1) 'test_foo' - default logs 2) 'test_bar' - default logs + /logfile 3) 'TestFoo' - default logs Fixes: https://pagure.io/freeipa/issue/8265 Signed-off-by: Stanislav Levin Reviewed-By: Rob Crittenden --- diff --git a/ipaplatform/base/paths.py b/ipaplatform/base/paths.py index 6ac9431..81e0331 100644 --- a/ipaplatform/base/paths.py +++ b/ipaplatform/base/paths.py @@ -407,6 +407,7 @@ class BasePathNamespace: DIRSRV_LOCK_DIR = "/run/lock/dirsrv" ALL_SLAPD_INSTANCE_SOCKETS = "/run/slapd-*.socket" VAR_LOG_DIRSRV_INSTANCE_TEMPLATE = "/var/log/dirsrv/slapd-%s" + VAR_LOG_DIRSRV = "/var/log/dirsrv/" SLAPD_INSTANCE_ACCESS_LOG_TEMPLATE = "/var/log/dirsrv/slapd-%s/access" SLAPD_INSTANCE_ERROR_LOG_TEMPLATE = "/var/log/dirsrv/slapd-%s/errors" SLAPD_INSTANCE_AUDIT_LOG_TEMPLATE = "/var/log/dirsrv/slapd-%s/audit" diff --git a/ipatests/pytest_ipa/integration/__init__.py b/ipatests/pytest_ipa/integration/__init__.py index 3b2ff3c..ab7f911 100644 --- a/ipatests/pytest_ipa/integration/__init__.py +++ b/ipatests/pytest_ipa/integration/__init__.py @@ -40,6 +40,51 @@ from . import tasks logger = logging.getLogger(__name__) +CLASS_LOGFILES = [ + # dirsrv logs + paths.VAR_LOG_DIRSRV, + # IPA install logs + paths.IPASERVER_INSTALL_LOG, + paths.IPACLIENT_INSTALL_LOG, + paths.IPAREPLICA_INSTALL_LOG, + paths.IPAREPLICA_CONNCHECK_LOG, + paths.IPAREPLICA_CA_INSTALL_LOG, + paths.IPASERVER_KRA_INSTALL_LOG, + paths.IPA_CUSTODIA_AUDIT_LOG, + paths.IPACLIENTSAMBA_INSTALL_LOG, + paths.IPACLIENTSAMBA_UNINSTALL_LOG, + paths.IPATRUSTENABLEAGENT_LOG, + # IPA uninstall logs + paths.IPASERVER_UNINSTALL_LOG, + paths.IPACLIENT_UNINSTALL_LOG, + # IPA upgrade logs + paths.IPAUPGRADE_LOG, + # IPA backup and restore logs + paths.IPARESTORE_LOG, + paths.IPABACKUP_LOG, + # kerberos related logs + paths.KADMIND_LOG, + paths.KRB5KDC_LOG, + # httpd logs + paths.VAR_LOG_HTTPD_DIR, + # dogtag logs + paths.VAR_LOG_PKI_DIR, + # selinux logs + paths.VAR_LOG_AUDIT, + # sssd + paths.VAR_LOG_SSSD_DIR, + # system + paths.RESOLV_CONF, + paths.HOSTS, +] + + +def make_class_logs(host): + logs = list(CLASS_LOGFILES) + env_filename = os.path.join(host.config.test_dir, 'env.sh') + logs.append(env_filename) + return logs + def pytest_addoption(parser): group = parser.getgroup("IPA integration tests") @@ -57,33 +102,39 @@ def _get_logname_from_node(node): return name -def collect_test_logs(node, logs_dict, test_config): +def collect_test_logs(node, logs_dict, test_config, suffix=''): """Collect logs from a test - Calls collect_logs + Calls collect_logs and collect_systemd_journal :param node: The pytest collection node (request.node) :param logs_dict: Mapping of host to list of log filnames to collect :param test_config: Pytest configuration + :param suffix: The custom suffix of the name of logfiles' directory """ + name = '{node}{suffix}'.format( + node=_get_logname_from_node(node), + suffix=suffix, + ) + logfile_dir = test_config.getoption('logfile_dir') collect_logs( - name=_get_logname_from_node(node), + name=name, logs_dict=logs_dict, - logfile_dir=test_config.getoption('logfile_dir'), + logfile_dir=logfile_dir, beakerlib_plugin=test_config.pluginmanager.getplugin('BeakerLibPlugin'), ) + hosts = logs_dict.keys() # pylint: disable=dict-keys-not-iterating + collect_systemd_journal(name, hosts, logfile_dir) + -def collect_systemd_journal(node, hosts, test_config): +def collect_systemd_journal(name, hosts, logfile_dir=None): """Collect systemd journal from remote hosts - :param node: The pytest collection node (request.node) + :param name: Name under which logs are collected, e.g. name of the test :param hosts: List of hosts from which to collect journal - :param test_config: Pytest configuration + :param logfile_dir: Directory to log to """ - name = _get_logname_from_node(node) - logfile_dir = test_config.getoption('logfile_dir') - if logfile_dir is None: return @@ -133,6 +184,8 @@ def collect_logs(name, logs_dict, logfile_dir=None, beakerlib_plugin=None): for host, logs in logs_dict.items(): logger.info('Collecting logs from: %s', host.hostname) + # make list of unique log filenames + logs = list(set(logs)) dirname = os.path.join(topdirname, host.hostname) if not os.path.isdir(dirname): os.makedirs(dirname) @@ -182,20 +235,116 @@ def collect_logs(name, logs_dict, logfile_dir=None, beakerlib_plugin=None): shutil.rmtree(topdirname) +class IntegrationLogs: + """Represent logfile collections + Collection is a mapping of IPA hosts and a list of logfiles to be + collected. There are two types of collections: class and method. + The former contains a list of logfiles which will be collected on + each test (within class) completion, while the latter contains + a list of logfiles which will be collected on only certain test + completion (once). + """ + def __init__(self): + self._class_logs = {} + self._method_logs = {} + + def set_logs(self, host, logs): + self._class_logs[host] = list(logs) + + @property + def method_logs(self): + return self._method_logs + + @property + def class_logs(self): + return self._class_logs + + def init_method_logs(self): + """Initilize method logs with the class ones""" + self._method_logs = {} + for k in self._class_logs: + self._method_logs[k] = list(self._class_logs[k]) + + def collect_class_log(self, host, filename): + """Add class scope log + The file with the given filename will be collected from the + host on an each test completion(within a test class). + """ + logger.info('Adding %s:%s to list of class logs to collect', + host.external_hostname, filename) + self._class_logs.setdefault(host, []).append(filename) + self._method_logs.setdefault(host, []).append(filename) + + def collect_method_log(self, host, filename): + """Add method scope log + The file with the given filename will be collected from the + host on a test completion. + """ + logger.info('Adding %s:%s to list of method logs to collect', + host.external_hostname, filename) + self._method_logs.setdefault(host, []).append(filename) + + @pytest.fixture(scope='class') -def class_integration_logs(): - """Internal fixture providing class-level logs_dict""" - return {} +def class_integration_logs(request): + """Internal fixture providing class-level logs_dict + For adjusting collection of logs, please, use 'integration_logs' + fixture. + """ + integration_logs = IntegrationLogs() + yield integration_logs + # since the main fixture of integration tests('mh') depends on + # this one the class logs collecting happens *after* the teardown + # of that fixture. The 'uninstall' is among the finalizers of 'mh'. + # This means that the logs collected here are the IPA *uninstall* + # logs. + class_logs = integration_logs.class_logs + collect_test_logs(request.node, class_logs, request.config, + suffix='-uninstall') @pytest.fixture def integration_logs(class_integration_logs, request): """Provides access to test integration logs, and collects after each test + To collect a logfile on a test completion one should add the dependency on + this fixture and call its 'collect_method_log' method. + For example, run TestFoo. + ``` + class TestFoo(IntegrationTest): + def test_foo(self): + pass + + def test_bar(self, integration_logs): + integration_logs.collect_method_log(self.master, '/logfile') + ``` + '/logfile' will be collected only for 'test_bar' test. + + To collect a logfile on a test class completion one should add the + dependency on this fixture and call its 'collect_class_log' method. + For example, run TestFoo. + ``` + class TestFoo(IntegrationTest): + def test_foo(self, integration_logs): + integration_logs.collect_class_log(self.master, '/logfile') + + def test_bar(self): + pass + ``` + '/logfile' will be collected 3 times: + 1) on 'test_foo' completion + 2) on 'test_bar' completion + 3) on 'TestFoo' completion + + Note, the registration of a collection works at the runtime. This means + that if the '/logfile' will be registered in 'test_bar' then + it will not be collected on 'test_foo' completion: + 1) on 'test_bar' completion + 2) on 'TestFoo' completion """ + class_integration_logs.init_method_logs() yield class_integration_logs - hosts = class_integration_logs.keys() - collect_test_logs(request.node, class_integration_logs, request.config) - collect_systemd_journal(request.node, hosts, request.config) + method_logs = class_integration_logs.method_logs + collect_test_logs(request.node, method_logs, request.config) @pytest.fixture(scope='class') @@ -255,17 +404,15 @@ def mh(request, class_integration_logs): for domain in ad_domains: mh.ad_treedomains.extend(domain.hosts_by_role('ad_treedomain')) - cls.logs_to_collect = class_integration_logs - - def collect_log(host, filename): - logger.info('Adding %s:%s to list of logs to collect', - host.external_hostname, filename) - class_integration_logs.setdefault(host, []).append(filename) + cls.logs_to_collect = class_integration_logs.class_logs if logger.isEnabledFor(logging.INFO): logger.info(pformat(mh.config.to_dict())) + + for ipa_host in mh.config.get_all_ipa_hosts(): + class_integration_logs.set_logs(ipa_host, make_class_logs(ipa_host)) + for host in mh.config.get_all_hosts(): - host.add_log_collector(collect_log) logger.info('Preparing host %s', host.hostname) tasks.prepare_host(host) @@ -278,13 +425,17 @@ def mh(request, class_integration_logs): try: yield mh.install() finally: - hosts = list(cls.get_all_hosts()) - for host in hosts: - host.remove_log_collector(collect_log) - collect_test_logs( - request.node, class_integration_logs, request.config - ) - collect_systemd_journal(request.node, hosts, request.config) + # the 'mh' fixture depends on 'class_integration_logs' one, + # thus, the class logs collecting happens *after* the teardown + # of 'mh' fixture. The 'uninstall' is among the finalizers of 'mh'. + # This means that the logs collected here are the IPA *uninstall* + # logs and the 'install' ones can be removed during the IPA + # uninstall phase. To address this problem(e.g. installation error) + # the install logs will be collected into '{nodeid}-install' directory + # while the uninstall ones into '{nodeid}-uninstall'. + class_logs = class_integration_logs.class_logs + collect_test_logs(request.node, class_logs, request.config, + suffix='-install') def add_compat_attrs(cls, mh): diff --git a/ipatests/pytest_ipa/integration/config.py b/ipatests/pytest_ipa/integration/config.py index 8a69eb0..6254ee2 100644 --- a/ipatests/pytest_ipa/integration/config.py +++ b/ipatests/pytest_ipa/integration/config.py @@ -87,6 +87,11 @@ class Config(pytest_multihost.config.Config): for host in domain.hosts: yield host + def get_all_ipa_hosts(self): + for ipa_domain in (d for d in self.domains if d.is_ipa_type): + for ipa_host in ipa_domain.hosts: + yield ipa_host + def to_dict(self): extra_args = self.extra_init_args - {'dirman_dn'} result = super(Config, self).to_dict(extra_args) diff --git a/ipatests/pytest_ipa/integration/tasks.py b/ipatests/pytest_ipa/integration/tasks.py index 5743407..27e1327 100755 --- a/ipatests/pytest_ipa/integration/tasks.py +++ b/ipatests/pytest_ipa/integration/tasks.py @@ -64,52 +64,6 @@ from .firewall import Firewall logger = logging.getLogger(__name__) -def setup_server_logs_collecting(host): - """ - This function setup logs to be collected on host. We should collect all - possible logs that may be helpful to debug IPA server - """ - # dirsrv logs - inst = host.domain.realm.replace('.', '-') - host.collect_log(paths.SLAPD_INSTANCE_ERROR_LOG_TEMPLATE % inst) - host.collect_log(paths.SLAPD_INSTANCE_ACCESS_LOG_TEMPLATE % inst) - host.collect_log(paths.SLAPD_INSTANCE_AUDIT_LOG_TEMPLATE % inst) - - # IPA install logs - host.collect_log(paths.IPASERVER_INSTALL_LOG) - host.collect_log(paths.IPASERVER_UNINSTALL_LOG) - host.collect_log(paths.IPACLIENT_INSTALL_LOG) - host.collect_log(paths.IPACLIENT_UNINSTALL_LOG) - host.collect_log(paths.IPAREPLICA_INSTALL_LOG) - host.collect_log(paths.IPAREPLICA_CONNCHECK_LOG) - host.collect_log(paths.IPAREPLICA_CA_INSTALL_LOG) - host.collect_log(paths.IPASERVER_KRA_INSTALL_LOG) - host.collect_log(paths.IPA_CUSTODIA_AUDIT_LOG) - - # IPA uninstall logs - host.collect_log(paths.IPACLIENT_UNINSTALL_LOG) - - # IPA backup and restore logs - host.collect_log(paths.IPARESTORE_LOG) - host.collect_log(paths.IPABACKUP_LOG) - - # kerberos related logs - host.collect_log(paths.KADMIND_LOG) - host.collect_log(paths.KRB5KDC_LOG) - - # httpd logs - host.collect_log(paths.VAR_LOG_HTTPD_ERROR) - - # dogtag logs - host.collect_log(os.path.join(paths.VAR_LOG_PKI_DIR)) - - # selinux logs - host.collect_log(paths.VAR_LOG_AUDIT) - - # SSSD debugging must be set after client is installed (function - # setup_sssd_debugging) - - def check_arguments_are(slice, instanceof): """ :param: slice - tuple of integers denoting the beginning and the end @@ -144,8 +98,6 @@ def prepare_host(host): # First we try to run simple echo command to test the connection host.run_command(['true'], set_env=False) - - host.collect_log(env_filename) try: host.transport.mkdir_recursive(host.config.test_dir) except IOError: @@ -337,7 +289,6 @@ def install_master(host, setup_dns=True, setup_kra=False, setup_adtrust=False, if domain_level is None: domain_level = host.config.domain_level check_domain_level(domain_level) - setup_server_logs_collecting(host) apply_common_fixes(host) fix_apache_semaphores(host) fw = Firewall(host) @@ -449,7 +400,6 @@ def install_replica(master, replica, setup_ca=True, setup_dns=False, domain_level = domainlevel(master) check_domain_level(domain_level) apply_common_fixes(replica) - setup_server_logs_collecting(replica) allow_sync_ptr(master) fw = Firewall(replica) fw_services = ["freeipa-ldap", "freeipa-ldaps"] @@ -515,8 +465,6 @@ def install_replica(master, replica, setup_ca=True, setup_dns=False, def install_client(master, client, extra_args=[], user=None, password=None, unattended=True, stdin_text=None): - client.collect_log(paths.IPACLIENT_INSTALL_LOG) - apply_common_fixes(client) allow_sync_ptr(master) # Now, for the situations where a client resides in a different subnet from @@ -558,9 +506,6 @@ def install_adtrust(host): Runs ipa-adtrust-install on the client and generates SIDs for the entries. Configures the compat tree for the legacy clients. """ - - setup_server_logs_collecting(host) - kinit_admin(host) host.run_command(['ipa-adtrust-install', '-U', '--enable-compat', @@ -782,8 +727,6 @@ def setup_sssd_debugging(host): paths.SSSD_CONF], raiseonerr=False) - host.collect_log(os.path.join(paths.VAR_LOG_SSSD_DIR)) - # Clear the cache and restart SSSD clear_sssd_cache(host) @@ -988,7 +931,6 @@ def kinit_admin(host, raiseonerr=True): def uninstall_master(host, ignore_topology_disconnect=True, ignore_last_of_role=True, clean=True, verbose=False): - host.collect_log(paths.IPASERVER_UNINSTALL_LOG) uninstall_cmd = ['ipa-server-install', '--uninstall', '-U'] host_domain_level = domainlevel(host) @@ -1028,8 +970,6 @@ def uninstall_master(host, ignore_topology_disconnect=True, def uninstall_client(host): - host.collect_log(paths.IPACLIENT_UNINSTALL_LOG) - host.run_command(['ipa-client-install', '--uninstall', '-U'], raiseonerr=False) unapply_fixes(host) @@ -1544,10 +1484,7 @@ def install_kra(host, domain_level=None, first_instance=False, raiseonerr=True): domain_level = domainlevel(host) check_domain_level(domain_level) command = ["ipa-kra-install", "-U", "-p", host.config.dirman_password] - try: - result = host.run_command(command, raiseonerr=raiseonerr) - finally: - setup_server_logs_collecting(host) + result = host.run_command(command, raiseonerr=raiseonerr) return result @@ -1570,10 +1507,7 @@ def install_ca( if cert_files: for fname in cert_files: command.extend(['--external-cert-file', fname]) - try: - result = host.run_command(command, raiseonerr=raiseonerr) - finally: - setup_server_logs_collecting(host) + result = host.run_command(command, raiseonerr=raiseonerr) return result