From 3004b1bc5d00694778d05aa84502ab6251e351e4 Mon Sep 17 00:00:00 2001 From: Andrei Stepanov Date: Dec 06 2018 12:53:06 +0000 Subject: Reformat provisioners: docker, local, rpm. Signed-off-by: Andrei Stepanov --- diff --git a/inventory/standard-inventory-docker b/inventory/standard-inventory-docker index f8441aa..bc3371d 100755 --- a/inventory/standard-inventory-docker +++ b/inventory/standard-inventory-docker @@ -1,41 +1,43 @@ #!/usr/bin/python3 + # SPDX Licence identifier MIT # Copyright (c) 2017-2018 Red Hat Inc. # Author: Merlin Mathesius +# Andrei Stepanov +# Bruno Goncalves -import argparse -import errno -import json import os -import shutil +import sys +import time +import json +import errno import shlex +import shutil import signal -import subprocess -import sys +import logging import tempfile -import time +import argparse +import subprocess import distutils.util -import logging -def print_bad_inventory(exctype, value, tb): - """Print empty inventory on any uncatched exception. This will prevent +EMPTY_INVENTORY = {} +LOG_FILE = "default_provisioners.log" + + +def print_bad_inventory(): + """Print bad inventory on any uncatched exception. This will prevent running playbook on localhost. """ - hosts = ["fake_hostname_failed_inventory"] + fake_host = "fake_host" + fake_hostname = "standard-inventory-qcow2_failed_check_logs" + hosts = [fake_host] bad_inv = {"localhost": {"hosts": hosts, "vars": {}}, "subjects": {"hosts": hosts, "vars": {}}, - "_meta": {"hostvars": {}}} + "_meta": {"hostvars": {fake_host: {"ansible_host": fake_hostname}}}} sys.stdout.write(json.dumps(bad_inv, indent=4, separators=(',', ': '))) -sys.excepthook = print_bad_inventory - - -EMPTY_INVENTORY = {} -LOG_FILE = "default_provisioners.log" - - def get_artifact_path(path=""): """Return path to an artifact file in artifacts directory. If path == "" than return path artifacts dir. Create artifacts dir if necessary. @@ -49,53 +51,6 @@ def get_artifact_path(path=""): return os.path.join(artifacts, path) -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -# stderr output -conhandler = logging.StreamHandler() -# Print to strerr by default messages with level >= warning, can be changed -# with setting TEST_DEBUG=1. -try: - diagnose = distutils.util.strtobool(os.getenv("TEST_DEBUG", "0")) -except ValueError: - diagnose = 0 -conhandler.setLevel(logging.WARNING) -if diagnose: - # Collect all messages with any log level to stderr. - conhandler.setLevel(logging.NOTSET) -# Log format for stderr. -log_format = "[%(levelname)-5.5s] {}: %(message)s".format(os.path.basename(__file__)) -formatter = logging.Formatter(log_format) -conhandler.setFormatter(formatter) -logger.addHandler(conhandler) - - -def main(argv): - parser = argparse.ArgumentParser(description="Inventory for a container image in a registry") - parser.add_argument("--list", action="store_true", help="Verbose output") - parser.add_argument('--host', help="Get host variables") - parser.add_argument('--docker-extra-args', help="Extra docker arguments for launching container", - default=os.environ.get("TEST_DOCKER_EXTRA_ARGS", "")) - parser.add_argument("subjects", nargs="*", default=shlex.split(os.environ.get("TEST_SUBJECTS", ""))) - opts = parser.parse_args() - # Send logs to common logfile for all default provisioners. - log_file = get_artifact_path(LOG_FILE) - fhandler = logging.FileHandler(log_file) - # Collect all messages with any log level to log file. - fhandler.setLevel(logging.NOTSET) - log_format = ("%(asctime)s [{}/%(threadName)-12.12s] [%(levelname)-5.5s]:" - "%(message)s").format(os.path.basename(__file__)) - logFormatter = logging.Formatter(log_format) - fhandler.setFormatter(logFormatter) - logger.addHandler(fhandler) - logger.info("Start provisioner.") - if opts.host: - _, data = inv_host(opts.host, opts.docker_extra_args) - else: - data = inv_list(opts.subjects, opts.docker_extra_args) - sys.stdout.write(json.dumps(data, indent=4, separators=(',', ': '))) - - def inv_list(subjects, docker_extra_args): hosts = [] variables = {} @@ -114,17 +69,14 @@ def inv_list(subjects, docker_extra_args): def inv_host(subject, docker_extra_args): if not subject.startswith("docker:"): return None, EMPTY_INVENTORY - image = subject[7:] null = open(os.devnull, 'w') - try: tty = os.open("/dev/tty", os.O_WRONLY) os.dup2(tty, 2) except OSError: tty = None pass - directory = tempfile.mkdtemp(prefix="inventory-docker") cidfile = os.path.join(directory, "cid") # Check for any additional arguments to include when starting docker container @@ -132,9 +84,7 @@ def inv_host(subject, docker_extra_args): extra_arg_list = shlex.split(docker_extra_args) except ValueError: raise RuntimeError("Could not parse DOCKER_EXTRA_ARGS") - logger.info("Launching Docker container for {0}".format(image)) - # Make sure the docker service is running cmd = [ "/usr/bin/systemctl", "start", "docker" @@ -143,7 +93,6 @@ def inv_host(subject, docker_extra_args): subprocess.check_call(cmd, stdout=sys.stderr.fileno()) except subprocess.CalledProcessError: raise RuntimeError("Could not start docker service") - # And launch the actual container cmd = [ "/usr/bin/docker", "run", "--detach", "--cidfile={0}".format(cidfile), @@ -154,7 +103,6 @@ def inv_host(subject, docker_extra_args): subprocess.check_call(cmd, stdout=sys.stderr.fileno()) except subprocess.CalledProcessError: raise RuntimeError("Could not start container image: {0}".format(image)) - # Read out the container environment variable for _ in range(1, 90): if os.path.exists(cidfile): @@ -162,10 +110,8 @@ def inv_host(subject, docker_extra_args): time.sleep(1) else: raise RuntimeError("Could not find container file for launched container") - with open(cidfile, "r") as f: name = f.read().strip() - # Need to figure out what python interpreter to use interpreters = ["/usr/bin/python3", "/usr/bin/python2"] for interpreter in interpreters: @@ -179,54 +125,42 @@ def inv_host(subject, docker_extra_args): else: logger.error("Could not set ansible_python_interpreter.") return None - # Directory to place artifacts artifacts = os.environ.get("TEST_ARTIFACTS", os.path.join(os.getcwd(), "artifacts")) - # The variables variables = { "ansible_connection": "docker", "ansible_python_interpreter": ansible_python_interpreter } - # Process of our parent ppid = os.getppid() - child = os.fork() if child: return name, variables - # Daemonize and watch the processes os.chdir("/") os.setsid() os.umask(0) - if tty is None: tty = null.fileno() - # Duplicate standard input to standard output and standard error. os.dup2(null.fileno(), 0) os.dup2(tty, 1) os.dup2(tty, 2) - # Now wait for the parent process to go away, then kill the VM logger.info("docker exec -it {0} /bin/bash".format(name)) while True: time.sleep(3) - try: os.kill(ppid, 0) except OSError: break # Either of the processes no longer exist - if diagnose: def _signal_handler(*args): logger.info("Diagnose ending.") - logger.info("kill {0} # when finished".format(os.getpid())) signal.signal(signal.SIGTERM, _signal_handler) signal.pause() - # Dump the container logs try: os.makedirs(artifacts) @@ -234,22 +168,68 @@ def inv_host(subject, docker_extra_args): if exc.errno != errno.EEXIST or not os.path.isdir(artifacts): raise log = os.path.join(artifacts, "{0}.log".format(os.path.basename(image))) - # Kill the container with open(log, "w") as f: subprocess.call(["/usr/bin/docker", "logs", name], stdout=f.fileno()) subprocess.call(["/usr/bin/docker", "rm", "-f", name], stdout=null) - shutil.rmtree(directory) sys.exit(0) +def main(argv): + global logger + global diagnose + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + # stderr output + conhandler = logging.StreamHandler() + # Print to strerr by default messages with level >= warning, can be changed + # with setting TEST_DEBUG=1. + try: + diagnose = distutils.util.strtobool(os.getenv("TEST_DEBUG", "0")) + except ValueError: + diagnose = 0 + conhandler.setLevel(logging.WARNING) + if diagnose: + # Collect all messages with any log level to stderr. + conhandler.setLevel(logging.NOTSET) + # Log format for stderr. + log_format = "[%(levelname)-5.5s] {}: %(message)s".format(os.path.basename(__file__)) + formatter = logging.Formatter(log_format) + conhandler.setFormatter(formatter) + logger.addHandler(conhandler) + parser = argparse.ArgumentParser(description="Inventory for a container image in a registry") + parser.add_argument("--list", action="store_true", help="Verbose output") + parser.add_argument('--host', help="Get host variables") + parser.add_argument('--docker-extra-args', help="Extra docker arguments for launching container", + default=os.environ.get("TEST_DOCKER_EXTRA_ARGS", "")) + parser.add_argument("subjects", nargs="*", default=shlex.split(os.environ.get("TEST_SUBJECTS", ""))) + opts = parser.parse_args() + # Send logs to common logfile for all default provisioners. + log_file = get_artifact_path(LOG_FILE) + fhandler = logging.FileHandler(log_file) + # Collect all messages with any log level to log file. + fhandler.setLevel(logging.NOTSET) + log_format = ("%(asctime)s [{}/%(threadName)-12.12s] [%(levelname)-5.5s]:" + "%(message)s").format(os.path.basename(__file__)) + logFormatter = logging.Formatter(log_format) + fhandler.setFormatter(logFormatter) + logger.addHandler(fhandler) + logger.info("Start provisioner.") + if opts.host: + _, data = inv_host(opts.host, opts.docker_extra_args) + else: + data = inv_list(opts.subjects, opts.docker_extra_args) + sys.stdout.write(json.dumps(data, indent=4, separators=(',', ': '))) + + if __name__ == '__main__': ret = -1 try: main(sys.argv) ret = 0 except Exception: + print_bad_inventory() # Backtrace stack goes to log file. If TEST_DEBUG == 1, it goes to stderr too. logger.info("Fatal error in provision script.", exc_info=True) sys.exit(ret) diff --git a/inventory/standard-inventory-local b/inventory/standard-inventory-local index 0f777a0..ff5dd09 100755 --- a/inventory/standard-inventory-local +++ b/inventory/standard-inventory-local @@ -1,35 +1,38 @@ #!/usr/bin/python3 + # SPDX Licence identifier MIT # Copyright (c) 2017-2018 Red Hat Inc. # Author: Stef Walter +# Andrei Stepanov +# Bruno Goncalves + -import argparse -import json import os import sys -import logging +import json import errno +import logging +import argparse import distutils.util -def print_bad_inventory(exctype, value, tb): - """Print empty inventory on any uncatched exception. This will prevent +EMPTY_INVENTORY = {} +LOG_FILE = "default_provisioners.log" + + +def print_bad_inventory(): + """Print bad inventory on any uncatched exception. This will prevent running playbook on localhost. """ - hosts = ["fake_hostname_failed_inventory"] + fake_host = "fake_host" + fake_hostname = "standard-inventory-qcow2_failed_check_logs" + hosts = [fake_host] bad_inv = {"localhost": {"hosts": hosts, "vars": {}}, "subjects": {"hosts": hosts, "vars": {}}, - "_meta": {"hostvars": {}}} + "_meta": {"hostvars": {fake_host: {"ansible_host": fake_hostname}}}} sys.stdout.write(json.dumps(bad_inv, indent=4, separators=(',', ': '))) -sys.excepthook = print_bad_inventory - - -EMPTY_INVENTORY = {} -LOG_FILE = "default_provisioners.log" - - def get_artifact_path(path=""): """Return path to an artifact file in artifacts directory. If path == "" than return path artifacts dir. Create artifacts dir if necessary. @@ -43,29 +46,49 @@ def get_artifact_path(path=""): return os.path.join(artifacts, path) -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -# stderr output -conhandler = logging.StreamHandler() -# Print to strerr by default messages with level >= warning, can be changed -# with setting TEST_DEBUG=1. -try: - diagnose = distutils.util.strtobool(os.getenv("TEST_DEBUG", "0")) -except ValueError: - diagnose = 0 -conhandler.setLevel(logging.WARNING) -if diagnose: - # Collect all messages with any log level to stderr. - conhandler.setLevel(logging.NOTSET) -# Log format for stderr. - -log_format = "[%(levelname)-5.5s] {}: %(message)s".format(os.path.basename(__file__)) -formatter = logging.Formatter(log_format) -conhandler.setFormatter(formatter) -logger.addHandler(conhandler) +def inv_list(): + hosts = [] + variables = {} + if os.environ.get("TEST_SUBJECTS", None) == "local": + host_vars = inv_host("local") + if host_vars: + hosts.append("local") + variables["local"] = host_vars + if not hosts: + return EMPTY_INVENTORY + return {"subjects": {"hosts": hosts, "vars": {}}, + "localhost": {"hosts": hosts, "vars": {}}, + "_meta": {"hostvars": variables}} + + +def inv_host(host): + if host == "local": + return {"ansible_connection": "local"} + return EMPTY_INVENTORY def main(argv): + global logger + global diagnose + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + # stderr output + conhandler = logging.StreamHandler() + # Print to strerr by default messages with level >= warning, can be changed + # with setting TEST_DEBUG=1. + try: + diagnose = distutils.util.strtobool(os.getenv("TEST_DEBUG", "0")) + except ValueError: + diagnose = 0 + conhandler.setLevel(logging.WARNING) + if diagnose: + # Collect all messages with any log level to stderr. + conhandler.setLevel(logging.NOTSET) + # Log format for stderr. + log_format = "[%(levelname)-5.5s] {}: %(message)s".format(os.path.basename(__file__)) + formatter = logging.Formatter(log_format) + conhandler.setFormatter(formatter) + logger.addHandler(conhandler) parser = argparse.ArgumentParser(description="Inventory for local") parser.add_argument("--list", action="store_true", help="Verbose output") parser.add_argument('--host', help="Get host variables") @@ -88,33 +111,13 @@ def main(argv): sys.stdout.write(json.dumps(data, indent=4, separators=(',', ': '))) -def inv_list(): - hosts = [] - variables = {} - if os.environ.get("TEST_SUBJECTS", None) == "local": - host_vars = inv_host("local") - if host_vars: - hosts.append("local") - variables["local"] = host_vars - if not hosts: - return EMPTY_INVENTORY - return {"subjects": {"hosts": hosts, "vars": {}}, - "localhost": {"hosts": hosts, "vars": {}}, - "_meta": {"hostvars": variables}} - - -def inv_host(host): - if host == "local": - return {"ansible_connection": "local"} - return EMPTY_INVENTORY - - if __name__ == '__main__': ret = -1 try: main(sys.argv) ret = 0 except Exception: + print_bad_inventory() # Backtrace stack goes to log file. If TEST_DEBUG == 1, it goes to stderr too. logger.info("Fatal error in provision script.", exc_info=True) sys.exit(ret) diff --git a/inventory/standard-inventory-rpm b/inventory/standard-inventory-rpm index bbaf7f7..a93f5e9 100755 --- a/inventory/standard-inventory-rpm +++ b/inventory/standard-inventory-rpm @@ -1,38 +1,41 @@ #!/usr/bin/python3 + # SPDX Licence identifier MIT # Copyright (c) 2017-2018 Red Hat Inc. # Authors: Merlin Mathesius # Stef Walter +# Andrei Stepanov +# Bruno Goncalves + -import argparse -import json import os -import shlex -import subprocess import sys -import logging +import json import errno +import shlex +import logging +import argparse +import subprocess import distutils.util -def print_bad_inventory(exctype, value, tb): - """Print empty inventory on any uncatched exception. This will prevent +EMPTY_INVENTORY = {} +LOG_FILE = "default_provisioners.log" + + +def print_bad_inventory(): + """Print bad inventory on any uncatched exception. This will prevent running playbook on localhost. """ - hosts = ["fake_hostname_failed_inventory"] + fake_host = "fake_host" + fake_hostname = "standard-inventory-qcow2_failed_check_logs" + hosts = [fake_host] bad_inv = {"localhost": {"hosts": hosts, "vars": {}}, "subjects": {"hosts": hosts, "vars": {}}, - "_meta": {"hostvars": {}}} + "_meta": {"hostvars": {fake_host: {"ansible_host": fake_hostname}}}} sys.stdout.write(json.dumps(bad_inv, indent=4, separators=(',', ': '))) -sys.excepthook = print_bad_inventory - - -EMPTY_INVENTORY = {} -LOG_FILE = "default_provisioners.log" - - def get_artifact_path(path=""): """Return path to an artifact file in artifacts directory. If path == "" than return path artifacts dir. Create artifacts dir if necessary. @@ -46,61 +49,13 @@ def get_artifact_path(path=""): return os.path.join(artifacts, path) -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -# stderr output -conhandler = logging.StreamHandler() -# Print to strerr by default messages with level >= warning, can be changed -# with setting TEST_DEBUG=1. -try: - diagnose = distutils.util.strtobool(os.getenv("TEST_DEBUG", "0")) -except ValueError: - diagnose = 0 -conhandler.setLevel(logging.WARNING) -if diagnose: - # Collect all messages with any log level to stderr. - conhandler.setLevel(logging.NOTSET) -# Log format for stderr. -log_format = "[%(levelname)-5.5s] {}: %(message)s".format(os.path.basename(__file__)) -formatter = logging.Formatter(log_format) -conhandler.setFormatter(formatter) -logger.addHandler(conhandler) - - -def main(argv): - parser = argparse.ArgumentParser(description="Inventory for local RPM installed host") - parser.add_argument("--list", action="store_true", help="Verbose output") - parser.add_argument('--host', help="Get host variables") - parser.add_argument("subjects", nargs="*", default=os.environ.get("TEST_SUBJECTS", "")) - opts = parser.parse_args() - # Send logs to common logfile for all default provisioners. - log_file = get_artifact_path(LOG_FILE) - fhandler = logging.FileHandler(log_file) - # Collect all messages with any log level to log file. - fhandler.setLevel(logging.NOTSET) - log_format = ("%(asctime)s [{}/%(threadName)-12.12s] [%(levelname)-5.5s]:" - "%(message)s").format(os.path.basename(__file__)) - logFormatter = logging.Formatter(log_format) - fhandler.setFormatter(logFormatter) - logger.addHandler(fhandler) - logger.info("Start provisioner.") - if opts.host: - data = gethost(opts.host) - else: - data = getlist(opts.subjects) - sys.stdout.write(json.dumps(data, indent=4, separators=(',', ': '))) - - def getlist(subjects): hosts = [] variables = {} - host_vars = gethost(subjects) if host_vars: hosts.append("rpms") variables["rpms"] = host_vars - if not hosts: return EMPTY_INVENTORY return {"subjects": {"hosts": hosts, "vars": {}}, @@ -112,28 +67,23 @@ def gethost(subjects): subjects = shlex.split(subjects) repos = [] rpms = [] - for subject in subjects: if subject.endswith(".rpm"): rpms.append(subject) elif isrepo(subject): repos.append(subject) - if not repos and not rpms: return EMPTY_INVENTORY - # The variables variables = { "ansible_connection": "local" } - try: tty = os.open("/dev/tty", os.O_WRONLY) os.dup2(tty, 2) except OSError: tty = None pass - # enable any provided repos first so RPMs can pull dependencies from them if needed for repo in repos: addrepo = ["/usr/bin/yum", "config-manager", "--add-repo", repo] @@ -141,14 +91,12 @@ def gethost(subjects): subprocess.check_call(addrepo, stdout=sys.stderr.fileno()) except subprocess.CalledProcessError: raise RuntimeError("could not add repo: {0}".format(repo)) - if rpms: install = ["/usr/bin/yum", "-y", "install"] + rpms try: subprocess.check_call(install, stdout=sys.stderr.fileno()) except subprocess.CalledProcessError: raise RuntimeError("could not install rpms: {0}".format(rpms)) - return variables @@ -156,12 +104,58 @@ def isrepo(subject): return os.path.isfile(os.path.join(subject, "repodata", "repomd.xml")) +def main(argv): + global logger + global diagnose + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + # stderr output + conhandler = logging.StreamHandler() + # Print to strerr by default messages with level >= warning, can be changed + # with setting TEST_DEBUG=1. + try: + diagnose = distutils.util.strtobool(os.getenv("TEST_DEBUG", "0")) + except ValueError: + diagnose = 0 + conhandler.setLevel(logging.WARNING) + if diagnose: + # Collect all messages with any log level to stderr. + conhandler.setLevel(logging.NOTSET) + # Log format for stderr. + log_format = "[%(levelname)-5.5s] {}: %(message)s".format(os.path.basename(__file__)) + formatter = logging.Formatter(log_format) + conhandler.setFormatter(formatter) + logger.addHandler(conhandler) + parser = argparse.ArgumentParser(description="Inventory for local RPM installed host") + parser.add_argument("--list", action="store_true", help="Verbose output") + parser.add_argument('--host', help="Get host variables") + parser.add_argument("subjects", nargs="*", default=os.environ.get("TEST_SUBJECTS", "")) + opts = parser.parse_args() + # Send logs to common logfile for all default provisioners. + log_file = get_artifact_path(LOG_FILE) + fhandler = logging.FileHandler(log_file) + # Collect all messages with any log level to log file. + fhandler.setLevel(logging.NOTSET) + log_format = ("%(asctime)s [{}/%(threadName)-12.12s] [%(levelname)-5.5s]:" + "%(message)s").format(os.path.basename(__file__)) + logFormatter = logging.Formatter(log_format) + fhandler.setFormatter(logFormatter) + logger.addHandler(fhandler) + logger.info("Start provisioner.") + if opts.host: + data = gethost(opts.host) + else: + data = getlist(opts.subjects) + sys.stdout.write(json.dumps(data, indent=4, separators=(',', ': '))) + + if __name__ == '__main__': ret = -1 try: main(sys.argv) ret = 0 except Exception: + print_bad_inventory() # Backtrace stack goes to log file. If TEST_DEBUG == 1, it goes to stderr too. logger.info("Fatal error in provision script.", exc_info=True) sys.exit(ret)