From 9d5f21824233bc959eb5946f0e7c9ad6cebadad7 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Aug 01 2017 17:46:46 +0000 Subject: [PATCH 1/2] Add scripts directory and README file. Add merge-standard-inventory script to run and merge output from the default standard inventory scripts. --- diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..8c61c34 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,13 @@ +# Standard Test Interface Scripts + +## `merge-standard-inventory` + +`merge-standard-inventory` is a wrapper script that runs _all_ of the available Standard Testing Interface dynamic inventory scripts from `/usr/share/ansible/inventory` and merges their output. This allows an individual test to provide a single customized dynamic inventory script that sets up certain environment variables before running all the default standard-inventory scripts. To make use of it, simply have `ansible` use the custom inventory script (which makes use of the `merge-standard-inventory` wrapper) in place of the `/usr/share/ansible/inventory` directory. + +The following short example custom inventory script sets the `TEST_DOCKER_EXTRA_ARGS` environment variable before running `merge-standard-inventory` with any command line arguments provided by ansible. By doing this, when `merge-standard-inventory` runs the `standard-inventory-docker` script it will launch the docker container with `CAP_SYS_ADMIN` capabilities (if the `TEST_SUBJECTS` environment variable triggers a container to be launched). Other `standard-inventory-*` scripts will simply ignore the `TEST_DOCKER_EXTRA_ARGS` environment variable--but may have their own environment variables set in the inventory script. + +``` +#!/bin/bash +export TEST_DOCKER_EXTRA_ARGS="--cap-add=SYS_ADMIN" +exec merge-standard-inventory "$@" +``` diff --git a/scripts/merge-standard-inventory b/scripts/merge-standard-inventory new file mode 100755 index 0000000..faeec61 --- /dev/null +++ b/scripts/merge-standard-inventory @@ -0,0 +1,180 @@ +#!/usr/bin/python2 + +import argparse +import json +import os +import sys +import subprocess +import time + + +def main(argv): + """ + Run all standard inventory scripts and return their merged output. + + Note the standard inventory scripts clean up their spawned hosts when they detect their parent processes go away. To accomodate that behavior, this script forks a child process to run the inventory scripts, send back the merged inventory, and then wait for the parent of this script (ansible) to die before silently exiting. In the mean time, this script outputs the merged inventory gathered by the child and then exits. + """ + + # Keep track of our parent + waitpid = os.getppid() + + pipein, pipeout = os.pipe() + + childpid = os.fork() + if childpid == 0: + # this is the child process + + # close the inherited input side of the pipe + os.close(pipein) + + # run and merge output from standard inventory scripts + merged_data = merge_standard_inventories(argv[1:]) + + # send merged data to parent via output pipe + os.write(pipeout, merged_data) + + # close the pipe so the parent knows we are done + os.close(pipeout) + + # wait for the grandparent process to exit + linger(waitpid) + + # exit cleanly + sys.exit(0) + + # this is the parent process + + # close the inherited output side of the pipe + os.close(pipeout) + + # send eveything from the child to stdout + while True: + data = os.read(pipein, 999) + if not data: + os.close(pipein) + break + sys.stdout.write(data) + + return 0 + + +def merge_standard_inventories(args): + + inventory_dir = os.environ.get( + "TEST_DYNAMIC_INVENTORY_DIRECTORY", "/usr/share/ansible/inventory") + inventory_ignore_extensions = ( + "~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo") + + merged = Inventory() + + for i in os.listdir(inventory_dir): + if not i.startswith("standard-inventory-"): + continue + if i.endswith(inventory_ignore_extensions): + continue + + cmd = [os.path.join(inventory_dir, i)] + args + + try: + inv_out = subprocess.check_output(cmd, stdin=None, close_fds=True) + except subprocess.CalledProcessError as ex: + raise RuntimeError("Could not run: {0}".format(str(cmd))) + + merged.merge(inv_out) + + return merged.dumps() + + +def linger(waitpid): + # Go into daemon mode and watch the process + null = open(os.devnull, 'w') + + try: + tty = os.open("/dev/tty", os.O_WRONLY) + os.dup2(tty, 2) + except OSError: + tty = None + pass + + os.chdir("/") + os.setsid() + os.umask(0) + + if tty is None: + tty = null.fileno() + + # Duplicate standard input to standard output and standard error. + os.dup2(null.fileno(), 0) + os.dup2(tty, 1) + os.dup2(tty, 2) + + # Now wait for the watched process to go away, then return + while True: + time.sleep(3) + + try: + os.kill(waitpid, 0) + except OSError: + break # The process no longer exists + + return + + +class Inventory: + + def __init__(self): + self.hosts = [] + self.variables = {} + + def merge_files(self, files): + for f in files: + with open(f) as ifile: + s = ifile.read() + self.merge(s) + + def merge(self, s): + # parse provided string as JSON + inventory = json.loads(s) + + if not isinstance(inventory, dict): + raise ValueError( + "inventory JSON does not contain the expected top level dictionary") + + if "subjects" not in inventory or not isinstance(inventory["subjects"], dict): + raise ValueError( + "inventory JSON does not contain the expected [subjects] dictionary") + if "hosts" not in inventory["subjects"] or not isinstance(inventory["subjects"]["hosts"], list): + raise ValueError( + "inventory JSON does not contain the expected [subjects][hosts] list") + + for h in inventory["subjects"]["hosts"]: + self.hosts.append(h) + + if "_meta" not in inventory or not isinstance(inventory["_meta"], dict): + raise ValueError( + "inventory JSON does not contain the expected [_meta] dictionary") + if "hostvars" not in inventory["_meta"] or not isinstance(inventory["_meta"]["hostvars"], dict): + raise ValueError( + "inventory JSON does not contain the expected [_meta][hostvars] dict") + + for h in inventory["_meta"]["hostvars"]: + if not isinstance(inventory["_meta"]["hostvars"][h], dict): + raise ValueError( + "inventory JSON does not contain the expected [_meta][hostvars][{0}] dict".format(h)) + + self.variables[h] = inventory["_meta"]["hostvars"][h] + + if "localhost" not in inventory or not isinstance(inventory["localhost"], dict): + raise ValueError( + "inventory JSON does not contain the expected [localhost] dictionary") + if "hosts" not in inventory["localhost"] or not isinstance(inventory["localhost"]["hosts"], list): + raise ValueError( + "inventory JSON does not contain the expected [localhost][hosts] list") + + def dumps(self): + data = {"subjects": {"hosts": self.hosts, "vars": {}}, "localhost": { + "hosts": self.hosts, "vars": {}}, "_meta": {"hostvars": self.variables}} + return json.dumps(data, indent=4, separators=(',', ': ')) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) From 8e1cb3ca09c1752863f867a9428d5441ed21f8a5 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Aug 03 2017 16:42:16 +0000 Subject: [PATCH 2/2] Updates to address review feedback. --- diff --git a/scripts/README.md b/scripts/README.md index 8c61c34..bf5da45 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,9 +2,26 @@ ## `merge-standard-inventory` -`merge-standard-inventory` is a wrapper script that runs _all_ of the available Standard Testing Interface dynamic inventory scripts from `/usr/share/ansible/inventory` and merges their output. This allows an individual test to provide a single customized dynamic inventory script that sets up certain environment variables before running all the default standard-inventory scripts. To make use of it, simply have `ansible` use the custom inventory script (which makes use of the `merge-standard-inventory` wrapper) in place of the `/usr/share/ansible/inventory` directory. +`merge-standard-inventory` is a wrapper script that runs _all_ of the +available Standard Testing Interface dynamic inventory scripts from +`/usr/share/ansible/inventory` and merges their output. This allows +an individual test to provide a single customized dynamic inventory +script that sets up certain environment variables before running all +the default standard-inventory scripts. To make use of it, simply +have `ansible` use the custom inventory script (which makes use of +the `merge-standard-inventory` wrapper) in place of the +`/usr/share/ansible/inventory` directory. -The following short example custom inventory script sets the `TEST_DOCKER_EXTRA_ARGS` environment variable before running `merge-standard-inventory` with any command line arguments provided by ansible. By doing this, when `merge-standard-inventory` runs the `standard-inventory-docker` script it will launch the docker container with `CAP_SYS_ADMIN` capabilities (if the `TEST_SUBJECTS` environment variable triggers a container to be launched). Other `standard-inventory-*` scripts will simply ignore the `TEST_DOCKER_EXTRA_ARGS` environment variable--but may have their own environment variables set in the inventory script. +The following short example custom inventory script sets the +`TEST_DOCKER_EXTRA_ARGS` environment variable before running +`merge-standard-inventory` with any command line arguments provided +by ansible. By doing this, when `merge-standard-inventory` runs the +`standard-inventory-docker` script it will launch the docker +container with `CAP_SYS_ADMIN` capabilities (if the `TEST_SUBJECTS` +environment variable triggers a container to be launched). Other +`standard-inventory-*` scripts will simply ignore the +`TEST_DOCKER_EXTRA_ARGS` environment variable--but may have their own +environment variables set in the inventory script. ``` #!/bin/bash diff --git a/scripts/merge-standard-inventory b/scripts/merge-standard-inventory index faeec61..64fcdc1 100755 --- a/scripts/merge-standard-inventory +++ b/scripts/merge-standard-inventory @@ -1,6 +1,5 @@ #!/usr/bin/python2 -import argparse import json import os import sys @@ -12,12 +11,19 @@ def main(argv): """ Run all standard inventory scripts and return their merged output. - Note the standard inventory scripts clean up their spawned hosts when they detect their parent processes go away. To accomodate that behavior, this script forks a child process to run the inventory scripts, send back the merged inventory, and then wait for the parent of this script (ansible) to die before silently exiting. In the mean time, this script outputs the merged inventory gathered by the child and then exits. + Note the standard inventory scripts clean up their spawned hosts + when they detect their parent processes go away. To accomodate + that behavior, this script forks a child process to run the + inventory scripts, send back the merged inventory, and then wait + for the parent of this script (ansible) to die before silently + exiting. In the mean time, this script outputs the merged + inventory gathered by the child and then exits. """ # Keep track of our parent waitpid = os.getppid() + tty = err_to_tty() pipein, pipeout = os.pipe() childpid = os.fork() @@ -37,7 +43,7 @@ def main(argv): os.close(pipeout) # wait for the grandparent process to exit - linger(waitpid) + linger(waitpid, tty) # exit cleanly sys.exit(0) @@ -68,12 +74,15 @@ def merge_standard_inventories(args): merged = Inventory() for i in os.listdir(inventory_dir): + ipath = os.path.join(inventory_dir, i) if not i.startswith("standard-inventory-"): continue if i.endswith(inventory_ignore_extensions): continue + if not os.access(ipath, os.X_OK): + continue - cmd = [os.path.join(inventory_dir, i)] + args + cmd = [ipath] + args try: inv_out = subprocess.check_output(cmd, stdin=None, close_fds=True) @@ -85,16 +94,20 @@ def merge_standard_inventories(args): return merged.dumps() -def linger(waitpid): - # Go into daemon mode and watch the process - null = open(os.devnull, 'w') - +def err_to_tty(): try: tty = os.open("/dev/tty", os.O_WRONLY) os.dup2(tty, 2) except OSError: tty = None - pass + + return tty + + +def linger(waitpid, tty=None): + # Go into daemon mode and watch the process + + null = open(os.devnull, 'w') os.chdir("/") os.setsid() @@ -121,6 +134,14 @@ def linger(waitpid): class Inventory: + """ + Merge JSON data from standard test dynamic inventory scripts. + + Note: This class is very specific to the JSON data written by the + ansible dynamic inventory scripts that are provided by the + standard-test-roles package. In particular, it insists on finding + and generating "subjects" and "localhost" members. + """ def __init__(self): self.hosts = []