#39 Fix standard-inventory-qcow2 standards compliance
Opened 6 years ago by cevich. Modified 6 years ago
cevich/standard-test-roles fix_qcow2  into  master

file added
+69
@@ -0,0 +1,69 @@ 

+ # Backup files

+ .*~

+ *~

+ 

+ # Vim swap files

+ .*.swp

+ 

+ # Ansible retry files

+ *.retry

+ 

+ # Byte-compiled / optimized / DLL files

+ __pycache__

+ *.py[cod]

+ *$py.class

+ 

+ # C extensions

+ *.so

+ 

+ # Distribution / packaging

+ .Python

+ env/

+ build/

+ develop-eggs/

+ dist/

+ downloads/

+ eggs/

+ .eggs/

+ lib/

+ lib64/

+ parts/

+ sdist/

+ var/

+ wheels/

+ *.egg-info/

+ .installed.cfg

+ *.egg

+ 

+ # PyInstaller

+ #  Usually these files are written by a python script from a template

+ #  before PyInstaller builds the exe, so as to inject date/other infos into it.

+ *.manifest

+ *.spec

+ 

+ # Installer logs

+ pip-log.txt

+ pip-delete-this-directory.txt

+ 

+ # Unit test / coverage reports

+ htmlcov/

+ .tox/

+ .coverage

+ .coverage.*

+ .cache

+ nosetests.xml

+ coverage.xml

+ *.cover

+ .hypothesis/

+ 

+ # Translations

+ *.mo

+ *.pot

+ 

+ # virtualenv

+ .venv

+ venv/

+ ENV/

+ 

+ # Default location if running from repo-clone

+ artifacts/

file modified
+689 -224
@@ -9,53 +9,50 @@ 

  import signal

  import multiprocessing

  import socket

+ from contextlib import contextmanager

+ import fcntl

  import subprocess

  import sys

  import tempfile

  import time

- import distutils.util

- 

- IDENTITY = """

- -----BEGIN RSA PRIVATE KEY-----

- MIIEpQIBAAKCAQEA1DrTSXQRF8isQQfPfK3U+eFC4zBrjur+Iy15kbHUYUeSHf5S

- jXPYbHYqD1lHj4GJajC9okle9rykKFYZMmJKXLI6987wZ8vfucXo9/kwS6BDAJto

- ZpZSj5sWCQ1PI0Ce8CbkazlTp5NIkjRfhXGP8mkNKMEhdNjaYceO49ilnNCIxhpb

- eH5dH5hybmQQNmnzf+CGCCLBFmc4g3sFbWhI1ldyJzES5ZX3ahjJZYRUfnndoUM/

- TzdkHGqZhL1EeFAsv5iV65HuYbchch4vBAn8jDMmHh8G1ixUCL3uAlosfarZLLyo

- 3HrZ8U/llq7rXa93PXHyI/3NL/2YP3OMxE8baQIDAQABAoIBAQCxuOUwkKqzsQ9W

- kdTWArfj3RhnKigYEX9qM+2m7TT9lbKtvUiiPc2R3k4QdmIvsXlCXLigyzJkCsqp

- IJiPEbJV98bbuAan1Rlv92TFK36fBgC15G5D4kQXD/ce828/BSFT2C3WALamEPdn

- v8Xx+Ixjokcrxrdeoy4VTcjB0q21J4C2wKP1wEPeMJnuTcySiWQBdAECCbeZ4Vsj

- cmRdcvL6z8fedRPtDW7oec+IPkYoyXPktVt8WsQPYkwEVN4hZVBneJPCcuhikYkp

- T3WGmPV0MxhUvCZ6hSG8D2mscZXRq3itXVlKJsUWfIHaAIgGomWrPuqC23rOYCdT

- 5oSZmTvFAoGBAPs1FbbxDDd1fx1hisfXHFasV/sycT6ggP/eUXpBYCqVdxPQvqcA

- ktplm5j04dnaQJdHZ8TPlwtL+xlWhmhFhlCFPtVpU1HzIBkp6DkSmmu0gvA/i07Z

- pzo5Z+HRZFzruTQx6NjDtvWwiXVLwmZn2oiLeM9xSqPu55OpITifEWNjAoGBANhH

- XwV6IvnbUWojs7uiSGsXuJOdB1YCJ+UF6xu8CqdbimaVakemVO02+cgbE6jzpUpo

- krbDKOle4fIbUYHPeyB0NMidpDxTAPCGmiJz7BCS1fCxkzRgC+TICjmk5zpaD2md

- HCrtzIeHNVpTE26BAjOIbo4QqOHBXk/WPen1iC3DAoGBALsD3DSj46puCMJA2ebI

- 2EoWaDGUbgZny2GxiwrvHL7XIx1XbHg7zxhUSLBorrNW7nsxJ6m3ugUo/bjxV4LN

- L59Gc27ByMvbqmvRbRcAKIJCkrB1Pirnkr2f+xx8nLEotGqNNYIawlzKnqr6SbGf

- Y2wAGWKmPyEoPLMLWLYkhfdtAoGANsFa/Tf+wuMTqZuAVXCwhOxsfnKy+MNy9jiZ

- XVwuFlDGqVIKpjkmJyhT9KVmRM/qePwgqMSgBvVOnszrxcGRmpXRBzlh6yPYiQyK

- 2U4f5dJG97j9W7U1TaaXcCCfqdZDMKnmB7hMn8NLbqK5uLBQrltMIgt1tjIOfofv

- BNx0raECgYEApAvjwDJ75otKz/mvL3rUf/SNpieODBOLHFQqJmF+4hrSOniHC5jf

- f5GS5IuYtBQ1gudBYlSs9fX6T39d2avPsZjfvvSbULXi3OlzWD8sbTtvQPuCaZGI

- Df9PUWMYZ3HRwwdsYovSOkT53fG6guy+vElUEDkrpZYczROZ6GUcx70=

- -----END RSA PRIVATE KEY-----

- """

- 

- 

- AUTH_KEY = ("AAAAB3NzaC1yc2EAAAADAQABAAABAQDUOtNJdBEXyKxBB898rdT54ULjMGuO6v4jLX"

-             "mRsdRhR5Id/lKNc9hsdioPWUePgYlqML2iSV72vKQoVhkyYkpcsjr3zvBny9+5xej3"

-             "+TBLoEMAm2hmllKPmxYJDU8jQJ7wJuRrOVOnk0iSNF+FcY/yaQ0owSF02Nphx47j2K"

-             "Wc0IjGGlt4fl0fmHJuZBA2afN/4IYIIsEWZziDewVtaEjWV3InMRLllfdqGMllhFR+"

-             "ed2hQz9PN2QcapmEvUR4UCy/mJXrke5htyFyHi8ECfyMMyYeHwbWLFQIve4CWix9qt"

-             "ksvKjcetnxT+WWrutdr3c9cfIj/c0v/Zg/c4zETxtp")

- 

+ from copy import deepcopy

+ 

+ 

+ SSH_PRIV_KEY = ("-----BEGIN RSA PRIVATE KEY-----"

+                 "MIIEpQIBAAKCAQEA1DrTSXQRF8isQQfPfK3U+eFC4zBrjur+Iy15kbHUYUeSHf5S"

+                 "jXPYbHYqD1lHj4GJajC9okle9rykKFYZMmJKXLI6987wZ8vfucXo9/kwS6BDAJto"

+                 "ZpZSj5sWCQ1PI0Ce8CbkazlTp5NIkjRfhXGP8mkNKMEhdNjaYceO49ilnNCIxhpb"

+                 "eH5dH5hybmQQNmnzf+CGCCLBFmc4g3sFbWhI1ldyJzES5ZX3ahjJZYRUfnndoUM/"

+                 "TzdkHGqZhL1EeFAsv5iV65HuYbchch4vBAn8jDMmHh8G1ixUCL3uAlosfarZLLyo"

+                 "3HrZ8U/llq7rXa93PXHyI/3NL/2YP3OMxE8baQIDAQABAoIBAQCxuOUwkKqzsQ9W"

+                 "kdTWArfj3RhnKigYEX9qM+2m7TT9lbKtvUiiPc2R3k4QdmIvsXlCXLigyzJkCsqp"

+                 "IJiPEbJV98bbuAan1Rlv92TFK36fBgC15G5D4kQXD/ce828/BSFT2C3WALamEPdn"

+                 "v8Xx+Ixjokcrxrdeoy4VTcjB0q21J4C2wKP1wEPeMJnuTcySiWQBdAECCbeZ4Vsj"

+                 "cmRdcvL6z8fedRPtDW7oec+IPkYoyXPktVt8WsQPYkwEVN4hZVBneJPCcuhikYkp"

+                 "T3WGmPV0MxhUvCZ6hSG8D2mscZXRq3itXVlKJsUWfIHaAIgGomWrPuqC23rOYCdT"

+                 "5oSZmTvFAoGBAPs1FbbxDDd1fx1hisfXHFasV/sycT6ggP/eUXpBYCqVdxPQvqcA"

+                 "ktplm5j04dnaQJdHZ8TPlwtL+xlWhmhFhlCFPtVpU1HzIBkp6DkSmmu0gvA/i07Z"

+                 "pzo5Z+HRZFzruTQx6NjDtvWwiXVLwmZn2oiLeM9xSqPu55OpITifEWNjAoGBANhH"

+                 "XwV6IvnbUWojs7uiSGsXuJOdB1YCJ+UF6xu8CqdbimaVakemVO02+cgbE6jzpUpo"

+                 "krbDKOle4fIbUYHPeyB0NMidpDxTAPCGmiJz7BCS1fCxkzRgC+TICjmk5zpaD2md"

+                 "HCrtzIeHNVpTE26BAjOIbo4QqOHBXk/WPen1iC3DAoGBALsD3DSj46puCMJA2ebI"

+                 "2EoWaDGUbgZny2GxiwrvHL7XIx1XbHg7zxhUSLBorrNW7nsxJ6m3ugUo/bjxV4LN"

+                 "L59Gc27ByMvbqmvRbRcAKIJCkrB1Pirnkr2f+xx8nLEotGqNNYIawlzKnqr6SbGf"

+                 "Y2wAGWKmPyEoPLMLWLYkhfdtAoGANsFa/Tf+wuMTqZuAVXCwhOxsfnKy+MNy9jiZ"

+                 "XVwuFlDGqVIKpjkmJyhT9KVmRM/qePwgqMSgBvVOnszrxcGRmpXRBzlh6yPYiQyK"

+                 "2U4f5dJG97j9W7U1TaaXcCCfqdZDMKnmB7hMn8NLbqK5uLBQrltMIgt1tjIOfofv"

+                 "BNx0raECgYEApAvjwDJ75otKz/mvL3rUf/SNpieODBOLHFQqJmF+4hrSOniHC5jf"

+                 "f5GS5IuYtBQ1gudBYlSs9fX6T39d2avPsZjfvvSbULXi3OlzWD8sbTtvQPuCaZGI"

+                 "Df9PUWMYZ3HRwwdsYovSOkT53fG6guy+vElUEDkrpZYczROZ6GUcx70="

+                 "-----END RSA PRIVATE KEY-----")

+ SSH_PUB_KEY = ("AAAAB3NzaC1yc2EAAAADAQABAAABAQDUOtNJdBEXyKxBB898rdT54ULjMGuO6v4jLX"

+                "mRsdRhR5Id/lKNc9hsdioPWUePgYlqML2iSV72vKQoVhkyYkpcsjr3zvBny9+5xej3"

+                "+TBLoEMAm2hmllKPmxYJDU8jQJ7wJuRrOVOnk0iSNF+FcY/yaQ0owSF02Nphx47j2K"

+                "Wc0IjGGlt4fl0fmHJuZBA2afN/4IYIIsEWZziDewVtaEjWV3InMRLllfdqGMllhFR+"

+                "ed2hQz9PN2QcapmEvUR4UCy/mJXrke5htyFyHi8ECfyMMyYeHwbWLFQIve4CWix9qt"

+                "ksvKjcetnxT+WWrutdr3c9cfIj/c0v/Zg/c4zETxtp")

  DEF_USER = "root"

  DEF_PASSWD = "foobar"

- 

  USER_DATA = """#cloud-config

  users:

    - default
@@ -67,232 +64,700 @@ 

    list: |

      {0}:{1}

    expire: False

- """.format(DEF_USER, DEF_PASSWD, AUTH_KEY)

- 

- 

- def main(argv):

-     parser = argparse.ArgumentParser(description="Inventory for a QCow2 test image")

-     parser.add_argument("--list", action="store_true", help="Verbose output")

-     parser.add_argument('--host', help="Get host variables")

-     parser.add_argument("subjects", nargs="*", default=shlex.split(os.environ.get("TEST_SUBJECTS", "")))

-     opts = parser.parse_args()

- 

-     try:

-         if opts.host:

-             data = inv_host(opts.host)

-         else:

-             data = inv_list(opts.subjects)

-         sys.stdout.write(json.dumps(data, indent=4, separators=(',', ': ')))

-     except RuntimeError as ex:

-         sys.stderr.write("{0}: {1}\n".format(os.path.basename(sys.argv[0]), str(ex)))

-         return 1

- 

+ """.format(DEF_USER, DEF_PASSWD, SSH_PUB_KEY)

+ 

+ VM_IPV4_ADDR = "127.0.0.3"

+ VM_START_TRIES = 5

+ VM_PING_TRIES = 30

+ 

+ # Private to manage_debug() and debug(), do not use.

+ _debug_fmt = 'DEBUG: {0}'

+ _debug_enabled = False

+ 

+ 

+ def manage_debug(enable=None, debug_fmt=None):

+     """

+     Change or retrieve state of debugging enablement and message format

+     """

+     global _debug_enabled, _debug_fmt

+     if enable is not None:

+         _debug_enabled = enable

+     if debug_fmt is not None:

+         _debug_fmt = debug_fmt

+     return _debug_enabled

+ 

+ 

+ def log(msg):

+     """

+     Format msg with a newline, then write to sys.stderr + flush

+     """

+     msgnl = '{0}\n'.format(msg)

+     result = sys.stderr.write(msgnl)

+     sys.stderr.flush()

+     return result

+ 

+ 

+ def debug(msg):

+     """

+     When enabled by manage_debug, format msg for log()

+     """

+     global _debug_fmt

+     if manage_debug():

+         return log(_debug_fmt.format(msg))

      return 0

  

  

- def inv_list(subjects):

-     hosts = []

-     variables = {}

-     for subject in subjects:

-         if subject.endswith((".qcow2", ".qcow2c")):

-             host_vars = inv_host(subject)

-             if host_vars:

-                 hosts.append(subject)

-                 variables[subject] = host_vars

-     return {"localhost": {"hosts": hosts, "vars": {}},

-             "subjects": {"hosts": hosts, "vars": {}},

-             "_meta": {"hostvars": variables}}

+ class InvCache(object):

+     """

+     Represents a single-source, on-disk cache of Ansible inventory details

+ 

+     :param cachefile_basedir: Existing directory path where persistent cache

+                               file should live.  If None, ``tempfile.gettempdir()``

+                               is used.

+     """

+ 

+     # When non-none, represents the "empty" default contents of newly created cache

+     DEFAULT_CACHE = None

+ 

+     # When non-none, contains the base-directory for the persistent cache file

+     basedir = None

+ 

+     # Private, do not use

+     _singleton = None

+     _invcache = None

+ 

+     def __new__(cls, cachefile_basedir=None):

+         if not getattr(cls, '_singleton', None):

+             debug("Initializing Inventory Cache")

+             cls.reset()

+             cls.DEFAULT_CACHE = dict(localhost=dict(hosts=[], vars={}),

+                                      subjects=dict(hosts=[], vars={}),

+                                      _meta=dict(hostvars={}))

+             cls._singleton = super(InvCache, cls).__new__(cls)

+         return cls._singleton  # ``__init__()`` runs next

+ 

+     def __init__(self, cachefile_basedir=None):

+         # Don't touch it if already set by class

+         if cachefile_basedir and not self.basedir:

+             self.basedir = cachefile_basedir

+         elif not self.basedir:

+             self.basedir = tempfile.gettempdir()

+         try:

+             debug("Using inventory cache file: {0}".format(self.filepath))

+         except AttributeError:

+             pass  # don't fail when unittesting

+                   # '_io.StringIO' object has no attribute 'name'

+ 

+     def __str__(self):

+         return "{0}\n".format(json.dumps(self(), indent=4, separators=(',', ': ')))

+ 

+     def __call__(self, new_obj=None):

+         """

+         Replace and/or return current cached JSON object

+ 

+         :param new_obj: When not None, replaces current cache.

+         :returns: Current cache object or None

+         """

+         if new_obj:

+             debug("Replacing persistent inventory cache file contents")

+             self.cachefile.seek(0)

+             self.cachefile.truncate()

+             json.dump(new_obj, self.cachefile)

+             return self()

+         else:

+             self.cachefile.seek(0)

+             try:

+                 debug("Loading inventory cache from persistent file")

+                 loaded_cache = json.load(self.cachefile)

+             except ValueError as xcpt:

+                 debug("Persistent cache file is empty/invalid, initializing with defaults")

+                 try:

+                     self.cachefile.truncate()

+                     loaded_cache = deepcopy(self.DEFAULT_CACHE)

+                     json.dump(loaded_cache, self.cachefile)

+                 except RecursionError:

+                     raise RuntimeError("Error loading or parsing default 'empty' cache"

+                                        " after writing to disk: {0}".format(str(self.DEFAULT_CACHE)))

+             return loaded_cache

+ 

+     @property

+     def cachefile(self):

+         """Represents the active file backing the cache"""

+         if self._invcache:

+             return self._invcache

+         # Truncate if new, open for r/w otherwise

+         self._invcache = open(self.filepath, 'a+')

+         return self._invcache

+ 

+     @property

+     def filepath(self):

+         """Represents complete path to on-disk cache file"""

+         if self._invcache:  # Might not be open yet

+             return self._invcache.name

+         return os.path.join(self.basedir,

+                             self.filename)

+ 

+     @property

+     def filename(self):

+         """Represents the filename component of the on-disk cache file"""

+         cls = self.__class__  # Can't be classmethod && property

+         return "standard-inventory.cache".format(os.path.basename(sys.argv[0]))

+ 

+     @classmethod

+     def reset(cls):

+         """

+         Wipe-out current cache state, except on-disk file

+         """

+         if cls._singleton and cls._singleton._invcache:

+             try:

+                 cls._singleton._invcache = None

+             except IOError:

+                 pass

+         cls._singleton = None

+ 

+     @contextmanager

+     def locked(self, mode=fcntl.LOCK_EX):

+         """

+         Context manager protecting returned cache with mode

+ 

+         :param mode: A value accepted by ``fcntl.flock()``'s ``op`` parameter.

+         :returns: Standard Ansible inventory dictionary

+         """

+         debug("Acquiring Lock for persistent inventory cache file")

+         try:

+             fcntl.flock(self.cachefile, mode)  # __enter__

+             yield self()

+         finally:

+             debug("Lock released")

+             fcntl.flock(self.cachefile, fcntl.LOCK_UN) # __exit__

+ 

+     def gethost(self, hostname):

+         """

+         Look up details about a host from inventory cache.

+ 

+         :param hostname: Name of host to retrieve.

+         :returns: Tuple containing a dictionary of host variables,

+                   and a list of groups.  None if host not

+                   found.

+         """

+         debug("Retrieving cached details about {0}.".format(hostname))

+         with self.locked(fcntl.LOCK_SH) as inventory:

+             groups = []

+             for group, hostvars in inventory.items():

+                 hosts = hostvars.get("hosts", [])

+                 if hostname in hosts:

+                     groups.append(group)

+             meta = inventory.get("_meta", dict(hostvars=dict()))

+             hostvars = meta["hostvars"].get(hostname, None)

+         if hostvars is not None:

+             debug("Host found")

+             return (hostvars, groups)

+         else:

+             debug("Host not found")

+             return None

+ 

+     def addhost(self, hostname, hostvars=None, groups=('localhost', 'subjects')):

+         """

+         Update cache, adding hostname with hostvars to specified groups.

+ 

+         :param hostname: An Ansible inventory-hostname (may not be actual hostname).

+         :param hostvars: A dictionary of host variables to add, or None

+         :param groups: A list of groups for the host to join.

+         :returns: Tuple containing a dictionary of host variables, and a list of groups.

+         """

+         debug("Adding host {0} to cache".format(hostname))

+         if not groups:

+             groups = ('all',)

+         if not hostvars:

+             hostvars = {}

+         with self.locked(fcntl.LOCK_EX) as inventory:

+             meta = inventory.get("_meta", dict(hostvars=dict()))

+             meta["hostvars"][hostname] = hostvars

+             for group in groups:

+                 inv_group = inventory.get(group, dict(hosts=[], vars=dict()))

+                 inv_group["hosts"].append(hostname)

+             # Update and write cache to disk

+             self(inventory)

+         return self.gethost(hostname)

+ 

+     def delhost(self, hostname, keep_empty=False):

+         """

+         Remove hostname from inventory, return tuple of host vars. dict and group list

+ 

+         :param hostname: Ansible hostname to remove from inventory.

+         :param keep_empty: If False, remove cache file & reset if no hosts remain in cache.

+         :returns: Tuple containing a former dictionary of host variables, and a list of

+                   groups or None

+         """

+         debug("Deleting host {0} from cache".format(hostname))

+         with self.locked(fcntl.LOCK_EX) as inventory:

+             hostvars_groups = self.gethost(hostname)

+             if hostvars_groups:

+                 hostvars, groups = hostvars_groups

+                 meta = inventory.get("_meta", dict(hostvars=dict()))

+                 if hostname in meta["hostvars"]:

+                     del meta["hostvars"][hostname]

+                 for group in groups:

+                     inv_group = inventory.get(group, dict(hosts=[], vars=dict()))

+                     if hostname in inv_group["hosts"]:

+                         inv_group["hosts"].remove(hostname)

+                 # Write out to disk

+                 self(inventory)

+             conditions = [not keep_empty,

+                           not hostvars_groups or not meta["hostvars"]]

+             if all(conditions):  # No more hosts

+                 debug("Inventory cache is empty, removing file.")

+                 self.reset()

+                 try:

+                     os.unlink(self.filepath)

+                 except FileNotFoundError:

+                     pass

+             else:

+                 return None

+         return hostvars_groups

+ 

+     @staticmethod

+     def make_hostvars(priv_key_file, port, host=VM_IPV4_ADDR,

+                       user=DEF_USER, passwd=DEF_PASSWD):

+         """Return dictionary of standard/boiler-plate  hostvars"""

+ 

+         return dict(ansible_ssh_private_key_file=str(priv_key_file),

+                     ansible_port=str(port),

+                     ansible_host=str(host),

+                     ansible_user=str(user),

+                     ansible_ssh_pass=str(passwd),

+                     ansible_ssh_common_args="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no")

+ 

+     def str_hostvars(self, hostname):

+         hostvars, groups = self.gethost(hostname)

+         if hostvars == groups == None:

+             raise ValueError("Host '{0}' not found in cache file"

+                              " '{1}'".format(hostname, self.cachefile.name))

+         del groups  # not used

+         return "{0}\n".format(json.dumps(hostvars, indent=4, separators=(',', ': ')))

+ 

+ 

+ def try_replace_stderr_devtty():

+     """

+     Duplicate the controling terminal as if it were stderr to make debugging easier

+     """

+     try:

+         tty = os.open("/dev/tty", os.O_WRONLY)

+         os.dup2(tty, 2)

+     except OSError:

+         pass

  

  

- def start_qemu(image, cloudinit, log, portrange=(2222, 5555)):

+ def main(argv=None, environ=None):

+     if argv is None:  # Makes unittesting easier

+         argv = sys.argv

+     if environ is None:

+         environ = os.environ  # Side-effects: this isn't a dumb-dictionary

+     test_debug_env = environ.get("TEST_DEBUG", "0")[0].lower() in ['t','y','1']

+ 

+     parser = argparse.ArgumentParser(description="Inventory for a QCow2 test image",

+                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)

+ 

+     # Ansible inventory 'script' plugin's interface.

+     # ref: http://docs.ansible.com/ansible/devel/plugins/inventory/script.html

+     parser.add_argument("--list", action="store_true", default=True,

+                        help="Get variables for entire inventory")

+ 

+     # This is not really needed since --list provides '_meta'.

+     # It's implemented (below) and functional, for nostalga sake.

+     parser.add_argument('--host', default=None, help="Get variables for a single host")

+ 

+     # Auxiliary arguments (not used by Ansible)

+     parser.add_argument('--debug', action="store_true", default=test_debug_env,

+                         help="Disable qemu process cleanup & inventory cache removal."

+                              " As an alternative, set $TEST_DEBUG == true, yes, or 1")

+     parser.add_argument("subjects", nargs="*", default=[],

+                         help=("With --list, an optional list of file"

+                               " path(s) to qcow2 image(s) - in addition to any"

+                               " specified in $TEST_SUBJECTS.  If --host, "

+                               " a single path to a qcow2 image is required."))

+ 

+     opts = parser.parse_args(args=argv[1:])

+     if opts.debug:

+         manage_debug(True)

+         debug('# Debugging enabled\n')

+     try_replace_stderr_devtty()

+ 

+     # Load / Create cache

+     artifacts = artifacts_dirpath(environ)

+     invcache = InvCache(artifacts)

+     if opts.host:

+         opts.list = False

+         hostname = os.path.basename(opts.host)

+         debug("Operating in --host mode for {0}".format(hostname))

+ 

+         if not invcache.gethost(hostname):  # Enforce unique hostnames

+             try_create_host(opts, opts.host, environ)

+         try:

+             sys.stdout.write(invcache.str_hostvars(hostname))

+         except TypeError:  # try_create_host() failed

+             sys.stdout.write('{}\n')

+ 

+     else:  # opts.list (subjects is optional)

+         opts.host = False

+         # De-duplicate paths with $TEST_SUBJECTS

+         subjects = set(s for s in shlex.split(environ.get("TEST_SUBJECTS", "")) if s.strip())

+         subjects |= set(s for s in opts.subjects if s.strip())

+         subjects -= set([None, ''])  # Just in case

+ 

+         # Sets are un-ordered, coorespondence is needed between paths and hostnames

+         subjects = tuple(subjects)

+         # Catch null/empty paths/subjects while forming dict

+         subject_hostname = dict(zip(subjects,

+                                     tuple(os.path.basename(subject.strip())

+                                           for subject in subjects)))

+         if len(subject_hostname) != len(subjects):

+             # Fatal, do not allow clashes in cache over hostnames, differing paths OK

+             log("Error: One or more subjects from parameters"

+                 " or $TEST_SUBJECTS found with clashing"

+                 " hostnames (basenames):"

+                 " {0}\n".format(subjects))

+             parser.print_help()  # side-effect: exits non-zero

+             sys.exit(1)  # Required for unitesting

+ 

+         debug("Operating in --list mode for subjects: {0}".format(subject_hostname.keys()))

+         for subject, hostname in subject_hostname.items():

+             if not invcache.gethost(hostname):  # doesn't already exist

+                 try_create_host(opts, subject, environ)

+ 

+         sys.stdout.write(str(invcache))

+ 

+ 

+ def start_qemu(image_path, cloudinit, console_log, portrange=(2222, 5555)):

+     """

+     Start QEMU, return tuple of subprocess instance, ssh and monitor port numbers.

+     """

+     ioxcept = None

      for port in range(*portrange):

-         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

-         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

+         ssh_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

+         ssh_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

+         mon_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

+         mon_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

+         cmd = ["/usr/bin/qemu-system-x86_64", "-m", "1024", image_path,

+                "-cpu", "host", "-smp", "{}".format(multiprocessing.cpu_count()),

+                "-enable-kvm", "-snapshot", "-cdrom", cloudinit,

+                "-net", "nic,model=virtio", "-net",

+                "user,hostfwd=tcp:{0}:{1}-:22".format(VM_IPV4_ADDR, port),

+                "-device", "isa-serial,chardev=pts2", "-chardev",

+                "file,id=pts2,path=" + console_log,

+                # Monitor port allows rebooting VM w/o causing qemu-kvm process to exit

+                "-chardev",

+                "socket,host={0},port={1},id=mon0,server,nowait"

+                "".format(VM_IPV4_ADDR, port + 1),

+                "-mon", "chardev=mon0,mode=readline",

+                "-display", "none"]

          try:

-             sock.bind(("127.0.0.3", port))

-         except IOError:

+             ssh_sock.bind((VM_IPV4_ADDR, port))

+             mon_sock.bind((VM_IPV4_ADDR, port + 1))

+             # If launching qemu fails, having this means we get to see why.

+             with open(console_log, 'a+') as console_logf:

+                 console_logf.write("Qemu command-line: {0}\n".format(' '.join(cmd)))

+                 console_logf.flush()

+                 return subprocess.Popen(cmd, stdout=console_logf,

+                                         stderr=subprocess.STDOUT), port, port + 1

+         except IOError as ioxcept:

              pass

-         else:

-             break

-         finally:

-             sock.close()

      else:

-         raise RuntimeError("unable to find free local port to map SSH to")

+         msgfmt = "Error launching qemu process: {0}: {1}"

+         if ioxcept:

+             raise RuntimeError(msgfmt.format(ioxcept.__class__.__name__, ioxcept))

+         else:

+             raise RuntimeError(msgfmt.format("Could not allocate a ssh & monitor port from range",

+                                              portrange))

  

-     # Use -cpu host and -smp by default.

-     # virtio-rng-pci: https://wiki.qemu.org/Features/VirtIORNG

-     return subprocess.Popen(["/usr/bin/qemu-system-x86_64",

-                              "-cpu", "host", "-smp", "{}".format(multiprocessing.cpu_count()),

-                              "-m", "1024", image, "-enable-kvm", "-snapshot", "-cdrom", cloudinit,

-                              "-net", "nic,model=virtio", "-net", "user,hostfwd=tcp:127.0.0.3:{0}-:22".format(port),

-                              "-device", "virtio-rng-pci", "-rtc", "base=utc",

-                              "-device", "isa-serial,chardev=pts2", "-chardev", "file,id=pts2,path=" + log,

-                              "-display", "none"], stdout=open(log, 'a'), stderr=subprocess.STDOUT), port

  

+ class NoQcow2Error(ValueError):

+     """A non-fatal exception if --list, fatal if --host"""

+     pass

  

- def inv_host(image):

-     null = open(os.devnull, 'w')

  

+ def artifacts_dirpath(environ=None):

+     """Return complete path to directory where testing artifacts should be stored"""

+     if environ is None:

+         environ = os.environ  # Side-effects: this isn't a dumb-dictionary

      try:

-         tty = os.open("/dev/tty", os.O_WRONLY)

-         os.dup2(tty, 2)

-     except OSError:

-         tty = None

- 

-     # A directory for temporary stuff

-     directory = tempfile.mkdtemp(prefix="inventory-cloud")

-     identity = os.path.join(directory, "identity")

-     with open(identity, 'w') as f:

-         f.write(IDENTITY)

-     os.chmod(identity, 0o600)

-     metadata = os.path.join(directory, "meta-data")

+         artifacts = environ.get("TEST_ARTIFACTS", os.path.join(os.getcwd(), "artifacts"))

+         os.makedirs(artifacts)

+     except OSError as exc:

+         if exc.errno != errno.EEXIST or not os.path.isdir(artifacts):

+             raise

+     # Behave like a singleton: Update environ w/ proper path

+     environ["TEST_ARTIFACTS"] = artifacts

+     return artifacts

+ 

+ 

+ def try_create_host(opts, image_path, environ):

+     try:

+         create_host(opts, image_path, environ)

+     except NoQcow2Error as xcept:  # Not fatal by itself

+         log("Warning: {0}".format(xcept))

+         pass  # Presume another inventory script will handle it

+ 

+ 

+ def create_host(opts, image_path, environ):

+     image_path = image_path.strip()

+     # directory-separators in hostnames could be bad

+     hostname = os.path.basename(image_path)

+     log("Creating host {0}".format(hostname))

+     if not image_path.endswith(".qcow2") and not image_path.endswith(".qcow2c"):

+         # Ignored in main(), printed if --debug used, required for unittesting

+         raise NoQcow2Error("Subject '{0}' / '{1}',"

+                            " image does not end in '.qcow2' or '.qcow2c'."

+                            "".format(hostname, image_path))

+ 

+     # temporary until subject destroyed, allow secondary debugging w/o artifacts-clutter

+     temp_dir = tempfile.mkdtemp(prefix=".{0}_".format(hostname),

+                                 suffix='.temp', dir=artifacts_dirpath())

+     ssh_priv_key_file = os.path.join(temp_dir, "ssh_priv_key")

+     debug("Using ssh key {0}".format(ssh_priv_key_file))

+     with open(ssh_priv_key_file, 'w') as f:

+         f.write(SSH_PRIV_KEY)

+         os.chmod(ssh_priv_key_file, 0o600)

+     metadata = os.path.join(temp_dir, "meta-data")

      with open(metadata, 'w') as f:

          f.write("")

-     userdata = os.path.join(directory, "user-data")

+     userdata = os.path.join(temp_dir, "user-data")

      with open(userdata, 'w') as f:

          f.write(USER_DATA)

  

      # Create our cloud init so we can log in

-     cloudinit = os.path.join(directory, "cloud-init.iso")

+     null = open(os.devnull, 'w')

+     cloudinit = os.path.join(temp_dir, "cloud-init.iso")

      subprocess.check_call(["/usr/bin/genisoimage", "-input-charset", "utf-8",

                             "-volid", "cidata", "-joliet", "-rock", "-quiet",

                             "-output", cloudinit, userdata, metadata], stdout=null)

- 

-     # Determine if virtual machine should be kept available for diagnosis after completion

-     try:

-         diagnose = distutils.util.strtobool(os.getenv("TEST_DEBUG", "0"))

-     except ValueError:

-         diagnose = 0

- 

-     sys.stderr.write("Launching virtual machine for {0}\n".format(image))

- 

+     debug("Using cloud-init {0}".format(cloudinit))

      # And launch the actual VM

-     artifacts = os.environ.get("TEST_ARTIFACTS", os.path.join(os.getcwd(), "artifacts"))

-     try:

-         os.makedirs(artifacts)

-     except OSError as exc:

-         if exc.errno != errno.EEXIST or not os.path.isdir(artifacts):

-             raise

-     log = os.path.join(artifacts, "{0}.log".format(os.path.basename(image)))

- 

+     console_log = os.path.join(artifacts_dirpath(environ), "{0}.log".format(hostname))

+     debug("Using qemu log {0}".format(log))

      proc = None  # for failure detection

      cpe = None  # for exception scoping

-     for _ in range(0, 5):

+     for tries in range(0, VM_START_TRIES):

+         log("    Launching {0}, attempt #{1}/{2}"

+             "".format(hostname, tries+1, VM_START_TRIES))

          try:

-             proc, port = start_qemu(image, cloudinit, log)

+             proc, ssh_port, mon_port = start_qemu(image_path, cloudinit, console_log)

              break

          except subprocess.CalledProcessError as cpe:

              time.sleep(1)

              continue

-     if proc is None:

-         raise RuntimeError("Could not launch VM for qcow2 image"

-                            " '{0}':{1}".format(image, cpe.output))

- 

-     for _ in range(0, 30):

+     if proc.poll():

+         raise RuntimeError("Could not launch VM, exited:{0}: {1}"

+                            "".format(proc.returncode, cpe.output))

+ 

+     # Ansible ping signals VM up, needs temporary inventory

+     hostvars = InvCache.make_hostvars(ssh_priv_key_file, int(ssh_port))

+     hostvars['qemu_monitor_port'] = mon_port

+     inv_vars_sfx = " ".join(tuple("{0}='{1}'".format(k,v) for k,v in hostvars.items()))

+     inventory_filepath = os.path.join(temp_dir, "inventory.ini")

+     with open(inventory_filepath, "w") as inventory_file:

+         # Exclude dictionary prefix/suffix in JSON

+         inventory_file.write("{0} {1}\n".format(hostname, inv_vars_sfx))

+ 

+     ping = [

+         "/usr/bin/ansible",

+         "--inventory",

+         inventory_filepath,

+         hostname,

+         "--module-name",

+         "raw",

+         "--args",

+         "/bin/true"

+     ]

+ 

+     for tries in range(0, VM_PING_TRIES):

+         log("    Contacting {0}, attempt #{1}/{2}"

+               "".format(hostname, tries+1, VM_PING_TRIES))

          try:

-             # The variables

-             variables = {

-                 "ansible_port": "{0}".format(port),

-                 "ansible_host": "127.0.0.3",

-                 "ansible_user": "root",

-                 "ansible_ssh_pass": "foobar",

-                 "ansible_ssh_private_key_file": identity,

-                 "ansible_ssh_common_args": "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

-             }

- 

-             # Write out a handy inventory file, for our use and for debugging

-             args = " ".join(["{0}='{1}'".format(*item) for item in variables.items()])

-             inventory = os.path.join(directory, "inventory")

-             with open(inventory, "w") as f:

-                 f.write("[subjects]\nlocalhost {0}\n".format(args))

- 

-             # Wait for ssh to come up

-             ping = [

-                 "/usr/bin/ansible",

-                 "--inventory",

-                 inventory,

-                 "localhost",

-                 "--module-name",

-                 "raw",

-                 "--args",

-                 "/bin/true"

-             ]

- 

-             (pid, _) = os.waitpid(proc.pid, os.WNOHANG)

-             if pid != 0:

-                 raise RuntimeError("qemu failed to launch qcow2 image: {0}".format(image))

-             subprocess.check_call(ping, stdout=null, stderr=null)

+             if proc.poll():

+                 raise RuntimeError("Error launching '{0}' for '{1}',"

+                                    " qemu exited {2}.\n".format(hostname, image_path,

+                                                                 proc.returncode))

+             subprocess.check_call(ping, stdout=null, stderr=null, close_fds=True)

              break

          except subprocess.CalledProcessError:

              time.sleep(3)

+     else:  # tries >= 30

+         raise RuntimeError("Error launching VM '{0}' for '{1}',"

+                            " excessive Ansible ping attempts.\n".format(hostname, image_path))

+ 

+     invcache = InvCache()

+     invcache.addhost(hostname, hostvars)

+ 

+     debug("Access host with:")

+     debug("    ssh -p {0} -o StrictHostKeyChecking=no"

+           " -o UserKnownHostsFile=/dev/null {2}@{1}"

+           "".format(ssh_port, VM_IPV4_ADDR, DEF_USER))

+     debug("    {0}'s password: {1}".format(DEF_USER, DEF_PASSWD))

+     debug("Access host's monitor with:")

+     debug("    telnet {0} {1}".format(VM_IPV4_ADDR, mon_port))

+     debug("")

+     # subject_watcher() will not kill QEMU when --debug was specified

+     debug("kill {0} # when finished\n".format(proc.pid))

+     # Even if debugging, the inventory cache file needs grooming on qemu exit

+     subject_watcher(proc.pid, hostname, temp_dir, environ, opts.debug)

+ 

+ 

+ def monitor(pid_or_filepath, qemu_pid):

+     """Helper for subject_watcher() to monitor processes"""

+ 

+     if isinstance(pid_or_filepath, int):

+         expected_errno = (errno.ESRCH, )

+         lockonfile = None

+         parent_pid = pid_or_filepath

+         mon_what = "qemu parent-pid {2}"

      else:

-         # Kill the qemu process

+         expected_errno = (errno.EACCES, errno.ENOENT,  # reg. file

+                           errno.EEXIST,                # pipe/fifo

+                           errno.ECONNRESET,            # socket

+                           errno.ECONNABORTED,          # socket

+                           errno.ECONNREFUSED)          # socket

+         lockonfile = pid_or_filepath

+         parent_pid = None

+         mon_what = "$LOCK_ON_FILE {2}"

+ 

+     mon_msg = 'QEMU PID {1} and ' + mon_what + ' at {0}'

+     monitor_signaled = False

+     qemu_died = False

+     # Monitor pid_or_filepath, and qemu-process

+     while all([not monitor_signaled, not qemu_died]):

+         log(mon_msg.format(int(time.time()), qemu_pid, pid_or_filepath))

          try:

-             os.kill(proc.pid, signal.SIGTERM)

-         except OSError:

-             pass

-         raise RuntimeError("could not access launched qcow2 image: {0}".format(image))

- 

-     # Process of our parent

-     ppid = os.getppid()

- 

-     child = os.fork()

-     if child:

-         return variables

+             monitor_signaled = True  # If exception thrown

+             if parent_pid:

+                 os.kill(parent_pid, 0)  # Exception if process doesn't exist

+             else:  # lockonfile

+                 open(lockonfile)        # Exception if read not allowed

+             monitor_signaled = False    # No exception thrown

+ 

+             # Monitoring unnecessary if qemu process doesn't exist

+             qemu_died = True   # If exception thrown

+             os.kill(qemu_pid, 0)

+             qemu_died = False  # No exception thrown

+         except (IOError, OSError) as xcpt:

+             if xcpt.errno not in expected_errno:

+                 debug("Unable to handle {0}: {1}".format(xcpt.__class__.__name__, xcpt))

+                 raise

+             continue  # exit loop immediatly

+         time.sleep(10)  # Don't busy-wait

+ 

+     return monitor_signaled, qemu_died

+ 

+ 

+ def subject_watcher(qemu_pid, hostname, temp_dir, environ, debugging=True):

+     """

+     Monitor process that called this script, kill off VMs and cleanup on exit.

+     """

+     # Parent of our pid is Ansible process

+     parent_pid = os.getppid()

+ 

+     if 1 in (parent_pid, qemu_pid):

+         raise RuntimeError("Cowardly refusing to monitor pid 1")

+ 

+     # Needed at end of watcher process

+     artifacts = artifacts_dirpath(environ)

+     lockonfile = environ.get('LOCK_ON_FILE', None)

+ 

+     info_msg_pfx = ("Watching VM {0} qemu pid {1} from watcher"

+                     "".format(hostname, qemu_pid))

+     if lockonfile:

+         info_msg_fmt = ("{0} pid {1} for $LOCK_ON_FILE {2}"

+                         "".format(info_msg_pfx, "{0}", lockonfile))

+         mon_args = (lockonfile, qemu_pid)

+         lockonfile = os.path.realpath(lockonfile)

+         try:

+             open(lockonfile)

+         except IOError as xcpt:

+             log("WARNING: Can't open $LOCK_ON_FILE {0}: {1}: {2}.\n"

+                    "".format(lockonfile, xcpt.__class__.__name__, xcpt))

+             if debugging:

+                 log("WARNING: Qemu process will be killed shortly after creation.\n")

+     else:

+         info_msg_fmt = ("{0} pid {1} for parent pid {2}"

+                         "".format(info_msg_pfx, "{0}", parent_pid))

+         mon_args = (parent_pid, qemu_pid)

+ 

+     watcher_pid = os.fork()

+     if watcher_pid:  # parent

+         log(info_msg_fmt.format(watcher_pid))

+         return

+     else:

+         watcher_pid = os.getpid()

  

-     # Daemonize and watch the processes

+     # Harden child process

      os.chdir("/")

      os.setsid()

      os.umask(0)

- 

-     if tty is None:

-         tty = null.fileno()

- 

-     # Duplicate standard input to standard output and standard error.

-     os.dup2(null.fileno(), 0)

-     os.dup2(tty, 1)

-     os.dup2(tty, 2)

- 

-     # alternatively, lock on a file

-     lock_file = os.environ.get("LOCK_ON_FILE", None)

-     while True:

-         time.sleep(3)

- 

-         if lock_file:

-             if not os.path.exists(lock_file):

-                 sys.stderr.write("Lock file is gone.")

-                 break

-         else:

-             # Now wait for the parent process to go away, then kill the VM

+     InvCache.reset()  # don't hold onto cachefile object

+     for _file in (sys.stdin, sys.stdout, sys.stderr):

+         if _file:

              try:

-                 os.kill(ppid, 0)

-                 os.kill(proc.pid, 0)

-             except OSError:

-                 sys.stderr.write("Either parent process or VM process is gone.")

-                 break  # Either of the processes no longer exist

- 

-     if diagnose:

-         sys.stderr.write("\n")

-         sys.stderr.write("DIAGNOSE: ssh -p {0} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "

-                          "root@{1} # password: {2}\n".format(port, "127.0.0.3", "foobar"))

-         sys.stderr.write("DIAGNOSE: export ANSIBLE_INVENTORY={0}\n".format(inventory))

-         sys.stderr.write("DIAGNOSE: kill {0} # when finished\n".format(os.getpid()))

- 

-         def _signal_handler(*args):

-             sys.stderr.write("\nDIAGNOSE ending...\n")

- 

-         signal.signal(signal.SIGTERM, _signal_handler)

-         signal.pause()

- 

-     # Kill the qemu process

-     try:

-         os.kill(proc.pid, signal.SIGTERM)

-     except OSError:

-         pass

- 

-     shutil.rmtree(directory)

+                 _file.close()

+             except IOError:

+                 pass

+     os.closerange(0, 255)  # Be certain nothing is open, ignore any errors

+ 

+     # Make multi-process debugging easier

+     logfn = '{0}_watcher.log'.format(os.path.basename(sys.argv[0]).split('.')[0])

+     sys.stderr = open(os.path.join(artifacts_dirpath(environ), logfn), 'a')

+     os.dup2(sys.stderr.fileno(), 2)

+     os.dup2(sys.stderr.fileno(), 1)

+     manage_debug(debugging, "WATCHER {0}: {1}".format(watcher_pid, '{0}'))

+     log(info_msg_fmt.format(watcher_pid))

+     invcache = InvCache(artifacts)  # re-opens cachefile

+ 

+     # Wait for monitor to signal exit time (blocking)

+     monitor_signaled, qemu_died = monitor(*mon_args)

+ 

+     oops_msg = ("Monitor function exited unexpectedly "

+                 "monitor_signaled={0}  and  qemu_died={1}"

+                 "".format(monitor_signaled, qemu_died))

+ 

+     if debugging:

+         msg_sfx = "NOT removing {0}".format(temp_dir)

+         if monitor_signaled:

+             debug("NOT killing qemu and {0}".format(msg_sfx))

+         elif qemu_died:

+             debug("Qemu process died, {0}"

+                    "".format(msg_sfx))

+         else:

+             debug(oops_msg)

+             debug(msg_sfx)

+         # Don't remove cache if empty

+         invcache.delhost(hostname, keep_empty=True)

+         debug("Removed host {0} from inventory cache.".format(hostname))

+     else:  # debugging==false

+         msg_sfx = "removing {0}".format(temp_dir)

+         try:

+             if monitor_signaled:

+                 debug("Killing qemu process and {0}".format(msg_sfx))

+                 os.kill(qemu_pid, signal.SIGTERM)  # Ensure it's actually dead

+             elif qemu_died:

+                 debug("Qemu process died, {0}".format(msg_sfx))

+             else:

+                 debug(oops_msg)

+                 debug(msg_sfx)

+ 

+         except OSError as xcpt:

+             # Don't care if qemu_pid doesn't exist, that was the goal.

+             if xcpt.errno != errno.ESRCH:

+                 debug("Qemu process dead, but unable to handle {0}: {1}"

+                        "".format(xcpt.__class__.__name__, xcpt))

+                 raise

+         finally:

+             shutil.rmtree(temp_dir, ignore_errors=True)

+             invcache.delhost(hostname, keep_empty=False)

+     log("Watcher {0} exiting".format(watcher_pid))

+     if debugging:

+         log("Remember to manually kill {0} if necessary".format(qemu_pid))

+     log("FIXME: ignore 'IOError: [Errno 9] Bad file descriptor' coming next...")

      sys.exit(0)

  

  

  if __name__ == '__main__':

-     sys.exit(main(sys.argv))

+     main(sys.argv)

@@ -0,0 +1,671 @@ 

+ #!/usr/bin/env python3

+ 

+ import sys

+ import os

+ import tempfile

+ import fcntl

+ import json

+ import shutil

+ import subprocess

+ from errno import ESRCH

+ from io import StringIO, SEEK_SET, FileIO

+ from contextlib import contextmanager, redirect_stdout, redirect_stderr

+ import unittest

+ from unittest.mock import MagicMock, patch, mock_open, call, create_autospec, ANY

+ from glob import glob

+ import importlib.machinery

+ from pdb import Pdb

+ 

+ 

+ # Assumes directory structure as-is from repo. clone

+ TEST_FILENAME = os.path.basename(sys.argv[0])

+ TESTS_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))

+ TESTS_DIR_PARENT = os.path.realpath(os.path.join(TESTS_DIR, '../'))

+ 

+ 

+ class TestCaseBase(unittest.TestCase):

+     """Exercize code from file based on TEST_FILENAME in TESTS_DIR_PARENT + SUBJECT_REL_PATH"""

+ 

+     # Mock'd call for opening files, used for patching

+     MockOpen = mock_open()

+ 

+     # When non-None, the file-like object returned by mock_open()

+     cachefile = None

+ 

+     # When non-None, a stand-in for fcntl module

+     mock_fcntl = None

+ 

+     # repo. relative path containing test subject python file

+     SUBJECT_REL_PATH = 'inventory'

+ 

+     # The name of the loaded code, as if it were a real module

+     SUBJECT_NAME = TEST_FILENAME[len('test_'):].split('.',1)[0]

+ 

+     # The complete path containing SUBJECT_NAME

+     SUBJECT_DIR = os.path.realpath(os.path.join(TESTS_DIR_PARENT, SUBJECT_REL_PATH))

+ 

+     # When non-none, reference to loaded subject as if it were a module

+     SUBJECT = None

+ 

+     # When non-none, complete path to unittest temporary directory

+     TEMPDIRPATH = None

+ 

+     # Fake proccess ID of parent process

+     PPID = 43

+ 

+     # When non-None, contains a tuple of patcher instances

+     patchers = None

+ 

+     # The complete path to the SUBJECT_NAME

+     for SUBJECT_PATH in glob(os.path.join(SUBJECT_DIR, '{}*'.format(SUBJECT_NAME))):

+         if os.path.isfile(SUBJECT_PATH):

+             # The subject may not conform to package.module standards

+             loader = importlib.machinery.SourceFileLoader(SUBJECT_NAME, SUBJECT_PATH)

+             # After python 3.6: Need loader for files w/o any extension

+             # so loader.exec_module() can be used.

+             SUBJECT = sys.modules[SUBJECT_NAME] = loader.load_module(SUBJECT_NAME)

+             break

+     else:

+         raise RuntimeError("Could not locate test subject: {} in {}".format(SUBJECT_NAME, SUBJECT_DIR))

+ 

+     def trace(self, statement=None):

+         """Enter the pdb debugger, 'n' will step back to self.trace() caller"""

+         return self._pdb.set_trace()

+ 

+     def subtests(self, items):

+         for item in items:

+             ctxmgr = self.subTest(item=item)

+             with ctxmgr:

+                 yield item

+ 

+     def reset(self):

+         self.SUBJECT.InvCache.reset()

+         self.MockOpen.reset_mock()

+         self.MockOpen.return_value = self.cachefile = StringIO()

+         self.mock_fcntl.reset_mock()

+         for attr in dir(fcntl):

+             mockattr = MagicMock(spec=getattr(fcntl, attr), spec_set=True)

+             # represent int constants properly

+             if attr.capitalize() == attr:

+                 mockattr.__int__ = getattr(fcntl, attr)

+             self.mock_fcntl.attach_mock(mockattr, attr)

+ 

+ 

+     def setUp(self):

+         super(TestCaseBase, self).setUp()

+         self._pdb = Pdb()

+         self.TEMPDIRPATH = tempfile.mkdtemp(prefix=os.path.basename(__file__))

+         self.mock_fcntl = create_autospec(spec=fcntl, spec_set=True, instance=True)

+         self.mock_makedirs = create_autospec(spec=os.makedirs, spec_set=True,

+                                                return_value=self.TEMPDIRPATH)

+         # TODO: All of ``os`` and ``sys`` should probably just be patched up inside a loop

+         if self.patchers is None:

+             self.patchers = []

+         self.patchers += [patch('{}.try_replace_stderr_devtty'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=None)),

+                           patch('{}.tempfile.gettempdir'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=self.TEMPDIRPATH)),

+                           patch('{}.tempfile.mkdtemp'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=self.TEMPDIRPATH)),

+                           patch('{}.os.makedirs'.format(TestCaseBase.SUBJECT_NAME),

+                                 self.mock_makedirs),

+                           patch('{}.os.getcwd'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=self.TEMPDIRPATH)),

+                           patch('{}.fcntl'.format(TestCaseBase.SUBJECT_NAME),

+                                 self.mock_fcntl),

+                           patch('{}.open'.format(format(TestCaseBase.SUBJECT_NAME)),

+                                 self.MockOpen, create=True),

+                           patch('{}.InvCache.filename'.format(TestCaseBase.SUBJECT_NAME),

+                                 'bar'),

+                           patch('{}.time.sleep'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(spec=self.SUBJECT.time.sleep)),

+                           patch('{}.os.chdir'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(spec=self.SUBJECT.os.chdir)),

+                           patch('{}.os.setsid'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(spec=self.SUBJECT.os.setsid)),

+                           patch('{}.os.umask'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(spec=self.SUBJECT.os.umask)),

+                           patch('{}.os.closerange'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(spec=self.SUBJECT.os.closerange)),

+                           patch('{}.os.fork'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=0)),

+                           patch('{}.os.getpid'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=42)),

+                           patch('{}.os.getppid'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=self.PPID)),

+                           patch('{}.os.getgid'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=44)),

+                           patch('{}.os.unlink'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=45)),

+                           patch('{}.os.chmod'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=46)),

+                           patch('{}.os.getuid'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=47)),

+                           patch('{}.shutil.rmtree'.format(TestCaseBase.SUBJECT_NAME),

+                                 MagicMock(return_value=48))]

+         for patcher in self.patchers:

+             patcher.start()

+         self.reset()

+ 

+     def tearDown(self):

+         if self.patchers:

+             for patcher in self.patchers:

+                 patcher.stop()

+         if self.TEMPDIRPATH:  # rm -rf /tmp/test_standard-inventory-qcow2.py*

+             for tempdirglob in glob('{}*'.format(self.TEMPDIRPATH)):

+                 shutil.rmtree(tempdirglob, ignore_errors=True)

+ 

+     @contextmanager

+     def silence_debug(self):

+         with patch('{}.debug'.format(self.SUBJECT_NAME), MagicMock(return_value=0)):

+             yield None

+ 

+     def validate_mock_fcntl(self):

+         """Helper to confirm cachefile-locking/unlocking was done properly"""

+         # Make sure unlock happens _after_ locking, and lock is released

+         locked = False

+         locks = 0

+         unlocks = 0

+         for args, dargs in self.mock_fcntl.flock.call_args_list:

+             if locked:

+                 if len(args) >= 2:

+                     op = args[1]

+                 elif 'op' in dargs:

+                     op = dargs['op']

+                 else:

+                     continue  # Don't care about this call

+                 # Is it a locking call?

+                 if op == self.mock_fcntl.LOCK_UN:

+                     locked = False

+                     unlocks += 1

+             else:

+                 if len(args) >= 2:

+                     op = args[1]

+                 elif 'op' in dargs:

+                     op = dargs['op']

+                 else:

+                     continue  # Don't care about this call

+                 if op in [self.mock_fcntl.LOCK_EX, self.mock_fcntl.LOCK_SH]:

+                     locked = True

+                     locks += 1

+         self.assertFalse(locked, "Mock cache file locked {} times but"

+                                  " unlocked {} times".format(locks, unlocks))

+ 

+ 

+ class TestToolFunctions(TestCaseBase):

+     """Tests for several misc. tooling functions"""

+ 

+     def validate_artifacts_dirpath(self, test_environ, expected_dirpath):

+         """Helper for actual tests"""

+         first = self.SUBJECT.artifacts_dirpath(test_environ)

+         second = self.SUBJECT.artifacts_dirpath(test_environ)

+         self.assertEqual(test_environ["TEST_ARTIFACTS"], first)

+         self.assertEqual(first, second)

+         self.assertEqual(len(self.mock_makedirs.mock_calls), 2)

+         self.assertEqual(self.mock_makedirs.mock_calls[0], self.mock_makedirs.mock_calls[1])

+         self.assertEqual(first, expected_dirpath)

+         self.mock_makedirs.assert_called_with(expected_dirpath)

+ 

+     def test_no_env_artifacts_dirpath(self):

+         """Verify artifacts_dirpath() always returns consistent path w/ env."""

+         self.validate_artifacts_dirpath({}, os.path.join(self.TEMPDIRPATH, 'artifacts'))

+ 

+     def test_no_env_artifacts_dirpath(self):

+         """Verify artifacts_dirpath() always returns consistent path w/ env."""

+         expected_dirpath = 'foo/../bar / baz../'

+         self.validate_artifacts_dirpath(dict(TEST_ARTIFACTS=expected_dirpath), expected_dirpath)

+ 

+ 

+ class TestInvCache(TestCaseBase):

+     """Tests for the InvCache class"""

+ 

+     def setUp(self):

+         super(TestInvCache, self).setUp()

+ 

+     def test_init_newcache(self):

+         """Verify InvCache initialization behavior"""

+         invcache = self.SUBJECT.InvCache()

+         self.assertDictEqual(invcache.DEFAULT_CACHE, invcache())

+         self.assertDictEqual(invcache.DEFAULT_CACHE, json.loads(self.cachefile.getvalue()))

+         self.validate_mock_fcntl()

+ 

+     def test_reset(self):

+         """Verify resetting results in new instance"""

+         invcache = self.SUBJECT.InvCache()

+         self.validate_mock_fcntl()

+         invcache.reset()

+         invcache_too = self.SUBJECT.InvCache()

+         self.validate_mock_fcntl()

+         self.assertTrue(id(invcache) != id(invcache_too))

+ 

+     def test_empty_gethost(self):

+         """Verify invcache.gethost() returns None when not found"""

+         invcache = self.SUBJECT.InvCache()

+         self.assertEqual(invcache.gethost('foobar'), None)

+         self.assertEqual(invcache.gethost(None), None)

+         self.validate_mock_fcntl()

+ 

+     def test_addgetdelhost(self):

+         """Verify invcache.gethost() == invcache.addhost() == invcache.delhost()"""

+         invcache = self.SUBJECT.InvCache()

+         added = invcache.addhost('foobar')

+         geted = invcache.gethost('foobar')

+         self.validate_mock_fcntl()

+         deled = invcache.delhost('foobar')

+         self.assertTrue(added)

+         self.assertTrue(geted)

+         self.assertTrue(deled)

+         self.assertEqual(added, geted)

+         self.assertEqual(geted, deled)

+         self.assertEqual(geted[0], {})

+         self.assertIn('localhost', geted[1])

+         self.assertIn('subjects', geted[1])

+         self.SUBJECT.os.unlink.assert_called_once_with(os.path.join(self.TEMPDIRPATH, 'bar'))

+         self.assertDictEqual(invcache.DEFAULT_CACHE, json.loads(self.cachefile.getvalue()))

+         self.validate_mock_fcntl()

+ 

+     def test_delhost_unlink(self):

+         """Verify empty invcache.delhost() returns None and unlinks file"""

+         invcache = self.SUBJECT.InvCache()

+         deled = invcache.delhost('foobar')

+         self.SUBJECT.os.unlink.assert_called_once_with(os.path.join(self.TEMPDIRPATH, 'bar'))

+         self.assertEqual(deled, None)

+         self.validate_mock_fcntl()

+         self.assertDictEqual(invcache.DEFAULT_CACHE, json.loads(self.cachefile.getvalue()))

+ 

+     def test_delhost_nounlink(self):

+         """Verify empty invcache.delhost() returns None and does NOT unlink"""

+         invcache = self.SUBJECT.InvCache()

+         deled = invcache.delhost('foobar', True)

+         self.assertFalse(self.SUBJECT.os.unlink.called)

+         self.assertEqual(deled, None)

+         self.validate_mock_fcntl()

+         self.assertDictEqual(invcache.DEFAULT_CACHE, json.loads(self.cachefile.getvalue()))

+ 

+ 

+ class TestMain(TestCaseBase):

+     """Tests for the ``main()`` function"""

+ 

+     mock_start_qemu = None

+     mock_create_host = None

+     mock_subject_watcher = None

+     fake_stdout = None

+     fake_stderr = None

+     exit_code = None

+     exit_msg = None

+ 

+     def mock_exit(self, code, msg):

+         self.exit_code = code

+         self.exit_msg = msg

+ 

+     def reset(self):

+         super(TestMain, self).reset()

+         self.fake_stdout = StringIO()

+         self.fake_stderr = StringIO()

+         self.mock_start_qemu.reset_mock()

+         self.mock_create_host.reset_mock()

+         self.mock_subject_watcher.reset_mock()

+         self.exit_code = None

+         self.exit_msg = None

+ 

+     def setUp(self):

+         self.fake_stdout = StringIO()

+         self.fake_stderr = StringIO()

+         self.mock_start_qemu = create_autospec(spec=self.SUBJECT.start_qemu,

+                                                spec_set=True, instance=True,

+                                                return_value=(MagicMock(), MagicMock()))

+         self.mock_create_host = create_autospec(spec=self.SUBJECT.create_host,

+                                                 spec_set=True, instance=True)

+         self.mock_subject_watcher = create_autospec(spec=self.SUBJECT.subject_watcher,

+                                                     spec_set=True, instance=True)

+         self.patchers = [patch('{}.argparse.ArgumentParser.exit'.format(TestCaseBase.SUBJECT_NAME),

+                                MagicMock(side_effect=self.mock_exit)),

+                          patch('{}.start_qemu'.format(TestCaseBase.SUBJECT_NAME),

+                                self.mock_start_qemu),

+                          patch('{}.create_host'.format(TestCaseBase.SUBJECT_NAME),

+                                self.mock_create_host),

+                          patch('{}.subject_watcher'.format(TestCaseBase.SUBJECT_NAME),

+                                self.mock_subject_watcher)]

+         super(TestMain, self).setUp()

+ 

+     def test_noargs(self):

+         """Script w/o args or environment behaves like --list"""

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+             self.SUBJECT.main(argv=[self.SUBJECT_PATH], environ={})

+         # No exceptions, parser exit, or stderr

+         self.assertEqual(self.exit_code, None)  # mock_exit NOT called

+         # output format checked more thrroughly elsewhere

+         for regex in ('localhost', 'subjects', '_meta'):

+             self.assertRegex(self.fake_stdout.getvalue(), regex)

+ 

+     def test_debug_arg(self):

+         """Script with --debug enables debug mode"""

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+             self.SUBJECT.main(argv=[self.SUBJECT_PATH, '--debug'], environ={})

+         # No exceptions, output, or input of any kind

+         self.assertRegex(self.fake_stdout.getvalue(), 'localhost')

+         self.assertRegex(self.fake_stderr.getvalue(), r'Debugging enabled')

+ 

+     def test_debug_env(self):

+         """Script with $TEST_DEBUG enables debug mode as expected"""

+         debug_enabled = []

+         for test_debug in ('TRUE', 'True', 'true', 'YES', 'Yes', 'yes', '1'):

+             with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+                 self.SUBJECT.main(argv=[self.SUBJECT_PATH],

+                                   environ=dict(TEST_DEBUG=test_debug))

+             # No exceptions, output, or input of any kind

+             self.assertRegex(self.fake_stdout.getvalue(), 'localhost')

+             self.assertRegex(self.fake_stderr.getvalue(), r'Debugging enabled')

+ 

+     def test_bad_mmhost(self):

+         """--host w/o any subjects exits non-zero."""

+         argv = [self.SUBJECT_PATH, '--host']

+         environ = {}

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+             try:

+                 self.SUBJECT.main(argv, environ)

+             except TypeError as xcept:

+                 pass  # No clue why argparse throws this only while under test

+         self.assertFalse(self.fake_stdout.getvalue())

+         self.assertTrue(self.exit_code)

+         self.assertRegex(self.fake_stderr.getvalue(), r'usage:')

+ 

+     def validate_happy_path(self, argv, environ, n_subs=0):

+         """Helper for checking happy-path + negative input == empty cache"""

+         self.MockOpen.return_value = mock_cachefile = StringIO()

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr), self.silence_debug():

+             # Actual call to main() for test_empty_mmlist() (below)

+             self.SUBJECT.main(argv, environ)

+         self.assertEqual(self.exit_code, None)

+         self.assertFalse(self.fake_stdout.closed)

+         self.assertFalse(self.fake_stderr.closed)

+         # Verify no errors or any other output

+         _stderr = self.fake_stderr.getvalue()

+         try:

+             if 'Debugging enabled' not in _stderr:

+                 self.assertFalse(_stderr)

+         except AssertionError:  # Only happens when '--debug' was specified

+             # A non-fatal warning was issued

+             self.assertRegex(_stderr, r"[Ss]kipping.+qcow2.+does not end in.+")

+         # Verify locking was done correctly

+         self.validate_mock_fcntl()

+         msg_fmt = '\n{} JSON:\n{}\n'

+         jsons = dict(stdout=self.fake_stdout.getvalue().replace('(Pdb) ', ''),

+                      cache=mock_cachefile.getvalue())

+         # In case pdb is in use :D

+         for jsons_name, json_str in jsons.items():

+             try:

+                 jsons[jsons_name] = json.loads(json_str)

+             except ValueError:  # Parsing failed, assist debugging

+                 print(msg_fmt.format(jsons_name, json_str))

+                 raise

+ 

+         # Confirm any/all JSON output parses and always matches DEFAULT_CACHE (i.e. empty)

+         if '--list' in argv:

+             self.assertDictEqual(jsons['stdout'], self.SUBJECT.InvCache.DEFAULT_CACHE)

+             # Cache contents match stdout

+             self.assertDictEqual(jsons['cache'], jsons['stdout'])

+         if '--host' in argv:

+             self.assertDictEqual(jsons['stdout'], dict())

+         count = self.mock_create_host.call_count

+         self.assertEqual(count, n_subs,

+                          'create_host() called {} times, expecting {}'.format(count, n_subs))

+         # Confirm if/that create_host() was called according to it's API

+         if self.mock_create_host.called:

+             for kall in self.mock_create_host.call_args_list:

+                 args = list(kall[0])

+                 dargs = kall[1]

+                 _opts = dargs.get('opts')

+                 _image_path = dargs.get('image_path')

+                 _environ = dargs.get('environ')

+                 if args:

+                     _opts = args.pop(0)

+                 if args:

+                     _image_path = args.pop(0)

+                 if args:

+                     _environ = args.pop(0)

+                 if '--list' in argv:

+                     self.assertTrue(_opts.list)

+                     self.assertFalse(_opts.host)

+                 if '--host' in argv:

+                     self.assertFalse(_opts.list)

+                     self.assertTrue(_opts.host)

+                 self.assertDictEqual(_environ, environ)

+                 # Image must be found at least once

+                 foundit = False

+                 for arg in argv:

+                     if arg.find(_image_path.strip()):

+                         foundit = True

+                         break

+                 if not foundit:

+                     for val in environ.values():

+                         if val.find(_image_path.strip()):

+                             foundit = True

+                             break

+                 self.assertTrue(foundit, 'image_path {} parameter from create_host'

+                                          ' was not found in argv {} or environ {}'

+                                          ''.format(_image_path, argv, environ))

+ 

+     def test_empty_mmlist(self):

+         """--list w/ invalid subjects always results in empty JSON output & empty cache."""

+         # Subtest mesage paired with arguments to validate_happy_path()

+         test_input = {

+                 "Validate output with no subjects":

+                     dict(argv=[self.SUBJECT_PATH, '--list'],

+                          environ=dict()),

+                 "Validate output with non-qcow2 argument":

+                     dict(argv=[self.SUBJECT_PATH, '--list', '/path/to/foobar'],

+                          environ=dict(), n_subs=1),

+                 "Validate output with non-qcow2 argument containing a space":

+                     dict(argv=[self.SUBJECT_PATH, '--list', '/space.qcow2 /foobar'],

+                          environ=dict(), n_subs=1),

+                 "Validate output with whitespace argument":

+                     dict(argv=[self.SUBJECT_PATH, '--list', ' ', '--debug'],

+                          environ=dict()),

+                 "Validate output with multiple non-qcow2 arguments":

+                     dict(argv=[self.SUBJECT_PATH, '--list', '--debug', '/path/to/foobar', '../baz'],

+                          environ=dict(), n_subs=2),

+                 "Validate output with multiple non-qcow2 arguments containing a space":

+                     dict(argv=[self.SUBJECT_PATH, '--list', '/space.qcow2 /foobar', '../baz'],

+                          environ=dict(), n_subs=2),

+                 "Validate output with multiple whitespace arguments":

+                     dict(argv=[self.SUBJECT_PATH, '--list', '', '\n\t\t'],

+                          environ=dict()),

+                 "Validate output with empty TEST_SUBJECTS":

+                     dict(argv=[self.SUBJECT_PATH, '--list'],

+                          environ=dict(TEST_SUBJECTS="")),

+                 "Validate output with non-qcow2 TEST_SUBJECTS":

+                     dict(argv=[self.SUBJECT_PATH, '--list'],

+                          environ=dict(TEST_SUBJECTS="/path/to/foobar"), n_subs=1),

+                 "Validate output with non-qcow2 TEST_SUBJECTS containing a space":

+                     dict(argv=[self.SUBJECT_PATH, '--list'],

+                          environ=dict(TEST_SUBJECTS="'/space.qcow2 /foobar'"), n_subs=1),

+                 "Validate output with multiple whitespace TEST_SUBJECTS":

+                     dict(argv=[self.SUBJECT_PATH, '--debug', '--list', ' ', '\n\t\t'],

+                          environ=dict(TEST_SUBJECTS="'\t\t\n ' '  \n  ' '\n     \t      \n'")),

+                 "Validate output with non-qcow2 argument and TEST_SUBJECTS":

+                     dict(argv=[self.SUBJECT_PATH, '--list', '/path/to/foo'],

+                          environ=dict(TEST_SUBJECTS="/path/to/bar"), n_subs=2),

+                 "Validate output with duplicate non-qcow2 argument and TEST_SUBJECTS":

+                     dict(argv=[self.SUBJECT_PATH, '--list', '/path/to/foobar'],

+                          environ=dict(TEST_SUBJECTS="/path/to/foobar"), n_subs=1),

+                 "Validate output with duplicate whitespace arguments and TEST_SUBJECTS":

+                     dict(argv=[self.SUBJECT_PATH, '--list', ' '],

+                          environ=dict(TEST_SUBJECTS=" ")),

+                 "Validate output with --host and non-qcow2 subject":

+                     dict(argv=[self.SUBJECT_PATH, '--host', '/foo/bar'],

+                          environ=dict(), n_subs=1),

+         }

+         for msg, dargs in test_input.items():

+             # Holds any failures until all subtests have run

+             with self.subTest(msg, **dargs):

+                 sys.stdout.write('.')  # Deserve some credit :D

+                 sys.stdout.flush()

+                 self.validate_happy_path(**dargs)

+                 for notused in (self.mock_start_qemu, self.mock_subject_watcher):

+                     self.assertFalse(notused.called)

+             # One subtest's mocks will fubar next subtest

+             self.reset()

+ 

+ 

+ class TestCreateHost(TestCaseBase):

+     """Tests for the ``create_host()`` function"""

+ 

+     fake_stdout = None

+     fake_stderr = None

+     fake_popen = None

+     poll_side_effect = None

+     mock_subprocess = None

+     make_opts = lambda self, **dargs: MagicMock(spec=self.SUBJECT.argparse.Namespace, **dargs)

+ 

+ 

+     def reset(self):

+         super(TestCreateHost, self).reset()

+         # Just pile up io in the same "file" for inspection

+         self.MockOpen.return_value.close = MagicMock()

+         self.fake_stdout = StringIO()

+         self.fake_stderr = StringIO()

+         self.fake_popen.reset_mock()

+         self.mock_start_qemu.reset_mock()

+         self.mock_subject_watcher.reset_mock()

+         self.mock_subprocess.reset_mock()

+         for attr in dir(subprocess):

+             mockattr = MagicMock(spec=getattr(subprocess, attr), spec_set=True)

+             # represent int constants properly

+             if attr.capitalize() == attr:

+                 mockattr.__int__ = getattr(subprocess, attr)

+             elif attr in ('Popen', 'CompletedProcess', 'call', 'run', 'check_call', 'check_output'):

+                 # Tests will most likely need to modify this for their purpose

+                 mockattr = self.fake_popen

+             elif isinstance(getattr(subprocess, attr), BaseException):

+                 # Use real exceptions

+                 mockattr = getattr(subprocess, attr)

+             self.mock_subprocess.attach_mock(mockattr, attr)

+ 

+     def setUp(self):

+         self.fake_popen = MagicMock(spec=subprocess.Popen)

+         for name, value in dict(stdin=0, stdout=1, stderr=2, pid=42, returncode=None).items():

+             setattr(self.fake_popen, name, value)

+         self.mock_start_qemu = create_autospec(spec=self.SUBJECT.start_qemu,

+                                                spec_set=True, instance=True,

+                                                return_value=(self.fake_popen, 12345, 67890))

+         self.mock_subject_watcher = create_autospec(spec=self.SUBJECT.subject_watcher,

+                                                     spec_set=True, instance=True)

+         self.mock_subprocess = create_autospec(spec=subprocess,

+                                                spec_set=True, instance=True)

+         self.patchers = [patch('{}.subprocess'.format(TestCaseBase.SUBJECT_NAME),

+                                self.mock_subprocess),

+                          patch('{}.start_qemu'.format(TestCaseBase.SUBJECT_NAME),

+                                self.mock_start_qemu),

+                          patch('{}.subject_watcher'.format(TestCaseBase.SUBJECT_NAME),

+                                self.mock_subject_watcher)]

+         super(TestCreateHost, self).setUp()

+ 

+     def test_no_qcow2(self):

+         """Verify calling create_host() with a non-qcow2 subject raises NoQcow2Error"""

+         imp = '/dev/null'

+         opts = self.make_opts(list=True, host=None, debug=False, subjects=[imp])

+         # N/B: This exception is normally ignored by main() but

+         #      printed if --debug ise enabled.  This is just testing the API/behavior

+         #      of create_host() detecting a request for a non-qcow2 subject.

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+             self.assertRaisesRegex(self.SUBJECT.NoQcow2Error, imp,

+                                    self.SUBJECT.create_host,

+                                    opts, imp, {})

+ 

+     def test_fugly_image(self):

+         """Verify calling create_host() with a fugly qcow2 image behaves"""

+         imp = '\t/sp ce/qcow2 / path/\0/ \n /@/foobar.qcow2 \n'

+         opts = self.make_opts(list=True, host=None, debug=False, subjects=[imp])

+         # poll() return order must match calling order :(

+         self.fake_popen.poll.side_effect = [None, None, 0]

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+             self.SUBJECT.create_host(opts, imp, {})

+ 

+ 

+     def test_qcow2_c(self):

+         for imp in ('/var/lib/libvirt/images/foo.qcow2', '/var/lib/libvirt/images/foo.qcow2c'):

+             opts = self.make_opts(list=True, host=None, debug=False, subjects=[imp])

+             self.fake_popen.poll.side_effect = [None, None, 0]

+             with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+                 self.SUBJECT.create_host(opts, imp, {})

+ 

+     def test_multi_debug_cache(self):

+         for imp in ('a.qcow2', 'b.qcow2c', 'c.qcow2', 'd.qcow2c'):

+             opts = self.make_opts(list=True, host=None, debug=True, subjects=[imp])

+             self.fake_popen.poll.side_effect = [None, None, 0]

+             with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+                 self.SUBJECT.create_host(opts, imp, {})

+             stderr = self.fake_stderr.getvalue()

+             stdout = self.fake_stdout.getvalue()

+             self.fake_stderr.truncate()

+             self.fake_stdout.truncate()

+             for expected in (r'Launching.+{}'.format(imp), r'Contacting.+{}'.format(imp)):

+                 self.assertRegex(stderr, expected)

+             for stdio in (stderr, stdout):

+                 self.assertNotRegex(stdio, '.*Traceback.*')

+             self.assertFalse(stdout)

+ 

+ 

+ class TestSubjectWatcher(TestCaseBase):

+     """Tests for the ``subject_watcher()`` function"""

+ 

+     fake_kill = None

+     fake_stdout = None

+     fake_stderr = None

+ 

+     def reset(self):

+         super(TestSubjectWatcher, self).reset()

+         self.MockOpen.return_value.close = MagicMock()

+         self.MockOpen.return_value.fileno = MagicMock(return_value=2)

+         for faker in ('fake_stdout', 'fake_stderr'):

+             setattr(self, faker, MagicMock(wraps=StringIO()))

+         self.fake_kill.reset_mock()

+ 

+     def setUp(self):

+         self.fake_kill = MagicMock(spec=os.kill)

+         self.patchers = [patch('{}.sys.exit'.format(TestCaseBase.SUBJECT_NAME),

+                                MagicMock()),

+                          patch('{}.os.dup2'.format(TestCaseBase.SUBJECT_NAME),

+                                MagicMock()),

+                          patch('{}.os.kill'.format(TestCaseBase.SUBJECT_NAME),

+                                self.fake_kill),

+                          patch('{}.sys.stdin.close'.format(TestCaseBase.SUBJECT_NAME),

+                                MagicMock(return_value=None)),

+                          patch('{}.InvCache'.format(TestCaseBase.SUBJECT_NAME),

+                                MagicMock(spec=self.SUBJECT.InvCache))]

+         super(TestSubjectWatcher, self).setUp()

+ 

+     def test_happy_path(self):

+         """Test calling subject_watcher normally works as expected"""

+         qemu_pid = 12345

+         hostname = 'bar'

+         tempdir = self.TEMPDIRPATH

+         self.fake_kill.side_effect = [0, 0, 0, OSError(ESRCH, "PID DIED!")]

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+             self.SUBJECT.subject_watcher(qemu_pid, hostname, tempdir, {})

+         self.assertTrue(self.fake_stdout.closed)

+         self.assertTrue(self.fake_stderr.closed)

+         self.assertTrue(self.fake_kill.called)

+         self.fake_kill.assert_any_call(qemu_pid, 0)

+         self.fake_kill.assert_any_call(self.PPID, 0)

+         self.validate_mock_fcntl()

+ 

+     def test_bad_path(self):

+         qemu_pid = 12345

+         hostname = 'bar'

+         tempdir = self.TEMPDIRPATH

+         # N/B: Inf. loop possible if kill doesn't eventually throw exception

+         msg = "SOMETHING BROKE WHILE UNITTESTING!"

+         self.fake_kill.side_effect = [OSError(-1, msg), 0, 0, OSError(ESRCH, "PID DIED!")]

+         with redirect_stdout(self.fake_stdout), redirect_stderr(self.fake_stderr):

+             self.assertRaisesRegex(OSError, msg, self.SUBJECT.subject_watcher,

+                                    qemu_pid, hostname, tempdir, {})

+         self.assertTrue(self.fake_stdout.closed)

+         self.assertTrue(self.fake_stderr.closed)

+         self.assertTrue(self.fake_kill.called)

+         self.validate_mock_fcntl()

+ 

+ 

+ if __name__ == '__main__':

+     unittest.main()

Fix standard-inventory-qcow2 standards compliance

It's possible for the inventory to be queried more than once.  However,
with current implementation this would result in starting multiple VMs,
possibly with clashing names and multiple useless logfiles and excess
resource consumption.

The --host CLI is expected to have identical output with every call
with identical input.  The current implementation results in
both varying output and launching multiple VMs, possibly with clashing
names and causing excessive and unintended resource consumption.

The inventory API expects dns-compliant hostnames, however the
image-path is assumed as CWD but never validated.  Therefor
it's easily possible for pathname components to end up as
part of the VMs hostname, possibly leading to odd behavior.

Fix the path vs hostname issue by passing the path through
``os.path.basename`` for all applicable calls and references.

Verify there are no conflicts/clashes over hostnames prior to
creating VMs or caching details.

Employ locking on persistent cache to prevent multiple parallel
calls from corrupting data.

All calls with either --host or --list, utilize the cache and
check for existing VMs before creating new ones.  Upon parent-process
exit, they are removed from cache.

Add unittests to cover most of the above features / changes.

Signed-off-by: Chris Evich <cevich@redhat.com>

If acceptable, I'm willing to do something similar for the other standard scripts as well.

.strip() needs to be .split(',') for this to work as expected.

We're working on making things python 3 compatible. (See PR #40) Please make this except EnvironmentError as ex:.

Where is the cache file expected to be written to? /usr/share/ansible/inventory/standard-inventory-qcow2.~?

Also, how does the cache file ever get cleaned up?

Are you aware that the standard-inventory-qcow2 script is intended to automatically spin up a VM based on a cloud image test subject when ansible-playbook is started, and destroy/cleanup the VM automatically when ansible-playbook exits?

I like the idea of caching things, but I can't get it to work after running a test playbook the first time. The cache file never gets cleaned up, so subsequent test runs attempt to connect to a VM (from a previous playbook run) that no longer exists.

Where is the cache file expected to be written to? /usr/share/ansible/inventory/standard- inventory-qcow2.~?

The cache file ends up wherever the inventory script itself lives. I s'pose it could go anywhere else, I just picked that spot because it was convenient.

Also, how does the cache file ever get cleaned up?

Good idea, that would better support manually running tests several times. I'll add that.

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

fixup! addresses all review feedback (except cache removal)

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

fixup! (untested) suggests method for safe cache cleanup + minor code cleanup and comments

Is this an acceptable cleanup approach, or is there a better idea?

I'm not married to the cache-file location, that could use a tempfile if preferable.

rebased

6 years ago

DNR: Finishing up much better caching and cleanup re-write.

I know you're still working on it, but I tested out the latest revision anyhow and had a few issues that I want to be sure are addressed.

  • If the script is run with $TEST_SUBJECTS unset, or set to something other than a qcow2 image, it caches empty data and will continue to return that cached (empty) data on future runs regardless of the setting of $TEST_SUBJECTS. The script needs to properly handle non-qcow2 values in $TEST_SUBJECTS since ansible will call it for any type of test subject(s).

For example:

$ sudo rm standard-inventory-qcow2.~.orig
$ sudo TEST_SUBJECTS=local ./standard-inventory-qcow2 --list
{
    "subjects": {
        "hosts": [],
        "vars": {}
    },
    "_meta": {
        "hostvars": {}
    },
    "localhost": {
        "hosts": [],
        "vars": {}
    }
}$
$ sudo TEST_SUBJECTS=$HOME/atomic26.qcow2 ./standard-inventory-qcow2 --list
{
    "subjects": {
        "hosts": [],
        "vars": {}
    },
    "_meta": {
        "hostvars": {}
    },
    "localhost": {
        "hosts": [],
        "vars": {}
    }
}$
$ cat standard-inventory-qcow2.~.orig
{"subjects": {"hosts": [], "vars": {}}, "_meta": {"hostvars": {}}, "localhost": {"hosts": [], "vars": {}}}$
$
  • I am unable to get the updated script to run without error in the simplest case. Any ideas what I could be doing wrong? (Note: the VM does get launched and is left running in the background.)
$ sudo rm standard-inventory-qcow2.~.orig
$ sudo TEST_SUBJECTS=$HOME/atomic26.qcow2 ./standard-inventory-qcow2 --list
Launching virtual machine for /home/merlinm/atomic26.qcow2
Traceback (most recent call last):
  File "./standard-inventory-qcow2", line 316, in <module>
    sys.exit(main(sys.argv))
  File "./standard-inventory-qcow2", line 86, in main
    data = list(opts.subjects)
  File "./standard-inventory-qcow2", line 147, in list
    vars = host(subject)
  File "./standard-inventory-qcow2", line 261, in host
    with lock_cache_filepath(fcntl.LOCK_SH):
  File "/usr/lib64/python2.7/contextlib.py", line 17, in __enter__
    return self.gen.next()
  File "./standard-inventory-qcow2", line 120, in lock_cache_filepath
    yield fcntl.flock(cache_filepath(environ, argv), mode)
TypeError: argument must be an int, or have a fileno() method.
$

This suffix list incorrect. "~" and ".orig" are two separate suffixes.

This is no long monitoring the correct parent PID. Since this is running in the context of the forked child and its parent will have have exited, os.getppid() here will always return 1--which will never go away. This needs to check the parent PID that is captured by os.getppid() before the fork (as was originally implemented).

I still have issue with writing a cache file to /usr/share/ansible/inventory/. Somewhere such as /var/cache/ would be much more appropriate and conform to the Filesystem Hierarchy Standard. Also, how are things supposed to work if multiple instances of standard-inventory-qcow2 are run simultaneously on the same system--such as when multiple tests are being run in parallel?

By the way, thank you for your work on this! The refactoring is taking shape nicely.

@merlinm thank you so much for looking at the early work, yes, it's not working :D

I'm finding the re-implementation of the cache-handling piece really needs unittesting. So I'm working on adding some python-3 based unittests (which requires some learning). For now I'm tossing them into /tests, with the idea we can drop in the standard Ansible interface playbooks later.

The script needs to properly handle non-qcow2 values in $TEST_SUBJECTS since ansible will call it for any type of test subject(s).

Yes, I agree. My understanding is that other inventory scripts would do their own caching so it was safe to ignore non-qcow2 input items. That's not the case then? What should the JSON look like for non-qcow2 inputs?

TypeError: argument must be an int

Yep, it's busted :D

This suffix list incorrect. "~" and ".orig" are two separate suffixes.

and

I still have issue with writing a cache file to /usr/share/ansible/inventory/.

These problems both go away by moving the persistent cache elsewhere as suggested.

Currently I'm forming the file-path as: /<tempfile.gettempdir()>/<inventory-script-name>_<uid>_<gid>_<ppid>.cache. Which should tie it sufficiently to the proper context. Though I'm happy to change this to anything else.

Also, I'm implementing the cache mechanism generically enough that it can be re-used by the other inventory scripts.

@cevich,

The script needs to properly handle non-qcow2 values in $TEST_SUBJECTS since ansible will call it for any type of test subject(s).

Yes, I agree. My understanding is that other inventory scripts would do their own caching so it was safe to ignore non-qcow2 input items. That's not the case then? What should the JSON look like for non-qcow2 inputs?

Sorry if I wasn't clear.

Your understanding is correct. Each script should operate and cache independently. However, ansible indiscriminately calls each and every script in the $ANSIBLE_INVENTORY directory regardless of the subject types in $TEST_SUBJECTS.

If you follow my previous transcript, you'll see the standard-inventory-qcow2 script was caching the empty results after it was run with TEST_SUBJECTS=local, and then when called with TEST_SUBJECTS=$HOME/atomic26.qcow2 (which referenced a valid dowloaded image) it would keep returning the empty cached results from the earlier non-qcow2 call instead of launching the qcow2.

It is correct for standard-inventory-qcow2 to return the empty results if no qcow2 subjects are found. However, it should never cache the results for non-qcow2 subjects. Each of the inventory scripts should deal only with caching for it's own subject type (if applicable).

Does that make sense now?

These problems both go away by moving the persistent cache elsewhere as suggested.
Currently I'm forming the file-path as: /<tempfile.gettempdir()>/<inventory-script-name><uid><gid>_<ppid>.cache. Which should tie it sufficiently to the proper context. Though I'm happy to change this to anything else.

That sounds like it should work.

Also, I'm implementing the cache mechanism generically enough that it can be re-used by the other inventory scripts.

Perfect!

rebased

6 years ago

Does that make sense now?

Yes, so this is a problem with "negative" caching, gotcha.

Latest push is probably not working code, however I've added unittests and those are working.

Let me stub in something for adding a negative-cache unittest, that's a really important thing to check.

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

Latest push is probably not working code, however I've added unittests and those are working.

Yeah. I'm not having much luck running it.
If I set TEST_SUBJECTS to a qcow2 image, I get a NameError: global name 'image' is not defined exception.
If I set TEST_SUBJECTS=local, I get a ValueError: Error launching VM 'local' for 'local',does not end in '.qcow2'. exception.
If I leave TEST_SUBJECTS unset, I get zero-length output. (Even in the null case, it needs to output a skeletal JSON dictionary like that shown in the earlier comment.)

Right, thanks for the reminder, I'll add a unittest for that, it's important.

Neck-deep into happy/sad path unittests for high-level entry/exit conditions. Turns out patching/mocking sys.stderr doesn't play nicely with using pdb for debuggin/troubleshooting :confounded:. Making slow/steady progress though.

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

Whew! I got it (finally). Okay, unittests (tests/test_standard-inventory-qcow2.py) are 100% passing again, including a new one for @merlinm's desire for correct output when $TEST_SUBJECTS and the subjects arg are both empty or invalid.

No idea if the actual inventory-script passes, haven't gotten that far yet. I'm doing more of a TDD thing here. So...more tests are needed...

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

rebased and pushed. Found a much easier way to handle unittesting w/ stdout/stderr that doesn't fuss with pdb. https://docs.python.org/3.6/library/contextlib.html?highlight=contextlib#contextlib.redirect_stdout

Implemented that along with a few bug fixes to the inventory script itself. The script now runs under argument-error conditions and does the right thing. No idea if it actually creates hosts, probably not. Testing that stuff is the next step.

3 new commits added

  • fixup! Fix standard-inventory-qcow2 standards compliance
  • fixup! Fix standard-inventory-qcow2 standards compliance
  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

Made lots and lots of progress, many bugs found & squashed. Only three more functions left to write tests for.

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

... including a new one for @merlinm's desire for correct output when $TEST_SUBJECTS and the subjects arg are both empty or invalid.

Thanks. But it's not just my desire to handle $TEST_SUBJECTS correctly, it's absolutely necessary. Just because standard-inventory-qcow2 thinks one of the subjects in $TEST_SUBJECTS is invalid, it may be the expected value to one of the other inventory scripts.

I keep trying to point this out but I must not be doing a very good job explaining it. Ansible indiscriminately calls each and every script in the $ANSIBLE_INVENTORY directory regardless of the value of $TEST_SUBJECTS. Each inventory script is responsible for detecting the subjects within $TEST_SUBJECTS that it is interested in and acting upon them accordingly. Thus, standard-inventory-qcow2 should identify and act only upon a qcow2 subject, and ignore anything else. Likewise, standard-inventory-docker identifies and acts only upon a docker subject within $TEST_SUBJECTS--ignoring qcow2s and anything else. (However, each inventory script still needs to output the skeletal JSON dictionary even when it ignores all the subjects.)

Please let me know if you have any questions about this point!

I've only glanced at the code, but I did run a few quick manual tests on the latest version of the script and here are a few initial observations.

  • As I attempted to explain in my previous comment, don't issue warnings just because a subject does not end with .qcow2--such as is shown in the following:
# TEST_SUBJECTS=local ./standard-inventory-qcow2  --list
Warning: Error launching VM 'local' with image 'local', image does not end in '.qcow2'.
{
    "localhost": {
        "hosts": [],
        "vars": {}
    },
    "subjects": {
        "hosts": [],
        "vars": {}
    },
    "_meta": {
        "hostvars": {}
    }
}
  • It can get rather verbose during the "Contacting..." phase on a slow machine
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #1/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #2/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #3/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #4/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #5/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #6/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #7/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #8/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #9/30                                                                                                  
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #10/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #11/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #12/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #13/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #14/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #15/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #16/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #17/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #18/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #19/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #20/30
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #21/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #22/30                                                                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #23/30                    
...
  • On the positive side, the VM launched successfully was contactable using the information reported in the inventory JSON!

  • Unfortunately, the VM doesn't get cleaned up when the parent process (eg., the one that launches standard-inventory-qcow2) exits. The forked standard-inventory-qcow2 process and the VM continue to run in the background. It should have been automatically stopped.

  • Upon manually killing the qemu process to simulate the VM exiting, a traceback is thrown:

Traceback (most recent call last):
  File "./standard-inventory-qcow2", line 528, in <module>
    main(sys.argv)
  File "./standard-inventory-qcow2", line 340, in main
    create_host(opts, subject, environ)
  File "./standard-inventory-qcow2", line 462, in create_host
    subject_watcher(proc, opts, port, hostname)
  File "./standard-inventory-qcow2", line 523, in subject_watcher
    hostname = os.path.basename(subject)
NameError: name 'subject' is not defined

But it's not just my desire to handle

@merlinm No need to essplain, I gotcha, we're on the same page :D

I added some unittests for main(), and they're passing (returning skeleton JSON, ignoring the request). If it's still breaking otherwise...there's a bug I needa fix...

Since the latest standard-inventory-qcow2 did successfully launch the requested VM, just for fun I tried using it in the normal manner as a dynamic inventory provider to ansible to run a playbook in the VM.

The first problem I ran into was that it caused ansible to hang. Suspecting it might be related to PR #47 (and Issue #52), I applied the associated update.

I tried again. This time the VM was launched and ansible proceeded (without hanging) to start running the playbook in the VM. It got a short way into the playbook and then stopped and prompted me for the VM's root password. Upon entering the password, ansible aborted:

root@127.0.0.3's password: 
fatal: [Fedora-Atomic-26-20170821.0.x86_64.qcow2]: FAILED! => {"changed": false, "cmd": "/bin/rsync --delay-updates -F --compress --archive --rsh=/bin/ssh -S none -i /tmp/inventory-cloudxzgoe9cu/ssh_priv_key -o Port=2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null --out-format=<<CHANGED>>%i %n%L /home/merlinm/upstreamfirst/sed/ root@127.0.0.3:/usr/local/bin/", "failed": true, "msg": "Warning: Identity file /tmp/inventory-cloudxzgoe9cu/ssh_priv_key not accessible: No such file or directory.\nWarning: Permanently added '[127.0.0.3]:2222' (ECDSA) to the list of known hosts.\r\nAuthentication failed.\r\nrsync: connection unexpectedly closed (0 bytes received so far) [sender]\nrsync error: unexplained error (code 255) at io.c(226) [sender=3.1.2]\n", "rc": 255}

Upon trying that again, I noticed the inventory script had already removed the /tmp/inventory-cloudXXXXXXXX directory (including the private key identity file needed by ansible). The inventory script shouldn't clean up that directory until after it has shut down the VM.

I also noticed the first "shebang" line of the inventory script forces the use of python3. Please make sure the final version of the script runs under either python2 or python3, but the shebang line should just reference "python".

Thank you again for all your work!

don't issue warnings

Oh okay, can-do. Or maybe I'll make that a --debug only option.

It can get rather verbose during the "Contacting..."

Yes, I agree. Okay, --debug only option then? Or I could make it print "......" instead?

On the positive side, the VM launched successfully

No way! I haven't even tested it that far. Cool!

Unfortunately, the VM doesn't get cleaned up

Upon manually killing the qemu process

I expect not, I haven't addressed any of that code in a long while, and there's no unittests (yet). I'm sure it's horribly broken. Baby steps.

the inventory script had already removed the

Oops, that's my bad. I'll fix that (and add a unittest for it).

I also noticed the first "shebang" line of the

Got it, yep, okay no prob.

Thanks for the quick-review, you're braver than I am :grinning:

@merlinm this unittest (of subtests) confirms empty-inventory output as required. The test_input dict. (above) contains all the sets of argv and environ inputs to main() that I could think of. Did I miss any?

This is the call to main() for the test_empty_mmlist unittest (below)

This is the confirmation that the JSON output for all non-qcow2 cases match the default (empty, boiler-plate) JSON.

All the rest here is just confirming that create_host() was called properly

(even though it did NOT create a host :grinning:)

I haven't actually looked at the tester script yet. I'll do that one of these days in the near future and then I'll take a shot at answering your question. :confused:

main() catches this exception and ignores it. But it's useful to have the exception for unittesting create_host() directly (w/o going through main()). That's why this test is here.

Note to me: Only check this if --debug was used?

Note to me: Only write() if --debug?

I haven't actually looked at the tester script yet.

Aww :cry: that's the tasty part. Testing is always 10-times harder than writing bad code :smile_cat: Python's unittest.mock never fails to turn my brain inside-out, make my eyes cross, and my butt-cheeks clench :astonished:

Note to me: Move this under subject_watcher(). Don't kill the ssh-key file before Ansible is done with the host.

nitpick: probably singe -> single

@cevich

Could you please make sure recently merged PRs #53 and #56 don't get lost the next time you rebase?

Also, many of the comments you made above on tests/test_standard-inventory-qcow2.py would be appropriate to include as in-line comments directly within the code. Would you consider doing so?

Thank you!

@cevich This PR addresses Issue #54, right? (At least for standard-inventory-qcow2.)

Note to me: Fix "singe -> single"

Could you please make sure recently merged PRs #53 and #56 don't get lost the next time you rebase?

Yes, thanks for the reminder. Those should cause conflicts (which I'll address).

Also, many of the comments you made above

Sure, if you think they're helpful, someone else might too. No prob.

This PR addresses Issue #54, right?

Yes it will.

4 new commits added

  • fixup! Fix standard-inventory-qcow2 standards compliance
  • fixup! Fix standard-inventory-qcow2 standards compliance
  • fixup! Fix standard-inventory-qcow2 standards compliance
  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

--force push after --rebase, addressed all the above concerns, integrated other mentioned PRs, updated comments, and fixed up unittests to pass :grinning:

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

Running for a non-qcow2 subject type returns template JSON. Good. But running it a second time results in an exception. I can't even re-run it for a qcow2 subject...

root# TEST_SUBJECTS="local" ./inventory/standard-inventory-qcow2 --list
{
    "subjects": {
        "hosts": [],
        "vars": {}
    },
    "_meta": {
        "hostvars": {}
    },
    "localhost": {
        "hosts": [],
        "vars": {}
    }
}root#
root# TEST_SUBJECTS="local" ./inventory/standard-inventory-qcow2 --list
Traceback (most recent call last):
  File "./inventory/standard-inventory-qcow2", line 548, in <module>
    main(sys.argv)
  File "./inventory/standard-inventory-qcow2", line 319, in main
    environ["TEST_ARTIFACTS"] = artifacts_dirpath(environ)
  File "/usr/lib64/python2.7/os.py", line 473, in __setitem__
    putenv(key, item)
TypeError: putenv() argument 2 must be string, not None
root#
root# TEST_SUBJECTS="/root/Fedora-Atomic-26-20170821.0.x86_64.qcow2" ./inventory/standard-inventory-qcow2 --list
Traceback (most recent call last):
  File "./inventory/standard-inventory-qcow2", line 548, in <module>
    main(sys.argv)
  File "./inventory/standard-inventory-qcow2", line 319, in main
    environ["TEST_ARTIFACTS"] = artifacts_dirpath(environ)
  File "/usr/lib64/python2.7/os.py", line 473, in __setitem__
    putenv(key, item)
TypeError: putenv() argument 2 must be string, not None
root#

If the artifacts directory exists, it never gets to the return artifacts--causing artifacts_dirpath() to return None and raise an exception. Move the return to after the try block.

qemu spews a warning message when starting the VM.

root# rm -rf artifacts
root# TEST_SUBJECTS="/root/Fedora-Atomic-26-20170821.0.x86_64.qcow2" ./inventory/standard-inventory-qcow2 --list                                                       
qemu-system-x86_64: -mon chardev=mon0,mode=readline,default: option 'default' does nothing and is deprecated                                                           
{                                        
    "subjects": {                        
        "hosts": [                       
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2"                             
        ],                               
        "vars": {}                       
    },                                   
    "_meta": {                           
        "hostvars": {                    
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2": {                          
                "ansible_ssh_common_args": "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no",                                                              
                "ansible_port": "2222",  
                "ansible_ssh_private_key_file": "/home/merlinm/pagure/_fork/cevich/standard-test-roles/artifacts/.inventory-cloudLgggl4.temp/ssh_priv_key",            
                "ansible_ssh_pass": "foobar",                                      
                "ansible_user": "root",  
                "ansible_host": "127.0.0.3",                                       
                "qemu_monitor_port": 2223                                          
            }                            
        }                                
    },                                   
    "localhost": {                       
        "hosts": [                       
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2"                             
        ],                               
        "vars": {}                       
    }                                    
}root# qemu-system-x86_64: terminating on signal 15 from pid 21441 (python)        
root# 

The VM also exits as soon as the script exits. And where's the background watcher process?

root# ps auxww | grep qemu
root       707  0.0  0.0  30744  1616 ?        Ss   Sep15   0:00 /usr/bin/qemu-ga
root     21452  0.0  0.0 119452   992 pts/2    S+   13:46   0:00 grep --color=auto qemu
root# ps auxww | grep inventory
root     21454  0.0  0.0 119452   928 pts/2    S+   13:46   0:00 grep --color=auto inventory
root# 

By the way, I made the following patch to my local copy of standard-inventory-qcow2 before running the commands in the previous comment. Otherwise, it just continued to raise exceptions as soon as it re-created the artifacts directory mid-run...

--- a/inventory/standard-inventory-qcow2
+++ b/inventory/standard-inventory-qcow2
@@ -403,10 +403,10 @@ def artifacts_dirpath(environ=None):
     try:
         artifacts = environ.get("TEST_ARTIFACTS", os.path.join(os.getcwd(), "artifacts"))
         os.makedirs(artifacts)
-        return artifacts
     except OSError as exc:
         if exc.errno != errno.EEXIST or not os.path.isdir(artifacts):
             raise
+    return artifacts

Environment variable TEST_DEBUG is ignored now. I see you added a --debug command line option--which is fine, except ansible will never be able to call the script with such a command line option. So, make sure $TEST_DEBUG can also be used to enable debugging!

root# TEST_DEBUG=1 TEST_SUBJECTS="/root/Fedora-Atomic-26-20170821.0.x86_64.qcow2" ./inventory/standard-inventory-qcow2 --list                                          
qemu-system-x86_64: -mon chardev=mon0,mode=readline,default: option 'default' does nothing and is deprecated                                                           
{                                        
    "subjects": {                        
        "hosts": [                       
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2"                             
        ],                               
        "vars": {}                       
    },                                   
    "_meta": {                           
        "hostvars": {                    
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2": {                          
                "ansible_ssh_common_args": "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no",                                                              
                "ansible_port": "2222",  
                "ansible_ssh_private_key_file": "/home/merlinm/pagure/_fork/cevich/standard-test-roles/artifacts/.inventory-cloud7dKVcy.temp/ssh_priv_key",            
                "ansible_ssh_pass": "foobar",                                      
                "ansible_user": "root",  
                "ansible_host": "127.0.0.3",                                       
                "qemu_monitor_port": 2223                                          
            }                            
        }                                
    },                                   
    "localhost": {                       
        "hosts": [                       
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2"                             
        ],                               
        "vars": {}                       
    }                                    
}root# qemu-system-x86_64: terminating on signal 15 from pid 21788 (python)        

An exception is raised when specifying the --debug command line option. But on the bright side, the VM is left running!

root# rm -rf artifacts
root# TEST_SUBJECTS="/root/Fedora-Atomic-26-20170821.0.x86_64.qcow2" ./inventory/standard-inventory-qcow2 --list --debug                                               
Launching VM Fedora-Atomic-26-20170821.0.x86_64.qcow2 for /root/Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #1/5                                                 
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #1/30              
qemu-system-x86_64: -mon chardev=mon0,mode=readline,default: option 'default' does nothing and is deprecated                                                           
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #2/30              
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #3/30              
DEBUG: Access host with:  ssh -p 2222 -o StrictHostKeyChecking=no,UserKnownHostsFile=/dev/null root@127.0.0.3  # (root's password: foobar)                             
Traceback (most recent call last):       
  File "./inventory/standard-inventory-qcow2", line 548, in <module>               
    main(sys.argv)                       
  File "./inventory/standard-inventory-qcow2", line 356, in main                   
    create_host(opts, subject, environ)  
  File "./inventory/standard-inventory-qcow2", line 505, in create_host            
    sew("Access host's monitor with:  telnet {1} {2}".format(VM_IPV4_ADDR, mon_port))                                                                                  
IndexError: tuple index out of range     
root# 
root# ps auxww | grep qemu
root       707  0.0  0.0  30744  1616 ?        Ss   Sep15   0:00 /usr/bin/qemu-ga
root     21794 11.5 24.9 1625560 511140 pts/2  Sl   14:09   0:58 /usr/bin/qemu-system-x86_64 -m 1024 /root/Fedora-Atomic-26-20170821.0.x86_64.qcow2 -enable-kvm -snapshot -cdrom /home/merlinm/pagure/_fork/cevich/standard-test-roles/artifacts/.inventory-cloudzmVoxM.temp/cloud-init.iso -net nic,model=virtio -net user,hostfwd=tcp:127.0.0.3:2222-:22 -device isa-serial,chardev=pts2 -chardev file,id=pts2,path=/home/merlinm/pagure/_fork/cevich/standard-test-roles/artifacts/Fedora-Atomic-26-20170821.0.x86_64.qcow2.log -chardev socket,host=127.0.0.3,port=2223,id=mon0,server,nowait -mon chardev=mon0,mode=readline,default -display none
root     21858  0.0  0.0 119452   984 pts/2    S+   14:18   0:00 grep --color=auto qemu
root# 

The diagnostic message providing the ssh command line to use to access the VM is invalid and should be fixed so it can be cut & pasted. (But after manually entering the corrected command, the VM can be accessed just fine.)

root# ssh -p 2222 -o StrictHostKeyChecking=no,UserKnownHostsFile=/dev/null root@127.0.0.3
command-line line 0: unsupported option "no,UserKnownHostsFile".
root# ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@127.0.0.3
Warning: Permanently added '[127.0.0.3]:2222' (ECDSA) to the list of known hosts.
root@127.0.0.3's password: 
[root@localhost ~]# exit
logout
Connection to 127.0.0.3 closed.
root# 

I know I've provided feedback on the above line of code before... This is not monitoring for the correct parent PID. Since this portion of the code is running in the context of the forked child and its parent will have have exited, os.getppid() here will always return 1--which will never go away. You need to capture the original parent PID of the script before the fork and monitor that.

So, if proc.pid exists (as just confirmed by the previous statement) in the try, you TERM it? Why? proc.pid is the qemu process running the VM. No wonder the VM immediately goes away! This script needs to wait around until either (1) it's original parent process goes away or (2) the qemu VM process goes away due to external causes. Then, and only then, should this script make sure the VM is dead and clean up the temporary directory.

@merlinm Oh wow, thanks! Yeah, the watcher thing is pretty well fubar'd, I started writing unittests for it, and now you've found a bunch more that are needed :D

TEST_DEBUG

Oops, didn't mean to kill that.

Double-run and die

Eek! okay, best add a unittest for that one. Not sure what's going on but the tests always find out :D

...and countless other blunders I made...

Thanks again, we're into the meat of 'er now, and VMs are actually starting, so must be nearer to the end than the beginning :grinning:

Running for a non-qcow2 subject type returns template JSON. Good. But running it a second time results in an exception. I can't even re-run it for a qcow2 subject...

Okay, found/fixed that problem.

qemu spews a warning message when starting the VM.

Fixed.

Environment variable TEST_DEBUG is ignored now.

Fixed.

The diagnostic message providing the ssh command

Fixed

background process

Did a little work here, should be more-betterer now, but still untested / needs unittests.

rebase & --force push. Old unittests broken, but new ones for the above are ready. Will continue hacking on this later this week.

rebased onto dc22804d4e1dfa43135f05cb0e61e4da9ad62a4b

6 years ago

I tried giving this latest version a whirl.

It spun up the VM just fine, but no watcher process was left running to clean up...

root# TEST_SUBJECTS="/root/Fedora-Atomic-26-20170821.0.x86_64.qcow2" ./inventory/standard-inventory-qcow2 --list                 
{                                        
    "subjects": {                        
        "hosts": [                                                                 
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2"                             
        ],                               
        "vars": {}                       
    },                                                                             
    "_meta": {
        "hostvars": {
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2": {
                "ansible_ssh_common_args": "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no",                                                              
                "ansible_port": "2222",                                                                                                                                
                "ansible_ssh_private_key_file": "/home/merlinm/pagure/_fork/cevich/standard-test-roles/artifacts/.inventory-cloud6zUKzF.temp/ssh_priv_key",            
                "ansible_ssh_pass": "foobar",
                "ansible_user": "root",
                "ansible_host": "127.0.0.3",
                "qemu_monitor_port": 2223
            }
        }
    },
    "localhost": {
        "hosts": [
            "Fedora-Atomic-26-20170821.0.x86_64.qcow2"
        ],
        "vars": {}
    }
}root# 
root# ps auxww | grep inventory
root     16766 14.7 24.9 1622384 509976 pts/3  Sl   14:21   0:59 /usr/bin/qemu-system-x86_64 -m 1024 /root/Fedora-Atomic-26-20170821.0.x86_64.qcow2 -enable-kvm -snapsh
ot -cdrom /home/merlinm/pagure/_fork/cevich/standard-test-roles/artifacts/.inventory-cloud6zUKzF.temp/cloud-init.iso -net nic,model=virtio -net user,hostfwd=tcp:127.0.
0.3:2222-:22 -device isa-serial,chardev=pts2 -chardev file,id=pts2,path=/home/merlinm/pagure/_fork/cevich/standard-test-roles/artifacts/Fedora-Atomic-26-20170821.0.x86
_64.qcow2.log -chardev socket,host=127.0.0.3,port=2223,id=mon0,server,nowait -mon chardev=mon0,mode=readline -display none
root     16851  0.0  0.0 119444   968 pts/3    S+   14:27   0:00 grep --color=auto inventory
root#

Yep, more unit-tests are needed to nail down the last few not working thingies. Good progress though, fixing all those other things you found + adding unittests for each of them :D

It's tricky to debug when enabling debugging triggers an exception. :stuck_out_tongue_winking_eye:

root# rm -rf artifacts
root# TEST_DEBUG=1 TEST_SUBJECTS="/root/Fedora-Atomic-26-20170821.0.x86_64.qcow2" ./inventory/standard-inventory-qcow2 --list
# DEBUG: Debugging enabled

Launching VM Fedora-Atomic-26-20170821.0.x86_64.qcow2 for /root/Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #1/5
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #1/30
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #2/30
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #3/30
Contacting VM Fedora-Atomic-26-20170821.0.x86_64.qcow2, attempt #4/30
# DEBUG: Access host with:  ssh -p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@127.0.0.3)
Traceback (most recent call last):
  File "./inventory/standard-inventory-qcow2", line 562, in <module>
    main(sys.argv)
  File "./inventory/standard-inventory-qcow2", line 358, in main
    create_host(opts, subject, environ)
  File "./inventory/standard-inventory-qcow2", line 508, in create_host
    debug("({1}'s password: {2})".format(DEF_USER, DEF_PASSWD))
IndexError: tuple index out of range
root# 

Note: there's a stray ')' at the end of the ssh copy and paste line.

Ouch. Closing all the file descriptors leaves no way to see debug messages, exceptions, etc. from the child from this point on...

It's tricky to debug when enabling debugging triggers an exception.

Lol, doh!

Note: there's a stray ')' at the end of the ssh copy and paste line.

Okay, I'll fix that and the fact that I can't count properly :D

Okay, fixed those two things and got the unit-tests working again. I still don't expect the watcher will work, some more unittests are needed for that, and to catch the debug-mode exception you found.

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

1 new commit added

  • fixup! Fix standard-inventory-qcow2 standards compliance
6 years ago

Added unittests to catch the debug-exceptions and ssh command output. Only unittests for the watcher remain.

rebased onto 5680ddeae5415df5973f3aae9ca671f48720514b

6 years ago

rebased, --autosquash, and --force push. All unittests are written, though code is untested because...

... @merlinm My concern is in the fixme comment :D What should the behavior be in this case?

As-written, nothing is done, the watcher simply exits.

Should it attempt some cleanup, removing the host from cache, removing temp files, etc. or maybe log the exception somewhere?

(question answered on IRC)

Ugg, something broke, can't push:

$ git push origin
FATAL: W any requests/forks/cevich/standard-test-roles cevich DENIED by fallthru
(or you mis-spelled the reponame)
fatal: Could not read from remote repository.

Please make sure you have the correct access rights
and the repository exists.

Yay! Got the watcher working w/ and w/o debug mode. All unittests passing.

There's one small exception happening at the end of the watcher under some circumstances. IIUC it correctly, it just needs to be try-wrapped-n-ignored.

Unf. Can't push b/c error above :confounded: No idea what the problem is.

Okay, problem's fixed now. --force pushed and removed [WIP]. Unittest coverage isn' t 100% but it's a pretty-bit better than the 0% it was before :grinning: I did some manual testing with two VMs and it appears to be working as intended with and without --debug.

This is ready for final reviews and any last-minute fixes/changes needed.

Uggg, nope, this PR shows HEAD as 5680ddeae5 but mine is 44755d6cc so there's still a problem with the PR tracking my branch :confounded: Working on it...

rebased onto 1c3d596b0e8854e45c9ea3dc2e83d7de0706ce5b

6 years ago

...should be fixed now. Ready for review.

@merlinm ping, is there something blocking final review of this?

is there something blocking final review of this?

@cevich: Other burning priorities at the moment. Sorry for the continuing delay.

@merlinm Ahh, okay, you're still alive, good. N/P, take your time, I was just worried the zombie-apocalypse had begun and I didn't notice yet :grinning:

@cevich is this still needed? Sorry for the delayed review.

@tflink NP I totally forgot about it as well. Yes it's needed to fix the issue in the description as well as a few actual open issues.

rebased onto 5042175bfb2e79d96c2d58cf50004050030d4a76

6 years ago

rebased onto 7c11a000f44901925d739cb2430455140cb4562e

6 years ago

Rebased & --force pushed. Incorporated support for $LOCK_ON_FILE env. var. Didn't add unittests for that yet, but all others pass.

@bgoncalv could you please test build for this PR? #7 from copr. thank you!

I tested standard-test-roles-2.9-1.fc26.7c11a00.1.noarch , but the command gets stuck and it looks like some python process crashes [python] <defunct>...

# TEST_DEBUG=1 ANSIBLE_INVENTORY=$(test -e inventory && echo inventory || echo /usr/share/ansible/inventory) TEST_SUBJECTS=../atomic.qcow2 ansible-playbook --tags=atomic tests.yml

-------------------------------------------------------------

root      5243  5124  0 11:50 pts/2    00:00:01 /usr/bin/python2 /usr/bin/ansible-playbook --tags=atomic tests.yml
root      5250  5243  0 11:50 pts/2    00:00:00 [python] <defunct>
root      5252     1 54 11:50 pts/2    00:01:08 /usr/bin/qemu-system-x86_64 -m 1024 ../atomic.qcow2 -enable-kvm -snapshot -cdrom /tmp/bgoncalv/curl/artifacts/.inventory-cloudxQ8OEG.temp/cloud-init.iso -net nic,m

@bgoncalv thanks for checking it out, I'll see if I can reproduce that, fix, and add a unittest.

@bgoncalv I couldn't reproduce the defunct process problem, but I think i spotted what might have been causing it. Can you give me a quick rundown of what you did to tested standard-test-roles-2.9-1.fc26.7c11a00.1.noarch so I can verify if I caught the bug or not?

rebased onto 4f66caf921c32c7013b20e75ef01a712913e1159

6 years ago

@cevich I've tested standard-test-roles-2.9-1.fc26.79ca0e4.1.noarch, but I've got the same result.

# TEST_DEBUG=1 ANSIBLE_INVENTORY=$(test -e inventory && echo inventory || echo /usr/share/ansible/inventory) TEST_SUBJECTS=../atomic.qcow2 ansible-playbook --tags=atomic tests.yml -vvv
ansible-playbook 2.4.3.0
config file = /etc/ansible/ansible.cfg
configured module search path = [u'/root/.ansible/plugins/modules', u'/usr/share/ansible/plugins/modules']
ansible python module location = /usr/lib/python2.7/site-packages/ansible
executable location = /usr/bin/ansible-playbook
python version = 2.7.14 (default, Nov  2 2017, 18:42:05) [GCC 7.2.1 20170915 (Red Hat 7.2.1-2)]
Using /etc/ansible/ansible.cfg as config file
Parsed /usr/share/ansible/inventory/standard-inventory-docker inventory source with script plugin
Parsed /usr/share/ansible/inventory/standard-inventory-local inventory source with script plugin
^C [ERROR]: User interrupted execution

Just after starting the playbook...

root      8344  6941  6 03:29 pts/2    00:00:01 /usr/bin/python2 /usr/bin/ansible-playbook --tags=atomic tests.yml
root      8351  8344  0 03:29 pts/2    00:00:00 python /usr/share/ansible/inventory/standard-inventory-qcow2 --list
root      8353  8351 99 03:29 pts/2    00:00:17 /usr/bin/qemu-system-x86_64 -m 1024 ../atomic.qcow2 -enable-kvm -snapshot -cdrom /tmp/bgoncalv/curl/artifacts/.atomic.qcow2__vGtG_.temp/cloud-init.iso -net nic,mod
root      8359     2  0 03:29 ?        00:00:00 [kvm-pit/8353]
root      8370   941  5 03:29 ?        00:00:00 sshd: root [priv]
root      8375  8370  0 03:29 ?        00:00:00 sshd: root@pts/3
root      8376  8375  0 03:29 pts/3    00:00:00 -bash
root      8395  8351 99 03:29 pts/2    00:00:01 /usr/bin/python2 /usr/bin/ansible --inventory /tmp/bgoncalv/curl/artifacts/.atomic.qcow2__vGtG_.temp/inventory.ini atomic.qcow2 --module-name raw --args /bin/true

After about 1 minute....

root      8344  6941  1 03:29 pts/2    00:00:01 /usr/bin/python2 /usr/bin/ansible-playbook --tags=atomic tests.yml
root      8351  8344  0 03:29 pts/2    00:00:00 [python] <defunct>
root      8353     1 99 03:29 pts/2    00:01:02 /usr/bin/qemu-system-x86_64 -m 1024 ../atomic.qcow2 -enable-kvm -snapshot -cdrom /tmp/bgoncalv/curl/artifacts/.atomic.qcow2__vGtG_.temp/cloud-init.iso -net nic,mod
root      8359     2  0 03:29 ?        00:00:00 [kvm-pit/8353]
root      8370   941  0 03:29 ?        00:00:00 sshd: root [priv]
root      8375  8370  0 03:29 ?        00:00:00 sshd: root@pts/3
root      8376  8375  0 03:29 pts/3    00:00:00 -bash
root      8435     1  0 03:30 ?        00:00:00 ssh: /root/.ansible/cp/9397280a35 [mux]
root      8438     1  0 03:30 ?        00:00:00 python /usr/share/ansible/inventory/standard-inventory-qcow2 --list

@bgoncalv Thanks for the extra details, I'll try to reproduce as you explained on IRC and reply with results. Thanks.

@bgoncalv Ha! Figured it out. Problem had to do with holding onto open file-descriptors from the parent process (duhhh). I've got that fixed now, and am scratching my skull with how in the heck to unittest for that...

@bgoncalv Okay, try this one in your environment, it should be better behaved. Though I'm still uncertain what's breaking WRT stderr and debug logging, it's "mostly working" :grin: Assuming it's no-longer hanging for you, I'll tackle those shenanigans and the unittests next.

rebased onto ab83c426eeae90d33b5b8f8f6928463d314d057a

6 years ago

@cevich thanks for the fix the tests now are now able to run :-)

I've noticed though some issue when running /usr/bin/merge-standard-inventory. I think we should use "--list" if no argument is passed.

using standard-test-roles-2.9-1.fc26.054dd75.1.noarch

# /usr/bin/merge-standard-inventory
usage: standard-inventory-qcow2 [-h] (--list | --host HOST) [--debug]
                                [subjects [subjects ...]]
standard-inventory-qcow2: error: one of the arguments --list --host is required
Traceback (most recent call last):
File "/usr/bin/merge-standard-inventory", line 208, in <module>
    sys.exit(main(sys.argv))
File "/usr/bin/merge-standard-inventory", line 37, in main
    merged_data = merge_standard_inventories(argv[1:])
File "/usr/bin/merge-standard-inventory", line 96, in merge_standard_inventories
    raise RuntimeError("Could not run: {0}".format(str(cmd)))
RuntimeError: Could not run: ['/usr/share/ansible/inventory/standard-inventory-qcow2']

while with standard-test-roles-2.9-1.fc26.noarch

# /usr/bin/merge-standard-inventory
{
    "subjects": {
        "hosts": [],
        "vars": {}
    },
    "_meta": {
        "hostvars": {}
    },
    "localhost": {
        "hosts": [],
        "vars": {}
    }
}

Not really an issue, but do you think the script could print an message when it tries to start the VM. Such as Launching virtual machine for <image name> that was there before?

I think we should use "--list" if no argument is passed.

I'm not sure what merge-standard-inventory is/does but ansible dynamic-inventory interface stipulates either --list or --host must be specified. The whole motivation/point of this PR is precisely to bring the script into standards-compliance. My intention (once this is merged) was to re-use the InvCache class among the other inventory scripts, to bring them into compliance as well.

Point is, there's are lots of good reasons to comply with standards, and mostly unpredictable/terrible side-effects that come from deviating. Especially in a testing-environment, the goal is to expose unpredictability and side-effects in the subject, not the infrastructure :D

Not really an issue, but do you think the script could print an message when it tries to start the VM. Such as Launching virtual machine for <image name> that was there before?

I spent a good part of yesterday and this morning troubleshooting this...it's really, really strange that output appears at all (during the playbook run). This appears like an example of "working-by-accident", but I'm still troubleshooting how/why the message appears at all. In all cases, I don't think it's something that should be relied on. The best thing is to log any/all output to a file, otherwise count on nobody seeing it. To see for yourself, try this inventory:

$ cat inventory/foobar.sh 
#!/bin/bash

echo -e "Localhost is all there is\n" > /dev/stderr

if [[ "$1" == "--list" ]]
then
    cat << EOF
{"all": { "hosts": ["localhost"], "vars": {} },
 "_meta": { "hostvars": { "localhost": {"ansible_connection": "local"} } } }
EOF
else
    echo "{}"
fi

and playbook:

---

- hosts: localhost
  gather_facts: False
  tasks:
    - debug: msg="test message"

This is what I get:

$ ansible-playbook -i inventory/foobar.sh ./test.yml 

PLAY [localhost] ****************************************************************************************

TASK [debug] ********************************************************************************************
ok: [localhost] => {
    "msg": "test message"
}

PLAY RECAP **********************************************************************************************
localhost                  : ok=1    changed=0    unreachable=0    failed=0   

@astepano as discussed, I added a commit to resolve --list/--host, and opened an issue regarding the new ansible-inventory command.

@bgoncalv Investigating how/why Ansible passes through stderr from an inventory script is turning into a ratt's hole. Assuming it's okay, I'm going to give up that effort.

However, I will commit to some minor cleanup work on the logging/debugging code (it's ugly), and leave this PR at that (unless a bug is found). I also expect any stderr output from other inventory scripts, is likely to unpredictably stop working at some point (Just as an FYI).

1 new commit added

  • Don't require --list or --host for qcow2 subjects
6 years ago

@cevich, I think I found out how the stderr messages was showing on console. If you write to tty it will show there.
The code on master does it by redirecting stderr to tty:

try:
    tty = os.open("/dev/tty", os.O_WRONLY)
    os.dup2(tty, 2)
except OSError:
    tty = None

I've tested the latest package standard-test-roles-2.9-1.fc26.bdc6945.1.noarch and noticed that some command output is not what we expected. For example:

inventory]# ./standard-inventory-qcow2 --list
{
    "subjects": {
        "hosts": [],
        "vars": {}
    },
    "_meta": {
        "hostvars": {}
    },
    "localhost": {
        "hosts": [],
        "vars": {}
    }
}

This is correct

# TEST_SUBJECTS=/tmp/bgoncalv/atomic.qcow2 ./standard-inventory-qcow2 --list
Launching VM atomic.qcow2 for /tmp/bgoncalv/atomic.qcow2, attempt #1/5
Contacting VM atomic.qcow2, attempt #1/30
Contacting VM atomic.qcow2, attempt #2/30
Contacting VM atomic.qcow2, attempt #3/30
Contacting VM atomic.qcow2, attempt #4/30
{
    "subjects": {
        "hosts": [
            "atomic.qcow2"
        ],
        "vars": {}
    },
    "_meta": {
        "hostvars": {
            "atomic.qcow2": {
                "ansible_ssh_common_args": "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no",
                "ansible_port": "2222",
                "ansible_ssh_private_key_file": "/usr/share/ansible/inventory/artifacts/.atomic.qcow2_vPxlLp.temp/ssh_priv_key",
                "ansible_ssh_pass": "foobar",
                "ansible_user": "root",
                "ansible_host": "127.0.0.3",
                "qemu_monitor_port": 2223
            }
        }
    },
    "localhost": {
        "hosts": [
            "atomic.qcow2"
        ],
        "vars": {}
    }
}

This output is also what we expected it to be.

Now when we run the first command again the output does not seem correct, we would expect the same output as in the first case, but it seems to output the cache inventory...

# ./standard-inventory-qcow2 --list
{
    "subjects": {
        "hosts": [
            "atomic.qcow2"
        ],
        "vars": {}
    },
    "_meta": {
        "hostvars": {
            "atomic.qcow2": {
                "ansible_ssh_common_args": "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no",
                "ansible_port": "2222",
                "ansible_ssh_private_key_file": "/usr/share/ansible/inventory/artifacts/.atomic.qcow2_vPxlLp.temp/ssh_priv_key",
                "ansible_ssh_pass": "foobar",
                "ansible_user": "root",
                "ansible_host": "127.0.0.3",
                "qemu_monitor_port": 2223
            }
        }
    },
    "localhost": {
        "hosts": [
            "atomic.qcow2"
        ],
        "vars": {}
    }
}

Now when we run the first command again the output does not seem correct, we would expect the same output as in the first case, but it seems to output the cache inventory...

Unless I'm misunderstanding the comment, that's exactly what it's supposed to do (and the whole motivation behind this PR, besides adding unittests).

Quoting from the documented standard:

the script must output a JSON encoded hash/dictionary of all the groups to be managed

Since Ansible can/will query inventory at any moment, (and possibly from multiple threads). Inventory scripts must always represent the complete, atomic, state.

The forked watcher process, will remove a host from the cache only if --debug/$TEST_DEBUG was not set, the qemu process exits, or if the parent-process (ansible) or $LOCK_ON_FILE goes away.

Am I misunderstanding something?

I think I found out how the stderr messages was showing on console

Ooohhhh, jeeze, that seems an evil thing to do. Writing directly to another process's controlling terminal? Okay, I can replicate that, but, man, it sure gives me the heebie-jeebies. Thanks for tracking that down.

Update: Struggling a lot with getting the tty-output change working w/ the unittests. Considering backing up, and applying my own advice. Breaking the code up into discrete modules. Should be much easier to unittest vs trying to do so much, all at once.

rebased onto a18281b

5 years ago

okay, rebased from master, and fixed up unittests to support writing to /dev/tty. I ended up not splitting things up into modules, since that can be done via another PR. Goal here is to get it working properly per ansible expectations and get it unittested. Unless there are any last-minute code-cleanup items needed, I think this is ready. PTAL. Thanks.

@astepano Thanks. I'm going to hold off on work here in light of #167 conclusions - seems like that means abandoning this work (which is fine - simpler overall architecture is better for everyone).