Merge pull request #1058 from ceph/pytest-harness

testing harness with py.test and Vagrant

.gitignore

@@ -28,3 +28,4 @@ rgw-standalone.yml
 take-over-existing-cluster.yml
 osd-configure.yml
 rolling_update.yml
+.tox

tests/README.rst

@@ -0,0 +1,50 @@
+Functional Testing
+==================
+The directory structure, files, and tests found in this directory all work
+together to provide:
+
+* a set of machines (or even a single one) so that ceph-ansible can run against
+* a "scenario" configuration file in Python, that defines what nodes are
+  configured to what roles and what 'components' they will test
+* tests (in functional/tests/) that will all run unless skipped explicitly when
+  testing a distinct feature dependant on the ansible run.
+
+
+Example run
+-----------
+The following is the easiest way to try this out locally. Both Vagrant and
+VirtualBox are required. Ensure that ``py.test`` and ``pytest-xdist`` are
+installed (with pip on a virtualenv) by using the ``requirements.txt`` file in
+the ``tests`` directory::
+
+    pip install -r requirements.txt
+
+Choose a directory in ``tests/functional`` that has 3 files:
+
+* ``Vagrantfile``
+* ``vagrant_variables.yml``
+* A Python ("scenario") file.
+
+For example in: ``tests/functional/ubuntu/16.04/mon/initial_members``::
+
+    tree .
+    .
+    ├── Vagrantfile -> ../../../../../../Vagrantfile
+    ├── scenario.py
+    └── vagrant_variables.yml
+
+    0 directories, 3 files
+
+It is *required* to be in that directory. It is what triggers all the
+preprocessing of complex arguments based on the cluster setup.
+
+Run vagrant first to setup the environment::
+
+   vagrant up --no-provision --provider=virtualbox
+
+Then run ceph-ansible against the hosts with the distinct role (in this case we
+are deploying a monitor using ``initial_members``).
+
+And finally run ``py.test``::
+
+    py.test -v

tests/conftest.py

@@ -0,0 +1,101 @@
+import os
+
+import pytest
+import imp
+
+
+def pytest_addoption(parser):
+    default = 'scenario.py'
+    parser.addoption(
+        "--scenario",
+        action="store",
+        default=default,
+        help="YAML file defining scenarios to test. Currently defaults to: %s" % default
+    )
+
+
+def load_scenario_config(filepath, **kw):
+    '''
+    Creates a configuration dictionary from a file.
+
+    :param filepath: The path to the file.
+    '''
+
+    abspath = os.path.abspath(os.path.expanduser(filepath))
+    conf_dict = {}
+    if not os.path.isfile(abspath):
+        raise RuntimeError('`%s` is not a file.' % abspath)
+
+    # First, make sure the code will actually compile (and has no SyntaxErrors)
+    with open(abspath, 'rb') as f:
+        compiled = compile(f.read(), abspath, 'exec')
+
+    # Next, attempt to actually import the file as a module.
+    # This provides more verbose import-related error reporting than exec()
+    absname, _ = os.path.splitext(abspath)
+    basepath, module_name = absname.rsplit(os.sep, 1)
+    imp.load_module(
+        module_name,
+        *imp.find_module(module_name, [basepath])
+    )
+
+    # If we were able to import as a module, actually exec the compiled code
+    exec(compiled, globals(), conf_dict)
+    conf_dict['__file__'] = abspath
+    return conf_dict
+
+
+def pytest_configure_node(node):
+    node_id = node.slaveinput['slaveid']
+    scenario_path = os.path.abspath(node.config.getoption('--scenario'))
+    scenario = load_scenario_config(scenario_path)
+    node.slaveinput['node_config'] = scenario['nodes'][node_id]
+    node.slaveinput['scenario_config'] = scenario
+
+
+@pytest.fixture(scope='session')
+def node_config(request):
+    return request.config.slaveinput['node_config']
+
+
+@pytest.fixture(scope="session")
+def scenario_config(request):
+    return request.config.slaveinput['scenario_config']
+
+
+def pytest_report_header(config):
+    """
+    Hook to add extra information about the execution environment and to be
+    able to debug what did the magical args got expanded to
+    """
+    lines = []
+    scenario_path = str(config.rootdir.join(config.getoption('--scenario')))
+    if not config.remote_execution:
+        lines.append('execution environment: local')
+    else:
+        lines.append('execution environment: remote')
+        lines.append('loaded scenario: %s' % scenario_path)
+    lines.append('expanded args: %s' % config.extended_args)
+    return lines
+
+
+def pytest_cmdline_preparse(args, config):
+    # Note: we can only do our magical args expansion if we aren't already in
+    # a remote node via xdist/execnet so return quickly if we can't do magic.
+    # TODO: allow setting an environment variable that helps to skip this kind
+    # of magical argument expansion
+    if os.getcwd().endswith('pyexecnetcache'):
+        return
+
+    scenario_path = os.path.abspath(config.getoption('--scenario'))
+
+    scenarios = load_scenario_config(scenario_path, args=args)
+    rsync_dir = os.path.dirname(str(config.rootdir.join('functional')))
+    test_path = str(config.rootdir.join('functional/tests'))
+    nodes = []
+    config.remote_execution = True
+    for node in scenarios.get('nodes', []):
+        nodes.append('--tx')
+        nodes.append('vagrant_ssh={node_name}//id={node_name}'.format(node_name=node))
+    args[:] = args + ['--max-slave-restart', '0', '--dist=each'] + nodes + ['--rsyncdir', rsync_dir, test_path]
+    config.extended_args = ' '.join(args)

tests/functional/tests/mon/test_initial_members.py

@@ -0,0 +1,32 @@
+import pytest
+
+
+
+uses_mon_initial_members = pytest.mark.skipif(
+    'mon_initial_members' not in pytest.config.slaveinput['node_config']['components'],
+    reason="only run in monitors configured with initial_members"
+)
+
+
+class TestMon(object):
+
+    def get_line_from_config(self, string, conf_path):
+        with open(conf_path) as ceph_conf:
+            ceph_conf_lines = ceph_conf.readlines()
+            for line in ceph_conf_lines:
+                if string in line:
+                    return line.strip().strip('\n')
+
+    @uses_mon_initial_members
+    def test_ceph_config_has_inital_members_line(self, scenario_config):
+        cluster_name = scenario_config.get('ceph', {}).get('cluster_name', 'ceph')
+        ceph_conf_path = '/etc/ceph/%s.conf' % cluster_name
+        initial_members_line = self.get_line_from_config('mon initial members', ceph_conf_path)
+        assert initial_members_line
+
+    @uses_mon_initial_members
+    def test_initial_members_line_has_correct_value(self, scenario_config):
+        cluster_name = scenario_config.get('ceph', {}).get('cluster_name', 'ceph')
+        ceph_conf_path = '/etc/ceph/%s.conf' % cluster_name
+        initial_members_line = self.get_line_from_config('mon initial members', ceph_conf_path)
+        assert initial_members_line == 'mon initial members = ceph-mon0'

tests/functional/tests/test_install.py

@@ -0,0 +1,10 @@
+import os
+
+
+class TestInstall(object):
+
+    def test_ceph_dir_exists(self):
+        assert os.path.isdir('/etc/ceph')
+
+    def test_ceph_conf_exists(self):
+        assert os.path.isfile('/etc/ceph/ceph.conf')

tests/functional/ubuntu/16.04/mon/initial_members/Vagrantfile

@@ -0,0 +1 @@
+../../../../../../Vagrantfile

tests/functional/ubuntu/16.04/mon/initial_members/hosts

@@ -0,0 +1,4 @@
+
+[mons]
+mon0
+

tests/functional/ubuntu/16.04/mon/initial_members/scenario.py

@@ -0,0 +1,15 @@
+# Basic information about ceph and its configuration
+ceph = {
+    'releases': ['infernalis', 'jewel'],
+    'cluster_name': 'ceph'
+}
+
+# remote nodes to test, with anything specific to them that might be useful for
+# tests to get. Each one of these can get requested as a py.test fixture to
+# validate information.
+nodes = {
+    'mon0': {
+        'username': 'vagrant',
+        'components': ['mon', 'mon_initial_members']
+    }
+}

tests/functional/ubuntu/16.04/mon/initial_members/vagrant_variables.yml

@@ -0,0 +1,64 @@
+---
+
+# DEPLOY CONTAINERIZED DAEMONS
+docker: false
+
+# DEFINE THE NUMBER OF VMS TO RUN
+mon_vms: 1
+osd_vms: 0
+mds_vms: 0
+rgw_vms: 0
+nfs_vms: 0
+rbd_mirror_vms: 0
+client_vms: 0
+iscsi_gw_vms: 0
+
+# Deploy RESTAPI on each of the Monitors
+restapi: true
+
+# INSTALL SOURCE OF CEPH
+# valid values are 'stable' and 'dev'
+ceph_install_source: stable
+
+# SUBNETS TO USE FOR THE VMS
+public_subnet: 192.168.42
+cluster_subnet: 192.168.43
+
+# MEMORY
+# set 1024 for CentOS
+memory: 512
+
+# Ethernet interface name
+# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial
+eth: 'eth1'
+
+# Disks
+# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]"
+# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]"
+disks: "[ '/dev/sdb', '/dev/sdc' ]"
+
+# VAGRANT BOX
+# Ubuntu: bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64
+# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet
+# libvirt CentOS: centos/7
+# parallels Ubuntu: parallels/ubuntu-14.04
+# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller'
+# For more boxes have a look at:
+#   - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q=
+#   - https://download.gluster.org/pub/gluster/purpleidea/vagrant/
+vagrant_box: geerlingguy/ubuntu1604
+#ssh_private_key_path: "~/.ssh/id_rsa"
+# The sync directory changes based on vagrant box
+# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant
+#vagrant_sync_dir: /home/vagrant/sync
+#vagrant_sync_dir: /
+# VAGRANT URL
+# This is a URL to download an image from an alternate location.  vagrant_box
+# above should be set to the filename of the image.
+# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
+# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box
+# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
+
+os_tuning_params:
+  - { name: kernel.pid_max, value: 4194303 }
+  - { name: fs.file-max, value: 26234859 }

tests/pytest.ini

@@ -0,0 +1,2 @@
+# this is just a placeholder so that we can define what the 'root' of the tests
+# dir really is. 

tests/requirements.txt

@@ -0,0 +1,3 @@
+# These are Python requirements needed to run the functional tests
+pytest
+pytest-xdist

tests/scenarios/example.py

@@ -0,0 +1,24 @@
+# This is the most basic tests that can be executed remotely. It will trigger
+# a series of checks for paths, permissions and flags. Whatever is not
+# dependant on particular component of ceph should go here (for example,
+# nothing related to just OSDs)
+
+# Basic information about ceph and its configuration
+ceph = {
+    'releases': ['jewel', 'infernalis'],
+    'cluster_name': 'ceph'
+}
+
+# remote nodes to test, with anything specific to them that might be useful for
+# tests to get. Each one of these can get requested as a py.test fixture to
+# validate information.
+nodes = {
+    'mon0': {
+        'username': 'vagrant',
+        'components': ['mon']
+    },
+    'osd0': {
+        'username': 'vagrant',
+        'components': ['osd']
+    },
+}

tests/scripts/generate_ssh_config.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Generate a custom ssh config from Vagrant so that it can then be used by
+# ansible.cfg 
+
+path=$1
+
+if [ $# -eq 0 ]
+  then
+    echo "A path to the scenario is required as an argument and it wasn't provided"
+    exit 1
+fi
+
+cd "$path"
+vagrant ssh-config > vagrant_ssh_config

tox.ini

@@ -0,0 +1,26 @@
+[tox]
+envlist = {ansible2.1,ansible2.2}-{initial-members}
+skipsdist = True
+
+[testenv]
+whitelist_externals =
+    vagrant
+    bash
+passenv=*
+setenv=
+  ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config
+  ANSIBLE_ACTION_PLUGINS = {toxinidir}/plugins/actions
+deps=
+  ansible2.1: ansible==2.1
+  ansible2.2: ansible==2.2
+  -r{toxinidir}/tests/requirements.txt
+changedir=
+  initial-members: {toxinidir}/tests/functional/ubuntu/16.04/mon/initial_members
+commands=
+  vagrant up --no-provision --provider=virtualbox
+  bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
+
+  initial-members: ansible-playbook -i {toxinidir}/tests/functional/ubuntu/16.04/mon/initial_members/hosts --extra-vars "ceph_stable=True public_network=192.168.42.0/24 cluster_network=192.168.43.0/24 journal_size=100 monitor_interface=eth1" {toxinidir}/site.yml.sample
+
+  py.test -v
+  vagrant destroy --force