Merge pull request #1424 from ceph/stable-docs

backport docs for stable-2.1

docs/.gitignore

@@ -0,0 +1 @@
+build

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = ceph-ansible
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+#
+# ceph-ansible documentation build configuration file, created by
+# sphinx-quickstart on Wed Apr  5 11:55:38 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'glossary'
+
+# General information about the project.
+project = u'ceph-ansible'
+copyright = u'2017, Ceph team and individual contributors'
+author = u'Ceph team and individual contributors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u'2.2.0'
+# The full version, including alpha/beta/rc tags.
+release = u'2.2.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ceph-ansibledoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'ceph-ansible.tex', u'ceph-ansible Documentation',
+     u'Ceph team and individual contributors', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'ceph-ansible', u'ceph-ansible Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'ceph-ansible', u'ceph-ansible Documentation',
+     author, 'ceph-ansible', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+

docs/source/glossary.rst

@@ -0,0 +1,11 @@
+
+Glossary
+========
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Contents:
+
+   testing/glossary
+   index
+

docs/source/index.rst

@@ -0,0 +1,31 @@
+.. ceph-ansible documentation master file, created by
+   sphinx-quickstart on Wed Apr  5 11:55:38 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+ceph-ansible
+============
+Ansible playbooks for Ceph, the distributed filesystem.
+
+
+Testing
+=======
+
+* :doc:`Testing with ceph-ansible <testing/index>`
+* :doc:`Glossary <testing/glossary>`
+
+
+OSDs
+====
+
+MONs
+====
+
+RGW
+===
+
+Configuration
+=============
+
+Docker
+======

docs/source/testing/development.rst

@@ -0,0 +1,5 @@
+
+.. development:
+
+ceph-ansible testing for development
+====================================

docs/source/testing/glossary.rst

@@ -0,0 +1,15 @@
+
+Glossary
+========
+
+.. toctree::
+   :maxdepth: 1
+
+   index
+   running.rst
+   development.rst
+   scenarios.rst
+   modifying.rst
+   layout.rst
+   tests.rst
+   tox.rst

docs/source/testing/index.rst

@@ -0,0 +1,33 @@
+
+.. _testing:
+
+Testing
+=======
+ceph-ansible has the ability to test different scenarios (collocated journals
+or dmcrypt OSDs for example) in an isolated, repeatable, and easy way.
+
+These tests can run locally with VirtualBox or via libvirt if available, which
+removes the need to solely rely on a CI system like Jenkins to verify
+a behavior.
+
+* **Getting started:**
+  :doc:`Running a Test Scenario <running>` |
+  :ref:`dependencies`
+
+* **Configuration and structure:**
+  :ref:`layout` |
+  :ref:`test_files` |
+  :ref:`scenario_files` |
+  :ref:`scenario_wiring`
+
+* **Adding or modifying tests:**
+  :ref:`test_conventions` |
+  :ref:`testinfra` |
+
+* **Adding or modifying a scenario:**
+  :ref:`scenario_conventions` |
+  :ref:`scenario_environment_configuration` |
+  :ref:`scenario_ansible_configuration` |
+
+* **Custom/development repositories and packages:**
+  :ref:`tox_environment_variables` |

docs/source/testing/layout.rst

@@ -0,0 +1,40 @@
+.. _layout:
+
+Layout and conventions
+----------------------
+Test files and directories follow a few conventions, which makes it easy to
+create (or expect) certain interactions between tests and scenarios.
+
+All tests are in the ``tests`` directory. Scenarios are defined in
+``tests/functional/`` and use the following convention for directory
+structure::
+
+    tests/functional/<distro>/<distro version>/<scenario name>/
+
+For example: ``tests/functional/centos/7/journal-collocation``
+
+Within a test scenario there are a few files that define what that specific
+scenario needs for the tests, like how many OSD nodes or MON nodes. Tls
+
+At the very least, a scenario will need these files:
+
+* ``Vagrantfile``: must be symlinked from the root directory of the project
+* ``hosts``: An Ansible hosts file that defines the machines part of the
+  cluster
+* ``group_vars/all``: if any modifications are needed for deployment, this
+  would override them. Additionally, further customizations can be done. For
+  example, for OSDs that would mean adding ``group_vars/osds``
+* ``vagrant_variables.yml``: Defines the actual environment for the test, where
+  machines, networks, disks, linux distro/version, can be defined.
+
+
+.. _test_conventions:
+
+Conventions
+-----------
+
+
+.. _testinfra:
+
+``testinfra``
+-------------

docs/source/testing/modifying.rst

@@ -0,0 +1,5 @@
+
+.. _modifying:
+
+Modifying (or adding) tests
+===========================

docs/source/testing/running.rst

@@ -0,0 +1,147 @@
+
+.. _running_tests:
+
+Running Tests
+=============
+Although tests run continuously in CI, a lot of effort was put into making it
+easy to run in any environment, as long as a couple of requirements are met.
+
+
+.. _dependencies:
+
+Dependencies
+------------
+There are some Python dependencies, which are listed in a ``requirements.txt``
+file within the ``tests/`` directory. These are meant to be installed using
+Python install tools (pip in this case)::
+
+    pip install -r tests/requirements.txt
+
+For virtualization, either libvirt or VirtualBox is needed (there is native
+support from the harness for both). This makes the test harness even more
+flexible as most platforms will be covered by either VirtualBox or libvirt.
+
+
+.. _running_a_scenario:
+
+Running a scenario
+------------------
+Tests are driven by ``tox``, a command line tool to run a matrix of tests defined in
+a configuration file (``tox.ini`` in this case at the root of the project).
+
+For a thorough description of a scenario see :ref:`test_scenarios`.
+
+To run a single scenario, make sure it is available (should be defined from
+``tox.ini``) by listing them::
+
+    tox -l
+
+In this example, we will use the ``kraken-ansible2.2-xenial_cluster`` one.  The
+harness defaults to ``VirtualBox`` as the backend, so if you have that
+installed in your system then this command should just work::
+
+    tox -e kraken-ansible2.2-xenial_cluster
+
+And for libvirt it would be::
+
+    tox -e kraken-ansible2.2-xenial_cluster -- --provider=libvirt
+
+
+.. warning:: Depending on the type of scenario and resources available, running
+    these tests locally in a personal computer can be very resource intensive.
+
+.. note:: Most test runs take between 20 and 40 minutes depending on system
+    resources
+
+The command should bring up the machines needed for the test, provision them
+with ceph-ansible, run the tests, and tear the whole environment down at the
+end.
+
+
+The output would look something similar to this trimmed version::
+
+    kraken-ansible2.2-xenial_cluster create: /Users/alfredo/python/upstream/ceph-ansible/.tox/kraken-ansible2.2-xenial_cluster
+    kraken-ansible2.2-xenial_cluster installdeps: ansible==2.2.2, -r/Users/alfredo/python/upstream/ceph-ansible/tests/requirements.txt
+    kraken-ansible2.2-xenial_cluster runtests: commands[0] | vagrant up --no-provision --provider=virtualbox
+    Bringing machine 'client0' up with 'virtualbox' provider...
+    Bringing machine 'rgw0' up with 'virtualbox' provider...
+    Bringing machine 'mds0' up with 'virtualbox' provider...
+    Bringing machine 'mon0' up with 'virtualbox' provider...
+    Bringing machine 'mon1' up with 'virtualbox' provider...
+    Bringing machine 'mon2' up with 'virtualbox' provider...
+    Bringing machine 'osd0' up with 'virtualbox' provider...
+    ...
+
+
+After all the nodes are up, ceph-ansible will provision them, and run the
+playbook(s)::
+
+    ...
+    PLAY RECAP *********************************************************************
+    client0                    : ok=4    changed=0    unreachable=0    failed=0
+    mds0                       : ok=4    changed=0    unreachable=0    failed=0
+    mon0                       : ok=4    changed=0    unreachable=0    failed=0
+    mon1                       : ok=4    changed=0    unreachable=0    failed=0
+    mon2                       : ok=4    changed=0    unreachable=0    failed=0
+    osd0                       : ok=4    changed=0    unreachable=0    failed=0
+    rgw0                       : ok=4    changed=0    unreachable=0    failed=0
+    ...
+
+
+Once the whole environment is all running the tests will be sent out to the
+hosts, with output similar to this::
+
+    kraken-ansible2.2-xenial_cluster runtests: commands[4] | testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory=/Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster/hosts /Users/alfredo/python/upstream/ceph-ansible/tests/functional/tests
+    ============================ test session starts ===========================
+    platform darwin -- Python 2.7.8, pytest-3.0.7, py-1.4.33, pluggy-0.4.0 -- /Users/alfredo/python/upstream/ceph-ansible/.tox/kraken-ansible2.2-xenial_cluster/bin/python
+    cachedir: ../../../../.cache
+    rootdir: /Users/alfredo/python/upstream/ceph-ansible/tests, inifile: pytest.ini
+    plugins: testinfra-1.5.4, xdist-1.15.0
+    [gw0] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+    [gw1] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+    [gw2] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+    [gw3] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+    [gw0] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+    [gw1] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+    [gw2] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+    [gw3] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+    gw0 [154] / gw1 [154] / gw2 [154] / gw3 [154]
+    scheduling tests via LoadScheduling
+
+    ../../../tests/test_install.py::TestInstall::test_ceph_dir_exists[ansible:/mon0]
+    ../../../tests/test_install.py::TestInstall::test_ceph_dir_is_a_directory[ansible:/mon0]
+    ../../../tests/test_install.py::TestInstall::test_ceph_conf_is_a_file[ansible:/mon0]
+    ../../../tests/test_install.py::TestInstall::test_ceph_dir_is_a_directory[ansible:/mon1]
+    [gw2] PASSED ../../../tests/test_install.py::TestCephConf::test_ceph_config_has_mon_host_line[ansible:/mon0]
+    ../../../tests/test_install.py::TestInstall::test_ceph_conf_exists[ansible:/mon1]
+    [gw3] PASSED ../../../tests/test_install.py::TestCephConf::test_mon_host_line_has_correct_value[ansible:/mon0]
+    ../../../tests/test_install.py::TestInstall::test_ceph_conf_is_a_file[ansible:/mon1]
+    [gw1] PASSED ../../../tests/test_install.py::TestInstall::test_ceph_command_exists[ansible:/mon1]
+    ../../../tests/test_install.py::TestCephConf::test_mon_host_line_has_correct_value[ansible:/mon1]
+    ...
+
+Finally the whole environment gets torn down::
+
+    kraken-ansible2.2-xenial_cluster runtests: commands[5] | vagrant destroy --force
+    ==> osd0: Forcing shutdown of VM...
+    ==> osd0: Destroying VM and associated drives...
+    ==> mon2: Forcing shutdown of VM...
+    ==> mon2: Destroying VM and associated drives...
+    ==> mon1: Forcing shutdown of VM...
+    ==> mon1: Destroying VM and associated drives...
+    ==> mon0: Forcing shutdown of VM...
+    ==> mon0: Destroying VM and associated drives...
+    ==> mds0: Forcing shutdown of VM...
+    ==> mds0: Destroying VM and associated drives...
+    ==> rgw0: Forcing shutdown of VM...
+    ==> rgw0: Destroying VM and associated drives...
+    ==> client0: Forcing shutdown of VM...
+    ==> client0: Destroying VM and associated drives...
+
+
+And a brief summary of the scenario(s) that ran is displayed::
+
+    ________________________________________________ summary _________________________________________________
+      kraken-ansible2.2-xenial_cluster: commands succeeded
+      congratulations :)
+

docs/source/testing/scenarios.rst

@@ -0,0 +1,30 @@
+
+.. _test_scenarios:
+
+Test Scenarios
+==============
+
+.. _scenario_files:
+
+Scenario Files
+--------------
+
+.. _scenario_wiring:
+
+Scenario Wiring
+---------------
+
+.. _scenario_conventions:
+
+Conventions
+-----------
+
+.. _scenario_environment_configuration:
+
+Environment configuration
+-------------------------
+
+.. _scenario_ansible_configuration:
+
+Ansible configuration
+---------------------

docs/source/testing/tests.rst

@@ -0,0 +1,10 @@
+.. _tests:
+
+Tests
+=====
+
+.. _test_files:
+
+Test Files
+----------
+

docs/source/testing/tox.rst

@@ -0,0 +1,22 @@
+.. _tox:
+
+``tox``
+=======
+
+
+.. _tox_environment_variables:
+
+Environment variables
+---------------------
+
+
+.. _tox_sections:
+
+Sections
+--------
+
+
+.. _tox_environments:
+
+Modifying or Adding environments
+--------------------------------

docs/tox.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = docs
+skipsdist = True
+
+[testenv:docs]
+basepython=python
+changedir=source
+deps=sphinx
+commands=
+    sphinx-build -W -b html -d {envtmpdir}/doctrees .  {envtmpdir}/html