Merge pull request #648 from ceph/docker

Docker

README.md

@@ -185,6 +185,57 @@ $ vagrant provision
 If you want to use "backports", you can set "true" to `ceph_use_distro_backports`.
 Attention, ceph-common doesn't manage backports repository, you must add it yourself.
 
+### For Atomic systems
+
+If you want to run containerized deployment on Atomic systems (RHEL/CentOS Atomic), please copy
+[vagrant.yml.atomic](vagrant_variables.yml.atomic) to vagrant_variables.yml, and copy [group_vars/all.docker](group_vars/all.docker) to `group_vars/all`.
+
+Since `centos/atomic-host` doesn't have spare storage controller to attach more disks, it is likely the first time `vagrant up --provider=virtualbox` runs, it will fail to attach to a storage controller. In such case, run the following command:
+
+```console
+VBoxManage storagectl `VBoxManage list vms |grep ceph-ansible_osd0|awk '{print $1}'|tr \" ' '` --name "SATA" --add sata
+```
+
+then run `vagrant up --provider=virtualbox` again.
+
+## Setup for Vagrant using OpenStack provider
+
+Install the Vagrant plugin for the openstack provider: `vagrant plugin install vagrant-openstack-provider`.
+
+```bash
+$ cp site.yml.sample site.yml
+$ cp group_vars/all.docker.sample group_vars/all
+$ cp vagrant_variables.yml.openstack vagrant_variables.yml
+```
+* Edit `vagrant_variables.yml`:
+  Set `mon_vms` and `osd_vms` to the numbers you want.
+  If you are using an Atomic image, un-comment out the `skip_tags` line.
+  Un-comment the `os_` lines.
+    Set `os_ssh_username` to 'centos' for Centos and 'cloud-user' for
+        RHEL images.
+    Set `os_ssh_private_key_path` to '~/.ssh/id_rsa'
+    Set `os_openstack_auth_url` to the auth url of your Open Stack cloud
+    Set `os_username` and `os_password` to what you provided for Open Stack 
+        registration or leave them as ENV vars if you have set the
+        corresponding env vars for your user.
+    Set `os_tenant_name` to your Open Stack cloud project name.
+    Set `os_region` to your Open Stack cloud region name.
+    Set `os_flavor` to 'm3.medium'.  This size has ephemeral storage that will
+        be used by the OSD for the /dev/vdb disk
+    Set the `os_image` to an image found in the Images list in the Open Stack
+        cloud Dashboard (i.e. 'centos-atomic-host').
+    Set the `os_keypair_name` to the keypair name you used when you did the 
+        Open Stack registration.
+```
+$ vagrant up --provider=openstack
+```
+Once the playbook is finished, you should be able to do `vagrant ssh mon0` or 
+`vagrant ssh osd0` to get to the VMs.  
+`sudo docker ps` should show the running containers
+When you are done, use `vagrant destroy` to get rid of the VMs.  You should 
+also remove the associated entries in .ssh/known_hosts so that if the IP 
+addresses get reused by future Open Stack Cloud instances there will not be
+old known_hosts entries.
 
 # Want to contribute?
 

Vagrantfile

@@ -19,9 +19,20 @@ STORAGECTL = settings['vagrant_storagectl']
 ETH        = settings['eth']
 DOCKER     = settings['docker']
 
+if BOX == 'openstack'
+  require 'vagrant-openstack-provider'
+  OSVM = true
+  USER = settings['os_ssh_username']
+else
+  OSVM = false
+end
+
 ansible_provision = proc do |ansible|
   if DOCKER then
     ansible.playbook = 'site-docker.yml'
+    if settings['skip_tags']
+      ansible.skip_tags = settings['skip_tags']
+    end
   else
     ansible.playbook = 'site.yml'
   end
@@ -83,11 +94,32 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     override.vm.synced_folder '.', '/home/vagrant/sync', disabled: true
   end
 
+  if BOX == 'openstack'
+    # OpenStack VMs
+    config.vm.provider :openstack do |os|
+      config.vm.synced_folder ".", "/home/#{USER}/vagrant", disabled: true
+      config.ssh.username = USER
+      config.ssh.private_key_path = settings['os_ssh_private_key_path']
+      config.ssh.pty = true
+      os.openstack_auth_url = settings['os_openstack_auth_url']
+      os.username = settings['os_username']
+      os.password = settings['os_password']
+      os.tenant_name = settings['os_tenant_name']
+      os.region = settings['os_region']
+      os.flavor = settings['os_flavor']
+      os.image = settings['os_image']
+      os.keypair_name = settings['os_keypair_name']
+      os.security_groups = ['default']
+      config.vm.provision "shell", inline: "true", upload_path: "/home/#{USER}/vagrant-shell"
+    end
+  end
+
   (0..CLIENTS - 1).each do |i|
     config.vm.define "client#{i}" do |client|
       client.vm.hostname = "ceph-client#{i}"
-      client.vm.network :private_network, ip: "#{SUBNET}.4#{i}"
+      if !OSVM
-
+        client.vm.network :private_network, ip: "#{SUBNET}.4#{i}"
+      end
       # Virtualbox
       client.vm.provider :virtualbox do |vb|
         vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
@@ -114,7 +146,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   (0..NRGWS - 1).each do |i|
     config.vm.define "rgw#{i}" do |rgw|
       rgw.vm.hostname = "ceph-rgw#{i}"
-      rgw.vm.network :private_network, ip: "#{SUBNET}.5#{i}"
+      if !OSVM
+        rgw.vm.network :private_network, ip: "#{SUBNET}.5#{i}"
+      end
 
       # Virtualbox
       rgw.vm.provider :virtualbox do |vb|
@@ -142,8 +176,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   (0..NMDSS - 1).each do |i|
     config.vm.define "mds#{i}" do |mds|
       mds.vm.hostname = "ceph-mds#{i}"
-      mds.vm.network :private_network, ip: "#{SUBNET}.7#{i}"
+      if !OSVM
-
+        mds.vm.network :private_network, ip: "#{SUBNET}.7#{i}"
+      end
       # Virtualbox
       mds.vm.provider :virtualbox do |vb|
         vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
@@ -158,7 +193,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       mds.vm.provider :libvirt do |lv|
         lv.memory = MEMORY
       end
-      
       # Parallels
       mds.vm.provider "parallels" do |prl|
         prl.name = "ceph-mds#{i}"
@@ -170,8 +204,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   (0..NMONS - 1).each do |i|
     config.vm.define "mon#{i}" do |mon|
       mon.vm.hostname = "ceph-mon#{i}"
-      mon.vm.network :private_network, ip: "#{SUBNET}.1#{i}"
+      if !OSVM
-
+        mon.vm.network :private_network, ip: "#{SUBNET}.1#{i}"
+      end
       # Virtualbox
       mon.vm.provider :virtualbox do |vb|
         vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
@@ -198,9 +233,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   (0..NOSDS - 1).each do |i|
     config.vm.define "osd#{i}" do |osd|
       osd.vm.hostname = "ceph-osd#{i}"
-      osd.vm.network :private_network, ip: "#{SUBNET}.10#{i}"
+      if !OSVM
-      osd.vm.network :private_network, ip: "#{SUBNET}.20#{i}"
+        osd.vm.network :private_network, ip: "#{SUBNET}.10#{i}"
-
+        osd.vm.network :private_network, ip: "#{SUBNET}.20#{i}"
+      end
       # Virtualbox
       osd.vm.provider :virtualbox do |vb|
         (0..1).each do |d|
@@ -244,10 +280,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         prl.name = "ceph-osd#{i}"
         prl.memory = "#{MEMORY}"
         (0..1).each do |d|
-          prl.customize ["set", :id, 
+          prl.customize ["set", :id,
-                         "--device-add", 
+                         "--device-add",
-                         "hdd", 
+                         "hdd",
-                         "--iface", 
+                         "--iface",
                          "sata"]
         end
       end

group_vars/all.docker.sample

@@ -0,0 +1,57 @@
+---
+dummy:
+
+#######
+# MON #
+#######
+#mon_containerized_deployment: true
+#mon_containerized_deployment_with_kv: false
+#mon_containerized_default_ceph_conf_with_kv: true
+#kv_type: etcd
+#kv_endpoint: 127.0.0.1
+#kv_port: 4001
+#mon_docker_privileged: true
+#ceph_mon_docker_username: ceph
+#ceph_mon_docker_imagename: daemon
+#ceph_mon_docker_interface: "{{ monitor_interface }}"
+#ceph_mon_docker_subnet: "{{ public_network }}" # subnet of the ceph_mon_docker_interface
+#ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables
+
+#######
+# OSD #
+#######
+#osd_containerized_deployment: true
+#ceph_osd_docker_username: ceph
+#ceph_osd_docker_imagename: daemon
+#ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE" # comma separated variables
+#ceph_osd_docker_prepare_env: "CEPH_DAEMON=OSD_CEPH_DISK_PREPARE,OSD_FORCE_ZAP=1" # comma separated variables
+#ceph_osd_docker_devices:
+# - /dev/sdb
+# - /dev/sdc
+
+#######
+# MDS #
+#######
+#mds_containerized_deployment: false
+#ceph_mds_docker_username: ceph
+#ceph_mds_docker_imagename: daemon
+#ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables
+
+#######
+# RGW #
+#######
+#rgw_containerized_deployment: true
+#ceph_rgw_docker_username: ceph
+#ceph_rgw_docker_imagename: daemon
+#ceph_rgw_civetweb_port: 80
+#ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables
+
+###########
+# RESTAPI #
+###########
+#restapi_containerized_deployment: false
+#ceph_restapi_docker_interface: eth0
+#ceph_restapi_port: 5000
+#ceph_restapi_docker_username: ceph
+#ceph_restapi_docker_imagename: daemon
+#ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables

group_vars/common-coreoss.sample

@@ -0,0 +1,17 @@
+---
+# Variables here are applicable to all host groups NOT roles
+
+# This sample file generated by generate_group_vars_sample.sh
+
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+#coreos_pypy_version: 4.0.1
+#coreos_pypy_arch: linux64
+#coreos_pypy_url: https://bitbucket.org/pypy/pypy/downloads/pypy-{{coreos_pypy_version}}-{{coreos_pypy_arch}}.tar.bz2
+#pypy_directory: /opt/pypy
+#pypy_binary_directory: /opt/bin
+#pip_url: https://bootstrap.pypa.io/get-pip.py
+#local_temp_directory: /tmp
+

group_vars/mdss.sample

@@ -32,4 +32,5 @@ dummy:
 #ceph_mds_docker_username: ceph
 #ceph_mds_docker_imagename: daemon
 #ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables
+#ceph_docker_on_openstack: false
 

group_vars/mons.sample

@@ -71,9 +71,13 @@ dummy:
 ##########
 
 #mon_containerized_deployment: false
+#mon_containerized_deployment_with_kv: false
+#mon_containerized_default_ceph_conf_with_kv: false
 #ceph_mon_docker_interface: eth0
 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface
 #ceph_mon_docker_username: ceph
 #ceph_mon_docker_imagename: daemon
 #ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables
+#ceph_docker_on_openstack: false
+#mon_docker_privileged: true
 

group_vars/osds.sample

@@ -128,4 +128,5 @@ dummy:
 #ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables
 #ceph_osd_docker_devices:
 # - /dev/sdb
+#ceph_docker_on_openstack: false
 

group_vars/restapis.sample

@@ -24,4 +24,5 @@ dummy:
 #ceph_restapi_docker_username: ceph
 #ceph_restapi_docker_imagename: daemon
 #ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables
+#ceph_docker_on_openstack: false
 

group_vars/rgws.sample

@@ -39,4 +39,5 @@ dummy:
 #ceph_rgw_docker_username: ceph
 #ceph_rgw_docker_imagename: daemon
 #ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables
+#ceph_docker_on_openstack: false
 

roles/ceph-common-coreos/README.md

@@ -0,0 +1,44 @@
+# Ansible role: Ceph Storage Agent
+
+This role installs python and pip on CoreOS.
+
+# Requirements
+
+This role has to be run without gathering facts and with sudo attribute.
+
+# Role variables
+
+Have a look at: `defaults/main.yml`.
+
+## Mandatory variables
+
+None.
+
+# Dependencies
+
+New CoreOS releases support pypy in version 2.4 and above. Unfortunetly CoreOS stable channel
+has to be used with pypy in version 2.4 and below due to some dependency issues.
+
+# Example Playbook
+
+```
+- hosts: servers
+  remote_user: core
+  become: True
+  gather_facts: false
+  roles:
+     - { role: ceph-common-coreos }
+```
+
+# Contribution
+
+**THIS REPOSITORY DOES NOT ACCEPT PULL REQUESTS**
+**PULL REQUESTS MUST GO THROUGH [CEPH-ANSIBLE](https://github.com/ceph/ceph-ansible)**
+
+# License
+
+Apache
+
+# Author Information
+
+This role was created by Piotr Prokop.

roles/ceph-common-coreos/defaults/main.yml

@@ -0,0 +1,7 @@
+coreos_pypy_version: 4.0.1
+coreos_pypy_arch: linux64
+coreos_pypy_url: https://bitbucket.org/pypy/pypy/downloads/pypy-{{coreos_pypy_version}}-{{coreos_pypy_arch}}.tar.bz2
+pypy_directory: /opt/pypy
+pypy_binary_directory: /opt/bin
+pip_url: https://bootstrap.pypa.io/get-pip.py
+local_temp_directory: /tmp

roles/ceph-common-coreos/tasks/install_pip.yml

@@ -0,0 +1,30 @@
+---
+- name: download get_pip.py
+  raw: cd $HOME && wget {{pip_url}}
+
+- name: run get-pip.py
+  raw: "{{pypy_binary_directory}}/python $HOME/get-pip.py"
+
+- name: create local temp directory
+  local_action: raw mkdir -p {{local_temp_directory}}
+  become: no
+
+- name: prepare install_pip.sh
+  local_action: template src=install_pip.sh.j2 dest={{local_temp_directory}}/install_pip.sh
+  become: no
+
+- name: run pip.sh
+  script: "{{local_temp_directory}}/install_pip.sh"
+
+- name: add execute permission
+  raw: chmod a+x {{pypy_directory}}/pip
+
+- name: move python to binary directory
+  raw: mv {{pypy_directory}}/pip {{pypy_binary_directory}}/pip
+
+- name: create .pip
+  raw: touch $HOME/.pip
+
+- name: remove pip.sh
+  local_action: file path="{{local_temp_directory}}/pip.sh" state=absent
+  become: no

roles/ceph-common-coreos/tasks/install_pypy.yml

@@ -0,0 +1,33 @@
+---
+- name: download python
+  raw: cd $HOME &&  wget -O - {{coreos_pypy_url}} |tar -xjf -
+
+- name: move pypy to pypy_install_directory
+  raw:  mv $HOME/pypy-{{coreos_pypy_version}}-{{coreos_pypy_arch}} {{pypy_directory}}
+
+- name: create local temp directory
+  local_action: raw mkdir -p {{local_temp_directory}}
+  become: no
+
+- name: prepare python executable
+  local_action: template src=install_python.sh.j2 dest={{local_temp_directory}}/install_python.sh
+  become: no
+
+- name: fix library
+  raw: ln -s /lib64/libncurses.so.5.9 {{pypy_directory}}/lib_pypy/libtinfo.so.5
+
+- name: run install_python.sh
+  script: "{{local_temp_directory}}/install_python.sh"
+
+- name: add execute permission
+  raw: chmod a+x {{pypy_directory}}/python
+
+- name: move python to binary directory
+  raw: mv {{pypy_directory}}/python {{pypy_binary_directory}}/python
+
+- name: create .python
+  raw: touch $HOME/.python
+
+- name: remove install_python.sh
+  local_action: file path="{{local_temp_directory}}/install_python.sh" state=absent
+  become: no

roles/ceph-common-coreos/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: check if there is python
+  raw: stat $HOME/.python
+  register: need_python
+  ignore_errors: true
+
+- include: install_pypy.yml
+  when: need_python | failed
+
+- name: check if there is pip
+  raw: stat $HOME/.pip
+  register: need_pip
+  ignore_errors: true
+
+- include: install_pip.yml
+  when: need_pip | failed and need_python | failed

roles/ceph-common-coreos/templates/install_pip.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+
+cat > {{pypy_directory}}/pip << EOF
+#!/bin/bash
+LD_LIBRARY_PATH={{pypy_directory}}/lib_pypy:$LD_LIBRARY_PATH exec {{pypy_directory}}/bin/pip "\$@"\

roles/ceph-common-coreos/templates/install_python.sh.j2

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cat > {{pypy_directory}}/python << EOF
+#!/bin/bash
+LD_LIBRARY_PATH={{pypy_directory}}/lib_pypy:$LD_LIBRARY_PATH exec {{pypy_directory}}/bin/pypy "\$@"\

roles/ceph-common/templates/ceph.conf.j2

@@ -15,7 +15,7 @@ auth service required = none
 auth client required = none
 auth supported = none
 {% endif %}
-fsid = {{ fsid }}
+{% if not mon_containerized_deployment_with_kv %}fsid = {{ fsid }}{% endif %}
 max open files = {{ max_open_files }}
 osd pool default pg num = {{ pool_default_pg_num }}
 osd pool default pgp num = {{ pool_default_pgp_num }}
@@ -88,6 +88,7 @@ debug mon = {{ debug_mon_level }}
 debug paxos = {{ debug_mon_level }}
 debug auth = {{ debug_mon_level }}
 {% endif %}
+{% if not mon_containerized_deployment_with_kv %}
 {% for host in groups[mon_group_name] %}
 {% if hostvars[host]['ansible_fqdn'] is defined and mon_use_fqdn %}
 [mon.{{ hostvars[host]['ansible_fqdn'] }}]
@@ -102,6 +103,7 @@ host = {{ hostvars[host]['ansible_hostname'] }}
 {% include 'mon_addr_address.j2' %}
 {% endif %}
 {% endfor %}
+{% endif %}
 
 [osd]
 osd mkfs type = {{ osd_mkfs_type }}

roles/ceph-mds/defaults/main.yml

@@ -24,3 +24,4 @@ mds_containerized_deployment: false
 ceph_mds_docker_username: ceph
 ceph_mds_docker_imagename: daemon
 ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables
+ceph_docker_on_openstack: false

roles/ceph-mds/tasks/docker/pre_requisite.yml

@@ -50,6 +50,13 @@
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"
 
+# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
+# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
+- name: install six
+  pip:
+    name: six
+    version: 1.9.0
+
 # NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
 - name: install docker-py
   pip:

roles/ceph-mon/defaults/main.yml

@@ -63,8 +63,12 @@ openstack_keys:
 ##########
 
 mon_containerized_deployment: false
+mon_containerized_deployment_with_kv: false
+mon_containerized_default_ceph_conf_with_kv: false
 ceph_mon_docker_interface: eth0
 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface
 ceph_mon_docker_username: ceph
 ceph_mon_docker_imagename: daemon
 ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables
+ceph_docker_on_openstack: false
+mon_docker_privileged: true

roles/ceph-mon/tasks/ceph_keys.yml

@@ -5,7 +5,7 @@
   wait_for:
     path: /etc/ceph/ceph.client.admin.keyring
 
-- name: create ceph rest api keyring
+- name: create ceph rest api keyring when mon is not containerized
   command: ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring
   args:
     creates: /etc/ceph/ceph.client.restapi.keyring

roles/ceph-mon/tasks/docker/create_configs.yml

@@ -0,0 +1,18 @@
+---
+- name: create ceph conf directory
+  file:
+      path: /etc/ceph
+      state: directory
+      owner: root
+      group: root
+      mode: 0644
+
+- name: generate ceph configuration file
+  config_template:
+    src: "{{ playbook_dir }}/roles/ceph-common/templates/ceph.conf.j2"
+    dest: /etc/ceph/ceph.conf
+    owner: "root"
+    group: "root"
+    mode: "0644"
+    config_overrides: "{{ ceph_conf_overrides }}"
+    config_type: ini

roles/ceph-mon/tasks/docker/fetch_configs.yml

@@ -2,8 +2,8 @@
 - name: set config and keys paths
   set_fact:
     ceph_config_keys:
-      - /etc/ceph/ceph.client.admin.keyring
       - /etc/ceph/ceph.conf
+      - /etc/ceph/ceph.client.admin.keyring
       - /etc/ceph/monmap
       - /etc/ceph/ceph.mon.keyring
       - /var/lib/ceph/bootstrap-osd/ceph.keyring
@@ -20,7 +20,7 @@
 
 - name: try to fetch ceph config and keys
   copy:
-    src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
     dest: "{{ item.0 }}"
     owner: root
     group: root

roles/ceph-mon/tasks/docker/main.yml

@@ -5,13 +5,42 @@
   changed_when: false
   failed_when: false
 
+- name: check if it is Atomic host
+  stat: path=/run/ostree-booted
+  register: stat_ostree
+
+- name: set fact for using Atomic host
+  set_fact:
+      is_atomic='{{ stat_ostree.stat.exists }}'
+
 - include: checks.yml
-  when: ceph_health.rc != 0
+  when: ceph_health.rc != 0 and not mon_containerized_deployment_with_kv
 
 - include: pre_requisite.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
+# let the first mon create configs and keyrings
+- include: create_configs.yml
+  when: inventory_hostname == groups.mons[0] and mon_containerized_default_ceph_conf_with_kv
+
 - include: fetch_configs.yml
+  when: not mon_containerized_deployment_with_kv
+
 - include: start_docker_monitor.yml
+
 - include: copy_configs.yml
+  when: not mon_containerized_deployment_with_kv
+
+- name: create ceph rest api keyring when mon is containerized
+  command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring
+  args:
+     creates: /etc/ceph/ceph.client.restapi.keyring
+  changed_when: false
+  when:
+    cephx and
+    mon_containerized_deployment and
+    groups[restapi_group_name] is defined and
+    inventory_hostname == groups.mons|last and
+    not mon_containerized_deployment_with_kv

roles/ceph-mon/tasks/docker/pre_requisite.yml

@@ -19,6 +19,8 @@
     - docker
     - docker.io
   when: ansible_distribution == 'Ubuntu'
+  tags:
+    with_pkg
 
 - name: install pip and docker on debian
   apt:
@@ -29,6 +31,8 @@
     - python-pip
     - docker-engine
   when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   yum:
@@ -40,6 +44,8 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   dnf:
@@ -51,9 +57,44 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"
+  tags:
+    with_pkg
+
+- name: install epel-release on redhat
+  yum:
+    name: epel-release
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
+# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
+- name: install six
+  pip:
+    name: six
+    version: 1.9.0
+  tags:
+    with_pkg
+
+- name: pause after docker install before starting (on openstack vms)
+  pause: seconds=5
+  when: ceph_docker_on_openstack
+  tags:
+    with_pkg
+
+- name: start docker service
+  service:
+    name: docker
+    state: started
+    enabled: yes
+  tags:
+    with_pkg
 
 # NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
 - name: install docker-py
   pip:
     name: docker-py
     version: 1.1.0
+  tags:
+    with_pkg

roles/ceph-mon/tasks/docker/start_docker_monitor.yml

@@ -1,9 +1,104 @@
 ---
-- name: run the ceph Monitor docker image
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
+
+- name: populate kv_store with default ceph.conf
+  docker:
+    name: populate-kv-store
+    image: ceph/daemon
+    command: populate_kvstore
+    net: host
+    env:
+      KV_TYPE: "{{kv_type}}"
+      KV_IP: "{{kv_endpoint}}"
+      KV_PORT: "{{kv_port}}"
+  run_once: true
+  when:
+    inventory_hostname == groups.mons[0] and
+    mon_containerized_deployment_with_kv and
+    mon_containerized_default_ceph_conf_with_kv
+
+- name: populate kv_store with custom ceph.conf
+  docker:
+    name: populate-kv-store
+    image: ceph/daemon
+    command: populate_kvstore
+    net: host
+    env:
+      KV_TYPE: "{{kv_type}}"
+      KV_IP: "{{kv_endpoint}}"
+      KV_PORT: "{{kv_port}}"
+    volumes:
+      - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults
+  run_once: true
+  when:
+    inventory_hostname == groups.mons[0] and
+    mon_containerized_deployment_with_kv and
+    not mon_containerized_default_ceph_conf_with_kv
+
+- name: delete populate-kv-store docker
+  docker:
+    name: populate-kv-store
+    state: absent
+    image: ceph/daemon
+
+# Use systemd to manage container on Atomic host and CoreOS
+- name: generate systemd unit file
+  become: true
+  template:
+    src: ceph-mon.service.j2
+    dest: /var/lib/ceph/ceph-mon@.service
+    owner: "root"
+    group: "root"
+    mode: "0644"
+
+- name: link systemd unit file for mon instance
+  file:
+    src: /var/lib/ceph/ceph-mon@.service
+    dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
+    state: link
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
+
+- name: enable systemd unit file for mon instance
+  shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
+  failed_when: false
+  changed_when: false
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
+
+- name: reload systemd unit files
+  shell: systemctl daemon-reload
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
+
+- name: systemd start mon container
+  service:
+    name: ceph-mon@{{ ansible_hostname }}
+    state: started
+    enabled: yes
+  changed_when: false
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
+
+- name: wait for ceph.conf exists
+  wait_for:
+      path: /etc/ceph/ceph.conf
+  when: is_atomic
+
+- name: run the ceph monitor docker image
   docker:
     image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
     name: "{{ ansible_hostname }}"
     net: "host"
     state: "running"
+    privileged: "{{ mon_docker_privileged }}"
     env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+  when:
+    not is_atomic and
+    ansible_os_family != 'CoreOS'

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -0,0 +1,31 @@
+[Unit]
+Description=Ceph Monitor
+After=docker.service
+
+[Service]
+EnvironmentFile=-/etc/environment
+ExecStartPre=-/usr/bin/docker rm %i
+ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon
+ExecStart=/usr/bin/docker run --rm --name %i --net=host \
+   {% if not mon_containerized_deployment_with_kv -%}
+   -v /var/lib/ceph:/var/lib/ceph \
+   -v /etc/ceph:/etc/ceph \
+   {% else -%}
+   -e KV_TYPE={{kv_type}} \
+   -e KV_IP={{kv_endpoint}}\
+   {% endif -%}
+   --privileged \
+   -e CEPH_DAEMON=MON \
+   -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \
+   -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
+   -e MON_NAME={{ ansible_hostname }} \
+   --name={{ ansible_hostname }} \
+    {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}
+ExecStopPost=-/usr/bin/docker stop %i
+Restart=always
+RestartSec=10s
+TimeoutStartSec=120
+TimeoutStopSec=15
+
+[Install]
+WantedBy=multi-user.target

roles/ceph-osd/defaults/main.yml

@@ -120,3 +120,4 @@ ceph_osd_docker_imagename: daemon
 ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables
 #ceph_osd_docker_devices:
 # - /dev/sdb
+ceph_docker_on_openstack: false

roles/ceph-osd/tasks/docker/fetch_configs.yml

@@ -5,6 +5,14 @@
       - /etc/ceph/ceph.conf
       - /var/lib/ceph/bootstrap-osd/ceph.keyring
 
+- name: wait for ceph.conf and keys
+  local_action: >
+    wait_for
+    path="{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+  become: false
+  with_together:
+      - ceph_config_keys
+
 - name: stat for ceph config and keys
   local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
   with_items: ceph_config_keys
@@ -13,9 +21,9 @@
   failed_when: false
   register: statconfig
 
-- name: try to fetch ceph config and keys
+- name: try to copy ceph config and keys
   copy:
-    src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
     dest: "{{ item.0 }}"
     owner: root
     group: root
@@ -23,5 +31,3 @@
   changed_when: false
   with_together:
     - ceph_config_keys
-    - statconfig.results
-  when: item.1.stat.exists == true

roles/ceph-osd/tasks/docker/main.yml

@@ -8,6 +8,14 @@
 - include: checks.yml
   when: ceph_health.rc != 0
 
+- name: check if it is Atomic host
+  stat: path=/run/ostree-booted
+  register: stat_ostree
+
+- name: set fact for using Atomic host
+  set_fact:
+      is_atomic='{{ stat_ostree.stat.exists }}'
+
 - include: pre_requisite.yml
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'

roles/ceph-osd/tasks/docker/pre_requisite.yml

@@ -17,6 +17,8 @@
     - docker
     - docker.io
   when: ansible_distribution == 'Ubuntu'
+  tags:
+    with_pkg
 
 - name: install pip and docker on debian
   apt:
@@ -27,6 +29,8 @@
     - python-pip
     - docker-engine
   when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   yum:
@@ -38,6 +42,8 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   dnf:
@@ -49,9 +55,44 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"
+  tags:
+    with_pkg
+
+- name: install epel-release on redhat
+  yum:
+    name: epel-release
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
+# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
+- name: install six
+  pip:
+    name: six
+    version: 1.9.0
+  tags:
+    with_pkg
+
+- name: pause after docker install before starting (on openstack vms)
+  pause: seconds=5
+  when: ceph_docker_on_openstack
+  tags:
+    with_pkg
+
+- name: start docker service
+  service:
+    name: docker
+    state: started
+    enabled: yes
+  tags:
+    with_pkg
 
 # NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
 - name: install docker-py
   pip:
     name: docker-py
     version: 1.1.0
+  tags:
+    with_pkg

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -1,4 +1,70 @@
 ---
+# For openstack VMs modify the mount point below depending on if the Openstack
+# VM deploy tool defaults to mounting ephemeral disks
+- name: umount ceph disk (if on openstack)
+  mount:
+    name: /mnt
+    src: /dev/vdb
+    fstype: ext3
+    state: unmounted
+  when: ceph_docker_on_openstack
+
+# (rootfs) for reasons I haven't figured out, docker pull and run will fail.
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+
+- name: prepare ceph osd disk
+  docker:
+    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
+    name: "{{ ansible_hostname }}-osd-prepare-{{ item | regex_replace('/', '') }}"
+    net: host
+    pid: host
+    state: running
+    privileged: yes
+    env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_prepare_env }}"
+    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/"
+  with_items: ceph_osd_docker_devices
+  when: ceph_osd_docker_prepare_env is defined
+
+# Use systemd to manage container on Atomic host
+- name: generate systemd unit file
+  become: true
+  template:
+    src: ceph-osd.service.j2
+    dest: /var/lib/ceph/ceph-osd@.service
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  failed_when: false
+
+- name: link systemd unit file for osd instance
+  file:
+    src: /var/lib/ceph/ceph-osd@.service
+    dest: /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
+    state: link
+  with_items: ceph_osd_docker_devices
+  when: is_atomic
+
+- name: enable systemd unit file for osd instance
+  shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
+  failed_when: false
+  changed_when: false
+  with_items: ceph_osd_docker_devices
+  when: is_atomic
+
+- name: reload systemd unit files
+  shell: systemctl daemon-reload
+  when: is_atomic
+
+- name: systemd start osd container
+  service:
+    name: ceph-osd@{{ item | basename }}
+    state: started
+    enabled: yes
+  changed_when: false
+  with_items: ceph_osd_docker_devices
+  when: is_atomic
+
 - name: run the ceph osd docker image
   docker:
     image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
@@ -10,3 +76,4 @@
     env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_extra_env }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/"
   with_items: ceph_osd_docker_devices
+  when: not is_atomic

roles/ceph-osd/tasks/templates/ceph-osd.service.j2

@@ -0,0 +1,25 @@
+[Unit]
+Description=Ceph OSD
+After=docker.service
+
+[Service]
+EnvironmentFile=/etc/environment
+ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
+ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}-osd-dev%i
+ExecStart=/usr/bin/docker run --rm --net=host --pid=host\
+   -v /var/lib/ceph:/var/lib/ceph \
+   -v /etc/ceph:/etc/ceph \
+   -v /dev:/dev \
+   --privileged \
+   -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
+   -e OSD_DEVICE=/dev/%i \
+   --name={{ ansible_hostname }}-osd-dev%i \
+   {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}
+ExecStop=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
+Restart=always
+RestartSec=10s
+TimeoutStartSec=120
+TimeoutStopSec=15
+
+[Install]
+WantedBy=multi-user.target

roles/ceph-restapi/defaults/main.yml

@@ -16,3 +16,4 @@ ceph_restapi_port: 5000
 ceph_restapi_docker_username: ceph
 ceph_restapi_docker_imagename: daemon
 ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables
+ceph_docker_on_openstack: false

roles/ceph-restapi/tasks/docker/fetch_configs.yml

@@ -15,7 +15,7 @@
 
 - name: try to fetch ceph config and keys
   copy:
-    src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
     dest: "{{ item.0 }}"
     owner: root
     group: root

roles/ceph-restapi/tasks/docker/pre_requisite.yml

@@ -9,6 +9,8 @@
     - docker
     - docker.io
   when: ansible_distribution == 'Ubuntu'
+  tags:
+    with_pkg
 
 - name: install pip and docker on debian
   apt:
@@ -19,6 +21,8 @@
     - python-pip
     - docker-engine
   when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   yum:
@@ -30,6 +34,8 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   dnf:
@@ -41,9 +47,36 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"
+  tags:
+    with_pkg
 
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
+- name: install epel-release on redhat
-- name: install docker-py
+  yum:
+    name: epel-release
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
+# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
+- name: install six
   pip:
-    name: docker-py
+    name: six
-    version: 1.1.0
+    version: 1.9.0
+  tags:
+    with_pkg
+
+- name: pause after docker install before starting (on openstack vms)
+  pause: seconds=5
+  when: ceph_docker_on_openstack
+  tags:
+    with_pkg
+
+- name: start docker service
+  service:
+    name: docker
+    state: started
+    enabled: yes
+  tags:
+    with_pkg

roles/ceph-rgw/defaults/main.yml

@@ -31,3 +31,4 @@ ceph_rgw_civetweb_port: 80
 ceph_rgw_docker_username: ceph
 ceph_rgw_docker_imagename: daemon
 ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables
+ceph_docker_on_openstack: false

roles/ceph-rgw/tasks/docker/fetch_configs.yml

@@ -15,7 +15,7 @@
 
 - name: try to fetch ceph config and keys
   copy:
-    src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
     dest: "{{ item.0 }}"
     owner: root
     group: root

roles/ceph-rgw/tasks/docker/pre_requisite.yml

@@ -17,6 +17,8 @@
     - docker
     - docker.io
   when: ansible_distribution == 'Ubuntu'
+  tags:
+    with_pkg
 
 - name: install pip and docker on debian
   apt:
@@ -27,6 +29,8 @@
     - python-pip
     - docker-engine
   when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   yum:
@@ -38,6 +42,8 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
 
 - name: install pip and docker on redhat
   dnf:
@@ -49,9 +55,36 @@
   when:
     ansible_os_family == 'RedHat' and
     ansible_pkg_mgr == "dnf"
+  tags:
+    with_pkg
 
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
+- name: install epel-release on redhat
-- name: install docker-py
+  yum:
+    name: epel-release
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
+# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
+- name: install six
   pip:
-    name: docker-py
+    name: six
-    version: 1.1.0
+    version: 1.9.0
+  tags:
+    with_pkg
+
+- name: pause after docker install before starting (on openstack vms)
+  pause: seconds=5
+  when: ceph_docker_on_openstack
+  tags:
+    with_pkg
+
+- name: start docker service
+  service:
+    name: docker
+    state: started
+    enabled: yes
+  tags:
+    with_pkg

roles/ceph-rgw/tasks/docker/start_docker_rgw.yml

@@ -1,10 +1,13 @@
 ---
+- name: pull ceph daemon image
+  shell: "docker pull {{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
+
 - name: run the rados gateway docker image
   docker:
     image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}"
     name: ceph-{{ ansible_hostname }}-rgw
-    expose=: "{{ ceph_rgw_civetweb_port }}"
+    expose: "{{ ceph_rgw_civetweb_port }}"
     ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}"
     state: running
-    env=: "CEPH_DAEMON=RGW,{{ ceph_rgw_docker_extra_env }}"
+    env: "CEPH_DAEMON=RGW,{{ ceph_rgw_docker_extra_env }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"

vagrant_variables.yml.atomic

@@ -0,0 +1,25 @@
+---
+
+# DEFINE THE NUMBER OF VMS TO RUN
+mon_vms: 1
+osd_vms: 1
+mds_vms: 0
+rgw_vms: 0
+client_vms: 0
+
+# SUBNET TO USE FOR THE VMS
+subnet: 192.168.0
+
+# MEMORY
+memory: 1024
+
+disks: "[ '/dev/sdb', '/dev/sdc' ]"
+
+eth: 'enp0s8'
+vagrant_box: centos/atomic-host
+
+# if vagrant fails to attach storage controller, add the storage controller name by:
+# VBoxManage storagectl `VBoxManage list vms |grep ceph-ansible_osd0|awk '{print $1}'|tr \" ' '` --name "SATA" --add sata
+# and "vagrant up" again
+vagrant_storagectl: 'SATA'
+skip_tags: 'with_pkg'

vagrant_variables.yml.openstack

@@ -0,0 +1,37 @@
+---
+
+# DEFINE THE NUMBER OF VMS TO RUN
+mon_vms: 1
+osd_vms: 1
+mds_vms: 0
+rgw_vms: 0
+client_vms: 0
+
+# SUBNET TO USE FOR THE VMS
+# Use whatever private subnet your Openstack VMs are given
+subnet: 172.17.72 
+
+# For Openstack VMs, the disk will depend on what you are allocated
+disks: "[ '/dev/vdb' ]"
+
+# For Openstack VMs, the lan is usually eth0
+eth: 'eth0'
+
+# For Openstack VMs, choose the following box instead
+vagrant_box: 'openstack'
+
+# For Atomic (RHEL or Cento) uncomment the line below
+#skip_tags: 'with_pkg'
+
+# For deploying on OpenStack VMs uncomment these vars and assign values.  
+# You can use env vars for the values if it makes sense.
+#os_ssh_username : 
+#os_ssh_private_key_path : 
+#os_openstack_auth_url : 
+#os_username : ENV['OS_USERNAME']
+#os_password : ENV['OS_PASSWORD']
+#os_tenant_name : 
+#os_region : 
+#os_flavor : 
+#os_image : 
+#os_keypair_name :