From 0647cad7be275a36a59e1a37fa31d65cbb68bf8a Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Wed, 13 Jan 2016 19:15:35 +0000 Subject: [PATCH 01/32] containerized mon --- group_vars/all.sample | 1 - roles/ceph-mon/tasks/docker/fetch_configs.yml | 3 +- roles/ceph-mon/tasks/docker/pre_requisite.yml | 41 +++++++++++++++++++ .../tasks/docker/start_docker_monitor.yml | 10 +++++ roles/ceph-osd/tasks/docker/fetch_configs.yml | 14 +++++-- roles/ceph-osd/tasks/docker/pre_requisite.yml | 41 +++++++++++++++++++ .../tasks/docker/start_docker_osd.yml | 4 ++ 7 files changed, 107 insertions(+), 7 deletions(-) diff --git a/group_vars/all.sample b/group_vars/all.sample index 48375f711..4284df595 100644 --- a/group_vars/all.sample +++ b/group_vars/all.sample @@ -337,4 +337,3 @@ dummy: ########## #docker: false - diff --git a/roles/ceph-mon/tasks/docker/fetch_configs.yml b/roles/ceph-mon/tasks/docker/fetch_configs.yml index 56e737aee..48438dfac 100644 --- a/roles/ceph-mon/tasks/docker/fetch_configs.yml +++ b/roles/ceph-mon/tasks/docker/fetch_configs.yml @@ -3,7 +3,6 @@ set_fact: ceph_config_keys: - /etc/ceph/ceph.client.admin.keyring - - /etc/ceph/ceph.conf - /etc/ceph/monmap - /etc/ceph/ceph.mon.keyring - /var/lib/ceph/bootstrap-osd/ceph.keyring @@ -20,7 +19,7 @@ - name: try to fetch ceph config and keys copy: - src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" dest: "{{ item.0 }}" owner: root group: root diff --git a/roles/ceph-mon/tasks/docker/pre_requisite.yml b/roles/ceph-mon/tasks/docker/pre_requisite.yml index 41fb0e7fd..f09655f64 100644 --- a/roles/ceph-mon/tasks/docker/pre_requisite.yml +++ b/roles/ceph-mon/tasks/docker/pre_requisite.yml @@ -19,6 +19,8 @@ - docker - docker.io when: ansible_distribution == 'Ubuntu' + tags: + with_pkg - name: install pip and docker on debian apt: @@ -29,6 +31,8 @@ - python-pip - docker-engine when: ansible_distribution == 'Debian' + tags: + with_pkg - name: install pip and docker on redhat yum: @@ -40,6 +44,8 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "yum" + tags: + with_pkg - name: install pip and docker on redhat dnf: @@ -51,9 +57,44 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "dnf" + tags: + with_pkg + +- name: install epel-release on redhat + yum: + name: epel-release + state: present + when: ansible_os_family == 'RedHat' + tags: + with_pkg + +# NOTE (jimcurtis): need at least version 1.9.0 of six or we get: +# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined +- name: install six + pip: + name: six + version: 1.9.0 + tags: + with_pkg + +- name: pause after docker install before starting (on openstack vms) + pause: seconds=5 + when: ceph_docker_on_openstack + tags: + with_pkg + +- name: start docker service + service: + name: docker + state: started + enabled: yes + tags: + with_pkg # NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227 - name: install docker-py pip: name: docker-py version: 1.1.0 + tags: + with_pkg diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index e943fd9af..3d962822d 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -1,4 +1,7 @@ --- +- name: pull ceph daemon image + shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + - name: run the ceph Monitor docker image docker: image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" @@ -7,3 +10,10 @@ state: "running" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" + +- name: fetch ceph.conf from mon + fetch: + dest: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid}}/etc/ceph/ceph.conf" + src: /etc/ceph/ceph.conf + flat: yes + \ No newline at end of file diff --git a/roles/ceph-osd/tasks/docker/fetch_configs.yml b/roles/ceph-osd/tasks/docker/fetch_configs.yml index f67200b44..970519807 100644 --- a/roles/ceph-osd/tasks/docker/fetch_configs.yml +++ b/roles/ceph-osd/tasks/docker/fetch_configs.yml @@ -5,6 +5,14 @@ - /etc/ceph/ceph.conf - /var/lib/ceph/bootstrap-osd/ceph.keyring +- name: wait for ceph.conf and keys + local_action: > + wait_for + path="{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + become: false + with_together: + - ceph_config_keys + - name: stat for ceph config and keys local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }} with_items: ceph_config_keys @@ -13,9 +21,9 @@ failed_when: false register: statconfig -- name: try to fetch ceph config and keys +- name: try to copy ceph config and keys copy: - src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" dest: "{{ item.0 }}" owner: root group: root @@ -23,5 +31,3 @@ changed_when: false with_together: - ceph_config_keys - - statconfig.results - when: item.1.stat.exists == true diff --git a/roles/ceph-osd/tasks/docker/pre_requisite.yml b/roles/ceph-osd/tasks/docker/pre_requisite.yml index 3f3a982c4..a4837f11f 100644 --- a/roles/ceph-osd/tasks/docker/pre_requisite.yml +++ b/roles/ceph-osd/tasks/docker/pre_requisite.yml @@ -17,6 +17,8 @@ - docker - docker.io when: ansible_distribution == 'Ubuntu' + tags: + with_pkg - name: install pip and docker on debian apt: @@ -27,6 +29,8 @@ - python-pip - docker-engine when: ansible_distribution == 'Debian' + tags: + with_pkg - name: install pip and docker on redhat yum: @@ -38,6 +42,8 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "yum" + tags: + with_pkg - name: install pip and docker on redhat dnf: @@ -49,9 +55,44 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "dnf" + tags: + with_pkg + +- name: install epel-release on redhat + yum: + name: epel-release + state: present + when: ansible_os_family == 'RedHat' + tags: + with_pkg + +# NOTE (jimcurtis): need at least version 1.9.0 of six or we get: +# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined +- name: install six + pip: + name: six + version: 1.9.0 + tags: + with_pkg + +- name: pause after docker install before starting (on openstack vms) + pause: seconds=5 + when: ceph_docker_on_openstack + tags: + with_pkg + +- name: start docker service + service: + name: docker + state: started + enabled: yes + tags: + with_pkg # NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227 - name: install docker-py pip: name: docker-py version: 1.1.0 + tags: + with_pkg diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index 847b07ebe..64fccae03 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -1,4 +1,8 @@ --- +# (rootfs) for reasons I haven't figured out, docker pull and run will fail. +- name: pull ceph daemon image + shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + - name: run the ceph osd docker image docker: image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" From 0d704b571045e260255a945b01a1f1d2b8100602 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Thu, 21 Jan 2016 17:22:46 +0000 Subject: [PATCH 02/32] copy bootstrap keyring and conf from mon to ansible server, so osd can get them to bootstrap Signed-off-by: Huamin Chen --- roles/ceph-mon/tasks/docker/fetch_configs.yml | 3 ++- .../tasks/docker/start_docker_monitor.yml | 27 ++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/roles/ceph-mon/tasks/docker/fetch_configs.yml b/roles/ceph-mon/tasks/docker/fetch_configs.yml index 48438dfac..46384d009 100644 --- a/roles/ceph-mon/tasks/docker/fetch_configs.yml +++ b/roles/ceph-mon/tasks/docker/fetch_configs.yml @@ -2,6 +2,7 @@ - name: set config and keys paths set_fact: ceph_config_keys: + - /etc/ceph/ceph.conf - /etc/ceph/ceph.client.admin.keyring - /etc/ceph/monmap - /etc/ceph/ceph.mon.keyring @@ -19,7 +20,7 @@ - name: try to fetch ceph config and keys copy: - src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" + src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" dest: "{{ item.0 }}" owner: root group: root diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 3d962822d..386f4ab54 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -1,4 +1,12 @@ --- +- name: set config and keys paths + set_fact: + ceph_bootstrap_config_keys: + - /etc/ceph/ceph.conf + - /var/lib/ceph/bootstrap-osd/ceph.keyring + - /var/lib/ceph/bootstrap-rgw/ceph.keyring + - /var/lib/ceph/bootstrap-mds/ceph.keyring + - name: pull ceph daemon image shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" @@ -11,9 +19,22 @@ env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" -- name: fetch ceph.conf from mon + +- name: stat for ceph config and keys + stat: path="{{ item }}" + with_items: ceph_bootstrap_config_keys + changed_when: false + failed_when: false + register: statmonconfig + +- name: fetch boostrap keys and conf from mon fetch: - dest: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid}}/etc/ceph/ceph.conf" - src: /etc/ceph/ceph.conf + dest: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" + src: "{{ item.0 }}" flat: yes + with_together: + - ceph_bootstrap_config_keys + - statmonconfig.results + when: item.1.stat.exists == true + \ No newline at end of file From 29b239a8f858a14aabbe1d1ae552856ca3bb866e Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Thu, 21 Jan 2016 19:54:41 +0000 Subject: [PATCH 03/32] containerized mon: create restapi keyring add sample config for containerized deployment --- group_vars/all.docker | 25 +++++++++++++++++++++++++ roles/ceph-mon/tasks/ceph_keys.yml | 13 ++++++++++++- roles/ceph-mon/tasks/docker/main.yml | 1 + 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 group_vars/all.docker diff --git a/group_vars/all.docker b/group_vars/all.docker new file mode 100644 index 000000000..67f995812 --- /dev/null +++ b/group_vars/all.docker @@ -0,0 +1,25 @@ +--- +dummy: +cephx_require_signatures: false # Kernel RBD does NOT support signatures! +cephx_cluster_require_signatures: false +restapi_group_name: restapis +fetch_directory: fetch/ +mon_containerized_deployment: true +ceph_mon_docker_username: hchen +ceph_mon_docker_imagename: rhceph +ceph_mon_docker_interface: "{{ monitor_interface }}" +ceph_mon_docker_subnet: "{{ public_network }}" # subnet of the ceph_mon_docker_interface +ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables +osd_containerized_deployment: true +ceph_osd_docker_username: hchen +ceph_osd_docker_imagename: rhceph +ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK,OSD_FORCE_ZAP=1" # comma separated variables +ceph_osd_docker_devices: + - /dev/sdb +rgw_containerized_deployment: true +ceph_rgw_docker_username: hchen +ceph_rgw_docker_imagename: rhceph +ceph_rgw_civetweb_port: 80 +ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables +rbd_client_directories: false +ceph_stable_release: hammer diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml index f2d33f467..f5eba729b 100644 --- a/roles/ceph-mon/tasks/ceph_keys.yml +++ b/roles/ceph-mon/tasks/ceph_keys.yml @@ -5,13 +5,24 @@ wait_for: path: /etc/ceph/ceph.client.admin.keyring -- name: create ceph rest api keyring +- name: create ceph rest api keyring when mon is not containerized command: ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring args: creates: /etc/ceph/ceph.client.restapi.keyring changed_when: false when: cephx and + not mon_containerized_deployment and + groups[restapi_group_name] is defined + +- name: create ceph rest api keyring when mon is containerized + command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring + args: + creates: /etc/ceph/ceph.client.restapi.keyring + changed_when: false + when: + cephx and + mon_containerized_deployment and groups[restapi_group_name] is defined - include: openstack_config.yml diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index 22489ff99..e2d02ba20 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -15,3 +15,4 @@ - include: fetch_configs.yml - include: start_docker_monitor.yml - include: copy_configs.yml +- include: ../ceph_keys.yml From dda83004c8065fabd9367baf796ebd0b0b776076 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Fri, 22 Jan 2016 19:10:41 +0000 Subject: [PATCH 04/32] containerize rgw and restapi Signed-off-by: Huamin Chen --- .../tasks/docker/fetch_configs.yml | 2 +- .../tasks/docker/pre_requisite.yml | 41 +++++++++++++++++-- roles/ceph-rgw/tasks/docker/fetch_configs.yml | 2 +- roles/ceph-rgw/tasks/docker/pre_requisite.yml | 41 +++++++++++++++++-- .../tasks/docker/start_docker_rgw.yml | 7 +++- 5 files changed, 81 insertions(+), 12 deletions(-) diff --git a/roles/ceph-restapi/tasks/docker/fetch_configs.yml b/roles/ceph-restapi/tasks/docker/fetch_configs.yml index 919bf0a5c..2faf8ade5 100644 --- a/roles/ceph-restapi/tasks/docker/fetch_configs.yml +++ b/roles/ceph-restapi/tasks/docker/fetch_configs.yml @@ -15,7 +15,7 @@ - name: try to fetch ceph config and keys copy: - src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" dest: "{{ item.0 }}" owner: root group: root diff --git a/roles/ceph-restapi/tasks/docker/pre_requisite.yml b/roles/ceph-restapi/tasks/docker/pre_requisite.yml index 3eef68689..f2d9ecd70 100644 --- a/roles/ceph-restapi/tasks/docker/pre_requisite.yml +++ b/roles/ceph-restapi/tasks/docker/pre_requisite.yml @@ -9,6 +9,8 @@ - docker - docker.io when: ansible_distribution == 'Ubuntu' + tags: + with_pkg - name: install pip and docker on debian apt: @@ -19,6 +21,8 @@ - python-pip - docker-engine when: ansible_distribution == 'Debian' + tags: + with_pkg - name: install pip and docker on redhat yum: @@ -30,6 +34,8 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "yum" + tags: + with_pkg - name: install pip and docker on redhat dnf: @@ -41,9 +47,36 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "dnf" + tags: + with_pkg -# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227 -- name: install docker-py +- name: install epel-release on redhat + yum: + name: epel-release + state: present + when: ansible_os_family == 'RedHat' + tags: + with_pkg + +# NOTE (jimcurtis): need at least version 1.9.0 of six or we get: +# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined +- name: install six pip: - name: docker-py - version: 1.1.0 + name: six + version: 1.9.0 + tags: + with_pkg + +- name: pause after docker install before starting (on openstack vms) + pause: seconds=5 + when: ceph_docker_on_openstack + tags: + with_pkg + +- name: start docker service + service: + name: docker + state: started + enabled: yes + tags: + with_pkg diff --git a/roles/ceph-rgw/tasks/docker/fetch_configs.yml b/roles/ceph-rgw/tasks/docker/fetch_configs.yml index bd65d7d8f..45dd3bebc 100644 --- a/roles/ceph-rgw/tasks/docker/fetch_configs.yml +++ b/roles/ceph-rgw/tasks/docker/fetch_configs.yml @@ -15,7 +15,7 @@ - name: try to fetch ceph config and keys copy: - src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" dest: "{{ item.0 }}" owner: root group: root diff --git a/roles/ceph-rgw/tasks/docker/pre_requisite.yml b/roles/ceph-rgw/tasks/docker/pre_requisite.yml index c0a98451a..d288e6f58 100644 --- a/roles/ceph-rgw/tasks/docker/pre_requisite.yml +++ b/roles/ceph-rgw/tasks/docker/pre_requisite.yml @@ -17,6 +17,8 @@ - docker - docker.io when: ansible_distribution == 'Ubuntu' + tags: + with_pkg - name: install pip and docker on debian apt: @@ -27,6 +29,8 @@ - python-pip - docker-engine when: ansible_distribution == 'Debian' + tags: + with_pkg - name: install pip and docker on redhat yum: @@ -38,6 +42,8 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "yum" + tags: + with_pkg - name: install pip and docker on redhat dnf: @@ -49,9 +55,36 @@ when: ansible_os_family == 'RedHat' and ansible_pkg_mgr == "dnf" + tags: + with_pkg -# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227 -- name: install docker-py +- name: install epel-release on redhat + yum: + name: epel-release + state: present + when: ansible_os_family == 'RedHat' + tags: + with_pkg + +# NOTE (jimcurtis): need at least version 1.9.0 of six or we get: +# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined +- name: install six pip: - name: docker-py - version: 1.1.0 + name: six + version: 1.9.0 + tags: + with_pkg + +- name: pause after docker install before starting (on openstack vms) + pause: seconds=5 + when: ceph_docker_on_openstack + tags: + with_pkg + +- name: start docker service + service: + name: docker + state: started + enabled: yes + tags: + with_pkg diff --git a/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml b/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml index 65781fbba..ce551848a 100644 --- a/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml +++ b/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml @@ -1,10 +1,13 @@ --- +- name: pull ceph daemon image + shell: "docker pull {{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}" + - name: run the rados gateway docker image docker: image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}" name: ceph-{{ ansible_hostname }}-rgw - expose=: "{{ ceph_rgw_civetweb_port }}" + expose: "{{ ceph_rgw_civetweb_port }}" ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}" state: running - env=: "CEPH_DAEMON=RGW,{{ ceph_rgw_docker_extra_env }}" + env: "CEPH_DAEMON=RGW,{{ ceph_rgw_docker_extra_env }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" From fd03074cd5dc1a19d2e32a051c1b084047992c8a Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Fri, 22 Jan 2016 19:56:15 +0000 Subject: [PATCH 05/32] remove unused files Signed-off-by: Huamin Chen --- roles/ceph-mon/tasks/docker/copy_configs.yml | 10 ---------- roles/ceph-mon/tasks/docker/main.yml | 1 - 2 files changed, 11 deletions(-) delete mode 100644 roles/ceph-mon/tasks/docker/copy_configs.yml diff --git a/roles/ceph-mon/tasks/docker/copy_configs.yml b/roles/ceph-mon/tasks/docker/copy_configs.yml deleted file mode 100644 index f2ba50e81..000000000 --- a/roles/ceph-mon/tasks/docker/copy_configs.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: push ceph files to the ansible server - fetch: - src: "{{ item.0 }}" - dest: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" - flat: yes - with_together: - - ceph_config_keys - - statconfig.results - when: item.1.stat.exists == false diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index e2d02ba20..bc15a362c 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -14,5 +14,4 @@ - include: fetch_configs.yml - include: start_docker_monitor.yml -- include: copy_configs.yml - include: ../ceph_keys.yml From f88eff37d7329db0892c0d7d660a8dbd4e237f23 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Mon, 25 Jan 2016 19:28:10 +0000 Subject: [PATCH 06/32] initial steps to provision CentOS Atomic host Signed-off-by: Huamin Chen --- Vagrantfile | 66 ++++++++++++++----- .../tasks/installs/install_on_redhat.yml | 16 ++--- vagrant_variables.yml.atomic | 34 ++++++++++ 3 files changed, 89 insertions(+), 27 deletions(-) create mode 100644 vagrant_variables.yml.atomic diff --git a/Vagrantfile b/Vagrantfile index ccd5773f0..dcfe2fc2c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -19,9 +19,20 @@ STORAGECTL = settings['vagrant_storagectl'] ETH = settings['eth'] DOCKER = settings['docker'] +if BOX == 'openstack' + require 'vagrant-openstack-provider' + OSVM = true + USER = settings['os_ssh_username'] +else + OSVM = false +end + ansible_provision = proc do |ansible| if DOCKER then ansible.playbook = 'site-docker.yml' + if settings['skip_tags'] + ansible.skip_tags = settings['skip_tags'] + end else ansible.playbook = 'site.yml' end @@ -83,11 +94,32 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| override.vm.synced_folder '.', '/home/vagrant/sync', disabled: true end + if BOX == 'openstack' + # OpenStack VMs + config.vm.provider :openstack do |os| + config.vm.synced_folder ".", "/home/#{USER}/vagrant", disabled: true + config.ssh.username = USER + config.ssh.private_key_path = settings['os_ssh_private_key_path'] + config.ssh.pty = true + os.openstack_auth_url = settings['os_openstack_auth_url'] + os.username = settings['os_username'] + os.password = settings['os_password'] + os.tenant_name = settings['os_tenant_name'] + os.region = settings['os_region'] + os.flavor = settings['os_flavor'] + os.image = settings['os_image'] + os.keypair_name = settings['os_keypair_name'] + os.security_groups = ['default'] + config.vm.provision "shell", inline: "true", upload_path: "/home/#{USER}/vagrant-shell" + end + end + (0..CLIENTS - 1).each do |i| config.vm.define "client#{i}" do |client| client.vm.hostname = "ceph-client#{i}" - client.vm.network :private_network, ip: "#{SUBNET}.4#{i}" - + if !OSVM + client.vm.network :private_network, ip: "#{SUBNET}.4#{i}" + end # Virtualbox client.vm.provider :virtualbox do |vb| vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"] @@ -114,7 +146,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| (0..NRGWS - 1).each do |i| config.vm.define "rgw#{i}" do |rgw| rgw.vm.hostname = "ceph-rgw#{i}" - rgw.vm.network :private_network, ip: "#{SUBNET}.5#{i}" + if !OSVM + rgw.vm.network :private_network, ip: "#{SUBNET}.5#{i}" + end # Virtualbox rgw.vm.provider :virtualbox do |vb| @@ -142,8 +176,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| (0..NMDSS - 1).each do |i| config.vm.define "mds#{i}" do |mds| mds.vm.hostname = "ceph-mds#{i}" - mds.vm.network :private_network, ip: "#{SUBNET}.7#{i}" - + if !OSVM + mds.vm.network :private_network, ip: "#{SUBNET}.7#{i}" + end # Virtualbox mds.vm.provider :virtualbox do |vb| vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"] @@ -158,7 +193,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| mds.vm.provider :libvirt do |lv| lv.memory = MEMORY end - # Parallels mds.vm.provider "parallels" do |prl| prl.name = "ceph-mds#{i}" @@ -170,8 +204,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| (0..NMONS - 1).each do |i| config.vm.define "mon#{i}" do |mon| mon.vm.hostname = "ceph-mon#{i}" - mon.vm.network :private_network, ip: "#{SUBNET}.1#{i}" - + if !OSVM + mon.vm.network :private_network, ip: "#{SUBNET}.1#{i}" + end # Virtualbox mon.vm.provider :virtualbox do |vb| vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"] @@ -198,9 +233,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| (0..NOSDS - 1).each do |i| config.vm.define "osd#{i}" do |osd| osd.vm.hostname = "ceph-osd#{i}" - osd.vm.network :private_network, ip: "#{SUBNET}.10#{i}" - osd.vm.network :private_network, ip: "#{SUBNET}.20#{i}" - + if !OSVM + osd.vm.network :private_network, ip: "#{SUBNET}.10#{i}" + osd.vm.network :private_network, ip: "#{SUBNET}.20#{i}" + end # Virtualbox osd.vm.provider :virtualbox do |vb| (0..1).each do |d| @@ -244,10 +280,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| prl.name = "ceph-osd#{i}" prl.memory = "#{MEMORY}" (0..1).each do |d| - prl.customize ["set", :id, - "--device-add", - "hdd", - "--iface", + prl.customize ["set", :id, + "--device-add", + "hdd", + "--iface", "sata"] end end diff --git a/roles/ceph-common/tasks/installs/install_on_redhat.yml b/roles/ceph-common/tasks/installs/install_on_redhat.yml index da0ce967e..670e529ef 100644 --- a/roles/ceph-common/tasks/installs/install_on_redhat.yml +++ b/roles/ceph-common/tasks/installs/install_on_redhat.yml @@ -46,9 +46,7 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and mon_group_name in group_names and - ansible_pkg_mgr == "yum" and - ceph_stable and - ceph_stable_release not in ceph_stable_releases + ansible_pkg_mgr == "yum" - name: install distro or red hat storage ceph mon dnf: @@ -60,9 +58,7 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and mon_group_name in group_names and - ansible_pkg_mgr == "dnf" and - ceph_stable and - ceph_stable_release not in ceph_stable_releases + ansible_pkg_mgr == "dnf" - name: install distro or red hat storage ceph osd yum: @@ -74,9 +70,7 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and osd_group_name in group_names and - ansible_pkg_mgr == "yum" and - ceph_stable and - ceph_stable_release not in ceph_stable_releases + ansible_pkg_mgr == "yum" - name: install distro or red hat storage ceph osd dnf: @@ -88,9 +82,7 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and osd_group_name in group_names and - ansible_pkg_mgr == "dnf" and - ceph_stable and - ceph_stable_release not in ceph_stable_releases + ansible_pkg_mgr == "dnf" - name: install ceph-test yum: diff --git a/vagrant_variables.yml.atomic b/vagrant_variables.yml.atomic new file mode 100644 index 000000000..a4932df97 --- /dev/null +++ b/vagrant_variables.yml.atomic @@ -0,0 +1,34 @@ +--- + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 1 +mds_vms: 0 +rgw_vms: 0 +client_vms: 0 + +# SUBNET TO USE FOR THE VMS +subnet: 192.168.0 + +# MEMORY +memory: 1024 + +disks: "[ '/dev/sdb', '/dev/sdc' ]" + +eth: 'enp0s3' +# VAGRANT BOX +# Fedora: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Ubuntu: ubuntu/trusty64 +# CentOS: chef/centos-7.0 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ + +vagrant_box: centos/atomic-host + +# if vagrant fails to attach storage controller, add the storage controller name by: +# VBoxManage storagectl `VBoxManage list vms |grep ceph-ansible-osd|awk '{print $1}'|tr \" ' '` --name "LsiLogic" --add scsi +# and "vagrant up" again +vagrant_storagectl: 'LsiLogic' +skip_tags: 'with_pkg' \ No newline at end of file From 70561b3fc30fd7a8bca656eeac3bc13d1b6a63d6 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 26 Jan 2016 20:01:03 +0000 Subject: [PATCH 07/32] add variable to allow containerized mon to run privileged mode. this is to allow ceph-authtool to read and write to /var/ and /etc on CentOS Atomic. Add doc on how to run containerized deployment on RHEL/CentOS Atomic Signed-off-by: Huamin Chen --- README.md | 14 ++++++++++++++ group_vars/all.docker | 1 + group_vars/mons.sample | 4 ++++ .../tasks/docker/start_docker_monitor.yml | 1 + vagrant_variables.yml.atomic | 17 ++++------------- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e3450753b..29ca84723 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,20 @@ $ vagrant provision If you want to use "backports", you can set "true" to `ceph_use_distro_backports`. Attention, ceph-common doesn't manage backports repository, you must add it yourself. +### For Atomic systems + +If you want to run containerized deployment on Atomic systems (RHEL/CentOS Atomic), please copy +[vagrant.yml.atomic](vagrant_variables.yml.atomic) to vagrant_variables.yml, and copy [group_vars/all.docker](group_vars/all.docker) to `group_vars/all`. + +Since `centos/atomic-host` doesn't have spare storage controller to attach more disks, it is likely the first time `vagrant up --provider=virtualbox` runs, it will fail to attach to a storage controller. In such case, run the following command: + +```console +VBoxManage storagectl `VBoxManage list vms |grep ceph-ansible_osd0|awk '{print $1}'|tr \" ' '` --name "SATA" --add sata +``` + +then run `vagrant up --provider=virtualbox` again. + + # Want to contribute? diff --git a/group_vars/all.docker b/group_vars/all.docker index 67f995812..f0c57139b 100644 --- a/group_vars/all.docker +++ b/group_vars/all.docker @@ -5,6 +5,7 @@ cephx_cluster_require_signatures: false restapi_group_name: restapis fetch_directory: fetch/ mon_containerized_deployment: true +mon_docker_privileged: true ceph_mon_docker_username: hchen ceph_mon_docker_imagename: rhceph ceph_mon_docker_interface: "{{ monitor_interface }}" diff --git a/group_vars/mons.sample b/group_vars/mons.sample index 75f9daf27..c6414a941 100644 --- a/group_vars/mons.sample +++ b/group_vars/mons.sample @@ -71,9 +71,13 @@ dummy: ########## #mon_containerized_deployment: false +#mon_containerized_deployment_with_kv: false +#mon_containerized_default_ceph_conf_with_kv: false #ceph_mon_docker_interface: eth0 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface #ceph_mon_docker_username: ceph #ceph_mon_docker_imagename: daemon #ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables +#ceph_docker_on_openstack: false +#mon_docker_privileged: true diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 386f4ab54..c90201b3b 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -16,6 +16,7 @@ name: "{{ ansible_hostname }}" net: "host" state: "running" + privileged: "{{ mon_docker_privileged }}" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" diff --git a/vagrant_variables.yml.atomic b/vagrant_variables.yml.atomic index a4932df97..9561efbdb 100644 --- a/vagrant_variables.yml.atomic +++ b/vagrant_variables.yml.atomic @@ -15,20 +15,11 @@ memory: 1024 disks: "[ '/dev/sdb', '/dev/sdc' ]" -eth: 'enp0s3' -# VAGRANT BOX -# Fedora: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box -# Ubuntu: ubuntu/trusty64 -# CentOS: chef/centos-7.0 -# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' -# For more boxes have a look at: -# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= -# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ - +eth: 'enp0s8' vagrant_box: centos/atomic-host # if vagrant fails to attach storage controller, add the storage controller name by: -# VBoxManage storagectl `VBoxManage list vms |grep ceph-ansible-osd|awk '{print $1}'|tr \" ' '` --name "LsiLogic" --add scsi +# VBoxManage storagectl `VBoxManage list vms |grep ceph-ansible_osd0|awk '{print $1}'|tr \" ' '` --name "SATA" --add sata # and "vagrant up" again -vagrant_storagectl: 'LsiLogic' -skip_tags: 'with_pkg' \ No newline at end of file +vagrant_storagectl: 'SATA' +skip_tags: 'with_pkg' From a3dbfba4c00363f48f2de048e8b238ded0a8b7db Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Wed, 27 Jan 2016 19:50:05 +0000 Subject: [PATCH 08/32] use systemd to manage ceph daemons Signed-off-by: Huamin Chen --- roles/ceph-mon/tasks/docker/main.yml | 8 ++++ .../tasks/docker/start_docker_monitor.yml | 46 +++++++++++++++++-- .../tasks/templates/ceph-mon.service.j2 | 28 +++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 roles/ceph-mon/tasks/templates/ceph-mon.service.j2 diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index bc15a362c..5c74a9201 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -5,6 +5,14 @@ changed_when: false failed_when: false +- name: check if it is Atomic host + stat: path=/run/ostree-booted + register: stat_ostree + +- name: set fact for using Atomic host + set_fact: + is_atomic='{{ stat_ostree.stat.exists }}' + - include: checks.yml when: ceph_health.rc != 0 diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index c90201b3b..0a269fcd0 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -10,6 +10,48 @@ - name: pull ceph daemon image shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" +# Use systemd to manage container on Atomic host +- name: generate systemd unit file + sudo: true + config_template: + src: ceph-mon.service.j2 + dest: /var/lib/ceph/ceph-mon@.service + owner: "root" + group: "root" + mode: "0644" + config_overrides: {} + config_type: ini + +- name: link systemd unit file for mon instance + file: + src: /var/lib/ceph/ceph-mon@.service + dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service + state: link + when: is_atomic + +- name: enable systemd unit file for mon instance + shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service + failed_when: false + changed_when: false + when: is_atomic + +- name: reload systemd unit files + shell: systemctl daemon-reload + when: is_atomic + +- name: systemd start mon container + service: + name: ceph-mon@{{ ansible_hostname }} + state: started + enabled: yes + changed_when: false + when: is_atomic + +- name: wait for ceph.conf exists + wait_for: + path: /etc/ceph/ceph.conf + when: is_atomic + - name: run the ceph Monitor docker image docker: image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" @@ -19,7 +61,7 @@ privileged: "{{ mon_docker_privileged }}" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" - + when: not is_atomic - name: stat for ceph config and keys stat: path="{{ item }}" @@ -37,5 +79,3 @@ - ceph_bootstrap_config_keys - statmonconfig.results when: item.1.stat.exists == true - - \ No newline at end of file diff --git a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 b/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 new file mode 100644 index 000000000..b97248e21 --- /dev/null +++ b/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 @@ -0,0 +1,28 @@ +[Unit] +Description=Ceph Monitor +After=docker.service + +[Service] +EnvironmentFile=/etc/environment +ExecStartPre=-/usr/bin/docker kill %p +ExecStartPre=-/usr/bin/docker rm %p +ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon +ExecStart=/usr/bin/docker run -d --rm --name %p --net=host \ + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + --privileged \ + -e CEPH_DAEMON=MON \ + -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \ + -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \ + -e MON_NAME={{ ansible_hostname }} \ + --name={{ ansible_hostname }} \ + {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }} +ExecStopPost=-/usr/bin/docker stop %p +ExecStopPost=-/usr/bin/docker rm %p +Restart=always +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=15 + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 6bd8d41a39610efe098dff0df5eb2f7b4a1c9946 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Mon, 1 Feb 2016 15:40:48 +0000 Subject: [PATCH 09/32] remove --rm option from docker run, since the image is already removed during pre start Signed-off-by: Huamin Chen --- roles/ceph-mon/tasks/templates/ceph-mon.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 b/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 index b97248e21..c8c5de7bc 100644 --- a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 @@ -7,7 +7,7 @@ EnvironmentFile=/etc/environment ExecStartPre=-/usr/bin/docker kill %p ExecStartPre=-/usr/bin/docker rm %p ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon -ExecStart=/usr/bin/docker run -d --rm --name %p --net=host \ +ExecStart=/usr/bin/docker run -d --name %p --net=host \ -v /var/lib/ceph:/var/lib/ceph \ -v /etc/ceph:/etc/ceph \ --privileged \ From e18154fbe552348667778e066157c44948bbedc4 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Fri, 5 Feb 2016 18:50:57 +0000 Subject: [PATCH 10/32] serialize containerized mon config Signed-off-by: Huamin Chen --- roles/ceph-mon/tasks/docker/create_configs.yml | 18 ++++++++++++++++++ roles/ceph-mon/tasks/docker/main.yml | 3 +++ .../tasks/docker/start_docker_monitor.yml | 1 + 3 files changed, 22 insertions(+) create mode 100644 roles/ceph-mon/tasks/docker/create_configs.yml diff --git a/roles/ceph-mon/tasks/docker/create_configs.yml b/roles/ceph-mon/tasks/docker/create_configs.yml new file mode 100644 index 000000000..ff59a9a3c --- /dev/null +++ b/roles/ceph-mon/tasks/docker/create_configs.yml @@ -0,0 +1,18 @@ +--- +- name: create ceph conf directory + file: + path: /etc/ceph + state: directory + owner: root + group: root + mode: 0644 + +- name: generate ceph configuration file + config_template: + src: "{{ playbook_dir }}/roles/ceph-common/templates/ceph.conf.j2" + dest: /etc/ceph/ceph.conf + owner: "root" + group: "root" + mode: "0644" + config_overrides: "{{ ceph_conf_overrides }}" + config_type: ini diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index 5c74a9201..e1965301f 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -20,6 +20,9 @@ - include: selinux.yml when: ansible_os_family == 'RedHat' +# let the first mon create configs and keyrings +- include: create_configs.yml + when: inventory_hostname == groups.mons[0] - include: fetch_configs.yml - include: start_docker_monitor.yml - include: ../ceph_keys.yml diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 0a269fcd0..7fa861852 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -79,3 +79,4 @@ - ceph_bootstrap_config_keys - statmonconfig.results when: item.1.stat.exists == true + and inventory_hostname == groups.mons[0] From a4b3885ac9ff74c93ab1b2aaf0074554327dd232 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Mon, 8 Feb 2016 17:23:37 +0000 Subject: [PATCH 11/32] allow multiple mon containers to reach quorum Signed-off-by: Huamin Chen --- roles/ceph-mon/tasks/ceph_keys.yml | 6 +++++- roles/ceph-mon/tasks/docker/main.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml index f5eba729b..00c716895 100644 --- a/roles/ceph-mon/tasks/ceph_keys.yml +++ b/roles/ceph-mon/tasks/ceph_keys.yml @@ -15,6 +15,8 @@ not mon_containerized_deployment and groups[restapi_group_name] is defined +- debug: msg="host {{ inventory_hostname }} last {{ groups.mons|last }}" + - name: create ceph rest api keyring when mon is containerized command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring args: @@ -23,7 +25,9 @@ when: cephx and mon_containerized_deployment and - groups[restapi_group_name] is defined + groups[restapi_group_name] is defined and + inventory_hostname == groups.mons|last + - include: openstack_config.yml when: openstack_config diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index e1965301f..c5d82070f 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -25,4 +25,4 @@ when: inventory_hostname == groups.mons[0] - include: fetch_configs.yml - include: start_docker_monitor.yml -- include: ../ceph_keys.yml +- include: ../ceph_keys.yml \ No newline at end of file From 6cf3fff4365dbf8a0b2fb9b765bdc8d076939d58 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 9 Feb 2016 01:49:47 +0000 Subject: [PATCH 12/32] split osd disk to prepare and activate Signed-off-by: Huamin Chen --- group_vars/all.docker | 6 +++++- roles/ceph-mon/tasks/ceph_keys.yml | 2 -- roles/ceph-osd/tasks/docker/start_docker_osd.yml | 13 +++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/group_vars/all.docker b/group_vars/all.docker index f0c57139b..12ae37383 100644 --- a/group_vars/all.docker +++ b/group_vars/all.docker @@ -14,7 +14,8 @@ ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variabl osd_containerized_deployment: true ceph_osd_docker_username: hchen ceph_osd_docker_imagename: rhceph -ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK,OSD_FORCE_ZAP=1" # comma separated variables +ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE" # comma separated variables +ceph_osd_docker_prepare_env: "CEPH_DAEMON=OSD_CEPH_DISK_PREPARE,OSD_FORCE_ZAP=1" # comma separated variables ceph_osd_docker_devices: - /dev/sdb rgw_containerized_deployment: true @@ -24,3 +25,6 @@ ceph_rgw_civetweb_port: 80 ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables rbd_client_directories: false ceph_stable_release: hammer + + + diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml index 00c716895..c9ddc6854 100644 --- a/roles/ceph-mon/tasks/ceph_keys.yml +++ b/roles/ceph-mon/tasks/ceph_keys.yml @@ -15,8 +15,6 @@ not mon_containerized_deployment and groups[restapi_group_name] is defined -- debug: msg="host {{ inventory_hostname }} last {{ groups.mons|last }}" - - name: create ceph rest api keyring when mon is containerized command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring args: diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index 64fccae03..43a93f91d 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -3,6 +3,19 @@ - name: pull ceph daemon image shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" +- name: prepare ceph osd disk + docker: + image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" + name: "{{ ansible_hostname }}-osd-prepare-{{ item | regex_replace('/', '') }}" + net: host + pid: host + state: running + privileged: yes + env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_prepare_env }}" + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/" + with_items: ceph_osd_docker_devices + when: ceph_osd_docker_prepare_env is defined + - name: run the ceph osd docker image docker: image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" From 326db629a02f8352c87fbe098abfaa753b048e4a Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 9 Feb 2016 17:35:41 +0000 Subject: [PATCH 13/32] start osd daemon via systemd if supported Signed-off-by: Huamin Chen --- .../tasks/templates/ceph-mon.service.j2 | 10 ++--- roles/ceph-osd/tasks/docker/main.yml | 8 ++++ .../tasks/docker/start_docker_osd.yml | 44 ++++++++++++++++++- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 b/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 index c8c5de7bc..484b2e3a1 100644 --- a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 @@ -4,10 +4,9 @@ After=docker.service [Service] EnvironmentFile=/etc/environment -ExecStartPre=-/usr/bin/docker kill %p -ExecStartPre=-/usr/bin/docker rm %p +ExecStartPre=-/usr/bin/docker rm %i ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon -ExecStart=/usr/bin/docker run -d --name %p --net=host \ +ExecStart=/usr/bin/docker run --rm --name %p --net=host \ -v /var/lib/ceph:/var/lib/ceph \ -v /etc/ceph:/etc/ceph \ --privileged \ @@ -16,9 +15,8 @@ ExecStart=/usr/bin/docker run -d --name %p --net=host \ -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \ -e MON_NAME={{ ansible_hostname }} \ --name={{ ansible_hostname }} \ - {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }} -ExecStopPost=-/usr/bin/docker stop %p -ExecStopPost=-/usr/bin/docker rm %p + {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }} +ExecStopPost=-/usr/bin/docker stop %i Restart=always RestartSec=10s TimeoutStartSec=120 diff --git a/roles/ceph-osd/tasks/docker/main.yml b/roles/ceph-osd/tasks/docker/main.yml index b0a2a27fd..2d1ba5b24 100644 --- a/roles/ceph-osd/tasks/docker/main.yml +++ b/roles/ceph-osd/tasks/docker/main.yml @@ -8,6 +8,14 @@ - include: checks.yml when: ceph_health.rc != 0 +- name: check if it is Atomic host + stat: path=/run/ostree-booted + register: stat_ostree + +- name: set fact for using Atomic host + set_fact: + is_atomic='{{ stat_ostree.stat.exists }}' + - include: pre_requisite.yml - include: selinux.yml when: ansible_os_family == 'RedHat' diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index 43a93f91d..265fe2df4 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -1,7 +1,7 @@ --- # (rootfs) for reasons I haven't figured out, docker pull and run will fail. - name: pull ceph daemon image - shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" + shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" - name: prepare ceph osd disk docker: @@ -16,6 +16,47 @@ with_items: ceph_osd_docker_devices when: ceph_osd_docker_prepare_env is defined +# Use systemd to manage container on Atomic host +- name: generate systemd unit file + sudo: true + config_template: + src: ceph-osd.service.j2 + dest: /var/lib/ceph/ceph-osd@.service + owner: "root" + group: "root" + mode: "0644" + config_overrides: {} + config_type: ini + failed_when: false + +- name: link systemd unit file for osd instance + file: + src: /var/lib/ceph/ceph-osd@.service + dest: /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service + state: link + with_items: ceph_osd_docker_devices + when: is_atomic + +- name: enable systemd unit file for osd instance + shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service + failed_when: false + changed_when: false + with_items: ceph_osd_docker_devices + when: is_atomic + +- name: reload systemd unit files + shell: systemctl daemon-reload + when: is_atomic + +- name: systemd start osd container + service: + name: ceph-osd@{{ item | basename }} + state: started + enabled: yes + changed_when: false + with_items: ceph_osd_docker_devices + when: is_atomic + - name: run the ceph osd docker image docker: image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" @@ -27,3 +68,4 @@ env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_extra_env }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev/:/dev/" with_items: ceph_osd_docker_devices + when: not is_atomic From 04f7b5923f8598bcaad2ffc26b6815ff2cfbf139 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 9 Feb 2016 19:49:22 +0000 Subject: [PATCH 14/32] review feedback Signed-off-by: Huamin Chen --- roles/ceph-mon/tasks/docker/start_docker_monitor.yml | 6 ++---- roles/ceph-osd/tasks/docker/start_docker_osd.yml | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 7fa861852..3dad4f710 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -13,15 +13,13 @@ # Use systemd to manage container on Atomic host - name: generate systemd unit file sudo: true - config_template: + template: src: ceph-mon.service.j2 dest: /var/lib/ceph/ceph-mon@.service owner: "root" group: "root" mode: "0644" - config_overrides: {} - config_type: ini - + - name: link systemd unit file for mon instance file: src: /var/lib/ceph/ceph-mon@.service diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index 265fe2df4..a19c0e220 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -19,14 +19,12 @@ # Use systemd to manage container on Atomic host - name: generate systemd unit file sudo: true - config_template: + template: src: ceph-osd.service.j2 dest: /var/lib/ceph/ceph-osd@.service owner: "root" group: "root" mode: "0644" - config_overrides: {} - config_type: ini failed_when: false - name: link systemd unit file for osd instance From 3b73b8f2d493bed5a18fba8574a5c9133e491501 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Wed, 10 Feb 2016 18:15:52 +0000 Subject: [PATCH 15/32] add missing osd systemd unit template Signed-off-by: Huamin Chen --- .../tasks/templates/ceph-osd.service.j2 | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 roles/ceph-osd/tasks/templates/ceph-osd.service.j2 diff --git a/roles/ceph-osd/tasks/templates/ceph-osd.service.j2 b/roles/ceph-osd/tasks/templates/ceph-osd.service.j2 new file mode 100644 index 000000000..03efbb144 --- /dev/null +++ b/roles/ceph-osd/tasks/templates/ceph-osd.service.j2 @@ -0,0 +1,25 @@ +[Unit] +Description=Ceph OSD +After=docker.service + +[Service] +EnvironmentFile=/etc/environment +ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i +ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}-osd-dev%i +ExecStart=/usr/bin/docker run --rm --net=host --pid=host\ + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + -v /dev:/dev \ + --privileged \ + -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \ + -e OSD_DEVICE=/dev/%i \ + --name={{ ansible_hostname }}-osd-dev%i \ + {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }} +ExecStop=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i +Restart=always +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=15 + +[Install] +WantedBy=multi-user.target \ No newline at end of file From d5f642c2068763cb662ea2de1b2e5373db6da9da Mon Sep 17 00:00:00 2001 From: Jim Curtis Date: Thu, 4 Feb 2016 12:24:56 -0800 Subject: [PATCH 16/32] Changes to allow ceph-ansible and vagrant to work on Openstack VMs --- Vagrantfile.openstack | 111 ++++++++++++++++++ Vagrantfile => Vagrantfile.sample | 0 group_vars/{all.docker => all.docker.sample} | 0 roles/ceph-mds/tasks/docker/pre_requisite.yml | 5 + .../tasks/docker/start_docker_monitor.yml | 1 + .../tasks/docker/start_docker_osd.yml | 6 + vagrant_variables.yml.openstack | 37 ++++++ 7 files changed, 160 insertions(+) create mode 100644 Vagrantfile.openstack rename Vagrantfile => Vagrantfile.sample (100%) rename group_vars/{all.docker => all.docker.sample} (100%) create mode 100644 vagrant_variables.yml.openstack diff --git a/Vagrantfile.openstack b/Vagrantfile.openstack new file mode 100644 index 000000000..52fa4e60b --- /dev/null +++ b/Vagrantfile.openstack @@ -0,0 +1,111 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'yaml' +require 'vagrant-openstack-provider' +VAGRANTFILE_API_VERSION = '2' + +config_file=File.expand_path(File.join(File.dirname(__FILE__), 'vagrant_variables.yml')) +settings=YAML.load_file(config_file) + +NMONS = settings['mon_vms'] +NOSDS = settings['osd_vms'] +NMDSS = settings['mds_vms'] +NRGWS = settings['rgw_vms'] +CLIENTS = settings['client_vms'] +SUBNET = settings['subnet'] +BOX = settings['vagrant_box'] +MEMORY = settings['memory'] +STORAGECTL = settings['vagrant_storagectl'] +ETH = settings['eth'] +USER = settings['os_ssh_username'] + +ansible_provision = proc do |ansible| + ansible.playbook = 'site.yml' + if settings['skip_tags'] + ansible.skip_tags = settings['skip_tags'] + end + + # Note: Can't do ranges like mon[0-2] in groups because + # these aren't supported by Vagrant, see + # https://github.com/mitchellh/vagrant/issues/3539 + ansible.groups = { + 'mons' => (0..NMONS - 1).map { |j| "mon#{j}" }, + 'restapis' => (0..NMONS - 1).map { |j| "mon#{j}" }, + 'osds' => (0..NOSDS - 1).map { |j| "osd#{j}" }, + 'mdss' => (0..NMDSS - 1).map { |j| "mds#{j}" }, + 'rgws' => (0..NRGWS - 1).map { |j| "rgw#{j}" }, + 'clients' => (0..CLIENTS - 1).map { |j| "client#{j}" } + } + + # In a production deployment, these should be secret + ansible.extra_vars = { + ceph_stable: 'true', + journal_collocation: 'true', + fsid: '4a158d27-f750-41d5-9e7f-26ce4c9d2d45', + monitor_secret: 'AQAWqilTCDh7CBAAawXt6kyTgLFCxSvJhTEmuw==', + journal_size: 100, + monitor_interface: ETH, + cluster_network: "#{SUBNET}.0/24", + public_network: "#{SUBNET}.0/24", + devices: settings['disks'], + os_tuning_params: settings['os_tuning_params'] + } + ansible.limit = 'all' +end + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.synced_folder ".", "/home/#{USER}/vagrant", disabled: true + config.vm.box = BOX + config.ssh.insert_key = false # workaround for https://github.com/mitchellh/vagrant/issues/5048 + + # OpenStack VMs + config.vm.provider :openstack do |os| + config.ssh.username = settings['os_ssh_username'] + config.ssh.private_key_path = settings['os_ssh_private_key_path'] + config.ssh.pty = true + os.openstack_auth_url = settings['os_openstack_auth_url'] + os.username = settings['os_username'] + os.password = settings['os_password'] + os.tenant_name = settings['os_tenant_name'] + os.region = settings['os_region'] + os.flavor = settings['os_flavor'] + os.image = settings['os_image'] + os.keypair_name = settings['os_keypair_name'] + os.security_groups = ['default'] + end + + config.vm.provision "shell", inline: "true", upload_path: "/home/#{USER}/vagrant-shell" + + (0..CLIENTS - 1).each do |i| + config.vm.define "client#{i}" do |client| + client.vm.hostname = "ceph-client#{i}" + end + end + + (0..NRGWS - 1).each do |i| + config.vm.define "rgw#{i}" do |rgw| + rgw.vm.hostname = "ceph-rgw#{i}" + end + end + + (0..NMDSS - 1).each do |i| + config.vm.define "mds#{i}" do |mds| + mds.vm.hostname = "ceph-mds#{i}" + end + end + + (0..NMONS - 1).each do |i| + config.vm.define "mon#{i}" do |mon| + mon.vm.hostname = "ceph-mon#{i}" + end + end + + (0..NOSDS - 1).each do |i| + config.vm.define "osd#{i}" do |osd| + osd.vm.hostname = "ceph-osd#{i}" + # Run the provisioner after the last machine comes up + osd.vm.provision 'ansible', &ansible_provision if i == (NOSDS - 1) + end + end +end diff --git a/Vagrantfile b/Vagrantfile.sample similarity index 100% rename from Vagrantfile rename to Vagrantfile.sample diff --git a/group_vars/all.docker b/group_vars/all.docker.sample similarity index 100% rename from group_vars/all.docker rename to group_vars/all.docker.sample diff --git a/roles/ceph-mds/tasks/docker/pre_requisite.yml b/roles/ceph-mds/tasks/docker/pre_requisite.yml index 8e40aa963..176bdf52b 100644 --- a/roles/ceph-mds/tasks/docker/pre_requisite.yml +++ b/roles/ceph-mds/tasks/docker/pre_requisite.yml @@ -50,6 +50,11 @@ ansible_os_family == 'RedHat' and ansible_pkg_mgr == "dnf" +- name: install six + pip: + name: six + version: 1.9.0 + # NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227 - name: install docker-py pip: diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 3dad4f710..8bc4a84c7 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -55,6 +55,7 @@ image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" name: "{{ ansible_hostname }}" net: "host" + pid: "host" state: "running" privileged: "{{ mon_docker_privileged }}" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index a19c0e220..7ce75e18e 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -1,4 +1,10 @@ --- +# For openstack VMs modify the mount point below depending on if the Openstack +# VM deploy tool defaults to mounting ephemeral disks +- name: umount ceph disk (if on openstack) + shell: "umount /mnt" + when: ceph_docker_on_openstack + # (rootfs) for reasons I haven't figured out, docker pull and run will fail. - name: pull ceph daemon image shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" diff --git a/vagrant_variables.yml.openstack b/vagrant_variables.yml.openstack new file mode 100644 index 000000000..64e4d2598 --- /dev/null +++ b/vagrant_variables.yml.openstack @@ -0,0 +1,37 @@ +--- + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 1 +mds_vms: 0 +rgw_vms: 0 +client_vms: 0 + +# SUBNET TO USE FOR THE VMS +# Use whatever private subnet your Openstack VMs are given +subnet: 172.17.72 + +# For Openstack VMs, the disk will depend on what you are allocated +disks: "[ '/dev/vdb' ]" + +# For Openstack VMs, the lan is usually eth0 +eth: 'eth0' + +# For Openstack VMs, choose the following box instead +vagrant_box: 'openstack' + +# For Atomic (RHEL or Cento) uncomment the line below +skip_tags: 'with_pkg' + +# For deploying on OpenStack VMs uncomment these vars and assign values. +# You can use env vars for the values if it makes sense. +#os_ssh_username : +#os_ssh_private_key_path : +#os_openstack_auth_url : +#os_username : +#os_password : +#os_tenant_name : +#os_region : +#os_flavor : +#os_image : +#os_keypair_name : From dae1bb072ebae4aad80b42a3146f18465aac5339 Mon Sep 17 00:00:00 2001 From: Jim Curtis Date: Tue, 9 Feb 2016 21:45:17 -0800 Subject: [PATCH 17/32] Review feedback fixes from PR #518 --- Vagrantfile.sample => Vagrantfile | 0 Vagrantfile.openstack | 111 ------------------ group_vars/all.docker.sample | 2 - roles/ceph-mds/tasks/docker/pre_requisite.yml | 3 + .../tasks/docker/start_docker_monitor.yml | 1 - .../tasks/docker/start_docker_osd.yml | 2 +- 6 files changed, 4 insertions(+), 115 deletions(-) rename Vagrantfile.sample => Vagrantfile (100%) delete mode 100644 Vagrantfile.openstack diff --git a/Vagrantfile.sample b/Vagrantfile similarity index 100% rename from Vagrantfile.sample rename to Vagrantfile diff --git a/Vagrantfile.openstack b/Vagrantfile.openstack deleted file mode 100644 index 52fa4e60b..000000000 --- a/Vagrantfile.openstack +++ /dev/null @@ -1,111 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -require 'yaml' -require 'vagrant-openstack-provider' -VAGRANTFILE_API_VERSION = '2' - -config_file=File.expand_path(File.join(File.dirname(__FILE__), 'vagrant_variables.yml')) -settings=YAML.load_file(config_file) - -NMONS = settings['mon_vms'] -NOSDS = settings['osd_vms'] -NMDSS = settings['mds_vms'] -NRGWS = settings['rgw_vms'] -CLIENTS = settings['client_vms'] -SUBNET = settings['subnet'] -BOX = settings['vagrant_box'] -MEMORY = settings['memory'] -STORAGECTL = settings['vagrant_storagectl'] -ETH = settings['eth'] -USER = settings['os_ssh_username'] - -ansible_provision = proc do |ansible| - ansible.playbook = 'site.yml' - if settings['skip_tags'] - ansible.skip_tags = settings['skip_tags'] - end - - # Note: Can't do ranges like mon[0-2] in groups because - # these aren't supported by Vagrant, see - # https://github.com/mitchellh/vagrant/issues/3539 - ansible.groups = { - 'mons' => (0..NMONS - 1).map { |j| "mon#{j}" }, - 'restapis' => (0..NMONS - 1).map { |j| "mon#{j}" }, - 'osds' => (0..NOSDS - 1).map { |j| "osd#{j}" }, - 'mdss' => (0..NMDSS - 1).map { |j| "mds#{j}" }, - 'rgws' => (0..NRGWS - 1).map { |j| "rgw#{j}" }, - 'clients' => (0..CLIENTS - 1).map { |j| "client#{j}" } - } - - # In a production deployment, these should be secret - ansible.extra_vars = { - ceph_stable: 'true', - journal_collocation: 'true', - fsid: '4a158d27-f750-41d5-9e7f-26ce4c9d2d45', - monitor_secret: 'AQAWqilTCDh7CBAAawXt6kyTgLFCxSvJhTEmuw==', - journal_size: 100, - monitor_interface: ETH, - cluster_network: "#{SUBNET}.0/24", - public_network: "#{SUBNET}.0/24", - devices: settings['disks'], - os_tuning_params: settings['os_tuning_params'] - } - ansible.limit = 'all' -end - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.synced_folder ".", "/home/#{USER}/vagrant", disabled: true - config.vm.box = BOX - config.ssh.insert_key = false # workaround for https://github.com/mitchellh/vagrant/issues/5048 - - # OpenStack VMs - config.vm.provider :openstack do |os| - config.ssh.username = settings['os_ssh_username'] - config.ssh.private_key_path = settings['os_ssh_private_key_path'] - config.ssh.pty = true - os.openstack_auth_url = settings['os_openstack_auth_url'] - os.username = settings['os_username'] - os.password = settings['os_password'] - os.tenant_name = settings['os_tenant_name'] - os.region = settings['os_region'] - os.flavor = settings['os_flavor'] - os.image = settings['os_image'] - os.keypair_name = settings['os_keypair_name'] - os.security_groups = ['default'] - end - - config.vm.provision "shell", inline: "true", upload_path: "/home/#{USER}/vagrant-shell" - - (0..CLIENTS - 1).each do |i| - config.vm.define "client#{i}" do |client| - client.vm.hostname = "ceph-client#{i}" - end - end - - (0..NRGWS - 1).each do |i| - config.vm.define "rgw#{i}" do |rgw| - rgw.vm.hostname = "ceph-rgw#{i}" - end - end - - (0..NMDSS - 1).each do |i| - config.vm.define "mds#{i}" do |mds| - mds.vm.hostname = "ceph-mds#{i}" - end - end - - (0..NMONS - 1).each do |i| - config.vm.define "mon#{i}" do |mon| - mon.vm.hostname = "ceph-mon#{i}" - end - end - - (0..NOSDS - 1).each do |i| - config.vm.define "osd#{i}" do |osd| - osd.vm.hostname = "ceph-osd#{i}" - # Run the provisioner after the last machine comes up - osd.vm.provision 'ansible', &ansible_provision if i == (NOSDS - 1) - end - end -end diff --git a/group_vars/all.docker.sample b/group_vars/all.docker.sample index 12ae37383..21fe01ea9 100644 --- a/group_vars/all.docker.sample +++ b/group_vars/all.docker.sample @@ -16,8 +16,6 @@ ceph_osd_docker_username: hchen ceph_osd_docker_imagename: rhceph ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE" # comma separated variables ceph_osd_docker_prepare_env: "CEPH_DAEMON=OSD_CEPH_DISK_PREPARE,OSD_FORCE_ZAP=1" # comma separated variables -ceph_osd_docker_devices: - - /dev/sdb rgw_containerized_deployment: true ceph_rgw_docker_username: hchen ceph_rgw_docker_imagename: rhceph diff --git a/roles/ceph-mds/tasks/docker/pre_requisite.yml b/roles/ceph-mds/tasks/docker/pre_requisite.yml index 176bdf52b..4b47e3cdf 100644 --- a/roles/ceph-mds/tasks/docker/pre_requisite.yml +++ b/roles/ceph-mds/tasks/docker/pre_requisite.yml @@ -50,6 +50,9 @@ ansible_os_family == 'RedHat' and ansible_pkg_mgr == "dnf" +# NOTE (jimcurtis): need at least version 1.9.0 of six or we get: +# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined + - name: install six pip: name: six diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 8bc4a84c7..3dad4f710 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -55,7 +55,6 @@ image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" name: "{{ ansible_hostname }}" net: "host" - pid: "host" state: "running" privileged: "{{ mon_docker_privileged }}" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index 7ce75e18e..a700a8ba6 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -2,7 +2,7 @@ # For openstack VMs modify the mount point below depending on if the Openstack # VM deploy tool defaults to mounting ephemeral disks - name: umount ceph disk (if on openstack) - shell: "umount /mnt" + mount: name=/mnt src=/dev/vdb fstype=ext3 state=unmounted when: ceph_docker_on_openstack # (rootfs) for reasons I haven't figured out, docker pull and run will fail. From b06229bafb14cc1e96183213e54875f35d165f88 Mon Sep 17 00:00:00 2001 From: Jim Curtis Date: Fri, 19 Feb 2016 11:46:33 -0800 Subject: [PATCH 18/32] review feedback --- README.md | 32 +++++++++++++++++++ .../tasks/docker/start_docker_osd.yml | 6 +++- vagrant_variables.yml.openstack | 6 ++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 29ca84723..a4831a17c 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,39 @@ VBoxManage storagectl `VBoxManage list vms |grep ceph-ansible_osd0|awk '{print $ then run `vagrant up --provider=virtualbox` again. +## Setup for Vagrant using OpenStack provider +Install the Vagrant plugin for the openstack provider: vagrant plugin install vagrant-openstack-provider. + +cp site.yml.sample site.yml +cp group_vars/all.docker.sample group_vars/all +cp vagrant_variables.yml.openstack vagrant_variables.yml +Edit vagrant_variables.yml: + Set mon_vms and osd_vms to the numbers you want. + If you are using an Atomic image, un-comment out the skip_tags line. + Un-comment the os_ lines. + Set os_ssh_username to 'centos' for Centos and 'cloud-user' for RHEL images. + Set os_ssh_private_key_path to '~/.ssh/id_rsa' + Set os_openstack_auth_url to the auth url of your Open Stack cloud + Set os_username and os_password to what you provided for Open Stack + registration or leave them as ENV vars if you have set the + corresponding env vars for your user. + Set os_tenant_name to your Open Stack cloud project name. + Set os_region to your Open Stack cloud region name. + Set os_flavor to 'm3.medium'. This size has ephemeral storage with will + be used by the OSD for the /dev/vdb disk + Set the os_image to an image found in the Images list in the Open Stack + cloud Dashboard (i.e. 'centos-atomic-host'). + Set the os_keypair_name to the keypair name you used when you did the + Open Stack registration. +vagrant up --provider=openstack +Once the playbook is finished, you should be able to do vagrant ssh mon0 or +vagrant ssh osd0 to get to the VMs. +sudo docker ps should show the running containers +When you are done, use vagrant destroy to get rid of the VMs. You should +also remove the associated entries in .ssh/known_hosts so that if the IP +addresses get reused by future Open Stack Cloud instances there will not be +old known_hosts entries. # Want to contribute? diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index a700a8ba6..919b98de9 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -2,7 +2,11 @@ # For openstack VMs modify the mount point below depending on if the Openstack # VM deploy tool defaults to mounting ephemeral disks - name: umount ceph disk (if on openstack) - mount: name=/mnt src=/dev/vdb fstype=ext3 state=unmounted + mount: + name: /mnt + src: /dev/vdb + fstype: ext3 + state: unmounted when: ceph_docker_on_openstack # (rootfs) for reasons I haven't figured out, docker pull and run will fail. diff --git a/vagrant_variables.yml.openstack b/vagrant_variables.yml.openstack index 64e4d2598..ddf01823c 100644 --- a/vagrant_variables.yml.openstack +++ b/vagrant_variables.yml.openstack @@ -21,15 +21,15 @@ eth: 'eth0' vagrant_box: 'openstack' # For Atomic (RHEL or Cento) uncomment the line below -skip_tags: 'with_pkg' +#skip_tags: 'with_pkg' # For deploying on OpenStack VMs uncomment these vars and assign values. # You can use env vars for the values if it makes sense. #os_ssh_username : #os_ssh_private_key_path : #os_openstack_auth_url : -#os_username : -#os_password : +#os_username : ENV['OS_USERNAME'] +#os_password : ENV['OS_PASSWORD'] #os_tenant_name : #os_region : #os_flavor : From bf4864dd72f754f1c3566def1058e7107cb5fd03 Mon Sep 17 00:00:00 2001 From: Jim Curtis Date: Sun, 21 Feb 2016 11:04:43 -0800 Subject: [PATCH 19/32] review feedback --- README.md | 49 ++++++++++--------- roles/ceph-mds/tasks/docker/pre_requisite.yml | 1 - 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index a4831a17c..809221244 100644 --- a/README.md +++ b/README.md @@ -200,34 +200,39 @@ then run `vagrant up --provider=virtualbox` again. ## Setup for Vagrant using OpenStack provider -Install the Vagrant plugin for the openstack provider: vagrant plugin install vagrant-openstack-provider. +Install the Vagrant plugin for the openstack provider: `vagrant plugin install vagrant-openstack-provider`. -cp site.yml.sample site.yml -cp group_vars/all.docker.sample group_vars/all -cp vagrant_variables.yml.openstack vagrant_variables.yml -Edit vagrant_variables.yml: - Set mon_vms and osd_vms to the numbers you want. - If you are using an Atomic image, un-comment out the skip_tags line. - Un-comment the os_ lines. - Set os_ssh_username to 'centos' for Centos and 'cloud-user' for RHEL images. - Set os_ssh_private_key_path to '~/.ssh/id_rsa' - Set os_openstack_auth_url to the auth url of your Open Stack cloud - Set os_username and os_password to what you provided for Open Stack +```bash +$ cp site.yml.sample site.yml +$ cp group_vars/all.docker.sample group_vars/all +$ cp vagrant_variables.yml.openstack vagrant_variables.yml +``` +* Edit `vagrant_variables.yml`: + Set `mon_vms` and `osd_vms` to the numbers you want. + If you are using an Atomic image, un-comment out the `skip_tags` line. + Un-comment the `os_` lines. + Set `os_ssh_username` to 'centos' for Centos and 'cloud-user' for + RHEL images. + Set `os_ssh_private_key_path` to '~/.ssh/id_rsa' + Set `os_openstack_auth_url` to the auth url of your Open Stack cloud + Set `os_username` and `os_password` to what you provided for Open Stack registration or leave them as ENV vars if you have set the corresponding env vars for your user. - Set os_tenant_name to your Open Stack cloud project name. - Set os_region to your Open Stack cloud region name. - Set os_flavor to 'm3.medium'. This size has ephemeral storage with will + Set `os_tenant_name` to your Open Stack cloud project name. + Set `os_region` to your Open Stack cloud region name. + Set `os_flavor` to 'm3.medium'. This size has ephemeral storage that will be used by the OSD for the /dev/vdb disk - Set the os_image to an image found in the Images list in the Open Stack + Set the `os_image` to an image found in the Images list in the Open Stack cloud Dashboard (i.e. 'centos-atomic-host'). - Set the os_keypair_name to the keypair name you used when you did the + Set the `os_keypair_name` to the keypair name you used when you did the Open Stack registration. -vagrant up --provider=openstack -Once the playbook is finished, you should be able to do vagrant ssh mon0 or -vagrant ssh osd0 to get to the VMs. -sudo docker ps should show the running containers -When you are done, use vagrant destroy to get rid of the VMs. You should +``` +$ vagrant up --provider=openstack +``` +Once the playbook is finished, you should be able to do `vagrant ssh mon0` or +`vagrant ssh osd0` to get to the VMs. +`sudo docker ps` should show the running containers +When you are done, use `vagrant destroy` to get rid of the VMs. You should also remove the associated entries in .ssh/known_hosts so that if the IP addresses get reused by future Open Stack Cloud instances there will not be old known_hosts entries. diff --git a/roles/ceph-mds/tasks/docker/pre_requisite.yml b/roles/ceph-mds/tasks/docker/pre_requisite.yml index 4b47e3cdf..69801ef77 100644 --- a/roles/ceph-mds/tasks/docker/pre_requisite.yml +++ b/roles/ceph-mds/tasks/docker/pre_requisite.yml @@ -52,7 +52,6 @@ # NOTE (jimcurtis): need at least version 1.9.0 of six or we get: # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined - - name: install six pip: name: six From 3e87da0f16ded93da7281f7983c9564ac250eca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 2 Mar 2016 14:52:00 +0100 Subject: [PATCH 20/32] docker: fix group_vars file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien Han --- group_vars/all.docker.sample | 70 ++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/group_vars/all.docker.sample b/group_vars/all.docker.sample index 21fe01ea9..c0d84362e 100644 --- a/group_vars/all.docker.sample +++ b/group_vars/all.docker.sample @@ -1,28 +1,52 @@ --- dummy: -cephx_require_signatures: false # Kernel RBD does NOT support signatures! -cephx_cluster_require_signatures: false -restapi_group_name: restapis -fetch_directory: fetch/ -mon_containerized_deployment: true -mon_docker_privileged: true -ceph_mon_docker_username: hchen -ceph_mon_docker_imagename: rhceph -ceph_mon_docker_interface: "{{ monitor_interface }}" -ceph_mon_docker_subnet: "{{ public_network }}" # subnet of the ceph_mon_docker_interface -ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables -osd_containerized_deployment: true -ceph_osd_docker_username: hchen -ceph_osd_docker_imagename: rhceph -ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE" # comma separated variables -ceph_osd_docker_prepare_env: "CEPH_DAEMON=OSD_CEPH_DISK_PREPARE,OSD_FORCE_ZAP=1" # comma separated variables -rgw_containerized_deployment: true -ceph_rgw_docker_username: hchen -ceph_rgw_docker_imagename: rhceph -ceph_rgw_civetweb_port: 80 -ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables -rbd_client_directories: false -ceph_stable_release: hammer +####### +# MON # +####### +#mon_containerized_deployment: true +#mon_docker_privileged: true +#ceph_mon_docker_username: ceph +#ceph_mon_docker_imagename: daemon +#ceph_mon_docker_interface: "{{ monitor_interface }}" +#ceph_mon_docker_subnet: "{{ public_network }}" # subnet of the ceph_mon_docker_interface +#ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables +####### +# OSD # +####### +#osd_containerized_deployment: true +#ceph_osd_docker_username: ceph +#ceph_osd_docker_imagename: daemon +#ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE" # comma separated variables +#ceph_osd_docker_prepare_env: "CEPH_DAEMON=OSD_CEPH_DISK_PREPARE,OSD_FORCE_ZAP=1" # comma separated variables +#ceph_osd_docker_devices: +# - /dev/sdb +# - /dev/sdc +####### +# MDS # +####### +#mds_containerized_deployment: false +#ceph_mds_docker_username: ceph +#ceph_mds_docker_imagename: daemon +#ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables + +####### +# RGW # +####### +#rgw_containerized_deployment: true +#ceph_rgw_docker_username: ceph +#ceph_rgw_docker_imagename: daemon +#ceph_rgw_civetweb_port: 80 +#ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables + +########### +# RESTAPI # +########### +#restapi_containerized_deployment: false +#ceph_restapi_docker_interface: eth0 +#ceph_restapi_port: 5000 +#ceph_restapi_docker_username: ceph +#ceph_restapi_docker_imagename: daemon +#ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables From 6b014d10ed5c0747660b70593570865cdb80ae80 Mon Sep 17 00:00:00 2001 From: pprokop Date: Mon, 14 Mar 2016 10:57:40 +0100 Subject: [PATCH 21/32] Adding role for deploying python on CoreOS --- roles/ceph-common-coreos/README.md | 44 +++++++++++++++++++ roles/ceph-common-coreos/defaults/main.yml | 7 +++ .../ceph-common-coreos/tasks/install_pip.yml | 30 +++++++++++++ .../ceph-common-coreos/tasks/install_pypy.yml | 33 ++++++++++++++ roles/ceph-common-coreos/tasks/main.yml | 16 +++++++ .../templates/install_pip.sh.j2 | 6 +++ .../templates/install_python.sh.j2 | 5 +++ 7 files changed, 141 insertions(+) create mode 100644 roles/ceph-common-coreos/README.md create mode 100644 roles/ceph-common-coreos/defaults/main.yml create mode 100644 roles/ceph-common-coreos/tasks/install_pip.yml create mode 100644 roles/ceph-common-coreos/tasks/install_pypy.yml create mode 100644 roles/ceph-common-coreos/tasks/main.yml create mode 100644 roles/ceph-common-coreos/templates/install_pip.sh.j2 create mode 100644 roles/ceph-common-coreos/templates/install_python.sh.j2 diff --git a/roles/ceph-common-coreos/README.md b/roles/ceph-common-coreos/README.md new file mode 100644 index 000000000..ae22a8b5d --- /dev/null +++ b/roles/ceph-common-coreos/README.md @@ -0,0 +1,44 @@ +# Ansible role: Ceph Storage Agent + +This role installs python and pip on CoreOS. + +# Requirements + +This role has to be run without gathering facts and with sudo attribute. + +# Role variables + +Have a look at: `defaults/main.yml`. + +## Mandatory variables + +None. + +# Dependencies + +New CoreOS releases support pypy in version 2.4 and above. Unfortunetly CoreOS stable channel +has to be used with pypy in version 2.4 and below due to some dependency issues. + +# Example Playbook + +``` +- hosts: servers + remote_user: core + sudo: True + gather_facts: false + roles: + - { role: ceph-common-coreos } +``` + +# Contribution + +**THIS REPOSITORY DOES NOT ACCEPT PULL REQUESTS** +**PULL REQUESTS MUST GO THROUGH [CEPH-ANSIBLE](https://github.com/ceph/ceph-ansible)** + +# License + +Apache + +# Author Information + +This role was created by Alfredo Deza. diff --git a/roles/ceph-common-coreos/defaults/main.yml b/roles/ceph-common-coreos/defaults/main.yml new file mode 100644 index 000000000..8b21fb830 --- /dev/null +++ b/roles/ceph-common-coreos/defaults/main.yml @@ -0,0 +1,7 @@ +coreos_pypy_version: 4.0.1 +coreos_pypy_arch: linux64 +coreos_pypy_url: https://bitbucket.org/pypy/pypy/downloads/pypy-{{coreos_pypy_version}}-{{coreos_pypy_arch}}.tar.bz2 +pypy_directory: /opt/pypy +pypy_binary_directory: /opt/bin +pip_url: https://bootstrap.pypa.io/get-pip.py +local_temp_directory: /tmp \ No newline at end of file diff --git a/roles/ceph-common-coreos/tasks/install_pip.yml b/roles/ceph-common-coreos/tasks/install_pip.yml new file mode 100644 index 000000000..2108f298f --- /dev/null +++ b/roles/ceph-common-coreos/tasks/install_pip.yml @@ -0,0 +1,30 @@ +--- +- name: download get_pip.py + raw: cd $HOME && wget {{pip_url}} + +- name: run get-pip.py + raw: "{{pypy_binary_directory}}/python $HOME/get-pip.py" + +- name: create local temp directory + local_action: raw mkdir -p {{local_temp_directory}} + sudo: no + +- name: prepare install_pip.sh + local_action: template src=install_pip.sh.j2 dest={{local_temp_directory}}/install_pip.sh + sudo: no + +- name: run pip.sh + script: "{{local_temp_directory}}/install_pip.sh" + +- name: add execute permission + raw: chmod a+x {{pypy_directory}}/pip + +- name: move python to binary directory + raw: mv {{pypy_directory}}/pip {{pypy_binary_directory}}/pip + +- name: create .pip + raw: touch $HOME/.pip + +- name: remove pip.sh + local_action: file path="{{local_temp_directory}}/pip.sh" state=absent + sudo: no \ No newline at end of file diff --git a/roles/ceph-common-coreos/tasks/install_pypy.yml b/roles/ceph-common-coreos/tasks/install_pypy.yml new file mode 100644 index 000000000..26d52c775 --- /dev/null +++ b/roles/ceph-common-coreos/tasks/install_pypy.yml @@ -0,0 +1,33 @@ +--- +- name: download python + raw: cd $HOME && wget -O - {{coreos_pypy_url}} |tar -xjf - + +- name: move pypy to pypy_install_directory + raw: mv $HOME/pypy-{{coreos_pypy_version}}-{{coreos_pypy_arch}} {{pypy_directory}} + +- name: create local temp directory + local_action: raw mkdir -p {{local_temp_directory}} + sudo: no + +- name: prepare python executable + local_action: template src=install_python.sh.j2 dest={{local_temp_directory}}/install_python.sh + sudo: no + +- name: fix library + raw: ln -s /lib64/libncurses.so.5.9 {{pypy_directory}}/lib_pypy/libtinfo.so.5 + +- name: run install_python.sh + script: "{{local_temp_directory}}/install_python.sh" + +- name: add execute permission + raw: chmod a+x {{pypy_directory}}/python + +- name: move python to binary directory + raw: mv {{pypy_directory}}/python {{pypy_binary_directory}}/python + +- name: create .python + raw: touch $HOME/.python + +- name: remove install_python.sh + local_action: file path="{{local_temp_directory}}/install_python.sh" state=absent + sudo: no \ No newline at end of file diff --git a/roles/ceph-common-coreos/tasks/main.yml b/roles/ceph-common-coreos/tasks/main.yml new file mode 100644 index 000000000..4f3404a14 --- /dev/null +++ b/roles/ceph-common-coreos/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: check if there is python + raw: stat $HOME/.python + register: need_python + ignore_errors: true + +- include: install_pypy.yml + when: need_python | failed + +- name: check if there is pip + raw: stat $HOME/.pip + register: need_pip + ignore_errors: true + +- include: install_pip.yml + when: need_pip | failed and need_python | failed \ No newline at end of file diff --git a/roles/ceph-common-coreos/templates/install_pip.sh.j2 b/roles/ceph-common-coreos/templates/install_pip.sh.j2 new file mode 100644 index 000000000..af7bc59f0 --- /dev/null +++ b/roles/ceph-common-coreos/templates/install_pip.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash + + +cat > {{pypy_directory}}/pip << EOF +#!/bin/bash +LD_LIBRARY_PATH={{pypy_directory}}/lib_pypy:$LD_LIBRARY_PATH exec {{pypy_directory}}/bin/pip "\$@"\ \ No newline at end of file diff --git a/roles/ceph-common-coreos/templates/install_python.sh.j2 b/roles/ceph-common-coreos/templates/install_python.sh.j2 new file mode 100644 index 000000000..04c14f0fa --- /dev/null +++ b/roles/ceph-common-coreos/templates/install_python.sh.j2 @@ -0,0 +1,5 @@ +#!/bin/bash + +cat > {{pypy_directory}}/python << EOF +#!/bin/bash +LD_LIBRARY_PATH={{pypy_directory}}/lib_pypy:$LD_LIBRARY_PATH exec {{pypy_directory}}/bin/pypy "\$@"\ From 3e935a27764e50ed3115b159c97ab1a7fab674bd Mon Sep 17 00:00:00 2001 From: pprokop Date: Mon, 14 Mar 2016 17:51:26 +0100 Subject: [PATCH 22/32] Removing extra spaces --- roles/ceph-common-coreos/tasks/install_pypy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/ceph-common-coreos/tasks/install_pypy.yml b/roles/ceph-common-coreos/tasks/install_pypy.yml index 26d52c775..fb0ad0c3c 100644 --- a/roles/ceph-common-coreos/tasks/install_pypy.yml +++ b/roles/ceph-common-coreos/tasks/install_pypy.yml @@ -14,20 +14,20 @@ sudo: no - name: fix library - raw: ln -s /lib64/libncurses.so.5.9 {{pypy_directory}}/lib_pypy/libtinfo.so.5 + raw: ln -s /lib64/libncurses.so.5.9 {{pypy_directory}}/lib_pypy/libtinfo.so.5 - name: run install_python.sh script: "{{local_temp_directory}}/install_python.sh" - name: add execute permission - raw: chmod a+x {{pypy_directory}}/python + raw: chmod a+x {{pypy_directory}}/python - name: move python to binary directory - raw: mv {{pypy_directory}}/python {{pypy_binary_directory}}/python + raw: mv {{pypy_directory}}/python {{pypy_binary_directory}}/python - name: create .python raw: touch $HOME/.python - name: remove install_python.sh local_action: file path="{{local_temp_directory}}/install_python.sh" state=absent - sudo: no \ No newline at end of file + sudo: no From b9b04a4cfff97987e041551652af9a3df027ea6d Mon Sep 17 00:00:00 2001 From: pprokop Date: Mon, 14 Mar 2016 17:53:18 +0100 Subject: [PATCH 23/32] Removing extra spaces in get_pip --- roles/ceph-common-coreos/tasks/install_pip.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/ceph-common-coreos/tasks/install_pip.yml b/roles/ceph-common-coreos/tasks/install_pip.yml index 2108f298f..bac7d6975 100644 --- a/roles/ceph-common-coreos/tasks/install_pip.yml +++ b/roles/ceph-common-coreos/tasks/install_pip.yml @@ -17,14 +17,14 @@ script: "{{local_temp_directory}}/install_pip.sh" - name: add execute permission - raw: chmod a+x {{pypy_directory}}/pip + raw: chmod a+x {{pypy_directory}}/pip - name: move python to binary directory - raw: mv {{pypy_directory}}/pip {{pypy_binary_directory}}/pip + raw: mv {{pypy_directory}}/pip {{pypy_binary_directory}}/pip - name: create .pip raw: touch $HOME/.pip - name: remove pip.sh local_action: file path="{{local_temp_directory}}/pip.sh" state=absent - sudo: no \ No newline at end of file + sudo: no From 613f7dbc6b7e5d16dbde3a0706fc1a28112879f9 Mon Sep 17 00:00:00 2001 From: pprokop Date: Tue, 15 Mar 2016 09:33:43 +0100 Subject: [PATCH 24/32] Changed creator name --- roles/ceph-common-coreos/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ceph-common-coreos/README.md b/roles/ceph-common-coreos/README.md index ae22a8b5d..297650853 100644 --- a/roles/ceph-common-coreos/README.md +++ b/roles/ceph-common-coreos/README.md @@ -41,4 +41,4 @@ Apache # Author Information -This role was created by Alfredo Deza. +This role was created by Piotr Prokop. From b68b9506f2ec1a96b47acdf3cc435dfa595986bb Mon Sep 17 00:00:00 2001 From: pprokop Date: Tue, 15 Mar 2016 12:48:22 +0100 Subject: [PATCH 25/32] Adding common-coreos.sample --- group_vars/common-coreoss.sample | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 group_vars/common-coreoss.sample diff --git a/group_vars/common-coreoss.sample b/group_vars/common-coreoss.sample new file mode 100644 index 000000000..4cfc5bfa3 --- /dev/null +++ b/group_vars/common-coreoss.sample @@ -0,0 +1,16 @@ +--- +# Variables here are applicable to all host groups NOT roles + +# This sample file generated by generate-group-vars.sh + +# Dummy variable to avoid error because ansible does not recognize the +# file as a good configuration file when no variable in it. +dummy: + +#coreos_pypy_version: 4.0.1 +#coreos_pypy_arch: linux64 +#coreos_pypy_url: https://bitbucket.org/pypy/pypy/downloads/pypy-{{coreos_pypy_version}}-{{coreos_pypy_arch}}.tar.bz2 +#pypy_directory: /opt/pypy +#pypy_binary_directory: /opt/bin +#pip_url: https://bootstrap.pypa.io/get-pip.py +#local_temp_directory: /tmp From 42e748a51479e6324d5cd0b2b27a4bd9bc2a5346 Mon Sep 17 00:00:00 2001 From: pprokop Date: Tue, 15 Mar 2016 17:32:17 +0100 Subject: [PATCH 26/32] Adding dockerized ceph-mon deployment with kv backend --- group_vars/all.docker.sample | 4 ++ roles/ceph-common/templates/ceph.conf.j2 | 3 +- roles/ceph-mon/defaults/main.yml | 4 ++ roles/ceph-mon/tasks/docker/main.yml | 10 ++++- .../tasks/docker/start_docker_monitor.yml | 41 +++++++++++++++---- .../{tasks => }/templates/ceph-mon.service.j2 | 9 +++- 6 files changed, 57 insertions(+), 14 deletions(-) rename roles/ceph-mon/{tasks => }/templates/ceph-mon.service.j2 (72%) diff --git a/group_vars/all.docker.sample b/group_vars/all.docker.sample index c0d84362e..fce0c22a9 100644 --- a/group_vars/all.docker.sample +++ b/group_vars/all.docker.sample @@ -5,6 +5,10 @@ dummy: # MON # ####### #mon_containerized_deployment: true +#mon_containerized_deployment_with_kv: false +#kv_type: etcd +#kv_endpoint: 127.0.0.1 +#kv_port: 4001 #mon_docker_privileged: true #ceph_mon_docker_username: ceph #ceph_mon_docker_imagename: daemon diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2 index 63f8140a4..c77e127a9 100644 --- a/roles/ceph-common/templates/ceph.conf.j2 +++ b/roles/ceph-common/templates/ceph.conf.j2 @@ -15,7 +15,6 @@ auth service required = none auth client required = none auth supported = none {% endif %} -fsid = {{ fsid }} max open files = {{ max_open_files }} osd pool default pg num = {{ pool_default_pg_num }} osd pool default pgp num = {{ pool_default_pgp_num }} @@ -88,6 +87,7 @@ debug mon = {{ debug_mon_level }} debug paxos = {{ debug_mon_level }} debug auth = {{ debug_mon_level }} {% endif %} +{% if not mon_containerized_deployment_with_kv %} {% for host in groups[mon_group_name] %} {% if hostvars[host]['ansible_fqdn'] is defined and mon_use_fqdn %} [mon.{{ hostvars[host]['ansible_fqdn'] }}] @@ -102,6 +102,7 @@ host = {{ hostvars[host]['ansible_hostname'] }} {% include 'mon_addr_address.j2' %} {% endif %} {% endfor %} +{% endif %} [osd] osd mkfs type = {{ osd_mkfs_type }} diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml index ff638c18f..abd30e505 100644 --- a/roles/ceph-mon/defaults/main.yml +++ b/roles/ceph-mon/defaults/main.yml @@ -63,8 +63,12 @@ openstack_keys: ########## mon_containerized_deployment: false +mon_containerized_deployment_with_kv: false +mon_containerized_default_ceph_conf_with_kv: false ceph_mon_docker_interface: eth0 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface ceph_mon_docker_username: ceph ceph_mon_docker_imagename: daemon ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables +ceph_docker_on_openstack: false +mon_docker_privileged: true diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index c5d82070f..e59dbba11 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -14,15 +14,21 @@ is_atomic='{{ stat_ostree.stat.exists }}' - include: checks.yml - when: ceph_health.rc != 0 + when: ceph_health.rc != 0 and not mon_containerized_deployment_with_kv - include: pre_requisite.yml + - include: selinux.yml when: ansible_os_family == 'RedHat' # let the first mon create configs and keyrings - include: create_configs.yml when: inventory_hostname == groups.mons[0] + - include: fetch_configs.yml + when: not mon_containerized_deployment_with_kv + - include: start_docker_monitor.yml -- include: ../ceph_keys.yml \ No newline at end of file + +- include: ../ceph_keys.yml + when: not mon_containerized_deployment_with_kv \ No newline at end of file diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 3dad4f710..3fa696f59 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -6,11 +6,33 @@ - /var/lib/ceph/bootstrap-osd/ceph.keyring - /var/lib/ceph/bootstrap-rgw/ceph.keyring - /var/lib/ceph/bootstrap-mds/ceph.keyring - + when: not mon_containerized_deployment_with_kv + - name: pull ceph daemon image shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" -# Use systemd to manage container on Atomic host +- name: populate kv_store + docker: + name: populate-kv-store + image: ceph/daemon + command: populate_kvstore + net: host + env: + KV_TYPE: "{{kv_type}}" + KV_IP: "{{kv_endpoint}}" + KV_PORT: "{{kv_port}}" + volumes: + - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults + run_once: true + when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv + +- name: delete populate-kv-store docker + docker: + name: populate-kv-store + state: absent + image: ceph/daemon + +# Use systemd to manage container on Atomic host and CoreOS - name: generate systemd unit file sudo: true template: @@ -19,23 +41,23 @@ owner: "root" group: "root" mode: "0644" - + - name: link systemd unit file for mon instance file: src: /var/lib/ceph/ceph-mon@.service dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service state: link - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: enable systemd unit file for mon instance shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service failed_when: false changed_when: false - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: reload systemd unit files shell: systemctl daemon-reload - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: systemd start mon container service: @@ -43,7 +65,7 @@ state: started enabled: yes changed_when: false - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: wait for ceph.conf exists wait_for: @@ -59,7 +81,7 @@ privileged: "{{ mon_docker_privileged }}" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" - when: not is_atomic + when: not is_atomic and ansible_os_family != 'CoreOS' - name: stat for ceph config and keys stat: path="{{ item }}" @@ -67,6 +89,7 @@ changed_when: false failed_when: false register: statmonconfig + when: not mon_containerized_deployment_with_kv - name: fetch boostrap keys and conf from mon fetch: @@ -76,5 +99,5 @@ with_together: - ceph_bootstrap_config_keys - statmonconfig.results - when: item.1.stat.exists == true + when: not mon_containerized_deployment_with_kv and item.1.stat.exists == true and inventory_hostname == groups.mons[0] diff --git a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2 similarity index 72% rename from roles/ceph-mon/tasks/templates/ceph-mon.service.j2 rename to roles/ceph-mon/templates/ceph-mon.service.j2 index 484b2e3a1..76f04226e 100644 --- a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/templates/ceph-mon.service.j2 @@ -3,12 +3,17 @@ Description=Ceph Monitor After=docker.service [Service] -EnvironmentFile=/etc/environment +EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker rm %i ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon -ExecStart=/usr/bin/docker run --rm --name %p --net=host \ +ExecStart=/usr/bin/docker run --rm --name %i --net=host \ + {% if not mon_containerized_deployment_with_kv -%} -v /var/lib/ceph:/var/lib/ceph \ -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}}\ + {% endif -%} --privileged \ -e CEPH_DAEMON=MON \ -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \ From 6fea8b17b6dd48491ea88798b4dff80ad7dfa1f8 Mon Sep 17 00:00:00 2001 From: pprokop Date: Tue, 15 Mar 2016 17:36:11 +0100 Subject: [PATCH 27/32] Adding missing fsid --- roles/ceph-common/templates/ceph.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2 index c77e127a9..3e93caf82 100644 --- a/roles/ceph-common/templates/ceph.conf.j2 +++ b/roles/ceph-common/templates/ceph.conf.j2 @@ -15,6 +15,7 @@ auth service required = none auth client required = none auth supported = none {% endif %} +fsid = {{ fsid }} max open files = {{ max_open_files }} osd pool default pg num = {{ pool_default_pg_num }} osd pool default pgp num = {{ pool_default_pgp_num }} From 72fa8da1dc2197fb985160867809f17e7fd9335f Mon Sep 17 00:00:00 2001 From: pprokop Date: Wed, 16 Mar 2016 11:46:06 +0100 Subject: [PATCH 28/32] Removing fsid with kv backend deployment --- roles/ceph-common/templates/ceph.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2 index 3e93caf82..cb542efb3 100644 --- a/roles/ceph-common/templates/ceph.conf.j2 +++ b/roles/ceph-common/templates/ceph.conf.j2 @@ -15,7 +15,7 @@ auth service required = none auth client required = none auth supported = none {% endif %} -fsid = {{ fsid }} +{% if not mon_containerized_deployment_with_kv%}fsid = {{ fsid }}{% endif %} max open files = {{ max_open_files }} osd pool default pg num = {{ pool_default_pg_num }} osd pool default pgp num = {{ pool_default_pgp_num }} From 832c3bedbadeb9ca5f96445a9a2b03b02238d408 Mon Sep 17 00:00:00 2001 From: pprokop Date: Wed, 16 Mar 2016 12:45:24 +0100 Subject: [PATCH 29/32] Adding comments --- roles/ceph-common/defaults/main.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml index 1a66d3c46..b810aae1b 100644 --- a/roles/ceph-common/defaults/main.yml +++ b/roles/ceph-common/defaults/main.yml @@ -42,7 +42,6 @@ centos_package_dependencies: - epel-release - ntp - python-setuptools - - libselinux-python redhat_package_dependencies: - python-pycurl @@ -183,10 +182,7 @@ rbd_default_format: 2 ## Monitor options # -# You must define either monitor_interface or monitor_address. Preference -# will go to monitor_interface if both are defined. monitor_interface: interface -monitor_address: 0.0.0.0 mon_use_fqdn: false # if set to true, the MON name used will be the fqdn in the ceph.conf mon_osd_down_out_interval: 600 mon_osd_min_down_reporters: 7 # number of OSDs per host + 1 @@ -281,7 +277,6 @@ email_address: foo@bar.com ## REST API options # restapi_interface: "{{ monitor_interface }}" -restapi_address: "{{ monitor_address }}" restapi_port: 5000 restapi_base_url: /api/v0.1 restapi_log_level: warning # available level are: critical, error, warning, info, debug From eef421716d969f05732261cd5486734041fc28a5 Mon Sep 17 00:00:00 2001 From: pprokop Date: Mon, 21 Mar 2016 09:42:46 +0100 Subject: [PATCH 30/32] Adding var to choose between custom and default ceph.conf populated into etcd --- group_vars/all.docker.sample | 1 + roles/ceph-mon/tasks/docker/main.yml | 2 +- .../tasks/docker/start_docker_monitor.yml | 17 ++++++++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/group_vars/all.docker.sample b/group_vars/all.docker.sample index fce0c22a9..0f8b52700 100644 --- a/group_vars/all.docker.sample +++ b/group_vars/all.docker.sample @@ -6,6 +6,7 @@ dummy: ####### #mon_containerized_deployment: true #mon_containerized_deployment_with_kv: false +#mon_containerized_default_ceph_conf_with_kv: true #kv_type: etcd #kv_endpoint: 127.0.0.1 #kv_port: 4001 diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index e59dbba11..469cf9357 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -23,7 +23,7 @@ # let the first mon create configs and keyrings - include: create_configs.yml - when: inventory_hostname == groups.mons[0] + when: inventory_hostname == groups.mons[0] and not mon_containerized_default_ceph_conf_with_kv - include: fetch_configs.yml when: not mon_containerized_deployment_with_kv diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 3fa696f59..1b5602cf5 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -11,7 +11,21 @@ - name: pull ceph daemon image shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" -- name: populate kv_store +- name: populate kv_store with default ceph.conf + docker: + name: populate-kv-store + image: ceph/daemon + command: populate_kvstore + net: host + env: + KV_TYPE: "{{kv_type}}" + KV_IP: "{{kv_endpoint}}" + KV_PORT: "{{kv_port}}" + run_once: true + when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv + and mon_containerized_default_ceph_conf_with_kv + +- name: populate kv_store with custom ceph.conf docker: name: populate-kv-store image: ceph/daemon @@ -25,6 +39,7 @@ - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults run_once: true when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv + and not mon_containerized_default_ceph_conf_with_kv - name: delete populate-kv-store docker docker: From 77bd4aff3765d5dc8694ad64c208e329c04bf407 Mon Sep 17 00:00:00 2001 From: pprokop Date: Mon, 21 Mar 2016 10:50:17 +0100 Subject: [PATCH 31/32] Adding missing space --- roles/ceph-common/templates/ceph.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2 index cb542efb3..81e46edc3 100644 --- a/roles/ceph-common/templates/ceph.conf.j2 +++ b/roles/ceph-common/templates/ceph.conf.j2 @@ -15,7 +15,7 @@ auth service required = none auth client required = none auth supported = none {% endif %} -{% if not mon_containerized_deployment_with_kv%}fsid = {{ fsid }}{% endif %} +{% if not mon_containerized_deployment_with_kv %}fsid = {{ fsid }}{% endif %} max open files = {{ max_open_files }} osd pool default pg num = {{ pool_default_pg_num }} osd pool default pgp num = {{ pool_default_pgp_num }} From b0f56590e0e4cde421031650961a7770b6712872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 22 Mar 2016 16:58:20 +0100 Subject: [PATCH 32/32] docker: fix tons of issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien Han --- group_vars/all.sample | 1 + group_vars/common-coreoss.sample | 3 +- group_vars/mdss.sample | 1 + group_vars/osds.sample | 1 + group_vars/restapis.sample | 1 + group_vars/rgws.sample | 1 + roles/ceph-common-coreos/README.md | 2 +- .../ceph-common-coreos/tasks/install_pip.yml | 6 +- .../ceph-common-coreos/tasks/install_pypy.yml | 6 +- roles/ceph-common/defaults/main.yml | 5 ++ .../tasks/installs/install_on_redhat.yml | 16 +++-- roles/ceph-mds/defaults/main.yml | 1 + roles/ceph-mon/tasks/ceph_keys.yml | 13 ---- roles/ceph-mon/tasks/docker/copy_configs.yml | 10 +++ roles/ceph-mon/tasks/docker/main.yml | 20 ++++-- .../tasks/docker/start_docker_monitor.yml | 64 ++++++++----------- roles/ceph-osd/defaults/main.yml | 1 + .../tasks/docker/start_docker_osd.yml | 6 +- roles/ceph-restapi/defaults/main.yml | 1 + .../tasks/docker/fetch_configs.yml | 2 +- roles/ceph-rgw/defaults/main.yml | 1 + roles/ceph-rgw/tasks/docker/fetch_configs.yml | 2 +- 22 files changed, 91 insertions(+), 73 deletions(-) create mode 100644 roles/ceph-mon/tasks/docker/copy_configs.yml diff --git a/group_vars/all.sample b/group_vars/all.sample index 4284df595..48375f711 100644 --- a/group_vars/all.sample +++ b/group_vars/all.sample @@ -337,3 +337,4 @@ dummy: ########## #docker: false + diff --git a/group_vars/common-coreoss.sample b/group_vars/common-coreoss.sample index 4cfc5bfa3..32ae29553 100644 --- a/group_vars/common-coreoss.sample +++ b/group_vars/common-coreoss.sample @@ -1,7 +1,7 @@ --- # Variables here are applicable to all host groups NOT roles -# This sample file generated by generate-group-vars.sh +# This sample file generated by generate_group_vars_sample.sh # Dummy variable to avoid error because ansible does not recognize the # file as a good configuration file when no variable in it. @@ -14,3 +14,4 @@ dummy: #pypy_binary_directory: /opt/bin #pip_url: https://bootstrap.pypa.io/get-pip.py #local_temp_directory: /tmp + diff --git a/group_vars/mdss.sample b/group_vars/mdss.sample index 3cb0d0e2e..aa7729bdf 100644 --- a/group_vars/mdss.sample +++ b/group_vars/mdss.sample @@ -32,4 +32,5 @@ dummy: #ceph_mds_docker_username: ceph #ceph_mds_docker_imagename: daemon #ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables +#ceph_docker_on_openstack: false diff --git a/group_vars/osds.sample b/group_vars/osds.sample index fb60f230c..11f0caee1 100644 --- a/group_vars/osds.sample +++ b/group_vars/osds.sample @@ -128,4 +128,5 @@ dummy: #ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables #ceph_osd_docker_devices: # - /dev/sdb +#ceph_docker_on_openstack: false diff --git a/group_vars/restapis.sample b/group_vars/restapis.sample index f87d0a85f..e58bfbb31 100644 --- a/group_vars/restapis.sample +++ b/group_vars/restapis.sample @@ -24,4 +24,5 @@ dummy: #ceph_restapi_docker_username: ceph #ceph_restapi_docker_imagename: daemon #ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables +#ceph_docker_on_openstack: false diff --git a/group_vars/rgws.sample b/group_vars/rgws.sample index 6c4adc9b1..f7d6425c6 100644 --- a/group_vars/rgws.sample +++ b/group_vars/rgws.sample @@ -39,4 +39,5 @@ dummy: #ceph_rgw_docker_username: ceph #ceph_rgw_docker_imagename: daemon #ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables +#ceph_docker_on_openstack: false diff --git a/roles/ceph-common-coreos/README.md b/roles/ceph-common-coreos/README.md index 297650853..8574a6f2d 100644 --- a/roles/ceph-common-coreos/README.md +++ b/roles/ceph-common-coreos/README.md @@ -24,7 +24,7 @@ has to be used with pypy in version 2.4 and below due to some dependency issues. ``` - hosts: servers remote_user: core - sudo: True + become: True gather_facts: false roles: - { role: ceph-common-coreos } diff --git a/roles/ceph-common-coreos/tasks/install_pip.yml b/roles/ceph-common-coreos/tasks/install_pip.yml index bac7d6975..366d93f85 100644 --- a/roles/ceph-common-coreos/tasks/install_pip.yml +++ b/roles/ceph-common-coreos/tasks/install_pip.yml @@ -7,11 +7,11 @@ - name: create local temp directory local_action: raw mkdir -p {{local_temp_directory}} - sudo: no + become: no - name: prepare install_pip.sh local_action: template src=install_pip.sh.j2 dest={{local_temp_directory}}/install_pip.sh - sudo: no + become: no - name: run pip.sh script: "{{local_temp_directory}}/install_pip.sh" @@ -27,4 +27,4 @@ - name: remove pip.sh local_action: file path="{{local_temp_directory}}/pip.sh" state=absent - sudo: no + become: no diff --git a/roles/ceph-common-coreos/tasks/install_pypy.yml b/roles/ceph-common-coreos/tasks/install_pypy.yml index fb0ad0c3c..a441101a9 100644 --- a/roles/ceph-common-coreos/tasks/install_pypy.yml +++ b/roles/ceph-common-coreos/tasks/install_pypy.yml @@ -7,11 +7,11 @@ - name: create local temp directory local_action: raw mkdir -p {{local_temp_directory}} - sudo: no + become: no - name: prepare python executable local_action: template src=install_python.sh.j2 dest={{local_temp_directory}}/install_python.sh - sudo: no + become: no - name: fix library raw: ln -s /lib64/libncurses.so.5.9 {{pypy_directory}}/lib_pypy/libtinfo.so.5 @@ -30,4 +30,4 @@ - name: remove install_python.sh local_action: file path="{{local_temp_directory}}/install_python.sh" state=absent - sudo: no + become: no diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml index b810aae1b..1a66d3c46 100644 --- a/roles/ceph-common/defaults/main.yml +++ b/roles/ceph-common/defaults/main.yml @@ -42,6 +42,7 @@ centos_package_dependencies: - epel-release - ntp - python-setuptools + - libselinux-python redhat_package_dependencies: - python-pycurl @@ -182,7 +183,10 @@ rbd_default_format: 2 ## Monitor options # +# You must define either monitor_interface or monitor_address. Preference +# will go to monitor_interface if both are defined. monitor_interface: interface +monitor_address: 0.0.0.0 mon_use_fqdn: false # if set to true, the MON name used will be the fqdn in the ceph.conf mon_osd_down_out_interval: 600 mon_osd_min_down_reporters: 7 # number of OSDs per host + 1 @@ -277,6 +281,7 @@ email_address: foo@bar.com ## REST API options # restapi_interface: "{{ monitor_interface }}" +restapi_address: "{{ monitor_address }}" restapi_port: 5000 restapi_base_url: /api/v0.1 restapi_log_level: warning # available level are: critical, error, warning, info, debug diff --git a/roles/ceph-common/tasks/installs/install_on_redhat.yml b/roles/ceph-common/tasks/installs/install_on_redhat.yml index 670e529ef..da0ce967e 100644 --- a/roles/ceph-common/tasks/installs/install_on_redhat.yml +++ b/roles/ceph-common/tasks/installs/install_on_redhat.yml @@ -46,7 +46,9 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and mon_group_name in group_names and - ansible_pkg_mgr == "yum" + ansible_pkg_mgr == "yum" and + ceph_stable and + ceph_stable_release not in ceph_stable_releases - name: install distro or red hat storage ceph mon dnf: @@ -58,7 +60,9 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and mon_group_name in group_names and - ansible_pkg_mgr == "dnf" + ansible_pkg_mgr == "dnf" and + ceph_stable and + ceph_stable_release not in ceph_stable_releases - name: install distro or red hat storage ceph osd yum: @@ -70,7 +74,9 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and osd_group_name in group_names and - ansible_pkg_mgr == "yum" + ansible_pkg_mgr == "yum" and + ceph_stable and + ceph_stable_release not in ceph_stable_releases - name: install distro or red hat storage ceph osd dnf: @@ -82,7 +88,9 @@ when: (ceph_origin == "distro" or ceph_stable_rh_storage) and osd_group_name in group_names and - ansible_pkg_mgr == "dnf" + ansible_pkg_mgr == "dnf" and + ceph_stable and + ceph_stable_release not in ceph_stable_releases - name: install ceph-test yum: diff --git a/roles/ceph-mds/defaults/main.yml b/roles/ceph-mds/defaults/main.yml index 2df43d9dd..9229d2a2d 100644 --- a/roles/ceph-mds/defaults/main.yml +++ b/roles/ceph-mds/defaults/main.yml @@ -24,3 +24,4 @@ mds_containerized_deployment: false ceph_mds_docker_username: ceph ceph_mds_docker_imagename: daemon ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables +ceph_docker_on_openstack: false diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml index c9ddc6854..d28d8776b 100644 --- a/roles/ceph-mon/tasks/ceph_keys.yml +++ b/roles/ceph-mon/tasks/ceph_keys.yml @@ -12,21 +12,8 @@ changed_when: false when: cephx and - not mon_containerized_deployment and groups[restapi_group_name] is defined -- name: create ceph rest api keyring when mon is containerized - command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring - args: - creates: /etc/ceph/ceph.client.restapi.keyring - changed_when: false - when: - cephx and - mon_containerized_deployment and - groups[restapi_group_name] is defined and - inventory_hostname == groups.mons|last - - - include: openstack_config.yml when: openstack_config diff --git a/roles/ceph-mon/tasks/docker/copy_configs.yml b/roles/ceph-mon/tasks/docker/copy_configs.yml new file mode 100644 index 000000000..f2ba50e81 --- /dev/null +++ b/roles/ceph-mon/tasks/docker/copy_configs.yml @@ -0,0 +1,10 @@ +--- +- name: push ceph files to the ansible server + fetch: + src: "{{ item.0 }}" + dest: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + flat: yes + with_together: + - ceph_config_keys + - statconfig.results + when: item.1.stat.exists == false diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index 469cf9357..49a381886 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -12,7 +12,7 @@ - name: set fact for using Atomic host set_fact: is_atomic='{{ stat_ostree.stat.exists }}' - + - include: checks.yml when: ceph_health.rc != 0 and not mon_containerized_deployment_with_kv @@ -23,12 +23,24 @@ # let the first mon create configs and keyrings - include: create_configs.yml - when: inventory_hostname == groups.mons[0] and not mon_containerized_default_ceph_conf_with_kv + when: inventory_hostname == groups.mons[0] and mon_containerized_default_ceph_conf_with_kv - include: fetch_configs.yml when: not mon_containerized_deployment_with_kv - include: start_docker_monitor.yml -- include: ../ceph_keys.yml - when: not mon_containerized_deployment_with_kv \ No newline at end of file +- include: copy_configs.yml + when: not mon_containerized_deployment_with_kv + +- name: create ceph rest api keyring when mon is containerized + command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring + args: + creates: /etc/ceph/ceph.client.restapi.keyring + changed_when: false + when: + cephx and + mon_containerized_deployment and + groups[restapi_group_name] is defined and + inventory_hostname == groups.mons|last and + not mon_containerized_deployment_with_kv diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 1b5602cf5..48fb8ad6d 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -1,13 +1,4 @@ --- -- name: set config and keys paths - set_fact: - ceph_bootstrap_config_keys: - - /etc/ceph/ceph.conf - - /var/lib/ceph/bootstrap-osd/ceph.keyring - - /var/lib/ceph/bootstrap-rgw/ceph.keyring - - /var/lib/ceph/bootstrap-mds/ceph.keyring - when: not mon_containerized_deployment_with_kv - - name: pull ceph daemon image shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" @@ -22,8 +13,10 @@ KV_IP: "{{kv_endpoint}}" KV_PORT: "{{kv_port}}" run_once: true - when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv - and mon_containerized_default_ceph_conf_with_kv + when: + inventory_hostname == groups.mons[0] and + mon_containerized_deployment_with_kv and + mon_containerized_default_ceph_conf_with_kv - name: populate kv_store with custom ceph.conf docker: @@ -38,8 +31,10 @@ volumes: - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults run_once: true - when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv - and not mon_containerized_default_ceph_conf_with_kv + when: + inventory_hostname == groups.mons[0] and + mon_containerized_deployment_with_kv and + not mon_containerized_default_ceph_conf_with_kv - name: delete populate-kv-store docker docker: @@ -49,7 +44,7 @@ # Use systemd to manage container on Atomic host and CoreOS - name: generate systemd unit file - sudo: true + become: true template: src: ceph-mon.service.j2 dest: /var/lib/ceph/ceph-mon@.service @@ -62,17 +57,23 @@ src: /var/lib/ceph/ceph-mon@.service dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service state: link - when: is_atomic or ansible_os_family == 'CoreOS' + when: + is_atomic or + ansible_os_family == 'CoreOS' - name: enable systemd unit file for mon instance shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service failed_when: false changed_when: false - when: is_atomic or ansible_os_family == 'CoreOS' + when: + is_atomic or + ansible_os_family == 'CoreOS' - name: reload systemd unit files shell: systemctl daemon-reload - when: is_atomic or ansible_os_family == 'CoreOS' + when: + is_atomic or + ansible_os_family == 'CoreOS' - name: systemd start mon container service: @@ -80,14 +81,16 @@ state: started enabled: yes changed_when: false - when: is_atomic or ansible_os_family == 'CoreOS' + when: + is_atomic or + ansible_os_family == 'CoreOS' - name: wait for ceph.conf exists wait_for: path: /etc/ceph/ceph.conf when: is_atomic -- name: run the ceph Monitor docker image +- name: run the ceph monitor docker image docker: image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" name: "{{ ansible_hostname }}" @@ -96,23 +99,6 @@ privileged: "{{ mon_docker_privileged }}" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" - when: not is_atomic and ansible_os_family != 'CoreOS' - -- name: stat for ceph config and keys - stat: path="{{ item }}" - with_items: ceph_bootstrap_config_keys - changed_when: false - failed_when: false - register: statmonconfig - when: not mon_containerized_deployment_with_kv - -- name: fetch boostrap keys and conf from mon - fetch: - dest: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" - src: "{{ item.0 }}" - flat: yes - with_together: - - ceph_bootstrap_config_keys - - statmonconfig.results - when: not mon_containerized_deployment_with_kv and item.1.stat.exists == true - and inventory_hostname == groups.mons[0] + when: + not is_atomic and + ansible_os_family != 'CoreOS' diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml index d7c13e0b8..0b86c1fe3 100644 --- a/roles/ceph-osd/defaults/main.yml +++ b/roles/ceph-osd/defaults/main.yml @@ -120,3 +120,4 @@ ceph_osd_docker_imagename: daemon ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables #ceph_osd_docker_devices: # - /dev/sdb +ceph_docker_on_openstack: false diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml index 919b98de9..2892f9262 100644 --- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml +++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml @@ -2,13 +2,13 @@ # For openstack VMs modify the mount point below depending on if the Openstack # VM deploy tool defaults to mounting ephemeral disks - name: umount ceph disk (if on openstack) - mount: + mount: name: /mnt src: /dev/vdb fstype: ext3 state: unmounted when: ceph_docker_on_openstack - + # (rootfs) for reasons I haven't figured out, docker pull and run will fail. - name: pull ceph daemon image shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" @@ -28,7 +28,7 @@ # Use systemd to manage container on Atomic host - name: generate systemd unit file - sudo: true + become: true template: src: ceph-osd.service.j2 dest: /var/lib/ceph/ceph-osd@.service diff --git a/roles/ceph-restapi/defaults/main.yml b/roles/ceph-restapi/defaults/main.yml index c8320c114..4ac192eed 100644 --- a/roles/ceph-restapi/defaults/main.yml +++ b/roles/ceph-restapi/defaults/main.yml @@ -16,3 +16,4 @@ ceph_restapi_port: 5000 ceph_restapi_docker_username: ceph ceph_restapi_docker_imagename: daemon ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables +ceph_docker_on_openstack: false diff --git a/roles/ceph-restapi/tasks/docker/fetch_configs.yml b/roles/ceph-restapi/tasks/docker/fetch_configs.yml index 2faf8ade5..8af56884e 100644 --- a/roles/ceph-restapi/tasks/docker/fetch_configs.yml +++ b/roles/ceph-restapi/tasks/docker/fetch_configs.yml @@ -15,7 +15,7 @@ - name: try to fetch ceph config and keys copy: - src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" + src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" dest: "{{ item.0 }}" owner: root group: root diff --git a/roles/ceph-rgw/defaults/main.yml b/roles/ceph-rgw/defaults/main.yml index 26b2b0913..076c93fe7 100644 --- a/roles/ceph-rgw/defaults/main.yml +++ b/roles/ceph-rgw/defaults/main.yml @@ -31,3 +31,4 @@ ceph_rgw_civetweb_port: 80 ceph_rgw_docker_username: ceph ceph_rgw_docker_imagename: daemon ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables +ceph_docker_on_openstack: false diff --git a/roles/ceph-rgw/tasks/docker/fetch_configs.yml b/roles/ceph-rgw/tasks/docker/fetch_configs.yml index 45dd3bebc..6902b5919 100644 --- a/roles/ceph-rgw/tasks/docker/fetch_configs.yml +++ b/roles/ceph-rgw/tasks/docker/fetch_configs.yml @@ -15,7 +15,7 @@ - name: try to fetch ceph config and keys copy: - src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}" + src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" dest: "{{ item.0 }}" owner: root group: root