rgw: ability to use ceph-ansible vars into containers

Since the container now simply reads the ceph.conf, we remove all the
unnecessary options.

Also this PR is the foundation to support multiple backend, such as the
new 'beast' from Ceph Mimic.

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1582411
Signed-off-by: Sébastien Han <seb@redhat.com>
(cherry picked from commit 4d64dd4686)

# Conflicts:
#	roles/ceph-rgw/tasks/docker/main.yml

group_vars/all.yml.sample

@@ -386,12 +386,19 @@ dummy:
 
 ## Rados Gateway options
 #
+#radosgw_frontend_type: civetweb # For additionnal frontends see: http://docs.ceph.com/docs/mimic/radosgw/frontends/
+
 #radosgw_civetweb_port: 8080
 #radosgw_civetweb_num_threads: 100
+#radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at
 # https://github.com/civetweb/civetweb/blob/master/docs/UserManual.md
-#radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
+
+#radosgw_frontend_port: "{{ radosgw_civetweb_port if radosgw_frontend_type == 'civetweb' else '8080' }}"
+#radosgw_frontend_options: "{{ radosgw_civetweb_options if radosgw_frontend_type == 'civetweb' }}"
+
+
 # You must define either radosgw_interface, radosgw_address.
 # These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
 # Eg. If you want to specify for each radosgw node which address the radosgw will bind to you can set it in your **inventory host file** by using 'radosgw_address' variable.

group_vars/rgws.yml.sample

@@ -20,7 +20,7 @@ dummy:
 #copy_admin_key: false
 
 # Multi-site remote pull URL variables
-#rgw_pull_port: "{{ radosgw_civetweb_port }}"
+#rgw_pull_port: "{{ radosgw_frontend_port }}"
 #rgw_pull_proto: "http"
 
 

group_vars/rhcs.yml.sample

@@ -386,12 +386,19 @@ ceph_rhcs_version: 3
 
 ## Rados Gateway options
 #
+#radosgw_frontend_type: civetweb # For additionnal frontends see: http://docs.ceph.com/docs/mimic/radosgw/frontends/
+
 #radosgw_civetweb_port: 8080
 #radosgw_civetweb_num_threads: 100
+#radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at
 # https://github.com/civetweb/civetweb/blob/master/docs/UserManual.md
-#radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
+
+#radosgw_frontend_port: "{{ radosgw_civetweb_port if radosgw_frontend_type == 'civetweb' else '8080' }}"
+#radosgw_frontend_options: "{{ radosgw_civetweb_options if radosgw_frontend_type == 'civetweb' }}"
+
+
 # You must define either radosgw_interface, radosgw_address.
 # These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
 # Eg. If you want to specify for each radosgw node which address the radosgw will bind to you can set it in your **inventory host file** by using 'radosgw_address' variable.

roles/ceph-common/tasks/checks/check_firewall.yml

@@ -79,7 +79,7 @@
 - name: check if rados gateway port is not filtered
   local_action:
     module: shell
-      set -o pipefail && nmap -p {{ radosgw_civetweb_port }} {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }} | grep -sqo -e filtered -e '0 hosts up'
+      set -o pipefail && nmap -p {{ radosgw_frontend_port }} {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }} | grep -sqo -e filtered -e '0 hosts up'
   changed_when: false
   failed_when: false
   register: rgwportstate
@@ -90,7 +90,7 @@
 
 - name: fail if rados gateway port is filtered
   fail:
-      msg: "Please allow port {{ radosgw_civetweb_port }} on your firewall"
+      msg: "Please allow port {{ radosgw_frontend_port }} on your firewall"
   when:
     - rgw_group_name in group_names
     - nmapexist.rc == 0

roles/ceph-common/tasks/misc/configure_firewall_rpm.yml

@@ -50,7 +50,7 @@
 
 - name: open rgw ports
   firewalld:
-    port: "{{ radosgw_civetweb_port }}/tcp"
+    port: "{{ radosgw_frontend_port }}/tcp"
     zone: "{{ ceph_rgw_firewall_zone }}"
     permanent: true
     immediate: false # if true then fails in case firewalld is stopped

roles/ceph-config/templates/ceph.conf.j2

@@ -180,41 +180,41 @@ keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_hos
 log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] }}.log
 {% if hostvars[host]['radosgw_address_block'] is defined and hostvars[host]['radosgw_address_block'] != 'subnet' %}
     {% if ip_version == 'ipv4' %}
-rgw frontends = civetweb port={{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% elif ip_version == 'ipv6' %}
-rgw frontends = civetweb port=[{{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}=[{{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}]:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% endif %}
 {% elif radosgw_address_block is defined and radosgw_address_block != 'subnet' -%}
     {% if ip_version == 'ipv4' %}
-rgw frontends = civetweb port={{ radosgw_address_block | ipaddr(radosgw_address_block) | first }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ radosgw_address_block | ipaddr(radosgw_address_block) | first }}:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% elif ip_version == 'ipv6' %}
-rgw frontends = civetweb port=[{{ radosgw_address_block | ipaddr(radosgw_address_block) | first }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}=[{{ radosgw_address_block | ipaddr(radosgw_address_block) | first }}]:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% endif %}
 {% elif hostvars[host]['radosgw_address'] is defined and hostvars[host]['radosgw_address'] != 'address' -%}
     {% if ip_version == 'ipv4' %}
-rgw frontends = civetweb port={{ hostvars[host]['radosgw_address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ hostvars[host]['radosgw_address'] }}:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% elif ip_version == 'ipv6' %}
-rgw frontends = civetweb port=[{{ hostvars[host]['radosgw_address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}=[{{ hostvars[host]['radosgw_address'] }}]:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% endif %}
 {% elif radosgw_address is defined and radosgw_address != 'address' -%}
     {% if ip_version == 'ipv4' %}
-rgw frontends = civetweb port={{ radosgw_address }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ radosgw_address }}:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% elif ip_version == 'ipv6' %}
-rgw frontends = civetweb port=[{{ radosgw_address }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}=[{{ radosgw_address }}]:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% endif %}
 {% elif hostvars[host]['radosgw_interface'] is defined -%}
     {% set interface='ansible_' + (hostvars[host]['radosgw_interface'] | replace('-', '_')) %}
     {% if ip_version == 'ipv4' %}
-rgw frontends = civetweb port={{ hostvars[host][interface][ip_version]['address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ hostvars[host][interface][ip_version]['address'] }}:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% elif ip_version == 'ipv6' %}
-rgw frontends = civetweb port=[{{ hostvars[host][interface][ip_version][0]['address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}=[{{ hostvars[host][interface][ip_version][0]['address'] }}]:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% endif %}
 {% else %}
     {% set interface='ansible_' + (radosgw_interface | replace('-', '_')) %}
     {% if ip_version == 'ipv4' %}
-rgw frontends = civetweb port={{ hostvars[host][interface][ip_version]['address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ hostvars[host][interface][ip_version]['address'] }}:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% elif ip_version == 'ipv6' %}
-rgw frontends = civetweb port=[{{ hostvars[host][interface][ip_version][0]['address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}=[{{ hostvars[host][interface][ip_version][0]['address'] }}]:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
     {% endif %}
 {% endif %}
 {% endif %}

roles/ceph-defaults/defaults/main.yml

@@ -378,12 +378,19 @@ mds_max_mds: 1
 
 ## Rados Gateway options
 #
+radosgw_frontend_type: civetweb # For additionnal frontends see: http://docs.ceph.com/docs/mimic/radosgw/frontends/
+
 radosgw_civetweb_port: 8080
 radosgw_civetweb_num_threads: 100
+radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at
 # https://github.com/civetweb/civetweb/blob/master/docs/UserManual.md
-radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
+
+radosgw_frontend_port: "{{ radosgw_civetweb_port if radosgw_frontend_type == 'civetweb' else '8080' }}"
+radosgw_frontend_options: "{{ radosgw_civetweb_options if radosgw_frontend_type == 'civetweb' }}"
+
+
 # You must define either radosgw_interface, radosgw_address.
 # These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
 # Eg. If you want to specify for each radosgw node which address the radosgw will bind to you can set it in your **inventory host file** by using 'radosgw_address' variable.

roles/ceph-defaults/templates/restart_rgw_daemon.sh.j2

@@ -3,7 +3,7 @@
 RETRIES="{{ handler_health_rgw_check_retries }}"
 DELAY="{{ handler_health_rgw_check_delay }}"
 RGW_NAME="{{ ansible_hostname }}"
-RGW_PORT="{{ radosgw_civetweb_port }}"
+RGW_PORT="{{ radosgw_frontend_port }}"
 {% if containerized_deployment %}
 DOCKER_EXEC="docker exec ceph-rgw-{{ ansible_hostname }}"
 {% endif %}

roles/ceph-rgw/defaults/main.yml

@@ -12,7 +12,7 @@
 copy_admin_key: false
 
 # Multi-site remote pull URL variables
-rgw_pull_port: "{{ radosgw_civetweb_port }}"
+rgw_pull_port: "{{ radosgw_frontend_port }}"
 rgw_pull_proto: "http"
 
 

roles/ceph-rgw/tasks/docker/container_env_facts.yml

@@ -1,16 +0,0 @@
----
-- name: set_fact docker_env_args '-e RGW_FRONTENDS={{ radosgw_civetweb_options }}'
-  set_fact:
-    docker_env_args: -e RGW_FRONTENDS={{ radosgw_civetweb_options }}
-
-- name: set_fact docker_env_args '-e RGW_ZONEGROUP={{ rgw_zonegroup }}'
-  set_fact:
-    docker_env_args: "{{ docker_env_args }} -e RGW_ZONEGROUP={{ rgw_zonegroup }}"
-  when:
-    - rgw_zonegroup != ""
-
-- name: set_fact docker_env_args '-e RGW_ZONE={{ rgw_zone }}'
-  set_fact:
-    docker_env_args: "{{ docker_env_args }} -e RGW_ZONE={{ rgw_zone }}"
-  when:
-    - rgw_zone != ""

roles/ceph-rgw/tasks/docker/main.yml

@@ -1,8 +1,11 @@
 ---
+<<<<<<< HEAD
 - name: include container_env_facts.yml
   include: container_env_facts.yml
   when:
     - containerized_deployment
 
+=======
+>>>>>>> 4d64dd46... rgw: ability to use ceph-ansible vars into containers
 - name: include start_docker_rgw.yml
   include: start_docker_rgw.yml

roles/ceph-rgw/tasks/multisite/master.yml

@@ -9,7 +9,7 @@
     - update period
 
 - name: create the zonegroup
-  command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_civetweb_port }} --master --default
+  command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_frontend_port }} --master --default
   run_once: true
   when:
     - 'No such file or directory' in zonegroupcheck.stderr
@@ -18,7 +18,7 @@
     - update period
 
 - name: create the zone
-  command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master
+  command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master
   run_once: true
   when:
     - 'No such file or directory' in zonecheck.stderr

roles/ceph-rgw/tasks/multisite/secondary.yml

@@ -28,7 +28,7 @@
     - update period # Might not need to update period here
 
 - name: create the zone
-  command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default
+  command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default
   run_once: true
   when:
     - 'No such file or directory' in zonecheck.stderr

roles/ceph-rgw/templates/ceph-radosgw.service.j2

@@ -22,49 +22,9 @@ ExecStart=/usr/bin/docker run --rm --net=host \
   -e KV_IP={{ kv_endpoint }} \
   -e KV_PORT={{ kv_port }} \
   {% endif -%}
-{% if hostvars[inventory_hostname]['radosgw_address_block'] is defined and hostvars[inventory_hostname]['radosgw_address_block'] != 'subnet' %}
-    {% if ip_version == 'ipv4' %}
-  -e RGW_CIVETWEB_IP={{ hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }} \
-    {% elif ip_version == 'ipv6' %}
-  -e RGW_CIVETWEB_IP=[{{ hostvars[inventory_hostname]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}] \
-    {% endif %}
-{% elif radosgw_address_block is defined and radosgw_address_block != 'subnet' -%}
-    {% if ip_version == 'ipv4' %}
-  -e RGW_CIVETWEB_IP={{ ['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }} \
-    {% elif ip_version == 'ipv6' %}
-  -e RGW_CIVETWEB_IP=[{{ ['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}] \
-    {% endif %}
-{% elif hostvars[inventory_hostname]['radosgw_address'] is defined and hostvars[inventory_hostname]['radosgw_address'] != 'address' -%}
-    {% if ip_version == 'ipv4' %}
-  -e RGW_CIVETWEB_IP={{ hostvars[inventory_hostname]['radosgw_address'] }} \
-    {% elif ip_version == 'ipv6' %}
-  -e RGW_CIVETWEB_IP=[{{ hostvars[inventory_hostname]['radosgw_address'] }}] \
-    {% endif %}
-{% elif radosgw_address is defined and radosgw_address != 'address' -%}
-    {% if ip_version == 'ipv4' %}
-  -e RGW_CIVETWEB_IP={{ radosgw_address }} \
-    {% elif ip_version == 'ipv6' %}
-  -e RGW_CIVETWEB_IP=[{{ radosgw_address }}] \
-    {% endif %}
-{% elif hostvars[inventory_hostname]['radosgw_interface'] is defined -%}
-    {% set interface = 'ansible_' + (hostvars[inventory_hostname]['radosgw_interface'] | replace('-', '_')) %}
-    {% if ip_version == 'ipv4' %}
-  -e RGW_CIVETWEB_IP={{ hostvars[inventory_hostname][interface][ip_version]['address'] }} \
-    {% elif ip_version == 'ipv6' %}
-  -e RGW_CIVETWEB_IP=[{{ hostvars[inventory_hostname][interface][ip_version][0]['address'] }}] \
-    {% endif %}
-{% else %}
-    {% set interface = 'ansible_' + (radosgw_interface | replace('-', '_')) %}
-    {% if ip_version == 'ipv4' %}
-  -e RGW_CIVETWEB_IP={{ hostvars[inventory_hostname][interface][ip_version]['address'] }} \
-    {% elif ip_version == 'ipv6' %}
-  -e RGW_CIVETWEB_IP=[{{ hostvars[inventory_hostname][interface][ip_version][0]['address'] }}] \
-    {% endif %}
-{% endif %}
   -v /etc/localtime:/etc/localtime:ro \
   -e CEPH_DAEMON=RGW \
   -e CLUSTER={{ cluster }} \
-  -e RGW_CIVETWEB_PORT={{ radosgw_civetweb_port }} \
   --name=ceph-rgw-{{ ansible_hostname }} \
   {{ ceph_rgw_docker_extra_env }} \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}