Common: changed civetweb line in rgw section(conf)

Resolves issue: Multiple RGW Ceph.conf Issue #1258

In multi-RGW setup, in ceph.conf the RGW sections
contain identical bind IP in civetweb line. So this
modification fixes that issue and puts the right IP
for each RGW.

Signed-off-by: SirishaGuduru SGuduru@walmartlabs.com

Modified ceph-defaults and ran generate_group_vars_sample.sh

group_vars/osds.yml.sample and group_vars/rhcs.yml.sample are
not part of the changes. But they got modified when
generate_group_vars_sample.sh is ran to generate group_vars/
all.yml.sample.

Uncommented added variables in ceph-defaults

Updated tests by adding value for radosgw_interface

Added radosgw_interface to centos cluster tests

Modified ceph-rgw role,rebased and ran generate_group_vars_sample.sh

In ceph-rgw role removed check_mandatory_vars.yml.
Rebased on master.
Ran generate_group_vars_sample.sh and then the below files got
modified.

Vagrantfile

@@ -81,6 +81,7 @@ ansible_provision = proc do |ansible|
       devices: settings['disks'],
       ceph_docker_on_openstack: BOX == 'openstack',
       ceph_rgw_civetweb_port: 8080,
+      radosgw_interface: ETH,
       generate_fsid: 'true',
     })
   else
@@ -88,6 +89,7 @@ ansible_provision = proc do |ansible|
       devices: settings['disks'],
       osd_scenario: 'collocated',
       monitor_interface: ETH,
+      radosgw_interface: ETH,
       os_tuning_params: settings['os_tuning_params'],
       pool_default_size: '2',
     })
@@ -97,11 +99,14 @@ ansible_provision = proc do |ansible|
     ansible.sudo = true
     # Use monitor_address_block instead of monitor_interface:
     ansible.extra_vars.delete(:monitor_interface)
+    # Use radosgw_address_block instead of radosgw_interface:
+    ansible.extra_vars.delete(:radosgw_interface)
     ansible.extra_vars = ansible.extra_vars.merge({
       cluster_network: "#{CLUSTER_SUBNET}.0/16",
       devices: ['/dev/sdc'], # hardcode leftover disk
       osd_scenario: 'collocated',
       monitor_address_block: "#{PUBLIC_SUBNET}.0/16",
+      radosgw_address_block: "#{PUBLIC_SUBNET}.0/16",
       public_network: "#{PUBLIC_SUBNET}.0/16",
     })
   end

group_vars/all.yml.sample

@@ -302,7 +302,15 @@ dummy:
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at
 # https://github.com/civetweb/civetweb/blob/master/docs/UserManual.md
-#radosgw_civetweb_options: "port={{ radosgw_civetweb_bind_ip }}:{{ radosgw_civetweb_port }} num_threads={{ radosgw_civetweb_num_threads }}"
+#radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
+# You must define either radosgw_interface, radosgw_address.
+# These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
+# Eg. If you want to specify for each radosgw node which address the radosgw will bind to you can set it in your **inventory host file** by using 'radosgw_address' variable.
+# Preference will go to radosgw_address if both radosgw_address and radosgw_interface are defined.
+# To use an IPv6 address, use the radosgw_address setting instead (and set ip_version to ipv6)
+#radosgw_interface: interface
+#radosgw_address: 0.0.0.0
+#radosgw_address_block: []
 #radosgw_keystone: false # activate OpenStack Keystone options full detail here: http://ceph.com/docs/master/radosgw/keystone/
 # Rados Gateway options
 #email_address: foo@bar.com

group_vars/mdss.yml.sample

@@ -28,6 +28,7 @@ dummy:
 #ceph_mds_docker_extra_env: -e CLUSTER={{ cluster }} -e MDS_NAME={{ ansible_hostname }}
 #ceph_config_keys: [] # DON'T TOUCH ME
 
+
 ###########
 # SYSTEMD #
 ###########

group_vars/mgrs.yml.sample

@@ -14,6 +14,7 @@ dummy:
 #ceph_mgr_docker_extra_env: -e CLUSTER={{ cluster }} -e MGR_NAME={{ ansible_hostname }}
 #ceph_config_keys: [] # DON'T TOUCH ME
 
+
 ###########
 # SYSTEMD #
 ###########

group_vars/mons.yml.sample

@@ -125,6 +125,7 @@ dummy:
 #mon_docker_net_host: true
 #ceph_config_keys: [] # DON'T TOUCH ME
 
+
 ###########
 # SYSTEMD #
 ###########

group_vars/osds.yml.sample

@@ -247,6 +247,7 @@ dummy:
 #ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }}
 #ceph_osd_docker_run_script_path: "/usr/share" # script called by systemd to run the docker command
 
+
 ###########
 # SYSTEMD #
 ###########

group_vars/rgws.yml.sample

@@ -60,6 +60,7 @@ dummy:
 #ceph_config_keys: [] # DON'T TOUCH ME
 #rgw_config_keys: "/" # DON'T TOUCH ME
 
+
 ###########
 # SYSTEMD #
 ###########

group_vars/rhcs.yml.sample

@@ -13,7 +13,7 @@ dummy:
 # GENERAL #
 ###########
 
-fetch_directory: ~/ceph-ansible-keys
+#fetch_directory: fetch/
 
 # The 'cluster' variable determines the name of the cluster.
 # Changing the default value to something else means that you will
@@ -154,7 +154,7 @@ fetch_directory: ~/ceph-ansible-keys
 # Commit 492518a2 changed variable names of rhcs installations
 # to not break backward compatiblity we re-declare these variables
 # with the content of the new variable
-ceph_rhcs: true
+#ceph_rhcs: "{{ ceph_stable_rh_storage | default(false) }}"
 # This will affect how/what repositories are enabled depending on the desired
 # version. The previous version was 1.3. The current version is 2.
 #ceph_rhcs_version: "{{ ceph_stable_rh_storage_version | default(2) }}"
@@ -302,7 +302,15 @@ ceph_rhcs: true
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at
 # https://github.com/civetweb/civetweb/blob/master/docs/UserManual.md
-#radosgw_civetweb_options: "port={{ radosgw_civetweb_bind_ip }}:{{ radosgw_civetweb_port }} num_threads={{ radosgw_civetweb_num_threads }}"
+#radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
+# You must define either radosgw_interface, radosgw_address.
+# These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
+# Eg. If you want to specify for each radosgw node which address the radosgw will bind to you can set it in your **inventory host file** by using 'radosgw_address' variable.
+# Preference will go to radosgw_address if both radosgw_address and radosgw_interface are defined.
+# To use an IPv6 address, use the radosgw_address setting instead (and set ip_version to ipv6)
+#radosgw_interface: interface
+#radosgw_address: 0.0.0.0
+#radosgw_address_block: []
 #radosgw_keystone: false # activate OpenStack Keystone options full detail here: http://ceph.com/docs/master/radosgw/keystone/
 # Rados Gateway options
 #email_address: foo@bar.com

roles/ceph-common/tasks/checks/check_mandatory_vars.yml

@@ -71,3 +71,11 @@
     - monitor_interface == 'interface'
     - monitor_address == "0.0.0.0"
     - mon_group_name in group_names
+
+- name: make sure radosgw_interface or radosgw_address is defined
+  fail:
+    msg: "you must set radosgw_interface or radosgw_address"
+  when:
+    - radosgw_interface == 'interface'
+    - radosgw_address == "0.0.0.0"
+    - rgw_group_name in group_names

roles/ceph-common/templates/ceph.conf.j2

@@ -135,7 +135,26 @@ keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_hos
 rgw socket path = /tmp/radosgw-{{ hostvars[host]['ansible_hostname'] }}.sock
 log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] }}.log
 rgw data = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ hostvars[host]['ansible_hostname'] }}
-rgw frontends = civetweb port={{ radosgw_civetweb_bind_ip }}:{{ radosgw_civetweb_port }} num_threads={{ radosgw_civetweb_num_threads }}
+{% if radosgw_address_block | length > 0 %}
+    {% if ip_version == 'ipv4' -%}
+      rgw frontends = civetweb port={{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- elif ip_version == 'ipv6' -%}
+      rgw frontends = civetweb port=[{{ hostvars[host]['ansible_all_' + ip_version + '_addresses'] | ipaddr(radosgw_address_block) | first }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- endif %}
+{% elif hostvars[host]['radosgw_address'] is defined and hostvars[host]['radosgw_address'] != '0.0.0.0' -%}
+    {% if ip_version == 'ipv4' -%}
+      rgw frontends = civetweb port={{ hostvars[host]['radosgw_address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- elif ip_version == 'ipv6' -%}
+      rgw frontends = civetweb port=[{{ hostvars[host]['radosgw_address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {% endif %}
+{%- else -%}
+    {% set interface = ["ansible_",radosgw_interface]|join %}
+    {% if ip_version == 'ipv6' -%}
+      rgw frontends = civetweb port=[{{ hostvars[host][interface][ip_version][0]['address'] }}]:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {%- elif ip_version == 'ipv4' -%}
+      rgw frontends = civetweb port={{ hostvars[host][interface][ip_version]['address'] }}:{{ radosgw_civetweb_port }} {{ radosgw_civetweb_options }}
+    {% endif %}
+{%- endif %}
 {% endif %}
 {% endfor %}
 {% endif %}

roles/ceph-defaults/defaults/main.yml

@@ -294,7 +294,15 @@ radosgw_civetweb_num_threads: 100
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at
 # https://github.com/civetweb/civetweb/blob/master/docs/UserManual.md
-radosgw_civetweb_options: "port={{ radosgw_civetweb_bind_ip }}:{{ radosgw_civetweb_port }} num_threads={{ radosgw_civetweb_num_threads }}"
+radosgw_civetweb_options: "num_threads={{ radosgw_civetweb_num_threads }}"
+# You must define either radosgw_interface, radosgw_address.
+# These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
+# Eg. If you want to specify for each radosgw node which address the radosgw will bind to you can set it in your **inventory host file** by using 'radosgw_address' variable.
+# Preference will go to radosgw_address if both radosgw_address and radosgw_interface are defined.
+# To use an IPv6 address, use the radosgw_address setting instead (and set ip_version to ipv6)
+radosgw_interface: interface
+radosgw_address: 0.0.0.0
+radosgw_address_block: []
 radosgw_keystone: false # activate OpenStack Keystone options full detail here: http://ceph.com/docs/master/radosgw/keystone/
 # Rados Gateway options
 email_address: foo@bar.com

tests/functional/centos/7/bluestore/group_vars/all

@@ -3,6 +3,7 @@
 ceph_stable: True
 cluster: test
 monitor_interface: eth1
+radosgw_interface: eth1
 public_network: "192.168.1.0/24"
 cluster_network: "192.168.2.0/24"
 journal_size: 100

tests/functional/centos/7/bs-crypt-ded-jrn/group_vars/all

@@ -5,6 +5,7 @@ public_network: "192.168.11.0/24"
 cluster_network: "192.168.12.0/24"
 journal_size: 100
 monitor_interface: eth1
+radosgw_interface: eth1
 osd_scenario: non-collocated
 dmcrypt: true
 osd_objectstore: "bluestore"

tests/functional/centos/7/bs-crypt-jrn-col/group_vars/all

@@ -5,6 +5,7 @@ public_network: "192.168.13.0/24"
 cluster_network: "192.168.14.0/24"
 journal_size: 100
 monitor_interface: eth1
+radosgw_interface: eth1
 osd_scenario: collocated
 dmcrypt: true
 osd_objectstore: "bluestore"

tests/functional/centos/7/bs-dock-crypt-jrn-col/group_vars/all

@@ -7,6 +7,7 @@ public_network: "192.168.23.0/24"
 cluster_network: "192.168.24.0/24"
 journal_size: 100
 monitor_interface: eth1
+radosgw_interface: eth1
 osd_scenario: collocated
 dmcrypt: true
 osd_objectstore: "bluestore"

tests/functional/centos/7/bs-dock-ded-jrn/group_vars/all

@@ -7,6 +7,7 @@ ceph_stable: True
 containerized_deployment: True
 cluster: test
 monitor_interface: eth1
+radosgw_interface: eth1
 ceph_mon_docker_subnet: "{{ public_network }}"
 journal_size: 100
 ceph_docker_on_openstack: False

tests/functional/centos/7/bs-docker/group_vars/all

@@ -7,6 +7,7 @@ ceph_stable: True
 containerized_deployment: True
 cluster: test
 monitor_interface: eth1
+radosgw_interface: eth1
 ceph_mon_docker_subnet: "{{ public_network }}"
 journal_size: 100
 ceph_docker_on_openstack: False

tests/functional/centos/7/bs-jrn-col/group_vars/all

@@ -3,6 +3,7 @@
 ceph_stable: True
 cluster: test
 monitor_interface: eth1
+radosgw_interface: eth1
 public_network: "192.168.3.0/24"
 cluster_network: "192.168.4.0/24"
 journal_size: 100

tests/functional/centos/7/cluster/group_vars/all

@@ -5,6 +5,7 @@ cluster: test
 public_network: "192.168.1.0/24"
 cluster_network: "192.168.2.0/24"
 journal_size: 100
+radosgw_interface: eth1
 osd_objectstore: "filestore"
 devices:
   - '/dev/sda'

tests/functional/centos/7/crypt-ded-jrn/group_vars/all

@@ -5,6 +5,7 @@ public_network: "192.168.11.0/24"
 cluster_network: "192.168.12.0/24"
 journal_size: 100
 monitor_interface: eth1
+radosgw_interface: eth1
 osd_scenario: non-collocated
 osd_objectstore: "filestore"
 devices:

tests/functional/centos/7/crypt-jrn-col/group_vars/all

@@ -5,6 +5,7 @@ public_network: "192.168.13.0/24"
 cluster_network: "192.168.14.0/24"
 journal_size: 100
 monitor_interface: eth1
+radosgw_interface: eth1
 osd_scenario: collocated
 osd_objectstore: "filestore"
 devices:

tests/functional/centos/7/docker-crypt-jrn-col/group_vars/all

@@ -7,6 +7,7 @@ ceph_stable: True
 containerized_deployment: True
 cluster: ceph
 monitor_interface: eth1
+radosgw_interface: eth1
 ceph_mon_docker_subnet: "{{ public_network }}"
 journal_size: 100
 ceph_docker_on_openstack: False

tests/functional/centos/7/docker-ded-jrn/group_vars/all

@@ -7,6 +7,7 @@ ceph_stable: True
 containerized_deployment: True
 cluster: test
 monitor_interface: eth1
+radosgw_interface: eth1
 ceph_mon_docker_subnet: "{{ public_network }}"
 journal_size: 100
 ceph_docker_on_openstack: False

tests/functional/centos/7/docker/group_vars/all

@@ -7,6 +7,7 @@ ceph_stable: True
 containerized_deployment: True
 cluster: test
 monitor_interface: eth1
+radosgw_interface: eth1
 ceph_mon_docker_subnet: "{{ public_network }}"
 journal_size: 100
 ceph_docker_on_openstack: False

tests/functional/centos/7/jrn-col/group_vars/all

@@ -5,6 +5,7 @@ cluster: test
 public_network: "192.168.3.0/24"
 cluster_network: "192.168.4.0/24"
 monitor_interface: eth1
+radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "filestore"
 devices:

tests/functional/centos/7/lvm-osds/group_vars/all

@@ -5,6 +5,7 @@ cluster: ceph
 public_network: "192.168.3.0/24"
 cluster_network: "192.168.4.0/24"
 monitor_interface: eth1
+radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "filestore"
 osd_scenario: lvm

tests/functional/ubuntu/16.04/cluster/group_vars/all

@@ -5,6 +5,7 @@ cluster: test
 public_network: "192.168.5.0/24"
 cluster_network: "192.168.6.0/24"
 monitor_interface: eth1
+radosgw_interface: eth1
 journal_size: 100
 devices:
   - '/dev/sdb'