Add RADOS Gateway support

This is a wip branch.
This works on Ubuntu precise, Debian Wheezy and CentOS 6.4.

Signed-off-by: Sébastien Han <sebastien.han@enovance.com>

Vagrantfile

@@ -8,6 +8,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = "precise64"
   config.vm.box_url = "http://files.vagrantup.com/precise64.box"
 
+  config.vm.define :rgw do |rgw|
+    rgw.vm.network :private_network, ip: "192.168.0.2"
+    rgw.vm.host_name = "ceph-rgw"
+  end
+
   (0..2).each do |i|
     config.vm.define "mon#{i}" do |mon|
       mon.vm.hostname = "ceph-mon#{i}"

group_vars/all

@@ -9,12 +9,18 @@ redhat_distro: el6 # supported distros are el6, rhel6, f18, f19, opensuse12.2, s
 
 # Ceph options
 cephx: true
-mds: false # disable mds configuration in ceph.conf
 fsid: # /!\ GENERATE ONE WITH 'uuidgen -r' /!\
 
 # Monitors options
 monitor_interface: eth1
 
+# MDS options
+mds: true # disable mds configuration in ceph.conf
+
+# Rados Gateway options
+radosgw: true
+redhat_distro_ceph_extra: centos6.4 # supported distros are centos6.3, centos6.4, centos6, fedora18, fedora19, opensuse12.2, rhel6.3, rhel6.4, rhel6.5, rhel6, sles11sp2
+
 # OSD options
 journal_size: 100
 pool_default_pg_num: 128

hosts

@@ -16,7 +16,8 @@ ceph-osd2:2204
 ceph-osd0:2202
 ceph-osd1:2203
 ceph-osd2:2204
-
+[rgws]
+ceph-rgw:2205
 
 # Colocation setup example
 #[mons]

roles/common/templates/ceph.conf.j2

@@ -62,3 +62,13 @@
   {% endif %}
 {% endfor %}
 {% endif %}
+
+{% if radosgw %}
+[client.radosgw.gateway]
+  host = {{ hostvars[host]['ansible_hostname'] }}
+  keyring = /etc/ceph/keyring.radosgw.gateway
+  rgw socket path = /tmp/radosgw.sock
+  log file = /var/log/ceph/radosgw.log
+  rgw data = /var/lib/ceph/radosgw/{{ hostvars[host]['ansible_hostname'] }}
+  rgw print continue = false
+{% endif %}

roles/mon/tasks/main.yml

@@ -27,6 +27,11 @@
   until: result.rc == 0
   changed_when: False
 
+- name: Create RGW keyring
+  command: ceph auth get-or-create client.radosgw.gateway osd 'allow rwx' mon 'allow rw' -o /etc/ceph/keyring.radosgw.gateway creates=/etc/ceph/keyring.radosgw.gateway
+  when: cephx and radosgw
+  changed_when: False
+
 - name: Copy keys to the ansible server
   fetch: src={{ item }} dest=fetch/
   when: ansible_fqdn == hostvars[groups['mons'][0]]['ansible_fqdn'] and cephx
@@ -34,3 +39,4 @@
     - /etc/ceph/ceph.client.admin.keyring # just in case another application needs it
     - /var/lib/ceph/bootstrap-osd/ceph.keyring # this handles the non-colocation case
     - /var/lib/ceph/bootstrap-mds/ceph.keyring
+    - /etc/ceph/keyring.radosgw.gateway

roles/radosgw/files/s3gw.fcgi

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/bin/radosgw -c /etc/ceph/ceph.conf -n client.radosgw.gateway

roles/radosgw/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: restart apache2
+  service: name=apache2 state=restarted enabled=yes
+  when: ansible_os_family == 'Debian'
+
+- name: restart apache2
+  service: name=httpd state=restarted enabled=yes
+  when: ansible_os_family == 'RedHat'

roles/radosgw/tasks/Debian.yml

@@ -0,0 +1,81 @@
+---
+## Deploy RADOS Gateway
+#
+
+- name: Copy RGW bootstrap key
+  copy: src=fetch/{{ hostvars[groups['mons'][0]]['ansible_hostname'] }}/etc/ceph/keyring.radosgw.gateway dest=/etc/ceph/keyring.radosgw.gateway owner=root group=root mode=600
+  when: cephx
+
+- name: Set RGW bootstrap key permissions
+  file: path=/etc/ceph/keyring.radosgw.gateway mode=0600 owner=root group=root
+  when: cephx
+
+#- name: Add optimized version of the apache2 package repository
+#  apt_repository: repo='deb http://gitbuilder.ceph.com/apache2-deb-{{ ansible_lsb.codename }}-x86_64-basic/ref/master {{ ansible_lsb.codename }} main' state=present
+#
+#- name: Add optimized version of the fastcgi package repository
+#  apt_repository: repo='deb http://gitbuilder.ceph.com/libapache-mod-fastcgi-deb-{{ ansible_lsb.codename }}-x86_64-basic/ref/master {{ ansible_lsb.codename }} main' state=present
+#
+
+- name: Add Ceph extra
+  apt_repository: repo='deb http://ceph.com/packages/ceph-extras/debian {{ ansible_lsb.codename }} main' state=present
+
+- name: Install Apache, fastcgi and Rados Gateway
+  apt: pkg={{ item }} state=present default_release=precise
+  with_items:
+    - apache2
+    - libapache2-mod-fastcgi
+    - radosgw
+
+## Prepare Apache
+#
+
+- name: Install default httpd.conf
+  template: src=httpd.conf dest=/etc/apache2/httpd.conf owner=root group=root
+
+- name: Enable some apache mod rewrite and fastcgi
+  command: "{{ item }}"
+  with_items:
+    - a2enmod rewrite
+    - a2enmod fastcgi
+
+- name: Install Rados Gateway vhost
+  template: src=rgw.conf dest=/etc/apache2/sites-available/rgw.conf owner=root group=root
+
+## Prepare RGW
+#
+
+- name: Create RGW directory
+  file: path=/var/lib/ceph/radosgw/{{ ansible_hostname }} state=directory owner=root group=root mode=0644
+
+- name: Enable Rados Gateway vhost and disable default site
+  command: "{{ item }}"
+  with_items:
+    - a2ensite rgw.conf
+    - a2dissite default
+  notify:
+    - restart apache2
+
+- name: Install s3gw.fcgi script
+  copy: src=s3gw.fcgi dest=/var/www/s3gw.fcgi mode=0555 owner=root group=root
+
+## If we don't perform this check Ansible will start multiple instance of radosgw
+- name: Check if RGW is started
+  command: /etc/init.d/radosgw status
+  register: rgwstatus
+  ignore_errors: True
+
+- name: Start RGW
+  command: /etc/init.d/radosgw start
+  when: rgwstatus.rc != 0
+
+- name: Create a user in radosgw
+  command: radosgw-admin --name client.radosgw.gateway user create --uid=johndoe --display-name="John Doe" --email=john@example.com
+
+- name: Create a swift subuser
+  command: radosgw-admin --name client.radosgw.gateway subuser create --uid=johndoe --subuser=johndoe:swift --access=full
+  ignore_errors: True
+
+- name: Create a swift subuser key
+  command: radosgw-admin --name client.radosgw.gateway key create --subuser=johndoe:swift --key-type=swift
+  ignore_errors: True

roles/radosgw/tasks/RedHat.yml

@@ -0,0 +1,70 @@
+---
+## Deploy RADOS Gateway
+#
+
+- name: Copy RGW bootstrap key
+  copy: src=fetch/{{ hostvars[groups['mons'][0]]['ansible_hostname'] }}/etc/ceph/keyring.radosgw.gateway dest=/etc/ceph/keyring.radosgw.gateway owner=root group=root mode=600
+  when: cephx
+
+- name: Set RGW bootstrap key permissions
+  file: path=/etc/ceph/keyring.radosgw.gateway mode=0644 owner=root group=root
+  when: cephx
+
+- name: Add Ceph extra
+  template: src=ceph-extra.repo dest=/etc/yum.repos.d owner=root group=root
+
+- name: Add special fastcgi repository key
+  rpm_key: key=http://dag.wieers.com/rpm/packages/RPM-GPG-KEY.dag.txt
+
+- name: Add special fastcgi repository
+  command: rpm -ivh http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm
+
+- name: Install Apache, fastcgi, and Rados Gateway
+  yum: name={{ item }} state=present
+  with_items:
+    - httpd
+    - mod_fastcgi
+    - mod_fcgid
+    - ceph-radosgw
+
+## Prepare Apache
+#
+
+- name: Install Rados Gateway vhost
+  template: src=rgw.conf dest=/etc/httpd/conf.d/rgw.conf owner=root group=root
+
+## Prepare RGW
+#
+
+- name: Create RGW directory
+  file: path=/var/lib/ceph/radosgw/{{ ansible_hostname }} state=directory owner=root group=root mode=0644
+
+- name: Install s3gw.fcgi script
+  copy: src=s3gw.fcgi dest=/var/www/s3gw.fcgi mode=0555 owner=root group=root
+
+- name: Disable default site
+  shell: sed -i "s/^[^+#]/#/g" /etc/httpd/conf.d/welcome.conf
+  changed_when: False
+  notify:
+    - restart apache2
+
+## If we don't perform this check Ansible will start multiple instance of radosgw
+- name: Check if RGW is started
+  command: /etc/init.d/ceph-radosgw status
+  register: rgwstatus
+  ignore_errors: True
+
+- name: Start RGW
+  command: /etc/init.d/ceph-radosgw start
+  when: rgwstatus.rc != 0
+
+- name: Create a user in radosgw
+  command: radosgw-admin --name client.radosgw.gateway user create --uid=johndoe --display-name="John Doe" --email=john@example.com
+
+- name: Create a swift subuser
+  command: radosgw-admin --name client.radosgw.gateway subuser create --uid=johndoe --subuser=johndoe:swift --access=full
+  ignore_errors: True
+
+- name: Create a swift subuser key
+  command: radosgw-admin --name client.radosgw.gateway key create --subuser=johndoe:swift --key-type=swift
+  ignore_errors: True

roles/radosgw/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+## Check OS family
+#
+
+- include: RedHat.yml
+  when: ansible_os_family == 'RedHat'
+
+- include: Debian.yml
+  when: ansible_os_family == 'Debian'

roles/radosgw/templates/ceph-extra.repo

@@ -0,0 +1,30 @@
+# {{ ansible_managed }}
+
+[ceph-extras]
+name=Ceph Extras Packages
+baseurl=http://ceph.com/packages/ceph-extras/rpm/{{ redhat_distro_ceph_extra }}/$basearch
+enabled=1
+priority=2
+gpgcheck=1
+type=rpm-md
+gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc
+
+{% if (redhat_distro_ceph_extra != "centos6.4" and redhat_distro_ceph_extra !=  "rhel6.4" and redhat_distro_ceph_extra !=  "rhel6.5") %}
+[ceph-extras-noarch]
+name=Ceph Extras noarch
+baseurl=http://ceph.com/packages/ceph-extras/rpm/{{ redhat_distro_ceph_extra }}/noarch
+enabled=1
+priority=2
+gpgcheck=1
+type=rpm-md
+gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc
+{% endif %}
+
+[ceph-extras-source]
+name=Ceph Extras Sources
+baseurl=http://ceph.com/packages/ceph-extras/rpm/{{ redhat_distro_ceph_extra }}/SRPMS
+enabled=1
+priority=2
+gpgcheck=1
+type=rpm-md
+gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc

roles/radosgw/templates/httpd.conf

@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+
+ServerName {{ ansible_hostname }}

roles/radosgw/templates/rgw.conf

@@ -0,0 +1,23 @@
+# {{ ansible_managed }}
+
+FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock
+<VirtualHost *:80>
+        ServerName {{ ansible_hostname }}
+        ServerAdmin {{ email_address }}@{{ ansible_fqdn }}
+        DocumentRoot /var/www
+
+        <IfModule mod_fastcgi.c>
+                <Directory /var/www>
+                        Options +ExecCGI
+                        AllowOverride All
+                        SetHandler fastcgi-script
+                        Order allow,deny
+                        Allow from all
+                        AuthBasicAuthoritative Off
+                </Directory>
+        </IfModule>
+
+        RewriteEngine On
+        RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1&params=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+</VirtualHost>

site.yml

@@ -20,3 +20,8 @@
   sudo: True
   roles:
   - mds
+
+- hosts: rgws
+  sudo: True
+  roles:
+  - radosgw