Create separate subnet for public/cluster networks

- All Ceph instances now communicate using public subnet and
  additionally OSDs communicate with each other using private cluster
  subnet
- Workaround for
  https://github.com/vagrant-libvirt/vagrant-libvirt/issues/645
- Fix for #952 to avoid concatenated MAC addresses caused by
  vagrant-libvirt bug.

Vagrantfile

@@ -19,7 +19,8 @@ RESTAPI        = settings['restapi']
 NRBD_MIRRORS   = settings['rbd_mirror_vms']
 CLIENTS        = settings['client_vms']
 NISCSI_GWS     = settings['iscsi_gw_vms']
-SUBNET         = settings['subnet']
+PUBLIC_SUBNET  = settings['public_subnet']
+CLUSTER_SUBNET = settings['cluster_subnet']
 BOX            = settings['vagrant_box']
 BOX_URL        = settings['vagrant_box_url']
 SYNC_DIR       = settings['vagrant_sync_dir']
@@ -68,9 +69,9 @@ ansible_provision = proc do |ansible|
   end
 
   ansible.extra_vars = {
-      cluster_network: "#{SUBNET}.0/24",
+      cluster_network: "#{CLUSTER_SUBNET}.0/24",
       journal_size: 100,
-      public_network: "#{SUBNET}.0/24",
+      public_network: "#{PUBLIC_SUBNET}.0/24",
   }
   if settings['ceph_install_source'] == 'dev' then
     ansible.extra_vars['ceph_dev'] = true
@@ -92,7 +93,7 @@ ansible_provision = proc do |ansible|
       restapi_containerized_deployment: 'true',
       rbd_mirror_containerized_deployment: 'true',
       ceph_mon_docker_interface: ETH,
-      ceph_mon_docker_subnet: "#{SUBNET}.0/24",
+      ceph_mon_docker_subnet: "#{PUBLIC_SUBNET}.0/24",
       ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE,OSD_JOURNAL_SIZE=100",
       ceph_osd_docker_devices: settings['disks'],
       ceph_docker_on_openstack: BOX == 'openstack',
@@ -114,11 +115,11 @@ ansible_provision = proc do |ansible|
     # Use monitor_address_block instead of monitor_interface:
     ansible.extra_vars.delete(:monitor_interface)
     ansible.extra_vars = ansible.extra_vars.merge({
-      cluster_network: "#{SUBNET}.0/16",
+      cluster_network: "#{CLUSTER_SUBNET}.0/16",
       devices: ['/dev/sdc'], # hardcode leftover disk
       journal_collocation: 'true',
-      monitor_address_block: "#{SUBNET}.0/16",
+      monitor_address_block: "#{PUBLIC_SUBNET}.0/16",
-      public_network: "#{SUBNET}.0/16",
+      public_network: "#{PUBLIC_SUBNET}.0/16",
     })
   end
 
@@ -183,7 +184,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "#{LABEL_PREFIX}client#{i}" do |client|
       client.vm.hostname = "#{LABEL_PREFIX}ceph-client#{i}"
       if ASSIGN_STATIC_IP
-        client.vm.network :private_network, ip: "#{SUBNET}.4#{i}"
+        client.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.4#{i}"
       end
       # Virtualbox
       client.vm.provider :virtualbox do |vb|
@@ -216,7 +217,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "#{LABEL_PREFIX}rgw#{i}" do |rgw|
       rgw.vm.hostname = "#{LABEL_PREFIX}ceph-rgw#{i}"
       if ASSIGN_STATIC_IP
-        rgw.vm.network :private_network, ip: "#{SUBNET}.5#{i}"
+        rgw.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.5#{i}"
       end
 
       # Virtualbox
@@ -250,7 +251,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "nfs#{i}" do |nfs|
       nfs.vm.hostname = "ceph-nfs#{i}"
       if ASSIGN_STATIC_IP
-        nfs.vm.network :private_network, ip: "#{SUBNET}.6#{i}"
+        nfs.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.6#{i}"
       end
 
       # Virtualbox
@@ -284,7 +285,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "#{LABEL_PREFIX}mds#{i}" do |mds|
       mds.vm.hostname = "#{LABEL_PREFIX}ceph-mds#{i}"
       if ASSIGN_STATIC_IP
-        mds.vm.network :private_network, ip: "#{SUBNET}.7#{i}"
+        mds.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.7#{i}"
       end
       # Virtualbox
       mds.vm.provider :virtualbox do |vb|
@@ -316,7 +317,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "#{LABEL_PREFIX}rbd_mirror#{i}" do |rbd_mirror|
       rbd_mirror.vm.hostname = "#{LABEL_PREFIX}ceph-rbd-mirror#{i}"
       if ASSIGN_STATIC_IP
-        rbd_mirror.vm.network :private_network, ip: "#{SUBNET}.8#{i}"
+        rbd_mirror.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.8#{i}"
       end
       # Virtualbox
       rbd_mirror.vm.provider :virtualbox do |vb|
@@ -348,7 +349,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "#{LABEL_PREFIX}iscsi_gw#{i}" do |iscsi_gw|
       iscsi_gw.vm.hostname = "#{LABEL_PREFIX}ceph-iscsi-gw#{i}"
       if ASSIGN_STATIC_IP
-        iscsi_gw.vm.network :private_network, ip: "#{SUBNET}.9#{i}"
+        iscsi_gw.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.9#{i}"
       end
       # Virtualbox
       iscsi_gw.vm.provider :virtualbox do |vb|
@@ -380,7 +381,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "#{LABEL_PREFIX}mon#{i}" do |mon|
       mon.vm.hostname = "#{LABEL_PREFIX}ceph-mon#{i}"
       if ASSIGN_STATIC_IP
-        mon.vm.network :private_network, ip: "#{SUBNET}.1#{i}"
+        mon.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.1#{i}"
       end
       # Virtualbox
       mon.vm.provider :virtualbox do |vb|
@@ -413,8 +414,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.define "#{LABEL_PREFIX}osd#{i}" do |osd|
       osd.vm.hostname = "#{LABEL_PREFIX}ceph-osd#{i}"
       if ASSIGN_STATIC_IP
-        osd.vm.network :private_network, ip: "#{SUBNET}.10#{i}"
+        osd.vm.network :private_network, ip: "#{PUBLIC_SUBNET}.10#{i}"
-        osd.vm.network :private_network, ip: "#{SUBNET}.20#{i}"
+        osd.vm.network :private_network, ip: "#{CLUSTER_SUBNET}.20#{i}"
       end
       # Virtualbox
       osd.vm.provider :virtualbox do |vb|

vagrant_variables.yml.atomic

@@ -15,8 +15,9 @@ iscsi_gw_vms: 0
 # Deploy RESTAPI on each of the Monitors
 #restapi: false
 
-# SUBNET TO USE FOR THE VMS
+# SUBNETS TO USE FOR THE VMS
-subnet: 192.168.0
+public_subnet: 192.168.0
+cluster_subnet: 192.168.1
 
 # MEMORY
 memory: 1024

vagrant_variables.yml.linode

@@ -16,7 +16,8 @@ cloud_datacenter: 'newark'
 memory: 2048
 
 # The private network on Linode, you probably don't want to change this.
-subnet: 192.168.0
+public_subnet: 192.168.0
+cluster_subnet: 192.168.1
 
 # DEFINE THE NUMBER OF VMS TO RUN
 mon_vms: 3

vagrant_variables.yml.openstack

@@ -16,7 +16,8 @@ client_vms: 0
 
 # SUBNET TO USE FOR THE VMS
 # Use whatever private subnet your Openstack VMs are given
-subnet: 172.17.72 
+public_subnet: 172.17.72
+cluster_subnet: 172.17.73
 
 # For Openstack VMs, the disk will depend on what you are allocated
 disks: "[ '/dev/vdb' ]"

vagrant_variables.yml.sample

@@ -20,8 +20,9 @@ restapi: true
 # valid values are 'stable' and 'dev'
 ceph_install_source: stable
 
-# SUBNET TO USE FOR THE VMS
+# SUBNETS TO USE FOR THE VMS
-subnet: 192.168.42
+public_subnet: 192.168.42
+cluster_subnet: 192.168.43
 
 # MEMORY
 # set 1024 for CentOS