302 lines
12 KiB
Ruby
302 lines
12 KiB
Ruby
# -*- mode: ruby -*-
|
|
# # vi: set ft=ruby :
|
|
|
|
# For help on using kubespray with vagrant, check out docs/developers/vagrant.md
|
|
|
|
require 'fileutils'
|
|
|
|
Vagrant.require_version ">= 2.0.0"
|
|
|
|
CONFIG = File.join(File.dirname(__FILE__), ENV['KUBESPRAY_VAGRANT_CONFIG'] || 'vagrant/config.rb')
|
|
|
|
FLATCAR_URL_TEMPLATE = "https://%s.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.json"
|
|
|
|
# Uniq disk UUID for libvirt
|
|
DISK_UUID = Time.now.utc.to_i
|
|
|
|
SUPPORTED_OS = {
|
|
"flatcar-stable" => {box: "flatcar-stable", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["stable"]},
|
|
"flatcar-beta" => {box: "flatcar-beta", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["beta"]},
|
|
"flatcar-alpha" => {box: "flatcar-alpha", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["alpha"]},
|
|
"flatcar-edge" => {box: "flatcar-edge", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]},
|
|
"ubuntu2004" => {box: "generic/ubuntu2004", user: "vagrant"},
|
|
"ubuntu2204" => {box: "generic/ubuntu2204", user: "vagrant"},
|
|
"ubuntu2404" => {box: "bento/ubuntu-24.04", user: "vagrant"},
|
|
"centos8" => {box: "centos/8", user: "vagrant"},
|
|
"centos8-bento" => {box: "bento/centos-8", user: "vagrant"},
|
|
"almalinux8" => {box: "almalinux/8", user: "vagrant"},
|
|
"almalinux8-bento" => {box: "bento/almalinux-8", user: "vagrant"},
|
|
"rockylinux8" => {box: "rockylinux/8", user: "vagrant"},
|
|
"rockylinux9" => {box: "rockylinux/9", user: "vagrant"},
|
|
"fedora37" => {box: "fedora/37-cloud-base", user: "vagrant"},
|
|
"fedora38" => {box: "fedora/38-cloud-base", user: "vagrant"},
|
|
"opensuse" => {box: "opensuse/Leap-15.4.x86_64", user: "vagrant"},
|
|
"opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
|
|
"oraclelinux" => {box: "generic/oracle7", user: "vagrant"},
|
|
"oraclelinux8" => {box: "generic/oracle8", user: "vagrant"},
|
|
"rhel8" => {box: "generic/rhel8", user: "vagrant"},
|
|
"debian11" => {box: "debian/bullseye64", user: "vagrant"},
|
|
"debian12" => {box: "debian/bookworm64", user: "vagrant"},
|
|
}
|
|
|
|
if File.exist?(CONFIG)
|
|
require CONFIG
|
|
end
|
|
|
|
# Defaults for config options defined in CONFIG
|
|
$num_instances ||= 3
|
|
$instance_name_prefix ||= "k8s"
|
|
$vm_gui ||= false
|
|
$vm_memory ||= 2048
|
|
$vm_cpus ||= 2
|
|
$shared_folders ||= {}
|
|
$forwarded_ports ||= {}
|
|
$subnet ||= "172.18.8"
|
|
$subnet_ipv6 ||= "fd3c:b398:0698:0756"
|
|
$os ||= "ubuntu2004"
|
|
$network_plugin ||= "flannel"
|
|
$inventory ||= "inventory/sample"
|
|
$inventories ||= [$inventory]
|
|
# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
|
|
$multi_networking ||= "False"
|
|
$download_run_once ||= "True"
|
|
$download_force_cache ||= "False"
|
|
# The first three nodes are etcd servers
|
|
$etcd_instances ||= [$num_instances, 3].min
|
|
# The first two nodes are kube masters
|
|
$kube_master_instances ||= [$num_instances, 2].min
|
|
# All nodes are kube nodes
|
|
$kube_node_instances ||= $num_instances
|
|
# The following only works when using the libvirt provider
|
|
$kube_node_instances_with_disks ||= false
|
|
$kube_node_instances_with_disks_size ||= "20G"
|
|
$kube_node_instances_with_disks_number ||= 2
|
|
$override_disk_size ||= false
|
|
$disk_size ||= "20GB"
|
|
$local_path_provisioner_enabled ||= "False"
|
|
$local_path_provisioner_claim_root ||= "/opt/local-path-provisioner/"
|
|
$libvirt_nested ||= false
|
|
# boolean or string (e.g. "-vvv")
|
|
$ansible_verbosity ||= false
|
|
$ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || ""
|
|
|
|
$vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant")
|
|
|
|
$playbook ||= "cluster.yml"
|
|
$extra_vars ||= {}
|
|
|
|
host_vars = {}
|
|
|
|
# throw error if os is not supported
|
|
if ! SUPPORTED_OS.key?($os)
|
|
puts "Unsupported OS: #{$os}"
|
|
puts "Supported OS are: #{SUPPORTED_OS.keys.join(', ')}"
|
|
exit 1
|
|
end
|
|
|
|
$box = SUPPORTED_OS[$os][:box]
|
|
|
|
if Vagrant.has_plugin?("vagrant-proxyconf")
|
|
$no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
|
|
(1..$num_instances).each do |i|
|
|
$no_proxy += ",#{$subnet}.#{i+100}"
|
|
end
|
|
end
|
|
|
|
Vagrant.configure("2") do |config|
|
|
|
|
config.vm.box = $box
|
|
if SUPPORTED_OS[$os].has_key? :box_url
|
|
config.vm.box_url = SUPPORTED_OS[$os][:box_url]
|
|
end
|
|
config.ssh.username = SUPPORTED_OS[$os][:user]
|
|
|
|
# plugin conflict
|
|
if Vagrant.has_plugin?("vagrant-vbguest") then
|
|
config.vbguest.auto_update = false
|
|
end
|
|
|
|
# always use Vagrants insecure key
|
|
config.ssh.insert_key = false
|
|
|
|
if ($override_disk_size)
|
|
unless Vagrant.has_plugin?("vagrant-disksize")
|
|
system "vagrant plugin install vagrant-disksize"
|
|
end
|
|
config.disksize.size = $disk_size
|
|
end
|
|
|
|
(1..$num_instances).each do |i|
|
|
config.vm.define vm_name = "%s-%01d" % [$instance_name_prefix, i] do |node|
|
|
|
|
node.vm.hostname = vm_name
|
|
|
|
if Vagrant.has_plugin?("vagrant-proxyconf")
|
|
node.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ""
|
|
node.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ""
|
|
node.proxy.no_proxy = $no_proxy
|
|
end
|
|
|
|
["vmware_fusion", "vmware_workstation"].each do |vmware|
|
|
node.vm.provider vmware do |v|
|
|
v.vmx['memsize'] = $vm_memory
|
|
v.vmx['numvcpus'] = $vm_cpus
|
|
end
|
|
end
|
|
|
|
node.vm.provider :virtualbox do |vb|
|
|
vb.memory = $vm_memory
|
|
vb.cpus = $vm_cpus
|
|
vb.gui = $vm_gui
|
|
vb.linked_clone = true
|
|
vb.customize ["modifyvm", :id, "--vram", "8"] # ubuntu defaults to 256 MB which is a waste of precious RAM
|
|
vb.customize ["modifyvm", :id, "--audio", "none"]
|
|
end
|
|
|
|
node.vm.provider :libvirt do |lv|
|
|
lv.nested = $libvirt_nested
|
|
lv.cpu_mode = "host-model"
|
|
lv.memory = $vm_memory
|
|
lv.cpus = $vm_cpus
|
|
lv.default_prefix = 'kubespray'
|
|
# Fix kernel panic on fedora 28
|
|
if $os == "fedora"
|
|
lv.cpu_mode = "host-passthrough"
|
|
end
|
|
end
|
|
|
|
if $kube_node_instances_with_disks
|
|
# Libvirt
|
|
driverletters = ('a'..'z').to_a
|
|
node.vm.provider :libvirt do |lv|
|
|
# always make /dev/sd{a/b/c} so that CI can ensure that
|
|
# virtualbox and libvirt will have the same devices to use for OSDs
|
|
(1..$kube_node_instances_with_disks_number).each do |d|
|
|
lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "scsi"
|
|
end
|
|
end
|
|
node.vm.provider :virtualbox do |vb|
|
|
# always make /dev/sd{a/b/c} so that CI can ensure that
|
|
# virtualbox and libvirt will have the same devices to use for OSDs
|
|
(1..$kube_node_instances_with_disks_number).each do |d|
|
|
vb.customize ['createhd', '--filename', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--size', $kube_node_instances_with_disks_size] # 10GB disk
|
|
vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', d, '--device', 0, '--type', 'hdd', '--medium', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--nonrotational', 'on', '--mtype', 'normal']
|
|
end
|
|
end
|
|
end
|
|
|
|
if $expose_docker_tcp
|
|
node.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
|
|
end
|
|
|
|
$forwarded_ports.each do |guest, host|
|
|
node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
|
|
end
|
|
|
|
if ["rhel7","rhel8"].include? $os
|
|
# Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot
|
|
# be installed until the host is registered with a valid Red Hat support subscription
|
|
node.vm.synced_folder ".", "/vagrant", disabled: false
|
|
$shared_folders.each do |src, dst|
|
|
node.vm.synced_folder src, dst
|
|
end
|
|
else
|
|
node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']
|
|
$shared_folders.each do |src, dst|
|
|
node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
|
|
end
|
|
end
|
|
|
|
ip = "#{$subnet}.#{i+100}"
|
|
node.vm.network :private_network,
|
|
:ip => ip,
|
|
:libvirt__guest_ipv6 => 'yes',
|
|
:libvirt__ipv6_address => "#{$subnet_ipv6}::#{i+100}",
|
|
:libvirt__ipv6_prefix => "64",
|
|
:libvirt__forward_mode => "none",
|
|
:libvirt__dhcp_enabled => false
|
|
|
|
# Disable swap for each vm
|
|
node.vm.provision "shell", inline: "swapoff -a"
|
|
|
|
# ubuntu2004 and ubuntu2204 have IPv6 explicitly disabled. This undoes that.
|
|
if ["ubuntu2004", "ubuntu2204"].include? $os
|
|
node.vm.provision "shell", inline: "rm -f /etc/modprobe.d/local.conf"
|
|
node.vm.provision "shell", inline: "sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf"
|
|
end
|
|
# Hack for fedora37/38 to get the IP address of the second interface
|
|
if ["fedora37", "fedora38"].include? $os
|
|
config.vm.provision "shell", inline: <<-SHELL
|
|
nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d "=" -f2)
|
|
nmcli conn modify 'Wired connection 2' ipv4.method manual
|
|
service NetworkManager restart
|
|
SHELL
|
|
end
|
|
|
|
# Rockylinux boxes needs UEFI
|
|
if ["rockylinux8", "rockylinux9"].include? $os
|
|
config.vm.provider "libvirt" do |domain|
|
|
domain.loader = "/usr/share/OVMF/x64/OVMF_CODE.fd"
|
|
end
|
|
end
|
|
|
|
# Disable firewalld on oraclelinux/redhat vms
|
|
if ["oraclelinux","oraclelinux8","rhel7","rhel8","rockylinux8"].include? $os
|
|
node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld"
|
|
end
|
|
|
|
host_vars[vm_name] = {
|
|
"ip": ip,
|
|
"flannel_interface": "eth1",
|
|
"kube_network_plugin": $network_plugin,
|
|
"kube_network_plugin_multus": $multi_networking,
|
|
"download_run_once": $download_run_once,
|
|
"download_localhost": "False",
|
|
"download_cache_dir": ENV['HOME'] + "/kubespray_cache",
|
|
# Make kubespray cache even when download_run_once is false
|
|
"download_force_cache": $download_force_cache,
|
|
# Keeping the cache on the nodes can improve provisioning speed while debugging kubespray
|
|
"download_keep_remote_cache": "False",
|
|
"docker_rpm_keepcache": "1",
|
|
# These two settings will put kubectl and admin.config in $inventory/artifacts
|
|
"kubeconfig_localhost": "True",
|
|
"kubectl_localhost": "True",
|
|
"local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
|
|
"local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
|
|
"ansible_ssh_user": SUPPORTED_OS[$os][:user],
|
|
"ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
|
|
"unsafe_show_logs": "True"
|
|
}
|
|
|
|
# Only execute the Ansible provisioner once, when all the machines are up and ready.
|
|
# And limit the action to gathering facts, the full playbook is going to be ran by testcases_run.sh
|
|
if i == $num_instances
|
|
node.vm.provision "ansible" do |ansible|
|
|
ansible.playbook = $playbook
|
|
ansible.compatibility_mode = "2.0"
|
|
ansible.verbose = $ansible_verbosity
|
|
ansible.become = true
|
|
ansible.limit = "all,localhost"
|
|
ansible.host_key_checking = false
|
|
ansible.raw_arguments = ["--forks=#{$num_instances}",
|
|
"--flush-cache",
|
|
"-e ansible_become_pass=vagrant"] +
|
|
$inventories.map {|inv| ["-i", inv]}.flatten
|
|
ansible.host_vars = host_vars
|
|
ansible.extra_vars = $extra_vars
|
|
if $ansible_tags != ""
|
|
ansible.tags = [$ansible_tags]
|
|
end
|
|
ansible.groups = {
|
|
"etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"],
|
|
"kube_control_plane" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"],
|
|
"kube_node" => ["#{$instance_name_prefix}-[1:#{$kube_node_instances}]"],
|
|
"k8s_cluster:children" => ["kube_control_plane", "kube_node"],
|
|
}
|
|
end
|
|
end
|
|
|
|
end
|
|
end
|
|
end
|