diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index 09c7c4ddb..be74b7c4e 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -264,6 +264,107 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. |`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage | |`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage | |`use_server_group` | Create and use openstack nova servergroups, default: false | +|`k8s_nodes` | Map containing worker node definition, see explanation below | + +##### k8s_nodes +Allows a custom defintion of worker nodes giving the operator full control over individual node flavor and +availability zone placement. To enable the use of this mode set the `number_of_k8s_nodes` and +`number_of_k8s_nodes_no_floating_ip` variables to 0. Then define your desired worker node configuration +using the `k8s_nodes` variable. + +For example: +``` +k8s_nodes = { + "1" = { + "az" = "sto1" + "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" + "floating_ip" = true + }, + "2" = { + "az" = "sto2" + "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" + "floating_ip" = true + }, + "3" = { + "az" = "sto3" + "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" + "floating_ip" = true + } +} +``` + +Would result in the same configuration as: +``` +number_of_k8s_nodes = 3 +flavor_k8s_node = "83d8b44a-26a0-4f02-a981-079446926445" +az_list = ["sto1", "sto2", "sto3"] +``` + +And: +``` +k8s_nodes = { + "ing-1" = { + "az" = "sto1" + "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" + "floating_ip" = true + }, + "ing-2" = { + "az" = "sto2" + "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" + "floating_ip" = true + }, + "ing-3" = { + "az" = "sto3" + "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" + "floating_ip" = true + }, + "big-1" = { + "az" = "sto1" + "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b" + "floating_ip" = false + }, + "big-2" = { + "az" = "sto2" + "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b" + "floating_ip" = false + }, + "big-3" = { + "az" = "sto3" + "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b" + "floating_ip" = false + }, + "small-1" = { + "az" = "sto1" + "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e" + "floating_ip" = false + }, + "small-2" = { + "az" = "sto2" + "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e" + "floating_ip" = false + }, + "small-3" = { + "az" = "sto3" + "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e" + "floating_ip" = false + } +} +``` + +Would result in three nodes in each availability zone each with their own separate naming, +flavor and floating ip configuration. + +The "schema": +``` +k8s_nodes = { + "key | node name suffix, must be unique" = { + "az" = string + "flavor" = string + "floating_ip" = bool + }, +} +``` +All values are required. #### Terraform state files @@ -504,3 +605,81 @@ $ ansible-playbook --become -i inventory/$CLUSTER/hosts ./contrib/network-storag ## What's next Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/). + +## Appendix + +### Migration from `number_of_k8s_nodes*` to `k8s_nodes` +If you currently have a cluster defined using the `number_of_k8s_nodes*` variables and wish +to migrate to the `k8s_nodes` style you can do it like so: + +```ShellSession +$ terraform state list +module.compute.data.openstack_images_image_v2.gfs_image +module.compute.data.openstack_images_image_v2.vm_image +module.compute.openstack_compute_floatingip_associate_v2.k8s_master[0] +module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0] +module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1] +module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2] +module.compute.openstack_compute_instance_v2.k8s_master[0] +module.compute.openstack_compute_instance_v2.k8s_node[0] +module.compute.openstack_compute_instance_v2.k8s_node[1] +module.compute.openstack_compute_instance_v2.k8s_node[2] +module.compute.openstack_compute_keypair_v2.k8s +module.compute.openstack_compute_servergroup_v2.k8s_etcd[0] +module.compute.openstack_compute_servergroup_v2.k8s_master[0] +module.compute.openstack_compute_servergroup_v2.k8s_node[0] +module.compute.openstack_networking_secgroup_rule_v2.bastion[0] +module.compute.openstack_networking_secgroup_rule_v2.egress[0] +module.compute.openstack_networking_secgroup_rule_v2.k8s +module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[0] +module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[1] +module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[2] +module.compute.openstack_networking_secgroup_rule_v2.k8s_master[0] +module.compute.openstack_networking_secgroup_rule_v2.worker[0] +module.compute.openstack_networking_secgroup_rule_v2.worker[1] +module.compute.openstack_networking_secgroup_rule_v2.worker[2] +module.compute.openstack_networking_secgroup_rule_v2.worker[3] +module.compute.openstack_networking_secgroup_rule_v2.worker[4] +module.compute.openstack_networking_secgroup_v2.bastion[0] +module.compute.openstack_networking_secgroup_v2.k8s +module.compute.openstack_networking_secgroup_v2.k8s_master +module.compute.openstack_networking_secgroup_v2.worker +module.ips.null_resource.dummy_dependency +module.ips.openstack_networking_floatingip_v2.k8s_master[0] +module.ips.openstack_networking_floatingip_v2.k8s_node[0] +module.ips.openstack_networking_floatingip_v2.k8s_node[1] +module.ips.openstack_networking_floatingip_v2.k8s_node[2] +module.network.openstack_networking_network_v2.k8s[0] +module.network.openstack_networking_router_interface_v2.k8s[0] +module.network.openstack_networking_router_v2.k8s[0] +module.network.openstack_networking_subnet_v2.k8s[0] +$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["1"]' +Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"1\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["2"]' +Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"2\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["3"]' +Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"3\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[0]' 'module.compute.openstack_compute_instance_v2.k8s_node["1"]' +Move "module.compute.openstack_compute_instance_v2.k8s_node[0]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"1\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[1]' 'module.compute.openstack_compute_instance_v2.k8s_node["2"]' +Move "module.compute.openstack_compute_instance_v2.k8s_node[1]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"2\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[2]' 'module.compute.openstack_compute_instance_v2.k8s_node["3"]' +Move "module.compute.openstack_compute_instance_v2.k8s_node[2]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"3\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[0]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["1"]' +Move "module.ips.openstack_networking_floatingip_v2.k8s_node[0]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"1\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[1]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["2"]' +Move "module.ips.openstack_networking_floatingip_v2.k8s_node[1]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"2\"]" +Successfully moved 1 object(s). +$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[2]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["3"]' +Move "module.ips.openstack_networking_floatingip_v2.k8s_node[2]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"3\"]" +Successfully moved 1 object(s). +``` + +Of course for nodes without floating ips those steps can be omitted. diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf index 746b5a550..511027479 100644 --- a/contrib/terraform/openstack/kubespray.tf +++ b/contrib/terraform/openstack/kubespray.tf @@ -26,6 +26,7 @@ module "ips" { external_net = "${var.external_net}" network_name = "${var.network_name}" router_id = "${module.network.router_id}" + k8s_nodes = "${var.k8s_nodes}" } module "compute" { @@ -43,6 +44,7 @@ module "compute" { number_of_bastions = "${var.number_of_bastions}" number_of_k8s_nodes_no_floating_ip = "${var.number_of_k8s_nodes_no_floating_ip}" number_of_gfs_nodes_no_floating_ip = "${var.number_of_gfs_nodes_no_floating_ip}" + k8s_nodes = "${var.k8s_nodes}" bastion_root_volume_size_in_gb = "${var.bastion_root_volume_size_in_gb}" etcd_root_volume_size_in_gb = "${var.etcd_root_volume_size_in_gb}" master_root_volume_size_in_gb = "${var.master_root_volume_size_in_gb}" @@ -63,6 +65,7 @@ module "compute" { k8s_master_fips = "${module.ips.k8s_master_fips}" k8s_master_no_etcd_fips = "${module.ips.k8s_master_no_etcd_fips}" k8s_node_fips = "${module.ips.k8s_node_fips}" + k8s_nodes_fips = "${module.ips.k8s_nodes_fips}" bastion_fips = "${module.ips.bastion_fips}" bastion_allowed_remote_ips = "${var.bastion_allowed_remote_ips}" master_allowed_remote_ips = "${var.master_allowed_remote_ips}" @@ -95,7 +98,7 @@ output "k8s_master_fips" { } output "k8s_node_fips" { - value = "${module.ips.k8s_node_fips}" + value = "${var.number_of_k8s_nodes > 0 ? module.ips.k8s_node_fips : [for key, value in module.ips.k8s_nodes_fips : value.address]}" } output "bastion_fips" { diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index ad6f0dc63..8be6346c4 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -465,6 +465,53 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" { } } +resource "openstack_compute_instance_v2" "k8s_nodes" { + for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {} + name = "${var.cluster_name}-k8s-node-${each.key}" + availability_zone = "${each.value.az}" + image_name = "${var.image}" + flavor_id = "${each.value.flavor}" + key_pair = "${openstack_compute_keypair_v2.k8s.name}" + + dynamic "block_device" { + for_each = var.node_root_volume_size_in_gb > 0 ? [var.image] : [] + content { + uuid = "${data.openstack_images_image_v2.vm_image.id}" + source_type = "image" + volume_size = "${var.node_root_volume_size_in_gb}" + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + } + + network { + name = "${var.network_name}" + } + + security_groups = ["${openstack_networking_secgroup_v2.k8s.name}", + "${openstack_networking_secgroup_v2.worker.name}", + ] + + dynamic "scheduler_hints" { + for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : [] + content { + group = "${openstack_compute_servergroup_v2.k8s_node[0].id}" + } + } + + metadata = { + ssh_user = "${var.ssh_user}" + kubespray_groups = "kube-node,k8s-cluster,%{if each.value.floating_ip == false}no-floating,%{endif}${var.supplementary_node_groups}" + depends_on = "${var.network_id}" + use_access_ip = "${var.use_access_ip}" + } + + provisioner "local-exec" { + command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > group_vars/no-floating.yml%{else}true%{endif}" + } +} + resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" { name = "${var.cluster_name}-gfs-node-nf-${count.index + 1}" count = "${var.number_of_gfs_nodes_no_floating_ip}" @@ -530,7 +577,14 @@ resource "openstack_compute_floatingip_associate_v2" "k8s_master_no_etcd" { resource "openstack_compute_floatingip_associate_v2" "k8s_node" { count = "${var.node_root_volume_size_in_gb == 0 ? var.number_of_k8s_nodes : 0}" floating_ip = "${var.k8s_node_fips[count.index]}" - instance_id = "${element(openstack_compute_instance_v2.k8s_node.*.id, count.index)}" + instance_id = "${element(openstack_compute_instance_v2.k8s_node[*].id, count.index)}" + wait_until_associated = "${var.wait_for_floatingip}" +} + +resource "openstack_compute_floatingip_associate_v2" "k8s_nodes" { + for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip } : {} + floating_ip = "${var.k8s_nodes_fips[each.key].address}" + instance_id = "${openstack_compute_instance_v2.k8s_nodes[each.key].id}" wait_until_associated = "${var.wait_for_floatingip}" } @@ -545,4 +599,4 @@ resource "openstack_compute_volume_attach_v2" "glusterfs_volume" { count = "${var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0}" instance_id = "${element(openstack_compute_instance_v2.glusterfs_node_no_floating_ip.*.id, count.index)}" volume_id = "${element(openstack_blockstorage_volume_v2.glusterfs_volume.*.id, count.index)}" -} \ No newline at end of file +} diff --git a/contrib/terraform/openstack/modules/compute/variables.tf b/contrib/terraform/openstack/modules/compute/variables.tf index 4fab83c57..b85f56250 100644 --- a/contrib/terraform/openstack/modules/compute/variables.tf +++ b/contrib/terraform/openstack/modules/compute/variables.tf @@ -76,6 +76,10 @@ variable "k8s_node_fips" { type = "list" } +variable "k8s_nodes_fips" { + type = "map" +} + variable "bastion_fips" { type = "list" } @@ -96,6 +100,8 @@ variable "k8s_allowed_egress_ips" { type = "list" } +variable "k8s_nodes" {} + variable "wait_for_floatingip" {} variable "supplementary_master_groups" { diff --git a/contrib/terraform/openstack/modules/ips/main.tf b/contrib/terraform/openstack/modules/ips/main.tf index a4d5cd637..15f6f1f05 100644 --- a/contrib/terraform/openstack/modules/ips/main.tf +++ b/contrib/terraform/openstack/modules/ips/main.tf @@ -27,3 +27,10 @@ resource "openstack_networking_floatingip_v2" "bastion" { pool = "${var.floatingip_pool}" depends_on = ["null_resource.dummy_dependency"] } + +resource "openstack_networking_floatingip_v2" "k8s_nodes" { + for_each = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip } : {} + pool = "${var.floatingip_pool}" + depends_on = ["null_resource.dummy_dependency"] +} + diff --git a/contrib/terraform/openstack/modules/ips/outputs.tf b/contrib/terraform/openstack/modules/ips/outputs.tf index 703e6f4cd..f74e1c930 100644 --- a/contrib/terraform/openstack/modules/ips/outputs.tf +++ b/contrib/terraform/openstack/modules/ips/outputs.tf @@ -10,6 +10,10 @@ output "k8s_node_fips" { value = "${openstack_networking_floatingip_v2.k8s_node[*].address}" } +output "k8s_nodes_fips" { + value = "${openstack_networking_floatingip_v2.k8s_nodes}" +} + output "bastion_fips" { value = "${openstack_networking_floatingip_v2.bastion[*].address}" } diff --git a/contrib/terraform/openstack/modules/ips/variables.tf b/contrib/terraform/openstack/modules/ips/variables.tf index a2cb54538..40e4a759f 100644 --- a/contrib/terraform/openstack/modules/ips/variables.tf +++ b/contrib/terraform/openstack/modules/ips/variables.tf @@ -14,4 +14,6 @@ variable "network_name" {} variable "router_id" { default = "" -} \ No newline at end of file +} + +variable "k8s_nodes" {} diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf index b0fe4a6ed..334e6c678 100644 --- a/contrib/terraform/openstack/variables.tf +++ b/contrib/terraform/openstack/variables.tf @@ -225,3 +225,8 @@ variable "router_id" { description = "uuid of an externally defined router to use" default = null } + +variable "k8s_nodes" { + default = {} +} +