kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove deprecated provider, fix flatcar configs, enable CI tests and refactor hetzner terraform (#10002) 

* Remove deprecated provider and fix flatcar configs

* Refactor for DRYness

* Add missing line endings

* Enable tests for hetzner terraform in CI

* Add missing inventory for CI tests
pull/10025/head
Qasim Mehmood 2023-05-08 05:15:16 +05:00 committed by GitHub
parent 426b8913c0
commit ab6d204641
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 179 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitlab-ci/terraform.yml
View File

@ -80,6 +80,12 @@ tf-validate-exoscale:
TF_VERSION: $TERRAFORM_VERSION TF_VERSION: $TERRAFORM_VERSION
PROVIDER: exoscale PROVIDER: exoscale
tf-validate-hetzner:
extends: .terraform_validate
variables:
TF_VERSION: $TERRAFORM_VERSION
PROVIDER: hetzner
tf-validate-vsphere: tf-validate-vsphere:
extends: .terraform_validate extends: .terraform_validate
variables: variables:

12
contrib/terraform/hetzner/default.tfvars
View File

@ -1,6 +1,6 @@
prefix = "default" prefix = "default"
zone = "hel1" zone = "hel1"
network_zone = "eu-central" network_zone = "eu-central"
inventory_file = "inventory.ini" inventory_file = "inventory.ini"
ssh_public_keys = [ ssh_public_keys = [
@ -15,17 +15,17 @@ machines = {
"master-0" : { "master-0" : {
"node_type" : "master", "node_type" : "master",
"size" : "cx21", "size" : "cx21",
"image" : "ubuntu-20.04", "image" : "ubuntu-22.04",
}, },
"worker-0" : { "worker-0" : {
"node_type" : "worker", "node_type" : "worker",
"size" : "cx21", "size" : "cx21",
"image" : "ubuntu-20.04", "image" : "ubuntu-22.04",
}, },
"worker-1" : { "worker-1" : {
"node_type" : "worker", "node_type" : "worker",
"size" : "cx21", "size" : "cx21",
"image" : "ubuntu-20.04", "image" : "ubuntu-22.04",
} }
} }

43
contrib/terraform/hetzner/main.tf
View File

@ -2,7 +2,7 @@ provider "hcloud" {}
module "kubernetes" { module "kubernetes" {
source = "./modules/kubernetes-cluster" source = "./modules/kubernetes-cluster"
#source = "./modules/kubernetes-cluster-flatcar" # source = "./modules/kubernetes-cluster-flatcar"
prefix = var.prefix prefix = var.prefix
@ -14,7 +14,7 @@ module "kubernetes" {
#ssh_private_key_path = var.ssh_private_key_path #ssh_private_key_path = var.ssh_private_key_path
ssh_public_keys = var.ssh_public_keys ssh_public_keys = var.ssh_public_keys
network_zone = var.network_zone network_zone = var.network_zone
ssh_whitelist = var.ssh_whitelist ssh_whitelist = var.ssh_whitelist
api_server_whitelist = var.api_server_whitelist api_server_whitelist = var.api_server_whitelist
@ -26,31 +26,32 @@ module "kubernetes" {
# Generate ansible inventory # Generate ansible inventory
# #
data "template_file" "inventory" { locals {
template = file("${path.module}/templates/inventory.tpl") inventory = templatefile(
"${path.module}/templates/inventory.tpl",
vars = { {
connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d", connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
keys(module.kubernetes.master_ip_addresses), keys(module.kubernetes.master_ip_addresses),
values(module.kubernetes.master_ip_addresses).*.public_ip, values(module.kubernetes.master_ip_addresses).*.public_ip,
values(module.kubernetes.master_ip_addresses).*.private_ip, values(module.kubernetes.master_ip_addresses).*.private_ip,
range(1, length(module.kubernetes.master_ip_addresses) + 1))) range(1, length(module.kubernetes.master_ip_addresses) + 1)))
connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s", connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
keys(module.kubernetes.worker_ip_addresses), keys(module.kubernetes.worker_ip_addresses),
values(module.kubernetes.worker_ip_addresses).*.public_ip, values(module.kubernetes.worker_ip_addresses).*.public_ip,
values(module.kubernetes.worker_ip_addresses).*.private_ip)) values(module.kubernetes.worker_ip_addresses).*.private_ip))
list_master = join("\n", keys(module.kubernetes.master_ip_addresses)) list_master = join("\n", keys(module.kubernetes.master_ip_addresses))
list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses)) list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
network_id = module.kubernetes.network_id network_id = module.kubernetes.network_id
} }
)
} }
resource "null_resource" "inventories" { resource "null_resource" "inventories" {
provisioner "local-exec" { provisioner "local-exec" {
command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}" command = "echo '${local.inventory}' > ${var.inventory_file}"
} }
triggers = { triggers = {
template = data.template_file.inventory.rendered template = local.inventory
} }
} }

156
contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf
View File

@ -15,12 +15,12 @@ resource "hcloud_ssh_key" "first" {
public_key = var.ssh_public_keys.0 public_key = var.ssh_public_keys.0
} }
resource "hcloud_server" "master" { resource "hcloud_server" "machine" {
for_each = { for_each = {
for name, machine in var.machines : for name, machine in var.machines :
name => machine name => machine
if machine.node_type == "master"
} }
name = "${var.prefix}-${each.key}" name = "${var.prefix}-${each.key}"
ssh_keys = [hcloud_ssh_key.first.id] ssh_keys = [hcloud_ssh_key.first.id]
# boot into rescue OS # boot into rescue OS
@ -30,11 +30,11 @@ resource "hcloud_server" "master" {
server_type = each.value.size server_type = each.value.size
location = var.zone location = var.zone
connection { connection {
host = self.ipv4_address host = self.ipv4_address
timeout = "5m" timeout = "5m"
private_key = file(var.ssh_private_key_path) private_key = file(var.ssh_private_key_path)
} }
firewall_ids = [hcloud_firewall.machine.id] firewall_ids = each.value.node_type == "master" ? [hcloud_firewall.master.id] : [hcloud_firewall.worker.id]
provisioner "file" { provisioner "file" {
content = data.ct_config.machine-ignitions[each.key].rendered content = data.ct_config.machine-ignitions[each.key].rendered
destination = "/root/ignition.json" destination = "/root/ignition.json"
@ -45,9 +45,9 @@ resource "hcloud_server" "master" {
"set -ex", "set -ex",
"apt update", "apt update",
"apt install -y gawk", "apt install -y gawk",
"curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install", "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install",
"chmod +x flatcar-install", "chmod +x flatcar-install",
"./flatcar-install -s -i /root/ignition.json", "./flatcar-install -s -i /root/ignition.json -C stable",
"shutdown -r +1", "shutdown -r +1",
] ]
} }
@ -55,9 +55,10 @@ resource "hcloud_server" "master" {
# optional: # optional:
provisioner "remote-exec" { provisioner "remote-exec" {
connection { connection {
host = self.ipv4_address host = self.ipv4_address
timeout = "3m" private_key = file(var.ssh_private_key_path)
user = var.user_flatcar timeout = "3m"
user = var.user_flatcar
} }
inline = [ inline = [
@ -66,65 +67,11 @@ resource "hcloud_server" "master" {
} }
} }
resource "hcloud_server_network" "master" { resource "hcloud_server_network" "machine" {
for_each = hcloud_server.master
server_id = each.value.id
subnet_id = hcloud_network_subnet.kubernetes.id
}
resource "hcloud_server" "worker" {
for_each = { for_each = {
for name, machine in var.machines : for name, machine in var.machines :
name => machine name => hcloud_server.machine[name]
if machine.node_type == "worker"
} }
name = "${var.prefix}-${each.key}"
ssh_keys = [hcloud_ssh_key.first.id]
# boot into rescue OS
rescue = "linux64"
# dummy value for the OS because Flatcar is not available
image = each.value.image
server_type = each.value.size
location = var.zone
connection {
host = self.ipv4_address
timeout = "5m"
private_key = file(var.ssh_private_key_path)
}
firewall_ids = [hcloud_firewall.machine.id]
provisioner "file" {
content = data.ct_config.machine-ignitions[each.key].rendered
destination = "/root/ignition.json"
}
provisioner "remote-exec" {
inline = [
"set -ex",
"apt update",
"apt install -y gawk",
"curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install",
"chmod +x flatcar-install",
"./flatcar-install -s -i /root/ignition.json",
"shutdown -r +1",
]
}
# optional:
provisioner "remote-exec" {
connection {
host = self.ipv4_address
timeout = "3m"
user = var.user_flatcar
}
inline = [
"sudo hostnamectl set-hostname ${self.name}",
]
}
}
resource "hcloud_server_network" "worker" {
for_each = hcloud_server.worker
server_id = each.value.id server_id = each.value.id
subnet_id = hcloud_network_subnet.kubernetes.id subnet_id = hcloud_network_subnet.kubernetes.id
} }
@ -134,38 +81,33 @@ data "ct_config" "machine-ignitions" {
for name, machine in var.machines : for name, machine in var.machines :
name => machine name => machine
} }
content = data.template_file.machine-configs[each.key].rendered
strict = false
content = templatefile(
"${path.module}/templates/machine.yaml.tmpl",
{
ssh_keys = jsonencode(var.ssh_public_keys)
user_flatcar = var.user_flatcar
name = each.key
}
)
} }
data "template_file" "machine-configs" { resource "hcloud_firewall" "master" {
for_each = { name = "${var.prefix}-master-firewall"
for name, machine in var.machines :
name => machine
}
template = file("${path.module}/templates/machine.yaml.tmpl")
vars = {
ssh_keys = jsonencode(var.ssh_public_keys)
user_flatcar = jsonencode(var.user_flatcar)
name = each.key
}
}
resource "hcloud_firewall" "machine" {
name = "${var.prefix}-machine-firewall"
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "22" port = "22"
source_ips = var.ssh_whitelist source_ips = var.ssh_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "6443" port = "6443"
source_ips = var.api_server_whitelist source_ips = var.api_server_whitelist
} }
} }
@ -173,30 +115,30 @@ resource "hcloud_firewall" "worker" {
name = "${var.prefix}-worker-firewall" name = "${var.prefix}-worker-firewall"
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "22" port = "22"
source_ips = var.ssh_whitelist source_ips = var.ssh_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "80" port = "80"
source_ips = var.ingress_whitelist source_ips = var.ingress_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "443" port = "443"
source_ips = var.ingress_whitelist source_ips = var.ingress_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "30000-32767" port = "30000-32767"
source_ips = var.nodeport_whitelist source_ips = var.nodeport_whitelist
} }
} }

22
contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf
View File

@ -1,20 +1,22 @@
output "master_ip_addresses" { output "master_ip_addresses" {
value = { value = {
for key, instance in hcloud_server.master : for name, machine in var.machines :
instance.name => { name => {
"private_ip" = hcloud_server_network.master[key].ip "private_ip" = hcloud_server_network.machine[name].ip
"public_ip" = hcloud_server.master[key].ipv4_address "public_ip" = hcloud_server.machine[name].ipv4_address
} }
if machine.node_type == "master"
} }
} }
output "worker_ip_addresses" { output "worker_ip_addresses" {
value = { value = {
for key, instance in hcloud_server.worker : for name, machine in var.machines :
instance.name => { name => {
"private_ip" = hcloud_server_network.worker[key].ip "private_ip" = hcloud_server_network.machine[name].ip
"public_ip" = hcloud_server.worker[key].ipv4_address "public_ip" = hcloud_server.machine[name].ipv4_address
} }
if machine.node_type == "worker"
} }
} }

5
contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl
View File

@ -1,8 +1,11 @@
--- variant: flatcar
version: 1.0.0
passwd: passwd:
users: users:
- name: ${user_flatcar} - name: ${user_flatcar}
ssh_authorized_keys: ${ssh_keys} ssh_authorized_keys: ${ssh_keys}
storage: storage:
files: files:
- path: /home/core/works - path: /home/core/works

4
contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf
View File

@ -1,6 +1,6 @@
variable "zone" { variable "zone" {
type = string type = string
default = "fsn1" default = "fsn1"
} }
@ -9,7 +9,7 @@ variable "prefix" {
} }
variable "user_flatcar" { variable "user_flatcar" {
type = string type = string
default = "core" default = "core"
} }

5
contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf
View File

@ -1,13 +1,14 @@
terraform { terraform {
required_providers { required_providers {
hcloud = { hcloud = {
source = "hetznercloud/hcloud" source = "hetznercloud/hcloud"
} }
ct = { ct = {
source = "poseidon/ct" source = "poseidon/ct"
version = "0.11.0"
} }
null = { null = {
source = "hashicorp/null" source = "hashicorp/null"
} }
} }
} }

48
contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf
View File

@ -75,17 +75,17 @@ resource "hcloud_firewall" "master" {
name = "${var.prefix}-master-firewall" name = "${var.prefix}-master-firewall"
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "22" port = "22"
source_ips = var.ssh_whitelist source_ips = var.ssh_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "6443" port = "6443"
source_ips = var.api_server_whitelist source_ips = var.api_server_whitelist
} }
} }
@ -93,30 +93,30 @@ resource "hcloud_firewall" "worker" {
name = "${var.prefix}-worker-firewall" name = "${var.prefix}-worker-firewall"
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "22" port = "22"
source_ips = var.ssh_whitelist source_ips = var.ssh_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "80" port = "80"
source_ips = var.ingress_whitelist source_ips = var.ingress_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "443" port = "443"
source_ips = var.ingress_whitelist source_ips = var.ingress_whitelist
} }
rule { rule {
direction = "in" direction = "in"
protocol = "tcp" protocol = "tcp"
port = "30000-32767" port = "30000-32767"
source_ips = var.nodeport_whitelist source_ips = var.nodeport_whitelist
} }
} }

4
contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf
View File

@ -1,8 +1,8 @@
terraform { terraform {
required_providers { required_providers {
hcloud = { hcloud = {
source = "hetznercloud/hcloud" source = "hetznercloud/hcloud"
version = "1.31.1" version = "1.38.2"
} }
} }
required_version = ">= 0.14" required_version = ">= 0.14"

46
contrib/terraform/hetzner/sample-inventory/cluster.tfvars 100644
View File

@ -0,0 +1,46 @@
prefix = "default"
zone = "hel1"
network_zone = "eu-central"
inventory_file = "inventory.ini"
ssh_public_keys = [
# Put your public SSH key here
"ssh-rsa I-did-not-read-the-docs",
"ssh-rsa I-did-not-read-the-docs 2",
]
ssh_private_key_path = "~/.ssh/id_rsa"
machines = {
"master-0" : {
"node_type" : "master",
"size" : "cx21",
"image" : "ubuntu-22.04",
},
"worker-0" : {
"node_type" : "worker",
"size" : "cx21",
"image" : "ubuntu-22.04",
},
"worker-1" : {
"node_type" : "worker",
"size" : "cx21",
"image" : "ubuntu-22.04",
}
}
nodeport_whitelist = [
"0.0.0.0/0"
]
ingress_whitelist = [
"0.0.0.0/0"
]
ssh_whitelist = [
"0.0.0.0/0"
]
api_server_whitelist = [
"0.0.0.0/0"
]

1
contrib/terraform/hetzner/sample-inventory/group_vars 120000
View File

@ -0,0 +1 @@
../../../../inventory/sample/group_vars

2
contrib/terraform/hetzner/variables.tf
View File

@ -3,7 +3,7 @@ variable "zone" {
} }
variable "network_zone" { variable "network_zone" {
description = "The network zone where the cluster is running" description = "The network zone where the cluster is running"
default = "eu-central" default = "eu-central"
} }
variable "prefix" { variable "prefix" {

5
contrib/terraform/hetzner/versions.tf
View File

@ -2,14 +2,11 @@ terraform {
required_providers { required_providers {
hcloud = { hcloud = {
source = "hetznercloud/hcloud" source = "hetznercloud/hcloud"
version = "1.31.1" version = "1.38.2"
} }
null = { null = {
source = "hashicorp/null" source = "hashicorp/null"
} }
template = {
source = "hashicorp/template"
}
} }
required_version = ">= 0.14" required_version = ">= 0.14"
} }