Remove deprecated provider, fix flatcar configs, enable CI tests and refactor hetzner terraform (#10002)

* Remove deprecated provider and fix flatcar configs

* Refactor for DRYness

* Add missing line endings

* Enable tests for hetzner terraform in CI

* Add missing inventory for CI tests

.gitlab-ci/terraform.yml

@@ -80,6 +80,12 @@ tf-validate-exoscale:
     TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: exoscale
 
+tf-validate-hetzner:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: hetzner
+
 tf-validate-vsphere:
   extends: .terraform_validate
   variables:

contrib/terraform/hetzner/default.tfvars

@@ -15,17 +15,17 @@ machines = {
   "master-0" : {
     "node_type" : "master",
     "size" : "cx21",
-    "image" : "ubuntu-20.04",
+    "image" : "ubuntu-22.04",
   },
   "worker-0" : {
     "node_type" : "worker",
     "size" : "cx21",
-    "image" : "ubuntu-20.04",
+    "image" : "ubuntu-22.04",
   },
   "worker-1" : {
     "node_type" : "worker",
     "size" : "cx21",
-    "image" : "ubuntu-20.04",
+    "image" : "ubuntu-22.04",
   }
 }
 

contrib/terraform/hetzner/main.tf

@@ -2,7 +2,7 @@ provider "hcloud" {}
 
 module "kubernetes" {
   source = "./modules/kubernetes-cluster"
-  #source = "./modules/kubernetes-cluster-flatcar"
+  # source = "./modules/kubernetes-cluster-flatcar"
 
   prefix = var.prefix
 
@@ -26,10 +26,10 @@ module "kubernetes" {
 # Generate ansible inventory
 #
 
-data "template_file" "inventory" {
-  template = file("${path.module}/templates/inventory.tpl")
-
-  vars = {
+locals {
+  inventory = templatefile(
+    "${path.module}/templates/inventory.tpl",
+    {
       connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
         keys(module.kubernetes.master_ip_addresses),
         values(module.kubernetes.master_ip_addresses).*.public_ip,
@@ -43,14 +43,15 @@ data "template_file" "inventory" {
       list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
       network_id  = module.kubernetes.network_id
     }
+  )
 }
 
 resource "null_resource" "inventories" {
   provisioner "local-exec" {
-    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
+    command = "echo '${local.inventory}' > ${var.inventory_file}"
   }
 
   triggers = {
-    template = data.template_file.inventory.rendered
+    template = local.inventory
   }
 }

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf

@@ -15,12 +15,12 @@ resource "hcloud_ssh_key" "first" {
   public_key = var.ssh_public_keys.0
 }
 
-resource "hcloud_server" "master" {
+resource "hcloud_server" "machine" {
   for_each = {
     for name, machine in var.machines :
     name => machine
-    if machine.node_type == "master"
   }
+
   name     = "${var.prefix}-${each.key}"
   ssh_keys = [hcloud_ssh_key.first.id]
   # boot into rescue OS
@@ -34,7 +34,7 @@ resource "hcloud_server" "master" {
     timeout     = "5m"
     private_key = file(var.ssh_private_key_path)
   }
-  firewall_ids = [hcloud_firewall.machine.id]
+  firewall_ids = each.value.node_type == "master" ? [hcloud_firewall.master.id] : [hcloud_firewall.worker.id]
   provisioner "file" {
     content     = data.ct_config.machine-ignitions[each.key].rendered
     destination = "/root/ignition.json"
@@ -45,9 +45,9 @@ resource "hcloud_server" "master" {
       "set -ex",
       "apt update",
       "apt install -y gawk",
-      "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install",
+      "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install",
       "chmod +x flatcar-install",
-      "./flatcar-install -s -i /root/ignition.json",
+      "./flatcar-install -s -i /root/ignition.json -C stable",
       "shutdown -r +1",
     ]
   }
@@ -56,6 +56,7 @@ resource "hcloud_server" "master" {
   provisioner "remote-exec" {
     connection {
       host        = self.ipv4_address
+      private_key = file(var.ssh_private_key_path)
       timeout     = "3m"
       user        = var.user_flatcar
     }
@@ -66,65 +67,11 @@ resource "hcloud_server" "master" {
   }
 }
 
-resource "hcloud_server_network" "master" {
-  for_each = hcloud_server.master
-  server_id = each.value.id
-  subnet_id = hcloud_network_subnet.kubernetes.id
-}
-
-resource "hcloud_server" "worker" {
+resource "hcloud_server_network" "machine" {
   for_each = {
     for name, machine in var.machines :
-    name => machine
-    if machine.node_type == "worker"
+    name => hcloud_server.machine[name]
   }
-  name     = "${var.prefix}-${each.key}"
-  ssh_keys = [hcloud_ssh_key.first.id]
-  # boot into rescue OS
-  rescue = "linux64"
-  # dummy value for the OS because Flatcar is not available
-  image       = each.value.image
-  server_type = each.value.size
-  location    = var.zone
-  connection {
-    host    = self.ipv4_address
-    timeout = "5m"
-    private_key = file(var.ssh_private_key_path)
-  }
-  firewall_ids = [hcloud_firewall.machine.id]
-  provisioner "file" {
-    content     = data.ct_config.machine-ignitions[each.key].rendered
-    destination = "/root/ignition.json"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "set -ex",
-      "apt update",
-      "apt install -y gawk",
-      "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install",
-      "chmod +x flatcar-install",
-      "./flatcar-install -s -i /root/ignition.json",
-      "shutdown -r +1",
-    ]
-  }
-
-  # optional:
-  provisioner "remote-exec" {
-    connection {
-      host    = self.ipv4_address
-      timeout = "3m"
-      user    = var.user_flatcar
-    }
-
-    inline = [
-      "sudo hostnamectl set-hostname ${self.name}",
-    ]
-  }
-}
-
-resource "hcloud_server_network" "worker" {
-  for_each = hcloud_server.worker
   server_id = each.value.id
   subnet_id = hcloud_network_subnet.kubernetes.id
 }
@@ -134,25 +81,20 @@ data "ct_config" "machine-ignitions" {
     for name, machine in var.machines :
     name => machine
   }
-  content  = data.template_file.machine-configs[each.key].rendered
-}
 
-data "template_file" "machine-configs" {
-  for_each = {
-    for name, machine in var.machines :
-    name => machine
-  }
-  template = file("${path.module}/templates/machine.yaml.tmpl")
-
-  vars = {
+  strict = false
+  content = templatefile(
+    "${path.module}/templates/machine.yaml.tmpl",
+    {
       ssh_keys     = jsonencode(var.ssh_public_keys)
-    user_flatcar = jsonencode(var.user_flatcar)
+      user_flatcar = var.user_flatcar
       name         = each.key
     }
+  )
 }
 
-resource "hcloud_firewall" "machine" {
-  name = "${var.prefix}-machine-firewall"
+resource "hcloud_firewall" "master" {
+  name = "${var.prefix}-master-firewall"
 
   rule {
     direction  = "in"

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf

@@ -1,20 +1,22 @@
 output "master_ip_addresses" {
   value = {
-    for key, instance in hcloud_server.master :
-    instance.name => {
-      "private_ip" = hcloud_server_network.master[key].ip
-      "public_ip"  = hcloud_server.master[key].ipv4_address
+    for name, machine in var.machines :
+      name => {
+        "private_ip" = hcloud_server_network.machine[name].ip
+        "public_ip"  = hcloud_server.machine[name].ipv4_address
       }
+      if machine.node_type == "master"
   }
 }
 
 output "worker_ip_addresses" {
   value = {
-    for key, instance in hcloud_server.worker :
-    instance.name => {
-      "private_ip" = hcloud_server_network.worker[key].ip
-      "public_ip"  = hcloud_server.worker[key].ipv4_address
+    for name, machine in var.machines :
+      name => {
+        "private_ip" = hcloud_server_network.machine[name].ip
+        "public_ip"  = hcloud_server.machine[name].ipv4_address
       }
+      if machine.node_type == "worker"
   }
 }
 

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl

@@ -1,8 +1,11 @@
----
+variant: flatcar
+version: 1.0.0
+
 passwd:
   users:
     - name: ${user_flatcar}
       ssh_authorized_keys: ${ssh_keys}
+
 storage:
   files:
     - path: /home/core/works

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf

@@ -5,6 +5,7 @@ terraform {
     }
     ct = {
       source  = "poseidon/ct"
+      version = "0.11.0"
     }
     null = {
       source = "hashicorp/null"

contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = "1.31.1"
+      version = "1.38.2"
     }
   }
   required_version = ">= 0.14"

contrib/terraform/hetzner/sample-inventory/cluster.tfvars

@@ -0,0 +1,46 @@
+prefix         = "default"
+zone           = "hel1"
+network_zone   = "eu-central"
+inventory_file = "inventory.ini"
+
+ssh_public_keys = [
+  # Put your public SSH key here
+  "ssh-rsa I-did-not-read-the-docs",
+  "ssh-rsa I-did-not-read-the-docs 2",
+]
+
+ssh_private_key_path = "~/.ssh/id_rsa"
+
+machines = {
+  "master-0" : {
+    "node_type" : "master",
+    "size" : "cx21",
+    "image" : "ubuntu-22.04",
+  },
+  "worker-0" : {
+    "node_type" : "worker",
+    "size" : "cx21",
+    "image" : "ubuntu-22.04",
+  },
+  "worker-1" : {
+    "node_type" : "worker",
+    "size" : "cx21",
+    "image" : "ubuntu-22.04",
+  }
+}
+
+nodeport_whitelist = [
+  "0.0.0.0/0"
+]
+
+ingress_whitelist = [
+  "0.0.0.0/0"
+]
+
+ssh_whitelist = [
+  "0.0.0.0/0"
+]
+
+api_server_whitelist = [
+  "0.0.0.0/0"
+]

contrib/terraform/hetzner/sample-inventory/group_vars

@@ -0,0 +1 @@
+../../../../inventory/sample/group_vars

contrib/terraform/hetzner/versions.tf

@@ -2,14 +2,11 @@ terraform {
   required_providers {
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = "1.31.1"
+      version = "1.38.2"
     }
     null = {
       source = "hashicorp/null"
     }
-    template = {
-      source = "hashicorp/template"
-    }
   }
   required_version = ">= 0.14"
 }