Remove deprecated provider, fix flatcar configs, enable CI tests and refactor hetzner terraform (#10002)

* Remove deprecated provider and fix flatcar configs

* Refactor for DRYness

* Add missing line endings

* Enable tests for hetzner terraform in CI

* Add missing inventory for CI tests

.gitlab-ci/terraform.yml

@@ -80,6 +80,12 @@ tf-validate-exoscale:
     TF_VERSION: $TERRAFORM_VERSION
     PROVIDER: exoscale
 
+tf-validate-hetzner:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: hetzner
+
 tf-validate-vsphere:
   extends: .terraform_validate
   variables:

contrib/terraform/hetzner/default.tfvars

@@ -1,6 +1,6 @@
-prefix = "default"
-zone   = "hel1"
-network_zone = "eu-central"
+prefix         = "default"
+zone           = "hel1"
+network_zone   = "eu-central"
 inventory_file = "inventory.ini"
 
 ssh_public_keys = [
@@ -15,17 +15,17 @@ machines = {
   "master-0" : {
     "node_type" : "master",
     "size" : "cx21",
-    "image" : "ubuntu-20.04",
+    "image" : "ubuntu-22.04",
   },
   "worker-0" : {
     "node_type" : "worker",
     "size" : "cx21",
-    "image" : "ubuntu-20.04",
+    "image" : "ubuntu-22.04",
   },
   "worker-1" : {
     "node_type" : "worker",
     "size" : "cx21",
-    "image" : "ubuntu-20.04",
+    "image" : "ubuntu-22.04",
   }
 }
 

contrib/terraform/hetzner/main.tf

@@ -2,7 +2,7 @@ provider "hcloud" {}
 
 module "kubernetes" {
   source = "./modules/kubernetes-cluster"
-  #source = "./modules/kubernetes-cluster-flatcar"
+  # source = "./modules/kubernetes-cluster-flatcar"
 
   prefix = var.prefix
 
@@ -14,7 +14,7 @@ module "kubernetes" {
   #ssh_private_key_path = var.ssh_private_key_path
 
   ssh_public_keys = var.ssh_public_keys
-  network_zone = var.network_zone
+  network_zone    = var.network_zone
 
   ssh_whitelist        = var.ssh_whitelist
   api_server_whitelist = var.api_server_whitelist
@@ -26,31 +26,32 @@ module "kubernetes" {
 # Generate ansible inventory
 #
 
-data "template_file" "inventory" {
-  template = file("${path.module}/templates/inventory.tpl")
-
-  vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
-      keys(module.kubernetes.master_ip_addresses),
-      values(module.kubernetes.master_ip_addresses).*.public_ip,
-      values(module.kubernetes.master_ip_addresses).*.private_ip,
-    range(1, length(module.kubernetes.master_ip_addresses) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
-      keys(module.kubernetes.worker_ip_addresses),
-      values(module.kubernetes.worker_ip_addresses).*.public_ip,
-    values(module.kubernetes.worker_ip_addresses).*.private_ip))
-    list_master = join("\n", keys(module.kubernetes.master_ip_addresses))
-    list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
-    network_id = module.kubernetes.network_id
-  }
+locals {
+  inventory = templatefile(
+    "${path.module}/templates/inventory.tpl",
+    {
+      connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+        keys(module.kubernetes.master_ip_addresses),
+        values(module.kubernetes.master_ip_addresses).*.public_ip,
+        values(module.kubernetes.master_ip_addresses).*.private_ip,
+      range(1, length(module.kubernetes.master_ip_addresses) + 1)))
+      connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+        keys(module.kubernetes.worker_ip_addresses),
+        values(module.kubernetes.worker_ip_addresses).*.public_ip,
+      values(module.kubernetes.worker_ip_addresses).*.private_ip))
+      list_master = join("\n", keys(module.kubernetes.master_ip_addresses))
+      list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
+      network_id  = module.kubernetes.network_id
+    }
+  )
 }
 
 resource "null_resource" "inventories" {
   provisioner "local-exec" {
-    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
+    command = "echo '${local.inventory}' > ${var.inventory_file}"
   }
 
   triggers = {
-    template = data.template_file.inventory.rendered
+    template = local.inventory
   }
 }

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf

@@ -15,12 +15,12 @@ resource "hcloud_ssh_key" "first" {
   public_key = var.ssh_public_keys.0
 }
 
-resource "hcloud_server" "master" {
+resource "hcloud_server" "machine" {
   for_each = {
     for name, machine in var.machines :
     name => machine
-    if machine.node_type == "master"
   }
+
   name     = "${var.prefix}-${each.key}"
   ssh_keys = [hcloud_ssh_key.first.id]
   # boot into rescue OS
@@ -30,11 +30,11 @@ resource "hcloud_server" "master" {
   server_type = each.value.size
   location    = var.zone
   connection {
-    host    = self.ipv4_address
-    timeout = "5m"
+    host        = self.ipv4_address
+    timeout     = "5m"
     private_key = file(var.ssh_private_key_path)
   }
-  firewall_ids = [hcloud_firewall.machine.id]
+  firewall_ids = each.value.node_type == "master" ? [hcloud_firewall.master.id] : [hcloud_firewall.worker.id]
   provisioner "file" {
     content     = data.ct_config.machine-ignitions[each.key].rendered
     destination = "/root/ignition.json"
@@ -45,9 +45,9 @@ resource "hcloud_server" "master" {
       "set -ex",
       "apt update",
       "apt install -y gawk",
-      "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install",
+      "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install",
       "chmod +x flatcar-install",
-      "./flatcar-install -s -i /root/ignition.json",
+      "./flatcar-install -s -i /root/ignition.json -C stable",
       "shutdown -r +1",
     ]
   }
@@ -55,9 +55,10 @@ resource "hcloud_server" "master" {
   # optional:
   provisioner "remote-exec" {
     connection {
-      host    = self.ipv4_address
-      timeout = "3m"
-      user    = var.user_flatcar
+      host        = self.ipv4_address
+      private_key = file(var.ssh_private_key_path)
+      timeout     = "3m"
+      user        = var.user_flatcar
     }
 
     inline = [
@@ -66,65 +67,11 @@ resource "hcloud_server" "master" {
   }
 }
 
-resource "hcloud_server_network" "master" {
-  for_each = hcloud_server.master
-  server_id = each.value.id
-  subnet_id = hcloud_network_subnet.kubernetes.id
-}
-
-resource "hcloud_server" "worker" {
+resource "hcloud_server_network" "machine" {
   for_each = {
     for name, machine in var.machines :
-    name => machine
-    if machine.node_type == "worker"
+    name => hcloud_server.machine[name]
   }
-  name     = "${var.prefix}-${each.key}"
-  ssh_keys = [hcloud_ssh_key.first.id]
-  # boot into rescue OS
-  rescue = "linux64"
-  # dummy value for the OS because Flatcar is not available
-  image       = each.value.image
-  server_type = each.value.size
-  location    = var.zone
-  connection {
-    host    = self.ipv4_address
-    timeout = "5m"
-    private_key = file(var.ssh_private_key_path)
-  }
-  firewall_ids = [hcloud_firewall.machine.id]
-  provisioner "file" {
-    content     = data.ct_config.machine-ignitions[each.key].rendered
-    destination = "/root/ignition.json"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "set -ex",
-      "apt update",
-      "apt install -y gawk",
-      "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install",
-      "chmod +x flatcar-install",
-      "./flatcar-install -s -i /root/ignition.json",
-      "shutdown -r +1",
-    ]
-  }
-
-  # optional:
-  provisioner "remote-exec" {
-    connection {
-      host    = self.ipv4_address
-      timeout = "3m"
-      user    = var.user_flatcar
-    }
-
-    inline = [
-      "sudo hostnamectl set-hostname ${self.name}",
-    ]
-  }
-}
-
-resource "hcloud_server_network" "worker" {
-  for_each = hcloud_server.worker
   server_id = each.value.id
   subnet_id = hcloud_network_subnet.kubernetes.id
 }
@@ -134,38 +81,33 @@ data "ct_config" "machine-ignitions" {
     for name, machine in var.machines :
     name => machine
   }
-  content  = data.template_file.machine-configs[each.key].rendered
+
+  strict = false
+  content = templatefile(
+    "${path.module}/templates/machine.yaml.tmpl",
+    {
+      ssh_keys     = jsonencode(var.ssh_public_keys)
+      user_flatcar = var.user_flatcar
+      name         = each.key
+    }
+  )
 }
 
-data "template_file" "machine-configs" {
-  for_each = {
-    for name, machine in var.machines :
-    name => machine
-  }
-  template = file("${path.module}/templates/machine.yaml.tmpl")
-
-  vars = {
-    ssh_keys = jsonencode(var.ssh_public_keys)
-    user_flatcar = jsonencode(var.user_flatcar)
-    name     = each.key
-  }
-}
-
-resource "hcloud_firewall" "machine" {
-  name = "${var.prefix}-machine-firewall"
+resource "hcloud_firewall" "master" {
+  name = "${var.prefix}-master-firewall"
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "22"
-   source_ips = var.ssh_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "22"
+    source_ips = var.ssh_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "6443"
-   source_ips = var.api_server_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "6443"
+    source_ips = var.api_server_whitelist
   }
 }
 
@@ -173,30 +115,30 @@ resource "hcloud_firewall" "worker" {
   name = "${var.prefix}-worker-firewall"
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "22"
-   source_ips = var.ssh_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "22"
+    source_ips = var.ssh_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "80"
-   source_ips = var.ingress_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "80"
+    source_ips = var.ingress_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "443"
-   source_ips = var.ingress_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "443"
+    source_ips = var.ingress_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "30000-32767"
-   source_ips = var.nodeport_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "30000-32767"
+    source_ips = var.nodeport_whitelist
   }
 }

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf

@@ -1,20 +1,22 @@
 output "master_ip_addresses" {
   value = {
-    for key, instance in hcloud_server.master :
-    instance.name => {
-      "private_ip" = hcloud_server_network.master[key].ip
-      "public_ip"  = hcloud_server.master[key].ipv4_address
-    }
+    for name, machine in var.machines :
+      name => {
+        "private_ip" = hcloud_server_network.machine[name].ip
+        "public_ip"  = hcloud_server.machine[name].ipv4_address
+      }
+      if machine.node_type == "master"
   }
 }
 
 output "worker_ip_addresses" {
   value = {
-    for key, instance in hcloud_server.worker :
-    instance.name => {
-      "private_ip" = hcloud_server_network.worker[key].ip
-      "public_ip"  = hcloud_server.worker[key].ipv4_address
-    }
+    for name, machine in var.machines :
+      name => {
+        "private_ip" = hcloud_server_network.machine[name].ip
+        "public_ip"  = hcloud_server.machine[name].ipv4_address
+      }
+      if machine.node_type == "worker"
   }
 }
 

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl

@@ -1,8 +1,11 @@
----
+variant: flatcar
+version: 1.0.0
+
 passwd:
   users:
     - name: ${user_flatcar}
       ssh_authorized_keys: ${ssh_keys}
+
 storage:
   files:
     - path: /home/core/works

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf

@@ -1,6 +1,6 @@
 
 variable "zone" {
-  type = string
+  type    = string
   default = "fsn1"
 }
 
@@ -9,7 +9,7 @@ variable "prefix" {
 }
 
 variable "user_flatcar" {
-  type = string
+  type    = string
   default = "core"
 }
 

contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf

@@ -1,13 +1,14 @@
 terraform {
   required_providers {
     hcloud = {
-      source  = "hetznercloud/hcloud"
+      source = "hetznercloud/hcloud"
     }
     ct = {
       source  = "poseidon/ct"
+      version = "0.11.0"
     }
     null = {
-      source  = "hashicorp/null"
+      source = "hashicorp/null"
     }
   }
 }

contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf

@@ -75,17 +75,17 @@ resource "hcloud_firewall" "master" {
   name = "${var.prefix}-master-firewall"
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "22"
-   source_ips = var.ssh_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "22"
+    source_ips = var.ssh_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "6443"
-   source_ips = var.api_server_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "6443"
+    source_ips = var.api_server_whitelist
   }
 }
 
@@ -93,30 +93,30 @@ resource "hcloud_firewall" "worker" {
   name = "${var.prefix}-worker-firewall"
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "22"
-   source_ips = var.ssh_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "22"
+    source_ips = var.ssh_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "80"
-   source_ips = var.ingress_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "80"
+    source_ips = var.ingress_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "443"
-   source_ips = var.ingress_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "443"
+    source_ips = var.ingress_whitelist
   }
 
   rule {
-   direction = "in"
-   protocol = "tcp"
-   port = "30000-32767"
-   source_ips = var.nodeport_whitelist
+    direction  = "in"
+    protocol   = "tcp"
+    port       = "30000-32767"
+    source_ips = var.nodeport_whitelist
   }
 }

contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf

@@ -1,8 +1,8 @@
 terraform {
   required_providers {
     hcloud = {
-      source = "hetznercloud/hcloud"
-      version = "1.31.1"
+      source  = "hetznercloud/hcloud"
+      version = "1.38.2"
     }
   }
   required_version = ">= 0.14"

contrib/terraform/hetzner/sample-inventory/cluster.tfvars

@@ -0,0 +1,46 @@
+prefix         = "default"
+zone           = "hel1"
+network_zone   = "eu-central"
+inventory_file = "inventory.ini"
+
+ssh_public_keys = [
+  # Put your public SSH key here
+  "ssh-rsa I-did-not-read-the-docs",
+  "ssh-rsa I-did-not-read-the-docs 2",
+]
+
+ssh_private_key_path = "~/.ssh/id_rsa"
+
+machines = {
+  "master-0" : {
+    "node_type" : "master",
+    "size" : "cx21",
+    "image" : "ubuntu-22.04",
+  },
+  "worker-0" : {
+    "node_type" : "worker",
+    "size" : "cx21",
+    "image" : "ubuntu-22.04",
+  },
+  "worker-1" : {
+    "node_type" : "worker",
+    "size" : "cx21",
+    "image" : "ubuntu-22.04",
+  }
+}
+
+nodeport_whitelist = [
+  "0.0.0.0/0"
+]
+
+ingress_whitelist = [
+  "0.0.0.0/0"
+]
+
+ssh_whitelist = [
+  "0.0.0.0/0"
+]
+
+api_server_whitelist = [
+  "0.0.0.0/0"
+]

contrib/terraform/hetzner/sample-inventory/group_vars

@@ -0,0 +1 @@
+../../../../inventory/sample/group_vars

contrib/terraform/hetzner/variables.tf

@@ -3,7 +3,7 @@ variable "zone" {
 }
 variable "network_zone" {
   description = "The network zone where the cluster is running"
-  default = "eu-central"
+  default     = "eu-central"
 }
 
 variable "prefix" {

contrib/terraform/hetzner/versions.tf

@@ -2,14 +2,11 @@ terraform {
   required_providers {
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = "1.31.1"
+      version = "1.38.2"
     }
     null = {
       source = "hashicorp/null"
     }
-    template = {
-      source = "hashicorp/template"
-    }
   }
   required_version = ">= 0.14"
 }