From 8267922a16193ba0ced3dccac6170b36449465de Mon Sep 17 00:00:00 2001 From: florianow <64468897+florianow@users.noreply.github.com> Date: Wed, 28 Dec 2022 03:17:28 +0100 Subject: [PATCH] add flatcar support for Hetzner (#9618) --- contrib/terraform/hetzner/README.md | 17 +- contrib/terraform/hetzner/default.tfvars | 2 + contrib/terraform/hetzner/main.tf | 6 +- .../kubernetes-cluster-flatcar/main.tf | 202 ++++++++++++++++++ .../kubernetes-cluster-flatcar/outputs.tf | 27 +++ .../templates/machine.yaml.tmpl | 16 ++ .../kubernetes-cluster-flatcar/variables.tf | 60 ++++++ .../kubernetes-cluster-flatcar/versions.tf | 13 ++ contrib/terraform/hetzner/variables.tf | 6 + 9 files changed, 346 insertions(+), 3 deletions(-) create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf diff --git a/contrib/terraform/hetzner/README.md b/contrib/terraform/hetzner/README.md index fdc43f9ff..63ef640e7 100644 --- a/contrib/terraform/hetzner/README.md +++ b/contrib/terraform/hetzner/README.md @@ -56,11 +56,24 @@ cd inventory/$CLUSTER Edit `default.tfvars` to match your requirement. +Flatcar Container Linux instead of the basic Hetzner Images. + +```bash +cd ../../contrib/terraform/hetzner +``` + +Edit `main.tf` and reactivate the module `source = "./modules/kubernetes-cluster-flatcar"`and +comment out the `#source = "./modules/kubernetes-cluster"`. + +activate `ssh_private_key_path = var.ssh_private_key_path`. The VM boots into +Rescue-Mode with the selected image of the `var.machines` but installs Flatcar instead. + Run Terraform to create the infrastructure. ```bash -terraform init ../../contrib/terraform/hetzner -terraform apply --var-file default.tfvars ../../contrib/terraform/hetzner/ +cd ./kubespray +terraform -chdir=./contrib/terraform/hetzner/ init +terraform -chdir=./contrib/terraform/hetzner/ apply --var-file=../../../inventory/$CLUSTER/default.tfvars ``` You should now have a inventory file named `inventory.ini` that you can use with kubespray. diff --git a/contrib/terraform/hetzner/default.tfvars b/contrib/terraform/hetzner/default.tfvars index 957b2d523..a56bab1ae 100644 --- a/contrib/terraform/hetzner/default.tfvars +++ b/contrib/terraform/hetzner/default.tfvars @@ -9,6 +9,8 @@ ssh_public_keys = [ "ssh-rsa I-did-not-read-the-docs 2", ] +ssh_private_key_path = "~/.ssh/id_rsa" + machines = { "master-0" : { "node_type" : "master", diff --git a/contrib/terraform/hetzner/main.tf b/contrib/terraform/hetzner/main.tf index 805c7bfb8..fc2f27108 100644 --- a/contrib/terraform/hetzner/main.tf +++ b/contrib/terraform/hetzner/main.tf @@ -2,6 +2,7 @@ provider "hcloud" {} module "kubernetes" { source = "./modules/kubernetes-cluster" + #source = "./modules/kubernetes-cluster-flatcar" prefix = var.prefix @@ -9,6 +10,9 @@ module "kubernetes" { machines = var.machines + #only for flatcar + #ssh_private_key_path = var.ssh_private_key_path + ssh_public_keys = var.ssh_public_keys network_zone = var.network_zone @@ -49,4 +53,4 @@ resource "null_resource" "inventories" { triggers = { template = data.template_file.inventory.rendered } -} +} \ No newline at end of file diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf new file mode 100644 index 000000000..804c5038e --- /dev/null +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf @@ -0,0 +1,202 @@ +resource "hcloud_network" "kubernetes" { + name = "${var.prefix}-network" + ip_range = var.private_network_cidr +} + +resource "hcloud_network_subnet" "kubernetes" { + type = "cloud" + network_id = hcloud_network.kubernetes.id + network_zone = var.network_zone + ip_range = var.private_subnet_cidr +} + +resource "hcloud_ssh_key" "first" { + name = var.prefix + public_key = var.ssh_public_keys.0 +} + +resource "hcloud_server" "master" { + for_each = { + for name, machine in var.machines : + name => machine + if machine.node_type == "master" + } + name = "${var.prefix}-${each.key}" + ssh_keys = [hcloud_ssh_key.first.id] + # boot into rescue OS + rescue = "linux64" + # dummy value for the OS because Flatcar is not available + image = each.value.image + server_type = each.value.size + location = var.zone + connection { + host = self.ipv4_address + timeout = "5m" + private_key = file(var.ssh_private_key_path) + } + firewall_ids = [hcloud_firewall.machine.id] + provisioner "file" { + content = data.ct_config.machine-ignitions[each.key].rendered + destination = "/root/ignition.json" + } + + provisioner "remote-exec" { + inline = [ + "set -ex", + "apt update", + "apt install -y gawk", + "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install", + "chmod +x flatcar-install", + "./flatcar-install -s -i /root/ignition.json", + "shutdown -r +1", + ] + } + + # optional: + provisioner "remote-exec" { + connection { + host = self.ipv4_address + timeout = "3m" + user = var.user_flatcar + } + + inline = [ + "sudo hostnamectl set-hostname ${self.name}", + ] + } +} + +resource "hcloud_server_network" "master" { + for_each = hcloud_server.master + server_id = each.value.id + subnet_id = hcloud_network_subnet.kubernetes.id +} + +resource "hcloud_server" "worker" { + for_each = { + for name, machine in var.machines : + name => machine + if machine.node_type == "worker" + } + name = "${var.prefix}-${each.key}" + ssh_keys = [hcloud_ssh_key.first.id] + # boot into rescue OS + rescue = "linux64" + # dummy value for the OS because Flatcar is not available + image = each.value.image + server_type = each.value.size + location = var.zone + connection { + host = self.ipv4_address + timeout = "5m" + private_key = file(var.ssh_private_key_path) + } + firewall_ids = [hcloud_firewall.machine.id] + provisioner "file" { + content = data.ct_config.machine-ignitions[each.key].rendered + destination = "/root/ignition.json" + } + + provisioner "remote-exec" { + inline = [ + "set -ex", + "apt update", + "apt install -y gawk", + "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install", + "chmod +x flatcar-install", + "./flatcar-install -s -i /root/ignition.json", + "shutdown -r +1", + ] + } + + # optional: + provisioner "remote-exec" { + connection { + host = self.ipv4_address + timeout = "3m" + user = var.user_flatcar + } + + inline = [ + "sudo hostnamectl set-hostname ${self.name}", + ] + } +} + +resource "hcloud_server_network" "worker" { + for_each = hcloud_server.worker + server_id = each.value.id + subnet_id = hcloud_network_subnet.kubernetes.id +} + +data "ct_config" "machine-ignitions" { + for_each = { + for name, machine in var.machines : + name => machine + } + content = data.template_file.machine-configs[each.key].rendered +} + +data "template_file" "machine-configs" { + for_each = { + for name, machine in var.machines : + name => machine + } + template = file("${path.module}/templates/machine.yaml.tmpl") + + vars = { + ssh_keys = jsonencode(var.ssh_public_keys) + user_flatcar = jsonencode(var.user_flatcar) + name = each.key + } +} + +resource "hcloud_firewall" "machine" { + name = "${var.prefix}-machine-firewall" + + rule { + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.ssh_whitelist + } + + rule { + direction = "in" + protocol = "tcp" + port = "6443" + source_ips = var.api_server_whitelist + } +} + +resource "hcloud_firewall" "worker" { + name = "${var.prefix}-worker-firewall" + + rule { + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.ssh_whitelist + } + + rule { + direction = "in" + protocol = "tcp" + port = "80" + source_ips = var.ingress_whitelist + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = var.ingress_whitelist + } + + rule { + direction = "in" + protocol = "tcp" + port = "30000-32767" + source_ips = var.nodeport_whitelist + } +} diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf new file mode 100644 index 000000000..c6bb276da --- /dev/null +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf @@ -0,0 +1,27 @@ +output "master_ip_addresses" { + value = { + for key, instance in hcloud_server.master : + instance.name => { + "private_ip" = hcloud_server_network.master[key].ip + "public_ip" = hcloud_server.master[key].ipv4_address + } + } +} + +output "worker_ip_addresses" { + value = { + for key, instance in hcloud_server.worker : + instance.name => { + "private_ip" = hcloud_server_network.worker[key].ip + "public_ip" = hcloud_server.worker[key].ipv4_address + } + } +} + +output "cluster_private_network_cidr" { + value = var.private_subnet_cidr +} + +output "network_id" { + value = hcloud_network.kubernetes.id +} \ No newline at end of file diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl new file mode 100644 index 000000000..26e05f867 --- /dev/null +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl @@ -0,0 +1,16 @@ +--- +passwd: + users: + - name: ${user_flatcar} + ssh_authorized_keys: ${ssh_keys} +storage: + files: + - path: /home/core/works + filesystem: root + mode: 0755 + contents: + inline: | + #!/bin/bash + set -euo pipefail + hostname="$(hostname)" + echo My name is ${name} and the hostname is $${hostname} \ No newline at end of file diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf new file mode 100644 index 000000000..4f2f8f51d --- /dev/null +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf @@ -0,0 +1,60 @@ + +variable "zone" { + type = string + default = "fsn1" +} + +variable "prefix" { + default = "k8s" +} + +variable "user_flatcar" { + type = string + default = "core" +} + +variable "machines" { + type = map(object({ + node_type = string + size = string + image = string + })) +} + + + +variable "ssh_public_keys" { + type = list(string) +} + +variable "ssh_private_key_path" { + type = string + default = "~/.ssh/id_rsa" +} + +variable "ssh_whitelist" { + type = list(string) +} + +variable "api_server_whitelist" { + type = list(string) +} + +variable "nodeport_whitelist" { + type = list(string) +} + +variable "ingress_whitelist" { + type = list(string) +} + +variable "private_network_cidr" { + default = "10.0.0.0/16" +} + +variable "private_subnet_cidr" { + default = "10.0.10.0/24" +} +variable "network_zone" { + default = "eu-central" +} diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf new file mode 100644 index 000000000..4291f8a61 --- /dev/null +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + } + ct = { + source = "poseidon/ct" + } + null = { + source = "hashicorp/null" + } + } +} \ No newline at end of file diff --git a/contrib/terraform/hetzner/variables.tf b/contrib/terraform/hetzner/variables.tf index e83676ad8..8fc8b2c97 100644 --- a/contrib/terraform/hetzner/variables.tf +++ b/contrib/terraform/hetzner/variables.tf @@ -25,6 +25,12 @@ variable "ssh_public_keys" { type = list(string) } +variable "ssh_private_key_path" { + description = "Private SSH key which connect to the VMs." + type = string + default = "~/.ssh/id_rsa" +} + variable "ssh_whitelist" { description = "List of IP ranges (CIDR) to whitelist for ssh" type = list(string)