kubeadm support (#1631)

* kubeadm support

* move k8s master to a subtask
* disable k8s secrets when using kubeadm
* fix etcd cert serial var
* move simple auth users to master role
* make a kubeadm-specific env file for kubelet
* add non-ha CI job

* change ci boolean vars to json format

* fixup

* Update create-gce.yml

* Update create-gce.yml

* Update create-gce.yml

.gitlab-ci.yml

@@ -53,6 +53,7 @@ before_script:
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   UPGRADE_TEST: "false"
+  KUBEADM_ENABLED: "false"
   RESOLVCONF_MODE: docker_dns
   LOG_LEVEL: "-vv"
   ETCD_DEPLOYMENT: "docker"
@@ -117,9 +118,9 @@ before_script:
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
+      -e "{deploy_netchecker: true}"
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
@@ -127,6 +128,9 @@ before_script:
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
       -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml
@@ -144,17 +148,19 @@ before_script:
       -e ansible_ssh_user=${SSH_USER}
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
+      -e "{deploy_netchecker: true}"
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
       -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
       -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       $PLAYBOOK;
@@ -178,14 +184,18 @@ before_script:
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{deploy_netchecker: true}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
-      -e deploy_netchecker=true
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -221,14 +231,18 @@ before_script:
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
+      -e "{deploy_netchecker: true}"
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
-      -e deploy_netchecker=true
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -280,6 +294,17 @@ before_script:
   UPGRADE_TEST: "graceful"
   STARTUP_SCRIPT: ""
 
+.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_MACHINE_TYPE: "n1-standard-2"
+  CLOUD_REGION: europe-west1-b
+  CLUSTER_MODE: default
+  KUBEADM_ENABLED: "true"
+  STARTUP_SCRIPT: ""
+
 .rhel7_weave_variables: &rhel7_weave_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: weave
@@ -470,6 +495,27 @@ ubuntu-canal-ha-rbac-triggers:
   when: on_success
   only: ['triggers']
 
+ubuntu-canal-kubeadm-rbac:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+ubuntu-canal-kubeadm-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
 rhel7-weave:
   stage: deploy-gce-part1
   <<: *job

cluster.yml

@@ -69,6 +69,17 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/master, tags: master }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 

inventory/group_vars/all.yml

@@ -82,6 +82,12 @@ bin_dir: /usr/local/bin
 #openstack_lbaas_monitor_timeout: "30s"
 #openstack_lbaas_monitor_max_retries: "3"
 
+## Uncomment to enable experimental kubeadm deployment mode
+#kubeadm_enabled: false
+#kubeadm_token_first: "{{ lookup('password', 'credentials/kubeadm_token_first length=6  chars=ascii_letters,digits') }}"
+#kubeadm_token_second: "{{ lookup('password', 'credentials/kubeadm_token_second length=16 chars=ascii_letters,digits') }}"
+#kubeadm_token: "{{ kubeadm_token_first }}.{{ kubeadm_token_second }}"
+#
 ## Set these proxy values in order to update docker daemon to use proxies
 #http_proxy: ""
 #https_proxy: ""

library/kube.py

@@ -135,12 +135,15 @@ class KubeManager(object):
             return None
         return out.splitlines()
 
-    def create(self, check=True):
+    def create(self, check=True, force=True):
         if check and self.exists():
             return []
 
         cmd = ['apply']
 
+        if force:
+            cmd.append('--force')
+
         if not self.filename:
             self.module.fail_json(msg='filename required to create')
 
@@ -148,11 +151,11 @@ class KubeManager(object):
 
         return self._execute(cmd)
 
-    def replace(self):
+    def replace(self, force=True):
 
         cmd = ['apply']
 
-        if self.force:
+        if force:
             cmd.append('--force')
 
         if not self.filename:

roles/download/defaults/main.yml

@@ -19,6 +19,7 @@ download_always_pull: False
 
 # Versions
 kube_version: v1.7.3
+kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.4
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
@@ -31,11 +32,13 @@ flannel_version: "v0.8.0"
 flannel_cni_version: "v0.2.0"
 pod_infra_version: 3.0
 
-# Download URL's
+# Download URLs
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
+kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"
 
 # Checksums
 etcd_checksum: "274c46a7f8d26f7ae99d6880610f54933cbcf7f3beafa19236c52eb5df8c7a0b"
+kubeadm_checksum: "378e6052f8b178f8e6a38e8637681c72d389443b66b78b51b8ddc9a162c655c3"
 
 # Containers
 # Possible values: host, docker
@@ -132,6 +135,15 @@ downloads:
     container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
+  kubeadm:
+    version: "{{ kubeadm_version }}"
+    dest: "kubeadm"
+    sha256: "{{ kubeadm_checksum }}"
+    source_url: "{{ kubeadm_download_url }}"
+    url: "{{ kubeadm_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
   hyperkube:
     container: true
     repo: "{{ hyperkube_image_repo }}"

roles/etcd/tasks/main.yml

@@ -11,7 +11,7 @@
 
 - name: "Gen_certs | Get etcd certificate serials"
   shell: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial | cut -d= -f2"
-  register: "node-{{ inventory_hostname }}_serial"
+  register: "etcd_client_cert_serial"
   when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort
 
 - include: "install_{{ etcd_deployment_type }}.yml"

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -8,6 +8,18 @@
   delay: 6
   when: inventory_hostname == groups['kube-master'][0]
 
+- name: kubeadm | Delete kubeadm kubedns
+  kube:
+    name: "kubedns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "deploy"
+    state: absent
+  when:
+    - kubeadm_enabled|default(false)
+    - kubeadm_init.changed|default(false)
+    - inventory_hostname == groups['kube-master'][0]
+
 - name: Kubernetes Apps | Lay Down KubeDNS Template
   template:
     src: "{{item.file}}"

roles/kubernetes/kubeadm/tasks/main.yml

@@ -0,0 +1,41 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint }}
+      {%- endif %}
+  when: not is_kube_master
+  tags: facts
+
+- name: Create kubeadm client config
+  template:
+    src: kubeadm-client.conf.j2
+    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    backup: yes
+  when: not is_kube_master
+  register: kubeadm_client_conf
+
+- name: Join to cluster if needed
+  command: kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks
+  register: kubeadm_join
+  when: not is_kube_master and kubeadm_client_conf.changed
+
+- name: Update server field in kubelet kubeconfig
+  replace:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '(\s+){{ first_kube_master }}:{{ kube_apiserver_port }}(\s+.*)?$'
+    replace: '\1{{ kube_apiserver_endpoint }}\2'
+    backup: yes
+  when: not is_kube_master and kubeadm_discovery_address != kube_apiserver_endpoint
+
+# FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
+- name: Symlink kubelet kubeconfig for calico/canal
+  file:
+    src: "{{ kube_config_dir }}//kubelet.conf"
+    dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    state: link
+    force: yes
+  when: kube_network_plugin in ['calico','canal']

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -0,0 +1,6 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: NodeConfiguration
+caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+token: {{ kubeadm_token }}
+discoveryTokenAPIServers:
+- {{ kubeadm_discovery_address | replace("https://", "")}}

roles/kubernetes/master/defaults/main.yml

@@ -66,3 +66,7 @@ apiserver_custom_flags: []
 controller_mgr_custom_flags: []
 
 scheduler_custom_flags: []
+
+# kubeadm settings
+# Value of 0 means it never expires
+kubeadm_token_ttl: 0

roles/kubernetes/master/handlers/main.yml

@@ -44,3 +44,7 @@
   until: result.status == 200
   retries: 20
   delay: 6
+
+- name: Master | set secret_changed
+  set_fact:
+    secret_changed: true

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -0,0 +1,35 @@
+---
+- name: kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: >-
+      kubernetes
+      kubernetes.default
+      kubernetes.default.svc
+      kubernetes.default.svc.{{ dns_domain }}
+      {{ kube_apiserver_ip }}
+      localhost
+      127.0.0.1
+      {{ ' '.join(groups['kube-master']) }}
+      {%- if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+      {{ apiserver_loadbalancer_domain_name }}
+      {%- endif %}
+      {%- for host in groups['kube-master'] -%}
+      {%- if hostvars[host]['access_ip'] is defined %}{{ hostvars[host]['access_ip'] }}{% endif -%}
+      {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+      {%- endfor %}
+  tags: facts
+
+- name: kubeadm | Copy etcd cert dir under k8s cert dir
+  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
+  changed_when: false
+
+- name: kubeadm | Create kubeadm config
+  template:
+    src: kubeadm-config.yaml.j2
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kubeadm_config
+
+- name: kubeadm | Initialize cluster
+  command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  register: kubeadm_init
+  when: kubeadm_config.changed

roles/kubernetes/master/tasks/main.yml

@@ -2,6 +2,9 @@
 - include: pre-upgrade.yml
   tags: k8s-pre-upgrade
 
+- include: users-file.yml
+  when: kube_basic_auth|default(true)
+
 - name: Copy kubectl from hyperkube container
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl"
   register: kube_task_result
@@ -25,63 +28,10 @@
   when: ansible_os_family in ["Debian","RedHat"]
   tags: [kubectl, upgrade]
 
-- name: Write kube-apiserver manifest
+- task: Include kubeadm setup if enabled
-  template:
+  include: kubeadm-setup.yml
-    src: manifests/kube-apiserver.manifest.j2
+  when: kubeadm_enabled|bool|default(false)
-    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
-  notify: Master | wait for the apiserver to be running
-  tags: kube-apiserver
 
-- meta: flush_handlers
+- task: Include static pod setup if not using kubeadm
-
+  include: static-pod-setup.yml
-- name: Write kube system namespace manifest
+  when: not kubeadm_enabled|bool|default(false)
-  template:
-    src: namespace.j2
-    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  run_once: yes
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Check if kube system namespace exists
-  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
-  register: 'kubesystem'
-  changed_when: False
-  failed_when: False
-  run_once: yes
-  tags: apps
-
-- name: Create kube system namespace
-  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  register: create_system_ns
-  until: create_system_ns.rc == 0
-  changed_when: False
-  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Write kube-scheduler kubeconfig
-  template:
-    src: kube-scheduler-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
-  tags: kube-scheduler
-
-- name: Write kube-scheduler manifest
-  template:
-    src: manifests/kube-scheduler.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
-  notify: Master | wait for kube-scheduler
-  tags: kube-scheduler
-
-- name: Write kube-controller-manager kubeconfig
-  template:
-    src: kube-controller-manager-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
-  tags: kube-controller-manager
-
-- name: Write kube-controller-manager manifest
-  template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
-  tags: kube-controller-manager

roles/kubernetes/master/tasks/static-pod-setup.yml

@@ -0,0 +1,61 @@
+---
+- name: Write kube-apiserver manifest
+  template:
+    src: manifests/kube-apiserver.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
+  notify: Master | wait for the apiserver to be running
+  tags: kube-apiserver
+
+- meta: flush_handlers
+
+- name: Write kube system namespace manifest
+  template:
+    src: namespace.j2
+    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  run_once: yes
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Check if kube system namespace exists
+  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
+  register: 'kubesystem'
+  changed_when: False
+  failed_when: False
+  run_once: yes
+  tags: apps
+
+- name: Create kube system namespace
+  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  register: create_system_ns
+  until: create_system_ns.rc == 0
+  changed_when: False
+  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Write kube-scheduler kubeconfig
+  template:
+    src: kube-scheduler-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+  tags: kube-scheduler
+
+- name: Write kube-scheduler manifest
+  template:
+    src: manifests/kube-scheduler.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
+  notify: Master | wait for kube-scheduler
+  tags: kube-scheduler
+
+- name: Write kube-controller-manager kubeconfig
+  template:
+    src: kube-controller-manager-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+  tags: kube-controller-manager
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
+  notify: Master | wait for kube-controller-manager
+  tags: kube-controller-manager

roles/kubernetes/master/tasks/users-file.yml

@@ -0,0 +1,14 @@
+---
+- name: Make sure the users directory exits
+  file:
+    path: "{{ kube_users_dir }}"
+    state: directory
+    mode: o-rwx
+    group: "{{ kube_cert_group }}"
+
+- name: Populate users for basic auth in API
+  template:
+    src: known_users.csv.j2
+    dest: "{{ kube_users_dir }}/known_users.csv"
+    backup: yes
+  notify: Master | set secret_changed

roles/kubernetes/master/templates/kubeadm-config.yaml.j2

@@ -0,0 +1,67 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  bindPort: "{{ kube_apiserver_port }}"
+etcd:
+  endpoints: 
+{% for endpoint in etcd_access_endpoint.split(',') %}
+  - {{ endpoint }}
+{% endfor %}
+  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: {{ kube_service_addresses }}
+  podSubnet: {{ kube_pods_subnet }}
+kubernetesVersion: {{ kube_version }}
+cloudProvider: {{ cloud_provider|default('') }}
+#TODO: cloud provider conf file
+authorizationModes: 
+{% for mode in authorization_modes %}
+- {{ mode }}
+{% endfor %}
+token: {{ kubeadm_token }}
+tokenTTL: {{ kubeadm_token_ttl }}
+selfHosted: false
+apiServerExtraArgs:
+  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+  insecure-port: "{{ kube_apiserver_insecure_port }}"
+  admission-control: {{ kube_apiserver_admission_control | join(',') }}
+  service-node-port-range: {{ kube_apiserver_node_port_range }}
+{% if kube_basic_auth|default(true) %}
+  basic-auth-file: {{ kube_users_dir }}/known_users.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+  oidc-issuer-url: {{ kube_oidc_url }}
+  oidc-client-id: {{ kube_oidc_client_id }}
+{%   if kube_oidc_ca_file is defined %}
+  oidc-ca-file: {{ kube_oidc_ca_file }}
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+  oidc-username-claim: {{ kube_oidc_username_claim }}
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+  oidc-groups-claim: {{ kube_oidc_groups_claim }}
+{%   endif %}
+{% endif %}
+  storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config is defined %}
+  runtime-config: {{ kube_api_runtime_config }}
+{% endif %}
+  allow-privileged: "true"
+#TODO: Custom flags compatible with kubeadm
+controllerManagerExtraArgs:
+  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+  node-monitor-period: {{ kube_controller_node_monitor_period }}
+  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kube_feature_gates %}
+  feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+#schedulerExtraArgs:
+apiServerCertSANs:
+{% for san in  apiserver_sans.split(' ') | unique %}
+  - {{ san }}
+{% endfor %}
+certificatesDir: {{ kube_config_dir }}/ssl

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -7,7 +7,7 @@ metadata:
     k8s-app: kube-apiserver
     kubespray: v2
   annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     kubespray.apiserver-cert/serial: "{{ apiserver_cert_serial }}"
 spec:
   hostNetwork: true

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -6,7 +6,7 @@ metadata:
   labels:
     k8s-app: kube-controller
   annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     kubespray.controller-manager-cert/serial: "{{ controller_manager_cert_serial }}"
 spec:
   hostNetwork: true

roles/kubernetes/node/meta/main.yml

@@ -10,7 +10,12 @@ dependencies:
     file: "{{ downloads.install_socat }}"
     tags: [download, kubelet]
     when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+  - role: download
+    file: "{{ downloads.kubeadm }}"
+    tags: [download, kubelet, kubeadm]
+    when: kubeadm_enabled
   - role: kubernetes/secrets
+    when: not kubeadm_enabled
     tags: k8s-secrets
   - role: download
     file: "{{ downloads.nginx }}"

roles/kubernetes/node/tasks/install.yml

@@ -13,6 +13,18 @@
     ]"
   tags: facts
 
+- name: Set kubelet deployment to host if kubeadm is enabled
+  set_fact:
+    kubelet_deployment_type: host
+  when: kubeadm_enabled
+  tags: kubeadm
+
+- name: install | Copy binary from download dir
+  command: rsync -piu "{{ local_release_dir }}/kubeadm" "{{ bin_dir }}/kubeadm"
+  changed_when: false
+  when: kubeadm_enabled
+  tags: kubeadm
+
 - include: "install_{{ kubelet_deployment_type }}.yml"
 
 - name: install | Write kubelet systemd init file

roles/kubernetes/node/tasks/main.yml

@@ -20,14 +20,24 @@
   when: is_kube_master == false and loadbalancer_apiserver_localhost|default(true)
   tags: nginx
 
-- name: Write kubelet config file
+- name: Write kubelet config file (non-kubeadm)
   template:
-    src: kubelet.j2
+    src: kubelet.standard.env.j2
     dest: "{{ kube_config_dir }}/kubelet.env"
     backup: yes
+  when: not kubeadm_enabled
   notify: restart kubelet
   tags: kubelet
 
+- name: Write kubelet config file (kubeadm)
+  template:
+    src: kubelet.kubeadm.env.j2
+    dest: "{{ kube_config_dir }}/kubelet.env"
+    backup: yes
+  when: kubeadm_enabled
+  notify: restart kubelet
+  tags: ['kubelet', 'kubeadm']
+
 - name: write the kubecfg (auth) file for kubelet
   template:
     src: "{{ item }}-kubeconfig.yaml.j2"
@@ -36,6 +46,7 @@
   with_items:
     - node
     - kube-proxy
+  when: not kubeadm_enabled
   notify: restart kubelet
   tags: kubelet
 
@@ -84,6 +95,7 @@
   template:
     src: manifests/kube-proxy.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-proxy.manifest"
+  when: not kubeadm_enabled
   tags: kube-proxy
 
 # reload-systemd

roles/kubernetes/node/templates/kubelet.host.service.j2

@@ -5,7 +5,7 @@ After=docker.service
 Wants=docker.socket
 
 [Service]
-EnvironmentFile={{kube_config_dir}}/kubelet.env
+EnvironmentFile=-{{kube_config_dir}}/kubelet.env
 ExecStart={{ bin_dir }}/kubelet \
 		$KUBE_LOGTOSTDERR \
 		$KUBE_LOG_LEVEL \

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -0,0 +1,57 @@
+### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+### All upstream values should be present in this file
+
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGGING="--logtostderr=true"
+KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
+# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
+KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}"
+# The port for the info server to serve on
+# KUBELET_PORT="--port=10250"
+# You may leave this blank to use the actual hostname
+{% if kube_override_hostname %}
+KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
+{% endif %}
+{# Base kubelet args #}
+{% set kubelet_args_base -%}
+{# start kubeadm specific settings #}
+--kubeconfig={{ kube_config_dir }}/kubelet.conf \
+--require-kubeconfig=true \
+--authorization-mode=Webhook \
+--client-ca-file={{ kube_cert_dir }}/ca.crt \
+--pod-manifest-path={{ kube_manifest_dir }} \
+--cadvisor-port=0 \
+{# end kubeadm specific settings #}
+--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
+--kube-reserved cpu={{ kubelet_cpu_limit }},memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }} \
+--node-status-update-frequency={{ kubelet_status_update_frequency }} \
+{% endset %}
+
+{# DNS settings for kubelet #}
+{% if dns_mode == 'kubedns' %}
+{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %}
+{% elif dns_mode == 'dnsmasq_kubedns' %}
+{% set kubelet_args_cluster_dns %}--cluster-dns={{ dns_server }}{% endset %}
+{% else %}
+{% set kubelet_args_cluster_dns %}{% endset %}
+{% endif %}
+{% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster-domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %}
+
+
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }}"
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave"] %}
+KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
+{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
+KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
+{% endif %}
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow-privileged=true"
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
+{% elif cloud_provider is defined and cloud_provider == "aws" %}
+KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}"
+{% else %}
+KUBELET_CLOUDPROVIDER=""
+{% endif %}
+
+PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

roles/kubernetes/preinstall/tasks/main.yml

@@ -34,29 +34,18 @@
       skip: true
   tags: facts
 
-- name: Create kubernetes config directory
+- name: Create kubernetes directories
   file:
-    path: "{{ kube_config_dir }}"
+    path: "{{ item }}"
     state: directory
     owner: kube
   when: inventory_hostname in groups['k8s-cluster']
   tags: [kubelet, k8s-secrets, kube-controller-manager, kube-apiserver, bootstrap-os, apps, network, master, node]
-
+  with_items:
-- name: Create kubernetes script directory
+    - "{{ kube_config_dir }}"
-  file:
+    - "{{ kube_config_dir }}/ssl"
-    path: "{{ kube_script_dir }}"
+    - "{{ kube_manifest_dir }}"
-    state: directory
+    - "{{ kube_script_dir }}"
-    owner: kube
-  when: "inventory_hostname in groups['k8s-cluster']"
-  tags: [k8s-secrets, bootstrap-os]
-
-- name: Create kubernetes manifests directory
-  file:
-    path: "{{ kube_manifest_dir }}"
-    state: directory
-    owner: kube
-  when: "inventory_hostname in groups['k8s-cluster']"
-  tags: [kubelet, bootstrap-os, master, node]
 
 - name: check cloud_provider value
   fail:

roles/kubernetes/preinstall/vars/centos.yml

@@ -2,3 +2,4 @@
 required_pkgs:
   - libselinux-python
   - device-mapper-libs
+  - ebtables

roles/kubernetes/preinstall/vars/debian.yml

@@ -4,3 +4,4 @@ required_pkgs:
   - aufs-tools
   - apt-transport-https
   - software-properties-common
+  - ebtables

roles/kubernetes/preinstall/vars/fedora.yml

@@ -2,3 +2,4 @@
 required_pkgs:
   - libselinux-python
   - device-mapper-libs
+  - ebtables

roles/kubernetes/preinstall/vars/redhat.yml

@@ -2,3 +2,4 @@
 required_pkgs:
   - libselinux-python
   - device-mapper-libs
+  - ebtables

roles/kubernetes/preinstall/vars/ubuntu.yml

@@ -0,0 +1,7 @@
+---
+required_pkgs:
+  - python-apt
+  - aufs-tools
+  - apt-transport-https
+  - software-properties-common
+  - ebtables

roles/kubernetes/secrets/tasks/main.yml

@@ -19,21 +19,6 @@
     mode: o-rwx
     group: "{{ kube_cert_group }}"
 
-- name: Make sure the users directory exits
-  file:
-    path: "{{ kube_users_dir }}"
-    state: directory
-    mode: o-rwx
-    group: "{{ kube_cert_group }}"
-
-- name: Populate users for basic auth in API
-  template:
-    src: known_users.csv.j2
-    dest: "{{ kube_users_dir }}/known_users.csv"
-    backup: yes
-  when: inventory_hostname in groups['kube-master'] and kube_basic_auth|default(true)
-  notify: set secret_changed
-
 #
 # The following directory creates make sure that the directories
 # exist on the first master for cases where the first master isn't
@@ -103,7 +88,7 @@
 
 - name: "Gen_certs | set kube node certificate serial facts"
   set_fact:
-    etcd_node_cert_serial: "{{ node_certificate_serials.results[0].stdout|default() }}"
+    kubelet_cert_serial: "{{ node_certificate_serials.results[0].stdout|default() }}"
     kube_proxy_cert_serial: "{{ node_certificate_serials.results[1].stdout|default() }}"
   when: inventory_hostname in groups['k8s-cluster']
 

roles/kubespray-defaults/defaults/main.yaml

@@ -114,6 +114,10 @@ kubelet_deployment_type: docker
 cert_management: script
 vault_deployment_type: docker
 
+# Enable kubeadm deployment (experimental)
+kubeadm_enabled: false
+kubeadm_token: "abcdef.0123456789abcdef"
+
 # K8s image pull policy (imagePullPolicy)
 k8s_image_pull_policy: IfNotPresent
 efk_enabled: false

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -19,7 +19,7 @@ spec:
         k8s-app: calico-node
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
-        kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+        kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     spec:
       hostNetwork: true
 {% if rbac_enabled %}

upgrade-cluster.yml

@@ -67,6 +67,7 @@
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/master, tags: master }
     - { role: network_plugin, tags: network }
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
 
 #Finally handle worker upgrades, based on given batch size
@@ -79,6 +80,7 @@
     - { role: kubernetes/node, tags: node }
     - { role: network_plugin, tags: network }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
     - { role: kubespray-defaults}
 
 - hosts: kube-master