From a70c3b661e12d75ca01f8c0b38ef6af625d3383a Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Wed, 13 Jul 2016 17:13:47 +0200 Subject: [PATCH] Add HA/LB endpoints for kube-apiserver * Add auto-evaluated internal endpoints and clarify the loadbalancer_apiserver vars and usecases. * Add loadbalancer_apiserver_localhost (default false). If enabled, override the external LB and expect localhost:443/8080 to be new internal only frontends. * Add kube_apiserver_multiaccess to ignore loadbalancers, and make clients to access the apiservers as a comma-separated list of access_ip/ip/ansible ip (a default mode). When disabled, allow clients to use the given loadbalancers. * Define connections security mode for kube controllers, schedulers, proxies. It is insecure be default, which is the current deployment choice. * Rework the groups['kube-master'][0] hardcode defining the apiserver endpoints. * Improve grouping of vars and add facts for kube_apiserver. * Define kube_apiserver_insecure_bind_address as a fact, add more facts for ease of use. Signed-off-by: Bogdan Dobrelya --- inventory/group_vars/all.yml | 77 +++++++++++++------ .../master/templates/kube-apiserver.j2 | 4 +- .../templates/kubectl-kubeconfig.yaml.j2 | 2 +- .../kube-controller-manager.manifest.j2 | 6 +- .../manifests/kube-scheduler.manifest.j2 | 6 +- roles/kubernetes/node/templates/kubelet.j2 | 2 +- .../manifests/kube-proxy.manifest.j2 | 10 +-- .../kubernetes/preinstall/tasks/set_facts.yml | 34 ++++++++ 8 files changed, 104 insertions(+), 37 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index e9b138c92..9c7000c0c 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -53,7 +53,7 @@ cluster_name: cluster.local # Service endpoints. May be a VIP or a load balanced frontend IP, like one # that a HAProxy or Nginx provides, or just a local service endpoint. -# + # Etcd endpoints use a local etcd-proxies to reach the etcd cluster via # auto-evaluated endpoints. Those will reuse the access_ip for etcd cluster, # if specified, or defer to the localhost:2379 as well. @@ -64,15 +64,64 @@ cluster_name: cluster.local # This may be the case if clients support and loadbalance multiple etcd servers natively. etcd_multiaccess: false -# -# TODO apiserver localhost:8080 and localhost:443 endpoints for kubelets and -# (hyper)kube-* and networking components. +# The port the API Server will be listening on. +kube_apiserver_port: 443 # (https) +kube_apiserver_insecure_port: 8080 # (http) + +# For multi masters architecture: +# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer +# This domain name will be inserted into the /etc/hosts file of all servers +# configuration example with haproxy: +# listen kubernetes-apiserver-https +# bind 10.99.0.21:8383 +# option ssl-hello-chk +# mode tcp +# timeout client 3h +# timeout server 3h +# server master1 10.99.0.26:443 +# server master2 10.99.0.27:443 +# balance roundrobin + +# And the corresponding example config vars: +# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" +# loadbalancer_apiserver: +# adress: 10.99.0.21 +# port: 8383 + +# For internal only multi-masters architecture: +# Assume there is no loadbalancers configured for internal access. Enable this flag, +# if a loadbalancer listens the localhost, which may be the case when there is neither +# loadbalancer_apiserver nor access_ip is configured, or you want to override both. +loadbalancer_apiserver_localhost: false + +# The internal kube-apiserver endpoints: +# The secure endpoint is auto-evaluated. If loadbalancer_apiserver_localhost=true, +# it uses the localhost:kube_apiserver_port. Otherwise, it uses the external +# apiserver_loadbalancer_domain_name:loadbalancer_apiserver.port. Or defers to the +# access_ip:kube_apiserver_port, then ip, then default ansible ip. + +# The insecure endpoint ignores the apiserver_loadbalancer_domain_name and access_ip. +# By default, it is a localhost:kube_apiserver_insecure_port. +# If loadbalancer_apiserver_localhost, it defers to ip, then default ansible ip. + +# Apiserver access modes: +# Configure clients to access all of the apiservers directly as the +# "http(s)://hostX:port, http(s)://hostY:port, ..." and ignore the loadbalancers, if any. +# This may be the case, if clients support and loadbalance multiple apiservers or +# when there is no loadbalancers configured. +kube_apiserver_multiaccess: true + +# Define connections security for kube controllers, schedulers and proxies +kube_proxy_insecure: true +kube_controller_insecure: true +kube_scheduler_insecure: true # Choose network plugin (calico, weave or flannel) kube_network_plugin: flannel # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 +kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" # internal network. When used, it will assign IP # addresses from this range to individual pods. @@ -93,11 +142,6 @@ peer_with_router: false # Warning : enabling router peering will disable calico's default behavior ('node mesh'). # The subnets of each nodes will be distributed by the datacenter router -# The port the API Server will be listening on. -kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" -kube_apiserver_port: 443 # (https) -kube_apiserver_insecure_port: 8080 # (http) - # Internal DNS configuration. # Kubernetes can create and mainatain its own DNS server to resolve service names # into appropriate IP addresses. It's highly advisable to run such DNS server, @@ -126,21 +170,6 @@ dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') # like you would do when using nova-client before starting the playbook. # cloud_provider: -# For multi masters architecture: -# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer -# This domain name will be inserted into the /etc/hosts file of all servers -# configuration example with haproxy : -# listen kubernetes-apiserver-https -# bind 10.99.0.21:8383 -# option ssl-hello-chk -# mode tcp -# timeout client 3h -# timeout server 3h -# server master1 10.99.0.26:443 -# server master2 10.99.0.27:443 -# balance roundrobin -# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" - ## Set these proxy values in order to update docker daemon to use proxies # http_proxy: "" # https_proxy: "" diff --git a/roles/kubernetes/master/templates/kube-apiserver.j2 b/roles/kubernetes/master/templates/kube-apiserver.j2 index 0e3a2710d..25719e91f 100644 --- a/roles/kubernetes/master/templates/kube-apiserver.j2 +++ b/roles/kubernetes/master/templates/kube-apiserver.j2 @@ -21,7 +21,7 @@ KUBE_ALLOW_PRIV="--allow_privileged=true" KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}" # Insecure API address (default is localhost) -KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}" +KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address }}" # Address range to use for services KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}" @@ -42,7 +42,7 @@ KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in k KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem" # Add you own! -KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}" +KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ kube_apiserver_address }}" {% if cloud_provider is defined and cloud_provider == "openstack" %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" diff --git a/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 b/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 index 5cc74cf9e..a9800d3ac 100644 --- a/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 +++ b/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 @@ -5,7 +5,7 @@ preferences: {} clusters: - cluster: certificate-authority-data: {{ kube_node_cert|b64encode }} - server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }} + server: {{ kube_apiserver_endpoint }} name: {{ cluster_name }} contexts: - context: diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 29c56e6a8..d185a863c 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -11,7 +11,11 @@ spec: command: - /hyperkube - controller-manager - - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} +{% if kube_controller_insecure %} + - --master={{ kube_apiserver_insecure_endpoint }} +{% else %} + - --master={{ kube_apiserver_endpoint }} +{% endif %} - --leader-elect=true - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem - --root-ca-file={{ kube_cert_dir }}/ca.pem diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index f642fbf70..6993bbb8d 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -12,7 +12,11 @@ spec: - /hyperkube - scheduler - --leader-elect=true - - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} +{% if kube_scheduler_insecure %} + - --master={{ kube_apiserver_insecure_endpoint }} +{% else %} + - --master={{ kube_apiserver_endpoint }} +{% endif %} - --v={{ kube_log_level | default('2') }} livenessProbe: httpGet: diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 20e521f73..27687eac7 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -8,7 +8,7 @@ KUBE_LOGGING="--logtostderr=true" KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" KUBE_ALLOW_PRIV="--allow_privileged=true" {% if inventory_hostname in groups['kube-node'] %} -KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" +KUBELET_API_SERVER="--api_servers={{ kube_apiserver_access_endpoint }}" {% endif %} # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}" diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index d094766b5..f4877784a 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -12,14 +12,10 @@ spec: - /hyperkube - proxy - --v={{ kube_log_level | default('2') }} -{% if inventory_hostname in groups['kube-master'] %} - - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} +{% if kube_proxy_insecure %} + - --master={{ kube_apiserver_insecure_endpoint }} {% else %} -{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} - - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} -{% else %} - - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }} -{% endif%} + - --master={{ kube_apiserver_access_endpoint }} - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml {% endif %} - --bind-address={{ ip | default(ansible_default_ipv4.address) }} diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 5dd283fef..055c77b07 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -1,4 +1,38 @@ --- +- set_fact: kube_apiserver_count="{{ groups['kube-master'] | length }}" +- set_fact: kube_apiserver_address="{{ ip | default(ansible_default_ipv4['address']) }}" +- set_fact: kube_apiserver_access_address="{{ access_ip | default(kube_apiserver_address) }}" +- set_fact: + kube_apiserver_insecure_bind_address: |- + {% if loadbalancer_apiserver_localhost %} + {{ kube_apiserver_address }}{% else %}127.0.0.1{% endif %} +- set_fact: + apiserver_access_port: |- + {% if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined %} + {{ loadbalancer_apiserver.port }}{% else %}{{ kube_apiserver_port }}{% endif %} +- set_fact: + kube_apiserver_endpoint: |- + {% if loadbalancer_apiserver_localhost %} + https://127.0.0.1:{{ apiserver_access_port }} + {% else -%} + https://{{ apiserver_loadbalancer_domain_name|default(kube_apiserver_access_address) }}:{{ apiserver_access_port }} + {%- endif %} +- set_fact: kube_apiserver_insecure_endpoint="http://{{ kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}" +- set_fact: + kube_apiserver_access_endpoint: |- + {% if kube_apiserver_multiaccess %} + {% for host in groups['kube-master'] -%} + https://{{ hostvars[host].kube_apiserver_access_address }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %} + {%- endfor %} + {% else %}{{ kube_apiserver_endpoint }}{% endif %} +- set_fact: + kube_apiserver_insecure_access_endpoint: |- + {% if kube_apiserver_multiaccess %} + {% for host in groups['kube-master'] -%} + https://{{ hostvars[host].kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}{% if not loop.last %},{% endif %} + {%- endfor %} + {% else %}{{ kube_apiserver_insecure_endpoint }}{% endif %} + - set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}" - set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}" - set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"