diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 9c7000c0c..e9b138c92 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -53,7 +53,7 @@ cluster_name: cluster.local # Service endpoints. May be a VIP or a load balanced frontend IP, like one # that a HAProxy or Nginx provides, or just a local service endpoint. - +# # Etcd endpoints use a local etcd-proxies to reach the etcd cluster via # auto-evaluated endpoints. Those will reuse the access_ip for etcd cluster, # if specified, or defer to the localhost:2379 as well. @@ -64,64 +64,15 @@ cluster_name: cluster.local # This may be the case if clients support and loadbalance multiple etcd servers natively. etcd_multiaccess: false -# The port the API Server will be listening on. -kube_apiserver_port: 443 # (https) -kube_apiserver_insecure_port: 8080 # (http) - -# For multi masters architecture: -# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer -# This domain name will be inserted into the /etc/hosts file of all servers -# configuration example with haproxy: -# listen kubernetes-apiserver-https -# bind 10.99.0.21:8383 -# option ssl-hello-chk -# mode tcp -# timeout client 3h -# timeout server 3h -# server master1 10.99.0.26:443 -# server master2 10.99.0.27:443 -# balance roundrobin - -# And the corresponding example config vars: -# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" -# loadbalancer_apiserver: -# adress: 10.99.0.21 -# port: 8383 - -# For internal only multi-masters architecture: -# Assume there is no loadbalancers configured for internal access. Enable this flag, -# if a loadbalancer listens the localhost, which may be the case when there is neither -# loadbalancer_apiserver nor access_ip is configured, or you want to override both. -loadbalancer_apiserver_localhost: false - -# The internal kube-apiserver endpoints: -# The secure endpoint is auto-evaluated. If loadbalancer_apiserver_localhost=true, -# it uses the localhost:kube_apiserver_port. Otherwise, it uses the external -# apiserver_loadbalancer_domain_name:loadbalancer_apiserver.port. Or defers to the -# access_ip:kube_apiserver_port, then ip, then default ansible ip. - -# The insecure endpoint ignores the apiserver_loadbalancer_domain_name and access_ip. -# By default, it is a localhost:kube_apiserver_insecure_port. -# If loadbalancer_apiserver_localhost, it defers to ip, then default ansible ip. - -# Apiserver access modes: -# Configure clients to access all of the apiservers directly as the -# "http(s)://hostX:port, http(s)://hostY:port, ..." and ignore the loadbalancers, if any. -# This may be the case, if clients support and loadbalance multiple apiservers or -# when there is no loadbalancers configured. -kube_apiserver_multiaccess: true - -# Define connections security for kube controllers, schedulers and proxies -kube_proxy_insecure: true -kube_controller_insecure: true -kube_scheduler_insecure: true +# +# TODO apiserver localhost:8080 and localhost:443 endpoints for kubelets and +# (hyper)kube-* and networking components. # Choose network plugin (calico, weave or flannel) kube_network_plugin: flannel # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 -kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" # internal network. When used, it will assign IP # addresses from this range to individual pods. @@ -142,6 +93,11 @@ peer_with_router: false # Warning : enabling router peering will disable calico's default behavior ('node mesh'). # The subnets of each nodes will be distributed by the datacenter router +# The port the API Server will be listening on. +kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" +kube_apiserver_port: 443 # (https) +kube_apiserver_insecure_port: 8080 # (http) + # Internal DNS configuration. # Kubernetes can create and mainatain its own DNS server to resolve service names # into appropriate IP addresses. It's highly advisable to run such DNS server, @@ -170,6 +126,21 @@ dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') # like you would do when using nova-client before starting the playbook. # cloud_provider: +# For multi masters architecture: +# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer +# This domain name will be inserted into the /etc/hosts file of all servers +# configuration example with haproxy : +# listen kubernetes-apiserver-https +# bind 10.99.0.21:8383 +# option ssl-hello-chk +# mode tcp +# timeout client 3h +# timeout server 3h +# server master1 10.99.0.26:443 +# server master2 10.99.0.27:443 +# balance roundrobin +# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" + ## Set these proxy values in order to update docker daemon to use proxies # http_proxy: "" # https_proxy: "" diff --git a/roles/kubernetes/master/templates/kube-apiserver.j2 b/roles/kubernetes/master/templates/kube-apiserver.j2 index 25719e91f..0e3a2710d 100644 --- a/roles/kubernetes/master/templates/kube-apiserver.j2 +++ b/roles/kubernetes/master/templates/kube-apiserver.j2 @@ -21,7 +21,7 @@ KUBE_ALLOW_PRIV="--allow_privileged=true" KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}" # Insecure API address (default is localhost) -KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address }}" +KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}" # Address range to use for services KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}" @@ -42,7 +42,7 @@ KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in k KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem" # Add you own! -KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ kube_apiserver_address }}" +KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}" {% if cloud_provider is defined and cloud_provider == "openstack" %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" diff --git a/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 b/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 index a9800d3ac..5cc74cf9e 100644 --- a/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 +++ b/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 @@ -5,7 +5,7 @@ preferences: {} clusters: - cluster: certificate-authority-data: {{ kube_node_cert|b64encode }} - server: {{ kube_apiserver_endpoint }} + server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }} name: {{ cluster_name }} contexts: - context: diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index d185a863c..29c56e6a8 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -11,11 +11,7 @@ spec: command: - /hyperkube - controller-manager -{% if kube_controller_insecure %} - - --master={{ kube_apiserver_insecure_endpoint }} -{% else %} - - --master={{ kube_apiserver_endpoint }} -{% endif %} + - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} - --leader-elect=true - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem - --root-ca-file={{ kube_cert_dir }}/ca.pem diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index 6993bbb8d..f642fbf70 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -12,11 +12,7 @@ spec: - /hyperkube - scheduler - --leader-elect=true -{% if kube_scheduler_insecure %} - - --master={{ kube_apiserver_insecure_endpoint }} -{% else %} - - --master={{ kube_apiserver_endpoint }} -{% endif %} + - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} - --v={{ kube_log_level | default('2') }} livenessProbe: httpGet: diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 27687eac7..20e521f73 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -8,7 +8,7 @@ KUBE_LOGGING="--logtostderr=true" KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" KUBE_ALLOW_PRIV="--allow_privileged=true" {% if inventory_hostname in groups['kube-node'] %} -KUBELET_API_SERVER="--api_servers={{ kube_apiserver_access_endpoint }}" +KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" {% endif %} # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}" diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index f4877784a..d094766b5 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -12,10 +12,14 @@ spec: - /hyperkube - proxy - --v={{ kube_log_level | default('2') }} -{% if kube_proxy_insecure %} - - --master={{ kube_apiserver_insecure_endpoint }} +{% if inventory_hostname in groups['kube-master'] %} + - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} {% else %} - - --master={{ kube_apiserver_access_endpoint }} +{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} + - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} +{% else %} + - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }} +{% endif%} - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml {% endif %} - --bind-address={{ ip | default(ansible_default_ipv4.address) }} diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 055c77b07..5dd283fef 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -1,38 +1,4 @@ --- -- set_fact: kube_apiserver_count="{{ groups['kube-master'] | length }}" -- set_fact: kube_apiserver_address="{{ ip | default(ansible_default_ipv4['address']) }}" -- set_fact: kube_apiserver_access_address="{{ access_ip | default(kube_apiserver_address) }}" -- set_fact: - kube_apiserver_insecure_bind_address: |- - {% if loadbalancer_apiserver_localhost %} - {{ kube_apiserver_address }}{% else %}127.0.0.1{% endif %} -- set_fact: - apiserver_access_port: |- - {% if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined %} - {{ loadbalancer_apiserver.port }}{% else %}{{ kube_apiserver_port }}{% endif %} -- set_fact: - kube_apiserver_endpoint: |- - {% if loadbalancer_apiserver_localhost %} - https://127.0.0.1:{{ apiserver_access_port }} - {% else -%} - https://{{ apiserver_loadbalancer_domain_name|default(kube_apiserver_access_address) }}:{{ apiserver_access_port }} - {%- endif %} -- set_fact: kube_apiserver_insecure_endpoint="http://{{ kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}" -- set_fact: - kube_apiserver_access_endpoint: |- - {% if kube_apiserver_multiaccess %} - {% for host in groups['kube-master'] -%} - https://{{ hostvars[host].kube_apiserver_access_address }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %} - {%- endfor %} - {% else %}{{ kube_apiserver_endpoint }}{% endif %} -- set_fact: - kube_apiserver_insecure_access_endpoint: |- - {% if kube_apiserver_multiaccess %} - {% for host in groups['kube-master'] -%} - https://{{ hostvars[host].kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}{% if not loop.last %},{% endif %} - {%- endfor %} - {% else %}{{ kube_apiserver_insecure_endpoint }}{% endif %} - - set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}" - set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}" - set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"