From a56738324acee45184e056327666e8fb41ea723b Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 4 Oct 2017 13:27:55 +0100 Subject: [PATCH] Move set_facts to kubespray-defaults defaults These facts can be generated in defaults with a performance boost. Also cleaned up duplicate etcd var names. --- roles/etcd/tasks/install_docker.yml | 2 +- roles/etcd/tasks/install_rkt.yml | 2 +- roles/etcd/templates/etcd.j2 | 4 +- .../templates/calico-policy-controller.yml.j2 | 2 +- .../master/templates/kubeadm-config.yaml.j2 | 2 +- .../manifests/kube-apiserver.manifest.j2 | 2 +- .../kubernetes/preinstall/tasks/set_facts.yml | 88 ------------------- roles/kubespray-defaults/defaults/main.yaml | 44 ++++++++++ .../calico/rr/templates/calico-rr.env.j2 | 2 +- .../calico/templates/calico-config.yml.j2 | 2 +- .../calico/templates/calicoctl-container.j2 | 2 +- .../calico/templates/cni-calico.conf.j2 | 2 +- .../canal/templates/calicoctl-container.j2 | 2 +- .../canal/templates/canal-config.yaml.j2 | 2 +- .../canal/templates/cni-canal.conf.j2 | 2 +- 15 files changed, 57 insertions(+), 103 deletions(-) diff --git a/roles/etcd/tasks/install_docker.yml b/roles/etcd/tasks/install_docker.yml index f0b277981..43f4e44d8 100644 --- a/roles/etcd/tasks/install_docker.yml +++ b/roles/etcd/tasks/install_docker.yml @@ -2,7 +2,7 @@ - name: Install | Copy etcdctl binary from docker container command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy; {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && - {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl && + {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcdctl {{ bin_dir }}/etcdctl && {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy" when: etcd_deployment_type == "docker" register: etcd_task_result diff --git a/roles/etcd/tasks/install_rkt.yml b/roles/etcd/tasks/install_rkt.yml index 85f875383..5f7004229 100644 --- a/roles/etcd/tasks/install_rkt.yml +++ b/roles/etcd/tasks/install_rkt.yml @@ -18,7 +18,7 @@ --mount=volume=bin-dir,target=/host/bin {{ etcd_image_repo }}:{{ etcd_image_tag }} --name=etcdctl-binarycopy - --exec=/bin/cp -- {{ etcd_container_bin_dir }}/etcdctl /host/bin/etcdctl + --exec=/bin/cp -- /usr/local/bin/etcdctl /host/bin/etcdctl register: etcd_task_result until: etcd_task_result.rc == 0 retries: 4 diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 11f8f74e3..9ac08e073 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -17,7 +17,5 @@ {% endif %} --name={{ etcd_member_name | default("etcd") }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ - {% if etcd_after_v3 %} - {{ etcd_container_bin_dir }}etcd \ - {% endif %} + /usr/local/bin/etcd \ "$@" diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 index ca1711463..d715358c8 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 @@ -40,7 +40,7 @@ spec: memory: {{ calico_policy_controller_memory_requests }} env: - name: ETCD_ENDPOINTS - value: "{{ etcd_access_endpoint }}" + value: "{{ etcd_access_addresses }}" - name: ETCD_CA_CERT_FILE value: "{{ calico_cert_dir }}/ca_cert.crt" - name: ETCD_CERT_FILE diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index c8dfd9524..78d94d31e 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -5,7 +5,7 @@ api: bindPort: {{ kube_apiserver_port }} etcd: endpoints: -{% for endpoint in etcd_access_endpoint.split(',') %} +{% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 58c762961..cad57b5f2 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -29,7 +29,7 @@ spec: - /hyperkube - apiserver - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - - --etcd-servers={{ etcd_access_endpoint }} + - --etcd-servers={{ etcd_access_addresses }} - --etcd-quorum-read=true - --etcd-cafile={{ etcd_cert_dir }}/ca.pem - --etcd-certfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 96ec25499..073033315 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -1,92 +1,4 @@ --- -- set_fact: - kube_apiserver_count: "{{ groups['kube-master'] | length }}" - -- set_fact: - kube_apiserver_address: "{{ ip | default(ansible_default_ipv4['address']) }}" - -- set_fact: - kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}" - -- set_fact: - is_kube_master: "{{ inventory_hostname in groups['kube-master'] }}" - -- set_fact: - first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}" - -- set_fact: - loadbalancer_apiserver_localhost: false - when: loadbalancer_apiserver is defined - -- set_fact: - kube_apiserver_endpoint: |- - {% if not is_kube_master and loadbalancer_apiserver_localhost|default(true) -%} - https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }} - {%- elif is_kube_master -%} - https://127.0.0.1:{{ kube_apiserver_port }} - {%- else -%} - {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%} - https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} - {%- else -%} - https://{{ first_kube_master }}:{{ kube_apiserver_port }} - {%- endif -%} - {%- endif %} - -- set_fact: - kube_apiserver_insecure_endpoint: >- - http://{{ kube_apiserver_insecure_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_insecure_port }} - -- set_fact: - etcd_address: "{{ ip | default(ansible_default_ipv4['address']) }}" - -- set_fact: - etcd_access_address: "{{ access_ip | default(etcd_address) }}" - -- set_fact: - etcd_peer_url: "https://{{ etcd_access_address }}:2380" - -- set_fact: - etcd_client_url: "https://{{ etcd_access_address }}:2379" - -- set_fact: - etcd_authority: "127.0.0.1:2379" - -- set_fact: - etcd_endpoint: "https://{{ etcd_authority }}" - -- set_fact: - etcd_access_addresses: |- - {% for item in groups['etcd'] -%} - https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %} - {%- endfor %} - -- set_fact: - etcd_access_endpoint: "{% if etcd_multiaccess|default(true) %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}" - -- set_fact: - etcd_member_name: |- - {% for host in groups['etcd'] %} - {% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %} - {% endfor %} - -- set_fact: - etcd_peer_addresses: |- - {% for item in groups['etcd'] -%} - {{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} - {%- endfor %} - -- set_fact: - is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" - -- set_fact: - etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") - -- set_fact: - etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" - -- set_fact: - peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" - - name: check if atomic host stat: path: /run/ostree-booted diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 6e84a0311..61f820c62 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -77,6 +77,9 @@ kube_users: # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico +# Determines if calico-rr group exists +peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" + # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 @@ -158,3 +161,44 @@ vault_cert_dir: "{{ vault_base_dir }}/ssl" vault_config_dir: "{{ vault_base_dir }}/config" vault_roles_dir: "{{ vault_base_dir }}/roles" vault_secrets_dir: "{{ vault_base_dir }}/secrets" + +# Vars for pointing to kubernetes api endpoints +is_kube_master: "{{ inventory_hostname in groups['kube-master'] }}" +kube_apiserver_count: "{{ groups['kube-master'] | length }}" +kube_apiserver_address: "{{ ip | default(ansible_default_ipv4['address']) }}" +kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}" +first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}" +loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" +kube_apiserver_endpoint: |- + {% if not is_kube_master and loadbalancer_apiserver_localhost|default(true) -%} + https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }} + {%- elif is_kube_master -%} + https://127.0.0.1:{{ kube_apiserver_port }} + {%- else -%} + {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%} + https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} + {%- else -%} + https://{{ first_kube_master }}:{{ kube_apiserver_port }} + {%- endif -%} + {%- endif %} +kube_apiserver_insecure_endpoint: >- + http://{{ kube_apiserver_insecure_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_insecure_port }} + +# Vars for pointing to etcd endpoints +is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" +etcd_address: "{{ ip | default(ansible_default_ipv4['address']) }}" +etcd_access_address: "{{ access_ip | default(etcd_address) }}" +etcd_peer_url: "https://{{ etcd_access_address }}:2380" +etcd_client_url: "https://{{ etcd_access_address }}:2379" +etcd_access_addresses: |- + {% for item in groups['etcd'] -%} + https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %} + {%- endfor %} +etcd_member_name: |- + {% for host in groups['etcd'] %} + {% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %} + {% endfor %} +etcd_peer_addresses: |- + {% for item in groups['etcd'] -%} + {{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} + {%- endfor %} diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 index 201caecfe..1cdb2659c 100644 --- a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 +++ b/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 @@ -1,4 +1,4 @@ -ETCD_ENDPOINTS="{{ etcd_access_endpoint }}" +ETCD_ENDPOINTS="{{ etcd_access_addresses }}" ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt" ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt" ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem" diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2 index a4207f1dc..fbae4eda1 100644 --- a/roles/network_plugin/calico/templates/calico-config.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -4,7 +4,7 @@ metadata: name: calico-config namespace: {{ system_namespace }} data: - etcd_endpoints: "{{ etcd_access_endpoint }}" + etcd_endpoints: "{{ etcd_access_addresses }}" etcd_ca: "/calico-secrets/ca_cert.crt" etcd_cert: "/calico-secrets/cert.crt" etcd_key: "/calico-secrets/key.pem" diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2 index b752fe340..c9a1b5d40 100644 --- a/roles/network_plugin/calico/templates/calicoctl-container.j2 +++ b/roles/network_plugin/calico/templates/calicoctl-container.j2 @@ -1,7 +1,7 @@ #!/bin/bash {{ docker_bin_dir }}/docker run -i --privileged --rm \ --net=host --pid=host \ --e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \ +-e ETCD_ENDPOINTS={{ etcd_access_addresses }} \ -e ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \ -e ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \ -e ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \ diff --git a/roles/network_plugin/calico/templates/cni-calico.conf.j2 b/roles/network_plugin/calico/templates/cni-calico.conf.j2 index 49be7e2ac..892391d11 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conf.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conf.j2 @@ -6,7 +6,7 @@ "nodename": "{{ ansible_hostname }}", {% endif %} "type": "calico", - "etcd_endpoints": "{{ etcd_access_endpoint }}", + "etcd_endpoints": "{{ etcd_access_addresses }}", "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem", "etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem", "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem", diff --git a/roles/network_plugin/canal/templates/calicoctl-container.j2 b/roles/network_plugin/canal/templates/calicoctl-container.j2 index 28f09b4c5..cc0a38bfc 100644 --- a/roles/network_plugin/canal/templates/calicoctl-container.j2 +++ b/roles/network_plugin/canal/templates/calicoctl-container.j2 @@ -1,7 +1,7 @@ #!/bin/bash {{ docker_bin_dir }}/docker run -i --privileged --rm \ --net=host --pid=host \ --e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \ +-e ETCD_ENDPOINTS={{ etcd_access_addresses }} \ -e ETCD_CA_CERT_FILE={{ canal_cert_dir }}/ca_cert.crt \ -e ETCD_CERT_FILE={{ canal_cert_dir }}/cert.crt \ -e ETCD_KEY_FILE={{ canal_cert_dir }}/key.pem \ diff --git a/roles/network_plugin/canal/templates/canal-config.yaml.j2 b/roles/network_plugin/canal/templates/canal-config.yaml.j2 index 1d0d3002a..ab6c276bd 100644 --- a/roles/network_plugin/canal/templates/canal-config.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-config.yaml.j2 @@ -7,7 +7,7 @@ metadata: name: canal-config data: # Configure this with the location of your etcd cluster. - etcd_endpoints: "{{ etcd_access_endpoint }}" + etcd_endpoints: "{{ etcd_access_addresses }}" # The interface used by canal for host <-> host communication. # If left blank, then the interface is chosing using the node's diff --git a/roles/network_plugin/canal/templates/cni-canal.conf.j2 b/roles/network_plugin/canal/templates/cni-canal.conf.j2 index b835443c7..b47d7f9dd 100644 --- a/roles/network_plugin/canal/templates/cni-canal.conf.j2 +++ b/roles/network_plugin/canal/templates/cni-canal.conf.j2 @@ -3,7 +3,7 @@ "type": "flannel", "delegate": { "type": "calico", - "etcd_endpoints": "{{ etcd_access_endpoint }}", + "etcd_endpoints": "{{ etcd_access_addresses }}", "log_level": "info", "policy": { "type": "k8s"