kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Individual etcd ssl certs 

Includes hooks for triggering calico, kubelet, and kube-apiserver restarts
if etcd certs changed.
pull/798/head
Matthew Mosesohn 2016-12-13 09:03:35 +00:00
parent de8cd5cd7f
commit ad796d188d
13 changed files with 140 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
roles/etcd/files/make-ssl-etcd.sh
View File

@ -16,7 +16,6 @@
set -o errexit
set -o pipefail
usage()
{
cat << EOF
@ -61,20 +60,43 @@ cd "${tmpdir}"
mkdir -p "${SSLDIR}"
# Root CA
openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
if [ -e "$SSLDIR/ca-key.pem" ]; then
# Reuse existing CA
cp $SSLDIR/{ca.pem,ca-key.pem} .
else
openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
fi
# ETCD member
openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
if [ -n "$MASTERS" ]; then
for host in $MASTERS; do
openssl genrsa -out member-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${host}" -config ${CONFIG} > /dev/null 2>&1
openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
done
else
openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
fi
# Nodes and Admin
for i in node admin; do
openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
done
# Node and admin keys
if [ -n "$HOSTS" ]; then
for host in $HOSTS; do
for i in node admin; do
openssl genrsa -out ${i}-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key ${i}-${host}-key.pem -out ${i}-${host}.csr -subj "/CN=kube-${i}-${host}" > /dev/null 2>&1
openssl x509 -req -in ${i}-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}-${host}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
done
done
else
for i in node admin; do
openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
done
fi
# Install certs
mv *.pem ${SSLDIR}/

37
roles/etcd/tasks/check_certs.yml
View File

@ -1,36 +1,51 @@
---
- name: "Check_certs | check if the certs have already been generated on first master"
- name: "Check_certs | check if all certs have already been generated on first master"
stat:
path: "{{ etcd_cert_dir }}/ca.pem"
path: "{{ etcd_cert_dir }}/{{ item }}"
delegate_to: "{{groups['etcd'][0]}}"
register: etcdcert_master
run_once: true
with_items: >-
['ca.pem',
{% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|unique %}
{% for host in all_etcd_hosts %}
'node-{{ host }}-key.pem'
{% if not loop.last %}{{','}}{% endif %}
{% endfor %}]
- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false"
- name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false"
set_fact:
sync_certs: false
gen_certs: false
etcd_secret_changed: false
- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true"
- name: "Check_certs | Set 'gen_certs' to true"
set_fact:
gen_certs: true
when: not etcdcert_master.stat.exists
when: "not {{item.stat.exists}}"
run_once: true
with_items: "{{etcdcert_master.results}}"
- name: "Check certs | check if a cert already exists"
stat:
path: "{{ etcd_cert_dir }}/ca.pem"
path: "{{ etcd_cert_dir }}/node-{{inventory_hostname}}-key.pem"
register: etcdcert
with_items:
- ca.pem
- node-{{ inventory_hostname }}-key.pem
- name: "Check_certs | Set 'sync_certs' to true"
set_fact:
sync_certs: true
when: >-
{%- set certs = {'sync': False} -%}
{%- for server in play_hosts
if (not hostvars[server].etcdcert.stat.exists|default(False)) or
(hostvars[server].etcdcert.stat.checksum|default('') != etcdcert_master.stat.checksum|default('')) -%}
{%- set _ = certs.update({'sync': True}) -%}
{% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|unique %}
{% for host in all_etcd_hosts %}
{% if host == inventory_hostname %}
{% if (not etcdcert.results[0].stat.exists|default(False)) or
(etcdcert.results[1].stat.checksum|default('') != etcdcert_master.results[loop.index].stat.checksum|default('')) -%}
{%- set _ = certs.update({'sync': True}) -%}
{% endif %}
{% endif %}
{%- endfor -%}
{{ certs.sync }}
run_once: true

39
roles/etcd/tasks/gen_certs.yml
View File

@ -34,29 +34,56 @@
- name: Gen_certs | run cert generation script
command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
environment:
- MASTERS: "{% for m in groups['etcd'] %}
{% if hostvars[m].sync_certs|default(false) %}
{{ m }}
{% endif %}
{% endfor %}"
- HOSTS: "{% for h in groups['k8s-cluster'] %}
{% if hostvars[h].sync_certs|default(false) %}
{{ h }}
{% endif %}
{% endfor %}"
run_once: yes
delegate_to: "{{groups['etcd'][0]}}"
when: gen_certs|default(false)
notify: set etcd_secret_changed
- set_fact:
master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem']
node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
all_master_certs: "['ca-key.pem',
{% for node in groups['etcd'] %}
'admin-{{ node }}.pem',
'admin-{{ node }}-key.pem',
'member-{{ node }}.pem',
'member-{{ node }}-key.pem',
{% endfor %}]"
my_master_certs: ['ca-key.pem',
'admin-{{ inventory_hostname }}.pem',
'admin-{{ inventory_hostname }}-key.pem',
'member-{{ inventory_hostname }}.pem',
'member-{{ inventory_hostname }}-key.pem'
]
all_node_certs: "['ca.pem',
{% for node in groups['k8s-cluster'] %}
'node-{{ node }}.pem',
'node-{{ node }}-key.pem',
{% endfor %}]"
my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem']
tags: facts
- name: Gen_certs | Gather etcd master certs
shell: "tar cfz - -C {{ etcd_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }}| base64 --wrap=0"
shell: "tar cfz - -C {{ etcd_cert_dir }} {{ my_master_certs|join(' ') }} {{ all_node_certs|join(' ') }}| base64 --wrap=0"
register: etcd_master_cert_data
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
#run_once: true
when: sync_certs|default(false)
notify: set etcd_secret_changed
- name: Gen_certs | Gather etcd node certs
shell: "tar cfz - -C {{ etcd_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
shell: "tar cfz - -C {{ etcd_cert_dir }} {{ my_node_certs|join(' ') }} | base64 --wrap=0"
register: etcd_node_cert_data
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
when: sync_certs|default(false)
notify: set etcd_secret_changed

3
roles/etcd/tasks/main.yml
View File

@ -22,10 +22,11 @@
enabled: yes
when: is_etcd_master
- name: Restart etcd if binary changed
- name: Restart etcd if binary or certs changed
command: /bin/true
notify: restart etcd
when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_master
or etcd_secret_changed|default(false)
# Reload systemd before starting service
- meta: flush_handlers

5
roles/etcd/tasks/pre_upgrade.yml
View File

@ -34,10 +34,6 @@
command: "docker rm -f {{item}}"
with_items: "{{etcd_proxy_container.stdout_lines}}"
- name: "Pre-upgrade | remove etcd-proxy if it exists"
command: "docker rm -f {{item}}"
with_items: "{{etcd_proxy_container.stdout_lines}}"
- name: "Pre-upgrade | check if member list is non-SSL"
command: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses | regex_replace('https','http') }} member list"
register: etcd_member_list
@ -53,4 +49,3 @@
awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash
run_once: true
when: 'etcd_member_list is defined and "http://" in etcd_member_list.stdout'

8
roles/etcd/templates/etcd.j2
View File

@ -13,9 +13,9 @@ ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
# TLS settings
ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
ETCD_CERT_FILE={{ etcd_cert_dir }}/node.pem
ETCD_KEY_FILE={{ etcd_cert_dir }}/node-key.pem
ETCD_CERT_FILE={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
ETCD_KEY_FILE={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member.pem
ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-key.pem
ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=true

4
roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2
View File

@ -31,9 +31,9 @@ spec:
- name: ETCD_CA_CERT_FILE
value: "{{ etcd_cert_dir }}/ca.pem"
- name: ETCD_CERT_FILE
value: "{{ etcd_cert_dir }}/node.pem"
value: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
- name: ETCD_KEY_FILE
value: "{{ etcd_cert_dir }}/node-key.pem"
value: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
# Location of the Kubernetes API - this shouldn't need to be
# changed so long as it is used in conjunction with
# CONFIGURE_ETC_HOSTS="true".

20
roles/kubernetes/master/tasks/pre-upgrade.yml
View File

@ -27,3 +27,23 @@
- /etc/systemd/system/kube-apiserver.service
- /etc/init.d/kube-apiserver
tags: kube-apiserver
- name: "Pre-upgrade | See if kube-apiserver manifest exists"
stat:
path: /etc/kubernetes/manifests/kube-apiserver.manifest
register: kube_apiserver_manifest
when: secret_changed|default(false) or etcd_secret_changed|default(false)
- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if secrets were changed"
replace:
dest: /etc/kubernetes/manifests/kube-apiserver.manifest
regexp: '(\s+)image:\s+.*?$'
replace: '\1image: kill.apiserver.using.fake.image.in:manifest'
register: kube_apiserver_manifest_replaced
when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists
- name: "Pre-upgrade | Pause while waiting for kubelet to delete kube-apiserver pod"
pause: seconds=20
when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists
tags: kube-apiserver

5
roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
View File

@ -5,6 +5,7 @@ metadata:
namespace: {{system_namespace}}
labels:
k8s-app: kube-apiserver
kargo: v2
spec:
hostNetwork: true
containers:
@ -18,8 +19,8 @@ spec:
- --etcd-servers={{ etcd_access_endpoint }}
- --etcd-quorum-read=true
- --etcd-cafile={{ etcd_cert_dir }}/ca.pem
- --etcd-certfile={{ etcd_cert_dir }}/node.pem
- --etcd-keyfile={{ etcd_cert_dir }}/node-key.pem
- --etcd-certfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
- --etcd-keyfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
- --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
- --apiserver-count={{ kube_apiserver_count }}
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota

17
roles/network_plugin/calico/tasks/main.yml
View File

@ -27,8 +27,8 @@
force: yes
with_items:
- {s: "ca.pem", d: "ca_cert.crt"}
- {s: "node.pem", d: "cert.crt"}
- {s: "node-key.pem", d: "key.pem"}
- {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
- {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
- name: Calico | Install calicoctl container script
template:
@ -72,8 +72,8 @@
command: |-
curl \
--cacert {{ etcd_cert_dir }}/ca.pem \
--cert {{ etcd_cert_dir}}/admin.pem \
--key {{ etcd_cert_dir }}/admin-key.pem \
--cert {{ etcd_cert_dir}}/admin-{{ groups['etcd'][0] }}.pem \
--key {{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem \
https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
register: calico_conf
delegate_to: "{{groups['etcd'][0]}}"
@ -131,8 +131,8 @@
command: |-
curl \
--cacert {{ etcd_cert_dir }}/ca.pem \
--cert {{ etcd_cert_dir}}/admin.pem \
--key {{ etcd_cert_dir }}/admin-key.pem \
--cert {{ etcd_cert_dir}}/admin-{{ groups['etcd'][0] }}.pem \
--key {{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem \
https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
register: calico_pools_raw
delegate_to: "{{groups['etcd'][0]}}"
@ -191,6 +191,11 @@
when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "RedHat"
notify: restart calico-node
- name: Calico | Restart calico-node if secrets changed
command: /bin/true
when: secret_changed|default(false) or etcd_secret_changed|default(false)
notify: restart calico-node
- meta: flush_handlers
- name: Calico | Enable calico-node

4
roles/network_plugin/calico/templates/cni-calico.conf.j2
View File

@ -5,8 +5,8 @@
{% endif %}
"type": "calico",
"etcd_endpoints": "{{ etcd_access_endpoint }}",
"etcd_cert_file": "{{ etcd_cert_dir }}/node.pem",
"etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem",
"etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem",
"etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem",
"etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
"log_level": "info",
"ipam": {

4
roles/network_plugin/canal/tasks/main.yml
View File

@ -21,8 +21,8 @@
force: yes
with_items:
- {s: "ca.pem", d: "ca_cert.crt"}
- {s: "node.pem", d: "cert.crt"}
- {s: "node-key.pem", d: "key.pem"}
- {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
- {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
- name: Canal | Set Flannel etcd configuration
command: |-

2
roles/network_plugin/flannel/templates/flannel-pod.yml
View File

@ -22,7 +22,7 @@
command:
- "/bin/sh"
- "-c"
- "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
- "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -etcd-keyfile {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
ports:
- hostPort: 10253
containerPort: 10253