From 9715962356828afe72abbc2d9af3d05117833ede Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 19 Jan 2016 15:23:19 +0100 Subject: [PATCH] etcd directly in host fix etcd configuration for nodes fix wrong calico checksums using a var name etcd_bin_dir fix etcd handlers for sysvinit using a var name etcd_bin_dir sysvinit script review etcd configuration --- README.md | 6 +- roles/download/defaults/main.yml | 32 +++++-- roles/etcd/defaults/main.yml | 3 + roles/etcd/handlers/main.yml | 14 +++ roles/etcd/tasks/configure.yml | 27 ++++++ roles/etcd/tasks/install.yml | 14 +++ roles/etcd/tasks/main.yml | 13 +-- roles/etcd/templates/deb-etcd.initd.j2 | 113 +++++++++++++++++++++++++ roles/etcd/templates/etcd-pod.yml | 54 ------------ roles/etcd/templates/etcd.j2 | 17 ++++ roles/etcd/templates/etcd.service.j2 | 18 ++++ 11 files changed, 234 insertions(+), 77 deletions(-) create mode 100644 roles/etcd/defaults/main.yml create mode 100644 roles/etcd/handlers/main.yml create mode 100644 roles/etcd/tasks/configure.yml create mode 100644 roles/etcd/tasks/install.yml create mode 100644 roles/etcd/templates/deb-etcd.initd.j2 delete mode 100644 roles/etcd/templates/etcd-pod.yml create mode 100644 roles/etcd/templates/etcd.j2 create mode 100644 roles/etcd/templates/etcd.service.j2 diff --git a/README.md b/README.md index cea77f6ab..8b3856da0 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ in order to avoid any issue during deployment you should **disable your firewall ### Components * [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.4 -* [etcd](https://github.com/coreos/etcd/releases) v2.2.2 +* [etcd](https://github.com/coreos/etcd/releases) v2.2.4 * [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.14.0 * [flanneld](https://github.com/coreos/flannel/releases) v0.5.5 * [docker](https://www.docker.com/) v1.9.1 @@ -255,7 +255,7 @@ That way is easier if you want to do some changes and commit them. ### Networking -#### Calico networking +#### Calico Check if the calico-node container is running ``` docker ps | grep calico @@ -277,7 +277,7 @@ calicoctl pool show calicoctl endpoint show --detail ``` -#### Flannel networking +#### Flannel * Flannel configuration file should have been created there ``` diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index ce142a50d..684c75bbb 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -1,27 +1,41 @@ --- local_release_dir: /tmp +# Versions +kube_version: v1.1.4 +etcd_version: v2.2.4 calico_version: v0.14.0 calico_plugin_version: v0.7.0 -kube_version: v1.1.4 +# Download URL's +kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64" +etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz" +calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl" +calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes" + +# Checksums +calico_checksum: "f251d7a8583233906aa6d059447c1e4fb32bf1369a51fdf96a68d50466d6a69c" +calico_plugin_checksum: "032f582f5eeec6fb26191d2fbcbf8bca4da3b14abb579db7baa7b3504d4dffec" +etcd_checksum: "6c4e5cdeaaac1a70b8f06b5dd6b82c37ff19993c9bca81248975610e555c4b9b" kubectl_checksum: "873ba19926d17a3287dc8639ea1434fe3cd0cb4e61d82101ba754922cfc7a633" kubelet_checksum: "f2d1eae3fa6e304f6cbc9b2621e4b86fc3bcb4e74a15d35f58bf00e45c706e0a" -kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64" - -calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl" - -calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes" - downloads: - name: calico dest: calico/bin/calicoctl - url: "{{calico_download_url}}" + sha256: "{{ calico_checksum }}" + url: "{{ calico_download_url }}" - name: calico-plugin dest: calico/bin/calico - url: "{{calico_plugin_download_url}}" + sha256: "{{ calico_plugin_checksum }}" + url: "{{ calico_plugin_download_url }}" + + - name: etcd + dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz" + sha256: "{{ etcd_checksum }}" + url: "{{ etcd_download_url }}" + unarchive: true - name: kubernetes-kubelet dest: kubernetes/bin/kubelet diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml new file mode 100644 index 000000000..b36784bb2 --- /dev/null +++ b/roles/etcd/defaults/main.yml @@ -0,0 +1,3 @@ +--- +etcd_version: v2.2.4 +etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/" diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml new file mode 100644 index 000000000..cda751429 --- /dev/null +++ b/roles/etcd/handlers/main.yml @@ -0,0 +1,14 @@ +--- +- name: restart systemd-etcd + command: /bin/true + notify: + - reload systemd + - restart etcd + +- name: reload systemd + command: systemctl daemon-reload + +- name: restart etcd + service: + name: etcd + state: restarted diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml new file mode 100644 index 000000000..6f11a743b --- /dev/null +++ b/roles/etcd/tasks/configure.yml @@ -0,0 +1,27 @@ +--- +- name: Copy etcd.service systemd file + template: + src: etcd.service.j2 + dest: /lib/systemd/system/etcd.service + backup: yes + when: init_system == "systemd" + notify: + - restart systemd-etcd + +- name: Write calico-node initd script + template: + src: deb-etcd.initd.j2 + dest: /etc/init.d/etcd + owner: root + mode: 755 + when: init_system == "sysvinit" and ansible_os_family == "Debian" + notify: restart etcd + +- name: Create etcd environment vars file + template: + src: etcd.j2 + dest: /etc/etcd.env + notify: restart etcd + +- name: Ensure etcd is running + service: name=etcd state=started enabled=yes diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml new file mode 100644 index 000000000..5b83f815a --- /dev/null +++ b/roles/etcd/tasks/install.yml @@ -0,0 +1,14 @@ +--- +- name: Create etcd user + user: name=etcd shell=/bin/nologin home=/var/lib/etcd + +- name: Install etcd binaries + copy: + src={{ etcd_bin_dir }}/{{ item }} + dest={{ bin_dir }} + owner=etcd + mode=0755 + with_items: + - etcdctl + - etcd + notify: restart etcd diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 7a9fd3fcd..fdb17cf51 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,12 +1,3 @@ --- -- name: Stop etcd2 service - service: name=etcd state=stopped - ignore_errors: yes - -- name: Create etcd pod manifest - template: src=etcd-pod.yml dest=/etc/kubernetes/manifests/etcd-pod.manifest - -- name: Check for etcd2 port (pulling image and running container) - wait_for: - port: 2379 - delay: 5 +- include: install.yml +- include: configure.yml diff --git a/roles/etcd/templates/deb-etcd.initd.j2 b/roles/etcd/templates/deb-etcd.initd.j2 new file mode 100644 index 000000000..2fb4b76b4 --- /dev/null +++ b/roles/etcd/templates/deb-etcd.initd.j2 @@ -0,0 +1,113 @@ +#!/bin/sh +set -a + +### BEGIN INIT INFO +# Provides: etcd +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: etcd distributed k/v store +# Description: +# etcd is a distributed, consistent key-value store for shared configuration and service discovery +### END INIT INFO + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="etcd k/v store" +NAME=etcd +DAEMON={{ bin_dir }}/etcd +{% if inventory_hostname in groups['etcd'] %} +DAEMON_ARGS="" +{% else %} +DAEMON_ARGS="-proxy on" +{% endif %} +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=etcd +STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}" +PID=/var/run/etcd.pid + +# Exit if the binary is not present +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -f /etc/etcd.env ] && . /etc/etcd.env + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +do_status() +{ + status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $? +} + +# Function that starts the daemon/service +# +do_start() +{ + start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON \ + $DAEMON_OPTS \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME + RETVAL="$?" + + sleep 1 + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + if do_stop; then + log_end_msg 0 + else + log_failure_msg "Can't stop etcd" + log_end_msg 1 + fi + ;; + status) + if do_status; then + log_end_msg 0 + else + log_failure_msg "etcd is not running" + log_end_msg 1 + fi + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + if do_stop; then + if do_start; then + log_end_msg 0 + exit 0 + else + rc="$?" + fi + else + rc="$?" + fi + log_failure_msg "Can't restart etcd" + log_end_msg ${rc} + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/roles/etcd/templates/etcd-pod.yml b/roles/etcd/templates/etcd-pod.yml deleted file mode 100644 index 93971bb6c..000000000 --- a/roles/etcd/templates/etcd-pod.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: etcd - namespace: kube-system -spec: - hostNetwork: true - containers: - - name: etcd - image: quay.io/coreos/etcd:v2.2.2 - resources: - limits: - cpu: 100m - memory: 256M - args: -{% if inventory_hostname in groups['etcd'] %} - - --name - - etcd-{{inventory_hostname}}-master - - --advertise-client-urls - - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379" - - --listen-peer-urls - - http://0.0.0.0:2380 - - --initial-advertise-peer-urls - - http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380 - - --data-dir - - /var/etcd/data - - --initial-cluster-state - - new -{% else %} - - --proxy - - 'on' -{% endif %} - - --listen-client-urls - - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379,http://127.0.0.1:2379" - - --initial-cluster - - "{% for host in groups['etcd'] %}etcd-{{host}}-master=http://{{ hostvars[host]['ip'] | default( hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" - - --initial-cluster-token - - etcd-k8s-cluster - ports: - - name: etcd-client - containerPort: 2379 - hostPort: 2379 - - name: etcd-peer - containerPort: 2380 - hostPort: 2380 - volumeMounts: - - name: varetcd - mountPath: /var/etcd - readOnly: false - volumes: - - name: varetcd - hostPath: - path: /containers/pods/etcd-{{inventory_hostname}}/rootfs/var/etcd diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 new file mode 100644 index 000000000..2b9ab0820 --- /dev/null +++ b/roles/etcd/templates/etcd.j2 @@ -0,0 +1,17 @@ +ETCD_DATA_DIR="/var/lib/etcd" +{% if inventory_hostname in groups['etcd'] %} +{% set etcd = {} %} +{% for host in groups['etcd'] %} +{% if inventory_hostname == host %} +{% set _dummy = etcd.update({'name':"etcd"+loop.index|string}) %} +{% endif %} +{% endfor %} +ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379" +ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" +ETCD_INITIAL_CLUSTER_STATE="new" +ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd" +ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" +ETCD_NAME="{{ etcd.name }}" +{% endif %} +ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" +ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379,http://127.0.0.1:2379" diff --git a/roles/etcd/templates/etcd.service.j2 b/roles/etcd/templates/etcd.service.j2 new file mode 100644 index 000000000..876dcaa69 --- /dev/null +++ b/roles/etcd/templates/etcd.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=etcd +Conflicts=etcd.service + +[Service] +User=etcd +EnvironmentFile=/etc/etcd.env +{% if inventory_hostname in groups['etcd'] %} +ExecStart={{ bin_dir }}/etcd +{% else %} +ExecStart={{ bin_dir }}/etcd -proxy on +{% endif %} +Restart=always +RestartSec=10s +LimitNOFILE=40000 + +[Install] +WantedBy=multi-user.target