diff --git a/README.md b/README.md index d1dd231e4..d6bdda450 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ kubernetes-ansible Install and configure a kubernetes cluster including network plugin. ### Requirements -Tested on **Debian Jessie** and **Ubuntu** (14.10, 15.04, 15.10). +Tested on **Debian Wheezy/Jessie** and **Ubuntu** (14.10, 15.04, 15.10). +Should work on RedHat/Fedora/Centos plateforms (to be tested) * The target servers must have access to the Internet in order to pull docker imaqes. * The firewalls are not managed, you'll need to implement your own rules the way you used to. @@ -54,14 +55,6 @@ You can jump directly to "*Available apps, installation procedure*" Ansible ------------------------- -### Download binaries -A role allows to download required binaries. They will be stored in a directory defined by the variable -**'local_release_dir'** (by default /tmp). -Please ensure that you have enough disk space there (about **300M**). - -**Note**: Whenever you'll need to change the version of a software, you'll have to erase the content of this directory. - - ### Variables The main variables to change are located in the directory ```inventory/group_vars/all.yml```. @@ -117,8 +110,10 @@ kube-master - hosts: k8s-cluster roles: - - { role: etcd, tags: etcd } + - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker } + - { role: kubernetes/node, tags: node } + - { role: etcd, tags: etcd } - { role: dnsmasq, tags: dnsmasq } - { role: network_plugin, tags: ['calico', 'flannel', 'network'] } @@ -126,10 +121,6 @@ kube-master roles: - { role: kubernetes/master, tags: master } -- hosts: kube-node - roles: - - { role: kubernetes/node, tags: node } - ``` ### Run diff --git a/cluster.yml b/cluster.yml index 5d3a5cdb8..3724554aa 100644 --- a/cluster.yml +++ b/cluster.yml @@ -6,6 +6,7 @@ - hosts: k8s-cluster roles: + - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker } - { role: kubernetes/node, tags: node } - { role: etcd, tags: etcd } diff --git a/inventory/inventory.example b/inventory/inventory.example index df5d077cc..ab0d1fca1 100644 --- a/inventory/inventory.example +++ b/inventory/inventory.example @@ -1,5 +1,5 @@ [downloader] -node1 ansible_ssh_host=10.99.0.26 +localhost ansible_connection=local ansible_python_interpreter=python2 [kube-master] node1 ansible_ssh_host=10.99.0.26 diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index 57bb95e97..2dda0f840 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -61,5 +61,10 @@ - name: disable resolv.conf modification by dhclient copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate mode=u+x backup=yes + when: ansible_os_family == "Debian" + +- name: disable resolv.conf modification by dhclient + copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x backup=yes + when: ansible_os_family == "RedHat" - meta: flush_handlers diff --git a/roles/docker/.gitignore b/roles/docker/.gitignore new file mode 100644 index 000000000..e11a4750e --- /dev/null +++ b/roles/docker/.gitignore @@ -0,0 +1,2 @@ +.*.swp +.vagrant diff --git a/roles/docker/files/systemd-docker.service b/roles/docker/files/systemd-docker.service deleted file mode 100644 index 25eb328d5..000000000 --- a/roles/docker/files/systemd-docker.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Docker Application Container Engine -Documentation=https://docs.docker.com -After=network.target docker.socket -Requires=docker.socket - -[Service] -EnvironmentFile=-/etc/default/docker -Type=notify -ExecStart=/usr/bin/docker daemon -H fd:// $DOCKER_OPTS -MountFlags=slave -LimitNOFILE=1048576 -LimitNPROC=1048576 -LimitCORE=infinity - -[Install] -WantedBy=multi-user.target diff --git a/roles/docker/handlers/main.yml b/roles/docker/handlers/main.yml deleted file mode 100644 index 2644b6dfd..000000000 --- a/roles/docker/handlers/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: restart docker - command: /bin/true - notify: - - reload systemd - - restart docker service - -- name: reload systemd - shell: systemctl daemon-reload - -- name: restart docker service - service: name=docker state=restarted diff --git a/roles/docker/tasks/configure.yml b/roles/docker/tasks/configure.yml deleted file mode 100644 index 1e337ce5d..000000000 --- a/roles/docker/tasks/configure.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: enable docker - service: - name: docker - enabled: yes - state: started - tags: - - docker - -#- name: login to arkena's docker registry -# shell : > -# docker login --username={{ dockerhub_user }} -# --password={{ dockerhub_pass }} -# --email={{ dockerhub_email }} - -- meta: flush_handlers diff --git a/roles/docker/tasks/install.yml b/roles/docker/tasks/install.yml deleted file mode 100644 index 473e132fb..000000000 --- a/roles/docker/tasks/install.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: Install prerequisites for https transport - apt: pkg={{ item }} state=present update_cache=yes - with_items: - - apt-transport-https - - ca-certificates - -- name: Configure docker apt repository - template: src=docker.list.j2 dest=/etc/apt/sources.list.d/docker.list backup=yes - -- name: Install docker-engine - apt: pkg={{ item }} state=present force=yes update_cache=yes - with_items: - - aufs-tools - - cgroupfs-mount - - docker-engine=1.9.1-0~{{ ansible_distribution_release }} - -- name: Copy default docker configuration - template: src=default-docker.j2 dest=/etc/default/docker backup=yes - notify: restart docker - -- name: Copy Docker systemd unit file - copy: src=systemd-docker.service dest=/lib/systemd/system/docker.service backup=yes - notify: restart docker diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index fdb17cf51..4f95be0db 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,3 +1,53 @@ --- -- include: install.yml -- include: configure.yml +- name: gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}.yml" + - defaults.yml + paths: + - ../vars + +- name: check for minimum kernel version + fail: + msg: > + docker requires a minimum kernel version of + {{ docker_kernel_min_version }} on + {{ ansible_distribution }}-{{ ansible_distribution_version }} + when: ansible_kernel|version_compare(docker_kernel_min_version, "<") + +- name: ensure docker requirements packages are installed + action: "{{ docker_package_info.pkg_mgr }}" + args: docker_package_info.args + with_items: docker_package_info.pre_pkgs + when: docker_package_info.pre_pkgs|length > 0 + +- name: ensure docker repository public key is installed + action: "{{ docker_repo_key_info.pkg_key }}" + args: docker_repo_key_info.args + with_items: docker_repo_key_info.repo_keys + when: docker_repo_key_info.repo_keys|length > 0 + +- name: ensure docker repository is enabled + action: "{{ docker_repo_info.pkg_repo }}" + args: docker_repo_info.args + with_items: docker_repo_info.repos + when: docker_repo_info.repos|length > 0 + +- name: ensure docker packages are installed + action: "{{ docker_package_info.pkg_mgr }}" + args: docker_package_info.args + with_items: docker_package_info.pkgs + when: docker_package_info.pkgs|length > 0 + +- name: ensure docker service is started and enabled + service: + name: "{{ item }}" + enabled: yes + state: started + with_items: + - docker diff --git a/roles/docker/templates/default-docker.j2 b/roles/docker/templates/default-docker.j2 deleted file mode 100644 index 3da7d3606..000000000 --- a/roles/docker/templates/default-docker.j2 +++ /dev/null @@ -1,13 +0,0 @@ -# Docker Upstart and SysVinit configuration file - -# Customize location of Docker binary (especially for development testing). -#DOCKER="/usr/local/bin/docker" - -# Use DOCKER_OPTS to modify the daemon startup options. -#DOCKER_OPTS="" - -# If you need Docker to use an HTTP proxy, it can also be specified here. -#export http_proxy="http://127.0.0.1:3128/" - -# This is also a handy place to tweak where Docker's temporary files go. -#export TMPDIR="/mnt/bigdrive/docker-tmp" diff --git a/roles/docker/templates/docker.list.j2 b/roles/docker/templates/docker.list.j2 deleted file mode 100644 index 2b2f4da63..000000000 --- a/roles/docker/templates/docker.list.j2 +++ /dev/null @@ -1 +0,0 @@ -deb https://apt.dockerproject.org/repo {{ansible_distribution|lower}}-{{ ansible_distribution_release}} main diff --git a/roles/docker/vars/centos-6.yml b/roles/docker/vars/centos-6.yml new file mode 100644 index 000000000..e6937d595 --- /dev/null +++ b/roles/docker/vars/centos-6.yml @@ -0,0 +1,24 @@ +docker_kernel_min_version: '2.6.32-431' + +docker_package_info: + pkg_mgr: yum + args: + name: "{{ item }}" + state: latest + update_cache: yes + pre_pkgs: + - epel-release + - curl + - device-mapper-libs + pkgs: + - docker-io + +docker_repo_key_info: + pkg_key: '' + args: {} + repo_keys: [] + +docker_repo_info: + pkg_repo: '' + args: {} + repos: [] diff --git a/roles/docker/vars/debian.yml b/roles/docker/vars/debian.yml new file mode 100644 index 000000000..37cf3e021 --- /dev/null +++ b/roles/docker/vars/debian.yml @@ -0,0 +1,36 @@ +docker_kernel_min_version: '3.2' + +docker_package_info: + pkg_mgr: apt + args: + pkg: "{{ item }}" + update_cache: yes + cache_valid_time: 600 + state: latest + pre_pkgs: + - apt-transport-https + - curl + - software-properties-common + pkgs: + - docker-engine + +docker_repo_key_info: + pkg_key: apt_key + args: + id: "{{ item }}" + keyserver: hkp://p80.pool.sks-keyservers.net:80 + state: present + repo_keys: + - 58118E89F3A912897C070ADBF76221572C52609D + +docker_repo_info: + pkg_repo: apt_repository + args: + repo: "{{ item }}" + update_cache: yes + state: present + repos: + - > + deb https://apt.dockerproject.org/repo + {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }} + main diff --git a/roles/docker/vars/fedora-20.yml b/roles/docker/vars/fedora-20.yml new file mode 100644 index 000000000..94e51ce27 --- /dev/null +++ b/roles/docker/vars/fedora-20.yml @@ -0,0 +1,22 @@ +docker_kernel_min_version: '0' + +docker_package_info: + pkg_mgr: yum + args: + name: "{{ item }}" + state: latest + update_cache: yes + pre_pkgs: + - curl + pkgs: + - docker-io + +docker_repo_key_info: + pkg_key: '' + args: {} + repo_keys: [] + +docker_repo_info: + pkg_repo: '' + args: {} + repos: [] diff --git a/roles/docker/vars/main.yml b/roles/docker/vars/main.yml deleted file mode 100644 index cc5ecdb5d..000000000 --- a/roles/docker/vars/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -#dockerhub_user: -#dockerhub_pass: -#dockerhub_email: diff --git a/roles/docker/vars/redhat.yml b/roles/docker/vars/redhat.yml new file mode 100644 index 000000000..6c251ccec --- /dev/null +++ b/roles/docker/vars/redhat.yml @@ -0,0 +1,22 @@ +docker_kernel_min_version: '0' + +docker_package_info: + pkg_mgr: yum + args: + name: "{{ item }}" + state: latest + update_cache: yes + pre_pkgs: + - curl + pkgs: + - docker + +docker_repo_key_info: + pkg_key: '' + args: {} + repo_keys: [] + +docker_repo_info: + pkg_repo: '' + args: {} + repos: [] diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 0a2b3a177..5ba5104fd 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -1,16 +1,42 @@ --- -etcd_version: v2.2.2 -flannel_version: 0.5.5 +local_release_dir: /tmp +flannel_version: 0.5.5 +calico_version: v0.13.0 +calico_plugin_version: v0.7.0 kube_version: v1.1.3 + kubectl_checksum: "01b9bea18061a27b1cf30e34fd8ab45cfc096c9a9d57d0ed21072abb40dd3d1d" kubelet_checksum: "62191c66f2d670dd52ddf1d88ef81048977abf1ffaa95ee6333299447eb6a482" -calico_version: v0.13.0 -calico_plugin_version: v0.7.0 - -etcd_download_url: "https://github.com/coreos/etcd/releases/download" -flannel_download_url: "https://github.com/coreos/flannel/releases/download" kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64" -calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download" + +flannel_download_url: "https://github.com/coreos/flannel/releases/download/v{{ flannel_version }}/flannel-{{ flannel_version }}-linux-amd64.tar.gz" + +calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl" + calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes" + +downloads: + - name: calico + dest: calico/bin/calicoctl + url: "{{calico_download_url}}" + + - name: calico-plugin + dest: calico/bin/calico + url: "{{calico_plugin_download_url}}" + + - name: flannel + dest: flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz + url: "{{flannel_download_url}}" + unarchive: yes + + - name: kubernetes-kubelet + dest: kubernetes/bin/kubelet + sha256: "{{kubelet_checksum}}" + url: "{{ kube_download_url }}/kubelet" + + - name: kubernetes-kubectl + dest: kubernetes/bin/kubectl + sha256: "{{kubectl_checksum}}" + url: "{{ kube_download_url }}/kubectl" diff --git a/roles/download/tasks/calico.yml b/roles/download/tasks/calico.yml deleted file mode 100644 index 9fa82cac1..000000000 --- a/roles/download/tasks/calico.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: Create calico release directory - local_action: file - path={{ local_release_dir }}/calico/bin - recurse=yes - state=directory - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Check if calicoctl has been downloaded - local_action: stat - path={{ local_release_dir }}/calico/bin/calicoctl - register: c_tar - delegate_to: "{{ groups['kube-master'][0] }}" - -# issues with get_url module and redirects, to be tested again in the near future -- name: Download calico - local_action: shell - curl -o {{ local_release_dir }}/calico/bin/calicoctl -Ls {{ calico_download_url }}/{{ calico_version }}/calicoctl - when: not c_tar.stat.exists - register: dl_calico - delegate_to: "{{ groups['kube-master'][0] }}" - - -- name: Download calico-kubernetes-plugin - local_action: get_url - url="{{calico_plugin_download_url}}" - dest="{{ local_release_dir }}/calico/bin/calico" diff --git a/roles/download/tasks/flannel.yml b/roles/download/tasks/flannel.yml deleted file mode 100644 index 2de0ae547..000000000 --- a/roles/download/tasks/flannel.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Create flannel release directory - local_action: file - path={{ local_release_dir }}/flannel - recurse=yes - state=directory - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Check if flannel release archive has been downloaded - local_action: stat - path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz - register: f_tar - delegate_to: "{{ groups['kube-master'][0] }}" - -# issues with get_url module and redirects, to be tested again in the near future -- name: Download flannel - local_action: shell - curl -o {{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz -Ls {{ flannel_download_url }}/v{{ flannel_version }}/flannel-{{ flannel_version }}-linux-amd64.tar.gz - when: not f_tar.stat.exists - register: dl_flannel - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Extract flannel archive - local_action: unarchive - src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz - dest={{ local_release_dir }}/flannel copy=no - when: dl_flannel|changed - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Pick up only flannel binaries - local_action: copy - src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}/flanneld - dest={{ local_release_dir }}/flannel/bin - when: dl_flannel|changed - -- name: Delete unused flannel files - local_action: file - path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }} state=absent - when: dl_flannel|changed diff --git a/roles/download/tasks/kubernetes.yml b/roles/download/tasks/kubernetes.yml deleted file mode 100644 index 0985a17d3..000000000 --- a/roles/download/tasks/kubernetes.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Create kubernetes binary directory - local_action: file - path="{{ local_release_dir }}/kubernetes/bin" - state=directory - recurse=yes - -- name: Download kubelet and kubectl - local_action: get_url - url="{{ kube_download_url }}/{{ item.name }}" - dest="{{ local_release_dir }}/kubernetes/bin" - sha256sum="{{ item.checksum }}" - with_items: - - name: kubelet - checksum: "{{ kubelet_checksum }}" - - name: kubectl - checksum: "{{ kubectl_checksum }}" diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index ded856ba0..0d881acfc 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -1,4 +1,19 @@ --- -- include: kubernetes.yml -- include: calico.yml -- include: flannel.yml +- name: Create dest directories + file: path={{local_release_dir}}/{{item.dest|dirname}} state=directory recurse=yes + with_items: downloads + +- name: Download items + get_url: + url: "{{item.url}}" + dest: "{{local_release_dir}}/{{item.dest}}" + sha256sum: "{{item.sha256 | default(omit)}}" + with_items: downloads + +- name: Extract archives + unarchive: + src: "{{ local_release_dir }}/{{item.dest}}" + dest: "{{ local_release_dir }}/{{item.dest|dirname}}" + copy: no + when: "{{item.unarchive is defined and item.unarchive == True}}" + with_items: downloads diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml index 8b00d1689..22af48901 100644 --- a/roles/kubernetes/master/handlers/main.yml +++ b/roles/kubernetes/master/handlers/main.yml @@ -2,24 +2,13 @@ - name: reload systemd command: systemctl daemon-reload -- name: restart kubelet +- name: restart systemd-kubelet command: /bin/true notify: - reload systemd - - restart reloaded-kubelet + - restart kubelet -- name: restart reloaded-kubelet +- name: restart kubelet service: name: kubelet state: restarted - -- name: restart proxy - command: /bin/true - notify: - - reload systemd - - restart reloaded-proxy - -- name: restart reloaded-proxy - service: - name: kube-proxy - state: restarted diff --git a/roles/kubernetes/node/handlers/main.yml b/roles/kubernetes/node/handlers/main.yml index 162c4cde1..22af48901 100644 --- a/roles/kubernetes/node/handlers/main.yml +++ b/roles/kubernetes/node/handlers/main.yml @@ -1,20 +1,14 @@ --- -- name: restart daemons - command: /bin/true - notify: - - reload systemd - - restart reloaded-kubelet - - name: reload systemd command: systemctl daemon-reload -- name: restart kubelet +- name: restart systemd-kubelet command: /bin/true notify: - reload systemd - - restart reloaded-kubelet + - restart kubelet -- name: restart reloaded-kubelet +- name: restart kubelet service: name: kubelet state: restarted diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 190b2d1c7..815a4bc46 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -1,6 +1,17 @@ --- - name: Write kubelet systemd init file template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes + when: init_system == "systemd" + notify: restart systemd-kubelet + +- name: Write kubelet initd script + template: src=deb-kubelet.initd.j2 dest=/etc/init.d/kubelet owner=root mode=755 backup=yes + when: init_system == "sysvinit" and ansible_os_family == "Debian" + notify: restart kubelet + +- name: Write kubelet initd script + template: src=rh-kubelet.initd.j2 dest=/etc/init.d/kubelet owner=root mode=755 backup=yes + when: init_system == "sysvinit" and ansible_os_family == "RedHat" notify: restart kubelet - name: Install kubelet binary @@ -22,5 +33,4 @@ dest=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico mode=0755 when: kube_network_plugin == "calico" - notify: - - restart kubelet \ No newline at end of file + notify: restart kubelet diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index a01314f0c..5e54c51be 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -28,7 +28,7 @@ - secrets - name: Write kubelet config file - template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet.conf backup=yes + template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet backup=yes notify: - restart kubelet @@ -42,9 +42,6 @@ src: manifests/kube-proxy.manifest.j2 dest: "{{ kube_manifest_dir }}/kube-proxy.manifest" -- name: Write network-environment - template: src=network-environment.j2 dest=/etc/network-environment mode=640 - - name: Enable kubelet service: name: kubelet diff --git a/roles/kubernetes/node/templates/deb-kubelet.initd.j2 b/roles/kubernetes/node/templates/deb-kubelet.initd.j2 new file mode 100644 index 000000000..4f823ca76 --- /dev/null +++ b/roles/kubernetes/node/templates/deb-kubelet.initd.j2 @@ -0,0 +1,119 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kubelet +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: The Kubernetes node container manager +# Description: +# The Kubernetes container manager maintains docker state against a state file. +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="The Kubernetes container manager" +NAME=kubelet +DAEMON={{ bin_dir }}/kubelet +DAEMON_ARGS="" +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=root + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/kubernetes/$NAME ] && . /etc/kubernetes/$NAME + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --background --no-close \ + --make-pidfile --pidfile $PIDFILE \ + --exec $DAEMON -c $DAEMON_USER --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --background --no-close \ + --make-pidfile --pidfile $PIDFILE \ + --exec $DAEMON -c $DAEMON_USER -- \ + $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index eff4ccc9b..38d5166e7 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -22,3 +22,7 @@ KUBELET_NETWORK_PLUGIN="--network_plugin={{ kube_network_plugin }}" {% endif %} # Should this cluster be allowed to run privileged docker containers KUBE_ALLOW_PRIV="--allow_privileged=true" +{% if init_system == "sysvinit" %} +DAEMON_ARGS="$KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBELET_API_SERVER $KUBELET_ADDRESS \ +$KUBELET_HOSTNAME $KUBELET_REGISTER_NODE $KUBELET_ARGS $KUBELET_ARGS $KUBELET_NETWORK_PLUGIN" +{% endif %} diff --git a/roles/kubernetes/node/templates/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.service.j2 index afbfeffff..c6fb53e4a 100644 --- a/roles/kubernetes/node/templates/kubelet.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.service.j2 @@ -8,8 +8,7 @@ After=docker.service {% endif %} [Service] -EnvironmentFile=/etc/kubernetes/kubelet.conf -EnvironmentFile=/etc/network-environment +EnvironmentFile=/etc/kubernetes/kubelet ExecStart={{ bin_dir }}/kubelet \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ diff --git a/roles/kubernetes/node/templates/rh-kubelet.initd.j2 b/roles/kubernetes/node/templates/rh-kubelet.initd.j2 new file mode 100644 index 000000000..95666afde --- /dev/null +++ b/roles/kubernetes/node/templates/rh-kubelet.initd.j2 @@ -0,0 +1,129 @@ +#!/bin/bash +# +# /etc/rc.d/init.d/kubelet +# +# chkconfig: 2345 95 95 +# description: Daemon for kubelet (kubernetes.io) + +### BEGIN INIT INFO +# Provides: kubelet +# Required-Start: $local_fs $network $syslog cgconfig +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: start and stop kubelet +# Description: +# The Kubernetes container manager maintains docker state against a state file. +### END INIT INFO + +# Source function library. +. /etc/rc.d/init.d/functions + +prog="kubelet" +exec="{{ bin_dir }}/$prog" +pidfile="/var/run/$prog.pid" +lockfile="/var/lock/subsys/$prog" +logfile="/var/log/$prog" + +[ -e /etc/kubernetes/$prog ] && . /etc/kubernetes/$prog + +start() { + if [ ! -x $exec ]; then + if [ ! -e $exec ]; then + echo "Docker executable $exec not found" + else + echo "You do not have permission to execute the Docker executable $exec" + fi + exit 5 + fi + + check_for_cleanup + + if ! [ -f $pidfile ]; then + printf "Starting $prog:\t" + echo "\n$(date)\n" >> $logfile + $exec $DAEMON_ARGS &>> $logfile & + pid=$! + echo $pid >> $pidfile + touch $lockfile + success + echo + else + failure + echo + printf "$pidfile still exists...\n" + exit 7 + fi +} + +stop() { + echo -n $"Stopping $prog: " + killproc -p $pidfile -d 300 $prog + retval=$? + echo + [ $retval -eq 0 ] && rm -f $lockfile + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + status -p $pidfile $prog +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + + +check_for_cleanup() { + if [ -f ${pidfile} ]; then + /bin/ps -fp $(cat ${pidfile}) > /dev/null || rm ${pidfile} + fi +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac + +exit $? diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml new file mode 100644 index 000000000..4b2e6e36f --- /dev/null +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -0,0 +1,15 @@ +--- +common_required_pkgs: + - python-httplib2 + - openssl + - curl + +debian_required_pkgs: + - python-apt + - python-pip + +rh_required_pkgs: + - libselinux-python + +pypy_version: 2.4.0 +python_pypy_url: "https://bitbucket.org/pypy/pypy/downloads/pypy-{{ pypy_version }}.tar.bz2" diff --git a/roles/kubernetes/preinstall/files/bootstrap.sh b/roles/kubernetes/preinstall/files/bootstrap.sh new file mode 100644 index 000000000..2b04a51a3 --- /dev/null +++ b/roles/kubernetes/preinstall/files/bootstrap.sh @@ -0,0 +1,29 @@ +#/bin/bash +set -e + +BINDIR="/usr/local/bin" + +cd $BINDIR + +if [[ -e $BINDIR/.bootstrapped ]]; then + exit 0 +fi + +PYPY_VERSION=2.4.0 + +wget -O - https://bitbucket.org/pypy/pypy/downloads/pypy-$PYPY_VERSION-linux64.tar.bz2 |tar -xjf - +mv -n pypy-$PYPY_VERSION-linux64 pypy + +## library fixup +mkdir -p pypy/lib +ln -snf /lib64/libncurses.so.5.9 $BINDIR/pypy/lib/libtinfo.so.5 + +cat > $BINDIR/python < + if $(pgrep systemd > /dev/null); then + echo systemd; + else + echo sysvinit; + fi + always_run: True + register: init_system_output + changed_when: False + +- set_fact: + init_system: "{{ init_system_output.stdout }}" + +- name: Install packages requirements + action: + module: "{{ ansible_pkg_mgr }}" + name: "{{ item }}" + state: latest + with_items: common_required_pkgs + +- name: Install debian packages requirements + apt: + name: "{{ item }}" + state: latest + when: ansible_os_family == "Debian" + with_items: debian_required_pkgs + +- name: Install redhat packages requirements + action: + module: "{{ ansible_pkg_mgr }}" + name: "{{ item }}" + state: latest + when: ansible_os_family == "RedHat" + with_items: rh_required_pkgs + +- include: python-bootstrap.yml + when: ansible_os_family not in [ "Debian", "RedHat" ] diff --git a/roles/kubernetes/preinstall/tasks/python-bootstrap.yml b/roles/kubernetes/preinstall/tasks/python-bootstrap.yml new file mode 100644 index 000000000..409944317 --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/python-bootstrap.yml @@ -0,0 +1,41 @@ +--- +- name: Python | Check if bootstrap is needed + raw: stat {{ bin_dir}}/.bootstrapped + register: need_bootstrap + ignore_errors: True + +- name: Python | Run bootstrap.sh + script: bootstrap.sh + when: need_bootstrap | failed + +- set_fact: + ansible_python_interpreter: "{{ bin_dir }}/python" + +- name: Python | Check if we need to install pip + shell: "{{ansible_python_interpreter}} -m pip --version" + register: need_pip + ignore_errors: True + changed_when: false + when: need_bootstrap | failed + +- name: Python | Copy get-pip.py + copy: src=get-pip.py dest=~/get-pip.py + when: need_pip | failed + +- name: Python | Install pip + shell: "{{ansible_python_interpreter}} ~/get-pip.py" + when: need_pip | failed + +- name: Python | Remove get-pip.py + file: path=~/get-pip.py state=absent + when: need_pip | failed + +- name: Python | Install pip launcher + copy: src=runner dest={{ bin_dir }}/pip mode=0755 + when: need_pip | failed + +- name: Install required python modules + pip: + name: "{{ item }}" + with_items: pip_python_modules + diff --git a/roles/network_plugin/handlers/main.yml b/roles/network_plugin/handlers/main.yml index b875863e3..d692e7d79 100644 --- a/roles/network_plugin/handlers/main.yml +++ b/roles/network_plugin/handlers/main.yml @@ -1,6 +1,17 @@ --- +- name : reload systemd + shell: systemctl daemon-reload + +- name: restart systemd-calico-node + command: /bin/true + notify: + - reload systemd + - restart calico-node + - name: restart calico-node - service: name=calico-node state=restarted + service: + name: calico-node + state: restarted - name: restart docker service: name=docker state=restarted @@ -23,6 +34,3 @@ - name: start docker service: name=docker state=started - -- name : reload systemd - shell: systemctl daemon-reload diff --git a/roles/network_plugin/tasks/calico.yml b/roles/network_plugin/tasks/calico.yml index 2cf3e2b2c..95803daf2 100644 --- a/roles/network_plugin/tasks/calico.yml +++ b/roles/network_plugin/tasks/calico.yml @@ -44,30 +44,24 @@ run_once: true delegate_to: "{{ groups['etcd'][0] }}" +- name: Calico | Write calico-node configuration + template: src=calico/calico.conf.j2 dest=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico_kubernetes.ini + notify: restart calico-node + - name: Calico | Write calico-node systemd init file template: src=calico/calico-node.service.j2 dest=/etc/systemd/system/calico-node.service - register: newservice - notify: - - reload systemd - - restart calico-node + when: init_system == "systemd" + notify: restart calico-node -- name: Calico | daemon-reload - command: systemctl daemon-reload - when: newservice|changed - changed_when: False +- name: Calico | Write calico-node initd script + template: src=calico/deb-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=755 + when: init_system == "sysvinit" and ansible_os_family == "Debian" + notify: restart calico-node + +- name: Calico | Write calico-node initd script + template: src=calico/rh-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=755 + when: init_system == "sysvinit" and ansible_os_family == "RedHat" + notify: restart calico-node - name: Calico | Enable calico-node service: name=calico-node enabled=yes state=started - -- name: Calico | Disable node mesh - shell: calicoctl bgp node-mesh off - environment: - ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" - when: peer_with_router|default(false) and inventory_hostname in groups['kube-node'] - -- name: Calico | Configure peering with router(s) - shell: calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }} - environment: - ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" - with_items: peers - when: peer_with_router|default(false) and inventory_hostname in groups['kube-node'] diff --git a/roles/network_plugin/templates/calico/calico-node.service.j2 b/roles/network_plugin/templates/calico/calico-node.service.j2 index a78ef4ffb..ba594c758 100644 --- a/roles/network_plugin/templates/calico/calico-node.service.j2 +++ b/roles/network_plugin/templates/calico/calico-node.service.j2 @@ -5,7 +5,6 @@ Requires=docker.service After=docker.service etcd2.service [Service] -EnvironmentFile=/etc/network-environment User=root PermissionsStartOnly=true {% if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%} diff --git a/roles/kubernetes/node/templates/network-environment.j2 b/roles/network_plugin/templates/calico/calico.conf.j2 old mode 100755 new mode 100644 similarity index 51% rename from roles/kubernetes/node/templates/network-environment.j2 rename to roles/network_plugin/templates/calico/calico.conf.j2 index 20bd60311..ec17f6e37 --- a/roles/kubernetes/node/templates/network-environment.j2 +++ b/roles/network_plugin/templates/calico/calico.conf.j2 @@ -1,24 +1,10 @@ -#! /usr/bin/bash -{% if kube_network_plugin == "calico" %} -# This node's IPv4 address +[config] CALICO_IPAM=true DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} -# The kubernetes master IP -{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} -KUBERNETES_MASTER=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} -{% else %} -KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }} -{% endif %} - # Location of etcd cluster used by Calico. By default, this uses the etcd # instance running on the Kubernetes Master -ETCD_AUTHORITY="127.0.0.1:2379" -#{% if inventory_hostname in groups['etcd'] %} -#ETCD_AUTHORITY="127.0.0.1:2379" -#{% else %} -#ETCD_AUTHORITY="127.0.0.1:23799" -#{% endif %} +ETCD_AUTHORITY=127.0.0.1:2379 # The kubernetes-apiserver location - used by the calico plugin {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} @@ -26,9 +12,7 @@ KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_a {% else %} KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/ {% endif %} -{% else %} -FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network" -{% endif %} +# Kubernetes authentication token {% if calico_token is defined | default('') %} KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }} {% endif %} diff --git a/roles/network_plugin/templates/calico/deb-calico.initd.j2 b/roles/network_plugin/templates/calico/deb-calico.initd.j2 new file mode 100644 index 000000000..cd66de53f --- /dev/null +++ b/roles/network_plugin/templates/calico/deb-calico.initd.j2 @@ -0,0 +1,114 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: calico-node +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Calico docker container +# Description: +# Runs calico as a docker container +### END INIT INFO + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Calico-node Docker" +NAME=calico-node +DAEMON={{ bin_dir }}/calicoctl +DAEMON_ARGS="" +DOCKER=$(which docker) +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=root + +# Exit if the binary is not present +[ -x "$DAEMON" ] || exit 0 + +# Exit if the docker package is not installed +[ -x "$DOCKER" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/network-environment ] && . /etc/network-environment + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +do_status() +{ + if [ $($DOCKER ps | awk '{ print $2 }' | grep calico/node | wc -l) -eq 1 ]; then + return 0 + else + return 1 + fi +} + +# Function that starts the daemon/service +# +do_start() +{ + do_status + retval=$? + if [ $retval -ne 0 ]; then + ${DAEMON} node --ip=${DEFAULT_IPV4} >>/dev/null && return 0 || return 2 + else + return 1 + fi +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + ${DAEMON} node stop >> /dev/null || ${DAEMON} node stop --force >> /dev/null +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + if do_stop; then + log_end_msg 0 + else + log_failure_msg "Can't stop calico-node" + log_end_msg 1 + fi + ;; + status) + if do_status; then + log_end_msg 0 + else + log_failure_msg "Calico-node is not running" + log_end_msg 1 + fi + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + if do_stop; then + if do_start; then + log_end_msg 0 + exit 0 + else + rc="$?" + fi + else + rc="$?" + fi + log_failure_msg "Can't restart Calico-node" + log_end_msg ${rc} + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/roles/network_plugin/templates/calico/rh-calico.initd.j2 b/roles/network_plugin/templates/calico/rh-calico.initd.j2 new file mode 100644 index 000000000..d4f798f56 --- /dev/null +++ b/roles/network_plugin/templates/calico/rh-calico.initd.j2 @@ -0,0 +1,130 @@ +#!/bin/bash +# +# /etc/rc.d/init.d/calico-node +# +# chkconfig: 2345 95 95 +# description: Daemon for calico-node (http://www.projectcalico.org/) + +### BEGIN INIT INFO +# Provides: calico-node +# Required-Start: $local_fs $network $syslog cgconfig +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: start and stop calico-node +# Description: +# Manage calico-docker container +### END INIT INFO + +# Source function library. +. /etc/rc.d/init.d/functions + +prog="calicoctl" +exec="{{ bin_dir }}/$prog" +dockerexec="$(which docker)" +logfile="/var/log/$prog" + +[ -e /etc/network-environment ] && for i in $(cat /etc/network-environment | egrep '(^$|^#)'); do export $i; done + +do_status() +{ + if [ $($dockerexec ps | awk '{ print $2 }' | grep calico/node | wc -l) -ne 1 ]; then + return 1 + fi +} + +do_start() { + if [ ! -x $exec ]; then + if [ ! -e $exec ]; then + echo "calico-node executable $exec not found" + else + echo "You do not have permission to execute the calico-node executable $exec" + fi + exit 5 + fi + + [ -x "$dockerexec" ] || exit 0 + + do_status + retval=$? + if [ $retval -ne 0 ]; then + printf "Starting $prog:\t" + echo "\n$(date)\n" >> $logfile + $exec node --ip=${DEFAULT_IPV4} &>>$logfile + success + echo + else + echo -n "calico-node's already running" + success + exit 0 + fi +} + +do_stop() { + echo -n $"Stopping $prog: " + $exec node stop >> /dev/null || $exec node stop --force >> /dev/null + retval=$? + echo + return $retval +} + +restart() { + do_stop + do_start +} + +reload() { + restart +} + +force_reload() { + restart +} + +case "$1" in + start) + do_start + case "$?" in + 0|1) success || exit 0 ;; + 2) failure || exit 1 ;; + esac + ;; + stop) + echo -n "Stopping $DESC" "$NAME" + if do_stop; then + success + echo + else + echo -n "Can't stop calico-node" + failure + echo + fi + ;; + restart) + $1 + ;; + reload) + $1 + ;; + force-reload) + force_reload + ;; + status) + if do_status; then + echo -n "Calico-node is running" + success + echo + else + echo -n "Calico-node is not running" + failure + echo + fi + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|reload|force-reload}" + exit 2 +esac + +exit $?