Merge pull request #132 from Smana/net_plugins_distinct_roles

split network plugins into distinct roles
pull/134/head
Smaine Kahlouch 2016-02-09 16:10:15 +01:00
commit 03dd43e97d
38 changed files with 235 additions and 168 deletions

View File

@ -25,7 +25,7 @@ in order to avoid any issue during deployment you should disable your firewall
### Components
* [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.4
* [etcd](https://github.com/coreos/etcd/releases) v2.2.4
* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.14.0
* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.16.0
* [flanneld](https://github.com/coreos/flannel/releases) v0.5.5
* [docker](https://www.docker.com/) v1.9.1
@ -107,21 +107,20 @@ kube-master
### Playbook
```
---
- hosts: k8s-cluster
roles:
- { role: adduser, tags: adduser }
- { role: download, tags: download }
- { role: kubernetes/preinstall, tags: preinstall }
- { role: etcd, tags: etcd }
- { role: docker, tags: docker }
- { role: kubernetes/node, tags: node }
- { role: etcd, tags: etcd }
- { role: network_plugin, tags: network }
- { role: dnsmasq, tags: dnsmasq }
- { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
- hosts: kube-master
roles:
- { role: kubernetes/master, tags: master }
```
### Run
@ -143,14 +142,14 @@ the server address has to be present on both groups 'kube-master' and 'kube-node
In order to do so, some variables have to be used '**loadbalancer_apiserver**' and '**apiserver_loadbalancer_domain_name**'
### Network Overlay
### Network Plugin
You can choose between 2 network plugins. Only one must be chosen.
* **flannel**: gre/vxlan (layer 2) networking. ([official docs](https://github.com/coreos/flannel))
* **calico**: bgp (layer 3) networking. ([official docs](http://docs.projectcalico.org/en/0.13/))
The choice is defined with the variable '**kube_network_plugin**'
The choice is defined with the variable **kube_network_plugin**
### Check cluster status

View File

@ -9,7 +9,6 @@
- { role: apps/k8s-elasticsearch, tags: 'elasticsearch' }
- { role: apps/k8s-memcached, tags: 'memcached' }
- { role: apps/k8s-redis, tags: 'redis' }
- { role: apps/k8s-mongodb-simple, tags: 'mongodb-simple' }
# Msg Broker
- { role: apps/k8s-rabbitmq, tags: 'rabbitmq' }
@ -28,6 +27,3 @@
# ETCD
- { role: apps/k8s-etcd, tags: 'etcd'}
# Chat Apps
- { role: apps/k8s-rocketchat, tags: 'rocketchat'}

View File

@ -4,11 +4,11 @@
- { role: adduser, tags: adduser }
- { role: download, tags: download }
- { role: kubernetes/preinstall, tags: preinstall }
- { role: etcd, tags: etcd }
- { role: docker, tags: docker }
- { role: kubernetes/node, tags: node }
- { role: etcd, tags: etcd }
- { role: network_plugin, tags: network }
- { role: dnsmasq, tags: dnsmasq }
- { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
- hosts: kube-master
roles:

View File

@ -24,9 +24,6 @@ kube_users:
# Kubernetes cluster name, also will be used as DNS domain
cluster_name: cluster.local
# set this variable to calico if needed. keep it empty if flannel is used
kube_network_plugin: calico
# For some environments, each node has a pubilcally accessible
# address and an address it should bind services to. These are
# really inventory level variables, but described here for consistency.
@ -49,6 +46,9 @@ kube_network_plugin: calico
# but don't know about that address themselves.
# access_ip: 1.1.1.1
# Choose network plugin (calico or flannel)
kube_network_plugin: calico
# Kubernetes internal network for services, unused block of space.
kube_service_addresses: 10.233.0.0/18

View File

@ -41,7 +41,7 @@
action: "{{ docker_package_info.pkg_mgr }}"
args:
pkg: "{{item}}"
state: latest
state: present
with_items: docker_package_info.pkgs
when: docker_package_info.pkgs|length > 0

View File

@ -1,9 +1,10 @@
docker_kernel_min_version: '3.2'
docker_version: 1.9.1-0~{{ ansible_distribution_release|lower }}
docker_package_info:
pkg_mgr: apt
pkgs:
- docker-engine
- docker-engine={{ docker_version }}
docker_repo_key_info:
pkg_key: apt_key

View File

@ -0,0 +1,21 @@
docker_kernel_min_version: '3.2'
docker_version: 1.9.0-0~{{ ansible_distribution_release }}
docker_package_info:
pkg_mgr: apt
pkgs:
- docker-engine={{ docker_version }}
docker_repo_key_info:
pkg_key: apt_key
keyserver: hkp://p80.pool.sks-keyservers.net:80
repo_keys:
- 58118E89F3A912897C070ADBF76221572C52609D
docker_repo_info:
pkg_repo: apt_repository
repos:
- >
deb https://apt.dockerproject.org/repo
{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}
main

View File

@ -4,18 +4,20 @@ local_release_dir: /tmp
# Versions
kube_version: v1.1.4
etcd_version: v2.2.4
calico_version: v0.14.0
calico_plugin_version: v0.7.0
calico_version: v0.16.0
calico_cni_version: v1.0.0
# Download URL's
kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64"
etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl"
calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes"
calico_cni_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico"
calico_cni_ipam_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico-ipam"
# Checksums
calico_checksum: "f251d7a8583233906aa6d059447c1e4fb32bf1369a51fdf96a68d50466d6a69c"
calico_plugin_checksum: "032f582f5eeec6fb26191d2fbcbf8bca4da3b14abb579db7baa7b3504d4dffec"
calico_checksum: "cfbbcad4b3b7d79be9a25bcdc153ec1d139eecd54840914a363b0710eebc5c51"
calico_cni_checksum: "cfbb95d4416cb65845a188f3bd991fff232bd5ce3463b2919d586ab77967aecd"
calico_cni_ipam_checksum: "93ebf8756b26314e1e3f612f1e824418cbb0a8df2942664422e697bcb109fbb2"
etcd_checksum: "6c4e5cdeaaac1a70b8f06b5dd6b82c37ff19993c9bca81248975610e555c4b9b"
kubectl_checksum: "873ba19926d17a3287dc8639ea1434fe3cd0cb4e61d82101ba754922cfc7a633"
kubelet_checksum: "f2d1eae3fa6e304f6cbc9b2621e4b86fc3bcb4e74a15d35f58bf00e45c706e0a"
@ -29,10 +31,17 @@ downloads:
owner: "root"
mode: "0755"
- name: calico-plugin
- name: calico-cni-plugin
dest: calico/bin/calico
sha256: "{{ calico_plugin_checksum }}"
url: "{{ calico_plugin_download_url }}"
sha256: "{{ calico_cni_checksum }}"
url: "{{ calico_cni_download_url }}"
owner: "root"
mode: "0755"
- name: calico-cni-plugin-ipam
dest: calico/bin/calico-ipam
sha256: "{{ calico_cni_ipam_checksum }}"
url: "{{ calico_cni_ipam_download_url }}"
owner: "root"
mode: "0755"

View File

@ -1,4 +1,11 @@
---
- name: tokens | copy the token gen script
copy:
src=kube-gen-token.sh
dest={{ kube_script_dir }}
mode=u+x
when: inventory_hostname == groups['kube-master'][0]
- name: tokens | generate tokens for master components
command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
environment:

View File

@ -69,11 +69,6 @@
shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver
changed_when: false
- name: Restart apiserver
command: "/bin/true"
notify: restart kube-apiserver
when: is_gentoken_calico|default(false)
- meta: flush_handlers
- include: start.yml

View File

@ -9,10 +9,6 @@
- reload systemd
- reload kubelet
- name: set is_gentoken_calico fact
set_fact:
is_gentoken_calico: true
- name: reload kubelet
service:
name: kubelet

View File

@ -1,27 +0,0 @@
---
- name: tokens | copy the token gen script
copy:
src=kube-gen-token.sh
dest={{ kube_script_dir }}
mode=u+x
when: inventory_hostname == groups['kube-master'][0]
- name: tokens | generate tokens for calico
command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
environment:
TOKEN_DIR: "{{ kube_token_dir }}"
with_nested:
- [ "system:calico" ]
- "{{ groups['k8s-cluster'] }}"
register: gentoken_calico
changed_when: "'Added' in gentoken_calico.stdout"
when: kube_network_plugin == "calico"
delegate_to: "{{ groups['kube-master'][0] }}"
notify: set is_gentoken_calico fact
- name: tokens | get the calico token values
slurp:
src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token"
register: calico_token
when: kube_network_plugin == "calico"
delegate_to: "{{ groups['kube-master'][0] }}"

View File

@ -1,32 +1,12 @@
---
- name: Create kubernetes config directory
file:
path: "{{ kube_config_dir }}"
state: directory
- name: Write Calico cni config
template:
src: "cni-calico.conf.j2"
dest: "/etc/cni/net.d/10-calico.conf"
owner: kube
- name: Create kubernetes script directory
file:
path: "{{ kube_script_dir }}"
state: directory
owner: kube
- name: Create kubernetes manifests directory
file:
path: "{{ kube_manifest_dir }}"
state: directory
owner: kube
- name: Create kubernetes logs directory
file:
path: "{{ kube_log_dir }}"
state: directory
owner: kube
when: init_system == "sysvinit"
when: kube_network_plugin == "calico"
- include: secrets.yml
tags:
- secrets
- include: install.yml

View File

@ -16,8 +16,6 @@
- include: gen_certs.yml
when: inventory_hostname == groups['kube-master'][0]
- include: gen_calico_tokens.yml
# Sync certs between nodes
- name: Secrets | create user
user:

View File

@ -0,0 +1,9 @@
{
"name": "calico-k8s-network",
"type": "calico",
"etcd_authority": "127.0.0.1:2379",
"log_level": "info",
"ipam": {
"type": "calico-ipam"
}
}

View File

@ -24,7 +24,7 @@ KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} -
KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
KUBELET_NETWORK_PLUGIN="--network_plugin={{ kube_network_plugin }}"
KUBELET_NETWORK_PLUGIN="--network_plugin=cni --network-plugin-dir=/etc/cni/net.d"
{% endif %}
# Should this cluster be allowed to run privileged docker containers
KUBE_ALLOW_PRIV="--allow_privileged=true"

View File

@ -33,6 +33,41 @@
always_run: True
tags: always
- name: Create kubernetes config directory
file:
path: "{{ kube_config_dir }}"
state: directory
owner: kube
- name: Create kubernetes script directory
file:
path: "{{ kube_script_dir }}"
state: directory
owner: kube
- name: Create kubernetes manifests directory
file:
path: "{{ kube_manifest_dir }}"
state: directory
owner: kube
- name: Create kubernetes logs directory
file:
path: "{{ kube_log_dir }}"
state: directory
owner: kube
when: init_system == "sysvinit"
- name: Create cni directories
file:
path: "{{ item }}"
state: directory
owner: kube
with_items:
- "/etc/cni/net.d"
- "/opt/cni/bin"
when: kube_network_plugin == "calico"
- name: Update package management cache (APT)
apt: update_cache=yes
when: ansible_pkg_mgr == 'apt'

View File

@ -0,0 +1,2 @@
---
# cloud_provider: no

View File

@ -0,0 +1,15 @@
---
- name: restart calico-node
command: /bin/true
notify:
- reload systemd
- reload calico-node
- name : reload systemd
shell: systemctl daemon-reload
when: init_system == "systemd"
- name: reload calico-node
service:
name: calico-node
state: restarted

View File

@ -1,9 +1,36 @@
---
- name: Calico | Set docker daemon options
template:
src: docker
dest: "/etc/default/docker"
owner: root
group: root
mode: 0644
notify:
- restart docker
- name: Calico | Write docker.service systemd file
template:
src: systemd-docker.service
dest: /lib/systemd/system/docker.service
notify: restart docker
when: init_system == "systemd"
- meta: flush_handlers
- name: Calico | Install calicoctl bin
command: rsync -piu "{{ local_release_dir }}/calico/bin/calicoctl" "{{ bin_dir }}/calicoctl"
register: calico_copy
changed_when: false
- name: Calico | Install calico cni bin
command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico"
changed_when: false
- name: Calico | Install calico-ipam cni bin
command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico-ipam"
changed_when: false
- name: Calico | install calicoctl
file: path={{ bin_dir }}/calicoctl mode=0755 state=file
@ -51,33 +78,32 @@
( not calico_pools.json['node']['nodes'][0]['key'] | search(".*{{ kube_pods_subnet | ipaddr('network') }}.*") )
run_once: true
- name: Calico | Write calico-node configuration
template: src=calico/calico.conf.j2 dest=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico_kubernetes.ini
notify: restart calico-node
- name: Calico | Write /etc/network-environment
template: src=calico/network-environment.j2 dest=/etc/network-environment
template: src=network-environment.j2 dest=/etc/network-environment
when: init_system == "sysvinit"
- name: Calico | Write calico-node systemd init file
template: src=calico/calico-node.service.j2 dest=/etc/systemd/system/calico-node.service
template: src=calico-node.service.j2 dest=/etc/systemd/system/calico-node.service
when: init_system == "systemd"
notify: restart calico-node
- name: Calico | Write calico-node initd script
template: src=calico/deb-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755
template: src=deb-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755
when: init_system == "sysvinit" and ansible_os_family == "Debian"
notify: restart calico-node
- name: Calico | Write calico-node initd script
template: src=calico/rh-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755
template: src=rh-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755
when: init_system == "sysvinit" and ansible_os_family == "RedHat"
notify: restart calico-node
- meta: flush_handlers
- name: Calico | Enable calico-node
service: name=calico-node enabled=yes state=started
service:
name: calico-node
state: started
enabled: yes
- name: Calico | Restart calico if binary changed
service:

View File

@ -8,9 +8,9 @@ After=docker.service etcd.service
User=root
PermissionsStartOnly=true
{% if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%}
ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false
ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false
{% else %}
ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false
ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false
{% endif %}
Restart=always
Restart=10

View File

@ -0,0 +1,9 @@
# This host's IPv4 address (the source IP address used to reach other nodes
# in the Kubernetes cluster).
DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }}
# The Kubernetes master IP
KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}
# IP and port of etcd instance used by Calico
ETCD_AUTHORITY=127.0.0.1:2379

View File

@ -1,5 +1,4 @@
---
# Flannel public IP
# The address that flannel should advertise as how to access the system
flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"
@ -7,5 +6,3 @@ flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address
## interface that should be used for flannel operations
## This is actually an inventory node-level item
# flannel_interface:
# cloud_provider: no

View File

@ -1,10 +1,4 @@
---
- name: restart calico-node
command: /bin/true
notify:
- reload systemd
- reload calico-node
- name: restart docker
command: /bin/true
notify:
@ -21,11 +15,6 @@
shell: systemctl daemon-reload
when: init_system == "systemd"
- name: reload calico-node
service:
name: calico-node
state: restarted
- name: reload docker
service:
name: docker

View File

@ -1,13 +1,13 @@
---
- name: Flannel | Write flannel configuration
template:
src: flannel/network.json
src: network.json
dest: /etc/flannel-network.json
backup: yes
- name: Flannel | Create flannel pod manifest
template:
src: flannel/flannel-pod.yml
src: flannel-pod.yml
dest: /etc/kubernetes/manifests/flannel-pod.manifest
notify: delete default docker bridge
@ -16,7 +16,7 @@
path: /run/flannel/subnet.env
delay: 5
- name: Get flannel_subnet from subnet.env
- name: Flannel | Get flannel_subnet from subnet.env
shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_SUBNET" {print $2}'
register: flannel_subnet_output
changed_when: false
@ -24,10 +24,29 @@
- set_fact:
flannel_subnet: "{{ flannel_subnet_output.stdout }}"
- name: Get flannel_mtu from subnet.env
- name: Flannel | Get flannel_mtu from subnet.env
shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_MTU" {print $2}'
register: flannel_mtu_output
changed_when: false
- set_fact:
flannel_mtu: "{{ flannel_mtu_output.stdout }}"
- name: Flannel | Set docker daemon options
template:
src: docker
dest: "/etc/default/docker"
owner: root
group: root
mode: 0644
notify:
- restart docker
- name: Flannel | Write docker.service systemd file
template:
src: systemd-docker.service
dest: /lib/systemd/system/docker.service
notify: restart docker
when: init_system == "systemd"
- meta: flush_handlers

View File

@ -0,0 +1,6 @@
# Deployed by Ansible
{% if init_system == "sysvinit" and kube_network_plugin == "flannel" and ansible_os_family == "Debian" %}
DOCKER_OPTS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"
{% elif kube_network_plugin == "flannel" %}
OPTIONS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"
{% endif %}

View File

@ -0,0 +1,28 @@
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
{% if ansible_os_family == "RedHat" %}
After=network.target
Wants=docker-storage-setup.service
{% elif ansible_os_family == "Debian" %}
After=network.target docker.socket
Requires=docker.socket
{% endif %}
[Service]
Type=notify
EnvironmentFile=-/etc/default/docker
Environment=GOTRACEBACK=crash
ExecStart=/usr/bin/docker daemon \
$OPTIONS \
$DOCKER_STORAGE_OPTIONS \
$DOCKER_NETWORK_OPTIONS \
$INSECURE_REGISTRY
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
MountFlags=slave
TimeoutStartSec=1min
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,6 @@
---
dependencies:
- role: network_plugin/calico
when: kube_network_plugin == 'calico'
- role: network_plugin/flannel
when: kube_network_plugin == 'flannel'

View File

@ -1,30 +0,0 @@
---
- name: "Test if network plugin is defined"
fail: msg="ERROR, One network_plugin variable must be defined (Flannel or Calico)"
when: ( kube_network_plugin is defined and kube_network_plugin == "calico" and kube_network_plugin == "flannel" ) or
kube_network_plugin is not defined
- include: flannel.yml
when: kube_network_plugin == "flannel"
- name: Set docker daemon options
template:
src: docker
dest: "{{ '/etc/sysconfig/docker-network' if ansible_os_family == 'RedHat' else '/etc/default/docker' }}"
owner: root
group: root
mode: 0644
notify:
- restart docker
- name: Write docker.service systemd file
template:
src: systemd-docker.service
dest: /lib/systemd/system/docker.service
notify: restart docker
when: init_system == "systemd"
- meta: flush_handlers
- include: calico.yml
when: kube_network_plugin == "calico"

View File

@ -1,17 +0,0 @@
[config]
CALICO_IPAM=true
# Location of etcd cluster used by Calico. By default, this uses the etcd
# instance running on the Kubernetes Master
ETCD_AUTHORITY=127.0.0.1:2379
# The kubernetes-apiserver location - used by the calico plugin
{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/
{% else %}
KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{kube_apiserver_port}}/api/v1/
{% endif %}
# Kubernetes authentication token
{% if calico_token is defined | default('') %}
KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }}
{% endif %}

View File

@ -1,2 +0,0 @@
DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }}
ETCD_AUTHORITY=127.0.0.1:2379