kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added cilium support (#2236) 

* Added cilium support

* Fix typo in debian test config

* Remove empty lines

* Changed cilium version from <latest> to <v1.0.0-rc3>

* Add missing changes for cilium

* Add cilium to CI pipeline

* Fix wrong file name

* Check kernel version for cilium

* fixed ci error

* fixed cilium-ds.j2 template

* added waiting for cilium pods to run

* Fixed missing EOF

* Fixed trailing spaces

* Fixed trailing spaces

* Fixed trailing spaces

* Fixed too many blank lines

* Updated tolerations,annotations in cilium DS template

* Set cilium_version to iptables-1.9 to see if bug is fixed in CI

* Update cilium image tag to v1.0.0-rc4

* Update Cilium test case CI vars filenames

* Add optional prometheus flag, adjust initial readiness delay

* Update README.md with cilium info
pull/2355/merge
melkosoft 2018-02-16 19:37:47 -08:00 committed by Chad Swenson
parent 5c0a41a6e0
commit f13e76d022
26 changed files with 486 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.gitlab-ci.yml
View File

@ -257,6 +257,10 @@ before_script:
# stage: deploy-special
MOVED_TO_GROUP_VARS: "true"
.ubuntu_cilium_sep_variables: &ubuntu_cilium_sep_variables
# stage: deploy-special
MOVED_TO_GROUP_VARS: "true"
.rhel7_weave_variables: &rhel7_weave_variables
# stage: deploy-part1
MOVED_TO_GROUP_VARS: "true"
@ -456,6 +460,17 @@ gce_ubuntu-contiv-sep:
except: ['triggers']
only: ['master', /^pr-.*$/]
gce_ubuntu-cilium-sep:
stage: deploy-special
<<: *job
<<: *gce
variables:
<<: *gce_variables
<<: *ubuntu_cilium_sep_variables
when: manual
except: ['triggers']
only: ['master', /^pr-.*$/]
gce_rhel7-weave:
stage: deploy-part2
<<: *job

5
README.md
View File

@ -80,6 +80,7 @@ Versions of supported components
- [flanneld](https://github.com/coreos/flannel/releases) v0.8.0
- [calico](https://docs.projectcalico.org/v2.5/releases/) v2.5.0
- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
- [cilium](https://github.com/cilium/cilium) v1.0.0-rc4
- [contiv](https://github.com/contiv/install/releases) v1.0.3
- [weave](http://weave.works/) v2.0.1
- [docker](https://www.docker.com/) v1.13 (see note)
@ -107,7 +108,7 @@ Requirements
Network Plugins
---------------
You can choose between 4 network plugins. (default: `calico`, except Vagrant uses `flannel`)
You can choose between 6 network plugins. (default: `calico`, except Vagrant uses `flannel`)
- [flannel](docs/flannel.md): gre/vxlan (layer 2) networking.
@ -115,6 +116,8 @@ You can choose between 4 network plugins. (default: `calico`, except Vagrant use
- [canal](https://github.com/projectcalico/canal): a composition of calico and flannel plugins.
- [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
- [contiv](docs/contiv.md): supports vlan, vxlan, bgp and Cisco SDN networking. This plugin is able to
apply firewall policies, segregate containers in multiple network and bridging pods onto physical networks.

2
inventory/sample/group_vars/k8s-cluster.yml
View File

@ -62,7 +62,7 @@ kube_users:
# kube_oidc_groups_claim: groups
# Choose network plugin (calico, contiv, weave or flannel)
# Choose network plugin (cilium, calico, contiv, weave or flannel)
# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
kube_network_plugin: calico

9
roles/download/defaults/main.yml
View File

@ -41,6 +41,7 @@ vault_version: 0.8.1
weave_version: 2.2.0
pod_infra_version: 3.0
contiv_version: 1.1.7
cilium_version: "v1.0.0-rc4"
# Download URLs
istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux"
@ -88,6 +89,8 @@ contiv_image_repo: "contiv/netplugin"
contiv_image_tag: "{{ contiv_version }}"
contiv_auth_proxy_image_repo: "contiv/auth_proxy"
contiv_auth_proxy_image_tag: "{{ contiv_version }}"
cilium_image_repo: "docker.io/cilium/cilium"
cilium_image_tag: "{{ cilium_version }}"
nginx_image_repo: nginx
nginx_image_tag: 1.13
@ -174,6 +177,12 @@ downloads:
repo: "{{ hyperkube_image_repo }}"
tag: "{{ hyperkube_image_tag }}"
sha256: "{{ hyperkube_digest_checksum|default(None) }}"
cilium:
enabled: "{{ kube_network_plugin == 'cilium' }}"
container: true
repo: "{{ cilium_image_repo }}"
tag: "{{ cilium_image_tag }}"
sha256: "{{ cilium_digest_checksum|default(None) }}"
flannel:
enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
container: true

20
roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml 100755
View File

@ -0,0 +1,20 @@
---
- name: Cilium | Start Resources
kube:
name: "{{item.item.name}}"
namespace: "{{ system_namespace }}"
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: "{{kube_config_dir}}/{{item.item.file}}"
state: "latest"
with_items: "{{ cilium_node_manifests.results }}"
when: inventory_hostname == groups['kube-master'][0] and not item|skipped
- name: Cilium | Wait for pods to run
command: "{{bin_dir}}/kubectl -n {{system_namespace}} get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'"
register: pods_not_ready
until: pods_not_ready.stdout.find("cilium")==-1
retries: 30
delay: 10
ignore_errors: yes
when: inventory_hostname == groups['kube-master'][0]

5
roles/kubernetes-apps/network_plugin/meta/main.yml
View File

@ -1,5 +1,10 @@
---
dependencies:
- role: kubernetes-apps/network_plugin/cilium
when: kube_network_plugin == 'cilium'
tags:
- cilium
- role: kubernetes-apps/network_plugin/calico
when: kube_network_plugin == 'calico'
tags:

2
roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
View File

@ -52,7 +52,7 @@ spec:
{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
- --configure-cloud-routes=true
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal"] %}
{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal", "cilium"] %}
- --allocate-node-cidrs=true
- --cluster-cidr={{ kube_pods_subnet }}
- --service-cluster-ip-range={{ kube_service_addresses }}

2
roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
View File

@ -60,7 +60,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_reserve }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv"] %}
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"

2
roles/kubernetes/node/templates/kubelet.rkt.service.j2
View File

@ -39,7 +39,7 @@ ExecStart=/usr/bin/rkt run \
{% if local_volume_provisioner_enabled == true %}
--volume local-volume-base-dir,kind=host,source={{ local_volume_base_dir }},readOnly=false,recursive=true \
{% endif %}
{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv"] %}
{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"] %}
--volume etc-cni,kind=host,source=/etc/cni,readOnly=true \
--volume opt-cni,kind=host,source=/opt/cni,readOnly=true \
--volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \

2
roles/kubernetes/node/templates/kubelet.standard.env.j2
View File

@ -83,7 +83,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% endif %}
KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kubelet_reserve }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv"] %}
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
{% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}
DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock"

3
roles/kubernetes/preinstall/tasks/main.yml
View File

@ -89,10 +89,11 @@
- "/etc/cni/net.d"
- "/opt/cni/bin"
when:
- kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv"]
- kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"]
- inventory_hostname in groups['k8s-cluster']
tags:
- network
- cilium
- calico
- weave
- canal

6
roles/kubernetes/preinstall/tasks/verify-settings.yml
View File

@ -89,3 +89,9 @@
that: rbac_enabled and kube_api_anonymous_auth
when: kube_apiserver_insecure_port == 0
ignore_errors: "{{ ignore_assert_errors }}"
- name: Stop if kernel version is too low
assert:
that: ansible_kernel.split('-')[0]|version_compare('4.8', '>=')
when: kube_network_plugin == 'cilium'
ignore_errors: "{{ ignore_assert_errors }}"

2
roles/kubespray-defaults/defaults/main.yaml
View File

@ -82,7 +82,7 @@ kube_users:
pass: "{{kube_api_pwd}}"
role: admin
# Choose network plugin (calico, weave or flannel)
# Choose network plugin (cilium, calico, weave or flannel)
# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
kube_network_plugin: calico

25
roles/network_plugin/cilium/defaults/main.yml 100755
View File

@ -0,0 +1,25 @@
---
# Log-level
cilium_debug: false
cilium_disable_ipv4: false
# Etcd SSL dirs
cilium_cert_dir: /etc/cilium/certs
etcd_cert_dir: /etc/ssl/etcd/ssl
# Cilium Network Policy directory
cilium_policy_dir: /etc/kubernetes/policy
# Limits for apps
cilium_memory_limit: 500M
cilium_cpu_limit: 200m
cilium_memory_requests: 64M
cilium_cpu_requests: 50m
# Optional features
cilium_enable_prometheus: false
rbac_resources:
- sa
- clusterrole
- clusterrolebinding

14
roles/network_plugin/cilium/handlers/main.yml 100644
View File

@ -0,0 +1,14 @@
---
- name: restart kubelet
command: /bin/true
notify:
- Kubelet | reload systemd
- Kubelet | reload kubelet
- name: Kubelet | reload systemd
command: systemctl daemon-reload
- name: Kubelet | reload kubelet
service:
name: kubelet
state: restarted

55
roles/network_plugin/cilium/tasks/main.yml 100755
View File

@ -0,0 +1,55 @@
---
- name: Cilium | Ensure BFPFS mounted
mount:
fstype: bpf
path: /sys/fs/bpf
src: bpffs
state: mounted
- name: Cilium | Create Cilium certs directory
file:
dest: "{{ cilium_cert_dir }}"
state: directory
mode: 0750
owner: root
group: root
- name: Cilium | Link etcd certificates for cilium
file:
src: "{{ etcd_cert_dir }}/{{ item.s }}"
dest: "{{ cilium_cert_dir }}/{{ item.d }}"
state: hard
force: yes
with_items:
- {s: "ca.pem", d: "ca_cert.crt"}
- {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
- {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
- name: Cilium | Create Cilium node manifests
template:
src: "{{item.file}}.j2"
dest: "{{kube_config_dir}}/{{item.file}}"
with_items:
- {name: cilium, file: cilium-config.yml, type: cm}
- {name: cilium, file: cilium-crb.yml, type: clusterrolebinding}
- {name: cilium, file: cilium-cr.yml, type: clusterrole}
- {name: cilium, file: cilium-ds.yml, type: ds}
- {name: cilium, file: cilium-sa.yml, type: sa}
register: cilium_node_manifests
when:
- inventory_hostname in groups['kube-master']
- rbac_enabled or item.type not in rbac_resources
- name: Cilium | Set CNI directory permissions
file:
path: /opt/cni/bin
state: directory
owner: kube
recurse: true
mode: 0755
register: cni_bin_dir
- name: Cilium | Create network policy directory
file:
path: "{{ cilium_policy_dir }}"
state: directory

29
roles/network_plugin/cilium/templates/cilium-config.yml.j2 100755
View File

@ -0,0 +1,29 @@
kind: ConfigMap
apiVersion: v1
metadata:
name: cilium-config
namespace: {{ system_namespace }}
data:
# This etcd-config contains the etcd endpoints of your cluster. If you use
# TLS please make sure you uncomment the ca-file line and add the respective
# certificate has a k8s secret, see explanation bellow in the comment labeled
# "ETCD-CERT"
etcd-config: |-
---
endpoints:
{% for ip_addr in etcd_access_addresses.split(',') %}
- {{ ip_addr }}
{% endfor %}
#
# In case you want to use TLS in etcd, uncomment the following line
# and add the certificate as explained in the comment labeled "ETCD-CERT"
ca-file: "{{ cilium_cert_dir }}/ca_cert.crt"
#
# In case you want client to server authentication, uncomment the following
# lines and add the certificate and key in cilium-etcd-secrets bellow
key-file: "{{ cilium_cert_dir }}/key.pem"
cert-file: "{{ cilium_cert_dir }}/cert.crt"
# If you want to run cilium in debug mode change this value to true
debug: "{{ cilium_debug }}"
disable-ipv4: "{{ cilium_disable_ipv4 }}"

62
roles/network_plugin/cilium/templates/cilium-cr.yml.j2 100755
View File

@ -0,0 +1,62 @@
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: cilium
rules:
- apiGroups:
- "networking.k8s.io"
resources:
- networkpolicies
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
- services
- nodes
- endpoints
- componentstatuses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- pods
- nodes
verbs:
- get
- list
- watch
- update
- apiGroups:
- extensions
resources:
- networkpolicies #FIXME remove this when we drop support for k8s NP-beta GH-1202
- thirdpartyresources
- ingresses
verbs:
- create
- get
- list
- watch
- apiGroups:
- "apiextensions.k8s.io"
resources:
- customresourcedefinitions
verbs:
- create
- get
- list
- watch
- apiGroups:
- cilium.io
resources:
- ciliumnetworkpolicies
verbs:
- "*"

15
roles/network_plugin/cilium/templates/cilium-crb.yml.j2 100755
View File

@ -0,0 +1,15 @@
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: cilium
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cilium
subjects:
- kind: ServiceAccount
name: cilium
namespace: {{ system_namespace }}
- kind: Group
name: system:nodes

163
roles/network_plugin/cilium/templates/cilium-ds.yml.j2 100755
View File

@ -0,0 +1,163 @@
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: cilium
namespace: {{ system_namespace }}
spec:
template:
metadata:
labels:
k8s-app: cilium
kubernetes.io/cluster-service: "true"
annotations:
# This annotation plus the CriticalAddonsOnly toleration makes
# cilium to be a critical pod in the cluster, which ensures cilium
# gets priority scheduling.
# https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: >-
[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]
{% if cilium_enable_prometheus %}
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
{% endif %}
spec:
{% if rbac_enabled %}
serviceAccountName: cilium
{% endif %}
containers:
- image: {{ cilium_image_repo }}:{{ cilium_image_tag }}
imagePullPolicy: Always
name: cilium-agent
command: [ "cilium-agent" ]
args:
- "--debug=$(CILIUM_DEBUG)"
- "-t"
- "vxlan"
- "--kvstore"
- "etcd"
- "--kvstore-opt"
- "etcd.config=/var/lib/etcd-config/etcd.config"
- "--disable-ipv4=$(DISABLE_IPV4)"
{% if cilium_enable_prometheus %}
ports:
- name: prometheus
containerPort: 9090
{% endif %}
lifecycle:
postStart:
exec:
command:
- "/cni-install.sh"
preStop:
exec:
command:
- "/cni-uninstall.sh"
env:
- name: "K8S_NODE_NAME"
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: "CILIUM_DEBUG"
valueFrom:
configMapKeyRef:
name: cilium-config
key: debug
- name: "DISABLE_IPV4"
valueFrom:
configMapKeyRef:
name: cilium-config
key: disable-ipv4
{% if cilium_enable_prometheus %}
# Note: this variable is a no-op if not defined, and is used in the
# prometheus examples.
- name: "CILIUM_PROMETHEUS_SERVE_ADDR"
valueFrom:
configMapKeyRef:
name: cilium-metrics-config
optional: true
key: prometheus-serve-addr
{% endif %}
livenessProbe:
exec:
command:
- cilium
- status
# The initial delay for the liveness probe is intentionally large to
# avoid an endless kill & restart cycle if in the event that the initial
# bootstrapping takes longer than expected.
initialDelaySeconds: 120
failureThreshold: 10
periodSeconds: 10
readinessProbe:
exec:
command:
- cilium
- status
initialDelaySeconds: 5
periodSeconds: 5
volumeMounts:
- name: bpf-maps
mountPath: /sys/fs/bpf
- name: cilium-run
mountPath: /var/run/cilium
- name: cni-path
mountPath: /host/opt/cni/bin
- name: etc-cni-netd
mountPath: /host/etc/cni/net.d
- name: docker-socket
mountPath: /var/run/docker.sock
readOnly: true
- name: etcd-config-path
mountPath: /var/lib/etcd-config
readOnly: true
- name: cilium-certs
mountPath: {{ cilium_cert_dir }}
readOnly: true
securityContext:
capabilities:
add:
- "NET_ADMIN"
privileged: true
hostNetwork: true
volumes:
# To keep state between restarts / upgrades
- name: cilium-run
hostPath:
path: /var/run/cilium
# To keep state between restarts / upgrades
- name: bpf-maps
hostPath:
path: /sys/fs/bpf
# To read docker events from the node
- name: docker-socket
hostPath:
path: /var/run/docker.sock
# To install cilium cni plugin in the host
- name: cni-path
hostPath:
path: /opt/cni/bin
# To install cilium cni configuration in the host
- name: etc-cni-netd
hostPath:
path: /etc/cni/net.d
- name: cilium-certs
hostPath:
path: {{ cilium_cert_dir }}
# To read the etcd config stored in config maps
- name: etcd-config-path
configMap:
name: cilium-config
items:
- key: etcd-config
path: etcd.config
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
# Mark cilium's pod as critical for rescheduling
- key: CriticalAddonsOnly
operator: "Exists"

6
roles/network_plugin/cilium/templates/cilium-sa.yml.j2 100755
View File

@ -0,0 +1,6 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cilium
namespace: {{ system_namespace }}

5
roles/network_plugin/meta/main.yml
View File

@ -1,5 +1,10 @@
---
dependencies:
- role: network_plugin/cilium
when: kube_network_plugin == 'cilium'
tags:
- cilium
- role: network_plugin/calico
when: kube_network_plugin == 'calico'
tags:

11
tests/files/gce_centos7-cilium.yml 100644
View File

@ -0,0 +1,11 @@
# Instance settings
cloud_image_family: centos-7
cloud_region: us-central1-c
cloud_machine_type: "n1-standard-1"
mode: default
# Deployment settings
kube_network_plugin: cilium
deploy_netchecker: true
kubedns_min_replicas: 1
cloud_provider: gce

13
tests/files/gce_coreos-cilium.yml 100644
View File

@ -0,0 +1,13 @@
# Instance settings
cloud_image_family: coreos-stable
cloud_region: us-central1-c
mode: default
startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd'
# Deployment settings
kube_network_plugin: cilium
bootstrap_os: coreos
resolvconf_mode: host_resolvconf # this is required as long as the coreos stable channel uses docker < 1.12
deploy_netchecker: true
kubedns_min_replicas: 1
cloud_provider: gce

10
tests/files/gce_rhel7-cilium.yml 100644
View File

@ -0,0 +1,10 @@
# Instance settings
cloud_image_family: rhel-7
cloud_region: us-central1-b
mode: default
# Deployment settings
kube_network_plugin: cilium
deploy_netchecker: true
kubedns_min_replicas: 1
cloud_provider: gce

11
tests/files/gce_ubuntu-cilium-sep.yml 100644
View File

@ -0,0 +1,11 @@
# Instance settings
cloud_image_family: ubuntu-1604-lts
cloud_region: us-central1-b
mode: separate
# Deployment settings
kube_network_plugin: cilium
deploy_netchecker: true
kubedns_min_replicas: 1
cloud_provider: gce