add network plugin: cilium

06.network.yml

@@ -4,5 +4,6 @@
   - kube-node
   roles:
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
   - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" }

20.addnode.yml

@@ -5,5 +5,6 @@
   - kube-node
   # 根据hosts中配置，以下两种网络只会安装一种
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
   - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" }

21.addmaster.yml

@@ -14,6 +14,7 @@
   - kube-node
   # 根据hosts中配置，以下两种网络只会安装一种
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
   - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" }
   # 禁止业务 pod调度到 master节点

90.setup.yml

@@ -57,6 +57,7 @@
   - kube-node
   roles:
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
   - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" }
 

99.clean.yml

@@ -85,6 +85,9 @@
     - "/etc/calico/"
     - "/var/run/calico/"
     - "/var/log/calico/"
+    - "/etc/cilium/"
+    - "/var/run/cilium/"
+    - "/sys/fs/bpf/tc/"
     - "/var/lib/cni/"
     - "/var/lib/docker/"
     - "/var/lib/kube-router/"

example/hosts.allinone.example

@@ -35,7 +35,7 @@ KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
 BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7"
 
-# 集群网络插件，目前支持calico, flannel, kube-router
+# 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 
 # 服务网段 (Service CIDR），注意不要与内网已有网段冲突

example/hosts.m-masters.example

@@ -49,7 +49,7 @@ KUBE_APISERVER="https://{{ MASTER_IP }}:8443"
 #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
 BOOTSTRAP_TOKEN="c30302226d4b810e08731702d3890f50"
 
-# 集群网络插件，目前支持calico, flannel, kube-router
+# 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 
 # 服务网段 (Service CIDR），注意不要与内网已有网段冲突

example/hosts.s-master.example

@@ -36,7 +36,7 @@ KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
 BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7"
 
-# 集群网络插件，目前支持calico, flannel, kube-router
+# 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 
 # 服务网段 (Service CIDR），注意不要与内网已有网段冲突

roles/cilium/defaults/main.yml

@@ -0,0 +1,18 @@
+# 部分cilium相关配置, Note: cilium 需要Linux kernel >= 4.9.17
+
+# debug mode
+Debug_Mode: "false"
+
+# Removes any Cilium state, e.g. BPF policy maps, before starting
+Clean_Start: "false"
+
+# If true, the policy with the entity 'reserved:host' allows traffic from 'world'
+Legacy_Host_Policy: "true"
+
+# 镜像版本
+busybox_ver: "1.28.4"
+cilium_ver: "v1.1.2"
+
+# 离线镜像tar包
+cilium_offline: "cilium_{{ cilium_ver }}.tar"
+busybox_offline: "busybox_{{ busybox_ver }}.tar"

roles/cilium/tasks/main.yml

@@ -0,0 +1,85 @@
+- name: 创建cilium 相关目录
+  file: name={{ item }} state=directory
+  with_items:
+  - /etc/cilium/ssl
+  - /etc/cni/net.d
+  - /var/run/cilium
+  - /opt/kube/kube-system/cilium
+  - /opt/kube/images 
+
+- name: Optional-Mount BPF FS
+  mount:
+    fstype: "bpf"
+    src: "bpffs"
+    path: "/sys/fs/bpf"
+    state: "mounted" 
+
+- name: 创建cilium 证书请求
+  template: src=cilium-csr.json.j2 dest=/etc/cilium/ssl/cilium-csr.json
+
+- name: 创建 cilium证书和私钥
+  shell: "cd /etc/cilium/ssl && {{ bin_dir }}/cfssl gencert \
+        -ca={{ ca_dir }}/ca.pem \
+        -ca-key={{ ca_dir }}/ca-key.pem \
+        -config={{ ca_dir }}/ca-config.json \
+        -profile=kubernetes cilium-csr.json | {{ bin_dir }}/cfssljson -bare cilium"
+
+- name: 创建 cilium-etcd-secrets
+  shell: "cd /etc/cilium/ssl && \
+	{{ bin_dir }}/kubectl create secret generic -n kube-system cilium-etcd-secrets \
+	--from-file=etcd-ca={{ ca_dir }}/ca.pem \
+	--from-file=etcd-client-key=cilium-key.pem \
+	--from-file=etcd-client-crt=cilium.pem"
+  ignore_errors: true
+  run_once: true
+
+- name: 准备 cilium DaemonSet yaml文件
+  template: src=cilium.yaml.j2 dest=/opt/kube/kube-system/cilium/cilium.yaml
+  tags: reconf  
+
+# 【可选】推送离线docker 镜像，可以忽略执行错误
+- name: 尝试推送离线docker 镜像（若执行失败，可忽略）
+  copy: src={{ base_dir }}/down/{{ item }} dest=/opt/kube/images/{{ item }}
+  with_items:
+  - "pause_3.1.tar"
+  - "{{ cilium_offline }}"
+  - "{{ busybox_offline }}"
+  ignore_errors: true
+
+- name: 获取目录下离线镜像信息
+  command: "ls /opt/kube/images"
+  register: image_info
+
+# 如果目录下有离线镜像，就把它导入到node节点上
+- name: 导入 cilium的离线镜像（若执行失败，可忽略）
+  shell: "{{ bin_dir }}/docker load -i /opt/kube/images/{{ cilium_offline }}"
+  when: 'cilium_offline in image_info.stdout'
+  ignore_errors: true
+
+- name: 导入 busybox的离线镜像（若执行失败，可忽略）
+  shell: "{{ bin_dir }}/docker load -i /opt/kube/images/{{ busybox_offline }}"
+  when: 'busybox_offline in image_info.stdout'
+  ignore_errors: true
+
+- name: 导入 pause的离线镜像（若执行失败，可忽略）
+  shell: "{{ bin_dir }}/docker load -i /opt/kube/images/pause_3.1.tar"
+  when: '"pause_3" in image_info.stdout'
+  ignore_errors: true
+
+# 只需单节点执行一次
+- name: 运行 cilium网络
+  shell: "{{ bin_dir }}/kubectl apply -f /opt/kube/kube-system/cilium/ && sleep 5"
+  run_once: true
+
+# 删除原有cni配置
+- name: 删除默认cni配置
+  file: path=/etc/cni/net.d/10-default.conf state=absent
+
+# 等待网络插件部署成功，视下载镜像速度而定
+- name: 轮询等待cilium-node 运行，视下载镜像速度而定
+  shell: "{{ bin_dir }}/kubectl get pod -n kube-system -o wide|grep 'cilium'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
+  register: pod_status
+  until: pod_status.stdout == "Running"
+  retries: 12
+  delay: 8
+

roles/cilium/templates/cilium-csr.json.j2

@@ -0,0 +1,17 @@
+{
+  "CN": "cilium",
+  "hosts": [],
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "CN",
+      "ST": "HangZhou",
+      "L": "XS",
+      "O": "k8s",
+      "OU": "System"
+    }
+  ]
+}

roles/cilium/templates/cilium.yaml.j2

@@ -0,0 +1,315 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium-config
+  namespace: kube-system
+data:
+  # This etcd-config contains the etcd endpoints of your cluster. If you use
+  # TLS please make sure you follow the tutorial in https://cilium.link/etcd-config
+  etcd-config: |-
+    ---
+    endpoints:
+{% for host in groups['etcd'] %}
+    - https://{{ host }}:2379
+{% endfor %}
+    #
+    # In case you want to use TLS in etcd, uncomment the 'ca-file' line
+    # and create a kubernetes secret by following the tutorial in
+    # https://cilium.link/etcd-config
+    ca-file: '/var/lib/etcd-secrets/etcd-ca'
+    #
+    # In case you want client to server authentication, uncomment the following
+    # lines and create a kubernetes secret by following the tutorial in
+    # https://cilium.link/etcd-config
+    key-file: '/var/lib/etcd-secrets/etcd-client-key'
+    cert-file: '/var/lib/etcd-secrets/etcd-client-crt'
+
+  # If you want to run cilium in debug mode change this value to true
+  debug: "{{ Debug_Mode }}"
+  disable-ipv4: "false"
+  # If you want to clean cilium state; change this value to true
+  clean-cilium-state: "{{ Clean_Start }}"
+  legacy-host-allows-world: "{{ Legacy_Host_Policy }}"
+
+  # Regular expression matching compatible Istio sidecar istio-proxy
+  # container image names
+  sidecar-istio-proxy-image: "cilium/istio_proxy"
+---
+kind: DaemonSet
+apiVersion: apps/v1beta2
+metadata:
+  name: cilium
+  namespace: kube-system
+spec:
+  updateStrategy:
+    type: "RollingUpdate"
+    rollingUpdate:
+      # Specifies the maximum number of Pods that can be unavailable during the update process.
+      maxUnavailable: 2
+  selector:
+    matchLabels:
+      k8s-app: cilium
+      kubernetes.io/cluster-service: "true"
+  template:
+    metadata:
+      labels:
+        k8s-app: cilium
+        kubernetes.io/cluster-service: "true"
+      annotations:
+        # This annotation plus the CriticalAddonsOnly toleration makes
+        # cilium to be a critical pod in the cluster, which ensures cilium
+        # gets priority scheduling.
+        # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: >-
+          [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9090"
+    spec:
+      serviceAccountName: cilium
+      initContainers:
+      - name: clean-cilium-state
+        image: library/busybox:{{ busybox_ver }}
+        imagePullPolicy: IfNotPresent
+        command: ['sh', '-c', 'if [ "${CLEAN_CILIUM_STATE}" = "true" ]; then rm -rf /var/run/cilium/state; rm -rf /sys/fs/bpf/tc/globals/cilium_*; fi']
+        volumeMounts:
+          - name: bpf-maps
+            mountPath: /sys/fs/bpf
+          - name: cilium-run
+            mountPath: /var/run/cilium
+        env:
+          - name: "CLEAN_CILIUM_STATE"
+            valueFrom:
+              configMapKeyRef:
+                name: cilium-config
+                optional: true
+                key: clean-cilium-state
+      containers:
+      - image: cilium/cilium:{{ cilium_ver }}
+        imagePullPolicy: IfNotPresent
+        name: cilium-agent
+        command: [ "cilium-agent" ]
+        args:
+          - "--debug=$(CILIUM_DEBUG)"
+          - "-t=vxlan"
+          - "--kvstore=etcd"
+          - "--kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config"
+          - "--disable-ipv4=$(DISABLE_IPV4)"
+        ports:
+          - name: prometheus
+            containerPort: 9090
+        lifecycle:
+          postStart:
+            exec:
+              command:
+                - "/cni-install.sh"
+          preStop:
+            exec:
+              command:
+                - "/cni-uninstall.sh"
+        env:
+          - name: "K8S_NODE_NAME"
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: "CILIUM_DEBUG"
+            valueFrom:
+              configMapKeyRef:
+                name: cilium-config
+                key: debug
+          - name: "DISABLE_IPV4"
+            valueFrom:
+              configMapKeyRef:
+                name: cilium-config
+                key: disable-ipv4
+          # Note: this variable is a no-op if not defined, and is used in the
+          # prometheus examples.
+          - name: "CILIUM_PROMETHEUS_SERVE_ADDR"
+            valueFrom:
+              configMapKeyRef:
+                name: cilium-metrics-config
+                optional: true
+                key: prometheus-serve-addr
+          - name: "CILIUM_LEGACY_HOST_ALLOWS_WORLD"
+            valueFrom:
+              configMapKeyRef:
+                name: cilium-config
+                optional: true
+                key: legacy-host-allows-world
+          - name: "CILIUM_SIDECAR_ISTIO_PROXY_IMAGE"
+            valueFrom:
+              configMapKeyRef:
+                name: cilium-config
+                key: sidecar-istio-proxy-image
+                optional: true
+        livenessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+          # The initial delay for the liveness probe is intentionally large to
+          # avoid an endless kill & restart cycle if in the event that the initial
+          # bootstrapping takes longer than expected.
+          initialDelaySeconds: 120
+          failureThreshold: 10
+          periodSeconds: 10
+        readinessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        volumeMounts:
+          - name: bpf-maps
+            mountPath: /sys/fs/bpf
+          - name: cilium-run
+            mountPath: /var/run/cilium
+          - name: cni-path
+            mountPath: /host/opt/cni/bin
+          - name: etc-cni-netd
+            mountPath: /host/etc/cni/net.d
+          - name: docker-socket
+            mountPath: /var/run/docker.sock
+            readOnly: true
+          - name: etcd-config-path
+            mountPath: /var/lib/etcd-config
+            readOnly: true
+          - name: etcd-secrets
+            mountPath: /var/lib/etcd-secrets
+            readOnly: true
+        securityContext:
+          capabilities:
+            add:
+              - "NET_ADMIN"
+          privileged: true
+      hostNetwork: true
+      volumes:
+          # To keep state between restarts / upgrades
+        - name: cilium-run
+          hostPath:
+            path: /var/run/cilium
+          # To keep state between restarts / upgrades
+        - name: bpf-maps
+          hostPath:
+            path: /sys/fs/bpf
+          # To read docker events from the node
+        - name: docker-socket
+          hostPath:
+            path: /var/run/docker.sock
+          # To install cilium cni plugin in the host
+        - name: cni-path
+          hostPath:
+            path: {{ bin_dir }}
+          # To install cilium cni configuration in the host
+        - name: etc-cni-netd
+          hostPath:
+              path: /etc/cni/net.d
+          # To read the etcd config stored in config maps
+        - name: etcd-config-path
+          configMap:
+            name: cilium-config
+            items:
+            - key: etcd-config
+              path: etcd.config
+          # To read the k8s etcd secrets in case the user might want to use TLS
+        - name: etcd-secrets
+          secret:
+            secretName: cilium-etcd-secrets
+            optional: true
+      restartPolicy: Always
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      # Mark cilium's pod as critical for rescheduling
+      - key: CriticalAddonsOnly
+        operator: "Exists"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: cilium
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium
+subjects:
+- kind: ServiceAccount
+  name: cilium
+  namespace: kube-system
+- kind: Group
+  name: system:nodes
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: cilium
+rules:
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - services
+  - nodes
+  - endpoints
+  - componentstatuses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - extensions
+  resources:
+  - networkpolicies #FIXME remove this when we drop support for k8s NP-beta GH-1202
+  - thirdpartyresources
+  - ingresses
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "apiextensions.k8s.io"
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumendpoints
+  verbs:
+  - "*"
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: cilium
+  namespace: kube-system
+---

tools/change_ip_aio.yml

@@ -37,5 +37,6 @@
 - hosts: deploy
   roles:
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
   - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" }

tools/change_k8s_network.yml

@@ -96,6 +96,7 @@
   # 安装新的网络插件
   roles:
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
   - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" }
 

tools/init_vars.yml

@@ -7,6 +7,7 @@
     file: name={{ base_dir }}/roles/{{ item }}/vars state=directory
     with_items:
     - calico
+    - cilium
     - cluster-addon
     - cluster-restore
     - cluster-storage
@@ -23,6 +24,7 @@
       force: "yes"
     with_items:
     - calico
+    - cilium
     - cluster-addon
     - cluster-restore
     - cluster-storage