diff --git a/docs/setup/network-plugin/calico-bgp-rr.md b/docs/setup/network-plugin/calico-bgp-rr.md index c95266c..ba71ba8 100644 --- a/docs/setup/network-plugin/calico-bgp-rr.md +++ b/docs/setup/network-plugin/calico-bgp-rr.md @@ -7,11 +7,11 @@ calico-node 版本 v3.3 开始支持内建路由反射器，非常方便，因此使用 calico 作为网络插件可以支持大规模节点数的`K8S`集群。 -本文档主要讲解配置 BGP Route Reflectors，建议首先阅读[基础calico文档](calico.md)。 +- 建议集群节点数大于50时，应用BGP Route Reflectors 特性 ## 前提条件 -实验环境为按照kubeasz安装的2主2从集群，calico 版本 v3.3.2 +k8s 集群使用calico网络插件部署成功。本实验环境为按照kubeasz安装的2主2从集群，calico 版本 v3.19.4。 ``` $ kubectl get node @@ -30,7 +30,7 @@ calico-node-xtspl 2/2 Running 0 179m 1 查看当前集群中BGP连接情况：可以看到集群中4个节点两两建立了 BGP 连接 ``` -$ ansible all -m shell -a '/opt/kube/bin/calicoctl node status' +$ dk ansible -i /etc/kubeasz/clusters/xxx/hosts all -m shell -a '/opt/kube/bin/calicoctl node status' 192.168.1.3 | SUCCESS | rc=0 >> Calico process is running. @@ -91,7 +91,52 @@ IPv4 BGP status IPv6 BGP status No IPv6 peers found. ``` -## 配置全局禁用全连接（BGP full mesh） + +## kubeasz 启用 route reflector + +- 修改`/etc/kubeasz/clusters/xxx/config.yml`文件，设置配置项`CALICO_RR_ENABLED: true` +- 重新执行网络安装 `dk ezctl setup xxx 07` + +## 详解route reflector 安装过程 + +- 选择并配置 Route Reflector 节点 + +首先查看当前集群中的节点： + +``` +$ calicoctl get node -o wide +NAME ASN IPV4 IPV6 +k8s401 (64512) 192.168.1.1/24 +k8s402 (64512) 192.168.1.2/24 +k8s403 (64512) 192.168.1.3/24 +k8s404 (64512) 192.168.1.4/24 +``` + +可以在集群中选择1个或多个节点作为 rr 节点，这里先选择节点：k8s401 + +``` bash +#配置routeReflectorClusterID +calicoctl patch node k8s401 -p '{"spec": {"bgp": {"routeReflectorClusterID": "244.0.0.1"}}}' + +#配置node label +calicoctl patch node k8s401 -p '{"metadata": {"labels": {"route-reflector": "true"}}}' +``` + +- 配置 BGP node 与 Route Reflector 的连接建立规则 + +``` bash +$ cat << EOF | calicoctl create -f - +kind: BGPPeer +apiVersion: projectcalico.org/v3 +metadata: + name: peer-with-route-reflectors +spec: + nodeSelector: all() + peerSelector: route-reflector == 'true' +EOF +``` + +- 配置全局禁用全连接（BGP full mesh） ``` $ cat << EOF | calicoctl create -f - @@ -106,93 +151,10 @@ spec: EOF ``` -上述命令配置完成后，再次使用命令`ansible all -m shell -a '/opt/kube/bin/calicoctl node status'`查看，可以看到之前所有的bgp连接都消失了。 - -## 配置 BGP node 与 Route Reflector 的连接建立规则 - -``` bash -$ cat << EOF | calicoctl create -f - -kind: BGPPeer -apiVersion: projectcalico.org/v3 -metadata: - name: peer-to-rrs -spec: - # 规则1：普通 bgp node 与 rr 建立连接 - nodeSelector: "!has(i-am-a-route-reflector)" - peerSelector: has(i-am-a-route-reflector) - ---- -kind: BGPPeer -apiVersion: projectcalico.org/v3 -metadata: - name: rr-mesh -spec: - # 规则2：route reflectors 之间也建立连接 - nodeSelector: has(i-am-a-route-reflector) - peerSelector: has(i-am-a-route-reflector) -EOF -``` - -上述命令配置完成后，使用命令：`calicoctl get bgppeer` `calicoctl get bgppeer rr-mesh -o yaml` 检查配置是否正确。 - -## 选择并配置 Route Reflector 节点 - -首先查看当前集群中的节点： - -``` -$ calicoctl get node -o wide -NAME ASN IPV4 IPV6 -k8s401 (64512) 192.168.1.1/24 -k8s402 (64512) 192.168.1.2/24 -k8s403 (64512) 192.168.1.3/24 -k8s404 (64512) 192.168.1.4/24 -``` - -可以在集群中选择1个或多个节点作为 rr 节点，这里先选择节点：k8s401 - -``` bash -# 1.先导出 node k8s401 的配置，准备修改 -$ calicoctl get node k8s401 --export -o yaml |tee rr01.yml -apiVersion: projectcalico.org/v3 -kind: Node -metadata: - creationTimestamp: null - name: k8s401 -spec: - bgp: - ipv4Address: 192.168.1.1/24 - ipv4IPIPTunnelAddr: 172.20.7.128 - orchRefs: - - nodeName: 192.168.1.1 - orchestrator: k8s - -# 2.修改上述 rr01.yml 的配置如下 -apiVersion: projectcalico.org/v3 -kind: Node -metadata: - creationTimestamp: null - name: k8s401 - labels: - # 设置标签 - i-am-a-route-reflector: true -spec: - bgp: - ipv4Address: 192.168.1.1/24 - ipv4IPIPTunnelAddr: 172.20.7.128 - # 设置集群ID - routeReflectorClusterID: 224.0.0.1 - orchRefs: - - nodeName: 192.168.1.1 - orchestrator: k8s - -# 3.应用修改后的 rr node 配置 -$ calicoctl apply -f rr01.yml -``` - -## 查看增加 rr 之后的bgp 连接情况 +- 验证增加 rr 之后的bgp 连接情况 ``` -$ ansible all -m shell -a '/opt/kube/bin/calicoctl node status' +$ dk ansible -i /etc/kubeasz/clusters/xxx/hosts all -m shell -a '/opt/kube/bin/calicoctl node status' 192.168.1.4 | SUCCESS | rc=0 >> Calico process is running. @@ -249,15 +211,11 @@ No IPv6 peers found. ``` 可以看到所有其他节点都与所选rr节点建立bgp连接。 -## 再增加一个 rr 节点 +- 再增加一个 rr 节点(略) -步骤同上述选择第1个 rr 节点，这里省略；添加成功后可以看到所有其他节点都与两个rr节点建立bgp连接，两个rr节点之间也建立bgp连接。 - -- 对于节点数较多的`K8S`集群建议配置3-4个 RR 节点 +步骤同上，添加成功后可以看到所有其他节点都与两个rr节点建立bgp连接，两个rr节点之间也建立bgp连接。对于节点数较多的`K8S`集群建议配置2-3个 RR 节点。 ## 参考文档 -- 1.[Calico 使用指南：Route Reflectors](https://docs.projectcalico.org/v3.3/usage/routereflector) +- 1.[Calico bgp 配置指南](https://projectcalico.docs.tigera.io/reference/resources/bgpconfig) - 2.[BGP路由反射器基础](https://www.sohu.com/a/140033025_761420) - -更多 BGP 路由协议相关知识请查阅思科/华为相关网络文档。 diff --git a/docs/setup/network-plugin/calico.md b/docs/setup/network-plugin/calico.md index 274487e..199b653 100644 --- a/docs/setup/network-plugin/calico.md +++ b/docs/setup/network-plugin/calico.md @@ -66,15 +66,15 @@ calico 使用客户端证书，所以hosts字段可以为空；后续可以看 ### [可选]配置calicoctl工具 [calicoctl.cfg.j2](roles/calico/templates/calicoctl.cfg.j2) ``` bash -apiVersion: v1 -kind: calicoApiConfig +apiVersion: projectcalico.org/v3 +kind: CalicoAPIConfig metadata: spec: - datastoreType: "etcdv2" + datastoreType: "etcdv3" etcdEndpoints: {{ ETCD_ENDPOINTS }} etcdKeyFile: /etc/calico/ssl/calico-key.pem etcdCertFile: /etc/calico/ssl/calico.pem - etcdCACertFile: /etc/calico/ssl/ca.pem + etcdCACertFile: {{ ca_dir }}/ca.pem ``` ### 验证calico网络 @@ -155,11 +155,5 @@ ETCDCTL_API=3 etcdctl --endpoints="http://127.0.0.1:2379" get --prefix /calico ETCDCTL_API=3 etcdctl --endpoints="http://127.0.0.1:2379" get --prefix /calico/ipam/v2/host ``` -+ calico 2.x 版本默认使用 etcd v2存储，**登录集群的一个etcd 节点**，查看命令： - -``` bash -# 查看所有calico相关数据 -etcdctl --endpoints=http://127.0.0.1:2379 --ca-file=/etc/kubernetes/ssl/ca.pem ls /calico -``` ## 下一步：[设置 BGP Route Reflector](calico-bgp-rr.md) diff --git a/example/config.yml b/example/config.yml index 85d132b..9255aec 100644 --- a/example/config.yml +++ b/example/config.yml @@ -112,6 +112,14 @@ IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}" # [calico]设置calico 网络 backend: brid, vxlan, none CALICO_NETWORKING_BACKEND: "brid" +# [calico]设置calico 是否使用route reflectors +# 如果集群规模超过50个节点，建议启用该特性 +CALICO_RR_ENABLED: false + +# CALICO_RR_NODES 配置route reflectors的节点，如果未设置默认使用集群master节点 +# CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"] +CALICO_RR_NODES: [] + # [calico]更新支持calico 版本: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x] calico_ver: "__calico__" diff --git a/ezdown b/ezdown index 8f39640..f510261 100755 --- a/ezdown +++ b/ezdown @@ -546,6 +546,10 @@ function start_kubeasz_docker() { ln -s /usr/bin/python3 /usr/bin/python fi + # create cmd alias in /root/.bashrc + sed -i '/docker exec/d' /root/.bashrc + echo "alias dk='docker exec -it kubeasz' # generated by kubeasz" >> /root/.bashrc + # run kubeasz docker container docker run --detach \ --env HOST_IP="$host_ip" \ diff --git a/roles/calico/tasks/calico-rr.yml b/roles/calico/tasks/calico-rr.yml new file mode 100644 index 0000000..4d23bd8 --- /dev/null +++ b/roles/calico/tasks/calico-rr.yml @@ -0,0 +1,37 @@ +- block: + - name: 选择rr节点(master节点) + set_fact: NODE_IPS="{% for host in groups['kube_master'] %}{{ host }} {% endfor %}" + when: "CALICO_RR_NODES|length == 0" + + - name: 选择rr节点 + set_fact: NODE_IPS="{% for host in CALICO_RR_NODES %}{{ host }} {% endfor %}" + when: "CALICO_RR_NODES|length > 0" + + - name: 显示rr节点 + debug: var="NODE_IPS" + + - name: 配置routeReflectorClusterID 和 node label + shell: 'for ip in {{ NODE_IPS }};do \ + node_name=$({{ bin_dir }}/calicoctl get node -owide|grep " $ip/"|cut -d" " -f1) && \ + {{ bin_dir }}/kubectl label node "$ip" route-reflector=true --overwrite && \ + {{ bin_dir }}/calicoctl patch node "$node_name" \ + -p "{\"spec\": {\"bgp\": {\"routeReflectorClusterID\": \"244.0.0.1\"}}}"; \ + done' + + - name: 配置 calico bgp yaml文件 + template: src={{ item }}.j2 dest=/etc/calico/{{ item }} + with_items: + - "bgp-default.yaml" + - "bgp-rr.yaml" + + - name: 应用 calico bgp 配置 + shell: "{{ bin_dir }}/calicoctl apply -f /etc/calico/bgp-rr.yaml && \ + sleep 5 && \ + {{ bin_dir }}/calicoctl apply -f /etc/calico/bgp-default.yaml && sleep 2" + run_once: true + +- name: 查看bgp连接 + shell: "{{ bin_dir }}/calicoctl node status" + register: bgp_status + +- debug: var="bgp_status.stdout_lines" diff --git a/roles/calico/tasks/main.yml b/roles/calico/tasks/main.yml index 129c0b6..6510d2d 100644 --- a/roles/calico/tasks/main.yml +++ b/roles/calico/tasks/main.yml @@ -66,3 +66,6 @@ retries: 15 delay: 15 ignore_errors: true + +- import_tasks: calico-rr.yml + when: 'CALICO_RR_ENABLED|bool' diff --git a/roles/calico/templates/bgp-default.yaml.j2 b/roles/calico/templates/bgp-default.yaml.j2 new file mode 100644 index 0000000..f38b7c5 --- /dev/null +++ b/roles/calico/templates/bgp-default.yaml.j2 @@ -0,0 +1,8 @@ +apiVersion: projectcalico.org/v3 +kind: BGPConfiguration +metadata: + name: default +spec: + logSeverityScreen: Info + nodeToNodeMeshEnabled: false + asNumber: {{ CALICO_AS_NUMBER }} diff --git a/roles/calico/templates/bgp-rr.yaml.j2 b/roles/calico/templates/bgp-rr.yaml.j2 new file mode 100644 index 0000000..05e21b9 --- /dev/null +++ b/roles/calico/templates/bgp-rr.yaml.j2 @@ -0,0 +1,7 @@ +kind: BGPPeer +apiVersion: projectcalico.org/v3 +metadata: + name: peer-with-route-reflectors +spec: + nodeSelector: all() + peerSelector: route-reflector == 'true' diff --git a/roles/calico/templates/calico-v3.8.yaml.j2 b/roles/calico/templates/calico-v3.8.yaml.j2 deleted file mode 100644 index 8ac982c..0000000 --- a/roles/calico/templates/calico-v3.8.yaml.j2 +++ /dev/null @@ -1,529 +0,0 @@ -# We use cmd-line-way( kubectl create) to create secrets 'calico-etcd-secrets' -# refer to 'roles/calico/tasks/main.yml' for details. - ---- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # Configure this with the location of your etcd cluster. - etcd_endpoints: "{{ ETCD_ENDPOINTS }}" - - # If you're using TLS enabled etcd uncomment the following. - # You must also populate the Secret below with these files. - etcd_ca: "/calico-secrets/etcd-ca" - etcd_cert: "/calico-secrets/etcd-cert" - etcd_key: "/calico-secrets/etcd-key" - - # Typha is disabled. - typha_service_name: "none" - # Configure the backend to use. - calico_backend: "{{ CALICO_NETWORKING_BACKEND }}" - - # Configure the MTU to use - veth_mtu: "1440" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "etcd_endpoints": "{{ ETCD_ENDPOINTS }}", - "etcd_key_file": "/etc/calico/ssl/calico-key.pem", - "etcd_cert_file": "/etc/calico/ssl/calico.pem", - "etcd_ca_cert_file": "/etc/kubernetes/ssl/ca.pem", - "mtu": 1500, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "/root/.kube/config" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - } - ] - } - ---- -# Source: calico/templates/rbac.yaml - -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Pods are monitored for changing labels. - # The node controller monitors Kubernetes nodes. - # Namespace and serviceaccount labels are used for policy. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - - serviceaccounts - verbs: - - watch - - list - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - annotations: - # This, along with the CriticalAddonsOnly toleration below, - # marks the pod as a critical add-on, ensuring it gets - # priority scheduling and that its resources are reserved - # if it ever gets evicted. - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - nodeSelector: - beta.kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: easzlab.io.local:5000/calico/cni:v3.8.8-1 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # The location of the etcd cluster. - - name: ETCD_ENDPOINTS - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_endpoints - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - - mountPath: /calico-secrets - name: etcd-certs - securityContext: - privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: easzlab.io.local:5000/calico/pod2daemon-flexvol:v3.8.8 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: easzlab.io.local:5000/calico/node:v3.8.8-1 - env: - # The location of the etcd cluster. - - name: ETCD_ENDPOINTS - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_endpoints - # Location of the CA certificate for etcd. - - name: ETCD_CA_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_ca - # Location of the client key for etcd. - - name: ETCD_KEY_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_key - # Location of the client certificate for etcd. - - name: ETCD_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_cert - # Set noderef for node controller. - - name: CALICO_K8S_NODE_REF - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" - - name: IP_AUTODETECTION_METHOD - value: "{{ IP_AUTODETECTION_METHOD }}" - # Enable IPIP -{% if CALICO_NETWORKING_BACKEND == "brid" %} - - name: CALICO_IPV4POOL_IPIP - value: "{{ CALICO_IPV4POOL_IPIP }}" -{% endif %} - # Enable or Disable VXLAN on the default IP pool. -{% if CALICO_NETWORKING_BACKEND == "vxlan" %} - - name: CALICO_IPV4POOL_VXLAN - value: "Never" -{% endif %} - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within `--cluster-cidr`. - - name: CALICO_IPV4POOL_CIDR - value: "{{ CLUSTER_CIDR }}" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "warning" - - name: FELIX_HEALTHENABLED - value: "true" - # Set Kubernetes NodePorts: If services do use NodePorts outside Calico’s expected range, - # Calico will treat traffic to those ports as host traffic instead of pod traffic. - - name: FELIX_KUBENODEPORTRANGES - value: "{{ NODE_PORT_RANGE.split('-')[0] }}:{{ NODE_PORT_RANGE.split('-')[1] }}" - - name: FELIX_PROMETHEUSMETRICSENABLED - value: "false" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live -{% if CALICO_NETWORKING_BACKEND == "brid" %} - - -bird-live -{% endif %} - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node -{% if CALICO_NETWORKING_BACKEND == "brid" %} - - -bird-ready -{% endif %} - - -felix-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - mountPath: /calico-secrets - name: etcd-certs - - name: policysync - mountPath: /var/run/nodeagent - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: {{ bin_dir }} - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Mount in the etcd TLS secrets with mode 400. - # See https://kubernetes.io/docs/concepts/configuration/secret/ - - name: etcd-certs - secret: - secretName: calico-etcd-secrets - defaultMode: 0400 - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-kube-controllers.yaml - -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - nodeSelector: - beta.kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - # The controllers must run in the host network namespace so that - # it isn't governed by policy that would prevent it from working. - hostNetwork: true - containers: - - name: calico-kube-controllers - image: easzlab.io.local:5000/calico/kube-controllers:v3.8.8 - env: - # The location of the etcd cluster. - - name: ETCD_ENDPOINTS - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_endpoints - # Location of the CA certificate for etcd. - - name: ETCD_CA_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_ca - # Location of the client key for etcd. - - name: ETCD_KEY_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_key - # Location of the client certificate for etcd. - - name: ETCD_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_cert - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: policy,namespace,serviceaccount,workloadendpoint,node - volumeMounts: - # Mount in the etcd TLS secrets. - - mountPath: /calico-secrets - name: etcd-certs - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - volumes: - # Mount in the etcd TLS secrets with mode 400. - # See https://kubernetes.io/docs/concepts/configuration/secret/ - - name: etcd-certs - secret: - secretName: calico-etcd-secrets - defaultMode: 0400 - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system ---- -# Source: calico/templates/calico-typha.yaml - ---- -# Source: calico/templates/configure-canal.yaml - ---- -# Source: calico/templates/kdd-crds.yaml - diff --git a/roles/calico/vars/main.yml b/roles/calico/vars/main.yml index da3d3b8..169ea1a 100644 --- a/roles/calico/vars/main.yml +++ b/roles/calico/vars/main.yml @@ -1,3 +1,6 @@ # etcd 集群服务地址列表, 根据etcd组成员自动生成 TMP_ENDPOINTS: "{% for h in groups['etcd'] %}https://{{ h }}:2379,{% endfor %}" ETCD_ENDPOINTS: "{{ TMP_ENDPOINTS.rstrip(',') }}" + +# calico AS number +CALICO_AS_NUMBER: 64512