kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move flannel to etcd datastore

pull/9144/head
Florian Ruynat 2022-07-22 15:28:07 +02:00 committed by Kubernetes Prow Robot
parent eb10249a75
commit 307f598bc8
10 changed files with 340 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
roles/network_plugin/canal/tasks/main.yml
View File

@ -28,10 +28,28 @@
- {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
- {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
- name: Slurp etcd cacert file
slurp:
src: "{{ canal_cert_dir }}/ca_cert.crt"
register: etcd_ca_cert_file
failed_when: false
- name: Slurp etcd cert file
slurp:
src: "{{ canal_cert_dir }}/cert.crt"
register: etcd_cert_file
failed_when: false
- name: Slurp etcd key file
slurp:
src: "{{ canal_cert_dir }}/key.pem"
register: etcd_key_file
failed_when: false
# Flannel need etcd v2 API
- name: Canal | Set Flannel etcd configuration
command: |-
{{ bin_dir }}/etcdctl set /{{ cluster_name }}/network/config \
{{ bin_dir }}/etcdctl set /coreos.com/network/config \
'{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
register: output
retries: 4
@ -53,14 +71,17 @@
dest: "{{ kube_config_dir }}/{{ item.file }}"
mode: 0644
with_items:
- {name: canal-calico-etcd-secret, file: canal-secret-calico-etcd.yml, type: secret}
- {name: canal-config, file: canal-config.yaml, type: cm}
- {name: canal-node, file: canal-node.yaml, type: ds}
- {name: canal-kube-controllers, file: canal-calico-kube-controllers.yml, type: deployment}
- {name: canal-cr, file: canal-cr.yml, type: clusterrole}
- {name: canal, file: canal-node-sa.yml, type: sa}
- {name: calico-cr, file: canal-cr-calico-node.yml, type: clusterrole}
- {name: calico-kube-cr, file: canal-cr-calico-kube-controllers.yml, type: clusterrole}
- {name: calico-crd, file: canal-crd-calico.yml, type: crd}
- {name: flannel, file: canal-cr-flannel.yml, type: clusterrole}
- {name: canal, file: canal-crb-canal.yml, type: clusterrolebinding}
- {name: canal-calico, file: canal-crb-calico.yml, type: clusterrolebinding}
- {name: canal-flannel, file: canal-crb-flannel.yml, type: clusterrolebinding}
register: canal_manifests

43
roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2
View File

@ -33,15 +33,45 @@ spec:
effect: NoSchedule
serviceAccountName: calico-kube-controllers
priorityClassName: system-cluster-critical
# The controllers must run in the host network namespace so that
# it isn't governed by policy that would prevent it from working.
hostNetwork: true
containers:
- name: calico-kube-controllers
image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
env:
# The location of the etcd cluster.
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_endpoints
# Location of the CA certificate for etcd.
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_ca
# Location of the client key for etcd.
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_key
# Location of the client certificate for etcd.
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_cert
# Choose which controllers to run.
- name: ENABLED_CONTROLLERS
value: node
- name: DATASTORE_TYPE
value: kubernetes
value: policy,namespace,serviceaccount,workloadendpoint,node
volumeMounts:
# Mount in the etcd TLS secrets.
- mountPath: /calico-secrets
name: etcd-certs
livenessProbe:
exec:
command:
@ -57,3 +87,10 @@ spec:
- /usr/bin/check-status
- -r
periodSeconds: 10
volumes:
# Mount in the etcd TLS secrets with mode 400.
# See https://kubernetes.io/docs/concepts/configuration/secret/
- name: etcd-certs
secret:
secretName: calico-etcd-secrets
defaultMode: 0440

71
roles/network_plugin/canal/templates/canal-config.yaml.j2
View File

@ -7,6 +7,14 @@ metadata:
name: canal-config
namespace: kube-system
data:
# Configure this with the location of your etcd cluster.
etcd_endpoints: "{{ etcd_access_addresses }}"
# If you're using TLS enabled etcd uncomment the following.
# You must also populate the Secret below with these files.
etcd_ca: "/calico-secrets/etcd-ca"
etcd_cert: "/calico-secrets/etcd-cert"
etcd_key: "/calico-secrets/etcd-key"
# Typha is disabled.
typha_service_name: "none"
@ -28,41 +36,39 @@ data:
# values in this config will be automatically populated.
cni_network_config: |-
{
"name": "k8s-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "calico",
"log_level": "info",
"name": "canal",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"type": "calico",
"include_default_routes": true,
"etcd_endpoints": "__ETCD_ENDPOINTS__",
"etcd_key_file": "__ETCD_KEY_FILE__",
"etcd_cert_file": "__ETCD_CERT_FILE__",
"etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
"log_level": "info",
{% if calico_cni_log_file_path %}
"log_file_path": "{{ calico_cni_log_file_path }}",
"log_file_path": "{{ calico_cni_log_file_path }}",
{% endif %}
"datastore_type": "kubernetes",
"nodename": "__KUBERNETES_NODE_NAME__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "host-local",
"subnet": "usePodCidr"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {"portMappings": true}
},
{
"type": "bandwidth",
"capabilities": {"bandwidth": true}
}
]
"policy": {
"type": "k8s",
"k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
"k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": true
}
]
}
# Flannel network configuration. Mounted into the flannel container.
net-conf.json: |
{
@ -71,3 +77,4 @@ data:
"Type": "vxlan"
}
}

2
roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2
View File

@ -1,4 +1,4 @@
---
# Flannel ClusterRole
# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1

30
roles/network_plugin/canal/templates/canal-cr.yml.j2 100644
View File

@ -0,0 +1,30 @@
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: canal
rules:
# Used for creating service account tokens to be used by the CNI plugin
- apiGroups: [""]
resources:
- serviceaccounts/token
verbs:
- create
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
verbs:
- get
# Pod CIDR auto-detection on kubeadm needs access to config maps.
- apiGroups: [""]
resources:
- configmaps
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list

12
roles/network_plugin/canal/templates/canal-crb-canal.yml.j2 100644
View File

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: canal
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: canal
subjects:
- kind: ServiceAccount
name: canal
namespace: kube-system

6
roles/network_plugin/canal/templates/canal-node-sa.yml.j2
View File

@ -4,3 +4,9 @@ kind: ServiceAccount
metadata:
name: canal
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: calico-kube-controllers
namespace: kube-system

185
roles/network_plugin/canal/templates/canal-node.yaml.j2
View File

@ -44,6 +44,7 @@ spec:
# and CNI network config file on each node.
- name: install-cni
image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
command: ["/opt/cni/bin/install"]
envFrom:
- configMapRef:
@ -71,6 +72,30 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# The location of the etcd cluster.
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_endpoints
# Location of the CA certificate for etcd.
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_ca
# Location of the client key for etcd.
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_key
# Location of the client certificate for etcd.
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_cert
# CNI MTU Config variable
- name: CNI_MTU
valueFrom:
@ -85,6 +110,8 @@ spec:
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
- mountPath: /calico-secrets
name: etcd-certs
securityContext:
privileged: true
# This init container mounts the necessary filesystems needed by the BPF data plane
@ -125,17 +152,32 @@ spec:
name: kubernetes-services-endpoint
optional: true
env:
# Use Kubernetes API as the backing datastore.
- name: DATASTORE_TYPE
value: "kubernetes"
# Configure route aggregation based on pod CIDR.
- name: USE_POD_CIDR
value: "true"
# Wait for the datastore.
- name: WAIT_FOR_DATASTORE
value: "true"
# Set based on the k8s node name.
- name: NODENAME
# The location of the etcd cluster.
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_endpoints
# Location of the CA certificate for etcd.
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_ca
# Location of the client key for etcd.
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_key
# Location of the client certificate for etcd.
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_cert
# Set noderef for node controller.
- name: CALICO_K8S_NODE_REF
valueFrom:
fieldRef:
fieldPath: spec.nodeName
@ -221,6 +263,8 @@ spec:
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
- mountPath: /calico-secrets
name: etcd-certs
- name: policysync
mountPath: /var/run/nodeagent
# For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
@ -230,22 +274,70 @@ spec:
- name: cni-log-dir
mountPath: /var/log/calico/cni
readOnly: true
# This container runs flannel using the kube-subnet-mgr backend
# for allocating subnets.
- name: kube-flannel
# Runs the flannel daemon to enable vxlan networking between
# container hosts.
- name: flannel
image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"]
imagePullPolicy: {{ k8s_image_pull_policy }}
command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
securityContext:
privileged: true
resources:
limits:
cpu: {{ flannel_cpu_limit }}
memory: {{ flannel_memory_limit }}
requests:
cpu: {{ flannel_cpu_requests }}
memory: {{ flannel_memory_requests }}
env:
# The location of the etcd cluster.
- name: FLANNELD_ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_endpoints
# Location of the CA certificate for etcd.
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_ca
# Location of the client key for etcd.
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_key
# Location of the client certificate for etcd.
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_cert
# Location of the CA certificate for etcd.
- name: FLANNELD_ETCD_CAFILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_ca
# Location of the client key for etcd.
- name: FLANNELD_ETCD_KEYFILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_key
# Location of the client certificate for etcd.
- name: FLANNELD_ETCD_CERTFILE
valueFrom:
configMapKeyRef:
name: canal-config
key: etcd_cert
# The interface flannel should run on.
- name: FLANNELD_IFACE
valueFrom:
configMapKeyRef:
name: canal-config
key: canal_iface
# Perform masquerade on traffic leaving the pod cidr.
- name: FLANNELD_IP_MASQ
valueFrom:
configMapKeyRef:
name: canal-config
key: masquerade
# Write the subnet.env file to the mounted directory.
- name: FLANNELD_SUBNET_FILE
value: "/run/flannel/subnet.env"
- name: POD_NAME
valueFrom:
fieldRef:
@ -254,24 +346,22 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: FLANNELD_IFACE
valueFrom:
configMapKeyRef:
name: canal-config
key: canal_iface
- name: FLANNELD_IP_MASQ
valueFrom:
configMapKeyRef:
name: canal-config
key: masquerade
securityContext:
privileged: true
volumeMounts:
- mountPath: /run/xtables.lock
name: xtables-lock
readOnly: false
- name: flannel-cfg
mountPath: /etc/kube-flannel/
- mountPath: /etc/resolv.conf
name: resolv
- mountPath: /run/flannel
name: run-flannel
- mountPath: /calico-secrets
name: etcd-certs
- name: flannel-cfg
mountPath: /etc/kube-flannel/
volumes:
# Used by canal.
- name: flannel-cfg
configMap:
name: canal-config
# Used by canal-node.
- name: lib-modules
hostPath:
path: /lib/modules
@ -298,9 +388,12 @@ spec:
hostPath:
path: /proc
# Used by flannel.
- name: flannel-cfg
configMap:
name: canal-config
- name: run-flannel
hostPath:
path: /run/flannel
- name: resolv
hostPath:
path: /etc/resolv.conf
# Used to install CNI.
- name: cni-bin-dir
hostPath:
@ -312,6 +405,12 @@ spec:
- name: cni-log-dir
hostPath:
path: /var/log/calico/cni
# Mount in the etcd TLS secrets with mode 400.
# See https://kubernetes.io/docs/concepts/configuration/secret/
- name: etcd-certs
secret:
secretName: calico-etcd-secrets
defaultMode: 0400
# Used to create per-pod Unix Domain Sockets
- name: policysync
hostPath:

18
roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2 100644
View File

@ -0,0 +1,18 @@
# Source: calico/templates/calico-etcd-secrets.yaml
# The following contains k8s Secrets for use with a TLS enabled etcd cluster.
# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: calico-etcd-secrets
namespace: kube-system
data:
# Populate the following with etcd TLS configuration if desired, but leave blank if
# not using TLS for etcd.
# The keys below should be uncommented and the values populated with the base64
# encoded contents of each file that would be associated with the TLS data.
# Example command for encoding a file contents: cat <file> | base64 -w 0
etcd-key: {{ etcd_key_file.content }}
etcd-cert: {{ etcd_cert_file.content }}
etcd-ca: {{ etcd_ca_cert_file.content }}

59
roles/network_plugin/canal/templates/cni-canal.conflist.j2
View File

@ -1,33 +1,34 @@
{
"name": "cni0",
"cniVersion":"0.3.1",
"plugins":[
{
"type": "flannel",
"delegate": {
"type": "calico",
"include_default_routes": true,
"etcd_endpoints": "{{ etcd_access_addresses }}",
"etcd_key_file": "{{ canal_cert_dir }}/key.pem",
"etcd_cert_file": "{{ canal_cert_dir }}/cert.crt",
"etcd_ca_cert_file": "{{ canal_cert_dir }}/ca_cert.crt",
"log_level": "info",
"name": "canal",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"type": "calico",
"include_default_routes": true,
"etcd_endpoints": "__ETCD_ENDPOINTS__",
"etcd_key_file": "__ETCD_KEY_FILE__",
"etcd_cert_file": "__ETCD_CERT_FILE__",
"etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
"log_level": "info",
{% if calico_cni_log_file_path %}
"log_file_path": "{{ calico_cni_log_file_path }}",
"log_file_path": "{{ calico_cni_log_file_path }}",
{% endif %}
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
}
},
{
"type":"portmap",
"capabilities":{
"portMappings":true
}
"policy": {
"type": "k8s",
"k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
"k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": true
}
]
}
]
}