use metallb as LoadBalancer Provider

roles/cluster-addon/defaults/main.yml

@@ -31,3 +31,10 @@ heapster_offline: "heapster_v1.5.4.tar"
 
 # prometheus 自动安装
 #prometheus_install: "no"
+
+# metallb 自动安装
+metallb_install: "no"
+# 模式选择: 二层 "layer2" 或者三层 "bgp"
+metallb_protocol: "layer2"
+metallb_offline: "metallb_v0.7.3.tar"
+metallb_vip_pool: "192.168.1.240/29"

roles/cluster-addon/tasks/main.yml

@@ -13,7 +13,7 @@
       - coredns
     
     - name: 获取所有已经创建的POD信息
-      command: "{{ bin_dir }}/kubectl get pod -n kube-system"
+      command: "{{ bin_dir }}/kubectl get pod --all-namespaces"
       register: pod_info
   delegate_to: "{{ groups.deploy[0] }}"
   run_once: true
@@ -127,3 +127,36 @@
   when: '"heapster" not in pod_info.stdout and heapster_install == "yes"'
   ignore_errors: true
 
+- block:
+    - name: 尝试推送离线 metallb镜像（若执行失败，可忽略）
+      copy: src={{ base_dir }}/down/{{ metallb_offline }} dest=/opt/kube/images/{{ metallb_offline }}
+      when: 'metallb_offline in download_info.stdout'
+    
+    - name: 获取metallb离线镜像推送情况
+      command: "ls /opt/kube/images"
+      register: image_info     
+      
+    - name: 导入 metallb的离线镜像（若执行失败，可忽略）
+      shell: "{{ bin_dir }}/docker load -i /opt/kube/images/{{ metallb_offline }}"
+      when: 'metallb_offline in image_info.stdout'
+
+    - name: 生成 metallb 相关 manifests
+      template: src=metallb/{{ item }}.j2 dest=/opt/kube/kube-system/{{ item }}
+      with_items:
+      - "metallb.yaml"
+      - "{{ metallb_protocol }}.yaml"
+      delegate_to: "{{ groups.deploy[0] }}"
+      run_once: true
+
+    - name: 创建 metallb controller 部署
+      shell: "{{ bin_dir }}/kubectl apply -f /opt/kube/kube-system/metallb.yaml"
+      delegate_to: "{{ groups.deploy[0] }}"
+      run_once: true
+
+    - name: 创建 metallb configmap
+      shell: "{{ bin_dir }}/kubectl apply -f /opt/kube/kube-system/{{ metallb_protocol }}.yaml"
+      delegate_to: "{{ groups.deploy[0] }}"
+      run_once: true
+  when: '"metallb" not in pod_info.stdout and metallb_install == "yes"'
+  ignore_errors: true
+

roles/cluster-addon/templates/metallb/bgp.yaml.j2

@@ -0,0 +1,113 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    # The peers section tells MetalLB what BGP routers to connect too. There
+    # is one entry for each router you want to peer with.
+    peers:
+    - # The target IP address for the BGP session.
+      peer-address: 10.0.0.1
+      # The BGP AS number that MetalLB expects to see advertised by
+      # the router.
+      peer-asn: 64512
+      # The BGP AS number that MetalLB should speak as.
+      my-asn: 64512
+      # (optional) the TCP port to talk to. Defaults to 179, you shouldn't
+      # need to set this in production.
+      peer-port: 179
+      # (optional) The proposed value of the BGP Hold Time timer. Refer to
+      # BGP reference material to understand what setting this implies.
+      hold-time: 120
+      # (optional) The router ID to use when connecting to this peer. Defaults
+      # to the node IP address. Generally only useful when you need to peer with
+      # another BGP router running on the same machine as MetalLB.
+      router-id: 1.2.3.4
+      # (optional) Password for TCPMD5 authenticated BGP sessions
+      # offered by some peers.
+      password: "yourPassword"
+      # (optional) The nodes that should connect to this peer. A node
+      # matches if at least one of the node selectors matches. Within
+      # one selector, a node matches if all the matchers are
+      # satisfied. The semantics of each selector are the same as the
+      # label- and set-based selectors in Kubernetes, documented at
+      # https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/.
+      # By default, all nodes are selected.
+      node-selectors:
+      - # Match by label=value
+        match-labels:
+          kubernetes.io/hostname: prod-01
+        # Match by 'key OP values' expressions
+        match-expressions:
+        - key: beta.kubernetes.io/arch
+          operator: In
+          values: [amd64, arm]
+
+    # The address-pools section lists the IP addresses that MetalLB is
+    # allowed to allocate, along with settings for how to advertise
+    # those addresses over BGP once assigned. You can have as many
+    # address pools as you want.
+    address-pools:
+    - # A name for the address pool. Services can request allocation
+      # from a specific address pool using this name, by listing this
+      # name under the 'metallb.universe.tf/address-pool' annotation.
+      name: my-ip-space
+      # Protocol can be used to select how the announcement is done.
+      # Supported values are bgp and layer2.
+      protocol: bgp
+      
+      # A list of IP address ranges over which MetalLB has
+      # authority. You can list multiple ranges in a single pool, they
+      # will all share the same settings. Each range can be either a
+      # CIDR prefix, or an explicit start-end range of IPs.
+      addresses:
+      - 198.51.100.0/24
+      - 192.168.0.150-192.168.0.200
+      # (optional) If true, MetalLB will not allocate any address that
+      # ends in .0 or .255. Some old, buggy consumer devices
+      # mistakenly block traffic to such addresses under the guise of
+      # smurf protection. Such devices have become fairly rare, but
+      # the option is here if you encounter serving issues.
+      avoid-buggy-ips: true
+      # (optional, default true) If false, MetalLB will not automatically
+      # allocate any address in this pool. Addresses can still explicitly
+      # be requested via loadBalancerIP or the address-pool annotation.
+      auto-assign: false
+      # (optional) A list of BGP advertisements to make, when
+      # protocol=bgp. Each address that gets assigned out of this pool
+      # will turn into this many advertisements. For most simple
+      # setups, you'll probably just want one.
+      #
+      # The default value for this field is a single advertisement with
+      # all parameters set to their respective defaults.
+      bgp-advertisements:
+      - # (optional) How much you want to aggregate up the IP address
+        # before advertising. For example, advertising 1.2.3.4 with
+        # aggregation-length=24 would end up advertising 1.2.3.0/24.
+        # For the majority of setups, you'll want to keep this at the
+        # default of 32, which advertises the entire IP address
+        # unmodified.
+        aggregation-length: 32
+        # (optional) The value of the BGP "local preference" attribute
+        # for this advertisement. Only used with IBGP peers,
+        # i.e. peers where peer-asn is the same as my-asn.
+        localpref: 100
+        # (optional) BGP communities to attach to this
+        # advertisement. Communities are given in the standard
+        # two-part form <asn>:<community number>. You can also use
+        # alias names (see below).
+        communities:
+        - 64512:1
+        - no-export
+    # (optional) BGP community aliases. Instead of using hard to
+    # read BGP community numbers in address pool advertisement
+    # configurations, you can define alias names here and use those
+    # elsewhere in the configuration. The "no-export" community used
+    # above is defined below.
+    bgp-communities:
+      # no-export is a well-known BGP community that prevents
+      # re-advertisement outside of the immediate autonomous system,
+      # but people don't usually recognize its numerical value. :)
+      no-export: 65535:65281

roles/cluster-addon/templates/metallb/layer2.yaml.j2

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: my-ip-space
+      protocol: layer2
+      addresses:
+      - {{ metallb_vip_pool }}

roles/cluster-addon/templates/metallb/metallb.yaml.j2

@@ -0,0 +1,232 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metallb-system
+  labels:
+    app: metallb
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: metallb-system
+  name: controller
+  labels:
+    app: metallb
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: metallb-system
+  name: speaker
+  labels:
+    app: metallb
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metallb-system:controller
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get", "list", "watch", "update"]
+- apiGroups: [""]
+  resources: ["services/status"]
+  verbs: ["update"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metallb-system:speaker
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["services", "endpoints", "nodes"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: metallb-system
+  name: config-watcher
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create"]
+---
+
+## Role bindings
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metallb-system:controller
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: controller
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:controller
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metallb-system:speaker
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: speaker
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:speaker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: metallb-system
+  name: config-watcher
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: controller
+- kind: ServiceAccount
+  name: speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: config-watcher
+---
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  namespace: metallb-system
+  name: speaker
+  labels:
+    app: metallb
+    component: speaker
+spec:
+  selector:
+    matchLabels:
+      app: metallb
+      component: speaker
+  template:
+    metadata:
+      labels:
+        app: metallb
+        component: speaker
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "7472"
+    spec:
+      serviceAccountName: speaker
+      terminationGracePeriodSeconds: 0
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/metallb-speaker: "true"
+      containers:
+      - name: speaker
+        image: metallb/speaker:v0.7.3
+        imagePullPolicy: IfNotPresent
+        args:
+        - --port=7472
+        - --config=config
+        env:
+        - name: METALLB_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ MASTER_IP }}"
+          #value: "{{ KUBE_APISERVER.split(':')[1].lstrip('/') }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "{{ KUBE_APISERVER.split(':')[2] }}"
+        ports:
+        - name: monitoring
+          containerPort: 7472
+        resources:
+          limits:
+            cpu: 100m
+            memory: 100Mi
+          
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+            - all
+            add:
+            - net_raw
+
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  namespace: metallb-system
+  name: controller
+  labels:
+    app: metallb
+    component: controller
+spec:
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: metallb
+      component: controller
+  template:
+    metadata:
+      labels:
+        app: metallb
+        component: controller
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "7472"
+    spec:
+      serviceAccountName: controller
+      terminationGracePeriodSeconds: 0
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534 # nobody
+      containers:
+      - name: controller
+        image: metallb/controller:v0.7.3
+        imagePullPolicy: IfNotPresent
+        args:
+        - --port=7472
+        - --config=config
+        ports:
+        - name: monitoring
+          containerPort: 7472
+        resources:
+          limits:
+            cpu: 100m
+            memory: 100Mi
+          
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+
+---
+
+