From e851a9e2c387ed54c7df4f7b4f6b19bec27ac3d0 Mon Sep 17 00:00:00 2001 From: gjmzj Date: Mon, 6 May 2019 20:42:55 +0800 Subject: [PATCH] update efk: es/kibana 6.6.1 --- .../efk/es-dynamic-pv/es-statefulset.yaml | 12 +- .../efk/es-static-pv/es-statefulset.yaml | 12 +- .../efk/es-without-pv/es-statefulset.yaml | 12 +- manifests/efk/fluentd-es-configmap.yaml | 402 +++++++++--------- manifests/efk/fluentd-es-ds.yaml | 29 +- manifests/efk/kibana-deployment.yaml | 13 +- 6 files changed, 243 insertions(+), 237 deletions(-) diff --git a/manifests/efk/es-dynamic-pv/es-statefulset.yaml b/manifests/efk/es-dynamic-pv/es-statefulset.yaml index 0f5b111..4f1d10f 100644 --- a/manifests/efk/es-dynamic-pv/es-statefulset.yaml +++ b/manifests/efk/es-dynamic-pv/es-statefulset.yaml @@ -47,14 +47,14 @@ roleRef: apiGroup: "" --- # Elasticsearch deployment itself -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: StatefulSet metadata: name: elasticsearch-logging namespace: kube-system labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: @@ -63,18 +63,18 @@ spec: selector: matchLabels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 template: metadata: labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 kubernetes.io/cluster-service: "true" spec: serviceAccountName: elasticsearch-logging containers: - #- image: gcr.io/google-containers/elasticsearch:v5.6.4 - - image: mirrorgooglecontainers/elasticsearch:v5.6.4 + #- image: gcr.io/fluentd-elasticsearch/elasticsearch:v6.6.1 + - image: easzlab/elasticsearch:v6.6.1 name: elasticsearch-logging resources: # need more cpu upon initialization, therefore burstable class diff --git a/manifests/efk/es-static-pv/es-statefulset.yaml b/manifests/efk/es-static-pv/es-statefulset.yaml index f8e7f4f..0816cc3 100644 --- a/manifests/efk/es-static-pv/es-statefulset.yaml +++ b/manifests/efk/es-static-pv/es-statefulset.yaml @@ -47,14 +47,14 @@ roleRef: apiGroup: "" --- # Elasticsearch deployment itself -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: StatefulSet metadata: name: elasticsearch-logging namespace: kube-system labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: @@ -63,18 +63,18 @@ spec: selector: matchLabels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 template: metadata: labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 kubernetes.io/cluster-service: "true" spec: serviceAccountName: elasticsearch-logging containers: - #- image: gcr.io/google-containers/elasticsearch:v5.6.4 - - image: mirrorgooglecontainers/elasticsearch:v5.6.4 + #- image: gcr.io/fluentd-elasticsearch/elasticsearch:v6.6.1 + - image: easzlab/elasticsearch:v6.6.1 name: elasticsearch-logging resources: # need more cpu upon initialization, therefore burstable class diff --git a/manifests/efk/es-without-pv/es-statefulset.yaml b/manifests/efk/es-without-pv/es-statefulset.yaml index 4bd6584..6111fb6 100644 --- a/manifests/efk/es-without-pv/es-statefulset.yaml +++ b/manifests/efk/es-without-pv/es-statefulset.yaml @@ -47,14 +47,14 @@ roleRef: apiGroup: "" --- # Elasticsearch deployment itself -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: StatefulSet metadata: name: elasticsearch-logging namespace: kube-system labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: @@ -63,18 +63,18 @@ spec: selector: matchLabels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 template: metadata: labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.6.1 kubernetes.io/cluster-service: "true" spec: serviceAccountName: elasticsearch-logging containers: - #- image: gcr.io/google-containers/elasticsearch:v5.6.4 - - image: mirrorgooglecontainers/elasticsearch:v5.6.4 + #- image: gcr.io/fluentd-elasticsearch/elasticsearch:v6.6.1 + - image: easzlab/elasticsearch:v6.6.1 name: elasticsearch-logging resources: # need more cpu upon initialization, therefore burstable class diff --git a/manifests/efk/fluentd-es-configmap.yaml b/manifests/efk/fluentd-es-configmap.yaml index 09fbad0..a6d0cbe 100644 --- a/manifests/efk/fluentd-es-configmap.yaml +++ b/manifests/efk/fluentd-es-configmap.yaml @@ -1,6 +1,16 @@ kind: ConfigMap apiVersion: v1 +metadata: + name: fluentd-es-config-v0.2.0 + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile data: + system.conf: |- + + root_dir /tmp/fluentd-buffers/ + + containers.input.conf: |- # This configuration file for Fluentd / td-agent is used # to watch changes to Docker log files. The kubelet creates symlinks that @@ -101,226 +111,232 @@ data: # CRI Log Example: # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here - type tail + @id fluentd-containers.log + @type tail path /var/log/containers/*.log pos_file /var/log/es-containers.log.pos - time_format %Y-%m-%dT%H:%M:%S.%NZ - tag kubernetes.* + tag raw.kubernetes.* read_from_head true - format multi_format - - format json - time_key time - time_format %Y-%m-%dT%H:%M:%S.%NZ - - - format /^(? + + @type multi_format + + format json + time_key time + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format /^(? + + + # Detect exceptions in the log output and forward them as one log entry. + + @id raw.kubernetes + @type detect_exceptions + remove_tag_prefix raw + message log + stream stream + multiline_flush_interval 5 + max_bytes 500000 + max_lines 1000 + + + # Concatenate multi-line logs + + @id filter_concat + @type concat + key message + multiline_end_regexp /\n$/ + separator "" + + + # Enriches records with Kubernetes metadata + + @id filter_kubernetes_metadata + @type kubernetes_metadata + + + # Fixes json fields in Elasticsearch + + @id filter_parser + @type parser + key_name log + reserve_data true + remove_key_name_field true + + @type multi_format + + format json + + + format none + + + + system.input.conf: |- - # Example: - # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 - - type tail - format /^(?