From 4cf4d37304eca44163e51c3e0226b1630b5a107c Mon Sep 17 00:00:00 2001 From: gjmzj Date: Thu, 7 Nov 2019 14:22:31 +0000 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20log-pilot=20=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=96=B9=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/guide/efk.md | 1 + docs/guide/log-pilot.md | 56 +++++++++++ manifests/efk/kibana-deployment.yaml | 3 + manifests/efk/kibana-service.yaml | 1 + .../efk/log-pilot/log-pilot-filebeat.yaml | 95 +++++++++++++++++++ 5 files changed, 156 insertions(+) create mode 100644 docs/guide/log-pilot.md create mode 100644 manifests/efk/log-pilot/log-pilot-filebeat.yaml diff --git a/docs/guide/efk.md b/docs/guide/efk.md index c67e8cd..72a4238 100644 --- a/docs/guide/efk.md +++ b/docs/guide/efk.md @@ -1,6 +1,7 @@ ### 第一部分：EFK `EFK` 插件是`k8s`项目的一个日志解决方案，它包括三个组件：[Elasticsearch](), [Fluentd](), [Kibana]()；Elasticsearch 是日志存储和日志搜索引擎，Fluentd 负责把`k8s`集群的日志发送给 Elasticsearch, Kibana 则是可视化界面查看和检索存储在 ES 中的数据。 +- 建议在熟悉本文档内容后使用[Log-Pilot + ES + Kibana 日志方案](log-pilot.md) ### 准备 diff --git a/docs/guide/log-pilot.md b/docs/guide/log-pilot.md new file mode 100644 index 0000000..11c893d --- /dev/null +++ b/docs/guide/log-pilot.md @@ -0,0 +1,56 @@ +# Log-Pilot Elasticsearch Kibana 日志解决方案 + +该方案是社区方案`EFK`的升级版，它支持两种搜集形式，对应容器标准输出日志和容器内的日志文件；个人使用了一把，在原有`EFK`经验的基础上非常简单、方便，值得推荐；更多的关于`log-pilot`的介绍详见链接： + +- github 项目地址: https://github.com/AliyunContainerService/log-pilot +- 阿里云介绍文档: https://help.aliyun.com/document_detail/86552.html +- 介绍文档2: https://yq.aliyun.com/articles/674327 + +## 安装步骤 + +- 1.安装 ES 集群，同[EFK](efk.md)文档 + +- 2.安装 Kibana，同[EFK](efk.md)文档 + +- 3.安装 Log-Pilot + +``` bash +kubectl apply -f /etc/ansible/manifests/efk/log-pilot/log-pilot-filebeat.yaml +``` + +- 4.创建示例应用，采集日志 + +``` bash +$ cat > tomcat.yaml << EOF +apiVersion: v1 +kind: Pod +metadata: + name: tomcat +spec: + containers: + - name: tomcat + image: "tomcat:7.0" + env: + # 1、stdout为约定关键字，表示采集标准输出日志 + # 2、配置标准输出日志采集到ES的catalina索引下 + - name: aliyun_logs_catalina + value: "stdout" + # 1、配置采集容器内文件日志，支持通配符 + # 2、配置该日志采集到ES的access索引下 + - name: aliyun_logs_access + value: "/usr/local/tomcat/logs/catalina.*.log" + volumeMounts: + - name: tomcat-log + mountPath: /usr/local/tomcat/logs + volumes: + # 容器内文件日志路径需要配置emptyDir + - name: tomcat-log + emptyDir: {} +EOF + +$ kubectl apply -f tomcat.yaml +``` + +- 5.在 kibana 创建 Index Pattern，验证日志已搜集，如上示例应用，应创建如下 index pattern + - catalina-* + - access-* diff --git a/manifests/efk/kibana-deployment.yaml b/manifests/efk/kibana-deployment.yaml index e7cf9cc..b75d9d1 100644 --- a/manifests/efk/kibana-deployment.yaml +++ b/manifests/efk/kibana-deployment.yaml @@ -32,6 +32,9 @@ spec: env: - name: ELASTICSEARCH_URL value: http://elasticsearch-logging:9200 + # if kibana service is exposed by nodePort, use lines commited out instead + #- name: SERVER_BASEPATH + # value: "" - name: SERVER_BASEPATH value: /api/v1/namespaces/kube-system/services/kibana-logging/proxy ports: diff --git a/manifests/efk/kibana-service.yaml b/manifests/efk/kibana-service.yaml index d241ef7..c6fc25f 100644 --- a/manifests/efk/kibana-service.yaml +++ b/manifests/efk/kibana-service.yaml @@ -15,3 +15,4 @@ spec: targetPort: ui selector: k8s-app: kibana-logging + #type: NodePort diff --git a/manifests/efk/log-pilot/log-pilot-filebeat.yaml b/manifests/efk/log-pilot/log-pilot-filebeat.yaml new file mode 100644 index 0000000..525b65f --- /dev/null +++ b/manifests/efk/log-pilot/log-pilot-filebeat.yaml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: log-pilot + labels: + app: log-pilot + namespace: kube-system +spec: + selector: + matchLabels: + app: log-pilot + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: log-pilot + spec: + # 是否允许部署到Master节点上 + #tolerations: + #- key: node-role.kubernetes.io/master + # effect: NoSchedule + # priorityClassName: system-cluster-critical + containers: + - name: log-pilot + # 版本请参考https://github.com/AliyunContainerService/log-pilot/releases + image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat + resources: + limits: + memory: 500Mi + requests: + cpu: 200m + memory: 200Mi + env: + - name: "NODE_NAME" + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: "LOGGING_OUTPUT" + value: "elasticsearch" + # 请确保集群到ES网络可达 + - name: "ELASTICSEARCH_HOSTS" + value: "elasticsearch-logging:9200" + # 配置ES访问权限 + - name: "ELASTICSEARCH_USER" + value: "" + - name: "ELASTICSEARCH_PASSWORD" + value: "" + volumeMounts: + - name: sock + mountPath: /var/run/docker.sock + - name: root + mountPath: /host + readOnly: true + - name: varlib + mountPath: /var/lib/filebeat + - name: varlog + mountPath: /var/log/filebeat + - name: localtime + mountPath: /etc/localtime + readOnly: true + livenessProbe: + failureThreshold: 3 + exec: + command: + - /pilot/healthz + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + securityContext: + capabilities: + add: + - SYS_ADMIN + terminationGracePeriodSeconds: 30 + imagePullSecrets: + - name: ydy-test-key + volumes: + - name: sock + hostPath: + path: /var/run/docker.sock + - name: root + hostPath: + path: / + - name: varlib + hostPath: + path: /var/lib/filebeat + type: DirectoryOrCreate + - name: varlog + hostPath: + path: /var/log/filebeat + type: DirectoryOrCreate + - name: localtime + hostPath: + path: /etc/localtime