kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding EFK logging stack

pull/971/head
Brad Beam 2017-01-26 17:33:01 -06:00
parent 5a7a3f6d4a
commit df3e11bdb8
19 changed files with 333 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cluster.yml
View File

@ -63,3 +63,4 @@
any_errors_fatal: true
roles:
- { role: kubernetes-apps, tags: apps }
- { role: kubernetes-apps/efk, tags: [ apps, efk ] }

2
inventory/group_vars/all.yml
View File

@ -204,3 +204,5 @@ kpm_packages: []
rkt_version: 1.21.0
etcd_deployment_type: docker
kubelet_deployment_type: docker
efk_enabled: false

24
roles/download/defaults/main.yml
View File

@ -82,6 +82,15 @@ kubedns_image_repo: "gcr.io/google_containers/kubedns-amd64"
kubedns_image_tag: "{{ kubedns_version }}"
test_image_repo: busybox
test_image_tag: latest
elasticsearch_version: "v2.4.1"
elasticsearch_image_repo: "gcr.io/google_containers/elasticsearch"
elasticsearch_image_tag: "{{ elasticsearch_version }}"
fluentd_version: "1.22"
fluentd_image_repo: "gcr.io/google_containers/fluentd-elasticsearch"
fluentd_image_tag: "{{ fluentd_version }}"
kibana_version: "v4.6.1"
kibana_image_repo: "gcr.io/google_containers/kibana"
kibana_image_tag: "{{ kibana_version }}"
downloads:
netcheck_server:
@ -203,6 +212,21 @@ downloads:
repo: "{{ exechealthz_image_repo }}"
tag: "{{ exechealthz_image_tag }}"
sha256: "{{ exechealthz_digest_checksum|default(None) }}"
elasticsearch:
container: true
repo: "{{ elasticsearch_image_repo }}"
tag: "{{ elasticsearch_image_tag }}"
sha256: "{{ elasticsearch_digest_checksum|default(None) }}"
fluentd:
container: true
repo: "{{ fluentd_image_repo }}"
tag: "{{ fluentd_image_tag }}"
sha256: "{{ fluentd_digest_checksum|default(None) }}"
kibana:
container: true
repo: "{{ kibana_image_repo }}"
tag: "{{ kibana_image_tag }}"
sha256: "{{ kibana_digest_checksum|default(None) }}"
download:
container: "{{ file.container|default('false') }}"

6
roles/kubernetes-apps/efk/elasticsearch/defaults/main.yml 100644
View File

@ -0,0 +1,6 @@
---
elasticsearch_cpu_limit: 1000m
elasticsearch_mem_limit: 0M
elasticsearch_cpu_requests: 100m
elasticsearch_mem_requests: 0M
elasticsearch_service_port: 9200

7
roles/kubernetes-apps/efk/elasticsearch/meta/main.yml 100644
View File

@ -0,0 +1,7 @@
dependencies:
- role: download
file: "{{ downloads.elasticsearch }}"
# TODO: bradbeam add in curator
# https://github.com/Skillshare/kubernetes-efk/blob/master/configs/elasticsearch.yml#L94
# - role: download
# file: "{{ downloads.curator }}"

35
roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml 100644
View File

@ -0,0 +1,35 @@
---
- name: "ElasticSearch | Write ES deployment"
template:
src: elasticsearch-deployment.yml.j2
dest: "{{ kube_config_dir }}/elasticsearch-deployment.yaml"
register: es_deployment_manifest
- name: "ElasticSearch | Create ES deployment"
kube:
filename: "{{kube_config_dir}}/elasticsearch-deployment.yaml"
kubectl: "{{bin_dir}}/kubectl"
name: "elasticsearch-logging-v1"
namespace: "{{system_namespace}}"
resource: "rc"
state: "{{ item | ternary('latest','present') }}"
#state: "latest"
with_items: "{{ es_deployment_manifest.changed }}"
run_once: true
- name: "ElasticSearch | Write ES service "
template:
src: elasticsearch-service.yml.j2
dest: "{{ kube_config_dir }}/elasticsearch-service.yaml"
register: es_service_manifest
- name: "ElasticSearch | Create ES service"
kube:
filename: "{{kube_config_dir}}/elasticsearch-service.yaml"
kubectl: "{{bin_dir}}/kubectl"
name: "elasticsearch-logging"
namespace: "{{system_namespace}}"
resource: "svc"
state: "{{ item | ternary('latest','present') }}"
with_items: "{{ es_service_manifest.changed }}"
run_once: true

51
roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 100644
View File

@ -0,0 +1,51 @@
---
# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-elasticsearch/es-controller.yaml
apiVersion: v1
kind: ReplicationController
metadata:
name: elasticsearch-logging-v1
namespace: "{{ system_namespace }}"
labels:
k8s-app: elasticsearch-logging
version: "{{ elasticsearch_image_tag }}"
kubernetes.io/cluster-service: "true"
spec:
replicas: 2
selector:
k8s-app: elasticsearch-logging
version: "{{ elasticsearch_image_tag }}"
template:
metadata:
labels:
k8s-app: elasticsearch-logging
version: "{{ elasticsearch_image_tag }}"
kubernetes.io/cluster-service: "true"
spec:
containers:
- image: "{{ elasticsearch_image_repo }}:{{ elasticsearch_image_tag }}"
name: elasticsearch-logging
resources:
# need more cpu upon initialization, therefore burstable class
limits:
cpu: {{ elasticsearch_cpu_limit }}
{% if elasticsearch_mem_limit is defined and elasticsearch_mem_limit != "0M" %}
mem: {{ elasticsearch_mem_limit }}
{% endif %}
requests:
cpu: {{ elasticsearch_cpu_requests }}
{% if elasticsearch_mem_requests is defined and elasticsearch_mem_requests != "0M" %}
mem: {{ elasticsearch_mem_requests }}
{% endif %}
ports:
- containerPort: 9200
name: db
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
volumeMounts:
- name: es-persistent-storage
mountPath: /data
volumes:
- name: es-persistent-storage
emptyDir: {}

18
roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 100644
View File

@ -0,0 +1,18 @@
---
apiVersion: v1
kind: Service
metadata:
name: elasticsearch-logging
namespace: "{{ system_namespace }}"
labels:
k8s-app: elasticsearch-logging
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "Elasticsearch"
spec:
ports:
- port: {{ elasticsearch_service_port }}
protocol: TCP
targetPort: db
selector:
k8s-app: elasticsearch-logging

5
roles/kubernetes-apps/efk/fluentd/defaults/main.yml 100644
View File

@ -0,0 +1,5 @@
---
fluentd_cpu_limit: 0m
fluentd_mem_limit: 200Mi
fluentd_cpu_requests: 100m
fluentd_mem_requests: 200Mi

3
roles/kubernetes-apps/efk/fluentd/meta/main.yml 100644
View File

@ -0,0 +1,3 @@
dependencies:
- role: download
file: "{{ downloads.fluentd }}"

17
roles/kubernetes-apps/efk/fluentd/tasks/main.yml 100644
View File

@ -0,0 +1,17 @@
---
- name: "Fluentd | Write fluentd daemonset"
template:
src: fluentd-ds.yml.j2
dest: "{{ kube_config_dir }}/fluentd-ds.yaml"
register: fluentd_ds_manifest
- name: "Fluentd | Create fluentd daemonset"
kube:
filename: "{{kube_config_dir}}/fluentd-ds.yaml"
kubectl: "{{bin_dir}}/kubectl"
name: "fluentd-es-v{{ fluentd_version }}"
namespace: "{{system_namespace}}"
resource: "ds"
state: "{{ item | ternary('latest','present') }}"
with_items: "{{ fluentd_ds_manifest.changed }}"
run_once: true

49
roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 100644
View File

@ -0,0 +1,49 @@
---
# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-elasticsearch/es-controller.yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: "fluentd-es-v{{ fluentd_version }}"
namespace: "{{ system_namespace }}"
labels:
k8s-app: fluentd-es
kubernetes.io/cluster-service: "true"
version: "v{{ fluentd_version }}"
spec:
template:
metadata:
labels:
k8s-app: fluentd-es
kubernetes.io/cluster-service: "true"
version: "v{{ fluentd_version }}"
spec:
containers:
- name: fluentd-es
image: "{{ fluentd_image_repo }}:{{ fluentd_image_tag }}"
command:
- '/bin/sh'
- '-c'
- '/usr/sbin/td-agent 2>&1 >> /var/log/fluentd.log'
resources:
limits:
{% if fluentd_cpu_limit is defined and fluentd_cpu_limit != "0m" %}
mem: {{ fluentd_cpu_limit }}
{% endif %}
memory: {{ fluentd_mem_limit }}
requests:
cpu: {{ fluentd_cpu_requests }}
memory: {{ fluentd_mem_requests }}
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers

7
roles/kubernetes-apps/efk/kibana/defaults/main.yml 100644
View File

@ -0,0 +1,7 @@
---
kibana_cpu_limit: 100m
kibana_mem_limit: 0M
kibana_cpu_requests: 100m
kibana_mem_requests: 0M
kibana_service_port: 5601
kibaba_base_url: "/api/v1/proxy/namespaces/kube-system/services/kibana-logging"

3
roles/kubernetes-apps/efk/kibana/meta/main.yml 100644
View File

@ -0,0 +1,3 @@
dependencies:
- role: download
file: "{{ downloads.kibana }}"

34
roles/kubernetes-apps/efk/kibana/tasks/main.yml 100644
View File

@ -0,0 +1,34 @@
---
- name: "Kibana | Write Kibana deployment"
template:
src: kibana-deployment.yml.j2
dest: "{{ kube_config_dir }}/kibana-deployment.yaml"
register: kibana_deployment_manifest
- name: "Kibana | Create Kibana deployment"
kube:
filename: "{{kube_config_dir}}/kibana-deployment.yaml"
kubectl: "{{bin_dir}}/kubectl"
name: "kibana-logging"
namespace: "{{system_namespace}}"
resource: "rc"
state: "{{ item | ternary('latest','present') }}"
with_items: "{{ kibana_deployment_manifest.changed }}"
run_once: true
- name: "Kibana | Write Kibana service "
template:
src: kibana-service.yml.j2
dest: "{{ kube_config_dir }}/kibana-service.yaml"
register: kibana_service_manifest
- name: "Kibana | Create Kibana service"
kube:
filename: "{{kube_config_dir}}/kibana-service.yaml"
kubectl: "{{bin_dir}}/kubectl"
name: "kibana-logging"
namespace: "{{system_namespace}}"
resource: "svc"
state: "{{ item | ternary('latest','present') }}"
with_items: "{{ kibana_service_manifest.changed }}"
run_once: true

46
roles/kubernetes-apps/efk/kibana/templates/kibana-deployment.yml.j2 100644
View File

@ -0,0 +1,46 @@
---
# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-kibana/kibana-controller.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: kibana-logging
namespace: "{{ system_namespace }}"
labels:
k8s-app: kibana-logging
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: kibana-logging
template:
metadata:
labels:
k8s-app: kibana-logging
spec:
containers:
- name: kibana-logging
image: "{{ kibana_image_repo }}:{{ kibana_image_tag }}"
resources:
# keep request = limit to keep this container in guaranteed class
limits:
cpu: {{ kibana_cpu_limit }}
{% if kibana_mem_limit is defined and kibana_mem_limit != "0M" %}
mem: {{ kibana_mem_limit }}
{% endif %}
requests:
cpu: {{ kibana_cpu_requests }}
{% if kibana_mem_requests is defined and kibana_mem_requests != "0M" %}
mem: {{ kibana_mem_requests }}
{% endif %}
env:
- name: "ELASTICSEARCH_URL"
value: "http://elasticsearch-logging:{{ elasticsearch_service_port }}"
{% if kibana_base_url is defined and kibana_base_url != "" %}
- name: "KIBANA_BASE_URL"
value: "{{ kibana_base_url }}"
{% endif %}
ports:
- containerPort: 5601
name: ui
protocol: TCP

18
roles/kubernetes-apps/efk/kibana/templates/kibana-service.yml.j2 100644
View File

@ -0,0 +1,18 @@
---
apiVersion: v1
kind: Service
metadata:
name: kibana-logging
namespace: "{{ system_namespace }}"
labels:
k8s-app: kibana-logging
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "Kibana"
spec:
ports:
- port: {{ kibana_service_port }}
protocol: TCP
targetPort: ui
selector:
k8s-app: kibana-logging

4
roles/kubernetes-apps/efk/meta/main.yml 100644
View File

@ -0,0 +1,4 @@
dependencies:
- role: kubernetes-apps/efk/elasticsearch
- role: kubernetes-apps/efk/fluentd
- role: kubernetes-apps/efk/kibana

3
roles/kubernetes-apps/meta/main.yaml
View File

@ -18,3 +18,6 @@ dependencies:
tags: [download, netchecker]
- {role: kubernetes-apps/ansible, tags: apps}
- {role: kubernetes-apps/kpm, tags: [apps, kpm]}
- role: kubernetes-apps/efk
when: efk_enabled
tags: [ apps, efk ]