using OpenEBS for persistent volume

2018-01-05 16:31:47 +08:00 · 2018-01-05 16:31:47 +08:00 · 50778c50ef
parent 5ce0b97032
commit 50778c50ef
7 changed files with 448 additions and 0 deletions
--- a/SUMMARY.md
+++ b/SUMMARY.md
@ -118,6 +118,8 @@
    - [在OpenShift中使用GlusterFS做持久化存储](practice/storage-for-containers-using-glusterfs-with-openshift.md)
  - [CephFS](practice/cephfs.md)
    - [使用Ceph做持久化存储](practice/using-ceph-for-persistent-storage.md)
+  - [OpenEBS](practice/openebs.md)
+    - [使用OpenEBS做持久化存储](practice/using-openebs-for-persistent-storage.md)
 - [集群与应用监控](practice/monitoring.md)
  - [Heapster](practice//heapster.md)
    - [使用Heapster获取集群和对象的metric数据](practice/using-heapster-to-get-object-metrics.md)
--- a/images/iscsi-on-kubernetes.png
+++ b/images/iscsi-on-kubernetes.png
--- a/manifests/openebs/jenkins.yml
+++ b/manifests/openebs/jenkins.yml
@ -0,0 +1,51 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: jenkins-claim
+  annotations:
+    volume.beta.kubernetes.io/storage-class: openebs-standard
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5G
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: jenkins
+spec:
+ replicas: 1
+ template:
+  metadata:
+   labels:
+    app: jenkins-app
+  spec:
+   securityContext:
+     fsGroup: 1000
+   containers:
+   - name: jenkins
+     imagePullPolicy: IfNotPresent
+     image: sz-pg-oam-docker-hub-001.tendcloud.com/library/jenkins:lts
+     ports:
+     - containerPort: 8080
+     volumeMounts:
+       - mountPath: /var/jenkins_home
+         name: jenkins-home
+   volumes:
+     - name: jenkins-home
+       persistentVolumeClaim:
+         claimName: jenkins-claim
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jenkins-svc
+spec:
+  ports:
+    - port: 80
+      targetPort: 8080
+  selector:
+    app: jenkins-app
+  type: NodePort
--- a/manifests/openebs/openebs-operator.yaml
+++ b/manifests/openebs/openebs-operator.yaml
@ -0,0 +1,184 @@
+# Define the Service Account
+# Define the RBAC rules for the Service Account
+# Launch the maya-apiserver ( deployment )
+# Launch the maya-storagemanager ( deameon set )
+
+# Create Maya Service Account 
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: openebs-maya-operator
+  namespace: default
+---
+# Define Role that allows operations on K8s pods/deployments
+#  in "default" namespace
+# TODO : change to new namespace, for isolated data network
+# TODO : the rules should be updated with required group/resources/verb
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  namespace: default
+  name: openebs-maya-operator
+rules:
+- apiGroups: ["*"]
+  resources: ["nodes","nodes/proxy"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["*"]
+  resources: ["services","pods","deployments", "events", "endpoints"]
+  verbs: ["*"]
+- apiGroups: ["*"]
+  resources: ["persistentvolumes","persistentvolumeclaims"]
+  verbs: ["*"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["*"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]
+---
+# Bind the Service Account with the Role Privileges.
+# TODO: Check if default account also needs to be there
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: openebs-maya-operator
+  namespace: default
+subjects:
+- kind: ServiceAccount
+  name: openebs-maya-operator
+  namespace: default
+- kind: User
+  name: system:serviceaccount:default:default
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: openebs-maya-operator
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: maya-apiserver
+  namespace: default
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: maya-apiserver
+    spec:
+      serviceAccountName: openebs-maya-operator
+      containers:
+      - name: maya-apiserver
+        imagePullPolicy: Always
+        image: sz-pg-oam-docker-hub-001.tendcloud.com/library/openebs-m-apiserver:0.5.1-RC1
+        ports:
+        - containerPort: 5656
+        env:
+        - name: OPENEBS_IO_JIVA_CONTROLLER_IMAGE
+          value: "sz-pg-oam-docker-hub-001.tendcloud.com/library/openebs-jiva:0.5.1-RC1"
+        - name: OPENEBS_IO_JIVA_REPLICA_IMAGE
+          value: "sz-pg-oam-docker-hub-001.tendcloud.com/library/openebs-jiva:0.5.1-RC1"
+        - name: OPENEBS_IO_VOLUME_MONITOR_IMAGE
+          value: "sz-pg-oam-docker-hub-001.tendcloud.com/library/openebs-m-exporter:0.5.0"
+        - name: OPENEBS_IO_JIVA_REPLICA_COUNT
+          value: "2"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: maya-apiserver-service
+spec:
+  ports:
+  - name: api
+    port: 5656
+    protocol: TCP
+    targetPort: 5656
+  selector:
+    name: maya-apiserver
+  sessionAffinity: None
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: openebs-provisioner
+  namespace: default
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: openebs-provisioner
+    spec:
+      serviceAccountName: openebs-maya-operator
+      containers:
+      - name: openebs-provisioner
+        imagePullPolicy: Always
+        image: sz-pg-oam-docker-hub-001.tendcloud.com/library/openebs-k8s-provisioner:0.5.1-RC2
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: OPENEBS_MONITOR_URL
+          value: "http://127.0.0.1:32515/dashboard/db/openebs-volume-stats?orgId=1"
+        - name: OPENEBS_MONITOR_VOLKEY
+          value: "&var-OpenEBS"
+        - name: MAYA_PORTAL_URL
+          value: "https://mayaonline.io/"
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: storagepoolclaims.openebs.io
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: openebs.io
+  # version name to use for REST API: /apis/<group>/<version>
+  version: v1alpha1
+  # either Namespaced or Cluster
+  scope: Cluster
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: storagepoolclaims
+    # singular name to be used as an alias on the CLI and for display
+    singular: storagepoolclaim
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: StoragePoolClaim
+    # shortNames allow shorter string to match your resource on the CLI
+    shortNames:
+    - spc
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: storagepools.openebs.io
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: openebs.io
+  # version name to use for REST API: /apis/<group>/<version>
+  version: v1alpha1
+  # either Namespaced or Cluster
+  scope: Cluster
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: storagepools
+    # singular name to be used as an alias on the CLI and for display
+    singular: storagepool
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: StoragePool
+    # shortNames allow shorter string to match your resource on the CLI
+    shortNames:
+    - sp
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-standard
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
--- a/manifests/openebs/openebs-storageclasses.yaml
+++ b/manifests/openebs/openebs-storageclasses.yaml
@ -0,0 +1,100 @@
+# Define a storage classes supported by OpenEBS
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-standalone
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "1"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-percona
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "1"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-jupyter
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-mongodb
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-cassandra
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-redis
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-kafka
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 10G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-zk
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: openebs-es-data-sc
+provisioner: openebs.io/provisioner-iscsi
+parameters:
+  openebs.io/storage-pool: "default"
+  openebs.io/jiva-replica-count: "2"
+  openebs.io/volume-monitor: "true"
+  openebs.io/capacity: 5G
+---
--- a/practice/openebs.md
+++ b/practice/openebs.md
@ -0,0 +1,5 @@
+# OpenEBS
+
+[OpenEBS](https://github.com/openebs/openebs)是一款使用Go语言编写的基于容器的快存储开源块存储软件。OpenEBS使得在容器中运行关键性任务和需要数据持久化的负载变得更可靠。
+
+我们知道AWS中提供了[EBS](https://amazonaws-china.com/cn/ebs/)（Elastic Block Storage），适用于 Amazon EC2 的持久性块存储，可以满足要求最苛刻的应用程序在功能和性能方面的要求，OpenEBS即其开源实现。
--- a/practice/using-openebs-for-persistent-storage.md
+++ b/practice/using-openebs-for-persistent-storage.md
@ -0,0 +1,106 @@
+# 使用OpenEBS做持久化存储
+
+本文将指导您如何在Kubernetes集群上安装[OpenEBS](https://github.com/openebs/openebs)作为持久化存储。
+
+我们将使用[Operator](https://coreos.com/operators/)的方式来安装OpenEBS，安装之前需要先确认您的节点上已经安装了iSCSI。
+
+## 先决条件
+
+OpenEBS依赖与iSCSI做存储管理，因此需要先确保您的集群上已有安装OpenEBS。
+
+**注意**：如果您使用kubeadm，容器方式安装的kublet，那么其中会自带iSCSI，不需要再手动安装，如果是直接使用二进制形式在裸机上安装的kubelet，则需要自己安装iSCSI。
+
+iSCSI( Internet Small Computer System Interface 互联网小型计算机系统接口)是一种基于TCP/IP 的协议，用来建立和管理IP存储设备、主机和客户机等之间的相互连接，并创建存储区域网络（SAN）。SAN 使得SCSI 协议应用于高速数据传输网络成为可能，这种传输以数据块级别（block-level）在多个数据存储网络间进行。SCSI 结构基于C/S模式，其通常应用环境是：设备互相靠近，并且这些设备由SCSI 总线连接。
+
+OpenEBS需要使用iSCSI作为存储协议，而CentOS上默认是没有安装该软件的，因此我们需要手动安装。
+
+iSCSI中包括两种类型的角色：
+
+- **target**：用来提供存储（server）
+- **initiator**：使用存储的客户端（client）
+
+下图在Kubernetes中使用iSCSI的架构图（图片来源：http://rootfs.github.io/iSCSI-Kubernetes/）。
+
+![Kubernetes iSCSI架构](../images/iscsi-on-kubernetes.png)
+
+安装iSCSI服务十分简单，不需要额外的配置，只要安装后启动服务即可。
+
+在每个node节点上执行下面的命令：
+
+```bash
+yum -y install iscsi-initiator-utils
+systemctl enable iscsid
+systemctl start iscsid
+```
+
+## 快速开始
+
+使用Operator运行OpenEBS服务：
+
+```bash
+wget https://raw.githubusercontent.com/openebs/openebs/master/k8s/openebs-operator.yaml
+kubectl apply -f openebs-operator.yaml
+```
+
+使用默认或自定义的storageclass：
+
+```bash
+wget https://raw.githubusercontent.com/openebs/openebs/master/k8s/openebs-storageclasses.yaml
+kubectl apply -f openebs-storageclasses.yaml
+```
+
+用到的镜像有：
+
+- openebs/m-apiserver:0.5.1-RC1
+- openebs/openebs-k8s-provisioner:0.5.1-RC2
+- openebs/jiva:0.5.1-RC1
+- openebs/m-exporter:0.5.0
+
+## 测试
+
+下面使用OpenEBS官方文档中的[示例]()，安装Jenkins测试
+
+```bash
+wget https://raw.githubusercontent.com/openebs/openebs/master/k8s/demo/jenkins/jenkins.yml
+kubectl apply -f jenkins.yml
+```
+
+查看PV和PVC
+
+```bash
+$ kubectl get pv
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS    CLAIM                     STORAGECLASS       REASON    AGE
+pvc-8e203e86-f1e5-11e7-aa47-f4e9d49f8ed0   5G         RWO            Delete           Bound     default/jenkins-claim     openebs-standard             1h
+
+$ kubectl get pvc
+kubectl get pvc
+NAME              STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS       AGE
+jenkins-claim     Bound     pvc-8e203e86-f1e5-11e7-aa47-f4e9d49f8ed0   5G         RWO            openebs-standard   1h
+```
+
+查看Jenkins pod
+
+```bash
+Events:
+  Type     Reason                 Age                From                   Message
+  ----     ------                 ----               ----                   -------
+  Warning  FailedScheduling       29m (x2 over 29m)  default-scheduler      PersistentVolumeClaim is not bound: "jenkins-claim" (repeated 3 times)
+  Normal   Scheduled              29m                default-scheduler      Successfully assigned jenkins-668dfbd847-vhg4c to 172.20.0.115
+  Normal   SuccessfulMountVolume  29m                kubelet, 172.20.0.115  MountVolume.SetUp succeeded for volume "default-token-3l9f0"
+  Warning  FailedMount            27m                kubelet, 172.20.0.115  Unable to mount volumes for pod "jenkins-668dfbd847-vhg4c_default(8e2ad467-f1e5-11e7-aa47-f4e9d49f8ed0)": timeout expired waiting for volumes to attach/mount for pod "default"/"jenkins-668dfbd847-vhg4c". list of unattached/unmounted volumes=[jenkins-home]
+  Warning  FailedSync             27m                kubelet, 172.20.0.115  Error syncing pod
+  Normal   SuccessfulMountVolume  26m                kubelet, 172.20.0.115  MountVolume.SetUp succeeded for volume "pvc-8e203e86-f1e5-11e7-aa47-f4e9d49f8ed0"
+  Normal   Pulling                26m                kubelet, 172.20.0.115  pulling image "sz-pg-oam-docker-hub-001.tendcloud.com/library/jenkins:lts"
+  Normal   Pulled                 26m                kubelet, 172.20.0.115  Successfully pulled image "sz-pg-oam-docker-hub-001.tendcloud.com/library/jenkins:lts"
+  Normal   Created                26m                kubelet, 172.20.0.115  Created container
+  Normal   Started                26m                kubelet, 172.20.0.115  Started container
+```
+
+启动成功。Jenkins配置使用的是**NodePort**方式访问，现在访问集群中任何一个节点的Jenkins service的NodePort即可。
+
+## 参考
+
+- [OpenEBS Documentation](http://openebs.readthedocs.io/)
+- [CentOS 7.x 下配置iSCSI网络存储](http://blog.csdn.net/wh211212/article/details/52981305)
+- [Configure iSCSI Initiator](https://www.server-world.info/en/note?os=CentOS_7&p=iscsi&f=2)
+- [RHEL7: Configure a system as either an iSCSI target or initiator that persistently mounts an iSCSI target.](https://www.certdepot.net/rhel7-configure-iscsi-target-initiator-persistently/)