From 53773810ccf6cb60f5956e636e28ae4f22a20d0a Mon Sep 17 00:00:00 2001 From: jmgao Date: Mon, 25 Dec 2017 23:04:00 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0harbor=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 11.harbor.yml | 18 +++ docs/00-集群规划和基础参数设定.md | 6 +- docs/guide/harbor.md | 148 +++++++++++++++++++ docs/guide/index.md | 2 +- down/download.sh | 11 ++ example/hosts.allinone.example | 11 +- example/hosts.m-masters.example | 11 +- example/hosts.s-master.example | 11 +- roles/harbor/tasks/main.yml | 44 ++++++ roles/harbor/templates/harbor-csr.json.j2 | 21 +++ roles/harbor/templates/harbor.cfg.j2 | 106 +++++++++++++ 11 files changed, 379 insertions(+), 10 deletions(-) create mode 100644 11.harbor.yml create mode 100644 docs/guide/harbor.md create mode 100644 roles/harbor/tasks/main.yml create mode 100644 roles/harbor/templates/harbor-csr.json.j2 create mode 100644 roles/harbor/templates/harbor.cfg.j2 diff --git a/11.harbor.yml b/11.harbor.yml new file mode 100644 index 0000000..f188dd0 --- /dev/null +++ b/11.harbor.yml @@ -0,0 +1,18 @@ +- hosts: harbor + roles: + - prepare + - docker + - harbor + +- hosts: kube-node + tasks: + - name: harbor证书目录创建 + file: name=/etc/docker/certs.d/{{ HARBOR_DOMAIN }} state=directory + + - name: harbor服务器证书安装 + copy: src={{ base_dir }}/roles/prepare/files/ca.pem dest=/etc/docker/certs.d/{{ HARBOR_DOMAIN }}/ca.crt + +# 如果你的环境中有dns服务器，可以跳过hosts文件设置 + - name: 增加harbor的hosts解析 + shell: "sed -i '/{{ HARBOR_DOMAIN }}/d' /etc/hosts && \ + echo {{ HARBOR_IP }} {{ HARBOR_DOMAIN }} >> /etc/hosts" diff --git a/docs/00-集群规划和基础参数设定.md b/docs/00-集群规划和基础参数设定.md index d098028..35ac151 100644 --- a/docs/00-集群规划和基础参数设定.md +++ b/docs/00-集群规划和基础参数设定.md @@ -66,8 +66,8 @@ ca_dir="/etc/kubernetes/ssl" base_dir="/etc/ansible" #私有仓库 harbor服务器 (域名或者IP) 【可选】 -#需要把 harbor服务器证书复制到roles/harbor/files/harbor-ca.crt -HARBOR_SERVER="harbor.mydomain.com" +HARBOR_IP="192.168.1.8" +HARBOR_DOMAIN="harbor.mydomain.com" ``` ## 部署步骤 @@ -134,7 +134,7 @@ mv kubeasz /etc/ansible # 下载已打包好的binaries，并且解压缩到/etc/ansible/bin目录 # 国内请从我分享的百度云链接下载 https://pan.baidu.com/s/1c4RFaA # 如果你有合适网络环境也可以按照/down/download.sh自行从官网下载各种tar包到 ./down目录，并执行download.sh -tar zxvf k8s.184.tar.gz +tar zxvf k8s.190.tar.gz mv bin/* /etc/ansible/bin # 配置ansible的hosts文件，并且根据上文实际规划修改此hosts文件 cd /etc/ansible diff --git a/docs/guide/harbor.md b/docs/guide/harbor.md new file mode 100644 index 0000000..30e1306 --- /dev/null +++ b/docs/guide/harbor.md @@ -0,0 +1,148 @@ +## harbor + +Habor是由VMWare中国团队开源的容器镜像仓库。事实上，Habor是在Docker Registry上进行了相应的企业级扩展，从而获得了更加广泛的应用，这些新的企业级特性包括：管理用户界面，基于角色的访问控制 ，水平扩展，同步，AD/LDAP集成以及审计日志等。本文档仅说明部署单个基础harbor服务的步骤。 + +### 安装步骤 + +1. 在deploy节点下载最新的 [docker-compose](https://github.com/docker/compose/releases) 二进制文件，改名后把它放到项目 `/etc/ansible/bin`目录下，后续版本会一起打包进百度云盘`k8s.xxx.tar.gz`文件中，可以省略该步骤 + +``` bash +wget https://github.com/docker/compose/releases/download/1.18.0/docker-compose-Linux-x86_64 +mv docker-compose-Linux-x86_64 /etc/ansible/bin/docker-compose +``` +2. 在deploy节点下载最新的 [harbor](https://github.com/vmware/harbor/releases) 离线安装包，把它放到项目 `/etc/ansible/down` 目录下，也可以从分享的百度云盘下载 + +3. 在deploy节点编辑/etc/ansible/hosts文件，可以参考 `example`目录下的模板，修改部分举例如下 + +``` bash +# 如果启用harbor，请配置后面harbor相关参数 +[harbor] +192.168.1.8 NODE_IP="192.168.1.8" + +#私有仓库 harbor服务器 (域名或者IP) +HARBOR_IP="192.168.1.8" +HARBOR_DOMAIN="harbor.test.com" +``` + +4. 在deploy节点执行 `cd /etc/ansible && ansible-playbook 11.harbor.yml`，完成harbor安装 + +### 安装讲解 + +根据 `11.harbor.yml`文件，harbor节点需要以下步骤： + +1. role `prepare` 基础系统环境准备 +1. role `docker` 安装docker +1. role `harbor` 安装harbor + +`kube-node`节点在harbor部署完之后，需要配置harbor的证书，并可以在hosts里面添加harbor的域名解析，如果你的环境中有dns服务器，可以跳过hosts文件设置 + +请在另外窗口打开 [roles/harbor/tasks/main.yml](../../roles/harbor/tasks/main.yml)，对照以下讲解 + +1. 下载docker-compose可执行文件到$PATH目录 +1. 自注册变量result判断是否已经安装harbor，避免重复安装问题 +1. 解压harbor离线安装包到指定目录 +1. 导入harbor所需 docker images +1. 创建harbor证书和私钥(复用集群的CA证书) +1. 修改harbor.cfg配置文件 +1. 启动harbor安装脚本 + +### 验证harbor + +1. 在harbor节点使用`docker ps -a` 查看harbor容器组件运行情况 +1. 浏览器访问harbor节点的IP地址 `https://{{ NODE_IP }}`，使用账号 admin 和 密码 Harbor12345 (harbor.cfg 配置文件中的默认)登陆系统 + +### 管理harbor + ++ 日志目录 `/var/log/harbor` ++ 数据目录 `/data` ，其中最主要是 `/data/database` 和 `/data/registry` 目录，如果你要彻底重新安装harbor，删除这两个目录即可 + +先进入harbor安装目录 `cd /root/local/harbor`，常规操作如下： + +1. 暂停harbor `docker-compose stop` : docker容器stop，并不删除容器 +2. 恢复harbor `docker-compose start` : 恢复docker容器运行 +3. 停止harbor `docker-compose down -v` : 停止并删除docker容器 +4. 启动harbor `docker-compose up -d` : 启动所有docker容器 + +修改harbor的运行配置，需要如下步骤： + +``` bash +# 停止 harbor + docker-compose down -v +# 修改配置 + vim harbor.cfg +# 执行./prepare已更新配置到docker-compose.yml文件 + ./prepare +# 启动 harbor + docker-compose up -d +``` +### 在k8s集群使用harbor + +admin用户web登陆后可以方便的创建项目，并指定项目属性(公开或者私有)；然后创建用户，并在项目`成员`选项中选择用户和权限； + +#### 镜像上传 + +在node上使用harbor私有镜像仓库首先需要在指定目录配置harbor的CA证书，详见 `11.harbor.yml`文件。 + +使用docker客户端登陆`harbor.test.com`，然后把镜像tag成 `harbor.test.com/$项目名/$镜像名:$TAG` 之后，即可使用docker push 上传 + +``` bash +docker login harbor.test.com +Username: +Password: +Login Succeeded +docker tag busybox:latest harbor.test.com/library/busybox:latest +docker push harbor.test.com/library/busybox:latest +The push refers to a repository [harbor.test.com/library/busybox] +0271b8eebde3: Pushed +latest: digest: sha256:91ef6c1c52b166be02645b8efee30d1ee65362024f7da41c404681561734c465 size: 527 +``` +#### k8s中使用harbor + +1. 如果镜像保存在harbor中的公开项目中，那么只需要在yaml文件中简单指定harbor私有镜像即可，例如 + +``` bash +apiVersion: v1 +kind: Pod +metadata: + name: test-busybox +spec: + containers: + - name: test-busybox + image: harbor.test.com/xxx/busybox:latest + imagePullPolicy: Always +``` + +2. 如果镜像保存在harbor中的私有项目中，那么yaml文件中使用该私有项目的镜像需要指定`imagePullSecrets`，例如 + +``` bash +apiVersion: v1 +kind: Pod +metadata: + name: test-busybox +spec: + containers: + - name: test-busybox + image: harbor.test.com/xxx/busybox:latest + imagePullPolicy: Always + imagePullSecrets: + - name: harborKey1 +``` +其中 `harborKey1`可以用以下两种方式生成： + ++ 1.使用 `kubectl create secret docker-registry harborkey1 --docker-server=harbor.test.com --docker-username=admin --docker-password=Harbor12345 --docker-email=team@test.com` ++ 2.使用yaml配置文件生成 + +``` bash +//harborkey1.yaml +apiVersion: v1 +kind: Secret +metadata: + name: harborkey1 + namespace: default +data: + .dockerconfigjson: {base64 -w 0 ~/.docker/config.json} +type: kubernetes.io/dockerconfigjson +``` +前面docker login会在~/.docker下面创建一个config.json文件保存鉴权串，这里secret yaml的.dockerconfigjson后面的数据就是那个json文件的base64编码输出（-w 0让base64输出在单行上，避免折行） + +[前一篇]() -- [目录](index.md) -- [后一篇]() diff --git a/docs/guide/index.md b/docs/guide/index.md index a9fdf16..6302104 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -7,6 +7,7 @@ - 安装 [heapster](heapster.md) - 安装 [ingress](ingress.md) - 安装 efk +- 安装 [harbor](harbor.md) ### 集群维护 @@ -26,4 +27,3 @@ ### 其他 -- Harbor 部署 diff --git a/down/download.sh b/down/download.sh index 64839d8..9bb53d3 100644 --- a/down/download.sh +++ b/down/download.sh @@ -4,6 +4,8 @@ export K8S_VER=v1.8.4 export ETCD_VER=v3.2.10 export DOCKER_VER=17.09.0-ce export CALICO_VER=v2.6.2 +export DOCKER_COMPOSE=1.18.0 +export HARBOR=v1.2.2 echo "\n建议直接下载本人打包好的所有必要二进制包k8s-184.all.tar.gz，然后解压到bin目录" echo "\n建议不使用此脚本，如果你想升级组件或者实验，请通读该脚本，必要时适当修改后使用" @@ -28,6 +30,15 @@ echo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 echo "\n----download calico at:" echo https://docs.projectcalico.org/v2.6/releases/ +echo "\n----download calico cni-plugin at:" +echo https://github.com/projectcalico/cni-plugin/releases + +echo "\n----download docker-compose at:" +echo https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE}/docker-compose-Linux-x86_64 + +echo "\n----download harbor-offline-installer at:" +echo https://github.com/vmware/harbor/releases/download/${HARBOR}/harbor-offline-installer-${HARBOR}.tgz + sleep 30 ### 准备证书工具程序 diff --git a/example/hosts.allinone.example b/example/hosts.allinone.example index ee8bd96..f17ba23 100644 --- a/example/hosts.allinone.example +++ b/example/hosts.allinone.example @@ -18,6 +18,10 @@ kube-node kube-master +# 如果启用harbor，请配置后面harbor相关参数 +[harbor] +#10.100.97.44 NODE_IP="192.168.1.8" + [all:vars] # ---------集群主要参数--------------- #集群 MASTER IP @@ -26,6 +30,9 @@ MASTER_IP="192.168.1.1" #集群 APISERVER KUBE_APISERVER="https://192.168.1.1:6443" +#pause镜像地址 +POD_INFRA_CONTAINER_IMAGE=mirrorgooglecontainers/pause-amd64:3.0 + #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成 BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7" @@ -68,5 +75,5 @@ ca_dir="/etc/kubernetes/ssl" base_dir="/etc/ansible" #私有仓库 harbor服务器 (域名或者IP) -#需要把 harbor服务器证书复制到roles/harbor/files/harbor-ca.crt -HARBOR_SERVER="harbor.yourdomain.com" +#HARBOR_IP="192.168.1.8" +#HARBOR_DOMAIN="harbor.yourdomain.com" diff --git a/example/hosts.m-masters.example b/example/hosts.m-masters.example index 1146048..fa76e7e 100644 --- a/example/hosts.m-masters.example +++ b/example/hosts.m-masters.example @@ -34,6 +34,10 @@ MASTER_PORT="8443" # api-server 服务端口 kube-node kube-master +# 如果启用harbor，请配置后面harbor相关参数 +[harbor] +#10.100.97.44 NODE_IP="192.168.1.8" + # 预留组，后续添加node节点使用 [new-node] #192.168.1.xx NODE_ID=node6 NODE_IP="192.168.1.xx" @@ -45,6 +49,9 @@ kube-master MASTER_IP="192.168.1.10" KUBE_APISERVER="https://192.168.1.10:8443" +#pause镜像地址 +POD_INFRA_CONTAINER_IMAGE=mirrorgooglecontainers/pause-amd64:3.0 + #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成 BOOTSTRAP_TOKEN="c30302226d4b810e08731702d3890f50" @@ -87,5 +94,5 @@ ca_dir="/etc/kubernetes/ssl" base_dir="/etc/ansible" #私有仓库 harbor服务器 (域名或者IP) -#需要把 harbor服务器证书复制到roles/harbor/files/harbor-ca.crt -HARBOR_SERVER="harbor.mydomain.com" +#HARBOR_IP="192.168.1.8" +#HARBOR_DOMAIN="harbor.yourdomain.com" diff --git a/example/hosts.s-master.example b/example/hosts.s-master.example index 35f9ac3..690ba37 100644 --- a/example/hosts.s-master.example +++ b/example/hosts.s-master.example @@ -22,6 +22,10 @@ kube-node kube-master +# 如果启用harbor，请配置后面harbor相关参数 +[harbor] +#10.100.97.44 NODE_IP="192.168.1.8" + [all:vars] # ---------集群主要参数--------------- #集群 MASTER IP @@ -30,6 +34,9 @@ MASTER_IP="192.168.1.1" #集群 APISERVER KUBE_APISERVER="https://192.168.1.1:6443" +#pause镜像地址 +POD_INFRA_CONTAINER_IMAGE=mirrorgooglecontainers/pause-amd64:3.0 + #TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成 BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7" @@ -72,5 +79,5 @@ ca_dir="/etc/kubernetes/ssl" base_dir="/etc/ansible" #私有仓库 harbor服务器 (域名或者IP) -#需要把 harbor服务器证书复制到roles/harbor/files/harbor-ca.crt -HARBOR_SERVER="harbor.yourdomain.com" +#HARBOR_IP="192.168.1.8" +#HARBOR_DOMAIN="harbor.yourdomain.com" diff --git a/roles/harbor/tasks/main.yml b/roles/harbor/tasks/main.yml new file mode 100644 index 0000000..744343e --- /dev/null +++ b/roles/harbor/tasks/main.yml @@ -0,0 +1,44 @@ +- name: 下载docker compose 二进制文件 + copy: src={{ base_dir }}/bin/docker-compose dest={{ bin_dir }}/docker-compose mode=0755 + +# 注册变量result，根据result结果判断是否已经安装过harbor +# result|failed 说明没有安装过harbor，下一步进行安装 +# result|succeeded 说明已经安装过harbor，下一步跳过安装 +- name: 注册变量result + command: ls /data/registry + register: result + ignore_errors: True + +- name: 解压harbor离线安装包 + unarchive: + src: "{{ base_dir }}/down/harbor-offline-installer-v1.2.2.tgz" + dest: /root/local + copy: yes + keep_newer: yes + when: result|failed + +- name: 导入harbor所需 docker images + shell: "{{ bin_dir }}/docker load -i /root/local/harbor/harbor.v1.2.2.tar.gz" + when: result|failed + +- name: 创建harbor证书请求 + template: src=harbor-csr.json.j2 dest={{ ca_dir }}/harbor-csr.json + when: result|failed + +- name: 创建harbor证书和私钥 + shell: "cd {{ ca_dir }} && {{ bin_dir }}/cfssl gencert \ + -ca={{ ca_dir }}/ca.pem \ + -ca-key={{ ca_dir }}/ca-key.pem \ + -config={{ ca_dir }}/ca-config.json \ + -profile=kubernetes harbor-csr.json | {{ bin_dir }}/cfssljson -bare harbor" + when: result|failed + +- name: 配置 harbor.cfg 文件 + template: src=harbor.cfg.j2 dest=/root/local/harbor/harbor.cfg + when: result|failed + +- name: 安装 harbor + shell: "cd /root/local/harbor && \ + export PATH={{ bin_dir }}:$PATH && \ + ./install.sh" + when: result|failed diff --git a/roles/harbor/templates/harbor-csr.json.j2 b/roles/harbor/templates/harbor-csr.json.j2 new file mode 100644 index 0000000..ccbc82c --- /dev/null +++ b/roles/harbor/templates/harbor-csr.json.j2 @@ -0,0 +1,21 @@ +{ + "CN": "harbor", + "hosts": [ + "127.0.0.1", + "{{ NODE_IP }}", + "{{ HARBOR_DOMAIN }}" + ], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "k8s", + "OU": "System" + } + ] +} diff --git a/roles/harbor/templates/harbor.cfg.j2 b/roles/harbor/templates/harbor.cfg.j2 new file mode 100644 index 0000000..6d766e5 --- /dev/null +++ b/roles/harbor/templates/harbor.cfg.j2 @@ -0,0 +1,106 @@ +## Configuration file of Harbor + +#The IP address or hostname to access admin UI and registry service. +#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname = {{ NODE_IP }} + +#The protocol for accessing the UI and token/notification service, by default it is http. +#It can be set to https if ssl is enabled on nginx. +ui_url_protocol = https + +#The password for the root user of mysql db, change this before any production use. +db_password = Harbor12345 + +#Maximum number of job workers in job service +max_job_workers = 3 + +#Determine whether or not to generate certificate for the registry's token. +#If the value is on, the prepare script creates new root cert and private key +#for generating token to access the registry. If the value is off the default key/cert will be used. +#This flag also controls the creation of the notary signer's cert. +customize_crt = on + +#The path of cert and key files for nginx, they are applied only the protocol is set to https +ssl_cert = {{ ca_dir }}/harbor.pem +ssl_cert_key = {{ ca_dir }}/harbor-key.pem + +#The path of secretkey storage +secretkey_path = /data + +#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone +admiral_url = NA + +#The password of the Clair's postgres database, only effective when Harbor is deployed with Clair. +#Please update it before deployment, subsequent update will cause Clair's API server and Harbor unable to access Clair's database. +clair_db_password = password + +#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES +#only take effect in the first boot, the subsequent changes of these properties +#should be performed on web ui + +#************************BEGIN INITIAL PROPERTIES************************ + +#Email account settings for sending out password resetting emails. + +#Email server uses the given username and password to authenticate on TLS connections to host and act as identity. +#Identity left blank to act as username. +email_identity = + +email_server = smtp.mydomain.com +email_server_port = 25 +email_username = sample_admin@mydomain.com +email_password = abc +email_from = admin +email_ssl = false + +##The initial password of Harbor admin, only works for the first time when Harbor starts. +#It has no effect after the first launch of Harbor. +#Change the admin password from UI after launching Harbor. +harbor_admin_password = Harbor12345 + +##By default the auth mode is db_auth, i.e. the credentials are stored in a local database. +#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server. +auth_mode = db_auth + +#The url for an ldap endpoint. +ldap_url = ldaps://ldap.mydomain.com + +#A user's DN who has the permission to search the LDAP/AD server. +#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd. +#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com + +#the password of the ldap_searchdn +#ldap_search_pwd = password + +#The base DN from which to look up a user in LDAP/AD +ldap_basedn = ou=people,dc=mydomain,dc=com + +#Search filter for LDAP/AD, make sure the syntax of the filter is correct. +#ldap_filter = (objectClass=person) + +# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD +ldap_uid = uid + +#the scope to search for users, 1-LDAP_SCOPE_BASE, 2-LDAP_SCOPE_ONELEVEL, 3-LDAP_SCOPE_SUBTREE +ldap_scope = 3 + +#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds. +ldap_timeout = 5 + +#Turn on or off the self-registration feature +self_registration = on + +#The expiration time (in minute) of token created by token service, default is 30 minutes +token_expiration = 30 + +#The flag to control what users have permission to create projects +#The default value "everyone" allows everyone to creates a project. +#Set to "adminonly" so that only admin user can create project. +project_creation_restriction = everyone + +#Determine whether the job service should verify the ssl cert when it connects to a remote registry. +#Set this flag to off when the remote registry uses a self-signed or untrusted certificate. +verify_remote_cert = on +#************************END INITIAL PROPERTIES************************ +############# +