From f8ebd08e7542d152d684d98e9a1d140cfc3c7695 Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Fri, 9 Feb 2018 15:47:57 +0800 Subject: [PATCH 1/3] Registry Addon Fixup --- inventory/sample/group_vars/k8s-cluster.yml | 3 + roles/kubernetes-apps/registry/README.md | 89 +++++++------- .../registry/defaults/main.yml | 4 + .../registry/files/images/Dockerfile | 26 ---- .../registry/files/images/Makefile | 24 ---- .../registry/files/images/rootfs/bin/boot | 23 ---- .../rootfs/etc/nginx/conf.d/default.conf.in | 28 ----- .../rootfs/etc/nginx/docker-registry.conf | 6 - .../files/images/rootfs/etc/nginx/nginx.conf | 26 ---- roles/kubernetes-apps/registry/tasks/main.yml | 37 +++++- .../registry/templates/auth/README.md | 92 -------------- .../templates/auth/registry-auth-rc.yml.j2 | 56 --------- .../registry/templates/gcs/README.md | 81 ------------ .../templates/gcs/registry-gcs-rc.yml.j2 | 52 -------- .../registry/templates/registry-ns.yml.j2 | 5 + ...try-ds.yml.j2 => registry-proxy-ds.yml.j2} | 21 ++-- .../registry/templates/registry-pv.yml.j2 | 17 --- .../registry/templates/registry-pvc.yml.j2 | 10 +- ...{registry-rc.yml.j2 => registry-rs.yml.j2} | 27 ++-- .../registry/templates/registry-svc.yml.j2 | 14 +-- .../registry/templates/tls/README.md | 116 ------------------ .../templates/tls/registry-tls-rc.yml.j2 | 57 --------- .../templates/tls/registry-tls-svc.yml.j2 | 17 --- 23 files changed, 135 insertions(+), 696 deletions(-) delete mode 100644 roles/kubernetes-apps/registry/files/images/Dockerfile delete mode 100644 roles/kubernetes-apps/registry/files/images/Makefile delete mode 100755 roles/kubernetes-apps/registry/files/images/rootfs/bin/boot delete mode 100644 roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/conf.d/default.conf.in delete mode 100644 roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/docker-registry.conf delete mode 100644 roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/nginx.conf delete mode 100644 roles/kubernetes-apps/registry/templates/auth/README.md delete mode 100644 roles/kubernetes-apps/registry/templates/auth/registry-auth-rc.yml.j2 delete mode 100644 roles/kubernetes-apps/registry/templates/gcs/README.md delete mode 100644 roles/kubernetes-apps/registry/templates/gcs/registry-gcs-rc.yml.j2 create mode 100644 roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 rename roles/kubernetes-apps/registry/templates/{registry-ds.yml.j2 => registry-proxy-ds.yml.j2} (56%) delete mode 100644 roles/kubernetes-apps/registry/templates/registry-pv.yml.j2 rename roles/kubernetes-apps/registry/templates/{registry-rc.yml.j2 => registry-rs.yml.j2} (62%) delete mode 100644 roles/kubernetes-apps/registry/templates/tls/README.md delete mode 100644 roles/kubernetes-apps/registry/templates/tls/registry-tls-rc.yml.j2 delete mode 100644 roles/kubernetes-apps/registry/templates/tls/registry-tls-svc.yml.j2 diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 8808aecfe..32b872ab5 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -171,6 +171,9 @@ istio_enabled: false # Registry deployment registry_enabled: false +# registry_namespace: "{{ system_namespace }}" +# registry_storage_class: "" +# registry_disk_size: "10Gi" # Local volume provisioner deployment local_volume_provisioner_enabled: false diff --git a/roles/kubernetes-apps/registry/README.md b/roles/kubernetes-apps/registry/README.md index 59542355e..81615631e 100644 --- a/roles/kubernetes-apps/registry/README.md +++ b/roles/kubernetes-apps/registry/README.md @@ -1,36 +1,39 @@ -# Private Docker Registry in Kubernetes +Private Docker Registry in Kubernetes +===================================== Kubernetes offers an optional private Docker registry addon, which you can turn -on when you bring up a cluster or install later. This gives you a place to +on when you bring up a cluster or install later. This gives you a place to store truly private Docker images for your cluster. -## How it works +How it works +------------ -The private registry runs as a `Pod` in your cluster. It does not currently +The private registry runs as a `Pod` in your cluster. It does not currently support SSL or authentication, which triggers Docker's "insecure registry" -logic. To work around this, we run a proxy on each node in the cluster, +logic. To work around this, we run a proxy on each node in the cluster, exposing a port onto the node (via a hostPort), which Docker accepts as "secure", since it is accessed by `localhost`. -## Turning it on +Turning it on +------------- -Some cluster installs (e.g. GCE) support this as a cluster-birth flag. The +Some cluster installs (e.g. GCE) support this as a cluster-birth flag. The `ENABLE_CLUSTER_REGISTRY` variable in `cluster/gce/config-default.sh` governs -whether the registry is run or not. To set this flag, you can specify -`KUBE_ENABLE_CLUSTER_REGISTRY=true` when running `kube-up.sh`. If your cluster -does not include this flag, the following steps should work. Note that some of +whether the registry is run or not. To set this flag, you can specify +`KUBE_ENABLE_CLUSTER_REGISTRY=true` when running `kube-up.sh`. If your cluster +does not include this flag, the following steps should work. Note that some of this is cloud-provider specific, so you may have to customize it a bit. ### Make some storage -The primary job of the registry is to store data. To do that we have to decide -where to store it. For cloud environments that have networked storage, we can -use Kubernetes's `PersistentVolume` abstraction. The following template is +The primary job of the registry is to store data. To do that we have to decide +where to store it. For cloud environments that have networked storage, we can +use Kubernetes's `PersistentVolume` abstraction. The following template is expanded by `salt` in the GCE cluster turnup, but can easily be adapted to other situations: -```yaml +``` yaml kind: PersistentVolume apiVersion: v1 metadata: @@ -64,14 +67,15 @@ just want to kick the tires on this without committing to it, you can easily adapt the `ReplicationController` specification below to use a simple `emptyDir` volume instead of a `persistentVolumeClaim`. -## Claim the storage +Claim the storage +----------------- Now that the Kubernetes cluster knows that some storage exists, you can put a -claim on that storage. As with the `PersistentVolume` above, you can start +claim on that storage. As with the `PersistentVolume` above, you can start with the `salt` template: -```yaml +``` yaml kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -90,15 +94,16 @@ spec: This tells Kubernetes that you want to use storage, and the `PersistentVolume` you created before will be bound to this claim (unless you have other -`PersistentVolumes` in which case those might get bound instead). This claim +`PersistentVolumes` in which case those might get bound instead). This claim gives you the right to use this storage until you release the claim. -## Run the registry +Run the registry +---------------- Now we can run a Docker registry: -```yaml +``` yaml apiVersion: v1 kind: ReplicationController metadata: @@ -146,12 +151,13 @@ spec: ``` -## Expose the registry in the cluster +Expose the registry in the cluster +---------------------------------- Now that we have a registry `Pod` running, we can expose it as a Service: -```yaml +``` yaml apiVersion: v1 kind: Service metadata: @@ -171,14 +177,15 @@ spec: ``` -## Expose the registry on each node +Expose the registry on each node +-------------------------------- Now that we have a running `Service`, we need to expose it onto each Kubernetes -`Node` so that Docker will see it as `localhost`. We can load a `Pod` on every +`Node` so that Docker will see it as `localhost`. We can load a `Pod` on every node by creating following daemonset. -```yaml +``` yaml apiVersion: extensions/v1beta1 kind: DaemonSet metadata: @@ -217,7 +224,7 @@ spec: When modifying replication-controller, service and daemon-set defintions, take -care to ensure _unique_ identifiers for the rc-svc couple and the daemon-set. +care to ensure *unique* identifiers for the rc-svc couple and the daemon-set. Failing to do so will have register the localhost proxy daemon-sets to the upstream service. As a result they will then try to proxy themselves, which will, for obvious reasons, not work. @@ -226,29 +233,30 @@ This ensures that port 5000 on each node is directed to the registry `Service`. You should be able to verify that it is running by hitting port 5000 with a web browser and getting a 404 error: -```console +``` console $ curl localhost:5000 404 page not found ``` -## Using the registry +Using the registry +------------------ To use an image hosted by this registry, simply say this in your `Pod`'s `spec.containers[].image` field: -```yaml +``` yaml image: localhost:5000/user/container ``` Before you can use the registry, you have to be able to get images into it, -though. If you are building an image on your Kubernetes `Node`, you can spell -out `localhost:5000` when you build and push. More likely, though, you are +though. If you are building an image on your Kubernetes `Node`, you can spell +out `localhost:5000` when you build and push. More likely, though, you are building locally and want to push to your cluster. You can use `kubectl` to set up a port-forward from your local node to a running Pod: -```console +``` console $ POD=$(kubectl get pods --namespace kube-system -l k8s-app=kube-registry-upstream \ -o template --template '{{range .items}}{{.metadata.name}} {{.status.phase}}{{"\n"}}{{end}}' \ | grep Running | head -1 | cut -f1 -d' ') @@ -260,15 +268,14 @@ Now you can build and push images on your local computer as `localhost:5000/yourname/container` and those images will be available inside your kubernetes cluster with the same name. -# More Extensions +More Extensions +=============== -- [Use GCS as storage backend](gcs/README.md) -- [Enable TLS/SSL](tls/README.md) -- [Enable Authentication](auth/README.md) +- [Use GCS as storage backend](gcs/README.md) +- [Enable TLS/SSL](tls/README.md) +- [Enable Authentication](auth/README.md) -## Future improvements +Future improvements +------------------- -* Allow port-forwarding to a Service rather than a pod (#15180) - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/registry/README.md?pixel)]() +- Allow port-forwarding to a Service rather than a pod (\#15180) diff --git a/roles/kubernetes-apps/registry/defaults/main.yml b/roles/kubernetes-apps/registry/defaults/main.yml index d13290b3b..93d1cfa2a 100644 --- a/roles/kubernetes-apps/registry/defaults/main.yml +++ b/roles/kubernetes-apps/registry/defaults/main.yml @@ -3,3 +3,7 @@ registry_image_repo: registry registry_image_tag: 2.6 registry_proxy_image_repo: gcr.io/google_containers/kube-registry-proxy registry_proxy_image_tag: 0.4 + +registry_namespace: "{{ system_namespace }}" +registry_storage_class: "" +registry_disk_size: "10Gi" diff --git a/roles/kubernetes-apps/registry/files/images/Dockerfile b/roles/kubernetes-apps/registry/files/images/Dockerfile deleted file mode 100644 index 4223025a8..000000000 --- a/roles/kubernetes-apps/registry/files/images/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM nginx:1.12 - -RUN apt-get update \ - && apt-get install -y \ - curl \ - --no-install-recommends \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc - -COPY rootfs / - -CMD ["/bin/boot"] diff --git a/roles/kubernetes-apps/registry/files/images/Makefile b/roles/kubernetes-apps/registry/files/images/Makefile deleted file mode 100644 index c1b64de1c..000000000 --- a/roles/kubernetes-apps/registry/files/images/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.PHONY: build push vet test clean - -TAG = 0.4 -REPO = gcr.io/google_containers/kube-registry-proxy - -build: - docker build --pull -t $(REPO):$(TAG) . - -push: - gcloud docker -- push $(REPO):$(TAG) diff --git a/roles/kubernetes-apps/registry/files/images/rootfs/bin/boot b/roles/kubernetes-apps/registry/files/images/rootfs/bin/boot deleted file mode 100755 index 04262b464..000000000 --- a/roles/kubernetes-apps/registry/files/images/rootfs/bin/boot +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# fail if no hostname is provided -REGISTRY_HOST=${REGISTRY_HOST:?no host} -REGISTRY_PORT=${REGISTRY_PORT:-5000} - -# we are always listening on port 80 -# https://github.com/nginxinc/docker-nginx/blob/43c112100750cbd1e9f2160324c64988e7920ac9/stable/jessie/Dockerfile#L25 -PORT=80 - -sed -e "s/%HOST%/$REGISTRY_HOST/g" \ - -e "s/%PORT%/$REGISTRY_PORT/g" \ - -e "s/%BIND_PORT%/$PORT/g" \ - /etc/nginx/conf.d/default.conf - -# wait for registry to come online -while ! curl -sS "$REGISTRY_HOST:$REGISTRY_PORT" &>/dev/null; do - printf "waiting for the registry (%s:%s) to come online...\n" "$REGISTRY_HOST" "$REGISTRY_PORT" - sleep 1 -done - -printf "starting proxy...\n" -exec nginx -g "daemon off;" "$@" diff --git a/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/conf.d/default.conf.in b/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/conf.d/default.conf.in deleted file mode 100644 index ecd95fd2f..000000000 --- a/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/conf.d/default.conf.in +++ /dev/null @@ -1,28 +0,0 @@ -# Docker registry proxy for api version 2 - -upstream docker-registry { - server %HOST%:%PORT%; -} - -# No client auth or TLS -# TODO(bacongobbler): experiment with authenticating the registry if it's using TLS -server { - listen %BIND_PORT%; - server_name localhost; - - # disable any limits to avoid HTTP 413 for large image uploads - client_max_body_size 0; - - # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486) - chunked_transfer_encoding on; - - location / { - # Do not allow connections from docker 1.5 and earlier - # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents - if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) { - return 404; - } - - include docker-registry.conf; - } -} diff --git a/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/docker-registry.conf b/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/docker-registry.conf deleted file mode 100644 index 7dc8cfff2..000000000 --- a/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/docker-registry.conf +++ /dev/null @@ -1,6 +0,0 @@ -proxy_pass http://docker-registry; -proxy_set_header Host $http_host; # required for docker client's sake -proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $scheme; -proxy_read_timeout 900; diff --git a/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/nginx.conf b/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/nginx.conf deleted file mode 100644 index 54ecc888e..000000000 --- a/roles/kubernetes-apps/registry/files/images/rootfs/etc/nginx/nginx.conf +++ /dev/null @@ -1,26 +0,0 @@ -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - - keepalive_timeout 65; - - include /etc/nginx/conf.d/*.conf; -} diff --git a/roles/kubernetes-apps/registry/tasks/main.yml b/roles/kubernetes-apps/registry/tasks/main.yml index a236d273c..a175064ee 100644 --- a/roles/kubernetes-apps/registry/tasks/main.yml +++ b/roles/kubernetes-apps/registry/tasks/main.yml @@ -3,29 +3,56 @@ - name: Registry | Create addon dir file: path: "{{ kube_config_dir }}/addons/registry" + state: directory owner: root group: root mode: 0755 - recurse: true - name: Registry | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/registry/{{ item.file }}" with_items: - - { name: registry-svc, file: registry-svc.yml, type: service } - - { name: registry-rc, file: registry-rc.yml, type: replicationcontroller } - - { name: registry-ds, file: registry-ds.yml, type: daemonset } + - { name: registry-ns, file: registry-ns.yml, type: ns } + - { name: registry-svc, file: registry-svc.yml, type: svc } + - { name: registry-rs, file: registry-rs.yml, type: rs } + - { name: registry-proxy-ds, file: registry-proxy-ds.yml, type: ds } register: registry_manifests when: inventory_hostname == groups['kube-master'][0] - name: Registry | Apply manifests kube: name: "{{ item.item.name }}" - namespace: "{{ system_namespace }}" + namespace: "{{ registry_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/registry/{{ item.item.file }}" state: "latest" with_items: "{{ registry_manifests.results }}" when: inventory_hostname == groups['kube-master'][0] + +- name: Registry | Create PVC manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/registry/{{ item.file }}" + with_items: + - { name: registry-pvc, file: registry-pvc.yml, type: pvc } + register: registry_manifests + when: + - registry_storage_class != none + - registry_disk_size != none + - inventory_hostname == groups['kube-master'][0] + +- name: Registry | Apply PVC manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ registry_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/registry/{{ item.item.file }}" + state: "latest" + with_items: "{{ registry_manifests.results }}" + when: + - registry_storage_class != none + - registry_disk_size != none + - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/registry/templates/auth/README.md b/roles/kubernetes-apps/registry/templates/auth/README.md deleted file mode 100644 index 040c54bcb..000000000 --- a/roles/kubernetes-apps/registry/templates/auth/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# Enable Authentication with Htpasswd for Kube-Registry - -Docker registry support a few authentication providers. Full list of supported provider can be found [here](https://docs.docker.com/registry/configuration/#auth). This document describes how to enable authentication with htpasswd for kube-registry. - -### Prepare Htpasswd Secret - -Please generate your own htpasswd file. Assuming the file you generated is `htpasswd`. -Creating secret to hold htpasswd... -```console -$ kubectl --namespace=kube-system create secret generic registry-auth-secret --from-file=htpasswd=htpasswd -``` - -### Run Registry - -Please be noted that this sample rc is using emptyDir as storage backend for simplicity. - - -```yaml -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: registry:2 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - - name: REGISTRY_AUTH_HTPASSWD_REALM - value: basic_realm - - name: REGISTRY_AUTH_HTPASSWD_PATH - value: /auth/htpasswd - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - - name: auth-dir - mountPath: /auth - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumes: - - name: image-store - emptyDir: {} - - name: auth-dir - secret: - secretName: registry-auth-secret -``` - - -No changes are needed for other components (kube-registry service and proxy). - -### To Verify - -Setup proxy or port-forwarding to the kube-registry. Image push/pull should fail without authentication. Then use `docker login` to authenticate with kube-registry and see if it works. - -### Configure Nodes to Authenticate with Kube-Registry - -By default, nodes assume no authentication is required by kube-registry. Without authentication, nodes cannot pull images from kube-registry. To solve this, more documentation can be found [Here](https://github.com/kubernetes/kubernetes.github.io/blob/master/docs/concepts/containers/images.md#configuring-nodes-to-authenticate-to-a-private-repository). - - - - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/registry/auth/README.md?pixel)]() diff --git a/roles/kubernetes-apps/registry/templates/auth/registry-auth-rc.yml.j2 b/roles/kubernetes-apps/registry/templates/auth/registry-auth-rc.yml.j2 deleted file mode 100644 index 1af623d09..000000000 --- a/roles/kubernetes-apps/registry/templates/auth/registry-auth-rc.yml.j2 +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: registry:2 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - - name: REGISTRY_AUTH_HTPASSWD_REALM - value: basic_realm - - name: REGISTRY_AUTH_HTPASSWD_PATH - value: /auth/htpasswd - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - - name: auth-dir - mountPath: /auth - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumes: - - name: image-store - emptyDir: {} - - name: auth-dir - secret: - secretName: registry-auth-secret \ No newline at end of file diff --git a/roles/kubernetes-apps/registry/templates/gcs/README.md b/roles/kubernetes-apps/registry/templates/gcs/README.md deleted file mode 100644 index 5706a848f..000000000 --- a/roles/kubernetes-apps/registry/templates/gcs/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Kube-Registry with GCS storage backend - -Besides local file system, docker registry also supports a number of cloud storage backends. Full list of supported backend can be found [here](https://docs.docker.com/registry/configuration/#storage). This document describes how to enable GCS for kube-registry as storage backend. - -A few preparation steps are needed. - 1. Create a bucket named kube-registry in GCS. - 1. Create a service account for GCS access and create key file in json format. Detail instruction can be found [here](https://cloud.google.com/storage/docs/authentication#service_accounts). - - -### Pack Keyfile into a Secret - -Assuming you have downloaded the keyfile as `keyfile.json`. Create secret with the `keyfile.json`... -```console -$ kubectl --namespace=kube-system create secret generic gcs-key-secret --from-file=keyfile=keyfile.json -``` - - -### Run Registry - - -```yaml -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: registry:2 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE - value: gcs - - name: REGISTRY_STORAGE_GCS_BUCKET - value: kube-registry - - name: REGISTRY_STORAGE_GCS_KEYFILE - value: /gcs/keyfile - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumeMounts: - - name: gcs-key - mountPath: /gcs - volumes: - - name: gcs-key - secret: - secretName: gcs-key-secret -``` - - - -No changes are needed for other components (kube-registry service and proxy). - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/registry/gcs/README.md?pixel)]() diff --git a/roles/kubernetes-apps/registry/templates/gcs/registry-gcs-rc.yml.j2 b/roles/kubernetes-apps/registry/templates/gcs/registry-gcs-rc.yml.j2 deleted file mode 100644 index e69740335..000000000 --- a/roles/kubernetes-apps/registry/templates/gcs/registry-gcs-rc.yml.j2 +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: registry:2 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE - value: gcs - - name: REGISTRY_STORAGE_GCS_BUCKET - value: kube-registry - - name: REGISTRY_STORAGE_GCS_KEYFILE - value: /gcs/keyfile - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumeMounts: - - name: gcs-key - mountPath: /gcs - volumes: - - name: gcs-key - secret: - secretName: gcs-key-secret diff --git a/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 new file mode 100644 index 000000000..bb1b209e9 --- /dev/null +++ b/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ registry_namespace }} diff --git a/roles/kubernetes-apps/registry/templates/registry-ds.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 similarity index 56% rename from roles/kubernetes-apps/registry/templates/registry-ds.yml.j2 rename to roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 index 4d6a7800b..de55d3018 100644 --- a/roles/kubernetes-apps/registry/templates/registry-ds.yml.j2 +++ b/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 @@ -1,28 +1,33 @@ --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: - name: kube-registry-proxy - namespace: {{ system_namespace }} + name: registry-proxy + namespace: {{ registry_namespace }} labels: - k8s-app: kube-registry-proxy + k8s-app: registry-proxy kubernetes.io/cluster-service: "true" version: v{{ registry_proxy_image_tag }} spec: + selector: + matchLabels: + k8s-app: registry-proxy + version: v{{ registry_proxy_image_tag }} template: metadata: labels: - k8s-app: kube-registry-proxy - kubernetes.io/name: "kube-registry-proxy" + k8s-app: registry-proxy + kubernetes.io/name: "registry-proxy" kubernetes.io/cluster-service: "true" version: v{{ registry_proxy_image_tag }} spec: containers: - - name: kube-registry-proxy + - name: registry-proxy image: {{ registry_proxy_image_repo }}:{{ registry_proxy_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} env: - name: REGISTRY_HOST - value: kube-registry.kube-system.svc.cluster.local + value: registry.{{ registry_namespace }}.svc.cluster.local - name: REGISTRY_PORT value: "5000" ports: diff --git a/roles/kubernetes-apps/registry/templates/registry-pv.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-pv.yml.j2 deleted file mode 100644 index 196efa928..000000000 --- a/roles/kubernetes-apps/registry/templates/registry-pv.yml.j2 +++ /dev/null @@ -1,17 +0,0 @@ -kind: PersistentVolume -apiVersion: v1 -metadata: - name: kube-system-kube-registry-pv - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile -spec: -{% if pillar.get('cluster_registry_disk_type', '') == 'gce' %} - capacity: - storage: {{ pillar['cluster_registry_disk_size'] }} - accessModes: - - ReadWriteOnce - gcePersistentDisk: - pdName: "{{ pillar['cluster_registry_disk_name'] }}" - fsType: "ext4" -{% endif %} diff --git a/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 index 35c787177..0db26db96 100644 --- a/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 +++ b/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 @@ -1,14 +1,16 @@ -kind: PersistentVolumeClaim +--- apiVersion: v1 +kind: PersistentVolumeClaim metadata: - name: kube-registry-pvc - namespace: kube-system + name: registry-pvc + namespace: {{ registry_namespace }} labels: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: accessModes: - ReadWriteOnce + storageClassName: {{ registry_storage_class }} resources: requests: - storage: {{ pillar['cluster_registry_disk_size'] }} + storage: {{ registry_disk_size }} diff --git a/roles/kubernetes-apps/registry/templates/registry-rc.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 similarity index 62% rename from roles/kubernetes-apps/registry/templates/registry-rc.yml.j2 rename to roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 index 90c01c4aa..730ce272b 100644 --- a/roles/kubernetes-apps/registry/templates/registry-rc.yml.j2 +++ b/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 @@ -1,41 +1,48 @@ --- -apiVersion: v1 -kind: ReplicationController +apiVersion: apps/v1 +kind: ReplicaSet metadata: - name: kube-registry-v{{ registry_image_tag }} - namespace: {{ system_namespace }} + name: registry-v{{ registry_image_tag }} + namespace: {{ registry_namespace }} labels: - k8s-app: kube-registry-upstream + k8s-app: registry version: v{{ registry_image_tag }} kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: replicas: 1 selector: - k8s-app: kube-registry-upstream - version: v{{ registry_image_tag }} + matchLabels: + k8s-app: registry + version: v{{ registry_image_tag }} template: metadata: labels: - k8s-app: kube-registry-upstream + k8s-app: registry version: v{{ registry_image_tag }} kubernetes.io/cluster-service: "true" spec: containers: - name: registry image: {{ registry_image_repo }}:{{ registry_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} env: - name: REGISTRY_HTTP_ADDR value: :5000 - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY value: /var/lib/registry volumeMounts: - - name: image-store + - name: registry-pvc mountPath: /var/lib/registry ports: - containerPort: 5000 name: registry protocol: TCP volumes: - - name: image-store + - name: registry-pvc +{% if registry_storage_class != none %} + persistentVolumeClaim: + claimName: registry-pvc +{% else %} emptyDir: {} +{% endif %} diff --git a/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 index 566962469..58d101d29 100644 --- a/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 +++ b/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 @@ -2,17 +2,17 @@ apiVersion: v1 kind: Service metadata: - name: kube-registry - namespace: {{ system_namespace }} + name: registry + namespace: {{ registry_namespace }} labels: - k8s-app: kube-registry-upstream + k8s-app: registry kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "KubeRegistry" spec: selector: - k8s-app: kube-registry-upstream + k8s-app: registry ports: - - name: registry - port: 5000 - protocol: TCP + - name: registry + port: 5000 + protocol: TCP diff --git a/roles/kubernetes-apps/registry/templates/tls/README.md b/roles/kubernetes-apps/registry/templates/tls/README.md deleted file mode 100644 index 7ba5cc628..000000000 --- a/roles/kubernetes-apps/registry/templates/tls/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Enable TLS for Kube-Registry - -This document describes how to enable TLS for kube-registry. Before you start, please check if you have all the prerequisite: - -- A domain for kube-registry. Assuming it is ` myregistrydomain.com`. -- Domain certificate and key. Assuming they are `domain.crt` and `domain.key` - -### Pack domain.crt and domain.key into a Secret - -```console -$ kubectl --namespace=kube-system create secret generic registry-tls-secret --from-file=domain.crt=domain.crt --from-file=domain.key=domain.key -``` - -### Run Registry - -Please be noted that this sample rc is using emptyDir as storage backend for simplicity. - - -```yaml -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: registry:2 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - - name: REGISTRY_HTTP_TLS_CERTIFICATE - value: /certs/domain.crt - - name: REGISTRY_HTTP_TLS_KEY - value: /certs/domain.key - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - - name: cert-dir - mountPath: /certs - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumes: - - name: image-store - emptyDir: {} - - name: cert-dir - secret: - secretName: registry-tls-secret -``` - - -### Expose External IP for Kube-Registry - -Modify the default kube-registry service to `LoadBalancer` type and point the DNS record of `myregistrydomain.com` to the service external ip. - - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: kube-registry - namespace: kube-system - labels: - k8s-app: kube-registry -# kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeRegistry" -spec: - selector: - k8s-app: kube-registry - type: LoadBalancer - ports: - - name: registry - port: 5000 - protocol: TCP -``` - - -### To Verify - -Now you should be able to access your kube-registry from another docker host. -```console -docker pull busybox -docker tag busybox myregistrydomain.com:5000/busybox -docker push myregistrydomain.com:5000/busybox -docker pull myregistrydomain.com:5000/busybox -``` - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/registry/tls/README.md?pixel)]() diff --git a/roles/kubernetes-apps/registry/templates/tls/registry-tls-rc.yml.j2 b/roles/kubernetes-apps/registry/templates/tls/registry-tls-rc.yml.j2 deleted file mode 100644 index c2411c052..000000000 --- a/roles/kubernetes-apps/registry/templates/tls/registry-tls-rc.yml.j2 +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - namespace: kube-system - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 -# kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: registry:2 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - - name: REGISTRY_HTTP_TLS_CERTIFICATE - value: /certs/domain.crt - - name: REGISTRY_HTTP_TLS_KEY - value: /certs/domain.key - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - - name: cert-dir - mountPath: /certs - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumes: - - name: image-store - emptyDir: {} - - name: cert-dir - secret: - secretName: registry-tls-secret - diff --git a/roles/kubernetes-apps/registry/templates/tls/registry-tls-svc.yml.j2 b/roles/kubernetes-apps/registry/templates/tls/registry-tls-svc.yml.j2 deleted file mode 100644 index a9d59f117..000000000 --- a/roles/kubernetes-apps/registry/templates/tls/registry-tls-svc.yml.j2 +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-registry - namespace: kube-system - labels: - k8s-app: kube-registry -# kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeRegistry" -spec: - selector: - k8s-app: kube-registry - type: LoadBalancer - ports: - - name: registry - port: 5000 - protocol: TCP From b0d7115e9b95460afdc9f97f52b0e9c031b9821d Mon Sep 17 00:00:00 2001 From: Keyvan Hedayati Date: Sun, 18 Feb 2018 11:16:03 +0330 Subject: [PATCH 2/3] hswong3i/kubespray#3: Use {{ cluster_name }} for valid FQDN in REGISTRY_HOST --- .../kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 index de55d3018..84bf1cf5a 100644 --- a/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 +++ b/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2 @@ -27,7 +27,7 @@ spec: imagePullPolicy: {{ k8s_image_pull_policy }} env: - name: REGISTRY_HOST - value: registry.{{ registry_namespace }}.svc.cluster.local + value: registry.{{ registry_namespace }}.svc.{{ cluster_name }} - name: REGISTRY_PORT value: "5000" ports: From bb1eb9fec813a638d069283bebe83b73e70df612 Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Mon, 5 Mar 2018 23:05:19 +0800 Subject: [PATCH 3/3] Add labels for namespace --- roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 b/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 index bb1b209e9..c224337af 100644 --- a/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 +++ b/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 @@ -3,3 +3,5 @@ apiVersion: v1 kind: Namespace metadata: name: {{ registry_namespace }} + labels: + name: {{ registry_namespace }}