#!/bin/bash #-------------------------------------------------- # This script is used for: # 1. to download the scripts/binaries/images needed for installing a k8s cluster with kubeasz # 2. to run kubeasz in a container (optional) # @author: gjmzj # @usage: ./ezdown # @repo: https://github.com/easzlab/kubeasz # @ref: https://github.com/kubeasz/dockerfiles #-------------------------------------------------- set -o nounset set -o errexit #set -o xtrace # default settings, can be overridden by cmd line options, see usage DOCKER_VER=20.10.16 KUBEASZ_VER=3.3.0 K8S_BIN_VER=v1.24.1 EXT_BIN_VER=1.1.0 SYS_PKG_VER=0.4.3 HARBOR_VER=v2.1.3 REGISTRY_MIRROR=CN # images needed by k8s cluster calicoVer=v3.19.4 flannelVer=v0.15.1 dnsNodeCacheVer=1.21.1 corednsVer=1.8.6 dashboardVer=v2.5.1 dashboardMetricsScraperVer=v1.0.8 metricsVer=v0.5.2 pauseVer=3.6 nfsProvisionerVer=v4.0.2 export ciliumVer=v1.4.1 export kubeRouterVer=v0.3.1 export kubeOvnVer=v1.5.3 export promChartVer=35.5.1 function usage() { echo -e "\033[33mUsage:\033[0m ezdown [options] [args]" cat < set docker-ce version, default "$DOCKER_VER" -e set kubeasz-ext-bin version, default "$EXT_BIN_VER" -k set kubeasz-k8s-bin version, default "$K8S_BIN_VER" -m set docker registry mirrors, default "CN"(used in Mainland,China) -p set kubeasz-sys-pkg version, default "$SYS_PKG_VER" -z set kubeasz version, default "$KUBEASZ_VER" EOF } function logger() { TIMESTAMP=$(date +'%Y-%m-%d %H:%M:%S') case "$1" in debug) echo -e "$TIMESTAMP \033[36mDEBUG\033[0m $2" ;; info) echo -e "$TIMESTAMP \033[32mINFO\033[0m $2" ;; warn) echo -e "$TIMESTAMP \033[33mWARN\033[0m $2" ;; error) echo -e "$TIMESTAMP \033[31mERROR\033[0m $2" ;; *) ;; esac } function download_docker() { if [[ "$REGISTRY_MIRROR" == CN ]];then DOCKER_URL="https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/static/stable/x86_64/docker-${DOCKER_VER}.tgz" else DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VER}.tgz" fi if [[ -f "$BASE/down/docker-${DOCKER_VER}.tgz" ]];then logger warn "docker binaries already existed" else logger info "downloading docker binaries, version $DOCKER_VER" if [[ -e /usr/bin/wget ]];then wget -c --no-check-certificate "$DOCKER_URL" || { logger error "downloading docker failed"; exit 1; } else curl -k -C- -O --retry 3 "$DOCKER_URL" || { logger error "downloading docker failed"; exit 1; } fi /bin/mv -f "./docker-$DOCKER_VER.tgz" "$BASE/down" fi tar zxf "$BASE/down/docker-$DOCKER_VER.tgz" -C "$BASE/down" && \ /bin/cp -f "$BASE"/down/docker/* "$BASE/bin" && \ /bin/mv -f "$BASE"/down/docker/* /opt/kube/bin && \ ln -sf /opt/kube/bin/docker /bin/docker } function install_docker() { # check if a container runtime is already installed systemctl status docker|grep Active|grep -q running && { logger warn "docker is already running."; return 0; } logger debug "generate docker service file" cat > /etc/systemd/system/docker.service << EOF [Unit] Description=Docker Application Container Engine Documentation=http://docs.docker.io [Service] Environment="PATH=/opt/kube/bin:/bin:/sbin:/usr/bin:/usr/sbin" ExecStart=/opt/kube/bin/dockerd ExecStartPost=/sbin/iptables -P FORWARD ACCEPT ExecReload=/bin/kill -s HUP \$MAINPID Restart=on-failure RestartSec=5 LimitNOFILE=infinity LimitNPROC=infinity LimitCORE=infinity Delegate=yes KillMode=process [Install] WantedBy=multi-user.target EOF # configuration for dockerd mkdir -p /etc/docker DOCKER_VER_MAIN=$(echo "$DOCKER_VER"|cut -d. -f1) CGROUP_DRIVER="cgroupfs" ((DOCKER_VER_MAIN>=20)) && CGROUP_DRIVER="systemd" logger debug "generate docker config: /etc/docker/daemon.json" if [[ "$REGISTRY_MIRROR" == CN ]];then logger debug "prepare register mirror for $REGISTRY_MIRROR" cat > /etc/docker/daemon.json << EOF { "exec-opts": ["native.cgroupdriver=$CGROUP_DRIVER"], "registry-mirrors": [ "https://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com" ], "insecure-registries": ["http://easzlab.io.local:5000"], "max-concurrent-downloads": 10, "log-driver": "json-file", "log-level": "warn", "log-opts": { "max-size": "10m", "max-file": "3" }, "data-root": "/var/lib/docker" } EOF else logger debug "standard config without registry mirrors" cat > /etc/docker/daemon.json << EOF { "exec-opts": ["native.cgroupdriver=$CGROUP_DRIVER"], "insecure-registries": ["http://easzlab.io.local:5000"], "max-concurrent-downloads": 10, "log-driver": "json-file", "log-level": "warn", "log-opts": { "max-size": "10m", "max-file": "3" }, "data-root": "/var/lib/docker" } EOF fi # docker proxy setting http_proxy=${http_proxy:-} HTTP_PROXY=${HTTP_PROXY:-$http_proxy} https_proxy=${https_proxy:-} HTTPS_PROXY=${HTTPS_PROXY:-$https_proxy} USE_PROXY=0 CONFIG="[Service]\n" if [[ ! -z ${HTTP_PROXY} ]]; then USE_PROXY=1 CONFIG=${CONFIG}"Environment=HTTP_PROXY=${HTTP_PROXY}\n" fi if [[ ! -z ${HTTPS_PROXY} ]]; then USE_PROXY=1 CONFIG=${CONFIG}"Environment=HTTPS_PROXY=${HTTPS_PROXY}\n" fi if [[ ${USE_PROXY} == 1 ]]; then logger debug "generate docker service http proxy file" mkdir -p /etc/systemd/system/docker.service.d c=$(echo -e "$CONFIG") cat > /etc/systemd/system/docker.service.d/http-proxy.conf << EOF ${c} EOF fi if [[ -e /etc/centos-release || -e /etc/redhat-release ]]; then logger debug "turn off selinux in CentOS/Redhat" getenforce|grep Disabled || setenforce 0 sed -i 's/^SELINUX=.*$/SELINUX=disabled/g' /etc/selinux/config fi logger debug "enable and start docker" systemctl enable docker systemctl daemon-reload && systemctl restart docker && sleep 4 } function get_kubeasz() { # check if kubeasz already existed [[ -d "$BASE/roles/kube-node" ]] && { logger warn "kubeasz already existed"; return 0; } logger info "downloading kubeasz: $KUBEASZ_VER" docker ps -a |grep -q temp_easz && { logger debug "remove existing container"; docker rm -f temp_easz; } logger debug " run a temporary container" docker run -d --name temp_easz easzlab/kubeasz:${KUBEASZ_VER} || { logger error "download failed."; exit 1; } [[ -f "$BASE/down/docker-${DOCKER_VER}.tgz" ]] && /bin/mv -f "$BASE/down/docker-${DOCKER_VER}.tgz" /tmp [[ -d "$BASE/bin" ]] && /bin/mv -f "$BASE/bin" /tmp rm -rf "$BASE" && \ logger debug "cp kubeasz code from the temporary container" && \ docker cp "temp_easz:$BASE" "$BASE" && \ logger debug "stop&remove temporary container" && \ docker rm -f temp_easz mkdir -p "$BASE/bin" "$BASE/down" [[ -f "/tmp/docker-${DOCKER_VER}.tgz" ]] && /bin/mv -f "/tmp/docker-${DOCKER_VER}.tgz" "$BASE/down" [[ -d "/tmp/bin" ]] && /bin/mv -f /tmp/bin/* "$BASE/bin" return 0 } function get_k8s_bin() { [[ -f "$BASE/bin/kubelet" ]] && { logger warn "kubernetes binaries existed"; return 0; } logger info "downloading kubernetes: $K8S_BIN_VER binaries" docker ps -a |grep -q temp_k8s_bin && { logger debug "remove existing container"; docker rm -f temp_k8s_bin; } docker pull easzlab/kubeasz-k8s-bin:"$K8S_BIN_VER" && \ logger debug "run a temporary container" && \ docker run -d --name temp_k8s_bin easzlab/kubeasz-k8s-bin:${K8S_BIN_VER} && \ logger debug "cp k8s binaries" && \ docker cp temp_k8s_bin:/k8s "$BASE/k8s_bin_tmp" && \ /bin/mv -f "$BASE"/k8s_bin_tmp/* "$BASE/bin" && \ logger debug "stop&remove temporary container" && \ docker rm -f temp_k8s_bin && \ rm -rf "$BASE/k8s_bin_tmp" } function get_ext_bin() { [[ -f "$BASE/bin/etcdctl" ]] && { logger warn "extra binaries existed"; return 0; } logger info "downloading extral binaries kubeasz-ext-bin:$EXT_BIN_VER" docker ps -a |grep -q temp_ext_bin && { logger debug "remove existing container"; docker rm -f temp_ext_bin; } docker pull "easzlab/kubeasz-ext-bin:$EXT_BIN_VER" && \ logger debug "run a temporary container" && \ docker run -d --name temp_ext_bin "easzlab/kubeasz-ext-bin:$EXT_BIN_VER" && \ logger debug "cp extral binaries" && \ docker cp temp_ext_bin:/extra "$BASE/extra_bin_tmp" && \ /bin/mv -f "$BASE"/extra_bin_tmp/* "$BASE/bin" && \ logger debug "stop&remove temporary container" && \ docker rm -f temp_ext_bin && \ rm -rf "$BASE/extra_bin_tmp" } function get_sys_pkg() { [[ -f "$BASE/down/packages/chrony_xenial.tar.gz" ]] && { logger warn "system packages existed"; return 0; } docker ps -a |grep -q temp_sys_pkg && { logger debug "remove existing container"; docker rm -f temp_sys_pkg; } logger info "downloading system packages kubeasz-sys-pkg:$SYS_PKG_VER" docker pull "easzlab/kubeasz-sys-pkg:$SYS_PKG_VER" && \ logger debug "run a temporary container" && \ docker run -d --name temp_sys_pkg "easzlab/kubeasz-sys-pkg:$SYS_PKG_VER" && \ logger debug "cp system packages" && \ docker cp temp_sys_pkg:/packages "$BASE/down" && \ logger debug "stop&remove temporary container" && \ docker rm -f temp_sys_pkg } function get_harbor_offline_pkg() { [[ -f "$BASE/down/harbor-offline-installer-$HARBOR_VER.tgz" ]] && { logger warn "harbor-offline existed"; return 0; } docker ps -a |grep -q temp_harbor && { logger debug "remove existing container"; docker rm -f temp_harbor; } logger info "downloading harbor-offline:$HARBOR_VER" docker pull "easzlab/harbor-offline:$HARBOR_VER" && \ logger debug "run a temporary container" && \ docker run -d --name temp_harbor "easzlab/harbor-offline:$HARBOR_VER" && \ logger debug "cp harbor-offline installer package" && \ docker cp "temp_harbor:/harbor-offline-installer-$HARBOR_VER.tgz" "$BASE/down" && \ logger debug "stop&remove temporary container" && \ docker rm -f temp_harbor } function get_offline_image() { logger info "download offline images, then upload to the local registry" # calico if [[ ! -f "$imageDir/calico_$calicoVer.tar" ]];then docker pull "calico/cni:$calicoVer" && \ docker pull "calico/pod2daemon-flexvol:$calicoVer" && \ docker pull "calico/kube-controllers:$calicoVer" && \ docker pull "calico/node:$calicoVer" && \ docker save -o "$imageDir/calico_$calicoVer.tar" "calico/cni:$calicoVer" "calico/kube-controllers:$calicoVer" "calico/node:$calicoVer" "calico/pod2daemon-flexvol:$calicoVer" else docker load -i "$imageDir/calico_$calicoVer.tar" fi docker tag "calico/cni:$calicoVer" "easzlab.io.local:5000/calico/cni:$calicoVer" docker push "easzlab.io.local:5000/calico/cni:$calicoVer" docker tag "calico/pod2daemon-flexvol:$calicoVer" "easzlab.io.local:5000/calico/pod2daemon-flexvol:$calicoVer" docker push "easzlab.io.local:5000/calico/pod2daemon-flexvol:$calicoVer" docker tag "calico/kube-controllers:$calicoVer" "easzlab.io.local:5000/calico/kube-controllers:$calicoVer" docker push "easzlab.io.local:5000/calico/kube-controllers:$calicoVer" docker tag "calico/node:$calicoVer" "easzlab.io.local:5000/calico/node:$calicoVer" docker push "easzlab.io.local:5000/calico/node:$calicoVer" # coredns if [[ ! -f "$imageDir/coredns_$corednsVer.tar" ]];then docker pull "coredns/coredns:$corednsVer" && \ docker save -o "$imageDir/coredns_$corednsVer.tar" "coredns/coredns:$corednsVer" else docker load -i "$imageDir/coredns_$corednsVer.tar" fi docker tag "coredns/coredns:$corednsVer" "easzlab.io.local:5000/coredns/coredns:$corednsVer" docker push "easzlab.io.local:5000/coredns/coredns:$corednsVer" # dns-node-cache if [[ ! -f "$imageDir/k8s-dns-node-cache_$dnsNodeCacheVer.tar" ]];then docker pull "easzlab/k8s-dns-node-cache:$dnsNodeCacheVer" && \ docker save -o "$imageDir/k8s-dns-node-cache_$dnsNodeCacheVer.tar" "easzlab/k8s-dns-node-cache:$dnsNodeCacheVer" else docker load -i "$imageDir/k8s-dns-node-cache_$dnsNodeCacheVer.tar" fi docker tag "easzlab/k8s-dns-node-cache:$dnsNodeCacheVer" "easzlab.io.local:5000/easzlab/k8s-dns-node-cache:$dnsNodeCacheVer" docker push "easzlab.io.local:5000/easzlab/k8s-dns-node-cache:$dnsNodeCacheVer" # dashboard if [[ ! -f "$imageDir/dashboard_$dashboardVer.tar" ]];then docker pull "kubernetesui/dashboard:$dashboardVer" && \ docker save -o "$imageDir/dashboard_$dashboardVer.tar" "kubernetesui/dashboard:$dashboardVer" else docker load -i "$imageDir/dashboard_$dashboardVer.tar" fi docker tag "kubernetesui/dashboard:$dashboardVer" "easzlab.io.local:5000/kubernetesui/dashboard:$dashboardVer" docker push "easzlab.io.local:5000/kubernetesui/dashboard:$dashboardVer" # flannel if [[ ! -f "$imageDir/flannel_$flannelVer.tar" ]];then docker pull "easzlab/flannel:$flannelVer" && \ docker save -o "$imageDir/flannel_$flannelVer.tar" "easzlab/flannel:$flannelVer" else docker load -i "$imageDir/flannel_$flannelVer.tar" fi docker tag "easzlab/flannel:$flannelVer" "easzlab.io.local:5000/easzlab/flannel:$flannelVer" docker push "easzlab.io.local:5000/easzlab/flannel:$flannelVer" # dashboard-metrics-scraper if [[ ! -f "$imageDir/metrics-scraper_$dashboardMetricsScraperVer.tar" ]];then docker pull "kubernetesui/metrics-scraper:$dashboardMetricsScraperVer" && \ docker save -o "$imageDir/metrics-scraper_$dashboardMetricsScraperVer.tar" "kubernetesui/metrics-scraper:$dashboardMetricsScraperVer" else docker load -i "$imageDir/metrics-scraper_$dashboardMetricsScraperVer.tar" fi docker tag "kubernetesui/metrics-scraper:$dashboardMetricsScraperVer" "easzlab.io.local:5000/kubernetesui/metrics-scraper:$dashboardMetricsScraperVer" docker push "easzlab.io.local:5000/kubernetesui/metrics-scraper:$dashboardMetricsScraperVer" # metrics-server if [[ ! -f "$imageDir/metrics-server_$metricsVer.tar" ]];then docker pull "easzlab/metrics-server:$metricsVer" && \ docker save -o "$imageDir/metrics-server_$metricsVer.tar" "easzlab/metrics-server:$metricsVer" else docker load -i "$imageDir/metrics-server_$metricsVer.tar" fi docker tag "easzlab/metrics-server:$metricsVer" "easzlab.io.local:5000/easzlab/metrics-server:$metricsVer" docker push "easzlab.io.local:5000/easzlab/metrics-server:$metricsVer" # pause if [[ ! -f "$imageDir/pause_$pauseVer.tar" ]];then docker pull "easzlab/pause:$pauseVer" && \ docker save -o "$imageDir/pause_$pauseVer.tar" "easzlab/pause:$pauseVer" else docker load -i "$imageDir/pause_$pauseVer.tar" fi docker tag "easzlab/pause:$pauseVer" "easzlab.io.local:5000/easzlab/pause:$pauseVer" docker push "easzlab.io.local:5000/easzlab/pause:$pauseVer" # nfs-provisioner if [[ ! -f "$imageDir/nfs-provisioner_$nfsProvisionerVer.tar" ]];then docker pull "easzlab/nfs-subdir-external-provisioner:$nfsProvisionerVer" && \ docker save -o "$imageDir/nfs-provisioner_$nfsProvisionerVer.tar" "easzlab/nfs-subdir-external-provisioner:$nfsProvisionerVer" else docker load -i "$imageDir/nfs-provisioner_$nfsProvisionerVer.tar" fi docker tag "easzlab/nfs-subdir-external-provisioner:$nfsProvisionerVer" "easzlab.io.local:5000/easzlab/nfs-subdir-external-provisioner:$nfsProvisionerVer" docker push "easzlab.io.local:5000/easzlab/nfs-subdir-external-provisioner:$nfsProvisionerVer" # kubeasz if [[ ! -f "$imageDir/kubeasz_$KUBEASZ_VER.tar" ]];then docker pull "easzlab/kubeasz:$KUBEASZ_VER" && \ docker save -o "$imageDir/kubeasz_$KUBEASZ_VER.tar" "easzlab/kubeasz:$KUBEASZ_VER" else docker load -i "$imageDir/kubeasz_$KUBEASZ_VER.tar" fi } function download_all() { mkdir -p /opt/kube/bin "$BASE/down" "$BASE/bin" download_docker && \ install_docker && \ get_kubeasz && \ get_k8s_bin && \ get_ext_bin && \ start_local_registry && \ get_offline_image } function start_local_registry() { docker ps -a --format="{{ .Names }}"|grep registry > /dev/null 2>&1 && \ { logger warn "registry is already running"; return 0; } if [[ ! -f "$imageDir/registry-2.tar" ]];then docker pull "registry:2" && \ docker save -o "$imageDir/registry-2.tar" "registry:2" fi logger info "start local registry ..." docker load -i "$imageDir/registry-2.tar" > /dev/null mkdir -p /opt/kube/registry docker run -d \ --name registry \ --network host \ --restart always \ --volume /opt/kube/registry:/var/lib/registry \ registry:2 sed -i "/easzlab.io.local/d" /etc/hosts echo "127.0.0.1 easzlab.io.local" >> /etc/hosts } function start_kubeasz_docker() { [[ -d "$BASE/roles/kube-node" ]] || { logger error "not initialized. try 'ezdown -D' first."; exit 1; } logger info "try to run kubeasz in a container" # get host's IP host_if=$(ip route|grep default|head -n1|cut -d' ' -f5) host_ip=$(ip a|grep "$host_if$"|head -n1|awk '{print $2}'|cut -d'/' -f1) logger debug "get host IP: $host_ip" # allow ssh login using key locally if [[ ! -e /root/.ssh/id_rsa ]]; then logger debug "generate ssh key pair" ssh-keygen -t rsa -b 2048 -N '' -f /root/.ssh/id_rsa > /dev/null cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys ssh-keyscan -t ecdsa -H "$host_ip" >> /root/.ssh/known_hosts fi # create a link '/usr/bin/python' in Ubuntu1604 if [[ ! -e /usr/bin/python && -e /usr/bin/python3 ]]; then logger debug "create a soft link '/usr/bin/python'" ln -s /usr/bin/python3 /usr/bin/python fi # run kubeasz docker container docker run --detach \ --env HOST_IP="$host_ip" \ --name kubeasz \ --network host \ --restart always \ --volume "$BASE":"$BASE" \ --volume /root/.kube:/root/.kube \ --volume /root/.ssh:/root/.ssh \ --volume /etc/docker:/etc/docker \ easzlab/kubeasz:${KUBEASZ_VER} sleep 36000 } function clean_container() { logger info "clean all running containers" docker ps -a|awk 'NR>1{print $1}'|xargs docker rm -f } ### Main Lines ################################################## function main() { BASE="/etc/kubeasz" imageDir="$BASE/down" # check if use bash shell readlink /proc/$$/exe|grep -q "bash" || { logger error "you should use bash shell, not sh"; exit 1; } # check if use with root [[ "$EUID" -ne 0 ]] && { logger error "you should run this script as root"; exit 1; } [[ "$#" -eq 0 ]] && { usage >&2; exit 1; } ACTION="" while getopts "CDPRSd:e:k:m:p:z:" OPTION; do case "$OPTION" in C) ACTION="clean_container" ;; D) ACTION="download_all" ;; P) ACTION="get_sys_pkg" ;; R) ACTION="get_harbor_offline_pkg" ;; S) ACTION="start_kubeasz_docker" ;; d) DOCKER_VER="$OPTARG" ;; e) EXT_BIN_VER="$OPTARG" ;; k) K8S_BIN_VER="$OPTARG" ;; m) REGISTRY_MIRROR="$OPTARG" ;; p) SYS_PKG_VER="$OPTARG" ;; z) KUBEASZ_VER="$OPTARG" ;; ?) usage exit 1 ;; esac done [[ "$ACTION" == "" ]] && { logger error "illegal option"; usage; exit 1; } # excute cmd "$ACTION" logger info "Action begin: $ACTION" ${ACTION} || { logger error "Action failed: $ACTION"; return 1; } logger info "Action successed: $ACTION" } main "$@"