kubeasz/ezctl

#!/bin/bash
#  Create & manage k8s clusters by 'kubeasz'

set -o nounset
set -o errexit
#set -o xtrace

function usage() {
    echo -e "\033[33mUsage:\033[0m ezctl COMMAND [args]"
    cat <<EOF
-------------------------------------------------------------------------------------
Cluster setups:
    list		             to list all of the managed clusters
    new         <cluster>            to start a new k8s deploy with name 'cluster'
    setup       <cluster>  <step>    to setup a cluster, also supporting a step-by-step way
    start       <cluster>            to start all of the k8s services stopped by 'ezctl stop'
    stop        <cluster>            to stop all of the k8s services temporarily
    upgrade     <cluster>            to upgrade the k8s cluster
    destroy     <cluster>            to destroy the k8s cluster
    start-aio		             to quickly setup an all-in-one cluster with 'default' settings

Cluster ops:
    add-etcd    <cluster>  <ip>      to add a etcd-node to the etcd cluster
    add-master  <cluster>  <ip>      to add a master node to the k8s cluster
    add-node    <cluster>  <ip>      to add a work node to the k8s cluster
    del-etcd    <cluster>  <ip>      to delete a etcd-node from the etcd cluster
    del-master  <cluster>  <ip>      to delete a master node from the k8s cluster
    del-node    <cluster>  <ip>      to delete a work node from the k8s cluster

Use "ezctl help <command>" for more information about a given command.
EOF
}

function usage-setup(){
  echo -e "\033[33mUsage:\033[0m ezctl setup <cluster> <step>"
  cat <<EOF
available steps:
    01     prepare         to prepare CA/certs & kubeconfig & other system settings 
    02     etcd            to setup the etcd cluster
    03     runtime         to setup the container runtime(docker or containerd)
    04     kube-master     to setup the master nodes
    05     kube-node       to setup the worker nodes
    06     network         to setup the network plugin
    07     cluster-addon   to setup other useful plugins
    all                    to run 01~07 all at once

examples: ./ezctl setup test-k8s 01
          ./ezctl setup test-k8s 02
          ./ezctl setup test-k8s all
EOF
}

function logger() {
  TIMESTAMP=$(date +'%Y-%m-%d %H:%M:%S')
  case "$1" in
    debug)
      echo -e "$TIMESTAMP \033[36mDEBUG\033[0m $2"
      ;;
    info)
      echo -e "$TIMESTAMP \033[32mINFO\033[0m $2"
      ;;
    warn)
      echo -e "$TIMESTAMP \033[33mWARN\033[0m $2"
      ;;
    error)
      echo -e "$TIMESTAMP \033[31mERROR\033[0m $2"
      ;;
    *)
      ;;
  esac
}

function help-info() {
    case "$1" in
        (setup)
	    usage-setup
            ;;
        (add-etcd)
            echo -e "Usage: easzctl add-etcd <new_etcd_ip>\n\nread 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-etcd.md'"
            ;;
        (add-master)
            echo -e "Usage: easzctl add-master <new_master_ip>\n\nread 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-master.md'"
            ;;
        (add-node)
            echo -e "Usage: easzctl add-node <new_node_ip>\n\nread 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-node.md'"
            ;;
        (del-etcd)
            echo -e "Usage: easzctl del-etcd <etcd_ip>\n\nread 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-etcd.md'"
            ;;
        (del-master)
            echo -e "Usage: easzctl del-master <master_ip>\n\nread 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-master.md'"
            ;;
        (del-node)
            echo -e "Usage: easzctl del-node <node_ip>\n\nread 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-node.md'"
            ;;
        (*)
            usage
            return 0
            ;;
    esac
}

### Cluster setups functions ##############################

function new() {
    # check if already existed
    [[ -d "clusters/$1" ]] && { logger error "cluster: $1 already existed"; exit 1; }
    [[ "$1" == default ]] && { logger error "name 'default' is reserved"; exit 1; }

    logger debug "generate custom cluster files in clusters/$1"
    mkdir -p "clusters/$1"
    cp example/hosts.multi-node "clusters/$1/hosts"
    sed -i "s/_cluster_name_/$1/g" "clusters/$1/hosts"
    cp example/config.yml "clusters/$1/config.yml"

    logger debug "set version of common plugins"
    calicoVer=$(grep 'calicoVer=' ezdown|cut -d'=' -f2)
    ciliumVer=$(grep 'ciliumVer=' ezdown|cut -d'=' -f2)
    flannelVer=$(grep 'flannelVer=' ezdown|cut -d'=' -f2)
    kubeRouterVer=$(grep 'kubeRouterVer=' ezdown|cut -d'=' -f2)
    kubeOvnVer=$(grep 'kubeOvnVer=' ezdown|cut -d'=' -f2)
    corednsVer=$(grep 'corednsVer=' ezdown|cut -d'=' -f2)
    dashboardVer=$(grep 'dashboardVer=' ezdown|cut -d'=' -f2)
    dashboardMetricsScraperVer=$(grep 'dashboardMetricsScraperVer=' ezdown|cut -d'=' -f2)
    metricsVer=$(grep 'metricsVer=' ezdown|cut -d'=' -f2)

    sed -i -e "s/__flannel__/$flannelVer/g" \
	   -e "s/__calico__/$calicoVer/g" \
	   -e "s/__cilium__/$ciliumVer/g" \
	   -e "s/__kube_ovn__/$kubeOvnVer/g" \
	   -e "s/__kube_router__/$kubeRouterVer/g" \
	   -e "s/__coredns__/$corednsVer/g" \
	   -e "s/__dashboard__/$dashboardVer/g" \
	   -e "s/__dash_metrics__/$dashboardMetricsScraperVer/g" \
	   -e "s/__metrics__/$metricsVer/g" "clusters/$1/config.yml"


    logger debug "cluster $1: files successfully created."
    logger info "next steps 1: to config 'clusters/$1/hosts'"
    logger info "next steps 2: to config 'clusters/$1/config.yml'"
}

function setup() {
    [[ -d "clusters/$1" ]] || { logger error "invalid config, run 'ezctl new $1' first"; return 1; }
    [[ -f "bin/kube-apiserver" ]] || { logger error "no binaries founded, run 'ezdown -D' fist"; return 1; }

    PLAY_BOOK="dummy.yml"
    case "$2" in
      (01)
          PLAY_BOOK="01.prepare.yml"
          ;;
      (02)
          PLAY_BOOK="02.etcd.yml"
          ;;
      (03)
          PLAY_BOOK="03.runtime.yml"
          ;;
      (04)
          PLAY_BOOK="04.kube-master.yml"
          ;;
      (05)
          PLAY_BOOK="05.kube-node.yml"
          ;;
      (06)
          PLAY_BOOK="06.network.yml"
          ;;
      (07)
          PLAY_BOOK="07.cluster-addon.yml"
          ;;
      (90)
          PLAY_BOOK="90.setup.yml"
          ;;
      (all)
          PLAY_BOOK="90.setup.yml"
          ;;
      (*)
          usage-setup
          exit 1 
          ;;
    esac

    logger info "cluster:$1 setup begins in 5s, press any key to abort:\n"
    ! (read -t5 -n1 ANS) || { logger warn "setup aborted"; return 1; }

    ansible-playbook -i "clusters/$1/hosts" -e "@clusters/$1/config.yml" "playbooks/$PLAY_BOOK" || return 1
}

function cmd() {
    [[ -d "clusters/$1" ]] || { logger error "invalid config, run 'ezctl new $1' first"; return 1; }

    PLAY_BOOK="dummy.yml"
    case "$2" in
      (start)
          PLAY_BOOK="91.start.yml"
          ;;
      (stop)
          PLAY_BOOK="92.stop.yml"
          ;;
      (upgrade)
          PLAY_BOOK="93.upgrade.yml"
          ;;
      (backup)
          PLAY_BOOK="94.backup.yml"
          ;;
      (restore)
          PLAY_BOOK="95.restore.yml"
          ;;
      (destroy)
          PLAY_BOOK="99.clean.yml"
          ;;
      (*)
          usage
          exit 1
          ;;
    esac

    logger info "cluster:$1 $2 begins in 5s, press any key to abort:\n"
    ! (read -t5 -n1 ANS) || { logger warn "$2 aborted"; return 1; }

    ansible-playbook -i "clusters/$1/hosts" -e "@clusters/$1/config.yml" "playbooks/$PLAY_BOOK" || return 1
}


function list() {
    [ -f "$BASEPATH/.cluster/current_cluster" ] || { echo "[ERROR] invalid context, run 'easzctl checkout <cluster_name>' first"; return 1; }
    CLUSTER=$(cat $BASEPATH/.cluster/current_cluster)
    echo -e "\nlist of managed contexts (current: \033[33m$CLUSTER\033[0m)"
    i=1; for c in $(ls $BASEPATH/.cluster/ |grep -Ev "backup|ssl|current_cluster|kubeconfig");
    do
        echo -e "==> context $i:\t$c"
        let "i++"
    done
    echo -e "\nlist of installed clusters (current: \033[33m$CLUSTER\033[0m)"
    i=1; for c in $(ls $BASEPATH/.cluster/ |grep -Ev "backup|ssl|current_cluster|kubeconfig");
    do
        KUBECONF=$BASEPATH/.cluster/$c/config
        if [ -f "$KUBECONF" ]; then
            echo -e "==> cluster $i:\t$c"
            $BASEPATH/bin/kubectl --kubeconfig=$KUBECONF get node
        fi
        let "i++"
    done
}





### in-cluster operation functions ##############################

function add-node() {
    # check new node's address regexp
    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "[ERROR] Invalid ip address!"; return 1; }

    # check if the new node already exsited
    sed -n '/^\[kube-master/,/^\[harbor/p' $BASEPATH/hosts|grep "^$1[^0-9]*$" && { echo "[ERROR] node $1 already existed!"; return 2; }

    # add a node into 'kube-node' group
    sed -i "/\[kube-node/a $1 NEW_NODE=yes ${@:2}" $BASEPATH/hosts

    # check if playbook runs successfully
    ansible-playbook $BASEPATH/tools/02.addnode.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NEW_NODE=yes/d" $BASEPATH/hosts; return 2; }

    # save current cluster context if needed
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}

function add-master() {
    # check new master's address regexp
    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "[ERROR] Invalid ip address!"; return 2; }

    # check if the new master already exsited
    sed -n '/^\[kube-master/,/^\[kube-node/p' $BASEPATH/hosts|grep "^$1[^0-9]*$" && { echo "[ERROR] master $1 already existed!"; return 2; }

    # add a node into 'kube-master' group
    sed -i "/\[kube-master/a $1 NEW_MASTER=yes ${@:2}" $BASEPATH/hosts

    # check if playbook runs successfully
    ansible-playbook $BASEPATH/tools/03.addmaster.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NEW_MASTER=yes/d" $BASEPATH/hosts; return 2; }

    # reconfigure and restart the haproxy service on 'kube-node' nodes
    ansible-playbook $BASEPATH/05.kube-node.yml -t restart_lb -e MASTER_CHG=yes || { echo "[ERROR] Failed to restart the haproxy service on 'kube-node' nodes!"; return 2; }

    # save current cluster context if needed
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}

function add-etcd() {
    # check new node's address regexp
    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "[ERROR] Invalid ip address!"; return 2; }

    # check if the new node already exsited
    sed -n '/^\[etcd/,/^\[kube-master/p' $BASEPATH/hosts|grep "^$1[^0-9]*$" && { echo "[ERROR] etcd $1 already existed!"; return 2; }

    # input an unique NODE_NAME of the node in etcd cluster
    echo "Please input an UNIQUE name(string) for the new node: "
    read -t15 NAME
    sed -n '/^\[etcd/,/^\[kube-master/p' $BASEPATH/hosts|grep "$NAME" && { echo "[ERROR] name [$NAME] already existed!"; return 2; }

    # add a node into 'etcd' group
    sed -i "/\[etcd/a $1 NODE_NAME=$NAME" $BASEPATH/hosts

    # check if playbook runs successfully
    ansible-playbook $BASEPATH/tools/01.addetcd.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NODE_NAME=$NAME/d" $BASEPATH/hosts; return 2; }

    # restart apiservers to use the new etcd cluster
    ansible-playbook $BASEPATH/04.kube-master.yml -t restart_master || { echo "[ERROR] Unexpected failures in master nodes!"; return 2; }

    # save current cluster context if needed
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}

function del-etcd() {
    # check node's address regexp
    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "[ERROR] Invalid ip address!"; return 2; }

    # 
    ansible-playbook $BASEPATH/tools/11.deletcd.yml -e ETCD_TO_DEL=$1 || { echo "[ERROR] Failed to delete etcd node: $1!"; return 2; }

    # restart apiservers to use the new etcd cluster
    ansible-playbook $BASEPATH/04.kube-master.yml -t restart_master || { echo "[ERROR] Unexpected failures in master nodes!"; return 2; }

    # save current cluster context if needed
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}

function del-node() {
    # check node's address regexp
    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "[ERROR] Invalid ip address!"; return 2; }

    # 
    ansible-playbook $BASEPATH/tools/12.delnode.yml -e NODE_TO_DEL=$1 || { echo "[ERROR] Failed to delete 'kube-node': $1!"; return 2; }

    # save current cluster context if needed
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}

function del-master() {
    # check node's address regexp
    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "[ERROR] Invalid ip address!"; return 2; }

    # 
    ansible-playbook $BASEPATH/tools/13.delmaster.yml -e NODE_TO_DEL=$1 || { echo "[ERROR] Failed to delete 'kube-master': $1!"; return 2; }

    # reconfig kubeconfig in ansible manage node
    ansible-playbook $BASEPATH/roles/deploy/deploy.yml -t create_kctl_cfg

    # reconfigure and restart the haproxy service on 'kube-node' nodes
    ansible-playbook $BASEPATH/05.kube-node.yml -t restart_lb -e MASTER_CHG=yes || { echo "[ERROR] Failed to restart the haproxy service on 'kube-node' nodes!"; return 2; }

    # save current cluster context if needed
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}

function upgrade() {
    echo -e "[INFO] prepare the new binaries in advance"
    echo -e "[INFO] upgrade begin in 5s, press any key to abort\n:"
    ! (read -t5 -n1 ANS) || { echo "[WARN] upgrade aborted"; return 1; }
    ansible-playbook -t upgrade_k8s $BASEPATH/22.upgrade.yml || return 1
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}
### cluster-wide operation functions ############################

function save_context() {
    [ -f "$BASEPATH/.cluster/current_cluster" ] || { echo "[WARN] Invalid Context"; return 0; }
    CLUSTER=$(cat $BASEPATH/.cluster/current_cluster)
    echo "[INFO] save context: $CLUSTER"
    echo "[INFO] save $CLUSTER roles' configration"
    for ROLE in $(ls $BASEPATH/roles);
    do
        if [ -d "$BASEPATH/roles/$ROLE/defaults" ]; then
            mkdir -p $BASEPATH/.cluster/$CLUSTER/roles/$ROLE/defaults/
            cp -fpr $BASEPATH/roles/$ROLE/defaults/* $BASEPATH/.cluster/$CLUSTER/roles/$ROLE/defaults/
        fi
    done
    
    if [ -f "$BASEPATH/hosts" ];then
        echo "[INFO] save $CLUSTER ansible hosts"
        cp -fp $BASEPATH/hosts $BASEPATH/.cluster/$CLUSTER/
    fi
    
    if [ -f /root/.kube/config ];then
        echo "[INFO] save $CLUSTER kubeconfig"
        cp -fp /root/.kube/config $BASEPATH/.cluster/$CLUSTER/
    fi

    if [ -f "$BASEPATH/.cluster/kube-proxy.kubeconfig" ];then
        echo "[INFO] save $CLUSTER kube-proxy.kubeconfig"
        cp -fp $BASEPATH/.cluster/kube-proxy.kubeconfig $BASEPATH/.cluster/$CLUSTER/kube-proxy.kubeconfig
    fi
    
    if [ -d "$BASEPATH/.cluster/ssl" ];then
        echo "[INFO] save $CLUSTER certs"
        cp -rfp $BASEPATH/.cluster/ssl $BASEPATH/.cluster/$CLUSTER/ssl
    fi
}

function install_context() {
    [ -f "$BASEPATH/.cluster/current_cluster" ] || { echo "[ERROR] Invalid Context"; return 1; }
    CLUSTER=$(cat $BASEPATH/.cluster/current_cluster)
    echo "[INFO] install context: $CLUSTER"
    echo "[INFO] install $CLUSTER roles' configration"
    for ROLE in $(ls $BASEPATH/.cluster/$CLUSTER/roles);
    do
        cp -fp $BASEPATH/.cluster/$CLUSTER/roles/$ROLE/defaults/* $BASEPATH/roles/$ROLE/defaults/
    done
    
    if [ -f "$BASEPATH/.cluster/$CLUSTER/hosts" ];then
        echo "[INFO] install $CLUSTER ansible hosts"
        cp -fp $BASEPATH/.cluster/$CLUSTER/hosts $BASEPATH/
    fi
    
    if [ -f "$BASEPATH/.cluster/$CLUSTER/config" ];then
        echo "[INFO] install $CLUSTER kubeconfig"
        cp -fp $BASEPATH/.cluster/$CLUSTER/config /root/.kube/
    fi

    if [ -f "$BASEPATH/.cluster/$CLUSTER/kube-proxy.kubeconfig" ];then
        echo "[INFO] install $CLUSTER kube-proxy.kubeconfig"
        cp -fp $BASEPATH/.cluster/$CLUSTER/kube-proxy.kubeconfig $BASEPATH/.cluster/kube-proxy.kubeconfig
    fi
    
    if [ -d "$BASEPATH/.cluster/$CLUSTER/ssl" ];then
        echo "[INFO] install $CLUSTER certs"
        cp -rfp $BASEPATH/.cluster/$CLUSTER/ssl $BASEPATH/.cluster/ssl
    fi
}

function list() {
    [ -f "$BASEPATH/.cluster/current_cluster" ] || { echo "[ERROR] invalid context, run 'easzctl checkout <cluster_name>' first"; return 1; }
    CLUSTER=$(cat $BASEPATH/.cluster/current_cluster)
    echo -e "\nlist of managed contexts (current: \033[33m$CLUSTER\033[0m)"
    i=1; for c in $(ls $BASEPATH/.cluster/ |grep -Ev "backup|ssl|current_cluster|kubeconfig");
    do
        echo -e "==> context $i:\t$c"
        let "i++"
    done
    echo -e "\nlist of installed clusters (current: \033[33m$CLUSTER\033[0m)"
    i=1; for c in $(ls $BASEPATH/.cluster/ |grep -Ev "backup|ssl|current_cluster|kubeconfig");
    do
        KUBECONF=$BASEPATH/.cluster/$c/config
        if [ -f "$KUBECONF" ]; then
            echo -e "==> cluster $i:\t$c"
            $BASEPATH/bin/kubectl --kubeconfig=$KUBECONF get node
        fi
        let "i++"
    done
}

function start-aio(){
    checkout aio
    set +u
    # Check ENV 'HOST_IP', if exist indecates running in a docker container, otherwise running in a host machine
    if [[ -z $HOST_IP ]];then
        # easzctl runs in a host machine, get host's ip
        HOST_IF=$(ip route|grep default|cut -d' ' -f5)
        HOST_IP=$(ip a|grep "$HOST_IF$"|awk '{print $2}'|cut -d'/' -f1)
    fi
    set -u
    cp -f $BASEPATH/example/hosts.allinone $BASEPATH/hosts
    sed -i "s/192.168.1.1/$HOST_IP/g" $BASEPATH/hosts
    setup
}

### extra operation functions ###################################

function basic-auth(){
    OPTIND=2
    CONFIG=$BASEPATH/roles/kube-master/defaults/main.yml
    EX_VARS=""
    while getopts "sSu:p:" OPTION; do
        case $OPTION in
          s)
            EX_VARS="BASIC_AUTH_ENABLE=yes $EX_VARS"
            ENABLED=yes
            ;;
          S)
            grep BASIC_AUTH_ENABLE $CONFIG|grep no > /dev/null && \
            { echo -e "\n[WARN]basic-auth already disabled!\n"; return 1; }
            EX_VARS="BASIC_AUTH_ENABLE=no $EX_VARS"
            ENABLED=no
            ;;
          u)
            EX_VARS="BASIC_AUTH_USER=$OPTARG $EX_VARS"
            sed -i "s/BASIC_AUTH_USER.*$/BASIC_AUTH_USER: '$OPTARG'/g" $CONFIG
            ;;
          p)
            EX_VARS="BASIC_AUTH_PASS=$OPTARG $EX_VARS"
            sed -i "s/BASIC_AUTH_PASS.*$/BASIC_AUTH_PASS: '$OPTARG'/g" $CONFIG
            ;;
          ?)
            help-info basic-auth
            return 1
            ;;
        esac
    done
    
    ansible-playbook $BASEPATH/04.kube-master.yml -t restart_master -e "$EX_VARS" || { return 1; } 
    sed -i "s/BASIC_AUTH_ENABLE.*$/BASIC_AUTH_ENABLE: '$ENABLED'/g" $CONFIG 
    if [[ $ENABLED == yes ]];then
        echo -e "\n[INFO]basic-auth for apiserver is enabled!"
        sed -n '/BASIC_AUTH_USER/p' $CONFIG
        sed -n '/BASIC_AUTH_PASS/p' $CONFIG
    elif [[ $ENABLED == no ]];then
        echo -e "\n[INFO]basic-auth for apiserver is disabled!\n"
    fi
    # save current cluster context if needed
    [ -f "$BASEPATH/.cluster/current_cluster" ] && save_context
    return 0
}

### Main Lines ##################################################
function main() {
  BASEPATH=$(cd `dirname $0`; pwd)
  cd "$BASEPATH"

  # check workdir
  [[ "$BASEPATH" == "/etc/kubeasz" ]] || { logger error "workdir should be '/etc/kubeasz'"; exit 1; }

  # check bash shell
  readlink /proc/$$/exe|grep -q "dash" && { logger error "you should use bash shell only"; exit 1; }

  # check 'ansible' executable
  which ansible > /dev/null 2>&1 || { logger error "need 'ansible', try: 'pip install ansible==2.6.18'"; exit 1; }
  
  [ "$#" -gt 0 ] || { usage >&2; exit 2; }
  
  case "$1" in
      ### in-cluster operations #####################
      (add-etcd)
          [ "$#" -gt 1 ] || { usage >&2; exit 2; }
          ACTION="Action: add a etcd node"
          CMD="add-etcd $2" 
          ;;
      (add-master)
          [ "$#" -gt 1 ] || { usage >&2; exit 2; }
          ACTION="Action: add a k8s master node"
          CMD="add-master $2 ${@:3}" 
          ;;
      (add-node)
          [ "$#" -gt 1 ] || { usage >&2; exit 2; }
          ACTION="Action: add a k8s work node"
          CMD="add-node $2 ${@:3}" 
          ;;
      (del-etcd)
          [ "$#" -gt 1 ] || { usage >&2; exit 2; }
          ACTION="Action: delete a etcd node"
          CMD="del-etcd $2" 
          ;;
      (del-master)
          [ "$#" -gt 1 ] || { usage >&2; exit 2; }
          ACTION="Action: delete a kube-master"
          CMD="del-master $2" 
          ;;
      (del-node)
          [ "$#" -gt 1 ] || { usage >&2; exit 2; }
          ACTION="Action: delete a kube-node"
          CMD="del-node $2" 
          ;;
      ### cluster-wide operations #######################
      (list)
          ACTION="Action: list all of clusters managed"
          CMD="list" 
          ;;
      (new)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          new "$2"
          ;;
      (setup)
          [ "$#" -eq 3 ] || { usage-setup >&2; exit 2; }
          setup "${@:2}"
          ;;
      (start)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          cmd "$2" start
          ;;
      (stop)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          cmd "$2" stop
          ;;
      (upgrade)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          cmd "$2" upgrade
          ;;
      (backup)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          cmd "$2" backup
          ;;
      (restore)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          cmd "$2" restore
          ;;
      (destroy)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          cmd "$2" destroy
          ;;
      (start-aio)
          ACTION="Action: start an AllInOne cluster"
          CMD="start-aio" 
          ;;
      (help)
          [ "$#" -gt 1 ] || { usage >&2; exit 2; }
          help-info $2
          exit 0
          ;;
      ### extra operations ##############################
      (basic-auth)
          [ "$#" -gt 1 ] || { help-info $1; exit 2; }
          ACTION="Action: enable/disable apiserver's basic-auth"
          CMD="basic-auth $*"
          ;;
      (*)
          usage
          exit 0
          ;;
  esac
 }

main "$@"