From ad68b23d8a6bbb39892570855768a9c6a63dca93 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Wed, 14 Dec 2016 17:00:38 +0100 Subject: [PATCH] Manual steps for Gitlab CI pipeline * Reduce default testcase to 2 nodes, add HA case. * Adjust gen_matrix script for Travis/Gitlab CIs. * Enable netchecker deploy foro gitlab CI. * Sync other things from travis matrix and reorder them as build steps for pull requests, master branch, auto/manual. * Do auto-step1 from part1 and manual step2,3 for branches/PRs. * Do manual steps from part2, special for master merges. Signed-off-by: Bogdan Dobrelya --- .gitlab-ci.yml | 94 +++++++++++++++------------- .travis.yml.bak | 15 ++--- docs/test_cases.md | 37 ++++++++--- scripts/gen_matrix.sh | 15 ++++- tests/cloud_playbooks/create-gce.yml | 11 +++- tests/cloud_playbooks/delete-gce.yml | 11 +++- tests/templates/inventory-gce.j2 | 17 +++-- 7 files changed, 134 insertions(+), 66 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69e646a67..ef60e447e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - unit-tests - deploy-gce-part1 - deploy-gce-part2 - - deploy-gce-separated + - deploy-gce-special variables: FAILFASTCI_NAMESPACE: 'kargo-ci' @@ -101,6 +101,7 @@ before_script: -e ansible_python_interpreter=${PYPATH} -e download_run_once=true -e download_localhost=true + -e deploy_netchecker=true -e local_release_dir=${PWD}/downloads cluster.yml @@ -109,15 +110,16 @@ before_script: ## Test Master API - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL - ## Create a POD - - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL - ## Ping the between 2 pod - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL + ## Advanced DNS checks + - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL + after_script: - > ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL + -e mode=${CLUSTER_MODE} -e test_id=${TEST_ID} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} -e gce_project_id=${GCE_PROJECT_ID} @@ -127,7 +129,21 @@ before_script: -e inventory_path=${PWD}/inventory/inventory.ini -e cloud_region=${CLOUD_REGION} -debian8-flannel: +coreos-calico: + stage: deploy-gce-part1 + <<: *job + <<: *gce + variables: + <<: *gce_variables + KUBE_NETWORK_PLUGIN: calico + CLOUD_IMAGE: coreos-stable + CLOUD_REGION: us-west1-b + CLUSTER_MODE: separated + BOOTSTRAP_OS: coreos + when: on_success + #only: [/^pr-.*$/] + +debian8-canal: stage: deploy-gce-part1 <<: *job <<: *gce @@ -135,21 +151,10 @@ debian8-flannel: <<: *gce_variables KUBE_NETWORK_PLUGIN: canal CLOUD_IMAGE: debian-8-kubespray - CLOUD_REGION: us-west1-a - CLUSTER_MODE: default - - -coreos-canal: - stage: deploy-gce-part1 - <<: *job - <<: *gce - variables: - <<: *gce_variables - KUBE_NETWORK_PLUGIN: canal - CLOUD_IMAGE: coreos-stable CLOUD_REGION: us-east1-b - CLUSTER_MODE: default - BOOTSTRAP_OS: coreos + CLUSTER_MODE: ha + when: manual + #only: [/^pr-.*$/] rhel7-weave: stage: deploy-gce-part1 @@ -161,6 +166,8 @@ rhel7-weave: CLOUD_IMAGE: rhel-7 CLOUD_REGION: europe-west1-b CLUSTER_MODE: default + when: manual + #only: [/^pr-.*$/] centos7-flannel: stage: deploy-gce-part2 @@ -172,8 +179,8 @@ centos7-flannel: CLOUD_IMAGE: centos-7 CLOUD_REGION: us-west1-a CLUSTER_MODE: default - only: - - master + when: manual + only: ['master'] debian8-calico: stage: deploy-gce-part2 @@ -185,24 +192,25 @@ debian8-calico: CLOUD_IMAGE: debian-8-kubespray CLOUD_REGION: us-central1-b CLUSTER_MODE: default - only: - - master + when: manual + only: ['master'] -centos7-calico: +coreos-canal: stage: deploy-gce-part2 <<: *job <<: *gce variables: <<: *gce_variables - KUBE_NETWORK_PLUGIN: calico - CLOUD_IMAGE: centos-7 - CLOUD_REGION: europe-west1-b + KUBE_NETWORK_PLUGIN: canal + CLOUD_IMAGE: coreos-stable + CLOUD_REGION: us-east1-b CLUSTER_MODE: default - only: - - master + BOOTSTRAP_OS: coreos + when: manual + only: ['master'] rhel7-canal: - stage: deploy-gce-separated + stage: deploy-gce-special <<: *job <<: *gce variables: @@ -211,11 +219,11 @@ rhel7-canal: CLOUD_IMAGE: rhel-7 CLOUD_REGION: us-east1-b CLUSTER_MODE: separated - only: - - master + when: manual + only: ['master'] ubuntu-weave: - stage: deploy-gce-separated + stage: deploy-gce-special <<: *job <<: *gce variables: @@ -224,23 +232,21 @@ ubuntu-weave: CLOUD_IMAGE: ubuntu-1604-xenial CLOUD_REGION: us-central1-b CLUSTER_MODE: separated - only: - - master + when: manual + only: ['master'] -coreos-calico: - stage: deploy-gce-separated +centos7-calico: + stage: deploy-gce-special <<: *job <<: *gce variables: <<: *gce_variables KUBE_NETWORK_PLUGIN: calico - CLOUD_IMAGE: coreos-stable - CLOUD_REGION: us-west1-b - CLUSTER_MODE: separated - BOOTSTRAP_OS: coreos - only: - - master - + CLOUD_IMAGE: centos-7 + CLOUD_REGION: europe-west1-b + CLUSTER_MODE: ha + when: manual + only: ['master'] syntax-check: <<: *job diff --git a/.travis.yml.bak b/.travis.yml.bak index 11e6bc881..7b948dcfe 100644 --- a/.travis.yml.bak +++ b/.travis.yml.bak @@ -24,24 +24,24 @@ env: KUBE_NETWORK_PLUGIN=canal CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=asia-east1-a - CLUSTER_MODE=default + CLUSTER_MODE=ha - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=debian-8-kubespray - CLOUD_REGION=asia-northeast1-b + CLOUD_REGION=europe-west1-c CLUSTER_MODE=default # Centos 7 - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=centos-7 - CLOUD_REGION=europe-west1-c + CLOUD_REGION=asia-northeast1-c CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=centos-7 CLOUD_REGION=us-central1-b - CLUSTER_MODE=default + CLUSTER_MODE=ha # Redhat 7 - >- @@ -55,7 +55,7 @@ env: # KUBE_NETWORK_PLUGIN=weave # CLOUD_IMAGE=coreos-stable # CLOUD_REGION=europe-west1-b - # CLUSTER_MODE=default + # CLUSTER_MODE=ha # BOOTSTRAP_OS=coreos - >- KUBE_NETWORK_PLUGIN=canal @@ -68,7 +68,7 @@ env: - >- KUBE_NETWORK_PLUGIN=canal CLOUD_IMAGE=rhel-7 - CLOUD_REGION=europe-west1-c + CLOUD_REGION=asia-northeast1-b CLUSTER_MODE=separate - >- KUBE_NETWORK_PLUGIN=weave @@ -84,7 +84,7 @@ env: matrix: allow_failures: - - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=default BOOTSTRAP_OS=coreos + - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=ha BOOTSTRAP_OS=coreos before_install: # Install Ansible. @@ -150,6 +150,7 @@ script: after_script: - > $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL + -e mode=${CLUSTER_MODE} -e test_id=${TEST_ID} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} -e gce_project_id=${GCE_PROJECT_ID} diff --git a/docs/test_cases.md b/docs/test_cases.md index 74c6d2bc8..11e4db6aa 100644 --- a/docs/test_cases.md +++ b/docs/test_cases.md @@ -8,19 +8,21 @@ Here is the test matrix for the Travis CI gates: | Network plugin| OS type| GCE region| Nodes layout| |-------------------------|-------------------------|-------------------------|-------------------------| -| canal| debian-8-kubespray| asia-east1-a| default| -| calico| debian-8-kubespray| asia-northeast1-b| default| -| flannel| centos-7| europe-west1-c| default| -| calico| centos-7| us-central1-b| default| +| canal| debian-8-kubespray| asia-east1-a| ha| +| calico| debian-8-kubespray| europe-west1-c| default| +| flannel| centos-7| asia-northeast1-c| default| +| calico| centos-7| us-central1-b| ha| | weave| rhel-7| us-east1-c| default| | canal| coreos-stable| us-west1-b| default| -| canal| rhel-7| europe-west1-c| separate| +| canal| rhel-7| asia-northeast1-b| separate| | weave| ubuntu-1604-xenial| europe-west1-d| separate| | calico| coreos-stable| us-central1-f| separate| -Where the nodes layout `default` is that is given in the example inventory file. -And the `separate` layout is when there is only node of each type, which is a kube master, -compute and etcd cluster member. +Where the nodes layout `default` is a non-HA two nodes setup with the separate `kube-node` +and the `etcd` group merged with the `kube-master`. The `separate` layout is when +there is only node of each type, which is a kube master, compute and etcd cluster member. +And the `ha` layout stands for a two etcd nodes, two masters and a single worker node, +partially intersecting though. Note, the canal network plugin deploys flannel as well plus calico policy controller. @@ -30,4 +32,23 @@ bash scripts/gen_matrix.sh ``` will (hopefully) generate the CI test cases from the current ``.travis.yml``. +Gitlab CI test matrix +===================== +GCE instances +------------- + +| Stage| Network plugin| OS type| GCE region| Nodes layout| When| +|------------------|------------------|------------------|------------------|------------------|------------------| +| part1| calico| coreos-stable| us-west1-b| separated| on_success| +| part1| canal|debian-8-kubespray| us-east1-b| ha| manual| +| part1| weave| rhel-7| europe-west1-b| default| manual| +| part2| flannel| centos-7| us-west1-a| default| manual| +| part2| calico|debian-8-kubespray| us-central1-b| default| manual| +| part2| canal| coreos-stable| us-east1-b| default| manual| +| special| canal| rhel-7| us-east1-b| separated| manual| +| special| weave|ubuntu-1604-xenial| us-central1-b| separated| manual| +| special| calico| centos-7| europe-west1-b| ha| manual| + +The "Stage" means a build step of the build pipeline. The steps are ordered as `part1->part2->special`. +The "When" stands for the build step trigger condition. diff --git a/scripts/gen_matrix.sh b/scripts/gen_matrix.sh index 5cb04647d..c3d9f588a 100644 --- a/scripts/gen_matrix.sh +++ b/scripts/gen_matrix.sh @@ -1,6 +1,7 @@ #!/bin/sh # Generates MD formatted CI matrix from the .travis.yml -a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml) +a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml.bak) +echo Travis-CI printf "|%25s|%25s|%25s|%25s|\n" "Network plugin" "OS type" "GCE region" "Nodes layout" echo "|-------------------------|-------------------------|-------------------------|-------------------------|" c=0 @@ -9,3 +10,15 @@ for i in `echo $a`; do [ $(($c % 4)) -eq 3 ] && printf "|\n" c=$(( c + 1)) done + +echo +a=$(perl -ne '/^\s{2}(stage:\sdeploy-gce-|when:\s)(\S+)$/ && print "$2\n";/^\s{4}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE):\s(\S+)$/ && print "$2\n"' .gitlab-ci.yml) +echo Gitlab-CI +printf "|%18s|%18s|%18s|%18s|%18s|%18s|\n" "Stage" "Network plugin" "OS type" "GCE region" "Nodes layout" "When" +echo "|------------------|------------------|------------------|------------------|------------------|------------------|" +c=0 +for i in `echo $a`; do + printf "|%18s" $i + [ $(($c % 6)) -eq 5 ] && printf "|\n" + c=$(( c + 1)) +done diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml index 6084d2746..fe8bad77a 100644 --- a/tests/cloud_playbooks/create-gce.yml +++ b/tests/cloud_playbooks/create-gce.yml @@ -4,15 +4,24 @@ gather_facts: no vars: cloud_machine_type: g1-small + mode: default tasks: - name: replace_test_id set_fact: test_name: "{{test_id |regex_replace('\\.', '-')}}" + - set_fact: + instance_names: >- + {%- if mode in ['separate', 'ha'] -%} + k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3 + {%- else -%} + k8s-{{test_name}}-1,k8s-{{test_name}}-2 + {%- endif -%} + - name: Create gce instances gce: - instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3" + instance_names: "{{instance_names}}" machine_type: "{{ cloud_machine_type }}" image: "{{ cloud_image }}" service_account_email: "{{ gce_service_account_email }}" diff --git a/tests/cloud_playbooks/delete-gce.yml b/tests/cloud_playbooks/delete-gce.yml index 7705c649e..be2137a3c 100644 --- a/tests/cloud_playbooks/delete-gce.yml +++ b/tests/cloud_playbooks/delete-gce.yml @@ -4,15 +4,24 @@ gather_facts: no vars: cloud_machine_type: f1-micro + mode: default tasks: - name: replace_test_id set_fact: test_name: "{{test_id |regex_replace('\\.', '-')}}" + - set_fact: + instance_names: >- + {%- if mode in ['separate', 'ha'] -%} + k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3 + {%- else -%} + k8s-{{test_name}}-1,k8s-{{test_name}}-2 + {%- endif -%} + - name: delete gce instances gce: - instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3" + instance_names: "{{instance_names}}" machine_type: "{{ cloud_machine_type }}" image: "{{ cloud_image }}" service_account_email: "{{ gce_service_account_email }}" diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 418910771..3ede8f529 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -1,6 +1,8 @@ node1 ansible_ssh_host={{gce.instance_data[0].public_ip}} node2 ansible_ssh_host={{gce.instance_data[1].public_ip}} +{% if mode is defined and mode in ["separate", "ha"] %} node3 ansible_ssh_host={{gce.instance_data[2].public_ip}} +{% endif %} {% if mode is defined and mode == "separate" %} [kube-master] @@ -11,19 +13,26 @@ node2 [etcd] node3 -{% else %} +{% elif mode is defined and mode == "ha" %} [kube-master] node1 node2 [kube-node] -node1 -node2 node3 [etcd] -node1 node2 +node3 +{% else %} +[kube-master] +node1 + +[kube-node] +node2 + +[etcd] +node1 {% endif %} [k8s-cluster:children]