kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add complete test integration

pull/138/head
ant31 2016-02-10 11:51:39 +01:00
parent 09851621de
commit caa2555b1d
18 changed files with 1566 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
.travis.yml
View File

@ -1,38 +1,128 @@
sudo: required
dist: trusty
language: python
python: "2.7"
sudo: false
addons:
hosts:
- node1
git:
depth: 5
env:
- SITE=cluster.yml ANSIBLE_VERSION=2.0.0
global:
GCE_USER=travis
SSH_USER=$GCE_USER
TEST_ID=$TRAVIS_JOB_NUMBER
CONTAINER_ENGINE=docker
PRIVATE_KEY=$GCE_PRIVATE_KEY
ANSIBLE_KEEP_REMOTE_FILES=1
matrix:
# Debian Jessie
- >-
KUBE_NETWORK_PLUGIN=flannel
CLOUD_IMAGE=debian-8
CLOUD_REGION=europe-west1-b
- >-
KUBE_NETWORK_PLUGIN=calico
CLOUD_IMAGE=debian-8
CLOUD_REGION=europe-west1-b
install:
# Centos 7
- >-
KUBE_NETWORK_PLUGIN=flannel
CLOUD_IMAGE=centos-7-sudo
CLOUD_REGION=us-central1-c
# - >-
# KUBE_NETWORK_PLUGIN=calico
# CLOUD_IMAGE=centos-7-sudo
# CLOUD_REGION=us-central1-c
# # Redhat 7
# - >-
# KUBE_NETWORK_PLUGIN=flannel
# CLOUD_IMAGE=rhel-7-sudo
# CLOUD_REGION=us-east1-d
# SSH_ARGS='--ssh-extra-args="-t"'
# - >-
# KUBE_NETWORK_PLUGIN=calico
# CLOUD_IMAGE=rhel-7-sudo
# CLOUD_REGION=us-east1-d
# Ubuntu 14.04
- >-
KUBE_NETWORK_PLUGIN=flannel
CLOUD_IMAGE=ubuntu-1404-trusty
CLOUD_REGION=europe-west1-c
- >-
KUBE_NETWORK_PLUGIN=calico
CLOUD_IMAGE=ubuntu-1404-trusty
CLOUD_REGION=europe-west1-c
# Ubuntu 15.10
- >-
KUBE_NETWORK_PLUGIN=flannel
CLOUD_IMAGE=ubuntu-1510-wily
CLOUD_REGION=us-central1-a
- >-
KUBE_NETWORK_PLUGIN=calico
CLOUD_IMAGE=ubuntu-1510-wily
CLOUD_REGION=us-central1-a
matrix:
allow_failures:
- env: KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=us-central1-c
before_install:
# Install Ansible.
- sudo -H pip install ansible==${ANSIBLE_VERSION}
- sudo -H pip install netaddr
- pip install --user boto -U
- pip install --user ansible
- pip install --user netaddr
- pip install --user apache-libcloud
cache:
directories:
- $HOME/releases
- directories:
- $HOME/.cache/pip
- $HOME/.local
before_script:
- export PATH=$PATH:/usr/local/bin
- echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE "
- mkdir -p $HOME/.ssh
- echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa
- echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce
- chmod 400 $HOME/.ssh/id_rsa
- chmod 755 $HOME/.local/bin/ansible-playbook
- $HOME/.local/bin/ansible-playbook --version
- cp tests/ansible.cfg .
# - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
script:
# Check the role/playbook's syntax.
- "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --syntax-check"
# Run the role/playbook with ansible-playbook.
- "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local"
# Run the role/playbook again, checking to make sure it's idempotent.
- >
sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local
| tee /dev/stderr | grep -q 'changed=0.*failed=0'
&& (echo 'Idempotence test: pass' && exit 0)
|| (echo 'Idempotence test: fail' && exit 1)
$HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts -c local $LOG_LEVEL
-e test_id=${TEST_ID}
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-e gce_project_id=${GCE_PROJECT_ID}
-e gce_service_account_email=${GCE_ACCOUNT}
-e gce_pem_file=${HOME}/.ssh/gce
-e cloud_image=${CLOUD_IMAGE}
-e inventory_path=${PWD}/inventory/inventory.ini
-e cloud_region=${CLOUD_REGION}
# Create cluster
- "$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} cluster.yml"
# Tests Cases
## Test Master API
- $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/testcases/010_check-apiserver.yml $LOG_LEVEL
## Create a POD
- $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
## Ping the between 2 pod
- $HOME/.local/bin/ansible-playbook -i setup-kubernetes/inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
after_script:
- >
$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL
-e test_id=${TEST_ID}
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-e gce_project_id=${GCE_PROJECT_ID}
-e gce_service_account_email=${GCE_ACCOUNT}
-e gce_pem_file=${HOME}/.ssh/gce
-e cloud_image=${CLOUD_IMAGE}
-e inventory_path=${PWD}/inventory/inventory.ini
-e cloud_region=${CLOUD_REGION}

4
ansible.cfg 100644
View File

@ -0,0 +1,4 @@
[ssh_connection]
pipelining=True
[defaults]
host_key_checking=False

33
tests/README.md 100644
View File

@ -0,0 +1,33 @@
# k8s-integration-tests
*Work In Progress*
## Test environment variables
### Common
Variable | Description | Required | Default
--------------------- | -------------------------------------- | ---------- | --------
`TEST_ID` | A unique execution ID for this test | Yes |
`KUBE_NETWORK_PLUGIN` | The network plugin (calico or flannel) | Yes |
`PRIVATE_KEY_FILE` | The path to the SSH private key file | No |
### AWS Tests
Variable | Description | Required | Default
--------------------- | ----------------------------------------------- | ---------- | ---------
`AWS_ACCESS_KEY` | The Amazon Access Key ID | Yes |
`AWS_SECRET_KEY` | The Amazon Secret Access Key | Yes |
`AWS_AMI_ID` | The AMI ID to deploy | Yes |
`AWS_KEY_PAIR_NAME` | The name of the EC2 key pair to use | Yes |
`AWS_SECURITY_GROUP` | The EC2 Security Group to use | No | default
`AWS_REGION` | The EC2 region | No | eu-central-1
#### Use private ssh key
##### Key
```bash
openssl pkcs12 -in gce-secure.p12 -passin pass:notasecret -nodes -nocerts | openssl rsa -out gce-secure.pem
cat gce-secure.pem |base64 -w0 > GCE_PEM_FILE`
```

4
tests/ansible.cfg 100644
View File

@ -0,0 +1,4 @@
[ssh_connection]
pipelining=True
[defaults]
host_key_checking=False

33
tests/cloud_playbooks/create-aws.yml 100644
View File

@ -0,0 +1,33 @@
---
- hosts: localhost
sudo: False
gather_facts: False
tasks:
- name: Provision a set of instances
ec2:
key_name: "{{ aws.key_name }}"
aws_access_key: "{{ aws.access_key }}"
aws_secret_key: "{{ aws.secret_key }}"
region: "{{ aws.region }}"
group_id: "{{ aws.group}}"
instance_type: "{{ aws.instance_type}}"
image: "{{ aws.ami_id }}"
wait: true
count: "{{ aws.count }}"
instance_tags: "{{ aws.tags }}"
register: ec2
- name: Template the inventory
template:
src: templates/inventory.ini.j2
dest: "{{ inventory_path }}"
- name: Wait until SSH is available
local_action:
module: wait_for
host: "{{ item.public_ip }}"
port: 22
timeout: 300
state: started
with_items: ec2.instances

33
tests/cloud_playbooks/create-gce.yml 100644
View File

@ -0,0 +1,33 @@
---
- hosts: localhost
sudo: False
gather_facts: no
vars:
cloud_machine_type: g1-small
tasks:
- name: replace_test_id
set_fact:
test_name: "{{test_id |regex_replace('\\.', '-')}}"
- name: Create gce instances
gce:
instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
machine_type: "{{ cloud_machine_type }}"
image: "{{ cloud_image }}"
service_account_email: "{{ gce_service_account_email }}"
pem_file: "{{ gce_pem_file }}"
project_id: "{{ gce_project_id }}"
zone: "{{cloud_region}}"
metadata: '{"test_id": "{{test_id}}", "network": "{{kube_network_plugin}}"}'
tags: "build-{{test_name}},{{kube_network_plugin}}"
register: gce
- name: Template the inventory
template:
src: ../templates/inventory-gce.j2
dest: "{{ inventory_path }}"
- name: Wait for SSH to come up
wait_for: host={{item.public_ip}} port=22 delay=10 timeout=180 state=started
with_items: gce.instance_data

15
tests/cloud_playbooks/delete-aws.yml 100644
View File

@ -0,0 +1,15 @@
---
- hosts: kube-node
sudo: False
tasks:
- name: Gather EC2 facts
action: ec2_facts
- name: Terminate EC2 instances
local_action:
module: ec2
state: absent
instance_ids: "{{ ansible_ec2_instance_id }}"
region: "{{ ansible_ec2_placement_region }}"
wait: True

24
tests/cloud_playbooks/delete-gce.yml 100644
View File

@ -0,0 +1,24 @@
---
- hosts: localhost
sudo: False
gather_facts: no
vars:
cloud_machine_type: f1-micro
tasks:
- name: replace_test_id
set_fact:
test_name: "{{test_id |regex_replace('\\.', '-')}}"
- name: delete gce instances
gce:
instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
machine_type: "{{ cloud_machine_type }}"
image: "{{ cloud_image }}"
service_account_email: "{{ gce_service_account_email }}"
pem_file: "{{ gce_pem_file }}"
project_id: "{{ gce_project_id }}"
zone: "{{cloud_region | default('europe-west1-b')}}"
metadata: '{"test_id": "{{test_id}}", "network": "{{kube_network_plugin}}"}'
state: 'absent'
register: gce

12
tests/local_inventory/host_vars/localhost 100644
View File

@ -0,0 +1,12 @@
aws:
key_name: "{{ key_name | default('ansibl8s') }}"
access_key: "{{ aws_access_key }}"
secret_key: "{{ aws_secret_key }}"
region: "{{ aws_region | default('eu-west-1') }}" # default to eu-west-1
group: "{{ aws_security_group | default ('default')}}"
instance_type: t2.micro
ami_id: "{{ aws_ami_id | default('ami-02724d1f') }}" # default to Debian Jessie
count: 3
tags:
test_id: "{{ test_id }}"
network_plugin: "{{ kube_network_plugin }}"

1
tests/local_inventory/hosts.cfg 100644
View File

@ -0,0 +1 @@
localhost ansible_connection=localx

8
tests/run-tests.sh 100755
View File

@ -0,0 +1,8 @@
#! /bin/bash
# curl -# -C - -o shebang-unit https://raw.github.com/arpinum-oss/shebang-unit/master/releases/shebang-unit
# chmod +x shebang-unit
now=$(date +"%Y%m%d%H%M%S")
mkdir -p ${PWD}/tests-results
./shebang-unit --reporters=simple,junit --output-file=${PWD}/tests-results/junit_report-${now}.xml tests

52
tests/scripts/ansibl8s_test.sh 100644
View File

@ -0,0 +1,52 @@
#! /bin/bash
global_setup() {
git clone https://github.com/ansibl8s/setup-kubernetes.git setup-kubernetes
private_key=""
if [ ! -z ${PRIVATE_KEY_FILE} ]
then
private_key="--private-key=${PRIVATE_KEY_FILE}"
fi
ansible-playbook create.yml -i hosts -u admin -s \
-e test_id=${TEST_ID} \
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN} \
-e aws_access_key=${AWS_ACCESS_KEY} \
-e aws_secret_key=${AWS_SECRET_KEY} \
-e aws_ami_id=${AWS_AMI_ID} \
-e aws_security_group=${AWS_SECURITY_GROUP} \
-e key_name=${AWS_KEY_PAIR_NAME} \
-e inventory_path=${PWD}/inventory.ini \
-e aws_region=${AWS_REGION}
}
global_teardown() {
if [ -f inventory.ini ];
then
ansible-playbook -i inventory.ini -u admin delete.yml
fi
rm -rf ${PWD}/setup-kubernetes
}
should_deploy_cluster() {
ansible-playbook -i inventory.ini -s ${private_key} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml
assertion__status_code_is_success $?
}
should_api_server_respond() {
ansible-playbook -i inventory.ini ${private_key} testcases/check-apiserver.yml
assertion__status_code_is_success $?
}
should_create_pod() {
ansible-playbook -i inventory.ini -s ${private_key} testcases/check-create-pod.yml -vv
assertion__status_code_is_success $?
}
should_pod_be_in_expected_subnet() {
ansible-playbook -i inventory.ini -s ${private_key} testcases/check-network.yml -vv
assertion__status_code_is_success $?
}

1146
tests/shebang-unit 100755
View File

File diff suppressed because it is too large Load Diff

20
tests/templates/inventory-gce.j2 100644
View File

@ -0,0 +1,20 @@
node1 ansible_ssh_host={{gce.instance_data[0].public_ip}}
node2 ansible_ssh_host={{gce.instance_data[1].public_ip}}
node3 ansible_ssh_host={{gce.instance_data[2].public_ip}}
[kube-master]
node1
node2
[kube-node]
node1
node2
node3
[etcd]
node1
node2
[k8s-cluster:children]
kube-node
kube-master

20
tests/templates/inventory.ini.j2 100644
View File

@ -0,0 +1,20 @@
node1 ansible_ssh_host={{ec2.instances[0].public_ip}} ansible_ssh_user=admin
node2 ansible_ssh_host={{ec2.instances[1].public_ip}} ansible_ssh_user=admin
node3 ansible_ssh_host={{ec2.instances[2].public_ip}} ansible_ssh_user=admin
[kube-master]
node1
node2
[kube-node]
node1
node2
node3
[etcd]
node1
node2
[k8s-cluster:children]
kube-node
kube-master

12
tests/testcases/010_check-apiserver.yml 100644
View File

@ -0,0 +1,12 @@
---
- hosts: kube-master
tasks:
- name: Check the API servers are responding
local_action:
module: uri
url: https://{{ansible_ssh_host}}/api/v1
user: kube
password: changeme
validate_certs: no
status_code: 200

13
tests/testcases/020_check-create-pod.yml 100644
View File

@ -0,0 +1,13 @@
---
- hosts: node1
tasks:
- name: Run a replica controller composed of 2 pods
shell: "kubectl run test --image=busybox --replicas=2 --command -- tail -f /dev/null"
- name: Pods are running
shell: "kubectl get pods --no-headers -o json"
register: run_pods_log
until: (run_pods_log.stdout | from_json)['items'] | map(attribute = 'status.phase') | join(',') == "Running,Running"
retries: 24
delay: 5

21
tests/testcases/030_check-network.yml 100644
View File

@ -0,0 +1,21 @@
---
- hosts: node1
tasks:
- name: Get pod names
shell: "kubectl get pods -o json"
register: pods
- set_fact:
pod_names: "{{ (pods.stdout | from_json)['items'] | map(attribute = 'metadata.name') | list }}"
pod_ips: "{{ (pods.stdout | from_json)['items'] | map(attribute = 'status.podIP') | list }}"
- name: Check pods IP are in correct network
assert:
that: item | ipaddr(kube_pods_subnet)
with_items: pod_ips
- name: Ping between pods is working
shell: "kubectl exec {{pod_names[0]}} -- ping -c 4 {{ pod_ips[1] }}"