Update gce CI (#1748)

Use image family for picking latest coreos image
Update python deps

.gitlab-ci.yml

@@ -45,6 +45,7 @@ before_script:
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CLOUD_MACHINE_TYPE: "g1-small"
+  GCE_PREEMPTIBLE: "false"
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   ANSIBLE_CONFIG: ./tests/ansible.cfg
   BOOTSTRAP_OS: none
@@ -99,6 +100,7 @@ before_script:
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e mode=${CLUSTER_MODE}
       -e test_id=${TEST_ID}
+      -e preemptible=$GCE_PREEMPTIBLE
       -e startup_script="'${STARTUP_SCRIPT}'"
 
     # Check out latest tag if testing upgrade
@@ -274,7 +276,7 @@ before_script:
 .coreos_calico_aio_variables: &coreos_calico_aio_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: calico
-  CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817
+  CLOUD_IMAGE: coreos-stable
   CLOUD_REGION: us-west1-b
   CLOUD_MACHINE_TYPE: "n1-standard-2"
   CLUSTER_MODE: aio
@@ -286,7 +288,7 @@ before_script:
 .ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: canal
-  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_IMAGE: ubuntu-1604-lts
   CLOUD_REGION: europe-west1-b
   CLUSTER_MODE: ha
   UPGRADE_TEST: "graceful"
@@ -306,7 +308,7 @@ before_script:
 .ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: canal
-  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_IMAGE:  ubuntu-1604-lts
   CLOUD_MACHINE_TYPE: "n1-standard-1"
   CLOUD_REGION: europe-west1-b
   CLUSTER_MODE: ha
@@ -326,7 +328,7 @@ before_script:
   KUBE_NETWORK_PLUGIN: flannel
   CLOUD_IMAGE: centos-7
   CLOUD_REGION: us-west1-a
-  CLOUD_MACHINE_TYPE: "n1-standard-2"
+  CLOUD_MACHINE_TYPE: "n1-standard-1"
   CLUSTER_MODE: default
   STARTUP_SCRIPT: ""
   
@@ -342,7 +344,7 @@ before_script:
 .coreos_canal_variables: &coreos_canal_variables
 # stage: deploy-gce-part2
   KUBE_NETWORK_PLUGIN: canal
-  CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817
+  CLOUD_IMAGE: coreos-stable
   CLOUD_REGION: us-east1-b
   CLUSTER_MODE: default
   BOOTSTRAP_OS: coreos
@@ -361,7 +363,7 @@ before_script:
 .ubuntu_weave_sep_variables: &ubuntu_weave_sep_variables
 # stage: deploy-gce-special
   KUBE_NETWORK_PLUGIN: weave
-  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_IMAGE: ubuntu-1604-lts
   CLOUD_REGION: us-central1-b
   CLUSTER_MODE: separate
   IDEMPOT_CHECK: "false"
@@ -381,7 +383,7 @@ before_script:
 .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
 # stage: deploy-gce-special
   KUBE_NETWORK_PLUGIN: weave
-  CLOUD_IMAGE: coreos-alpha-1506-0-0-v20170817
+  CLOUD_IMAGE: coreos-alpha
   CLOUD_REGION: us-west1-a
   CLUSTER_MODE: ha-scale
   BOOTSTRAP_OS: coreos
@@ -391,7 +393,7 @@ before_script:
 .ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: flannel
-  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_IMAGE: ubuntu-1604-lts
   CLOUD_REGION: us-central1-b
   CLUSTER_MODE: separate
   ETCD_DEPLOYMENT: rkt
@@ -400,10 +402,10 @@ before_script:
 
 .ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
 # stage: deploy-gce-part1
-  CLOUD_MACHINE_TYPE: "n1-standard-2"
+  CLOUD_MACHINE_TYPE: "n1-standard-1"
   KUBE_NETWORK_PLUGIN: canal
   CERT_MGMT: vault
-  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_IMAGE: ubuntu-1604-lts
   CLOUD_REGION: us-central1-b
   CLUSTER_MODE: separate
   STARTUP_SCRIPT: ""
@@ -411,7 +413,7 @@ before_script:
 .ubuntu_flannel_variables: &ubuntu_flannel_variables
 # stage: deploy-gce-special
   KUBE_NETWORK_PLUGIN: flannel
-  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_IMAGE: ubuntu-1604-lts
   CLOUD_REGION: europe-west1-b
   CLUSTER_MODE: separate
   STARTUP_SCRIPT: ""

requirements.txt

@@ -1,4 +1,4 @@
 pbr>=1.6
-ansible>=2.3.2
+ansible>=2.4.0
 netaddr
 jinja2>=2.9.6

tests/cloud_playbooks/create-gce.yml

@@ -5,7 +5,7 @@
   vars:
     cloud_machine_type: g1-small
     mode: default
-
+    preemptible: no
   tasks:
     - name: replace_test_id
       set_fact:
@@ -25,8 +25,8 @@
       gce:
         instance_names: "{{instance_names}}"
         machine_type: "{{ cloud_machine_type }}"
-        image: "{{ cloud_image }}"
-        preemptible: no
+        image_family: "{{ cloud_image }}"
+        preemptible: "{{ preemptible }}"
         service_account_email: "{{ gce_service_account_email }}"
         pem_file: "{{ gce_pem_file | default(omit)}}"
         credentials_file: "{{gce_credentials_file | default(omit)}}"
@@ -34,6 +34,8 @@
         zone: "{{cloud_region}}"
         metadata: '{"test_id": "{{test_id}}", "network": "{{kube_network_plugin}}", "startup-script": "{{startup_script}}"}'
         tags: "build-{{test_name}},{{kube_network_plugin}}"
+        ip_forward: yes
+        service_account_permissions: ['compute-rw']
       register: gce
 
     - name: Add instances to host group

tests/requirements.txt

@@ -1,5 +1,5 @@
 -r ../requirements.txt
 yamllint
-apache-libcloud==0.20.1
+apache-libcloud==2.2.1
 boto==2.9.0
 tox