From 3e687bbe9a64b02e0008a6f4b0f44aae0d4bd898 Mon Sep 17 00:00:00 2001 From: Aleksandr Didenko Date: Wed, 16 Nov 2016 11:59:39 +0100 Subject: [PATCH 1/2] Fix download_run_once for containers Add one more step (task) to containers download/upload sequence - copy saved .tar containers to ansible host (delegate_to: localhost). Then upload images to target nodes. It uses synchronize module so if ansible host (localhost) is the same host as kube-master[0] then new task causes no issues and the copy to localhost process is basically skipped. --- .travis.yml | 3 +++ roles/download/tasks/main.yml | 24 +++++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d8ab8f16..0412ee732 100644 --- a/.travis.yml +++ b/.travis.yml @@ -119,6 +119,7 @@ cache: - directories: - $HOME/.cache/pip - $HOME/.local + - /var/tmp/releases before_script: - echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE " @@ -151,6 +152,8 @@ script: -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} -e bootstrap_os=${BOOTSTRAP_OS} -e ansible_python_interpreter=${PYPATH} + -e download_run_once=true + -e local_release_dir=/var/tmp/releases cluster.yml # Tests Cases diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index e715f380d..4edb0b2d0 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -44,7 +44,7 @@ when: "{{ download.enabled|bool and download.container|bool }}" - name: Create dest directory for saved/loaded container images - file: path="{{local_release_dir}}/containers" state=directory recurse=yes + file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755 owner="{{ansible_ssh_user}}" when: "{{ download.enabled|bool and download.container|bool }}" #NOTE(bogdando) this brings no docker-py deps for nodes @@ -78,11 +78,29 @@ run_once: true when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool -- name: Download | get container images +- name: Download | create local directory for saved/loaded container images + file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755 + delegate_to: localhost + become: false + run_once: true + when: "{{ download.enabled|bool and download.container|bool }}" + +- name: Download | copy container images to ansible host synchronize: src: "{{ fname }}" - dest: "{{local_release_dir}}/containers" + dest: "{{ fname }}" + mode: pull + delegate_to: localhost + become: false + when: ansible_os_family != "CoreOS" and inventory_hostname == groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool + +- name: Download | upload container images to nodes + synchronize: + src: "{{ fname }}" + dest: "{{ fname }}" mode: push + delegate_to: localhost + become: false register: get_task until: get_task|success retries: 4 From a03540dabcc20f8df1f1c9b9333eb29400b06249 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Fri, 18 Nov 2016 13:29:03 +0100 Subject: [PATCH 2/2] Add download localhost and enable for CI * Add download_localhost for the download_run_once mode, which is use the ansible host (a travis node for CI case) to store and distribute containers across cluster nodes in inventory. Defaults to false. * Rework download_run_once logic to fix idempotency of uploading containers. * For Travis CI, enable docker images caching and run Travis workers with sudo enabled as a dependency * For Travis CI, deploy with download_localhost and download_run_once enabled to shourten dev path drastically. * Add compression for saved container images. Defaults to 'best'. Signed-off-by: Bogdan Dobrelya Co-authored-by: Aleksandr Didenko --- .travis.yml | 9 ++- .../terraform/openstack/group_vars/all.yml | 2 +- roles/download/defaults/main.yml | 12 +++- roles/download/tasks/main.yml | 55 +++++++++++++------ scripts/collect-info.yaml | 2 +- tests/cloud_playbooks/upload-logs-gcs.yml | 4 +- 6 files changed, 61 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0412ee732..af0bc96f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,7 @@ -sudo: false +sudo: required + +services: + - docker git: depth: 5 @@ -6,6 +9,7 @@ git: env: global: GCE_USER=travis + LOG_LEVEL=-v SSH_USER=$GCE_USER TEST_ID=$TRAVIS_JOB_NUMBER CONTAINER_ENGINE=docker @@ -114,6 +118,8 @@ before_install: # W/A https://github.com/ansible/ansible-modules-core/issues/5196#issuecomment-253766186 - pip install --user apache-libcloud==0.20.1 - pip install --user boto==2.9.0 -U + # Load cached docker images + - if [ -d /var/tmp/releases ]; then find /var/tmp/releases -type f -name "*.tar" | xargs -I {} sh -c "zcat {} | docker load"; fi cache: - directories: @@ -153,6 +159,7 @@ script: -e bootstrap_os=${BOOTSTRAP_OS} -e ansible_python_interpreter=${PYPATH} -e download_run_once=true + -e download_localhost=true -e local_release_dir=/var/tmp/releases cluster.yml diff --git a/contrib/terraform/openstack/group_vars/all.yml b/contrib/terraform/openstack/group_vars/all.yml index 3dbbddbf7..e13628e76 100644 --- a/contrib/terraform/openstack/group_vars/all.yml +++ b/contrib/terraform/openstack/group_vars/all.yml @@ -1,5 +1,5 @@ # Valid bootstrap options (required): ubuntu, coreos, none -bootstrap_os: "none" +bootstrap_os: "none" # Directory where the binaries will be installed bin_dir: /usr/local/bin diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 59a58b3a3..49756a9aa 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -1,8 +1,18 @@ --- local_release_dir: /tmp -# if this is set to true will only download files once +# if this is set to true will only download files once. Doesn't work +# on CoreOS unless the download_localhost is true and localhost +# is running another OS type. Default compress level is 9 (best). download_run_once: False +download_compress: 9 + +# if this is set to true, uses the localhost for download_run_once mode +# (requires docker and sudo to access docker). You may want this option for +# local caching of docker images or for CoreOS cluster nodes. +# Otherwise, uses the first node in the kube-master group to store images +# in the download_run_once mode. +download_localhost: False # Versions kube_version: v1.4.6 diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 4edb0b2d0..66e635935 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -43,19 +43,36 @@ msg: "{{ download.repo }}:{{ download.tag }}" when: "{{ download.enabled|bool and download.container|bool }}" +- set_fact: + download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}" + - name: Create dest directory for saved/loaded container images - file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755 owner="{{ansible_ssh_user}}" + file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755 owner={{ansible_ssh_user}} when: "{{ download.enabled|bool and download.container|bool }}" +# This is required for the download_localhost delegate to work smooth with CoreOS cluster nodes +- name: Hack python binary path for localhost + raw: sh -c "mkdir -p /opt/bin; ln -sf /usr/bin/python /opt/bin/python" + when: "{{ download_delegate == 'localhost' }}" + delegate_to: localhost + run_once: true + +- name: Download | create local directory for saved/loaded container images + file: path="{{local_release_dir}}/containers" state=directory recurse=yes + delegate_to: localhost + become: false + run_once: true + when: "{{ download_run_once|bool and download.enabled|bool and download.container|bool and download_delegate == 'localhost' }}" + #NOTE(bogdando) this brings no docker-py deps for nodes - name: Download containers command: "/usr/bin/docker pull {{ download.repo }}:{{ download.tag }}" register: pull_task_result - until: pull_task_result.rc == 0 + until: pull_task_result|success retries: 4 delay: "{{ retry_stagger | random + 3 }}" when: "{{ download.enabled|bool and download.container|bool }}" - delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}" + delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" - set_fact: @@ -69,21 +86,25 @@ set_fact: container_changed: "{{ not 'up to date' in pull_task_result.stdout }}" when: "{{ download.enabled|bool and download.container|bool }}" - delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}" + delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" -- name: Download | save container images - shell: docker save "{{ download.repo }}:{{ download.tag }}" > "{{ fname }}" - delegate_to: "{{groups['kube-master'][0]}}" - run_once: true - when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool - -- name: Download | create local directory for saved/loaded container images - file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755 - delegate_to: localhost +- name: Stat saved container image + stat: path="{{fname}}" + register: img + changed_when: false + when: "{{ download.enabled|bool and download.container|bool and download_run_once|bool }}" + delegate_to: "{{ download_delegate }}" become: false run_once: true - when: "{{ download.enabled|bool and download.container|bool }}" + +- name: Download | save container images + shell: docker save "{{ download.repo }}:{{ download.tag }}" | gzip -{{ download_compress }} > "{{ fname }}" + delegate_to: "{{ download_delegate }}" + register: saved + run_once: true + become: false + when: (ansible_os_family != "CoreOS" or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool and (container_changed|bool or not img.stat.exists) - name: Download | copy container images to ansible host synchronize: @@ -92,7 +113,7 @@ mode: pull delegate_to: localhost become: false - when: ansible_os_family != "CoreOS" and inventory_hostname == groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool + when: ansible_os_family != "CoreOS" and inventory_hostname == groups['kube-master'][0] and download_delegate != "localhost" and download_run_once|bool and download.enabled|bool and download.container|bool and saved.changed - name: Download | upload container images to nodes synchronize: @@ -105,8 +126,8 @@ until: get_task|success retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool + when: (ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool - name: Download | load container images shell: docker load < "{{ fname }}" - when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool + when: (ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool diff --git a/scripts/collect-info.yaml b/scripts/collect-info.yaml index 6501da995..eb1252d04 100644 --- a/scripts/collect-info.yaml +++ b/scripts/collect-info.yaml @@ -86,7 +86,7 @@ with_items: "{{logs}}" - name: Pack results and logs - local_action: shell GZIP=-9 tar --remove-files -cvzf {{dir|default(".")}}/logs.tar.gz -C /tmp collect-info + local_action: raw GZIP=-9 tar --remove-files -cvzf {{dir|default(".")}}/logs.tar.gz -C /tmp collect-info run_once: true - name: Clean up collected command outputs diff --git a/tests/cloud_playbooks/upload-logs-gcs.yml b/tests/cloud_playbooks/upload-logs-gcs.yml index 80d651ba4..438467440 100644 --- a/tests/cloud_playbooks/upload-logs-gcs.yml +++ b/tests/cloud_playbooks/upload-logs-gcs.yml @@ -8,12 +8,12 @@ tasks: - name: Generate uniq bucket name prefix - shell: date +%Y%m%d + raw: date +%Y%m%d register: out - name: replace_test_id set_fact: - test_name: "kargo-ci-{{ out.stdout }}" + test_name: "kargo-ci-{{ out.stdout_lines[0] }}" - set_fact: file_name: "{{ostype}}-{{kube_network_plugin}}-{{commit}}-logs.tar.gz"