add support for installation checkpoint

This was taken from the openshift ansible repository here:
https://github.com/leseb/openshift-ansible/tree/master/roles/installer_checkpoint

Rationale:

A complete OpenShift cluster installation is comprised of many different
components which can take 30 minutes to several hours to complete. If
the installation should fail, it could be confusing to understand at
which component the failure occurred. Additionally, it may be desired to
re-run only the component which failed instead of starting over from the
beginning. Components which came after the failed component would also
need to be run individually.

Ceph has a similar situation so we can benefit from that
callback_plugin.

Signed-off-by: Sébastien Han <seb@redhat.com>
pull/2436/head
Sébastien Han 2018-02-28 17:08:07 +01:00 committed by Guillaume Abrioux
parent 04ca685ba7
commit f2e0ceed78
3 changed files with 463 additions and 1 deletions

View File

@ -0,0 +1,139 @@
"""Ansible callback plugin to print a summary completion status of installation
phases.
"""
from datetime import datetime
from ansible.plugins.callback import CallbackBase
from ansible import constants as C
class CallbackModule(CallbackBase):
"""This callback summarizes installation phase status."""
CALLBACK_VERSION = 2.0
CALLBACK_TYPE = 'aggregate'
CALLBACK_NAME = 'installer_checkpoint'
CALLBACK_NEEDS_WHITELIST = False
def __init__(self):
super(CallbackModule, self).__init__()
def v2_playbook_on_stats(self, stats):
# Set the order of the installer phases
installer_phases = [
'installer_phase_ceph_mon',
'installer_phase_ceph_mgr',
'installer_phase_ceph_agent',
'installer_phase_ceph_osd',
'installer_phase_ceph_mds',
'installer_phase_ceph_rgw',
'installer_phase_ceph_nfs',
'installer_phase_ceph_restapi',
'installer_phase_ceph_rbdmirror',
'installer_phase_ceph_client',
'installer_phase_ceph_iscsi_gw',
]
# Define the attributes of the installer phases
phase_attributes = {
'installer_phase_ceph_mon': {
'title': 'Install Ceph Monitor',
'playbook': 'roles/ceph-mon/tasks/main.yml'
},
'installer_phase_ceph_mgr': {
'title': 'Install Ceph Manager',
'playbook': 'roles/ceph-mgr/tasks/main.yml'
},
'installer_phase_ceph_agent': {
'title': 'Install Ceph Agent',
'playbook': 'roles/ceph-agent/tasks/main.yml'
},
'installer_phase_ceph_osd': {
'title': 'Install Ceph OSD',
'playbook': 'roles/ceph-osd/tasks/main.yml'
},
'installer_phase_ceph_mds': {
'title': 'Install Ceph MDS',
'playbook': 'roles/ceph-mds/tasks/main.yml'
},
'installer_phase_ceph_rgw': {
'title': 'Install Ceph RGW',
'playbook': 'roles/ceph-rgw/tasks/main.yml'
},
'installer_phase_ceph_nfs': {
'title': 'Install Ceph NFS',
'playbook': 'roles/ceph-nfs/tasks/main.yml'
},
'installer_phase_ceph_restapi': {
'title': 'Install Ceph REST API',
'playbook': 'roles/ceph-restapi/tasks/main.yml'
},
'installer_phase_ceph_rbdmirror': {
'title': 'Install Ceph RBD Mirror',
'playbook': 'roles/ceph-rbd-mirror/tasks/main.yml'
},
'installer_phase_ceph_client': {
'title': 'Install Ceph Client',
'playbook': 'roles/ceph-client/tasks/main.yml'
},
'installer_phase_ceph_iscsi_gw': {
'title': 'Install Ceph iSCSI Gateway',
'playbook': 'roles/ceph-iscsi-gw/tasks/main.yml'
},
}
# Find the longest phase title
max_column = 0
for phase in phase_attributes:
max_column = max(max_column, len(phase_attributes[phase]['title']))
if '_run' in stats.custom:
self._display.banner('INSTALLER STATUS')
for phase in installer_phases:
phase_title = phase_attributes[phase]['title']
padding = max_column - len(phase_title) + 2
if phase in stats.custom['_run']:
phase_status = stats.custom['_run'][phase]['status']
phase_time = phase_time_delta(stats.custom['_run'][phase])
self._display.display(
'{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time),
color=self.phase_color(phase_status))
if phase_status == 'In Progress' and phase != 'installer_phase_initialize':
self._display.display(
'\tThis phase can be restarted by running: {}'.format(
phase_attributes[phase]['playbook']))
self._display.display("", screen_only=True)
def phase_color(self, status):
""" Return color code for installer phase"""
valid_status = [
'In Progress',
'Complete',
]
if status not in valid_status:
self._display.warning('Invalid phase status defined: {}'.format(status))
if status == 'Complete':
phase_color = C.COLOR_OK
elif status == 'In Progress':
phase_color = C.COLOR_ERROR
else:
phase_color = C.COLOR_WARN
return phase_color
def phase_time_delta(phase):
""" Calculate the difference between phase start and end times """
time_format = '%Y%m%d%H%M%SZ'
phase_start = datetime.strptime(phase['start'], time_format)
if 'end' not in phase:
# The phase failed so set the end time to now
phase_end = datetime.now()
else:
phase_end = datetime.strptime(phase['end'], time_format)
delta = str(phase_end - phase_start).split(".")[0] # Trim microseconds
return delta

View File

@ -63,6 +63,16 @@
- is_atomic
- (ceph_docker_dev_image is undefined or not ceph_docker_dev_image)
- hosts: mons
tasks:
- name: set ceph monitor install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_mon:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: mons
become: True
gather_facts: false
@ -73,74 +83,212 @@
- ceph-mon
serial: 1 # MUST be '1' WHEN DEPLOYING MONITORS ON DOCKER CONTAINERS
- hosts: mons
tasks:
- name: set ceph monitor install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_mon:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: mgrs
become: True
gather_facts: false
pre_tasks:
- name: set ceph manager install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_mgr:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
- { role: ceph-mgr, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
post_tasks:
- name: set ceph manager install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_mgr:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: osds
become: True
gather_facts: false
pre_tasks:
- name: set ceph osd install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_osd:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-osd
post_tasks:
- name: set ceph osd install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_osd:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: mdss
become: True
gather_facts: false
pre_tasks:
- name: set ceph mds install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_mds:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-mds
post_tasks:
- name: set ceph mds install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_mds:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: rgws
become: True
gather_facts: false
pre_tasks:
- name: set ceph rgw install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_rgw:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-rgw
post_tasks:
- name: set ceph rgw install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_rgw:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: nfss
become: True
gather_facts: false
pre_tasks:
- name: set ceph nfs install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_nfs:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
- { role: ceph-nfs, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
post_tasks:
- name: set ceph nfs install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_nfs:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: rbdmirrors
become: True
gather_facts: false
pre_tasks:
- name: set ceph rbd mirror install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_rbdmirror:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-rbd-mirror
post_tasks:
- name: set ceph rbd mirror install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_rbdmirror:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: restapis
become: True
gather_facts: false
pre_tasks:
- name: set ceph rest api install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_restapi:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-restapi
post_tasks:
- name: set ceph rest api install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_restapi:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: clients
become: True
gather_facts: false
pre_tasks:
- name: set ceph client install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_client:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-docker-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-client
post_tasks:
- name: set ceph client install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_client:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

View File

@ -71,99 +71,274 @@
- hosts: mons
gather_facts: false
become: True
pre_tasks:
- name: set ceph monitor install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_mon:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-mon
post_tasks:
- name: set ceph monitor install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_mon:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: mgrs
gather_facts: false
become: True
pre_tasks:
- name: set ceph manager install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_mgr:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
- { role: ceph-mgr, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
post_tasks:
- name: set ceph manager install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_mgr:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: agents
gather_facts: false
become: True
pre_tasks:
- name: set ceph agent install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_agent:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-agent
post_tasks:
- name: set ceph agent install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_agent:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: osds
gather_facts: false
become: True
pre_tasks:
- name: set ceph osd install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_osd:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-osd
post_tasks:
- name: set ceph osd install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_osd:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: mdss
gather_facts: false
become: True
pre_tasks:
- name: set ceph mds install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_mds:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-mds
post_tasks:
- name: set ceph mds install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_mds:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: rgws
gather_facts: false
become: True
pre_tasks:
- name: set ceph rgw install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_rgw:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-rgw
post_tasks:
- name: set ceph rgw install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_rgw:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: nfss
gather_facts: false
become: True
pre_tasks:
- name: set ceph nfs install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_nfs:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
- { role: ceph-nfs, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
post_tasks:
- name: set ceph nfs install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_nfs:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: restapis
gather_facts: false
become: True
pre_tasks:
- name: set ceph rest api install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_restapi:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-restapi
post_tasks:
- name: set ceph rest api install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_restapi:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: rbdmirrors
gather_facts: false
become: True
pre_tasks:
- name: set ceph rbd mirror install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_rbdmirror:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-rbd-mirror
post_tasks:
- name: set ceph rbd mirror install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_rbdmirror:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: clients
gather_facts: false
become: True
pre_tasks:
- name: set ceph client install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_client:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'] }
- ceph-client
post_tasks:
- name: set ceph client install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_client:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
- hosts: iscsi-gws
gather_facts: false
become: True
pre_tasks:
- name: set ceph iscsi gateway install 'In Progress'
run_once: true
set_stats:
data:
installer_phase_ceph_iscsi_gw:
status: "In Progress"
start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
roles:
- { role: ceph-defaults, tags: ['ceph_update_config'] }
- ceph-common
- { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
- { role: ceph-iscsi-gw, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
post_tasks:
- name: set ceph iscsi gw install 'Complete'
run_once: true
set_stats:
data:
installer_phase_ceph_iscsi_gw:
status: "Complete"
end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"