refactor vault role (#2733)

* Move front-proxy-client certs back to kube mount

We want the same CA for all k8s certs

* Refactor vault to use a third party module

The module adds idempotency and reduces some of the repetitive
logic in the vault role

Requires ansible-modules-hashivault on ansible node and hvac
on the vault hosts themselves

Add upgrade test scenario
Remove bootstrap-os tags from tasks

* fix upgrade issues

* improve unseal logic

* specify ca and fix etcd check

* Fix initialization check

bump machine size

.gitlab-ci.yml

@@ -304,6 +304,10 @@ before_script:
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
 
+.coreos_vault_upgrade_variables: &coreos_vault_upgrade_variables
+# stage: deploy-part1
+  UPGRADE_TEST: "basic"
+
 .ubuntu_flannel_variables: &ubuntu_flannel_variables
 # stage: deploy-special
   MOVED_TO_GROUP_VARS: "true"
@@ -560,7 +564,7 @@ gce_rhel7-canal-sep:
     <<: *rhel7_canal_sep_variables
   when: manual
   except: ['triggers']
-  only: ['master', /^pr-.*$/,]
+  only: ['master', /^pr-.*$/]
 
 gce_rhel7-canal-sep-triggers:
   stage: deploy-part2
@@ -638,6 +642,17 @@ gce_ubuntu-vault-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
+gce_coreos-vault-upgrade:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_vault_upgrade_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
 gce_ubuntu-flannel-sep:
   stage: deploy-special
   <<: *job

requirements.txt

@@ -1,4 +1,6 @@
-pbr>=1.6
 ansible>=2.4.0
-netaddr
 jinja2>=2.9.6
+netaddr
+pbr>=1.6
+ansible-modules-hashivault>=3.9.4
+hvac

roles/bootstrap-os/tasks/bootstrap-ubuntu.yml

@@ -8,6 +8,7 @@
   changed_when: false
   with_items:
     - python
+    - python-apt
     - pip
     - dbus-daemon
   tags:

roles/dnsmasq/tasks/main.yml

@@ -3,15 +3,11 @@
   file:
     path: /etc/dnsmasq.d
     state: directory
-  tags:
-    - bootstrap-os
 
 - name: ensure dnsmasq.d-available directory exists
   file:
     path: /etc/dnsmasq.d-available
     state: directory
-  tags:
-    - bootstrap-os
 
 - name: check system nameservers
   shell: awk '/^nameserver/ {print $NF}' /etc/resolv.conf

roles/download/defaults/main.yml

@@ -37,7 +37,7 @@ calico_rr_version: "v0.4.2"
 flannel_version: "v0.10.0"
 flannel_cni_version: "v0.3.0"
 istio_version: "0.2.6"
-vault_version: 0.8.1
+vault_version: 0.10.1
 weave_version: 2.3.0
 pod_infra_version: 3.0
 contiv_version: 1.1.7

roles/etcd/defaults/main.yml

@@ -50,7 +50,7 @@ etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr',
 
 etcd_compaction_retention: "8"
 
-etcd_vault_mount_path: etcd
+etcd_vault_mount_path: "/etcd"
 
 # Force clients like etcdctl to use TLS certs (different than peer security)
 etcd_secure_client: true

roles/kubernetes/master/defaults/main.yml

@@ -100,3 +100,6 @@ kube_encrypt_secret_data: false
 kube_encrypt_token: "{{ lookup('password', inventory_dir + '/credentials/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}"
 # Must be either: aescbc, secretbox or aesgcm
 kube_encryption_algorithm: "aescbc"
+
+# You may want to use ca.pem depending on your situation
+kube_front_proxy_ca: "front-proxy-ca.pem"

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -117,8 +117,13 @@ spec:
     - --feature-gates={{ kube_feature_gates|join(',') }}
 {% endif %}
 {% if kube_version | version_compare('v1.9', '>=') %}
-    - --requestheader-client-ca-file={{ kube_cert_dir }}/front-proxy-ca.pem
+    - --requestheader-client-ca-file={{ kube_cert_dir }}/{{ kube_front_proxy_ca }}
+{# FIXME(mattymo): Vault certs do not work with front-proxy-client #}
+{% if cert_management == "vault" %}
+    - --requestheader-allowed-names=
+{% else %}
     - --requestheader-allowed-names=front-proxy-client
+{% endif %}
     - --requestheader-extra-headers-prefix=X-Remote-Extra-
     - --requestheader-group-headers=X-Remote-Group
     - --requestheader-username-headers=X-Remote-User

roles/kubernetes/node/defaults/main.yml

@@ -128,13 +128,13 @@ vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('')
 
 ## When azure is used, you need to also set the following variables.
 ## see docs/azure.md for details on how to get these values
-#azure_tenant_id:
-#azure_subscription_id:
-#azure_aad_client_id:
-#azure_aad_client_secret:
-#azure_resource_group:
-#azure_location:
-#azure_subnet_name:
-#azure_security_group_name:
-#azure_vnet_name:
-#azure_route_table_name:
+# azure_tenant_id:
+# azure_subscription_id:
+# azure_aad_client_id:
+# azure_aad_client_secret:
+# azure_resource_group:
+# azure_location:
+# azure_subnet_name:
+# azure_security_group_name:
+# azure_vnet_name:
+# azure_route_table_name:

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -13,6 +13,5 @@
 
 - import_tasks: set_resolv_facts.yml
   tags:
-    - bootstrap-os
     - resolvconf
     - facts

roles/kubernetes/secrets/defaults/main.yml

@@ -1,4 +1,3 @@
 ---
 kube_cert_group: kube-cert
-kube_vault_mount_path: kube
-front_proxy_vault_mount_path: front-proxy
+kube_vault_mount_path: "/kube"

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -12,7 +12,6 @@
     - k8s-secrets
     - kube-controller-manager
     - kube-apiserver
-    - bootstrap-os
     - apps
     - network
     - master
@@ -28,7 +27,6 @@
   when: gen_certs|default(false)
   tags:
     - k8s-secrets
-    - bootstrap-os
 
 - name: Gen_certs | write openssl config
   template:

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -127,7 +127,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: front-proxy-client
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
-    issue_cert_mount_path: "{{ front_proxy_vault_mount_path }}"
+    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
   with_items: "{{ kube_front_proxy_clients_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
   notify: set secret_changed

roles/kubernetes/secrets/tasks/main.yml

@@ -41,7 +41,6 @@
     - k8s-secrets
     - kube-controller-manager
     - kube-apiserver
-    - bootstrap-os
     - apps
     - network
     - master
@@ -57,7 +56,6 @@
   when: gen_certs|default(false) or gen_tokens|default(false)
   tags:
     - k8s-secrets
-    - bootstrap-os
 
 - name: "Get_tokens | Make sure the tokens directory exits (on {{groups['kube-master'][0]}})"
   file:

roles/vault/defaults/main.yml

@@ -16,10 +16,11 @@ vault_cert_dir: "{{ vault_base_dir }}/ssl"
 vault_config_dir: "{{ vault_base_dir }}/config"
 vault_roles_dir: "{{ vault_base_dir }}/roles"
 vault_secrets_dir: "{{ vault_base_dir }}/secrets"
+vault_lib_dir: "/var/lib/vault"
 vault_log_dir: "/var/log/vault"
 
-vault_version: 0.8.1
-vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188
+vault_version: 0.10.1
+vault_binary_checksum: 66f0f1b0b221d664dd5913f8697409d7401df4bb2a19c7277e8fbad152063fae
 vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
 vault_download_vars:
   container: "{{ vault_deployment_type != 'host' }}"
@@ -64,10 +65,10 @@ vault_config:
     etcd:
       address: "{{ vault_etcd_url }}"
       ha_enabled: "true"
-      redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}"
-      tls_ca_file: "{{ vault_etcd_cert_dir }}/ca.pem"
-      tls_cert_file: "{{ vault_etcd_cert_dir}}/node-{{ inventory_hostname }}.pem"
-      tls_key_file: "{{ vault_etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem"
+      redirect_addr: "https://{{ inventory_hostname }}:{{ vault_port }}"
+      tls_ca_file: "{{ etcd_cert_dir }}/ca.pem"
+      tls_cert_file: "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem"
+      tls_key_file: "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem"
   cluster_name: "kubernetes-vault"
   default_lease_ttl: "{{ vault_default_lease_ttl }}"
   max_lease_ttl: "{{ vault_max_lease_ttl }}"
@@ -80,6 +81,8 @@ vault_config:
 vault_secret_shares: 1
 vault_secret_threshold: 1
 
+vault_successful_http_codes: ["200", "429", "500", "501", "503"]
+
 vault_ca_options:
   vault:
     common_name: vault
@@ -97,20 +100,29 @@ vault_ca_options:
     format: pem
     ttl: "{{ vault_max_lease_ttl }}"
     exclude_cn_from_sans: true
-  front_proxy:
-    common_name: front-proxy
-    format: pem
-    ttl: "{{ vault_max_lease_ttl }}"
-    exclude_cn_from_sans: true
 
 vault_client_headers:
   Accept: "application/json"
   Content-Type: "application/json"
 
-vault_etcd_cert_dir: /etc/ssl/etcd/ssl
-vault_kube_cert_dir: /etc/kubernetes/ssl
+etcd_cert_dir: /etc/ssl/etcd/ssl
+kube_cert_dir: /etc/kubernetes/ssl
 
 vault_pki_mounts:
+  userpass:
+    name: userpass
+    default_lease_ttl: "{{ vault_default_lease_ttl }}"
+    max_lease_ttl: "{{ vault_max_lease_ttl }}"
+    description: "Userpass"
+    cert_dir: "{{ vault_cert_dir }}"
+    roles:
+      - name: userpass
+        group: userpass
+        password: "{{ lookup('password', inventory_dir + '/credentials/vault/userpass.creds length=15') }}"
+        policy_rules: default
+        role_options:
+          allow_any_name: true
+
   vault:
     name: vault
     default_lease_ttl: "{{ vault_default_lease_ttl }}"
@@ -122,13 +134,14 @@ vault_pki_mounts:
         group: vault
         password: "{{ lookup('password', inventory_dir + '/credentials/vault/vault.creds length=15') }}"
         policy_rules: default
-        role_options: default
+        role_options:
+          allow_any_name: true
   etcd:
     name: etcd
     default_lease_ttl: "{{ vault_default_lease_ttl }}"
     max_lease_ttl: "{{ vault_max_lease_ttl }}"
     description: "Etcd Root CA"
-    cert_dir: "{{ vault_etcd_cert_dir }}"
+    cert_dir: "{{ etcd_cert_dir }}"
     roles:
       - name: etcd
         group: etcd
@@ -143,7 +156,7 @@ vault_pki_mounts:
     default_lease_ttl: "{{ vault_default_lease_ttl }}"
     max_lease_ttl: "{{ vault_max_lease_ttl }}"
     description: "Kubernetes Root CA"
-    cert_dir: "{{ vault_kube_cert_dir }}"
+    cert_dir: "{{ kube_cert_dir }}"
     roles:
       - name: kube-master
         group: kube-master
@@ -153,6 +166,14 @@ vault_pki_mounts:
           allow_any_name: true
           enforce_hostnames: false
           organization: "system:masters"
+      - name: front-proxy-client
+        group: kube-master
+        password: "{{ lookup('password', inventory_dir + '/credentials/vault/kube-proxy.creds length=15') }}"
+        policy_rules: default
+        role_options:
+          allow_any_name: true
+          enforce_hostnames: false
+          organization: "system:front-proxy-client"
       - name: kube-node
         group: k8s-cluster
         password: "{{ lookup('password', inventory_dir + '/credentials/vault/kube-node.creds length=15') }}"
@@ -169,18 +190,3 @@ vault_pki_mounts:
           allow_any_name: true
           enforce_hostnames: false
           organization: "system:node-proxier"
-  front_proxy:
-    name: front-proxy
-    default_lease_ttl: "{{ vault_default_lease_ttl }}"
-    max_lease_ttl: "{{ vault_max_lease_ttl }}"
-    description: "Kubernetes Front Proxy CA"
-    cert_dir: "{{ vault_kube_cert_dir }}"
-    roles:
-      - name: front-proxy-client
-        group: k8s-cluster
-        password: "{{ lookup('password', inventory_dir + '/credentials/vault/front-proxy-client.creds length=15') }}"
-        policy_rules: default
-        role_options:
-          allow_any_name: true
-          enforce_hostnames: false
-          organization: "system:front-proxy"

roles/vault/handlers/main.yml

@@ -0,0 +1,50 @@
+---
+- name: restart vault
+  command: /bin/true
+  notify:
+    - restart vault service
+    - set facts about local Vault health
+    - unseal vault
+
+- name: wait for vault up
+  uri:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: "{{ vault_successful_http_codes | join(',') }}"
+  register: vault_health_check
+  until: vault_health_check|succeeded
+  retries: 10
+  delay: "{{ retry_stagger | random + 3 }}"
+  run_once: yes
+  notify: set facts about local Vault health
+
+- name: wait for vault up nowait
+  uri:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: "{{ vault_successful_http_codes | join(',') }}"
+  register: vault_health_check
+  run_once: yes
+  failed_when: false
+  notify: set facts about local Vault health
+
+- name: set facts about local Vault health
+  set_fact:
+    vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}"
+    vault_cluster_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}"
+
+- name: restart vault service
+  systemd:
+    daemon_reload: true
+    enabled: yes
+    name: vault
+    state: restarted
+
+- name: unseal vault
+  hashivault_unseal:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    keys: "{{ item }}"
+  with_items: "{{ vault_unseal_keys|default([]) }}"

roles/vault/tasks/bootstrap/create_mounts.yml

@@ -1,12 +1,13 @@
 ---
 - include_tasks: ../shared/create_mount.yml
   vars:
-    create_mount_path: "{{ item.name }}"
+    create_mount_path: "/{{ item.name }}"
     create_mount_default_lease_ttl: "{{ item.default_lease_ttl }}"
     create_mount_max_lease_ttl: "{{ item.max_lease_ttl }}"
     create_mount_description: "{{ item.description }}"
     create_mount_cert_dir: "{{ item.cert_dir }}"
     create_mount_config_ca_needed: "{{ item.config_ca }}"
   with_items:
+    - "{{ vault_pki_mounts.userpass|combine({'config_ca': not vault_ca_cert_needed}) }}"
     - "{{ vault_pki_mounts.vault|combine({'config_ca': not vault_ca_cert_needed}) }}"
     - "{{ vault_pki_mounts.etcd|combine({'config_ca': not vault_etcd_ca_cert_needed}) }}"

roles/vault/tasks/bootstrap/create_roles.yml

@@ -6,5 +6,5 @@
     create_role_policy_rules: "{{ item.policy_rules }}"
     create_role_password: "{{ item.password }}"
     create_role_options: "{{ item.role_options }}"
-    create_role_mount_path: "{{ mount.name }}"
+    create_role_mount_path: "/{{ mount.name }}"
   with_items: "{{ mount.roles }}"

roles/vault/tasks/bootstrap/gen_vault_certs.yml

@@ -14,7 +14,7 @@
         {%- endfor -%}
         "127.0.0.1","::1"
         ]
-    issue_cert_mount_path: "{{ vault_pki_mounts.vault.name }}"
+    issue_cert_mount_path: "/{{ vault_pki_mounts.vault.name }}"
     issue_cert_path: "{{ vault_cert_dir }}/api.pem"
     issue_cert_role: "{{ vault_pki_mounts.vault.roles[0].name }}"
     issue_cert_url: "{{ vault_leader_url }}"

roles/vault/tasks/bootstrap/main.yml

@@ -1,4 +1,9 @@
 ---
+- import_tasks: ../shared/check_etcd.yml
+  vars:
+    vault_etcd_needed: no
+  when: inventory_hostname in groups.vault
+
 - import_tasks: ../shared/check_vault.yml
   when: inventory_hostname in groups.vault
 
@@ -23,14 +28,14 @@
   when: not vault_cluster_is_initialized
 
 - import_tasks: create_mounts.yml
-  when: inventory_hostname == groups.vault|first
+  when: inventory_hostname == groups.vault|first and not vault_cluster_is_initialized
 
 - include_tasks: ../shared/auth_backend.yml
   vars:
     auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
     auth_backend_path: userpass
     auth_backend_type: userpass
-  when: inventory_hostname == groups.vault|first
+  when: inventory_hostname == groups.vault|first and not vault_cluster_is_initialized
 
 - include_tasks: create_roles.yml
   with_items:
@@ -38,14 +43,15 @@
     - "{{ vault_pki_mounts.etcd }}"
   loop_control:
     loop_var: mount
-  when: inventory_hostname in groups.vault
+  when: inventory_hostname in groups.vault and not vault_cluster_is_initialized
 
 - include_tasks: ../shared/gen_ca.yml
   vars:
     gen_ca_cert_dir: "{{ vault_pki_mounts.vault.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.vault.name }}"
+    gen_ca_mount_path: "/{{ vault_pki_mounts.vault.name }}"
     gen_ca_vault_headers: "{{ vault_headers }}"
     gen_ca_vault_options: "{{ vault_ca_options.vault }}"
+    gen_ca_copy_group: "kube-master"
   when: >-
         inventory_hostname in groups.vault
         and not vault_cluster_is_initialized
@@ -54,13 +60,13 @@
 - include_tasks: ../shared/gen_ca.yml
   vars:
     gen_ca_cert_dir: "{{ vault_pki_mounts.etcd.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.etcd.name }}"
+    gen_ca_mount_path: "/{{ vault_pki_mounts.etcd.name }}"
     gen_ca_vault_headers: "{{ vault_headers }}"
     gen_ca_vault_options: "{{ vault_ca_options.etcd }}"
     gen_ca_copy_group: "etcd"
-  when: inventory_hostname in groups.etcd and vault_etcd_ca_cert_needed
+  when: inventory_hostname in groups.etcd and not vault_cluster_is_initialized and vault_etcd_ca_cert_needed
 
 - import_tasks: gen_vault_certs.yml
-  when: inventory_hostname in groups.vault and vault_api_cert_needed
+  when: inventory_hostname in groups.vault and not vault_cluster_is_initialized and vault_api_cert_needed
 
 - import_tasks: ca_trust.yml

roles/vault/tasks/bootstrap/start_vault_temp.yml

@@ -12,19 +12,14 @@
            -v /etc/vault:/etc/vault
            {{ vault_image_repo }}:{{ vault_version }} server
 
-# FIXME(mattymo): Crashes on first start with aufs docker storage. See hashicorp/docker-vault#19
 - name: bootstrap/start_vault_temp | Start again single node Vault with file backend
   command: docker start {{ vault_temp_container_name }}
 
 - name: bootstrap/start_vault_temp | Initialize vault-temp
-  uri:
-    url: "http://localhost:{{ vault_port }}/v1/sys/init"
-    headers: "{{ vault_client_headers }}"
-    method: PUT
-    body_format: json
-    body:
-      secret_shares: 1
-      secret_threshold: 1
+  hashivault_init:
+    url: "http://localhost:{{ vault_port }}/"
+    secret_shares: 1
+    secret_threshold: 1
   until: "vault_temp_init|succeeded"
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
@@ -34,16 +29,14 @@
 - name: bootstrap/start_vault_temp | Set needed vault facts
   set_fact:
     vault_leader_url: "http://{{ inventory_hostname }}:{{ vault_port }}"
-    vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}"
-    vault_temp_root_token: "{{ vault_temp_init.json.root_token }}"
-    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}"
+    vault_temp_unseal_keys: "{{ vault_temp_init.keys_base64 }}"
+    vault_root_token: "{{ vault_temp_init.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.root_token}) }}"
 
 - name: bootstrap/start_vault_temp | Unseal vault-temp
-  uri:
-    url: "http://localhost:{{ vault_port }}/v1/sys/unseal"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      key: "{{ item }}"
+  hashivault_unseal:
+    url: "http://localhost:{{ vault_port }}/"
+    token: "{{ vault_root_token }}"
+    keys: "{{ item }}"
   with_items: "{{ vault_temp_unseal_keys|default([]) }}"
+  no_log: true

roles/vault/tasks/bootstrap/sync_etcd_certs.yml

@@ -3,7 +3,7 @@
 - include_tasks: ../shared/sync_file.yml
   vars:
     sync_file: "ca.pem"
-    sync_file_dir: "{{ vault_etcd_cert_dir }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
     sync_file_hosts: "{{ groups.etcd }}"
     sync_file_is_cert: true
 

roles/vault/tasks/bootstrap/sync_secrets.yml

@@ -29,19 +29,23 @@
 - name: bootstrap/sync_secrets | Cat root_token from a vault host
   command: "cat {{ vault_secrets_dir }}/root_token"
   register: vault_root_token_cat
-  when: vault_secrets_available and inventory_hostname == groups.vault|first
+  run_once: yes
+  when: vault_secrets_available
 
 - name: bootstrap/sync_secrets | Cat unseal_keys from a vault host
   command: "cat {{ vault_secrets_dir }}/unseal_keys"
   register: vault_unseal_keys_cat
-  when: vault_secrets_available and inventory_hostname == groups.vault|first
+  run_once: yes
+  when: vault_secrets_available
 
 - name: bootstrap/sync_secrets | Set needed facts for Vault API interaction when Vault is already running
   set_fact:
-    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}"
-    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}"
+    vault_root_token: "{{ vault_root_token_cat.stdout }}"
+    vault_unseal_keys: "{{ vault_unseal_keys_cat.stdout_lines }}"
+  run_once: yes
   when: vault_secrets_available
 
+# FIXME: Remove all uri calls
 - name: bootstrap/sync_secrets | Update vault_headers if we have the root_token
   set_fact:
     vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}"

roles/vault/tasks/bootstrap/sync_vault_certs.yml

@@ -14,6 +14,23 @@
   set_fact:
     sync_file_results: []
 
+# FIXME: Distribute ca.pem alone in a better way
+- include_tasks: ../shared/sync_file.yml
+  vars:
+    sync_file: "ca.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_is_cert: false
+
+- name: bootstrap/sync_vault_certs | Set facts for vault sync_file results
+  set_fact:
+    vault_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
+
+- name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []
+
+
 - include_tasks: ../shared/sync_file.yml
   vars:
     sync_file: "api.pem"

roles/vault/tasks/cluster/binary.yml

@@ -3,7 +3,7 @@
 - name: cluster/binary | Copy vault binary from downloaddir
   copy:
     src: "{{ local_release_dir }}/vault/vault"
-    dest: "/usr/bin/vault"
+    dest: "{{ bin_dir }}/vault"
     remote_src: true
     mode: "0755"
     owner: vault

roles/vault/tasks/cluster/configure.yml

@@ -1,10 +1,19 @@
 ---
-
-- name: cluster/configure | Ensure the vault/config directory exists
+- name: cluster/configure | Ensure the vault directories exist
   file:
-    dest: "{{ vault_config_dir }}"
+    dest: "{{ item }}"
+    owner: vault
     mode: 0750
     state: directory
+    recurse: true
+  with_items:
+    - "{{ vault_base_dir }}"
+    - "{{ vault_cert_dir }}"
+    - "{{ vault_config_dir }}"
+    - "{{ vault_roles_dir }}"
+    - "{{ vault_secrets_dir }}"
+    - "{{ vault_log_dir }}"
+    - "{{ vault_lib_dir }}"
 
 - name: cluster/configure | Lay down the configuration file
   copy:

roles/vault/tasks/cluster/create_mounts.yml

@@ -1,14 +1,13 @@
 ---
 - include_tasks: ../shared/create_mount.yml
   vars:
-    create_mount_path: "{{ item.name }}"
+    create_mount_path: "/{{ item.name }}"
     create_mount_default_lease_ttl: "{{ item.default_lease_ttl }}"
     create_mount_max_lease_ttl: "{{ item.max_lease_ttl }}"
     create_mount_description: "{{ item.description }}"
     create_mount_cert_dir: "{{ item.cert_dir }}"
-    create_mount_config_ca_needed: item.name != vault_pki_mounts.kube.name and item.name != vault_pki_mounts.front_proxy.name
+    create_mount_config_ca_needed: item.name != vault_pki_mounts.kube.name
   with_items:
     - "{{ vault_pki_mounts.vault }}"
     - "{{ vault_pki_mounts.etcd }}"
     - "{{ vault_pki_mounts.kube }}"
-    - "{{ vault_pki_mounts.front_proxy }}"

roles/vault/tasks/cluster/create_roles.yml

@@ -6,5 +6,5 @@
     create_role_password: "{{ item.password }}"
     create_role_policy_rules: "{{ item.policy_rules }}"
     create_role_options: "{{ item.role_options }}"
-    create_role_mount_path: "{{ mount.name }}"
+    create_role_mount_path: "/{{ mount.name }}"
   with_items: "{{ mount.roles }}"

roles/vault/tasks/cluster/init.yml

@@ -1,36 +1,28 @@
 ---
 
+- name: cluster/init | wait for vault
+  command: /bin/true
+  notify: wait for vault up
+
+- meta: flush_handlers
+
 - name: cluster/init | Initialize Vault
-  uri:
-    url: "https://{{ groups.vault|first }}:{{ vault_port }}/v1/sys/init"
-    headers: "{{ vault_client_headers }}"
-    method: POST
-    body_format: json
-    body:
-      secret_shares: "{{ vault_secret_shares }}"
-      secret_threshold: "{{ vault_secret_threshold }}"
-    validate_certs: false
+  hashivault_init:
+    url: "https://localhost:{{ vault_port }}/"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret_shares: "{{ vault_secret_shares }}"
+    secret_threshold: "{{ vault_secret_threshold }}"
+  run_once: true
   register: vault_init_result
-  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
+  when: not vault_cluster_is_initialized
 
 - name: cluster/init | Set facts on the results of the initialization
   set_fact:
-    vault_unseal_keys: "{{ vault_init_result.json['keys'] }}"
-    vault_root_token: "{{ vault_init_result.json.root_token }}"
-    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.json.root_token}) }}"
-  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
-
-- name: cluster/init | Ensure all hosts have these facts
-  set_fact:
-    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys'] }}"
-    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token'] }}"
-  when: not vault_cluster_is_initialized and inventory_hostname != groups.vault|first
-
-- name: cluster/init | Ensure the vault_secrets_dir exists
-  file:
-    mode: 0750
-    path: "{{ vault_secrets_dir }}"
-    state: directory
+    vault_unseal_keys: "{{ vault_init_result.keys_base64 }}"
+    vault_root_token: "{{ vault_init_result.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.root_token}) }}"
+  run_once: true
+  when: not vault_cluster_is_initialized
 
 - name: cluster/init | Ensure all in groups.vault have the unseal_keys locally
   copy:
@@ -48,5 +40,5 @@
 
 - name: cluster/init | Ensure vault_headers and vault statuses are updated
   set_fact:
-    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token})}}"
     vault_cluster_is_initialized: true
+  run_once: true

roles/vault/tasks/cluster/main.yml

@@ -1,8 +1,8 @@
 ---
-- import_tasks: ../shared/check_vault.yml
+- import_tasks: ../shared/check_etcd.yml
   when: inventory_hostname in groups.vault
 
-- import_tasks: ../shared/check_etcd.yml
+- import_tasks: ../shared/check_vault.yml
   when: inventory_hostname in groups.vault
 
 - import_tasks: configure.yml
@@ -14,6 +14,9 @@
 - import_tasks: systemd.yml
   when: inventory_hostname in groups.vault
 
+- import_tasks: ../shared/find_leader.yml
+  when: inventory_hostname in groups.vault
+
 - import_tasks: init.yml
   when: inventory_hostname in groups.vault
 
@@ -29,20 +32,12 @@
 - include_tasks: ../shared/gen_ca.yml
   vars:
     gen_ca_cert_dir: "{{ vault_pki_mounts.kube.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.kube.name }}"
+    gen_ca_mount_path: "/{{ vault_pki_mounts.kube.name }}"
     gen_ca_vault_headers: "{{ vault_headers }}"
     gen_ca_vault_options: "{{ vault_ca_options.kube }}"
     gen_ca_copy_group: "kube-master"
   when: inventory_hostname in groups.vault
 
-- include_tasks: ../shared/gen_ca.yml
-  vars:
-    gen_ca_cert_dir: "{{ vault_pki_mounts.front_proxy.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.front_proxy.name }}"
-    gen_ca_vault_headers: "{{ vault_headers }}"
-    gen_ca_vault_options: "{{ vault_ca_options.front_proxy }}"
-  when: inventory_hostname in groups.vault
-
 - include_tasks: ../shared/auth_backend.yml
   vars:
     auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
@@ -55,7 +50,6 @@
     - "{{ vault_pki_mounts.vault }}"
     - "{{ vault_pki_mounts.etcd }}"
     - "{{ vault_pki_mounts.kube }}"
-    - "{{ vault_pki_mounts.front_proxy }}"
   loop_control:
     loop_var: mount
   when: inventory_hostname in groups.vault

roles/vault/tasks/cluster/systemd.yml

@@ -1,32 +1,11 @@
 ---
-
-- name: cluster/systemd | Ensure mount points exist prior to vault.service startup
-  file:
-    mode: 0750
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - "{{ vault_config_dir }}"
-    - "{{ vault_log_dir }}"
-    - "{{ vault_secrets_dir }}"
-    - /var/lib/vault/
-
-- name: cluster/systemd | Ensure the vault user has access to needed directories
-  file:
-    owner: vault
-    path: "{{ item }}"
-    recurse: true
-  with_items:
-    - "{{ vault_base_dir }}"
-    - "{{ vault_log_dir }}"
-    - /var/lib/vault
-
 - name: cluster/systemd | Copy down vault.service systemd file
   template:
     src: "{{ vault_deployment_type }}.service.j2"
     dest: /etc/systemd/system/vault.service
     backup: yes
   register: vault_systemd_placement
+  notify: restart vault
 
 - name: Create vault service systemd directory
   file:
@@ -39,6 +18,7 @@
     dest: /etc/systemd/system/vault.service.d/http-proxy.conf
     backup: yes
   when: http_proxy is defined or https_proxy is defined
+  notify: restart vault
 
 - name: cluster/systemd | Enable vault.service
   systemd:
@@ -46,13 +26,4 @@
     enabled: yes
     name: vault
     state: started
-
-- name: cluster/systemd | Query local vault until service is up
-  uri:
-    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_client_headers }}"
-    status_code: 200,429,500,501
-  register: vault_health_check
-  until: vault_health_check|succeeded
-  retries: 10
-  delay: "{{ retry_stagger | random + 3 }}"
+  notify: wait for vault up

roles/vault/tasks/cluster/unseal.yml

@@ -1,25 +1,16 @@
 ---
 
 - name: cluster/unseal | Current sealed state
-  debug: " Sealed? {{vault_is_sealed}}"
+  debug:
+    msg: "Sealed? {{ vault_is_sealed }}"
 
 - name: cluster/unseal | Unseal Vault
-  uri:
-    url: "https://localhost:{{ vault_port }}/v1/sys/unseal"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      key: "{{ item }}"
+  hashivault_unseal:
+    url: "https://localhost:{{ vault_port }}/"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    keys: "{{ item }}"
+  no_log: true
   with_items: "{{ vault_unseal_keys|default([]) }}"
+  notify: wait for vault up
   when: vault_is_sealed
-
-- name: cluster/unseal | Wait until server is ready
-  uri:
-    url: "https://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_headers }}"
-    method: HEAD
-    status_code: 200, 429
-  register: vault_node_ready
-  until: vault_node_ready|succeeded
-  retries: 5

roles/vault/tasks/main.yml

@@ -10,6 +10,11 @@
 #    Vault cluster using Etcd as the backend. The same Root CA is mounted as
 #    used during step 1, allowing all certs to have the same chain of trust.
 
+- name: install hvac
+  pip:
+    name: "hvac"
+    state: "present"
+
 ## Bootstrap
 - include_tasks: bootstrap/main.yml
   when: cert_management == 'vault' and vault_bootstrap | d()

roles/vault/tasks/shared/auth_backend.yml

@@ -1,20 +1,10 @@
 ---
-- name: shared/auth_backend | Test if the auth backend exists
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}/tune"
-    headers: "{{ vault_headers }}"
-    validate_certs: false
-  ignore_errors: true
-  register: vault_auth_backend_check
-
-- name: shared/auth_backend | Add the cert auth backend if needed
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      description: "{{ auth_backend_description|d('') }}"
-      type: "{{ auth_backend_type }}"
-    status_code: 204
-  when: vault_auth_backend_check|failed
+- name: shared/auth_backend | Enable auth backend {{ auth_backend_path }}
+  hashivault_auth_enable:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ auth_backend_type }}"
+    mount_point: "{{ auth_backend_path }}"
+    description: "{{ auth_backend_description|d('') }}"
+  register: result

roles/vault/tasks/shared/cert_auth_mount.yml

@@ -10,11 +10,10 @@
         max_lease_ttl: "{{ vault_max_lease_ttl }}"
 
 - name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth-pki/roles/dummy"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body:
-      {'allow_any_name': true}
-    status_code: 204
+  hashivault_approle_role_create:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "auth-pki/roles/dummy"
+    policies:
+      allow_any_name: true

roles/vault/tasks/shared/check_etcd.yml

@@ -4,9 +4,12 @@
   uri:
     url: "{{ vault_etcd_url }}/health"
     validate_certs: no
+    client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
     return_content: yes
   until: vault_etcd_health_check.status == 200 or vault_etcd_health_check.status == 401
-  retries: 10
+  retries: 3
   delay: 2
   delegate_to: "{{groups['etcd'][0]}}"
   run_once: true

roles/vault/tasks/shared/check_vault.yml

@@ -8,24 +8,42 @@
 
 # Check if vault is reachable on the localhost
 - name: check_vault | Attempt to pull local https Vault health
-  uri:
-    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_client_headers }}"
-    status_code: 200,429,500,501,503
-    validate_certs: no
-  ignore_errors: true
-  register: vault_local_service_health
+  command: /bin/true
+  notify: wait for vault up nowait
+
+- meta: flush_handlers
 
 - name: check_vault | Set facts about local Vault health
   set_fact:
-    vault_is_running: "{{ vault_local_service_health|succeeded }}"
-    vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}"
-    vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}"
-    # vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}"
+    vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}"
+
+- name: check_vault | Set facts about local Vault health
+  set_fact:
+    vault_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}"
+    # vault_in_standby: "{{ vault_health_check.get('json', {}).get('standby', true) }}"
     # vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}"
 
+- name: check_vault | Check is vault is initialized in etcd if vault is not running
+  command: |-
+    curl \
+      --cacert {{ etcd_cert_dir }}/ca.pem \
+      --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \
+      --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
+      -X POST -d '{"key": "{{ "/vault/core/seal-config" | b64encode }}"}' \
+      {{ etcd_access_addresses.split(',') | first }}/v3alpha/kv/range
+  register: vault_etcd_exists
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  run_once: true
+  when: not vault_is_running and vault_etcd_available
+  changed_when: false
+
 - name: check_vault | Set fact about the Vault cluster's initialization state
   set_fact:
-    vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}"
+    vault_cluster_is_initialized: >-
+      {{ vault_is_initialized or
+         hostvars[item]['vault_is_initialized'] or
+         'Key not found' not in vault_etcd_exists.stdout|default('Key not found') }}
   with_items: "{{ groups.vault }}"
   run_once: true

roles/vault/tasks/shared/config_ca.yml

@@ -4,26 +4,26 @@
   register: vault_ca_cert_cat
 
 - name: config_ca | Pull current CA cert from Vault
-  uri:
-    url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/ca/pem"
-    headers: "{{ vault_headers }}"
-    return_content: true
-    status_code: 200,204
-    validate_certs: no
+  hashivault_read:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ config_ca_mount_path }}/ca"
+    key: "pem"
   register: vault_pull_current_ca
+  failed_when: false
 
 - name: config_ca | Read root CA key for Vault
   command: "cat {{ config_ca_ca_key }}"
   register: vault_ca_key_cat
-  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip()
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("data","").strip()
 
 - name: config_ca | Configure pki mount to use the found root CA cert and key
-  uri:
-    url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/config/ca"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
+  hashivault_write:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ config_ca_mount_path }}/config/ca"
+    data:
       pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}"
-    status_code: 204
-  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("content","").strip()
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("data","").strip()

roles/vault/tasks/shared/create_role.yml

@@ -1,42 +1,36 @@
 ---
-# The JSON inside JSON here is intentional (Vault API wants it)
-- name: create_role | Create a policy for the new role allowing issuing
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/sys/policy/{{ create_role_name }}"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: PUT
-    body_format: json
-    body:
-      rules: >-
-             {%- if create_role_policy_rules|d("default") == "default" -%}
-             {{
-             { 'path': {
-                 create_role_mount_path + '/issue/' + create_role_name: {'policy': 'write'},
-                 create_role_mount_path + '/roles/' + create_role_name: {'policy': 'read'}
-             }} | to_json + '\n'
-             }}
-             {%- else -%}
-             {{ create_role_policy_rules | to_json + '\n' }}
-             {%- endif -%}
-    status_code: 204
-  delegate_to: "{{ groups.vault|first }}"
-  run_once: true
+- name: create_role | Create a policy for the new role
+  hashivault_policy_set:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ create_role_name }}"
+    rules: >-
+           {%- if create_role_policy_rules|d("default") == "default" -%}
+           {{
+           { 'path': {
+                create_role_mount_path + '/issue/' + create_role_name: {'policy': 'write'},
+                create_role_mount_path + '/roles/' + create_role_name: {'policy': 'read'}
+            }} | to_json + '\n'
+            }}
+            {%- else -%}
+            {{ create_role_policy_rules | to_json + '\n' }}
+            {%- endif -%}
 
 - name: create_role | Create {{ create_role_name }} role in the {{ create_role_mount_path }} pki mount
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/{{ create_role_mount_path }}/roles/{{ create_role_name }}"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body: >-
-          {%- if create_role_options|d("default") == "default" -%}
-          {'allow_any_name': true}
-          {%- else -%}
-          {{ create_role_options }}
-          {%- endif -%}
-    status_code: 204
-  delegate_to: "{{ groups.vault|first }}"
-  run_once: true
+  hashivault_write:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ create_role_mount_path }}/roles/{{ create_role_name }}"
+    data: |
+     {%- if create_role_options|d("default") == "default" -%}
+     {
+     allow_any_name: true
+     }
+     {%- else -%}
+     {{ create_role_options | to_json }}
+     {%- endif -%}
 
 ## Userpass based auth method
 

roles/vault/tasks/shared/find_leader.yml

@@ -5,16 +5,16 @@
     url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
     headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
     method: HEAD
-    status_code: 200,429,503
+    status_code: 200,429,501,503
   register: vault_leader_check
   until: "vault_leader_check|succeeded"
   retries: 10
 
 - name: find_leader | Set fact for current http leader
   set_fact:
-    vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}"
+    vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ inventory_hostname }}:{{ vault_port }}"
   with_items: "{{ groups.vault }}"
-  when: "hostvars[item]['vault_leader_check'].get('status') in [200,503]"
+  when: "hostvars[item]['vault_leader_check'].get('status') in [200,501,503]"
   # run_once: true
 
 - name: find_leader| show vault_leader_url

roles/vault/tasks/shared/gen_ca.yml

@@ -1,35 +1,38 @@
 ---
-- name: "bootstrap/gen_ca | Ensure cert_dir {{ gen_ca_cert_dir }} exists"
+- name: "bootstrap/gen_ca | Ensure cert_dir {{ gen_ca_cert_dir }} exists on necessary hosts"
   file:
     mode: 0755
     path: "{{ gen_ca_cert_dir }}"
     state: directory
+  delegate_to: "{{ item }}"
+  with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}"
 
 - name: "bootstrap/gen_ca | Generate {{ gen_ca_mount_path }} root CA"
-  uri:
-    url: "{{ vault_leader_url }}/v1/{{ gen_ca_mount_path }}/root/generate/exported"
-    headers: "{{ gen_ca_vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ gen_ca_vault_options }}"
-    status_code: 200,204
-  register: vault_ca_gen
-  delegate_to: "{{ groups.vault|first }}"
+  hashivault_write:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ gen_ca_mount_path }}/root/generate/exported"
+    data: "{{ gen_ca_vault_options }}"
   run_once: true
+  no_log: true
+  register: vault_ca_gen
 
 - name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA cert locally"
   copy:
-    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}"
+    content: "{{ vault_ca_gen['data']['data']['certificate'] }}"
     dest: "{{ gen_ca_cert_dir }}/ca.pem"
     mode: 0644
-  when: vault_ca_gen.status == 200
+  when: '"data" in vault_ca_gen.keys()'
+  delegate_to: "{{ item }}"
+  with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}"
 
 
 - name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA key to necessary hosts"
   copy:
-    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}"
+    content: "{{ vault_ca_gen['data']['data']['private_key']}}"
     dest: "{{ gen_ca_cert_dir }}/ca-key.pem"
     mode: 0640
-  when: vault_ca_gen.status == 200
+  when: '"data" in vault_ca_gen.keys()'
   delegate_to: "{{ item }}"
   with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}"

roles/vault/tasks/shared/gen_userpass.yml

@@ -1,16 +1,13 @@
 ---
 - name: shared/gen_userpass | Create the Username/Password combo for the role
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/users/{{ gen_userpass_username }}"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body:
-      username: "{{ gen_userpass_username }}"
-      password: "{{ gen_userpass_password }}"
-      policies: "{{ gen_userpass_role }}"
-    status_code: 204
-  delegate_to: "{{ groups.vault|first }}"
+  hashivault_userpass_create:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ gen_userpass_username }}"
+    pass: "{{ gen_userpass_password }}"
+    policies:
+      - "{{ gen_userpass_role }}"
   run_once: true
 
 - name: shared/gen_userpass | Ensure destination directory exists

roles/vault/tasks/shared/issue_cert.yml

@@ -39,52 +39,58 @@
   delegate_to: "{{ groups.vault|first }}"
   run_once: true
 
-- name: gen_certs_vault | Log into Vault and obtain an token
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ user_vault_creds.username }}"
-    headers:
-      Accept: application/json
-      Content-Type: application/json
-    method: POST
-    body_format: json
-    body:
-      password: "{{ user_vault_creds.password }}"
-  register: vault_login_result
-  delegate_to: "{{ groups.vault|first }}"
+- name: gen_certs_vault | Ensure vault cert dir exists
+  file:
+    path: "{{ vault_cert_dir }}"
+    state: directory
+    recurse: yes
+    owner: "vault"
+    group: "vault"
+    mode: 0755
+
+- name: gen_certs_vault | install hvac
+  pip:
+    name: "hvac"
+    state: "present"
+
+- name: gen_certs_vault | Pull vault CA
+  get_url:
+    url: "{{ issue_cert_url }}/v1/vault/ca/pem"
+    dest: "{{ vault_cert_dir }}/ca.pem"
+    validate_certs: no
+  when: '"https" in issue_cert_url'
+
+- name: gen_certs_vault | Log into Vault and obtain a scoped token
+  hashivault_token_create:
+    url: "{{ issue_cert_url }}"
+    token: "{{ vault_root_token | default(hostvars[groups.vault|first]['vault_root_token']) }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    policies: "{{ user_vault_creds.username }}"
+    display_name: "{{ user_vault_creds.username }}"
+  register: vault_client_token_request
   run_once: true
 
-- name: gen_certs_vault | Set fact for vault_client_token
+- name: gen_certs_vault | Pull token from request
   set_fact:
-    vault_client_token: "{{ vault_login_result.get('json', {}).get('auth', {}).get('client_token') }}"
+    vault_client_token: "{{ vault_client_token_request['token']['auth']['client_token'] }}"
   run_once: true
 
-- name: gen_certs_vault | Set fact for Vault API token
-  set_fact:
-    issue_cert_headers:
-      Accept: application/json
-      Content-Type: application/json
-      X-Vault-Token: "{{ vault_client_token }}"
-  run_once: true
-  when: vault_client_token != ""
-
 - name: "issue_cert | Generate {{ issue_cert_path }} for {{ issue_cert_role }} role"
-  uri:
-    url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount_path|d('pki') }}/issue/{{ issue_cert_role }}"
-    headers: "{{ issue_cert_headers }}"
-    method: POST
-    body_format: json
-    body:
+  hashivault_write:
+    url: "{{ issue_cert_url }}"
+    token: "{{ vault_client_token }}"
+    ca_cert: "{% if 'https' in issue_cert_url %}{{ vault_cert_dir }}/ca.pem{% endif %}"
+    secret: "{{ issue_cert_mount_path|d('/pki') }}/issue/{{ issue_cert_role }}"
+    data:
       alt_names: "{{ issue_cert_alt_names | d([]) | join(',') }}"
       common_name: "{{ issue_cert_common_name | d(issue_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0]) }}"
       format: "{{ issue_cert_format | d('pem') }}"
       ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}"
   register: issue_cert_result
-  delegate_to: "{{ issue_cert_hosts|first }}"
-  run_once: true
 
 - name: "issue_cert | Copy {{ issue_cert_path }} cert to all hosts"
   copy:
-    content: "{{ issue_cert_result['json']['data']['certificate'] }}\n"
+    content: "{{ issue_cert_result['data']['data']['certificate'] }}\n"
     dest: "{{ issue_cert_path }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0644') }}"
@@ -92,7 +98,7 @@
 
 - name: "issue_cert | Copy key for {{ issue_cert_path }} to all hosts"
   copy:
-    content: "{{ issue_cert_result['json']['data']['private_key'] }}"
+    content: "{{ issue_cert_result['data']['data']['private_key'] }}"
     dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0640') }}"
@@ -100,7 +106,7 @@
 
 - name: issue_cert | Copy issuing CA cert
   copy:
-    content: "{{ issue_cert_result['json']['data']['issuing_ca'] }}\n"
+    content: "{{ issue_cert_result['data']['data']['issuing_ca'] }}\n"
     dest: "{{ issue_cert_path | dirname }}/{{ issue_cert_ca_filename | default('ca.pem') }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0644') }}"
@@ -109,7 +115,7 @@
 
 - name: issue_cert | Copy certificate serial to all hosts
   copy:
-    content: "{{ issue_cert_result['json']['data']['serial_number'] }}"
+    content: "{{ issue_cert_result['data']['data']['serial_number'] }}"
     dest: "{{ issue_cert_path.rsplit('.', 1)|first }}.serial"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0640') }}"

roles/vault/tasks/shared/pki_mount.yml

@@ -1,27 +1,12 @@
 ---
-- name: "shared/mount | Test if {{ pki_mount_path }} PKI mount exists"
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}/tune"
-    headers: "{{ vault_headers }}"
-  ignore_errors: true
-  register: vault_pki_mount_check
 
-- name: shared/mount | Set pki mount type
-  set_fact:
-    mount_options: "{{ pki_mount_options | combine({'type': 'pki'}) }}"
-  when: vault_pki_mount_check|failed
-
-- name: shared/mount | Mount {{ pki_mount_path }} PKI mount if needed
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ mount_options|d() }}"
-    status_code: 204
-  when: vault_pki_mount_check|failed
-
-- name: shared/mount | Unset mount options
-  set_fact:
-    mount_options: {}
-  when: vault_pki_mount_check|failed
+- name: shared/mount | Enable {{ pki_mount_path }} PKI mount
+  hashivault_secret_enable:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ pki_mount_path }}"
+    backend: "pki"
+    config: "{{ pki_mount_options }}"
+  register: secret_enable_result
+  failed_when: 'secret_enable_result.rc !=0 and "existing mount" not in secret_enable_result.msg'

roles/vault/tasks/shared/sync.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: "sync_file | Cat the file"
   command: "cat {{ sync_file_path }}"
   register: sync_file_cat

roles/vault/tasks/shared/sync_file.yml

@@ -1,7 +1,6 @@
 ---
 
 # NOTE: This should be a role (or custom module), but currently include_role is too buggy to use
-
 - name: "sync_file | Set facts for directory and file when sync_file_path is defined"
   set_fact:
     sync_file_dir: "{{ sync_file_path | dirname }}"

roles/vault/templates/docker.service.j2

@@ -21,13 +21,15 @@ ExecStart={{ docker_bin_dir }}/docker run \
 --cap-add=IPC_LOCK \
 -v {{ vault_cert_dir }}:{{ vault_cert_dir }} \
 -v {{ vault_config_dir }}:{{ vault_config_dir }} \
--v {{ vault_etcd_cert_dir }}:{{ vault_etcd_cert_dir }} \
+-v /etc/ssl:/etc/ssl \
+-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }} \
 -v {{ vault_log_dir }}:/vault/logs \
 -v {{ vault_roles_dir }}:{{ vault_roles_dir }} \
 -v {{ vault_secrets_dir }}:{{ vault_secrets_dir }} \
 --entrypoint=vault \
 {{ vault_image_repo }}:{{ vault_image_tag }} \
-server --config={{ vault_config_dir }}/config.json
+server --config={{ vault_config_dir }}/config.json \
+--log-level=trace
 
 [Install]
 WantedBy=multi-user.target

roles/vault/templates/rkt.service.j2

@@ -26,8 +26,8 @@ ExecStart=/usr/bin/rkt run \
 --mount=volume=vault-secrets-dir,target={{ vault_secrets_dir }} \
 --volume=vault-roles-dir,kind=host,source={{ vault_roles_dir }} \
 --mount=volume=vault-roles-dir,target={{ vault_roles_dir }} \
---volume=vault-etcd-cert-dir,kind=host,source={{ vault_etcd_cert_dir }} \
---mount=volume=vault-etcd-cert-dir,target={{ vault_etcd_cert_dir }} \
+--volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }} \
+--mount=volume=etcd-cert-dir,target={{ etcd_cert_dir }} \
 docker://{{ vault_image_repo }}:{{ vault_image_tag }} \
 --name={{ vault_container_name }} --net=host \
 --caps-retain=CAP_IPC_LOCK \

tests/files/gce_coreos-vault-upgrade.yml

@@ -0,0 +1,13 @@
+# Instance settings
+cloud_machine_type: "n1-standard-4"
+cloud_image_family: coreos-stable
+cloud_region: us-central1-b
+mode: aio
+
+# Instance settings
+bootstrap_os: coreos
+cert_management: vault
+kube_network_plugin: flannel
+deploy_netchecker: true
+kubedns_min_replicas: 1
+cloud_provider: gce