[release-2.25] pre-commit: make hooks self contained + ci config (#11359)
* Use alternate self-sufficient shellcheck precommit This pre-commit does not require prerequisite on the host, making it easier to run in CI workflows. * Switch to upstream ansible-lint pre-commit hook This way, the hook is self contained and does not depend on a previous virtualenv installation. * pre-commit: fix hooks dependencies - ansible-syntax-check - tox-inventory-builder - jinja-syntax-check * Fix ci-matrix pre-commit hook - Remove dependency of pydblite which fails to setup on recent pythons - Discard shell script and put everything into pre-commit * pre-commit: apply autofixes hooks and fix the rest manually - markdownlint (manual fix) - end-of-file-fixer - requirements-txt-fixer - trailing-whitespace * Convert check_typo to pre-commit + use maintained version client9/misspell is unmaintained, and has been forked by the golangci team, see https://github.com/client9/misspell/issues/197#issuecomment-1596318684. They haven't yet added a pre-commit config, so use my fork with the pre-commit hook config until the pull request is merged. * collection-build-install convert to pre-commit * Run pre-commit hooks in dynamic pipeline Use gitlab dynamic child pipelines feature to have one source of truth for the pre-commit jobs, the pre-commit config file. Use one cache per pre-commit. This should reduce the "fetching cache" time steps in gitlab-ci, since each job will have a separate cache with only its hook installed. * Remove gitlab-ci job done in pre-commit * pre-commit: adjust mardownlint default, md fixes Use a style file as recommended by upstream. This makes for only one source of truth. Conserve previous upstream default for MD007 (upstream default changed here https://github.com/markdownlint/markdownlint/pull/373) * Update pre-commit hooks --------- Co-authored-by: Max Gautier <mg@max.gautier.name>pull/11375/head
k8s-infra-cherrypick-robot
GitHub
parent
8d80b99941
commit
9b122fb5a8
|
|
@ -77,7 +77,6 @@ ci-authorized:
|
|||
include:
|
||||
- .gitlab-ci/build.yml
|
||||
- .gitlab-ci/lint.yml
|
||||
- .gitlab-ci/shellcheck.yml
|
||||
- .gitlab-ci/terraform.yml
|
||||
- .gitlab-ci/packet.yml
|
||||
- .gitlab-ci/vagrant.yml
|
||||
|
|
|
|||
|
|
@ -1,13 +1,24 @@
|
|||
---
|
||||
yamllint:
|
||||
extends: .job
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
variables:
|
||||
LANG: C.UTF-8
|
||||
generate-pre-commit:
|
||||
image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411'
|
||||
stage: build
|
||||
before_script: []
|
||||
script:
|
||||
- yamllint --strict .
|
||||
except: ['triggers', 'master']
|
||||
- >
|
||||
yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' |
|
||||
sed 's/^/ - /' |
|
||||
cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml
|
||||
artifacts:
|
||||
paths:
|
||||
- pre-commit-generated.yml
|
||||
|
||||
run-pre-commit:
|
||||
stage: unit-tests
|
||||
trigger:
|
||||
include:
|
||||
- artifact: pre-commit-generated.yml
|
||||
job: generate-pre-commit
|
||||
strategy: depend
|
||||
|
||||
vagrant-validate:
|
||||
extends: .job
|
||||
|
|
@ -19,108 +30,11 @@ vagrant-validate:
|
|||
- ./tests/scripts/vagrant-validate.sh
|
||||
except: ['triggers', 'master']
|
||||
|
||||
ansible-lint:
|
||||
extends: .job
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
script:
|
||||
- ansible-lint -v
|
||||
except: ['triggers', 'master']
|
||||
|
||||
jinja-syntax-check:
|
||||
extends: .job
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
script:
|
||||
- "find -name '*.j2' -exec tests/scripts/check-templates.py {} +"
|
||||
except: ['triggers', 'master']
|
||||
|
||||
syntax-check:
|
||||
extends: .job
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
variables:
|
||||
ANSIBLE_INVENTORY: inventory/local-tests.cfg
|
||||
ANSIBLE_REMOTE_USER: root
|
||||
ANSIBLE_BECOME: "true"
|
||||
ANSIBLE_BECOME_USER: root
|
||||
ANSIBLE_VERBOSITY: "3"
|
||||
script:
|
||||
- ansible-playbook --syntax-check cluster.yml
|
||||
- ansible-playbook --syntax-check playbooks/cluster.yml
|
||||
- ansible-playbook --syntax-check upgrade-cluster.yml
|
||||
- ansible-playbook --syntax-check playbooks/upgrade_cluster.yml
|
||||
- ansible-playbook --syntax-check reset.yml
|
||||
- ansible-playbook --syntax-check playbooks/reset.yml
|
||||
- ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml
|
||||
except: ['triggers', 'master']
|
||||
|
||||
collection-build-install-sanity-check:
|
||||
extends: .job
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
variables:
|
||||
ANSIBLE_COLLECTIONS_PATH: "./ansible_collections"
|
||||
script:
|
||||
- ansible-galaxy collection build
|
||||
- ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
|
||||
- ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
|
||||
- test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
|
||||
- test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml
|
||||
except: ['triggers', 'master']
|
||||
|
||||
tox-inventory-builder:
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
extends: .job
|
||||
before_script:
|
||||
- ./tests/scripts/rebase.sh
|
||||
script:
|
||||
- pip3 install tox
|
||||
- cd contrib/inventory_builder && tox
|
||||
except: ['triggers', 'master']
|
||||
|
||||
markdownlint:
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
image: node
|
||||
before_script:
|
||||
- npm install -g markdownlint-cli@0.22.0
|
||||
script:
|
||||
- markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md
|
||||
|
||||
generate-sidebar:
|
||||
extends: .job
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
script:
|
||||
- scripts/gen_docs_sidebar.sh
|
||||
- git diff --exit-code
|
||||
|
||||
check-readme-versions:
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
image: python:3
|
||||
script:
|
||||
- tests/scripts/check_readme_versions.sh
|
||||
|
||||
# TODO: convert to pre-commit hook
|
||||
check-galaxy-version:
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
image: python:3
|
||||
script:
|
||||
- tests/scripts/check_galaxy_version.sh
|
||||
|
||||
check-typo:
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
image: python:3
|
||||
script:
|
||||
- tests/scripts/check_typo.sh
|
||||
|
||||
ci-matrix:
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
image: python:3
|
||||
script:
|
||||
- tests/scripts/md-table/test.sh
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
---
|
||||
# stub pipeline for dynamic generation
|
||||
pre-commit:
|
||||
tags:
|
||||
- light
|
||||
image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
|
||||
variables:
|
||||
PRE_COMMIT_HOME: /pre-commit-cache
|
||||
script:
|
||||
- pre-commit run -a $HOOK_ID
|
||||
cache:
|
||||
key: pre-commit-$HOOK_ID
|
||||
paths:
|
||||
- /pre-commit-cache
|
||||
parallel:
|
||||
matrix:
|
||||
- HOOK_ID:
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
shellcheck:
|
||||
extends: .job
|
||||
stage: unit-tests
|
||||
tags: [light]
|
||||
variables:
|
||||
SHELLCHECK_VERSION: v0.7.1
|
||||
before_script:
|
||||
- ./tests/scripts/rebase.sh
|
||||
- curl --silent --location "https://github.com/koalaman/shellcheck/releases/download/"${SHELLCHECK_VERSION}"/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv
|
||||
- cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
|
||||
- shellcheck --version
|
||||
script:
|
||||
# Run shellcheck for all *.sh
|
||||
- find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error
|
||||
except: ['triggers', 'master']
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
---
|
||||
MD013: false
|
||||
MD029: false
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
all
|
||||
exclude_rule 'MD013'
|
||||
exclude_rule 'MD029'
|
||||
rule 'MD007', :indent => 2
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v3.4.0
|
||||
rev: v4.6.0
|
||||
hooks:
|
||||
- id: check-added-large-files
|
||||
- id: check-case-conflict
|
||||
|
|
@ -15,47 +15,59 @@ repos:
|
|||
- id: trailing-whitespace
|
||||
|
||||
- repo: https://github.com/adrienverge/yamllint.git
|
||||
rev: v1.27.1
|
||||
rev: v1.35.1
|
||||
hooks:
|
||||
- id: yamllint
|
||||
args: [--strict]
|
||||
|
||||
- repo: https://github.com/markdownlint/markdownlint
|
||||
rev: v0.11.0
|
||||
rev: v0.12.0
|
||||
hooks:
|
||||
- id: markdownlint
|
||||
args: [-r, "~MD013,~MD029"]
|
||||
exclude: "^.git"
|
||||
exclude: "^.github|(^docs/_sidebar\\.md$)"
|
||||
|
||||
- repo: https://github.com/jumanjihouse/pre-commit-hooks
|
||||
rev: 3.0.0
|
||||
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||
rev: v0.10.0.1
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
args: [--severity, "error"]
|
||||
args: ["--severity=error"]
|
||||
exclude: "^.git"
|
||||
files: "\\.sh$"
|
||||
|
||||
- repo: local
|
||||
- repo: https://github.com/ansible/ansible-lint
|
||||
rev: v24.5.0
|
||||
hooks:
|
||||
- id: ansible-lint
|
||||
name: ansible-lint
|
||||
entry: ansible-lint -v
|
||||
language: python
|
||||
pass_filenames: false
|
||||
additional_dependencies:
|
||||
- .[community]
|
||||
- ansible==9.5.1
|
||||
- jsonschema==4.22.0
|
||||
- jmespath==1.0.1
|
||||
- netaddr==1.2.1
|
||||
|
||||
- repo: https://github.com/VannTen/misspell
|
||||
# Waiting on https://github.com/golangci/misspell/pull/19 to get merged
|
||||
rev: 8592a4e
|
||||
hooks:
|
||||
- id: misspell
|
||||
exclude: "OWNERS_ALIASES$"
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: ansible-syntax-check
|
||||
name: ansible-syntax-check
|
||||
entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check
|
||||
language: python
|
||||
files: "^cluster.yml|^upgrade-cluster.yml|^reset.yml|^extra_playbooks/upgrade-only-k8s.yml"
|
||||
additional_dependencies:
|
||||
- ansible==9.5.1
|
||||
|
||||
- id: tox-inventory-builder
|
||||
name: tox-inventory-builder
|
||||
entry: bash -c "cd contrib/inventory_builder && tox"
|
||||
language: python
|
||||
pass_filenames: false
|
||||
additional_dependencies:
|
||||
- tox==4.15.0
|
||||
|
||||
- id: check-readme-versions
|
||||
name: check-readme-versions
|
||||
|
|
@ -63,6 +75,14 @@ repos:
|
|||
language: script
|
||||
pass_filenames: false
|
||||
|
||||
- id: collection-build-install
|
||||
name: Build and install kubernetes-sigs.kubespray Ansible collection
|
||||
language: python
|
||||
additional_dependencies:
|
||||
- ansible-core>=2.16.4
|
||||
entry: tests/scripts/collection-build-install.sh
|
||||
pass_filenames: false
|
||||
|
||||
- id: generate-docs-sidebar
|
||||
name: generate-docs-sidebar
|
||||
entry: scripts/gen_docs_sidebar.sh
|
||||
|
|
@ -71,9 +91,13 @@ repos:
|
|||
|
||||
- id: ci-matrix
|
||||
name: ci-matrix
|
||||
entry: tests/scripts/md-table/test.sh
|
||||
language: script
|
||||
entry: tests/scripts/md-table/main.py
|
||||
language: python
|
||||
pass_filenames: false
|
||||
additional_dependencies:
|
||||
- jinja2
|
||||
- pathlib
|
||||
- pyaml
|
||||
|
||||
- id: jinja-syntax-check
|
||||
name: jinja-syntax-check
|
||||
|
|
@ -82,4 +106,4 @@ repos:
|
|||
types:
|
||||
- jinja
|
||||
additional_dependencies:
|
||||
- Jinja2
|
||||
- jinja2
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ The setup looks like following
|
|||
|
||||
```bash
|
||||
./generate-inventory.sh > sample-inventory/inventory.ini
|
||||
```
|
||||
|
||||
* Export Variables:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
# OpenStack
|
||||
|
||||
## Known compatible public clouds
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
# Recovering the control plane
|
||||
|
||||
To recover from broken nodes in the control plane use the "recover\-control\-plane.yml" playbook.
|
||||
|
|
@ -8,7 +7,6 @@ Examples of what broken means in this context:
|
|||
* One or more bare metal node(s) suffer from unrecoverable hardware failure
|
||||
* One or more node(s) fail during patching or upgrading
|
||||
* Etcd database corruption
|
||||
|
||||
* Other node related failures leaving your control plane degraded or nonfunctional
|
||||
|
||||
__Note that you need at least one functional node to be able to recover using this method.__
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ ansible==9.5.1
|
|||
cryptography==42.0.7
|
||||
jinja2==3.1.4
|
||||
jmespath==1.0.1
|
||||
jsonschema==4.22.0
|
||||
MarkupSafe==2.1.5
|
||||
netaddr==1.2.1
|
||||
pbr==6.0.0
|
||||
ruamel.yaml==0.18.6
|
||||
ruamel.yaml.clib==0.2.8
|
||||
jsonschema==4.22.0
|
||||
|
|
|
|||
|
|
@ -107,4 +107,3 @@ oom_score = {{ containerd_oom_score }}
|
|||
sampling_ratio = {{ containerd_tracing_sampling_ratio }}
|
||||
service_name = "{{ containerd_tracing_service_name }}"
|
||||
{% endif %}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,4 +32,3 @@ data:
|
|||
- name: helper-pod
|
||||
image: "{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}"
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
|
|
|
|||
|
|
@ -11,4 +11,3 @@ subjects:
|
|||
- kind: ServiceAccount
|
||||
name: {{ node_feature_discovery_worker_sa_name }}
|
||||
namespace: {{ node_feature_discovery_namespace }}
|
||||
|
||||
|
|
|
|||
|
|
@ -102,4 +102,3 @@ data:
|
|||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -102,4 +102,3 @@ spec:
|
|||
protocol: TCP
|
||||
targetPort: 4244
|
||||
internalTrafficPolicy: Local
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ ara[server]==1.7.1
|
|||
dopy==0.3.7
|
||||
molecule==24.2.1
|
||||
molecule-plugins[vagrant]==23.5.3
|
||||
python-vagrant==1.0.0
|
||||
pytest-testinfra==10.1.0
|
||||
python-vagrant==1.0.0
|
||||
tox==4.15.0
|
||||
yamllint==1.35.1
|
||||
tzdata==2024.1
|
||||
yamllint==1.35.1
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# cd to the root directory of kubespray
|
||||
cd $(dirname $0)/../../
|
||||
|
||||
rm ./misspell*
|
||||
|
||||
set -e
|
||||
wget https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz
|
||||
tar -zxvf ./misspell_0.3.4_linux_64bit.tar.gz
|
||||
chmod 755 ./misspell
|
||||
git ls-files | grep -v OWNERS_ALIASES | xargs ./misspell -error
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/sh -e
|
||||
export ANSIBLE_COLLECTIONS_PATH="./ansible_collections"
|
||||
ansible-galaxy collection build --force
|
||||
ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
|
||||
ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
|
||||
test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
|
||||
test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml
|
||||
|
|
@ -4,7 +4,6 @@ import sys
|
|||
import glob
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
from pydblite import Base
|
||||
import re
|
||||
import jinja2
|
||||
import sys
|
||||
|
|
@ -14,6 +13,7 @@ from pprint import pprint
|
|||
|
||||
parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')
|
||||
parser.add_argument('--dir', default='tests/files/', help='folder with test yml files')
|
||||
parser.add_argument('--output', default='docs/developers/ci.md', help='output file')
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
|
@ -24,25 +24,26 @@ env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]))
|
|||
# Data represents CI coverage data matrix
|
||||
class Data:
|
||||
def __init__(self):
|
||||
self.db = Base(':memory:')
|
||||
self.db.create('container_manager', 'network_plugin', 'operating_system')
|
||||
self.container_managers = set()
|
||||
self.network_plugins = set()
|
||||
self.os = set()
|
||||
self.combination = set()
|
||||
|
||||
|
||||
def set(self, container_manager, network_plugin, operating_system):
|
||||
self.db.insert(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
|
||||
self.db.commit()
|
||||
def exists(self, container_manager, network_plugin, operating_system):
|
||||
return len((self.db("container_manager") == container_manager) & (self.db("network_plugin") == network_plugin) & (self.db("operating_system") == operating_system)) > 0
|
||||
def set(self, container_manager, network_plugin, os):
|
||||
self.container_managers.add(container_manager)
|
||||
self.network_plugins.add(network_plugin)
|
||||
self.os.add(os)
|
||||
self.combination.add(container_manager+network_plugin+os)
|
||||
|
||||
def exists(self, container_manager, network_plugin, os):
|
||||
return (container_manager+network_plugin+os) in self.combination
|
||||
|
||||
def jinja(self):
|
||||
template = env.get_template('table.md.j2')
|
||||
container_engines = list(self.db.get_unique_ids('container_manager'))
|
||||
network_plugins = list(self.db.get_unique_ids("network_plugin"))
|
||||
operating_systems = list(self.db.get_unique_ids("operating_system"))
|
||||
|
||||
container_engines.sort()
|
||||
network_plugins.sort()
|
||||
operating_systems.sort()
|
||||
container_engines = sorted(self.container_managers)
|
||||
network_plugins = sorted(self.network_plugins)
|
||||
operating_systems = sorted(self.os)
|
||||
|
||||
return template.render(
|
||||
container_engines=container_engines,
|
||||
|
|
@ -91,6 +92,5 @@ for f in files:
|
|||
network_plugin = y.get('kube_network_plugin', 'calico')
|
||||
x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
|
||||
operating_system = x.group(1)
|
||||
data.set(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
|
||||
#print(data.markdown())
|
||||
print(data.jinja())
|
||||
data.set(container_manager=container_manager, network_plugin=network_plugin, os=operating_system)
|
||||
print(data.jinja(), file=open(args.output, 'w'))
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
jinja2
|
||||
pathlib ; python_version < '3.10'
|
||||
pyaml
|
||||
pydblite
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -euxo pipefail
|
||||
|
||||
echo "Install requirements..."
|
||||
pip install -r ./tests/scripts/md-table/requirements.txt
|
||||
|
||||
echo "Generate current file..."
|
||||
./tests/scripts/md-table/main.py > tmp.md
|
||||
|
||||
echo "Compare docs/developers/ci.md with actual tests in tests/files/*.yml ..."
|
||||
cmp docs/developers/ci.md tmp.md
|
||||
Loading…
Reference in New Issue