[docs] Add CI matrix and script (#5461)

* Rename CI jobs from ubuntu to ubuntu16

* Add CI matrix and script

.gitlab-ci/packet.yml

@@ -23,37 +23,37 @@ packet_centos7-flannel-addons:
 
 # ### MANUAL JOBS
 
-packet_centos-weave-kubeadm-sep:
+packet_centos7-weave-kubeadm-sep:
   stage: deploy-part2
   extends: .packet
   when: on_success
   variables:
     UPGRADE_TEST: basic
 
-packet_ubuntu-weave-sep:
+packet_ubuntu16-weave-sep:
   stage: deploy-part2
   extends: .packet
   when: manual
 
 # # More builds for PRs/merges (manual) and triggers (auto)
 
-packet_ubuntu-canal-ha:
+packet_ubuntu16-canal-ha:
   stage: deploy-special
   extends: .packet
   when: manual
 
-packet_ubuntu-canal-kubeadm:
+packet_ubuntu16-canal-kubeadm:
   stage: deploy-part2
   extends: .packet
   when: on_success
 
-packet_ubuntu-flannel-ha:
+packet_ubuntu16-flannel-ha:
   stage: deploy-part2
   extends: .packet
   when: manual
 
 # Contiv does not work in k8s v1.16
-# packet_ubuntu-contiv-sep:
+# packet_ubuntu16-contiv-sep:
 #   stage: deploy-part2
 #   extends: .packet
 #   when: on_success
@@ -110,12 +110,12 @@ packet_opensuse-canal:
   extends: .packet
   when: manual
 
-packet_oracle-7-canal:
+packet_oracle7-canal:
   stage: deploy-part2
   extends: .packet
   when: manual
 
-packet_ubuntu-kube-router-sep:
+packet_ubuntu16-kube-router-sep:
   stage: deploy-part2
   extends: .packet
   when: manual

docs/_sidebar.md

@@ -36,4 +36,5 @@
 * Developers
   * [Test cases](docs/test_cases.md)
   * [Vagrant](docs/vagrant.md)
+  * [CI Matrix](docs/ci.md)
 * [Roadmap](docs/roadmap.md)

docs/ci.md

@@ -0,0 +1,39 @@
+# CI test coverage
+
+To generate this Matrix run `./tests/scripts/md-table/main.py`
+
+## docker
+
+| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: |
+coreos |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: |
+debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rhel7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+ubuntu |  :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
+ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
+ubuntu18 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+
+## containerd
+
+| OS / CNI | calico | canal | cilium | contiv | flannel | kube-ovn | kube-router | macvlan | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+coreos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rhel7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu18 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |

tests/scripts/md-table/main.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+import argparse
+import sys
+import glob
+from pathlib import Path
+import yaml
+from pydblite import Base
+import re
+import jinja2
+import sys
+
+from pprint import pprint
+
+
+parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')
+parser.add_argument('--dir', default='tests/files/', help='folder with test yml files')
+
+
+args = parser.parse_args()
+p = Path(args.dir)
+
+env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]))
+
+# Data represents CI coverage data matrix
+class Data:
+    def __init__(self):
+        self.db = Base(':memory:')
+        self.db.create('container_manager', 'network_plugin', 'operating_system')
+
+    
+    def set(self, container_manager, network_plugin, operating_system):
+        self.db.insert(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
+        self.db.commit()
+    def exists(self, container_manager, network_plugin, operating_system):
+        return len((self.db("container_manager") == container_manager) & (self.db("network_plugin") == network_plugin) & (self.db("operating_system") == operating_system)) > 0
+
+    def jinja(self):
+        template = env.get_template('table.md.j2')
+        container_engines = list(self.db.get_unique_ids('container_manager'))
+        network_plugins = list(self.db.get_unique_ids("network_plugin"))
+        operating_systems = list(self.db.get_unique_ids("operating_system"))
+
+        container_engines.sort()
+        container_engines.reverse() # reverse sort container_engines to get Docker first in the list
+        network_plugins.sort()
+        operating_systems.sort()
+
+        return template.render(
+            container_engines=container_engines,
+            network_plugins=network_plugins,
+            operating_systems=operating_systems,
+            exists=self.exists
+        )
+
+    def markdown(self):
+        out = ''
+        for container_manager in self.db.get_unique_ids('container_manager'):
+            # Prepare the headers
+            out += "# " + container_manager + "\n"
+            headers = '|OS / CNI| '
+            underline = '|----|'
+            for network_plugin in self.db.get_unique_ids("network_plugin"):
+                headers += network_plugin + ' | '
+                underline += '----|'
+            out += headers + "\n" + underline + "\n"
+            for operating_system in self.db.get_unique_ids("operating_system"):
+                out += '| ' + operating_system + ' | '
+                for network_plugin in self.db.get_unique_ids("network_plugin"):
+                    if self.exists(container_manager, network_plugin, operating_system):
+                        emoji = ':white_check_mark:'
+                    else:
+                        emoji = ':x:'
+                    out += emoji + ' | '
+                out += "\n"
+
+        pprint(self.db.get_unique_ids('operating_system'))
+        pprint(self.db.get_unique_ids('network_plugin'))
+        return out
+
+
+
+if not p.is_dir():
+    print("Path is not a directory")
+    sys.exit(2)
+
+data = Data()
+files = p.glob('*.yml')
+for f in files:
+    y = yaml.load(f.open(), Loader=yaml.FullLoader)
+
+    container_manager = y.get('container_manager', 'docker')
+    network_plugin = y.get('kube_network_plugin', 'calico')
+    x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
+    operating_system = x.group(1)
+    data.set(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
+#print(data.markdown())
+print(data.jinja())

tests/scripts/md-table/requirements.txt

@@ -0,0 +1,4 @@
+pyaml
+jinja2
+pathlib
+pydblite

tests/scripts/md-table/table.md.j2

@@ -0,0 +1,15 @@
+# CI test coverage
+
+To generate this Matrix run `./tests/scripts/md-table/main.py` 
+
+{%- for container_engine in container_engines %}
+
+## {{ container_engine }}
+
+| OS / CNI |{% for cni in network_plugins %} {{ cni }} |{% endfor %}
+|---|{% for cni in network_plugins %} --- |{% endfor %}
+{%- for os in operating_systems %}
+{{ os }} | {% for cni in network_plugins %} {{ ':white_check_mark:' if exists(container_engine, cni, os) else ':x:' }} |{% endfor %}
+{%- endfor %}
+
+{%- endfor %}

tests/scripts/md-table/test.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -euxo pipefail
+
+echo "Install requirements..."
+pip install -r ./tests/scripts/md-table/requirements.txt
+
+echo "Generate current file..."
+./tests/scripts/md-table/main.py > tmp.md
+
+echo "Compare docs/ci.md with actual tests in tests/files/*.yml ..."
+cmp docs/ci.md tmp.md

tests/scripts/md-table/tmp.md

@@ -0,0 +1,30 @@
+# CI test coverage
+
+## docker
+
+| OS / CNI | canal | cilium | weave | contiv | kube-router | calico | macvlan | flannel | kube-ovn |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+centos7 |  :x: | :x: | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: |
+coreos |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+ubuntu18 |  :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: |
+opensuse |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: |
+oracle7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+
+## containerd
+
+| OS / CNI | canal | cilium | weave | contiv | kube-router | calico | macvlan | flannel | kube-ovn |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+coreos |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu18 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+--