diff --git a/.gitlab-ci/build.yml b/.gitlab-ci/build.yml
index 3934591d7..0f1824b56 100644
--- a/.gitlab-ci/build.yml
+++ b/.gitlab-ci/build.yml
@@ -1,16 +1,40 @@
 ---
-pipeline image:
+.build:
   stage: build
-  image: docker:20.10.22-cli
+  image:
+    name: moby/buildkit:rootless
+    entrypoint: [""]
   variables:
-    DOCKER_TLS_CERTDIR: ""
-  services:
-    - name: docker:20.10.22-dind
-      # See https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300 for why this is required
-      command: ["--tls=false"]
+    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
   before_script:
-    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+    - mkdir ~/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json
+
+pipeline image:
+  extends: .build
   script:
-    # DOCKER_HOST is overwritten if we set it as a GitLab variable
-    - DOCKER_HOST=tcp://docker:2375; docker build --network host --file pipeline.Dockerfile --tag $PIPELINE_IMAGE .
-    - docker push $PIPELINE_IMAGE
+    - |
+      buildctl-daemonless.sh build \
+        --frontend=dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=. \
+        --opt filename=./pipeline.Dockerfile \
+        --output type=image,name=$PIPELINE_IMAGE,push=true \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache
+  rules:
+    - if: '$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH'
+
+pipeline image and build cache:
+  extends: .build
+  script:
+    - |
+      buildctl-daemonless.sh build \
+        --frontend=dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=. \
+        --opt filename=./pipeline.Dockerfile \
+        --output type=image,name=$PIPELINE_IMAGE,push=true \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache \
+        --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache,mode=max
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'
diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile
index f729a5ff1..be151195d 100644
--- a/pipeline.Dockerfile
+++ b/pipeline.Dockerfile
@@ -36,11 +36,13 @@ RUN apt update -q \
     && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*
 
 WORKDIR /kubespray
-COPY . .
 
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
+RUN --mount=type=bind,target=./requirements-2.12.txt,src=./requirements-2.12.txt \
+    --mount=type=bind,target=./tests/requirements-2.12.txt,src=./tests/requirements-2.12.txt \
+    --mount=type=bind,target=./roles/kubespray-defaults/defaults/main.yaml,src=./roles/kubespray-defaults/defaults/main.yaml \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
-    && pip install --no-compile --no-cache-dir -r tests/requirements.txt -r requirements.txt \
+    && pip install --no-compile --no-cache-dir -r tests/requirements-2.12.txt \
     && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \
     && curl -L https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
     && echo $(curl -L https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
@@ -52,6 +54,4 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && vagrant plugin install vagrant-libvirt \
     # Install Kubernetes collections
     && pip install --no-compile --no-cache-dir kubernetes \
-    && ansible-galaxy collection install kubernetes.core \
-    # Clean cache python
-    && find / -type d -name '*__pycache__' -prune -exec rm -rf {} \;
+    && ansible-galaxy collection install kubernetes.core