diff --git a/scripts/premoderator.sh b/scripts/premoderator.sh
index 11c49f3d3..94713efc9 100644
--- a/scripts/premoderator.sh
+++ b/scripts/premoderator.sh
@@ -7,6 +7,7 @@
 
 CURL_ARGS="-fs --retry 4 --retry-delay 5"
 MAGIC="${MAGIC:-ci check this}"
+exit_code=0
 
 # Get PR number from CI_BUILD_REF_NAME
 issue=$(echo ${CI_BUILD_REF_NAME} | perl -ne '/^pr-(\d+)-\S+$/ && print $1')
@@ -16,27 +17,35 @@ if [ "$issue" = "" ]; then
   exit 2
 fi
 
+echo "Fetching labels from PR $issue"
+labels=$(curl ${CURL_ARGS} "https://api.github.com/repos/kubernetes-sigs/kubespray/issues/${issue}?access_token=${GITHUB_TOKEN}" | jq '{labels: .labels}' | jq '.labels[].name' | jq -s '')
+labels_to_patch=$(echo -n $labels | jq '. + ["needs-ci-auth"]' | tr -d "\n")
+
 echo "Checking for '$MAGIC' comment in PR $issue"
 
 # Get the user name from the PR comments with the wanted magic incantation casted
-user=$(curl ${CURL_ARGS} "https://api.github.com/repos/kubernetes-sigs/kubespray/issues/${issue}/comments" \
-  | jq -M "map(select(.body | contains (\"$MAGIC\"))) | .[0] .user.login" | tr -d '"')
+user=$(curl ${CURL_ARGS} "https://api.github.com/repos/kubernetes-sigs/kubespray/issues/${issue}/comments" | jq -M "map(select(.body | contains (\"$MAGIC\"))) | .[0] .user.login" | tr -d '"')
 
 # Check for the required user group membership to allow (exit 0) or decline (exit >0) the pipeline
 if [ "$user" = "" ] || [ "$user" = "null" ]; then
   echo "Missing '$MAGIC' comment from one of the OWNERS"
-  exit 3
+  exit_code=3
 else
   echo "Found comment from user: $user"
+
+  curl ${CURL_ARGS} "https://api.github.com/orgs/kubernetes-sigs/members/${user}"
+
+  if [ $? -ne 0 ]; then
+    echo "User does not have permissions to start CI run"
+    exit_code=1
+  else
+    labels_to_patch=$(echo -n $labels | jq '. - ["needs-ci-auth"]' | tr -d "\n")
+    exit_code=0
+    echo "$user has allowed CI to start"
+  fi
 fi
 
-curl ${CURL_ARGS} "https://api.github.com/orgs/kubernetes-sigs/members/${user}"
+# Patch labels on PR
+curl ${CURL_ARGS} --request PATCH "https://api.github.com/repos/kubernetes-sigs/kubespray/issues/${issue}?access_token=${GITHUB_TOKEN}" -H "Content-Type: application/json" -d "{\"labels\": ${labels_to_patch}}"
 
-if [ $? -ne 0 ]; then
-  echo "User does not have permissions to start CI run"
-  exit 1
-else
-  echo "$user has allowed CI to start"
-fi
-
-exit 0
+exit $exit_code