diff --git a/.github/workflows/RollPyTorch.yml b/.github/workflows/RollPyTorch.yml
index d5df5e1f2..0a7fa5672 100644
--- a/.github/workflows/RollPyTorch.yml
+++ b/.github/workflows/RollPyTorch.yml
@@ -34,14 +34,18 @@ jobs:
         python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torch==${PT_RELEASE}"
         # Read the commit hash from the downloaded whl file without extracting it
         PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'")
+        echo "${PT_HASH}" | cmp - pytorch-version.txt --quiet
+        PT_HASH_CHANGED=$?
         echo "${PT_HASH}" > pytorch-version.txt
         rm torch-"${PT_RELEASE}"*.whl
         # Write the release and hash to the environment file so that we can
         # retrieve them when creating a PR
         echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV}
         echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV}
+        echo "PT_HASH_CHANGED=${PT_HASH_CHANGED}" >> ${GITHUB_ENV}
 
     - name: Build and test
+      if: env.PT_HASH_CHANGED != '0'
       run: |
         cd ${GITHUB_WORKSPACE}
         TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \
@@ -50,12 +54,12 @@ jobs:
         ./build_tools/python_deploy/build_linux_packages.sh
 
     - name: Push changes to main branch
+      if: env.PT_HASH_CHANGED != '0'
       run: |
         cd ${GITHUB_WORKSPACE}
         git config user.email "torch-mlir@users.noreply.github.com"
         git config user.name "Roll PyTorch Action"
-        git fetch
+        git fetch --recurse-submodules=no
         git checkout main
         git add pytorch-version.txt pytorch-requirements.txt
-        git commit -m "update PyTorch version to ${{ env.PT_RELEASE }}"
-        git push --set-upstream origin main
+        git diff --cached --exit-code || (git commit -m "update PyTorch version to ${{ env.PT_RELEASE }}" && git push --set-upstream origin main)