diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml
index 1321dd10b..c2ac7c59d 100644
--- a/.github/workflows/buildAndTest.yml
+++ b/.github/workflows/buildAndTest.yml
@@ -27,15 +27,12 @@ jobs:
       matrix:
         os-arch: [ubuntu-x86_64, macos-arm64, windows-x86_64]
         llvm-build: [in-tree, out-of-tree]
-        torch-binary: [ON, OFF]
+        torch-binary: [ON]
         torch-version: [nightly, stable]
         exclude:
-          # Exclude llvm in-tree and pytorch source
-          - llvm-build: in-tree
-            torch-binary: OFF
-          # Exclude llvm out-of-tree and pytorch source
+          # Exclude llvm out-of-tree and pytorch stable (to save resources)
           - llvm-build: out-of-tree
-            torch-binary: OFF
+            torch-version: stable
           # Exclude macos-arm64 and llvm out-of-tree altogether
           - os-arch: macos-arm64
             llvm-build: out-of-tree
@@ -45,9 +42,6 @@ jobs:
             llvm-build: out-of-tree
           - os-arch: windows-x86_64
             torch-version: stable
-          # For PyTorch stable builds, we don't build PyTorch from source
-          - torch-version: stable
-            torch-binary: OFF
         include:
           # Specify OS versions
           - os-arch: ubuntu-x86_64