diff --git a/.github/actions/setup-build/action.yml b/.github/actions/setup-build/action.yml
index f53c6933b..85c3f7516 100644
--- a/.github/actions/setup-build/action.yml
+++ b/.github/actions/setup-build/action.yml
@@ -26,7 +26,8 @@ runs:
 
   - name: Install PyTorch nightly depends
     run: |
-      python -m pip install -r requirements.txt
+      python -m pip install -r pytorch-requirements.txt
+      python -m pip install -r build-requirements.txt
     shell: bash
 
   - name: Install prerequisites (Linux)
diff --git a/.github/workflows/RollPyTorch.yml b/.github/workflows/RollPyTorch.yml
index 24e64922f..cc8a8ea09 100644
--- a/.github/workflows/RollPyTorch.yml
+++ b/.github/workflows/RollPyTorch.yml
@@ -44,7 +44,8 @@ jobs:
         # Read the version from the downloaded whl file without extracting it
         PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/')
         echo "Found torch release ${PT_RELEASE}"
-        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\ntorchvision==%s\n" "${PT_RELEASE}" "${VISION_RELEASE}" > pytorch-requirements.txt
+        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
+        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt
 
         # Read the commit hash from the downloaded whl file without extracting it
         PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'")
@@ -96,7 +97,7 @@ jobs:
         git fetch --recurse-submodules=no
         git checkout main
         git pull origin main
-        git add pytorch-hash.txt pytorch-requirements.txt lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td
+        git add pytorch-hash.txt pytorch-requirements.txt torchvision-requirements.txt lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td
         git diff --cached --exit-code || (git commit -m "update PyTorch version to ${{ env.PT_RELEASE }}" && git push --set-upstream origin main)
 
     - name: Update PyTorch Build Cache (if running on main branch)
diff --git a/build_tools/python_deploy/build_macos_packages.sh b/build_tools/python_deploy/build_macos_packages.sh
index 9a859527e..083d575ce 100755
--- a/build_tools/python_deploy/build_macos_packages.sh
+++ b/build_tools/python_deploy/build_macos_packages.sh
@@ -77,7 +77,8 @@ function build_torch_mlir() {
   python"${python_version}" -m venv "$output_dir"/build_venv
   source "$output_dir"/build_venv/bin/activate
   python"${python_version}" -m pip install -U pip
-  python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
+  python"${python_version}" -m pip install -r "$repo_root"/pytorch-requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
+  python"${python_version}" -m pip install -r "$repo_root"/build-requirements.txt
   CMAKE_GENERATOR=Ninja \
   TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \
   MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
@@ -107,7 +108,8 @@ function run_audit_wheel() {
     python"${python_version}" -m venv "$output_dir"/test_venv
     source "$output_dir"/test_venv/bin/activate
     python"${python_version}" -m pip install -U pip
-    python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
+    python"${python_version}" -m pip install -r "$repo_root"/pytorch-requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
+    python"${python_version}" -m pip install -r "$repo_root"/build-requirements.txt
     python"${python_version}" -m pip install "$generic_wheel" --extra-index-url https://download.pytorch.org/whl/nightly/cpu
     DYLD_LIBRARY_PATH="$output_dir"/test_venv/lib/python"${python_version}"/site-packages/torch/lib delocate-wheel -v "$generic_wheel"
     deactivate
diff --git a/build_tools/python_deploy/build_windows.ps1 b/build_tools/python_deploy/build_windows.ps1
index 44c767875..808a16cb1 100644
--- a/build_tools/python_deploy/build_windows.ps1
+++ b/build_tools/python_deploy/build_windows.ps1
@@ -13,7 +13,8 @@
 Write-Host "Installing Build Dependencies"
 python -m venv .\mlir_venv\
 .\mlir_venv\Scripts\Activate.PS1
-pip install -r .\requirements.txt
+pip install -r .\pytorch-requirements.txt
+pip install -r .\build-requirements.txt
 pip install delvewheel
 Write-Host "Build Deps installation completed successfully"
 
diff --git a/pytorch-requirements.txt b/pytorch-requirements.txt
index 32f2c638d..51a519083 100644
--- a/pytorch-requirements.txt
+++ b/pytorch-requirements.txt
@@ -1,4 +1,3 @@
 -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
 --pre
 torch==2.0.0.dev20230212
-torchvision==0.15.0.dev20230213
diff --git a/requirements.txt b/requirements.txt
index f47a87da7..f346b53da 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,3 @@
--r build-requirements.txt
 -r pytorch-requirements.txt
-
-# Test Requirements
-pillow
-dill
-multiprocess
+-r build-requirements.txt
+-r test-requirements.txt
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 000000000..e752531e2
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1,5 @@
+-r torchvision-requirements.txt
+
+pillow
+dill
+multiprocess
diff --git a/torchvision-requirements.txt b/torchvision-requirements.txt
new file mode 100644
index 000000000..e3fccd906
--- /dev/null
+++ b/torchvision-requirements.txt
@@ -0,0 +1,3 @@
+-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+--pre
+torchvision==0.15.0.dev20230213