From cb8ec30070ded59a4550cd05daad81f0b7a7183d Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Tue, 30 Jan 2024 21:30:13 -0800 Subject: [PATCH] Remove build automation that is no longer used. Most of this can't actually work given the token/security issues, so even if we re-introduce it, we are better off fetching from history vs carrying things that will lead folks astray. --- .github/workflows/RollPyTorch.yml | 146 ------ .github/workflows/buildAndTest.yml | 158 ------ .github/workflows/buildRelease.yml | 309 ----------- .github/workflows/merge-rollpytorch.yml | 34 -- .github/workflows/oneshotSnapshotPackage.yml | 70 --- .github/workflows/releaseSnapshotPackage.yml | 73 --- build_tools/build_python_wheels.sh | 35 -- build_tools/build_standalone.sh | 29 - build_tools/python_deploy/.gitignore | 1 - .../python_deploy/build_linux_packages.sh | 494 ------------------ .../python_deploy/build_macos_packages.sh | 144 ----- .../python_deploy/build_upload_m1_snapshot.sh | 40 -- build_tools/python_deploy/build_windows.ps1 | 30 -- build_tools/python_deploy/build_windows_ci.sh | 21 - .../python_deploy/install_macos_deps.sh | 63 --- build_tools/scrape_releases.py | 36 -- docs/development.md | 100 ---- 17 files changed, 1783 deletions(-) delete mode 100644 .github/workflows/RollPyTorch.yml delete mode 100644 .github/workflows/buildAndTest.yml delete mode 100644 .github/workflows/buildRelease.yml delete mode 100644 .github/workflows/merge-rollpytorch.yml delete mode 100644 .github/workflows/oneshotSnapshotPackage.yml delete mode 100644 .github/workflows/releaseSnapshotPackage.yml delete mode 100755 build_tools/build_python_wheels.sh delete mode 100755 build_tools/build_standalone.sh delete mode 100644 build_tools/python_deploy/.gitignore delete mode 100755 build_tools/python_deploy/build_linux_packages.sh delete mode 100755 build_tools/python_deploy/build_macos_packages.sh delete mode 100755 build_tools/python_deploy/build_upload_m1_snapshot.sh delete mode 100644 build_tools/python_deploy/build_windows.ps1 delete mode 100644 build_tools/python_deploy/build_windows_ci.sh delete mode 100755 build_tools/python_deploy/install_macos_deps.sh delete mode 100644 build_tools/scrape_releases.py diff --git a/.github/workflows/RollPyTorch.yml b/.github/workflows/RollPyTorch.yml deleted file mode 100644 index 1c0f8f568..000000000 --- a/.github/workflows/RollPyTorch.yml +++ /dev/null @@ -1,146 +0,0 @@ -# yamllint disable rule:line-length -name: Roll PyTorch - -on: - schedule: - - cron: '0 13 * * *' - workflow_dispatch: - -jobs: - build_linux: - name: Manylinux Build - runs-on: torch-mlir-cpubuilder-manylinux-x86-64 - - # Don't run this in everyone's forks. - if: github.repository == 'llvm/torch-mlir' - - steps: - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* - - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'false' - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - - - name: Get LLVM and StableHlo submodules - run: | - set -eo pipefail - cd ${GITHUB_WORKSPACE} - - # Fetching the submodules concurrently may cause problems, so we fetch - # them one after another. - rm -f .git/modules/externals/llvm-project/index.lock - rm -f .git/modules/externals/stablehlo/index.lock - git submodule update --init --recursive externals/llvm-project - git submodule update --init --recursive externals/stablehlo - - - name: Setup ccache - uses: ./.github/actions/setup-build - with: - cache-suffix: 'rollPyTorch' - - - name: Determine nightly PyTorch version - run: | - set -eo pipefail - - cd ${GITHUB_WORKSPACE} - python -m pip install wheel - sudo apt-get install unzip - - # Fetch the most recent nightly torchvision release - VISION_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torchvision | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/') - echo "Found torchvision release ${VISION_RELEASE}" - - # Fetch the whl file associated with the nightly torchvision release - rm -f torch*.whl - python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torchvision==${VISION_RELEASE}" - - # Downloading the torchvision WHL also downloads the PyTorch WHL file - # Read the version from the downloaded whl file without extracting it - PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/') - echo "Found torch release ${PT_RELEASE}" - printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt - printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt - - # Read the commit hash from the downloaded whl file without extracting it - PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | tail -1 | awk '{ print $3 }' | tr -d "'") - echo "Found torch commit hash ${PT_HASH}" - - PT_HASH_CHANGED=0 - echo "${PT_HASH}" | cmp - pytorch-hash.txt --quiet || PT_HASH_CHANGED=$? - echo "${PT_HASH}" > pytorch-hash.txt - rm torch-"${PT_RELEASE}"*.whl - - # Write the release and hash to the environment file so that we can - # retrieve them when creating a PR - echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV} - echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV} - echo "PTVISION_RELEASE=${VISION_RELEASE}" >> ${GITHUB_ENV} - echo "PT_HASH_CHANGED=${PT_HASH_CHANGED}" >> ${GITHUB_ENV} - - - name: Build and test (out-of-tree), also update ODS and abstract interpretation library - if: env.PT_HASH_CHANGED != '0' - run: | - cd ${GITHUB_WORKSPACE} - TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \ - TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \ - TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \ - TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB="ON" \ - ./build_tools/python_deploy/build_linux_packages.sh - - - name: Post issue comment on build failure - if: failure() - uses: peter-evans/create-or-update-comment@v2 - with: - issue-number: 1690 - body: | - The RollPyTorch action has failed. See [CI log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. - - The following information may come handy when fixing the code. - ``` - torch version: ${{ env.PT_RELEASE }} - torch commit hash: ${{ env.PT_HASH }} - torchvision version: ${{ env.PTVISION_RELEASE }} - ``` - - - name: Update PyTorch Build Cache (if running on main branch) - if: github.ref_name == 'main' - id: cache-pytorch - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse - key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} - - - name: Commit changes locally - if: env.PT_HASH_CHANGED != '0' - run: | - cd ${GITHUB_WORKSPACE} - git config user.email "torch-mlir@users.noreply.github.com" - git config user.name "Roll PyTorch Action" - git fetch --recurse-submodules=no - git checkout main - git pull origin main - - - name: Create pull request - uses: peter-evans/create-pull-request@v5.0.1 - with: - author: Roll PyTorch Action - branch: rollpytorch - body: | - torch version: ${{ env.PT_RELEASE }} - torch commit hash: ${{ env.PT_HASH }} - torchvision version: ${{ env.PTVISION_RELEASE }} - commit-message: | - update PyTorch version to ${{ env.PT_RELEASE }} - - - torch version: ${{ env.PT_RELEASE }} - - torch commit hash: ${{ env.PT_HASH }} - - torchvision version: ${{ env.PTVISION_RELEASE }} - committer: Roll PyTorch Action - title: update PyTorch version to ${{ env.PT_RELEASE }} - token: ${{ secrets.ROLLPYTORCH_TOKEN0 }} diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml deleted file mode 100644 index 08ab86571..000000000 --- a/.github/workflows/buildAndTest.yml +++ /dev/null @@ -1,158 +0,0 @@ -# yamllint disable rule:line-length -name: Build and Test - -on: - # pull_request: - # branches: [main] - # push: - # branches: [main] - # workflow_dispatch: - -# Ensure that only a single job or workflow using the same -# concurrency group will run at a time. This would cancel -# any in-progress jobs in the same github workflow and github -# ref (e.g. refs/heads/main or refs/pull//merge). -concurrency: - # A PR number if a pull request and otherwise the commit hash. This cancels - # queued and in-progress runs for the same PR (presubmit) or commit - # (postsubmit). The workflow name is prepended to avoid conflicts between - # different workflows. - group: ${{ github.workflow }}-${{ github.event.number || github.sha }} - cancel-in-progress: true - - -# Provisioned Jobs: -# ubuntu/docker - x86_64 - llvm in-tree - pytorch binary - build+test # most used dev flow and fastest signal -# ubuntu/docker - x86_64 - llvm out-of-tree - pytorch source - build+test # most elaborate build -# macos - arm64 - llvm in-tree - pytorch binary - build only # cross compile, can't test arm64 -jobs: - build-test: - strategy: - fail-fast: true - matrix: - os-arch: [macos-arm64, windows-x86_64] - llvm-build: [in-tree, out-of-tree] - torch-binary: [ON] - torch-version: [nightly, stable] - exclude: - # Exclude llvm out-of-tree and pytorch stable (to save resources) - - llvm-build: out-of-tree - torch-version: stable - # Exclude macos-arm64 and llvm out-of-tree altogether - - os-arch: macos-arm64 - llvm-build: out-of-tree - - os-arch: macos-arm64 - torch-version: stable - - os-arch: windows-x86_64 - llvm-build: out-of-tree - - os-arch: windows-x86_64 - torch-version: stable - include: - # Specify OS versions - - os-arch: ubuntu-x86_64 - os: a100 - - os-arch: macos-arm64 - os: macos-latest - - os-arch: windows-x86_64 - os: windows-latest - runs-on: ${{ matrix.os }} - - steps: - - name: Prepare workspace - if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* - - - name: Checkout torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - fetch-depth: 0 - - - name: Fetch PyTorch commit hash - if: ${{ matrix.os-arch != 'windows-x86_64' }} - run: | - PT_HASH="$(cat ${GITHUB_WORKSPACE}/pytorch-hash.txt)" - echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV} - - - name: Setup ccache - uses: ./.github/actions/setup-build - with: - cache-suffix: 'build-${{ matrix.llvm-build }}-${{ matrix.torch-version }}' - torch-version: ${{ matrix.torch-version }} - - - name: Set up Visual Studio shell - if: ${{ matrix.os-arch == 'windows-x86_64' }} - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - - name: Try to Restore PyTorch Build Cache - if: ${{ matrix.torch-binary == 'OFF' }} - id: cache-pytorch - uses: actions/cache/restore@v3 - with: - path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse - key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} - - - name: Build and Test os-arch='ubuntu-x86_64' llvm-build='${{ matrix.llvm-build }}' torch-binary='${{ matrix.torch-binary }}' - if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} - run: | - cd $GITHUB_WORKSPACE - TORCH_MLIR_SRC_PYTORCH_BRANCH="$(cat pytorch-hash.txt)" \ - TM_PACKAGES="${{ matrix.llvm-build }}" \ - TM_USE_PYTORCH_BINARY="${{ matrix.torch-binary }}" \ - TM_PYTORCH_INSTALL_WITHOUT_REBUILD="${{ steps.cache-pytorch.outputs.cache-hit }}" \ - TM_TORCH_VERSION="${{ matrix.torch-version }}" \ - ./build_tools/python_deploy/build_linux_packages.sh - - - name: Configure os-arch='macos-arm64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}' - # cross compile, can't test arm64 - if: ${{ matrix.os-arch == 'macos-arm64' && matrix.llvm-build == 'in-tree' }} - run: | - # TODO: Reenable LTC after build on macOS-arm64 is fixed (https://github.com/llvm/torch-mlir/issues/1253) - cmake -GNinja -Bbuild_arm64 \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_LINKER=lld \ - -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ - -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$GITHUB_WORKSPACE" \ - -DLLVM_TARGETS_TO_BUILD=AArch64 \ - -DLLVM_USE_HOST_TOOLS=ON \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DTORCH_MLIR_ENABLE_STABLEHLO=OFF \ - -DTORCH_MLIR_ENABLE_LTC=OFF \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${{ matrix.torch-binary }}" \ - -DMACOSX_DEPLOYMENT_TARGET=12.0 \ - -DPython3_EXECUTABLE="$(which python)" \ - $GITHUB_WORKSPACE/externals/llvm-project/llvm - - - name: Build torch-mlir (cross-compile) - if: ${{ matrix.os-arch == 'macos-arm64' }} - run: | - cmake --build build_arm64 - - - name: Build (Windows) - if: ${{ matrix.os-arch == 'windows-x86_64' }} - shell: bash - run: ./build_tools/python_deploy/build_windows_ci.sh - - - name: Save PyTorch Build Cache - if: ${{ github.ref_name == 'main' && matrix.torch-binary == 'OFF' }} - uses: actions/cache/save@v3 - with: - path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse - key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} - - - name: Print ccache statistics - shell: bash - run: ccache --show-stats diff --git a/.github/workflows/buildRelease.yml b/.github/workflows/buildRelease.yml deleted file mode 100644 index e84aabb4b..000000000 --- a/.github/workflows/buildRelease.yml +++ /dev/null @@ -1,309 +0,0 @@ -# yamllint disable rule:line-length -name: Release Build - -on: - workflow_dispatch: - inputs: - release_id: - description: 'Release id to upload artifacts to' - default: '' - python_package_version: - description: 'Version to use for creating the Python package' - default: '' - -jobs: - build_linux: - name: Manylinux x86_64 Build - runs-on: a100 - strategy: - matrix: - package: [torch-mlir] - py_version: [cp38-cp38, cp311-cp311] - - steps: - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* - - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - fetch-depth: 0 - - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Build Python wheels and smoke test. - run: | - cd $GITHUB_WORKSPACE - TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} - printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version - TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} ./build_tools/python_deploy/build_linux_packages.sh - - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - build_linux_arm64: - name: Manylinux arm64 Build - runs-on: linux-arm64 - strategy: - matrix: - package: [torch-mlir] - py_version: [cp311-cp311] - - steps: - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* - - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - fetch-depth: 0 - - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Build Python wheels and smoke test. - run: | - cd $GITHUB_WORKSPACE - TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} - printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version - TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} TORCH_MLIR_ENABLE_LTC='0' ./build_tools/python_deploy/build_linux_packages.sh - - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - build_macos: - name: MacOS Build - runs-on: macos-latest - strategy: - matrix: - package: [torch-mlir] - steps: - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Build Python wheels and smoke test. - run: | - cd $GITHUB_WORKSPACE - python -m pip install wheel - TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} - printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version - sudo ./build_tools/python_deploy/install_macos_deps.sh - packages=${{ matrix.package }} TORCH_MLIR_PYTHON_VERSIONS="3.11" ./build_tools/python_deploy/build_macos_packages.sh - - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - build_windows: - name: Windows Build - runs-on: windows-latest - strategy: - matrix: - package: [torch-mlir] - steps: - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Set up Visual Studio shell - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - name: Build Python wheels and smoke test. - shell: pwsh - run: | - $env:TORCH_MLIR_ENABLE_JIT_IR_IMPORTER='1' - $env:TORCH_MLIR_ENABLE_ONLY_MLIR_PYTHON_BINDINGS='0' - $env:TORCH_MLIR_PYTHON_PACKAGE_VERSION = '${{ github.event.inputs.python_package_version }}' - ./build_tools/python_deploy/build_windows.ps1 - - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - continue-on-error: true - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp ./wheelhouse/torch_mlir*.whl dist/ - - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - publish_releases: - runs-on: ubuntu-latest - needs: - - build_linux - - build_linux_arm64 - - build_macos - - build_windows - - # Publish even if one of the builds failed - if: ${{ always() }} - - steps: - - name: Invoke Publish Releases Page - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Publish releases page - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - # - # We're temporarily disabling pypi publishing until we can fix audit wheel - # ODR torch issues. See https://github.com/llvm/torch-mlir/issues/1709 - # - #- name: Download wheels for publishing to PyPI - # uses: actions/download-artifact@v3 - # with: - # name: wheels - # path: dist - #- name: Publish to PyPI - # if: github.event.inputs.release_id != '' - # uses: pypa/gh-action-pypi-publish@v1.5.1 - # with: - # password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/merge-rollpytorch.yml b/.github/workflows/merge-rollpytorch.yml deleted file mode 100644 index 58a91fd1d..000000000 --- a/.github/workflows/merge-rollpytorch.yml +++ /dev/null @@ -1,34 +0,0 @@ -# yamllint disable rule:line-length -name: RollPyTorch Merge - -on: - workflow_run: - workflows: [Build and Test] - types: [completed] - branches: [rollpytorch] - -jobs: - merge-pr: - runs-on: ubuntu-latest - if: | - github.repository == 'llvm/torch-mlir' && - github.event.workflow_run.actor.login == 'stellaraccident' && - github.event.workflow_run.conclusion == 'success' - - steps: - # Fetch the repo first so that the gh command knows where to look for the PR - - name: Fetch Repo - uses: actions/checkout@v3 - with: - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - - - name: Merge RollPyTorch PR - run: | - for pr_id in ${{ join(github.event.workflow_run.pull_requests.*.number, ' ') }} - do - echo "Merging PR: $pr_id" - gh pr merge $pr_id --delete-branch --squash - done - shell: bash - env: - GH_TOKEN: ${{ secrets.ROLLPYTORCH_TOKEN1 }} diff --git a/.github/workflows/oneshotSnapshotPackage.yml b/.github/workflows/oneshotSnapshotPackage.yml deleted file mode 100644 index ec1878606..000000000 --- a/.github/workflows/oneshotSnapshotPackage.yml +++ /dev/null @@ -1,70 +0,0 @@ -# yamllint disable rule:line-length -name: Release oneshot snapshot package - -on: - workflow_dispatch: - -jobs: - release_snapshot_package: - name: "Tag snapshot release" - runs-on: ubuntu-latest - # Don't run this in everyone's forks. - if: github.repository == 'llvm/torch-mlir' - steps: - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* - - - name: Checking out repository - uses: actions/checkout@v3 - with: - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - - - name: Compute version - run: | - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - package_version="$(printf '%(%Y%m%d)T.${{ github.run_number }}')" - tag_name="oneshot-${package_version}" - echo "package_version=${package_version}" >> $GITHUB_ENV - echo "tag_name=${tag_name}" >> $GITHUB_ENV - - - name: Updating snapshot tag - run: | - git tag "${tag_name}" - - - name: Pushing changes - uses: ad-m/github-push-action@v0.6.0 - with: - github_token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - branch: ${{ github.ref_name }} - tags: true - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - tag_name: ${{ env.tag_name }} - release_name: torch-mlir snapshot ${{ env.tag_name }} - body: | - Automatic snapshot release of torch-mlir. - draft: true - prerelease: false - - - name: "Invoke workflow :: Build and Test" - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Build and Test - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - ref: "${{ env.tag_name }}" - - - name: "Invoke workflow :: Release Build" - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Release Build - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - ref: "${{ env.tag_name }}" - inputs: '{"release_id": "${{ steps.create_release.outputs.id }}", "python_package_version": "${{ env.package_version }}"}' diff --git a/.github/workflows/releaseSnapshotPackage.yml b/.github/workflows/releaseSnapshotPackage.yml deleted file mode 100644 index 8a0ec9144..000000000 --- a/.github/workflows/releaseSnapshotPackage.yml +++ /dev/null @@ -1,73 +0,0 @@ -# yamllint disable rule:line-length -name: Release snapshot package - -on: - # schedule: - # - cron: '0 11 * * *' - workflow_dispatch: - -jobs: - release_snapshot_package: - name: "Tag snapshot release" - runs-on: ubuntu-latest - # Don't run this in everyone's forks. - if: github.repository == 'llvm/torch-mlir' - steps: - - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* - - - name: Checking out repository - uses: actions/checkout@v3 - with: - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - - - name: Compute version - run: | - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - package_version="$(printf '%(%Y%m%d)T.${{ github.run_number }}')" - tag_name="snapshot-${package_version}" - echo "package_version=${package_version}" >> $GITHUB_ENV - echo "tag_name=${tag_name}" >> $GITHUB_ENV - - - name: Updating snapshot tag - run: | - git tag "${tag_name}" - - - name: Pushing changes - uses: ad-m/github-push-action@v0.6.0 - with: - github_token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - branch: main - tags: true - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - tag_name: ${{ env.tag_name }} - release_name: torch-mlir snapshot ${{ env.tag_name }} - body: | - Automatic snapshot release of torch-mlir. - draft: true - prerelease: false - - - name: "Invoke workflow :: Build and Test" - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Build and Test - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - ref: "${{ env.tag_name }}" - - - name: "Invoke workflow :: Release Build" - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Release Build - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - ref: "${{ env.tag_name }}" - inputs: '{"release_id": "${{ steps.create_release.outputs.id }}", "python_package_version": "${{ env.package_version }}"}' diff --git a/build_tools/build_python_wheels.sh b/build_tools/build_python_wheels.sh deleted file mode 100755 index 371bf6263..000000000 --- a/build_tools/build_python_wheels.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -eu -o pipefail - -if [ -z "$PYTHON" ]; then - PYTHON="$(which python)" -fi -version="$("$PYTHON" --version)" -echo "Using python: $PYTHON (version $version)" - -repo_root="$(cd "$(dirname "$0")"/.. && pwd)" -wheelhouse="$repo_root/wheelhouse" -package_test_venv="$wheelhouse/package-test.venv" -mkdir -p "$wheelhouse" -cd "$wheelhouse" - -echo "---- BUILDING torch-mlir ----" -CMAKE_GENERATOR=Ninja \ -$PYTHON "${repo_root}/setup.py" bdist_wheel --dist-dir "$wheelhouse" -v - -# Smoke test: create a venv, install the package, and run an example. - -echo "---- CREATING VENV ----" -python -m venv "$package_test_venv" -VENV_PYTHON="$package_test_venv/bin/python" - -# Install the Torch-MLIR package. -# Note that we also need to pass in the `-r requirements.txt` here to pick up -# the right --find-links flag for the nightly PyTorch wheel registry. -echo "---- INSTALLING torch-mlir and dependencies ----" -$VENV_PYTHON -m pip install -f "$wheelhouse" --force-reinstall torch_mlir -r "${repo_root}/requirements.txt" -echo "---- INSTALLING other deps for smoke test ----" -$VENV_PYTHON -m pip install requests pillow - -echo "---- RUNNING SMOKE TEST ----" -$VENV_PYTHON "$repo_root/examples/torchscript_resnet18.py" diff --git a/build_tools/build_standalone.sh b/build_tools/build_standalone.sh deleted file mode 100755 index 02e2ec161..000000000 --- a/build_tools/build_standalone.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# Also available under a BSD-style license. See LICENSE. - -# Simple script that does a CMake configure of this project as an external -# LLVM project so it can be tested in isolation to larger assemblies. -# This is meant for CI's and project maintainers. - -set -eu -o errtrace - -project_dir="$(cd "$(dirname "$0")"/.. && pwd)" -llvm_project_dir="$project_dir/externals/llvm-project" -build_dir="$project_dir/build" - -cmake -GNinja -B"$build_dir" "$llvm_project_dir/llvm" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ - -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$project_dir" \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_TARGETS_TO_BUILD=host - -cd "$build_dir" -ninja tools/torch-mlir/all check-torch-mlir-all diff --git a/build_tools/python_deploy/.gitignore b/build_tools/python_deploy/.gitignore deleted file mode 100644 index 450df1609..000000000 --- a/build_tools/python_deploy/.gitignore +++ /dev/null @@ -1 +0,0 @@ -manylinux2014-x64 diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh deleted file mode 100755 index f0336b2a1..000000000 --- a/build_tools/python_deploy/build_linux_packages.sh +++ /dev/null @@ -1,494 +0,0 @@ -#!/bin/bash -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# build_linux_packages.sh -# One stop build of IREE Python packages for Linux. The Linux build is -# complicated because it has to be done via a docker container that has -# an LTS glibc version, all Python packages and other deps. -# This script handles all of those details. -# -# Usage: -# Build everything (all packages, all python versions): -# ./build_tools/python_deploy/build_linux_packages.sh -# -# Build specific Python versions and packages to custom directory: -# TM_PYTHON_VERSIONS="cp38-cp38 cp39-cp39" \ -# TM_PACKAGES="torch-mlir" \ -# TM_OUTPUT_DIR="/tmp/wheelhouse" \ -# ./build_tools/python_deploy/build_linux_packages.sh -# -# Valid Python versions match a subdirectory under /opt/python in the docker -# image. Typically: -# cp39-cp39 cp310-cp310 -# -# Valid packages: -# torch-mlir, in-tree, out-of-tree -# -# Note that this script is meant to be run on CI and it will pollute both the -# output directory and in-tree build/ directories with docker created, root owned builds. -# Sorry - there is no good way around it but TODO: move to using user UID/GID. -# -# It can be run on a workstation but recommend using a git worktree dedicated -# to packaging to avoid stomping on development artifacts. -set -eu -o errtrace - -this_dir="$(cd "$(dirname "$0")" && pwd)" -repo_root="$(cd "$this_dir"/../../ && pwd)" -arch="$(uname -m)" -echo "Running on Arch: ${arch}" -# This needs to be a manylinux image so we can ship pip packages -TM_RELEASE_DOCKER_IMAGE="${TM_RELEASE_DOCKER_IMAGE:-quay.io/pypa/manylinux2014_${arch}}" -# This assumes an Ubuntu LTS like image. You can build your own with -# ./build_tools/docker/Dockerfile -TM_CI_DOCKER_IMAGE="${TM_CI_DOCKER_IMAGE:-powderluv/torch-mlir-ci:latest}" -# Version of Python to use in Release builds. Ignored in CIs. -TM_PYTHON_VERSIONS="${TM_PYTHON_VERSIONS:-cp38-cp38 cp310-cp310 cp311-cp311}" -# Location to store Release wheels -TM_OUTPUT_DIR="${TM_OUTPUT_DIR:-${this_dir}/wheelhouse}" -# What "packages to build" -TM_PACKAGES="${TM_PACKAGES:-torch-mlir torch-mlir-core}" -# Use pre-built Pytorch -TM_USE_PYTORCH_BINARY="${TM_USE_PYTORCH_BINARY:-ON}" -# Skip running tests if you want quick iteration -TM_SKIP_TESTS="${TM_SKIP_TESTS:-OFF}" -# Update ODS and abstract interpretation library files -TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB="${TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB:-OFF}" -# Determine if we use a stable or a nightly torch build -TM_TORCH_VERSION="${TM_TORCH_VERSION:-nightly}" - -PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE" -TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" -echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" - -export TORCH_MLIR_SRC_PYTORCH_REPO="${TORCH_MLIR_SRC_PYTORCH_REPO:-pytorch/pytorch}" -echo "Setting torch-mlir PyTorch Repo for source builds to: ${TORCH_MLIR_SRC_PYTORCH_REPO}" -export TORCH_MLIR_SRC_PYTORCH_BRANCH="${TORCH_MLIR_SRC_PYTORCH_BRANCH:-master}" -echo "Setting torch-mlir PyTorch version for source builds to: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}" - -# If using PyTorch source, install from the existing build instead of rebuilding -# all of PyTorch. This option is useful in CI, when it determines that the -# PyTorch version has not changed between consecutive runs. -export TM_PYTORCH_INSTALL_WITHOUT_REBUILD="${TM_PYTORCH_INSTALL_WITHOUT_REBUILD:-false}" - -function run_on_host() { - echo "Running on host for $1:$@" - echo "Outputting to ${TM_OUTPUT_DIR}" - if [[ $TM_PYTORCH_INSTALL_WITHOUT_REBUILD != "true" ]]; then - # We want to use the cached files, so don't remove them. - rm -rf "${TM_OUTPUT_DIR}" - fi - mkdir -p "${TM_OUTPUT_DIR}" - case "$package" in - torch-mlir) - TM_CURRENT_DOCKER_IMAGE=${TM_RELEASE_DOCKER_IMAGE} - export USERID=0 - export GROUPID=0 - ;; - torch-mlir-core) - TM_CURRENT_DOCKER_IMAGE=${TM_RELEASE_DOCKER_IMAGE} - export USERID=0 - export GROUPID=0 - ;; - out-of-tree) - TM_CURRENT_DOCKER_IMAGE=${TM_CI_DOCKER_IMAGE} - # CI uses only Python3.10 - TM_PYTHON_VERSIONS="cp310-cp310" - export USERID=$(id -u) - export GROUPID=$(id -g) - ;; - in-tree) - TM_CURRENT_DOCKER_IMAGE=${TM_CI_DOCKER_IMAGE} - # CI uses only Python3.10 - TM_PYTHON_VERSIONS="cp310-cp310" - export USERID=$(id -u) - export GROUPID=$(id -g) - ;; - *) - echo "Unrecognized package '$package'" - exit 1 - ;; - esac - echo "Launching docker image ${TM_CURRENT_DOCKER_IMAGE} with UID:${USERID} GID:${GROUPID}" - docker run --rm \ - -v "${repo_root}:/main_checkout/torch-mlir" \ - -v "${TM_OUTPUT_DIR}:/wheelhouse" \ - -v "${HOME}:/home/${USER}" \ - --user ${USERID}:${GROUPID} \ - --workdir="/home/$USER" \ - --volume="/etc/group:/etc/group:ro" \ - --volume="/etc/passwd:/etc/passwd:ro" \ - --volume="/etc/shadow:/etc/shadow:ro" \ - --ipc=host \ - -e __MANYLINUX_BUILD_WHEELS_IN_DOCKER=1 \ - -e "TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" \ - -e "TM_PYTHON_VERSIONS=${TM_PYTHON_VERSIONS}" \ - -e "TM_PACKAGES=${package}" \ - -e "TM_SKIP_TESTS=${TM_SKIP_TESTS}" \ - -e "TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB=${TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB}" \ - -e "TM_USE_PYTORCH_BINARY=${TM_USE_PYTORCH_BINARY}" \ - -e "TORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO}" \ - -e "TORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH}" \ - -e "TM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD}" \ - -e "TM_TORCH_VERSION=${TM_TORCH_VERSION}" \ - -e "CCACHE_DIR=/main_checkout/torch-mlir/.ccache" \ - "${TM_CURRENT_DOCKER_IMAGE}" \ - /bin/bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh -} - -function run_in_docker() { - echo "Running in docker" - echo "Using python versions: ${TM_PYTHON_VERSIONS}" - - local orig_path="$PATH" - - # Build phase. - for package in $TM_PACKAGES; do - echo "******************** BUILDING PACKAGE ${package} (docker) ************" - for python_version in $TM_PYTHON_VERSIONS; do - python_dir="/opt/python/$python_version" - if ! [ -x "$python_dir/bin/python" ]; then - echo "Could not find python: $python_dir (using system default Python3)" - python_dir=`which python3` - echo "Defaulting to $python_dir (expected for CI builds)" - fi - export PATH=$python_dir/bin:$orig_path - echo ":::: Python version $(python3 --version)" - case "$package" in - torch-mlir) - clean_wheels torch_mlir "$python_version" - build_torch_mlir "$TM_TORCH_VERSION" - - # Disable audit wheel until we can fix ODR torch issues. See - # https://github.com/llvm/torch-mlir/issues/1709 - # - #run_audit_wheel torch_mlir "$python_version" - - clean_build torch_mlir "$python_version" - ;; - torch-mlir-core) - clean_wheels torch_mlir_core "$python_version" - build_torch_mlir_core - run_audit_wheel torch_mlir_core "$python_version" - clean_build torch_mlir_core "$python_version" - ;; - out-of-tree) - setup_venv "$python_version" "$TM_TORCH_VERSION" - build_out_of_tree "$TM_USE_PYTORCH_BINARY" "$python_version" "$TM_TORCH_VERSION" - if [ "${TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB}" == "ON" ]; then - pushd /main_checkout/torch-mlir - TORCH_MLIR_BUILD_DIR=/main_checkout/torch-mlir/build_oot ./build_tools/update_torch_ods.sh - TORCH_MLIR_BUILD_DIR=/main_checkout/torch-mlir/build_oot ./build_tools/update_abstract_interp_lib.sh - popd - fi - if [ "${TM_SKIP_TESTS}" == "OFF" ]; then - test_out_of_tree - fi - ;; - in-tree) - setup_venv "$python_version" "$TM_TORCH_VERSION" - build_in_tree "$TM_USE_PYTORCH_BINARY" "$python_version" "$TM_TORCH_VERSION" - if [ "${TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB}" == "ON" ]; then - pushd /main_checkout/torch-mlir - ./build_tools/update_torch_ods.sh - ./build_tools/update_abstract_interp_lib.sh - popd - fi - if [ "${TM_SKIP_TESTS}" == "OFF" ]; then - test_in_tree "$TM_TORCH_VERSION"; - fi - ;; - *) - echo "Unrecognized package '$package'" - exit 1 - ;; - esac - done - done -} - - -function build_in_tree() { - local torch_from_bin="$1" - local python_version="$2" - - local torch_version="$3" - local enable_ltc="ON" - if [[ "${torch_version}" == "stable" ]] - then - enable_ltc="OFF" - fi - - echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version" - cmake -GNinja -B/main_checkout/torch-mlir/build \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ - -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="/main_checkout/torch-mlir" \ - -DLLVM_TARGETS_TO_BUILD=host \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DTORCH_MLIR_ENABLE_LTC=${enable_ltc} \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \ - -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \ - -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \ - -DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \ - -DPython3_EXECUTABLE="$(which python3)" \ - /main_checkout/torch-mlir/externals/llvm-project/llvm - cmake --build /main_checkout/torch-mlir/build --target tools/torch-mlir/all - ccache -s -} - -function _check_file_not_changed_by() { - # _check_file_not_changed_by - cmd="$1" - file="$2" - file_backup="$PWD/$(basename $file)" - file_new="$PWD/$(basename $file).new" - # Save the original file. - cp "$file" "$file_backup" - # Run the command to regenerate it. - "$1" || return 1 - # Save the new generated file. - cp "$file" "$file_new" - # Restore the original file. We want this function to not change the user's - # working tree state. - mv "$file_backup" "$file" - # We use git-diff as "just a diff program" (no SCM stuff) because it has - # nicer output than regular `diff`. - if ! git diff --no-index --quiet "$file" "$file_new"; then - echo "#######################################################" - echo "Generated file '${file}' is not up to date (see diff below)" - echo ">>> Please run '${cmd}' to update it <<<" - echo "#######################################################" - git diff --no-index --color=always "$file" "$file_new" - # TODO: Is there a better cleanup strategy that doesn't require duplicating - # this inside and outside the `if`? - rm "$file_new" - return 1 - fi - rm "$file_new" -} - -function test_in_tree() { - local torch_version="$1" - - echo ":::: Test in-tree" - cmake --build /main_checkout/torch-mlir/build --target check-torch-mlir-all - - cd /main_checkout/torch-mlir/ - export PYTHONPATH="/main_checkout/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:/main_checkout/torch-mlir/projects/pt1" - - case $torch_version in - nightly) - echo ":::: Test with nightly torch" - - echo ":::: Check that update_abstract_interp_lib.sh has been run" - _check_file_not_changed_by ./build_tools/update_abstract_interp_lib.sh lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp - - echo ":::: Check that update_torch_ods.sh has been run" - _check_file_not_changed_by ./build_tools/update_torch_ods.sh include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td - - echo ":::: Run Lazy Tensor Core e2e integration tests" - python -m e2e_testing.main --config=lazy_tensor_core -v - - echo ":::: Run Linalg e2e integration tests" - python -m e2e_testing.main --config=linalg -v - - # Dynamo is changing a lot in nightly versions, and thus the implementation - # tends to become incompatible to the stable version. - echo ":::: Run TorchDynamo e2e integration tests" - python -m e2e_testing.main --config=torchdynamo -v - ;; - stable) - echo ":::: Test with stable torch" - - # Disabled until the next stable PyTorch release (v2.1) is available - # echo ":::: Run Lazy Tensor Core e2e integration tests in experimental mode" - # python -m e2e_testing.main --config=lazy_tensor_core -v --ignore_failures - ;; - *) - echo "Unrecognized torch version '$torch_version'" - exit 1 - ;; - esac - - echo ":::: Run make_fx + TOSA e2e integration tests" - python -m e2e_testing.main --config=make_fx_tosa -v - - echo ":::: Run TOSA e2e integration tests" - python -m e2e_testing.main --config=tosa -v -} - -function setup_venv() { - local python_version="$1" - local torch_version="$2" - - echo ":::: Setting up VENV with Python: $python_version PyTorch $torch_version" - python3 -m venv /main_checkout/torch-mlir/docker_venv - source /main_checkout/torch-mlir/docker_venv/bin/activate - - echo ":::: pip installing dependencies" - python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt - - case $torch_version in - nightly) - echo ":::: Using nightly dependencies" - python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/requirements.txt - python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/torchvision-requirements.txt - ;; - stable) - echo ":::: Using stable dependencies" - python3 -m pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu - python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/build-requirements.txt - ;; - *) - echo "Unrecognized torch version '$torch_version'" - exit 1 - ;; - esac - - python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/test-requirements.txt -} - -function build_out_of_tree() { - local torch_from_bin="$1" - local python_version="$2" - local torch_version="$3" - echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version ($torch_version)" - - local enable_ltc="ON" - if [[ "${torch_version}" == "stable" ]] - then - enable_ltc="OFF" - fi - - if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ] - then - echo ":::: LLVM / MLIR is not built so building it first.." - cmake -GNinja -B/main_checkout/torch-mlir/llvm-build \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_TARGETS_TO_BUILD=host \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE="$(which python3)" \ - /main_checkout/torch-mlir/externals/llvm-project/llvm - cmake --build /main_checkout/torch-mlir/llvm-build - fi - - # Incremental builds come here directly and can run cmake if required. - cmake -GNinja -B/main_checkout/torch-mlir/build_oot \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ - -DLLVM_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/llvm/" \ - -DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \ - -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \ - -DTORCH_MLIR_ENABLE_LTC=${enable_ltc} \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \ - -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \ - -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \ - -DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \ - -DPython3_EXECUTABLE="$(which python3)" \ - /main_checkout/torch-mlir - cmake --build /main_checkout/torch-mlir/build_oot - ccache -s -} - -function test_out_of_tree() { - echo ":::: Test out-of-tree" - cmake --build /main_checkout/torch-mlir/build_oot --target check-torch-mlir-all -} - -function clean_build() { - # clean up for recursive runs - local package="$1" - local python_version="$2" - echo ":::: Clean build dir $package $python_version" - rm -rf /main_checkout/torch-mlir/build /main_checkout/torch-mlir/llvm-build /main_checkout/torch-mlir/docker_venv /main_checkout/torch-mlir/libtorch -} - -function build_torch_mlir() { - local torch_version="$1" - case $torch_version in - nightly) - echo ":::: Using nightly dependencies" - python -m pip install --no-cache-dir -r /main_checkout/torch-mlir/requirements.txt \ - --extra-index-url https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - CMAKE_GENERATOR=Ninja \ - TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ - python -m pip wheel -v -w /wheelhouse /main_checkout/torch-mlir \ - -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html \ - -r /main_checkout/torch-mlir/whl-requirements.txt - ;; - stable) - echo ":::: Using stable dependencies" - python3 -m pip install --no-cache-dir torch torchvision - python3 -m pip install --no-cache-dir -r /main_checkout/torch-mlir/build-requirements.txt - CMAKE_GENERATOR=Ninja \ - TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ - python -m pip wheel -v -w /wheelhouse /main_checkout/torch-mlir - ;; - *) - echo "Unrecognized torch version '$torch_version'" - exit 1 - ;; - esac -} - -function run_audit_wheel() { - local wheel_basename="$1" - local python_version="$2" - generic_wheel="/wheelhouse/${wheel_basename}-${TORCH_MLIR_PYTHON_PACKAGE_VERSION}-${python_version}-linux_${arch}.whl" - echo ":::: Auditwheel $generic_wheel" - auditwheel repair -w /wheelhouse "$generic_wheel" - rm "$generic_wheel" -} - -function build_torch_mlir_core() { - python -m pip install --no-cache-dir -r /main_checkout/torch-mlir/build-requirements.txt - CMAKE_GENERATOR=Ninja \ - TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ - TORCH_MLIR_ENABLE_JIT_IR_IMPORTER=0 \ - TORCH_MLIR_ENABLE_ONLY_MLIR_PYTHON_BINDINGS=1 \ - python -m pip wheel -v -w /wheelhouse /main_checkout/torch-mlir -} - -function clean_wheels() { - local wheel_basename="$1" - local python_version="$2" - echo ":::: Clean wheels $wheel_basename $python_version" - rm -f /wheelhouse/"${wheel_basename}"-*-"${python_version}"-*.whl -} - -# Trampoline to the docker container if running on the host. -if [ -z "${__MANYLINUX_BUILD_WHEELS_IN_DOCKER-}" ]; then - for package in $TM_PACKAGES; do - echo "******************** BUILDING PACKAGE ${package} (host) *************" - run_on_host "${package} $@" - done -else - run_in_docker "$@" -fi diff --git a/build_tools/python_deploy/build_macos_packages.sh b/build_tools/python_deploy/build_macos_packages.sh deleted file mode 100755 index b928c1e48..000000000 --- a/build_tools/python_deploy/build_macos_packages.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# build_macos_packages.sh -# One stop build of IREE Python packages for MacOS. This presumes that -# dependencies are installed from install_macos_deps.sh. This will build -# for a list of Python versions synchronized with that script and corresponding -# with directory names under: -# /Library/Frameworks/Python.framework/Versions -# -# MacOS convention is to refer to this as major.minor (i.e. "3.9", "3.10"). -# Valid packages: -# torch-mlir - -set -eu -o errtrace - -this_dir="$(cd "$(dirname "$0")" && pwd)" -repo_root="$(cd "$this_dir"/../../ && pwd)" -python_versions="${TORCH_MLIR_PYTHON_VERSIONS:-3.9 3.10 3.11}" -output_dir="${output_dir:-${this_dir}/wheelhouse}" -packages="${packages:-torch-mlir}" - -PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE" -export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" -echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" - -# Note that this typically is selected to match the version that the official -# Python distributed is built at. -export MACOSX_DEPLOYMENT_TARGET="${TORCH_MLIR_OSX_TARGET:-11.0}" -export CMAKE_OSX_ARCHITECTURES="${TORCH_MLIR_OSX_ARCH:-arm64;x86_64}" -echo "CMAKE_OSX_ARCHITECTURES: $CMAKE_OSX_ARCHITECTURES" -echo "MACOSX_DEPLOYMENT_TARGET $MACOSX_DEPLOYMENT_TARGET" - -# Disable LTC build on MacOS to avoid linkage issues -# https://github.com/llvm/torch-mlir/issues/1253 -export TORCH_MLIR_ENABLE_LTC=0 - -function run() { - echo "Using python versions: ${python_versions}" - - local orig_path="$PATH" - - # Build phase. - for package in $packages; do - echo "******************** BUILDING PACKAGE ${package} ********************" - for python_version in $python_versions; do - python_dir="/Library/Frameworks/Python.framework/Versions/$python_version" - if ! [ -x "$python_dir/bin/python3" ]; then - echo "ERROR: Could not find python3: $python_dir (skipping)" - continue - fi - export PATH=$python_dir/bin:$orig_path - echo ":::: Python version $(python3 --version)" - case "$package" in - torch-mlir) - clean_wheels torch_mlir "$python_version" - build_torch_mlir torch_mlir "$python_version" - run_audit_wheel torch_mlir "$python_version" - ;; - torch-mlir-core) - clean_wheels torch_mlir_core "$python_version" - build_torch_mlir_core torch_mlir_core "$python_version" - run_audit_wheel torch_mlir_core "$python_version" - ;; - *) - echo "Unrecognized package '$package'" - exit 1 - ;; - esac - done - done -} - -function build_torch_mlir() { - local wheel_basename="$1" - local python_version="$2" - rm -rf "$output_dir"/build_venv - python"${python_version}" -m venv "$output_dir"/build_venv - source "$output_dir"/build_venv/bin/activate - python"${python_version}" -m pip install -U pip - python"${python_version}" -m pip install -r "$repo_root"/pytorch-requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu - python"${python_version}" -m pip install -r "$repo_root"/build-requirements.txt - CMAKE_GENERATOR=Ninja \ - TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ - MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ - CMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ - python"${python_version}" -m pip wheel -v -w "$output_dir" "$repo_root" --extra-index-url https://download.pytorch.org/whl/nightly/cpu - deactivate - rm -rf "$output_dir"/build_venv -} - -function build_torch_mlir_core() { - local wheel_basename="$1" - local python_version="$2" - rm -rf "$output_dir"/build_venv - python"${python_version}" -m venv "$output_dir"/build_venv - source "$output_dir"/build_venv/bin/activate - python"${python_version}" -m pip install -U pip delocate - python"${python_version}" -m pip install -r "$repo_root"/build-requirements.txt - CMAKE_GENERATOR=Ninja \ - TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ - MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ - CMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ - TORCH_MLIR_ENABLE_JIT_IR_IMPORTER=0 \ - TORCH_MLIR_ENABLE_ONLY_MLIR_PYTHON_BINDINGS=1 \ - python"${python_version}" -m pip wheel -v -w "$output_dir" "$repo_root" - deactivate - rm -rf "$output_dir"/build_venv -} - -function clean_wheels() { - local wheel_basename="$1" - local python_version="$2" - echo ":::: Clean wheels $wheel_basename $python_version" - rm -rf "$repo_root"/build/ - rm -f "$output_dir"/"${wheel_basename}"-*-"${python_version//./}"-*.whl -} - -function run_audit_wheel() { - set +x - local wheel_basename="$1" - local python_version="$2" - generic_wheel=$(ls "$output_dir"/"${wheel_basename}"-* | grep "${python_version//./}") - echo "Looking for $generic_wheel" - if [ -f "$generic_wheel" ]; then - echo "$generic_wheel found. Delocating it.." - rm -rf "$output_dir"/test_venv - python"${python_version}" -m venv "$output_dir"/test_venv - source "$output_dir"/test_venv/bin/activate - python"${python_version}" -m pip install -U pip - python"${python_version}" -m pip install -r "$repo_root"/pytorch-requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu - python"${python_version}" -m pip install -r "$repo_root"/build-requirements.txt - python"${python_version}" -m pip install "$generic_wheel" --extra-index-url https://download.pytorch.org/whl/nightly/cpu - DYLD_LIBRARY_PATH="$output_dir"/test_venv/lib/python"${python_version}"/site-packages/torch/lib delocate-wheel -v "$generic_wheel" - deactivate - rm -rf "$output_dir"/test_venv - fi -} - -run diff --git a/build_tools/python_deploy/build_upload_m1_snapshot.sh b/build_tools/python_deploy/build_upload_m1_snapshot.sh deleted file mode 100755 index b876801c7..000000000 --- a/build_tools/python_deploy/build_upload_m1_snapshot.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# build_upload_m1_snapshot.sh -# This is a wrapper to build_macos_packages.sh to be run on Apple M1 systems -# since GH Actions don't support M1 runners yet and Universal builds -# don't work for Torch-MLIR since we don't have universal PyTorch binaries -# This presumes that dependencies are installed from install_macos_deps.sh and -# you have the gh credentials to upload to the release - -set -eu -o errtrace - -this_dir="$(cd "$(dirname "$0")" && pwd)" -repo_root="$(cd "$this_dir"/../../ && pwd)" -output_dir="${output_dir:-${this_dir}/wheelhouse}" -rm -rf "${output_dir}" - -git fetch --all -latest_snapshot=$(git for-each-ref --sort=creatordate --format '%(refname:short)' refs/tags | tail -n 1) -git checkout "${latest_snapshot}" -git submodule update --init -package_version=${latest_snapshot#"snapshot-"} -echo "Latest snapshot tag is: ${latest_snapshot}" -echo "Latest version is: ${package_version}" - -export TORCH_MLIR_PYTHON_VERSIONS="3.9 3.10" -echo "Using Python Versions: ${TORCH_MLIR_PYTHON_VERSIONS}" -export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${package_version}" -echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" - -TORCH_MLIR_OSX_ARCH=arm64 \ -TORCH_MLIR_OSX_TARGET=11.0 \ -TORCH_MLIR_PYTHON_PACKAGE_VERSION="${package_version}" \ -TORCH_MLIR_PYTHON_VERSIONS="${TORCH_MLIR_PYTHON_VERSIONS}" \ -"${repo_root}"/build_tools/python_deploy/build_macos_packages.sh - -gh release upload "${latest_snapshot}" "${repo_root}"/build_tools/python_deploy/wheelhouse/torch*.whl diff --git a/build_tools/python_deploy/build_windows.ps1 b/build_tools/python_deploy/build_windows.ps1 deleted file mode 100644 index 808a16cb1..000000000 --- a/build_tools/python_deploy/build_windows.ps1 +++ /dev/null @@ -1,30 +0,0 @@ -# Uncomment if you want to install Python. GHA provides this - -#Write-Host "Installing python" - -#Start-Process choco 'install python --version=3.10.8' -wait -NoNewWindow - -#Write-Host "python installation completed successfully" - -#Write-Host "Reload environment variables" -#$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") -#Write-Host "Reloaded environment variables" - -Write-Host "Installing Build Dependencies" -python -m venv .\mlir_venv\ -.\mlir_venv\Scripts\Activate.PS1 -pip install -r .\pytorch-requirements.txt -pip install -r .\build-requirements.txt -pip install delvewheel -Write-Host "Build Deps installation completed successfully" - -Write-Host "Building torch-mlir" -$env:CMAKE_GENERATOR='Ninja' -$env:TORCH_MLIR_ENABLE_LTC='0' -python -m pip wheel -v -w wheelhouse ./ -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html -r whl-requirements.txt - -Write-Host "Build completed successfully" - -Write-Host "Fixing up wheel dependencies" -delvewheel repair --add-path .\build\cmake_build\tools\torch-mlir\python_packages\torch_mlir\torch_mlir\_mlir_libs --add-dll TorchMLIRAggregateCAPI.dll --no-dll 'c10.dll;torch_python.dll;torch_cpu.dll' -v (get-item .\wheelhouse\torch_mlir*.whl).FullName -Write-Host "All Done." diff --git a/build_tools/python_deploy/build_windows_ci.sh b/build_tools/python_deploy/build_windows_ci.sh deleted file mode 100644 index c5da1adf6..000000000 --- a/build_tools/python_deploy/build_windows_ci.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -echo "Building torch-mlir" - -cmake -GNinja -Bbuild \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_TARGETS_TO_BUILD=host \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_FIND_VIRTUALENV=ONLY \ - -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ - -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$PWD" \ - -DPython3_EXECUTABLE="$(which python)" \ - $GITHUB_WORKSPACE/externals/llvm-project/llvm - -cmake --build build --config Release - -echo "Build completed successfully" diff --git a/build_tools/python_deploy/install_macos_deps.sh b/build_tools/python_deploy/install_macos_deps.sh deleted file mode 100755 index 4d91a244c..000000000 --- a/build_tools/python_deploy/install_macos_deps.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/zsh -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Installs dependencies on MacOS necessary to build IREE. -# Additional dependencies (i.e MoltenVK) may be needed to use all functionality. -# -# Usage: -# sudo install_macos_deps.sh - -set -eu -o pipefail - -if [[ "$(whoami)" != "root" ]]; then - echo "ERROR: Must setup deps as root" - exit 1 -fi - -PYTHON_INSTALLER_URLS=( - "https://www.python.org/ftp/python/3.11.2/python-3.11.2-macos11.pkg" - "https://www.python.org/ftp/python/3.10.10/python-3.10.10-macos11.pkg" - "https://www.python.org/ftp/python/3.9.13/python-3.9.13-macos11.pkg" -) - -PYTHON_SPECS=( - 3.11@https://www.python.org/ftp/python/3.11.2/python-3.11.2-macos11.pkg - 3.10@https://www.python.org/ftp/python/3.10.5/python-3.10.5-macos11.pkg - 3.9@https://www.python.org/ftp/python/3.9.13/python-3.9.13-macos11.pkg -) - -for python_spec in $PYTHON_SPECS; do - python_version="${python_spec%%@*}" - url="${python_spec##*@}" - echo "-- Installing Python $python_version from $url" - python_path="/Library/Frameworks/Python.framework/Versions/$python_version" - python_exe="$python_path/bin/python3" - - # Install Python. - if ! [ -x "$python_exe" ]; then - package_basename="$(basename "$url")" - download_path="/tmp/torch_mlir_python_install/$package_basename" - mkdir -p "$(dirname "$download_path")" - echo "Downloading $url -> $download_path" - curl "$url" -o "$download_path" - - echo "Installing $download_path" - installer -pkg "$download_path" -target / - else - echo ":: Python version already installed. Not reinstalling." - fi - - echo ":: Python version $python_version installed:" - $python_exe --version - $python_exe -m pip --version - - echo ":: Installing system pip packages" - $python_exe -m pip install --upgrade pip - $python_exe -m pip install --upgrade delocate -done - -echo "*** All done ***" diff --git a/build_tools/scrape_releases.py b/build_tools/scrape_releases.py deleted file mode 100644 index b8c7265d3..000000000 --- a/build_tools/scrape_releases.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Scrapes the github releases API to generate a static pip-install-able releases page. - -See https://github.com/llvm/torch-mlir/issues/1374 -""" -import argparse -import json - -import requests - -# Parse arguments -parser = argparse.ArgumentParser() -parser.add_argument('owner', type=str) -parser.add_argument('repo', type=str) -args = parser.parse_args() - -# Get releases -response = requests.get( - f"https://api.github.com/repos/{args.owner}/{args.repo}/releases") -body = json.loads(response.content) - -# Parse releases -releases = [] -for row in body: - for asset in row['assets']: - releases.append((asset["name"], asset["browser_download_url"])) - -# Output HTML -html = """ - - -""" -for name, url in releases: - html += f" {name}
\n" -html += """ -""" -print(html) \ No newline at end of file diff --git a/docs/development.md b/docs/development.md index 782058a63..5ef52ef65 100644 --- a/docs/development.md +++ b/docs/development.md @@ -225,106 +225,6 @@ is built as part of the LLVM project along with MLIR. We mount a ccache and pip cache inside the docker container to speed up iterative builds. Iterative builds should be as fast as running without docker. -### In-Tree builds - -Build MLIR and Torch-MLIR together as part of the LLVM repo. - -```shell -TM_PACKAGES="in-tree" ./build_tools/python_deploy/build_linux_packages.sh -``` - -### Out-of-Tree builds - -Build LLVM/MLIR first and then build Torch-MLIR referencing that build -```shell -TM_PACKAGES="out-of-tree" ./build_tools/python_deploy/build_linux_packages.sh -``` - -### Release builds - -Build in a manylinux Docker image so we can upload artifacts to PyPI. - -```shell -TM_PACKAGES="torch-mlir" ./build_tools/python_deploy/build_linux_packages.sh -``` - -### Mimicing CI+Release builds - -If you wanted to build all the CIs locally - -```shell -TM_PACKAGES="out-of-tree in-tree" ./build_tools/python_deploy/build_linux_packages.sh -``` - -If you wanted to build all the CIs and the Release builds (just with Python 3.10 since most other Python builds are redundant) - -```shell -TM_PACKAGES="torch-mlir out-of-tree in-tree" TM_PYTHON_VERSIONS="cp310-cp310" ./build_tools/python_deploy/build_linux_packages.sh -``` - -Note: The Release docker still runs as root so it may generate some files owned by root:root. We hope to move it to run as a user in the future. - -### Cleaning up - -Docker builds tend to leave a wide variety of files around. Luckily most are owned by the user but there are still some that need to be removed -as superuser. - -```shell -rm -rf build build_oot llvm-build docker_venv externals/pytorch/build .ccache -``` - -## Building your own Docker image - -If you would like to build your own docker image (usually not necessary). You can run: - -```shell -cd ./build_tools/docker -docker build -t your-name/torch-mlir-ci --no-cache . -``` - -### Other configurable environmental variables - -The following additional environmental variables can be used to customize your docker build: - -* Custom Release Docker image: - Defaults to `stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest` -```shell - TM_RELEASE_DOCKER_IMAGE="stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest" -``` -* Custom CI Docker image: - Defaults to `powderluv/torch-mlir-ci:latest`. This assumes an Ubuntu LTS like image. You can build your own with `./build_tools/docker/Dockerfile` -```shell - TM_CI_DOCKER_IMAGE="powderluv/torch-mlir-ci:latest" -``` - -* Custom Python Versions for Release builds: - Version of Python to use in Release builds. Ignored in CIs. Defaults to `cp38-cp38 cp39-cp39 cp310-cp310` -```shell - TM_PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310" -``` - -* Location to store Release build wheels -```shell - TM_OUTPUT_DIR="./build_tools/python_deploy/wheelhouse" -``` - -* What "packages" to build: - Defaults to torch-mlir. Options are `torch-mlir out-of-tree in-tree` -```shell - TM_PACKAGES="torch-mlir out-of-tree in-tree" -``` -* Use pre-built Pytorch: - Defaults to using pre-built Pytorch. Setting it to `OFF` builds from source -```shell - TM_USE_PYTORCH_BINARY="OFF" -``` -* Skip running tests - Skip running tests if you want quick build only iteration. Default set to `OFF` -```shell - TM_SKIP_TESTS="OFF" -``` - - ## Build Python Packages We have preliminary support for building Python packages. This can be done