From 4a4d80a6ad18b01b7eaf096c5231106228fd719c Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Sat, 27 Jan 2024 15:48:06 -0800 Subject: [PATCH] [ci] Add lint job and enable yaml linting of GH files. (#2819) --- .github/actions/setup-build/action.yml | 99 ++-- .github/workflows/RollPyTorch.yml | 222 ++++----- .github/workflows/bazelBuildAndTest.yml | 159 +++--- .github/workflows/buildAndTest.yml | 182 +++---- .github/workflows/buildRelease.yml | 491 +++++++++---------- .github/workflows/gh-pages-releases.yml | 1 + .github/workflows/lint.yml | 17 + .github/workflows/merge-rollpytorch.yml | 31 +- .github/workflows/oneshotSnapshotPackage.yml | 1 + .github/workflows/releaseSnapshotPackage.yml | 1 + 10 files changed, 613 insertions(+), 591 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/actions/setup-build/action.yml b/.github/actions/setup-build/action.yml index 73592a7dc..a21c9a1d7 100644 --- a/.github/actions/setup-build/action.yml +++ b/.github/actions/setup-build/action.yml @@ -1,3 +1,4 @@ +# yamllint disable rule:line-length name: "Setup build environment" description: "Setup the build environment. An action so that it can be shared between in-tree/out-of-tree jobs" @@ -24,59 +25,59 @@ runs: using: "composite" steps: - - name: Set up Python - if: ${{ runner.arch == 'X64' }} - uses: actions/setup-python@v4 - with: - python-version: '3.11' + - name: Set up Python + if: ${{ runner.arch == 'X64' }} + uses: actions/setup-python@v4 + with: + python-version: '3.11' - - name: Install MLIR Python depends - if: ${{ runner.os != 'Linux' }} - run: | - python -m pip install -r $GITHUB_WORKSPACE/externals/llvm-project/mlir/python/requirements.txt - shell: bash + - name: Install MLIR Python depends + if: ${{ runner.os != 'Linux' }} + run: | + python -m pip install -r $GITHUB_WORKSPACE/externals/llvm-project/mlir/python/requirements.txt + shell: bash - - name: Install PyTorch nightly depends - if: ${{ runner.os != 'Linux' }} - run: | - python -m pip install -r pytorch-requirements.txt - python -m pip install -r build-requirements.txt - shell: bash + - name: Install PyTorch nightly depends + if: ${{ runner.os != 'Linux' }} + run: | + python -m pip install -r pytorch-requirements.txt + python -m pip install -r build-requirements.txt + shell: bash - - name: Install prerequisites (Linux) - if: ${{ runner.os == 'Linux' }} - run: sudo apt-get install --yes ccache ninja-build - shell: bash + - name: Install prerequisites (Linux) + if: ${{ runner.os == 'Linux' }} + run: sudo apt-get install --yes ccache ninja-build + shell: bash - - name: Install prerequisites (macOS) - if: ${{ runner.os == 'macOS' }} - run: brew install ccache ninja - shell: bash + - name: Install prerequisites (macOS) + if: ${{ runner.os == 'macOS' }} + run: brew install ccache ninja + shell: bash - - name: Install prerequisites (Windows) - if: ${{ runner.os == 'Windows' }} - run: | - pip install ninja - choco install ccache --yes - shell: bash + - name: Install prerequisites (Windows) + if: ${{ runner.os == 'Windows' }} + run: | + pip install ninja + choco install ccache --yes + shell: bash - - name: Configure ccache - if: ${{ inputs.cache-enabled == 'true' }} - run: | - rm -rf ${{ github.workspace }}/.ccache - mkdir -p ${{ github.workspace }}/.ccache - ccache --set-config "cache_dir=${{ github.workspace }}/.ccache" - ccache --set-config "compression=true" - ccache --set-config "max_size=300M" - ccache --zero-stats - shell: bash + - name: Configure ccache + if: ${{ inputs.cache-enabled == 'true' }} + run: | + rm -rf ${{ github.workspace }}/.ccache + mkdir -p ${{ github.workspace }}/.ccache + ccache --set-config "cache_dir=${{ github.workspace }}/.ccache" + ccache --set-config "compression=true" + ccache --set-config "max_size=300M" + ccache --zero-stats + shell: bash - - name: Enable ccache - if: ${{ inputs.cache-enabled == 'true' }} - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/.ccache - key: ${{ runner.os }}-${{ inputs.cache-suffix }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ inputs.cache-suffix }}- - ${{ runner.os }}- + - name: Enable ccache + if: ${{ inputs.cache-enabled == 'true' }} + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/.ccache + key: ${{ runner.os }}-${{ inputs.cache-suffix }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ inputs.cache-suffix }}- + ${{ runner.os }}- diff --git a/.github/workflows/RollPyTorch.yml b/.github/workflows/RollPyTorch.yml index 4f2d9d8c5..975b538c5 100644 --- a/.github/workflows/RollPyTorch.yml +++ b/.github/workflows/RollPyTorch.yml @@ -1,3 +1,4 @@ +# yamllint disable rule:line-length name: Roll PyTorch on: @@ -14,133 +15,132 @@ jobs: if: github.repository == 'llvm/torch-mlir' steps: + - name: Prepare workspace + run: | + # Clear the workspace directory so that we don't run into errors about + # existing lock files. + sudo rm -rf $GITHUB_WORKSPACE/* - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* + - name: Get torch-mlir + uses: actions/checkout@v3 + with: + submodules: 'false' + token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'false' - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + - name: Get LLVM and StableHlo submodules + run: | + set -eo pipefail + cd ${GITHUB_WORKSPACE} - - name: Get LLVM and StableHlo submodules - run: | - set -eo pipefail - cd ${GITHUB_WORKSPACE} + # Fetching the submodules concurrently may cause problems, so we fetch + # them one after another. + rm -f .git/modules/externals/llvm-project/index.lock + rm -f .git/modules/externals/stablehlo/index.lock + git submodule update --init --recursive externals/llvm-project + git submodule update --init --recursive externals/stablehlo - # Fetching the submodules concurrently may cause problems, so we fetch - # them one after another. - rm -f .git/modules/externals/llvm-project/index.lock - rm -f .git/modules/externals/stablehlo/index.lock - git submodule update --init --recursive externals/llvm-project - git submodule update --init --recursive externals/stablehlo + - name: Setup ccache + uses: ./.github/actions/setup-build + with: + cache-suffix: 'rollPyTorch' - - name: Setup ccache - uses: ./.github/actions/setup-build - with: - cache-suffix: 'rollPyTorch' + - name: Determine nightly PyTorch version + run: | + set -eo pipefail - - name: Determine nightly PyTorch version - run: | - set -eo pipefail + cd ${GITHUB_WORKSPACE} + python -m pip install wheel + sudo apt-get install unzip - cd ${GITHUB_WORKSPACE} - python -m pip install wheel - sudo apt-get install unzip + # Fetch the most recent nightly torchvision release + VISION_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torchvision | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/') + echo "Found torchvision release ${VISION_RELEASE}" - # Fetch the most recent nightly torchvision release - VISION_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torchvision | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/') - echo "Found torchvision release ${VISION_RELEASE}" + # Fetch the whl file associated with the nightly torchvision release + rm -f torch*.whl + python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torchvision==${VISION_RELEASE}" - # Fetch the whl file associated with the nightly torchvision release - rm -f torch*.whl - python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torchvision==${VISION_RELEASE}" + # Downloading the torchvision WHL also downloads the PyTorch WHL file + # Read the version from the downloaded whl file without extracting it + PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/') + echo "Found torch release ${PT_RELEASE}" + printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt + printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt - # Downloading the torchvision WHL also downloads the PyTorch WHL file - # Read the version from the downloaded whl file without extracting it - PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/') - echo "Found torch release ${PT_RELEASE}" - printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt - printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt + # Read the commit hash from the downloaded whl file without extracting it + PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | tail -1 | awk '{ print $3 }' | tr -d "'") + echo "Found torch commit hash ${PT_HASH}" - # Read the commit hash from the downloaded whl file without extracting it - PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | tail -1 | awk '{ print $3 }' | tr -d "'") - echo "Found torch commit hash ${PT_HASH}" + PT_HASH_CHANGED=0 + echo "${PT_HASH}" | cmp - pytorch-hash.txt --quiet || PT_HASH_CHANGED=$? + echo "${PT_HASH}" > pytorch-hash.txt + rm torch-"${PT_RELEASE}"*.whl - PT_HASH_CHANGED=0 - echo "${PT_HASH}" | cmp - pytorch-hash.txt --quiet || PT_HASH_CHANGED=$? - echo "${PT_HASH}" > pytorch-hash.txt - rm torch-"${PT_RELEASE}"*.whl + # Write the release and hash to the environment file so that we can + # retrieve them when creating a PR + echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV} + echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV} + echo "PTVISION_RELEASE=${VISION_RELEASE}" >> ${GITHUB_ENV} + echo "PT_HASH_CHANGED=${PT_HASH_CHANGED}" >> ${GITHUB_ENV} - # Write the release and hash to the environment file so that we can - # retrieve them when creating a PR - echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV} - echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV} - echo "PTVISION_RELEASE=${VISION_RELEASE}" >> ${GITHUB_ENV} - echo "PT_HASH_CHANGED=${PT_HASH_CHANGED}" >> ${GITHUB_ENV} + - name: Build and test (out-of-tree), also update ODS and abstract interpretation library + if: env.PT_HASH_CHANGED != '0' + run: | + cd ${GITHUB_WORKSPACE} + TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \ + TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \ + TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \ + TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB="ON" \ + ./build_tools/python_deploy/build_linux_packages.sh - - name: Build and test (out-of-tree), also update ODS and abstract interpretation library - if: env.PT_HASH_CHANGED != '0' - run: | - cd ${GITHUB_WORKSPACE} - TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \ - TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \ - TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \ - TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB="ON" \ - ./build_tools/python_deploy/build_linux_packages.sh + - name: Post issue comment on build failure + if: failure() + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: 1690 + body: | + The RollPyTorch action has failed. See [CI log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. - - name: Post issue comment on build failure - if: failure() - uses: peter-evans/create-or-update-comment@v2 - with: - issue-number: 1690 - body: | - The RollPyTorch action has failed. See [CI log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + The following information may come handy when fixing the code. + ``` + torch version: ${{ env.PT_RELEASE }} + torch commit hash: ${{ env.PT_HASH }} + torchvision version: ${{ env.PTVISION_RELEASE }} + ``` - The following information may come handy when fixing the code. - ``` - torch version: ${{ env.PT_RELEASE }} - torch commit hash: ${{ env.PT_HASH }} - torchvision version: ${{ env.PTVISION_RELEASE }} - ``` + - name: Update PyTorch Build Cache (if running on main branch) + if: github.ref_name == 'main' + id: cache-pytorch + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse + key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} - - name: Update PyTorch Build Cache (if running on main branch) - if: github.ref_name == 'main' - id: cache-pytorch - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse - key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} + - name: Commit changes locally + if: env.PT_HASH_CHANGED != '0' + run: | + cd ${GITHUB_WORKSPACE} + git config user.email "torch-mlir@users.noreply.github.com" + git config user.name "Roll PyTorch Action" + git fetch --recurse-submodules=no + git checkout main + git pull origin main - - name: Commit changes locally - if: env.PT_HASH_CHANGED != '0' - run: | - cd ${GITHUB_WORKSPACE} - git config user.email "torch-mlir@users.noreply.github.com" - git config user.name "Roll PyTorch Action" - git fetch --recurse-submodules=no - git checkout main - git pull origin main + - name: Create pull request + uses: peter-evans/create-pull-request@v5.0.1 + with: + author: Roll PyTorch Action + branch: rollpytorch + body: | + torch version: ${{ env.PT_RELEASE }} + torch commit hash: ${{ env.PT_HASH }} + torchvision version: ${{ env.PTVISION_RELEASE }} + commit-message: | + update PyTorch version to ${{ env.PT_RELEASE }} - - name: Create pull request - uses: peter-evans/create-pull-request@v5.0.1 - with: - author: Roll PyTorch Action - branch: rollpytorch - body: | - torch version: ${{ env.PT_RELEASE }} - torch commit hash: ${{ env.PT_HASH }} - torchvision version: ${{ env.PTVISION_RELEASE }} - commit-message: | - update PyTorch version to ${{ env.PT_RELEASE }} - - - torch version: ${{ env.PT_RELEASE }} - - torch commit hash: ${{ env.PT_HASH }} - - torchvision version: ${{ env.PTVISION_RELEASE }} - committer: Roll PyTorch Action - title: update PyTorch version to ${{ env.PT_RELEASE }} - token: ${{ secrets.ROLLPYTORCH_TOKEN0 }} + - torch version: ${{ env.PT_RELEASE }} + - torch commit hash: ${{ env.PT_HASH }} + - torchvision version: ${{ env.PTVISION_RELEASE }} + committer: Roll PyTorch Action + title: update PyTorch version to ${{ env.PT_RELEASE }} + token: ${{ secrets.ROLLPYTORCH_TOKEN0 }} diff --git a/.github/workflows/bazelBuildAndTest.yml b/.github/workflows/bazelBuildAndTest.yml index b3cb3b8fb..23f2addbe 100644 --- a/.github/workflows/bazelBuildAndTest.yml +++ b/.github/workflows/bazelBuildAndTest.yml @@ -1,8 +1,9 @@ +# yamllint disable rule:line-length name: Bazel Build and Test on: push: - branches: [ main ] + branches: [main] workflow_dispatch: # Ensure that only a single job or workflow using the same @@ -24,90 +25,90 @@ jobs: runs-on: ubuntu-latest steps: - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* + - name: Prepare workspace + run: | + # Clear the workspace directory so that we don't run into errors about + # existing lock files. + sudo rm -rf $GITHUB_WORKSPACE/* - - name: Checkout torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' + - name: Checkout torch-mlir + uses: actions/checkout@v3 + with: + submodules: 'true' - # Continually update cache even if there's a "hit" during - # restore to avoid the cache going stale over time - # https://github.com/actions/cache/blob/main/workarounds.md#update-a-cache - - name: Setup cache for bazel - uses: actions/cache@v3 - with: - path: ~/.cache/bazel - key: torch_mlir-bazel-build-cache-${{ runner.os }}-${{ github.sha }} - restore-keys: | - torch_mlir-bazel-build-cache-${{ runner.os }} + # Continually update cache even if there's a "hit" during + # restore to avoid the cache going stale over time + # https://github.com/actions/cache/blob/main/workarounds.md#update-a-cache + - name: Setup cache for bazel + uses: actions/cache@v3 + with: + path: ~/.cache/bazel + key: torch_mlir-bazel-build-cache-${{ runner.os }}-${{ github.sha }} + restore-keys: | + torch_mlir-bazel-build-cache-${{ runner.os }} - # Change bazel cache directory to root ownership - # to allow writing to it from within the docker container. - # If no cache hits, this directory is not present - # so don't run chown (will error otherwise). - - name: Set bazel cache permissions - run: | - if [ -d "${HOME}/.cache/bazel" ]; then - sudo chown -R root:root "${HOME}/.cache/bazel" - fi + # Change bazel cache directory to root ownership + # to allow writing to it from within the docker container. + # If no cache hits, this directory is not present + # so don't run chown (will error otherwise). + - name: Set bazel cache permissions + run: | + if [ -d "${HOME}/.cache/bazel" ]; then + sudo chown -R root:root "${HOME}/.cache/bazel" + fi - - name: Build docker image - run: | - docker build -f utils/bazel/docker/Dockerfile \ - -t torch-mlir:ci \ - . + - name: Build docker image + run: | + docker build -f utils/bazel/docker/Dockerfile \ + -t torch-mlir:ci \ + . - - name: Verify buildifier was run (bazel lint) - run: | - docker run --rm \ - -v "$(pwd)":"/opt/src/torch-mlir" \ - -v "${HOME}/.cache/bazel":"/root/.cache/bazel" \ - torch-mlir:ci \ - bazel run @torch-mlir//:buildifier - if [ -n "$(git status --porcelain)" ]; then - echo "Please 'bazel run @torch-mlir//:buildifier' and commit changes." - exit 1 - fi + - name: Verify buildifier was run (bazel lint) + run: | + docker run --rm \ + -v "$(pwd)":"/opt/src/torch-mlir" \ + -v "${HOME}/.cache/bazel":"/root/.cache/bazel" \ + torch-mlir:ci \ + bazel run @torch-mlir//:buildifier + if [ -n "$(git status --porcelain)" ]; then + echo "Please 'bazel run @torch-mlir//:buildifier' and commit changes." + exit 1 + fi - - name: Bazel build torch-mlir - run: | - docker run --rm \ - -v "$(pwd)":"/opt/src/torch-mlir" \ - -v "${HOME}/.cache/bazel":"/root/.cache/bazel" \ - torch-mlir:ci \ - bazel build @torch-mlir//:torch-mlir-opt + - name: Bazel build torch-mlir + run: | + docker run --rm \ + -v "$(pwd)":"/opt/src/torch-mlir" \ + -v "${HOME}/.cache/bazel":"/root/.cache/bazel" \ + torch-mlir:ci \ + bazel build @torch-mlir//:torch-mlir-opt - - name: Bazel test torch-mlir (lit tests) - run: | - docker run --rm \ - -v "$(pwd)":"/opt/src/torch-mlir" \ - -v "${HOME}/.cache/bazel":"/root/.cache/bazel" \ - torch-mlir:ci \ - bazel test @torch-mlir//test/... + - name: Bazel test torch-mlir (lit tests) + run: | + docker run --rm \ + -v "$(pwd)":"/opt/src/torch-mlir" \ + -v "${HOME}/.cache/bazel":"/root/.cache/bazel" \ + torch-mlir:ci \ + bazel test @torch-mlir//test/... - # Switch back bazel cache directory to user ownership - # to allow GHA post-cache step to save cache without - # permissions issue. - - name: Switch bazel cache permissions - run: | - if [ -d "${HOME}/.cache/bazel" ]; then - sudo chown -R "$USER":"$USER" "${HOME}/.cache/bazel" - fi + # Switch back bazel cache directory to user ownership + # to allow GHA post-cache step to save cache without + # permissions issue. + - name: Switch bazel cache permissions + run: | + if [ -d "${HOME}/.cache/bazel" ]; then + sudo chown -R "$USER":"$USER" "${HOME}/.cache/bazel" + fi - - name: Send mail - if: failure() - uses: dawidd6/action-send-mail@v3 - with: - server_address: ${{ secrets.SMTP_SERVER }} - server_port: ${{ secrets.SMTP_PORT }} - username: ${{ secrets.SMTP_USERNAME }} - password: ${{ secrets.SMTP_PASSWORD }} - subject: GitHub Action Bazel Build and Test failed! - body: Bazel Build job failed! See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} for more information. - to: ${{ secrets.MAIL_RECEIVER }} - from: Torch-MLIR Bazel Build GitHub Actions + - name: Send mail + if: failure() + uses: dawidd6/action-send-mail@v3 + with: + server_address: ${{ secrets.SMTP_SERVER }} + server_port: ${{ secrets.SMTP_PORT }} + username: ${{ secrets.SMTP_USERNAME }} + password: ${{ secrets.SMTP_PASSWORD }} + subject: GitHub Action Bazel Build and Test failed! + body: Bazel Build job failed! See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} for more information. + to: ${{ secrets.MAIL_RECEIVER }} + from: Torch-MLIR Bazel Build GitHub Actions diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml index f52ddf9b4..c6d345860 100644 --- a/.github/workflows/buildAndTest.yml +++ b/.github/workflows/buildAndTest.yml @@ -1,10 +1,11 @@ +# yamllint disable rule:line-length name: Build and Test on: pull_request: - branches: [ main ] + branches: [main] push: - branches: [ main ] + branches: [main] workflow_dispatch: # Ensure that only a single job or workflow using the same @@ -57,102 +58,101 @@ jobs: runs-on: ${{ matrix.os }} steps: - - - name: Prepare workspace - if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* - - - name: Checkout torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - fetch-depth: 0 + - name: Prepare workspace + if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} + run: | + # Clear the workspace directory so that we don't run into errors about + # existing lock files. + sudo rm -rf $GITHUB_WORKSPACE/* - - name: Fetch PyTorch commit hash - if: ${{ matrix.os-arch != 'windows-x86_64' }} - run: | - PT_HASH="$(cat ${GITHUB_WORKSPACE}/pytorch-hash.txt)" - echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV} + - name: Checkout torch-mlir + uses: actions/checkout@v3 + with: + submodules: 'true' + fetch-depth: 0 - - name: Setup ccache - uses: ./.github/actions/setup-build - with: - cache-suffix: 'build-${{ matrix.llvm-build }}-${{ matrix.torch-version }}' - torch-version: ${{ matrix.torch-version }} + - name: Fetch PyTorch commit hash + if: ${{ matrix.os-arch != 'windows-x86_64' }} + run: | + PT_HASH="$(cat ${GITHUB_WORKSPACE}/pytorch-hash.txt)" + echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV} - - name: Set up Visual Studio shell - if: ${{ matrix.os-arch == 'windows-x86_64' }} - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 + - name: Setup ccache + uses: ./.github/actions/setup-build + with: + cache-suffix: 'build-${{ matrix.llvm-build }}-${{ matrix.torch-version }}' + torch-version: ${{ matrix.torch-version }} - - name: Try to Restore PyTorch Build Cache - if: ${{ matrix.torch-binary == 'OFF' }} - id: cache-pytorch - uses: actions/cache/restore@v3 - with: - path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse - key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} + - name: Set up Visual Studio shell + if: ${{ matrix.os-arch == 'windows-x86_64' }} + uses: egor-tensin/vs-shell@v2 + with: + arch: x64 - - name: Build and Test os-arch='ubuntu-x86_64' llvm-build='${{ matrix.llvm-build }}' torch-binary='${{ matrix.torch-binary }}' - if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} - run: | - cd $GITHUB_WORKSPACE - TORCH_MLIR_SRC_PYTORCH_BRANCH="$(cat pytorch-hash.txt)" \ - TM_PACKAGES="${{ matrix.llvm-build }}" \ - TM_USE_PYTORCH_BINARY="${{ matrix.torch-binary }}" \ - TM_PYTORCH_INSTALL_WITHOUT_REBUILD="${{ steps.cache-pytorch.outputs.cache-hit }}" \ - TM_TORCH_VERSION="${{ matrix.torch-version }}" \ - ./build_tools/python_deploy/build_linux_packages.sh + - name: Try to Restore PyTorch Build Cache + if: ${{ matrix.torch-binary == 'OFF' }} + id: cache-pytorch + uses: actions/cache/restore@v3 + with: + path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse + key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} - - name: Configure os-arch='macos-arm64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}' - # cross compile, can't test arm64 - if: ${{ matrix.os-arch == 'macos-arm64' && matrix.llvm-build == 'in-tree' }} - run: | - # TODO: Reenable LTC after build on macOS-arm64 is fixed (https://github.com/llvm/torch-mlir/issues/1253) - cmake -GNinja -Bbuild_arm64 \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_LINKER=lld \ - -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ - -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$GITHUB_WORKSPACE" \ - -DLLVM_TARGETS_TO_BUILD=AArch64 \ - -DLLVM_USE_HOST_TOOLS=ON \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DTORCH_MLIR_ENABLE_STABLEHLO=OFF \ - -DTORCH_MLIR_ENABLE_LTC=OFF \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${{ matrix.torch-binary }}" \ - -DMACOSX_DEPLOYMENT_TARGET=12.0 \ - -DPython3_EXECUTABLE="$(which python)" \ - $GITHUB_WORKSPACE/externals/llvm-project/llvm + - name: Build and Test os-arch='ubuntu-x86_64' llvm-build='${{ matrix.llvm-build }}' torch-binary='${{ matrix.torch-binary }}' + if: ${{ matrix.os-arch == 'ubuntu-x86_64' }} + run: | + cd $GITHUB_WORKSPACE + TORCH_MLIR_SRC_PYTORCH_BRANCH="$(cat pytorch-hash.txt)" \ + TM_PACKAGES="${{ matrix.llvm-build }}" \ + TM_USE_PYTORCH_BINARY="${{ matrix.torch-binary }}" \ + TM_PYTORCH_INSTALL_WITHOUT_REBUILD="${{ steps.cache-pytorch.outputs.cache-hit }}" \ + TM_TORCH_VERSION="${{ matrix.torch-version }}" \ + ./build_tools/python_deploy/build_linux_packages.sh - - name: Build torch-mlir (cross-compile) - if: ${{ matrix.os-arch == 'macos-arm64' }} - run: | - cmake --build build_arm64 + - name: Configure os-arch='macos-arm64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}' + # cross compile, can't test arm64 + if: ${{ matrix.os-arch == 'macos-arm64' && matrix.llvm-build == 'in-tree' }} + run: | + # TODO: Reenable LTC after build on macOS-arm64 is fixed (https://github.com/llvm/torch-mlir/issues/1253) + cmake -GNinja -Bbuild_arm64 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_LINKER=lld \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ + -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$GITHUB_WORKSPACE" \ + -DLLVM_TARGETS_TO_BUILD=AArch64 \ + -DLLVM_USE_HOST_TOOLS=ON \ + -DLLVM_ENABLE_ZSTD=OFF \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DTORCH_MLIR_ENABLE_STABLEHLO=OFF \ + -DTORCH_MLIR_ENABLE_LTC=OFF \ + -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${{ matrix.torch-binary }}" \ + -DMACOSX_DEPLOYMENT_TARGET=12.0 \ + -DPython3_EXECUTABLE="$(which python)" \ + $GITHUB_WORKSPACE/externals/llvm-project/llvm - - name: Build (Windows) - if: ${{ matrix.os-arch == 'windows-x86_64' }} - shell: bash - run: ./build_tools/python_deploy/build_windows_ci.sh + - name: Build torch-mlir (cross-compile) + if: ${{ matrix.os-arch == 'macos-arm64' }} + run: | + cmake --build build_arm64 - - name: Save PyTorch Build Cache - if: ${{ github.ref_name == 'main' && matrix.torch-binary == 'OFF' }} - uses: actions/cache/save@v3 - with: - path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse - key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} + - name: Build (Windows) + if: ${{ matrix.os-arch == 'windows-x86_64' }} + shell: bash + run: ./build_tools/python_deploy/build_windows_ci.sh - - name: Print ccache statistics - shell: bash - run: ccache --show-stats + - name: Save PyTorch Build Cache + if: ${{ github.ref_name == 'main' && matrix.torch-binary == 'OFF' }} + uses: actions/cache/save@v3 + with: + path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse + key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }} + + - name: Print ccache statistics + shell: bash + run: ccache --show-stats diff --git a/.github/workflows/buildRelease.yml b/.github/workflows/buildRelease.yml index 318882204..e84aabb4b 100644 --- a/.github/workflows/buildRelease.yml +++ b/.github/workflows/buildRelease.yml @@ -1,3 +1,4 @@ +# yamllint disable rule:line-length name: Release Build on: @@ -16,295 +17,293 @@ jobs: runs-on: a100 strategy: matrix: - package: [ torch-mlir ] - py_version: [ cp38-cp38, cp311-cp311 ] + package: [torch-mlir] + py_version: [cp38-cp38, cp311-cp311] steps: + - name: Prepare workspace + run: | + # Clear the workspace directory so that we don't run into errors about + # existing lock files. + sudo rm -rf $GITHUB_WORKSPACE/* - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* + - name: Get torch-mlir + uses: actions/checkout@v3 + with: + submodules: 'true' + fetch-depth: 0 - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - fetch-depth: 0 + - uses: ./.github/actions/setup-build + with: + cache-enabled: 'false' + - name: Build Python wheels and smoke test. + run: | + cd $GITHUB_WORKSPACE + TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} + printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version + TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} ./build_tools/python_deploy/build_linux_packages.sh - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Build Python wheels and smoke test. - run: | - cd $GITHUB_WORKSPACE - TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} - printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version - TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} ./build_tools/python_deploy/build_linux_packages.sh + # If we were given a release_id, then upload the package we just built + # to the github releases page. + - name: Upload Release Assets (if requested) + if: github.event.inputs.release_id != '' + id: upload-release-assets + uses: dwenegar/upload-release-assets@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl + # Publishing is necessary to make the release visible to `pip` + # on the github releases page. + - name: Publish Release (if requested) + if: github.event.inputs.release_id != '' + id: publish_release + uses: eregon/publish-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + - name: Create dist directory + if: github.event.inputs.release_id != '' + run: mkdir dist + - name: Copy releases to publish to dist directory + if: github.event.inputs.release_id != '' + run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist + # Wheels must be published from a linux environment. + # + # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 + - name: Store the binary wheel + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist build_linux_arm64: name: Manylinux arm64 Build runs-on: linux-arm64 strategy: matrix: - package: [ torch-mlir ] - py_version: [ cp311-cp311 ] + package: [torch-mlir] + py_version: [cp311-cp311] steps: + - name: Prepare workspace + run: | + # Clear the workspace directory so that we don't run into errors about + # existing lock files. + sudo rm -rf $GITHUB_WORKSPACE/* - - name: Prepare workspace - run: | - # Clear the workspace directory so that we don't run into errors about - # existing lock files. - sudo rm -rf $GITHUB_WORKSPACE/* + - name: Get torch-mlir + uses: actions/checkout@v3 + with: + submodules: 'true' + fetch-depth: 0 - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - fetch-depth: 0 + - uses: ./.github/actions/setup-build + with: + cache-enabled: 'false' + - name: Build Python wheels and smoke test. + run: | + cd $GITHUB_WORKSPACE + TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} + printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version + TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} TORCH_MLIR_ENABLE_LTC='0' ./build_tools/python_deploy/build_linux_packages.sh - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Build Python wheels and smoke test. - run: | - cd $GITHUB_WORKSPACE - TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} - printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version - TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} TORCH_MLIR_ENABLE_LTC='0' ./build_tools/python_deploy/build_linux_packages.sh + # If we were given a release_id, then upload the package we just built + # to the github releases page. + - name: Upload Release Assets (if requested) + if: github.event.inputs.release_id != '' + id: upload-release-assets + uses: dwenegar/upload-release-assets@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl + # Publishing is necessary to make the release visible to `pip` + # on the github releases page. + - name: Publish Release (if requested) + if: github.event.inputs.release_id != '' + id: publish_release + uses: eregon/publish-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + - name: Create dist directory + if: github.event.inputs.release_id != '' + run: mkdir dist + - name: Copy releases to publish to dist directory + if: github.event.inputs.release_id != '' + run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist + # Wheels must be published from a linux environment. + # + # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 + - name: Store the binary wheel + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist build_macos: name: MacOS Build runs-on: macos-latest strategy: matrix: - package: [ torch-mlir ] + package: [torch-mlir] steps: - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Build Python wheels and smoke test. - run: | - cd $GITHUB_WORKSPACE - python -m pip install wheel - TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} - printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version - sudo ./build_tools/python_deploy/install_macos_deps.sh - packages=${{ matrix.package }} TORCH_MLIR_PYTHON_VERSIONS="3.11" ./build_tools/python_deploy/build_macos_packages.sh + - name: Get torch-mlir + uses: actions/checkout@v3 + with: + submodules: 'true' + - uses: ./.github/actions/setup-build + with: + cache-enabled: 'false' + - name: Build Python wheels and smoke test. + run: | + cd $GITHUB_WORKSPACE + python -m pip install wheel + TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }} + printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version + sudo ./build_tools/python_deploy/install_macos_deps.sh + packages=${{ matrix.package }} TORCH_MLIR_PYTHON_VERSIONS="3.11" ./build_tools/python_deploy/build_macos_packages.sh - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ + # If we were given a release_id, then upload the package we just built + # to the github releases page. + - name: Upload Release Assets (if requested) + if: github.event.inputs.release_id != '' + id: upload-release-assets + uses: dwenegar/upload-release-assets@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl + # Publishing is necessary to make the release visible to `pip` + # on the github releases page. + - name: Publish Release (if requested) + if: github.event.inputs.release_id != '' + id: publish_release + uses: eregon/publish-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + - name: Create dist directory + if: github.event.inputs.release_id != '' + run: mkdir dist + - name: Copy releases to publish to dist directory + if: github.event.inputs.release_id != '' + run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/ - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist + # Wheels must be published from a linux environment. + # + # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 + - name: Store the binary wheel + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist build_windows: name: Windows Build runs-on: windows-latest strategy: matrix: - package: [ torch-mlir ] + package: [torch-mlir] steps: - - name: Get torch-mlir - uses: actions/checkout@v3 - with: - submodules: 'true' - - uses: ./.github/actions/setup-build - with: - cache-enabled: 'false' - - name: Set up Visual Studio shell - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - name: Build Python wheels and smoke test. - shell: pwsh - run: | - $env:TORCH_MLIR_ENABLE_JIT_IR_IMPORTER='1' - $env:TORCH_MLIR_ENABLE_ONLY_MLIR_PYTHON_BINDINGS='0' - $env:TORCH_MLIR_PYTHON_PACKAGE_VERSION = '${{ github.event.inputs.python_package_version }}' - ./build_tools/python_deploy/build_windows.ps1 + - name: Get torch-mlir + uses: actions/checkout@v3 + with: + submodules: 'true' + - uses: ./.github/actions/setup-build + with: + cache-enabled: 'false' + - name: Set up Visual Studio shell + uses: egor-tensin/vs-shell@v2 + with: + arch: x64 + - name: Build Python wheels and smoke test. + shell: pwsh + run: | + $env:TORCH_MLIR_ENABLE_JIT_IR_IMPORTER='1' + $env:TORCH_MLIR_ENABLE_ONLY_MLIR_PYTHON_BINDINGS='0' + $env:TORCH_MLIR_PYTHON_PACKAGE_VERSION = '${{ github.event.inputs.python_package_version }}' + ./build_tools/python_deploy/build_windows.ps1 - # If we were given a release_id, then upload the package we just built - # to the github releases page. - - name: Upload Release Assets (if requested) - if: github.event.inputs.release_id != '' - id: upload-release-assets - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - assets_path: ./wheelhouse/torch*.whl - # Publishing is necessary to make the release visible to `pip` - # on the github releases page. - - name: Publish Release (if requested) - if: github.event.inputs.release_id != '' - id: publish_release - uses: eregon/publish-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - with: - release_id: ${{ github.event.inputs.release_id }} - - name: Create dist directory - if: github.event.inputs.release_id != '' - run: mkdir dist - continue-on-error: true - - name: Copy releases to publish to dist directory - if: github.event.inputs.release_id != '' - run: cp ./wheelhouse/torch_mlir*.whl dist/ + # If we were given a release_id, then upload the package we just built + # to the github releases page. + - name: Upload Release Assets (if requested) + if: github.event.inputs.release_id != '' + id: upload-release-assets + uses: dwenegar/upload-release-assets@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + assets_path: ./wheelhouse/torch*.whl + # Publishing is necessary to make the release visible to `pip` + # on the github releases page. + - name: Publish Release (if requested) + if: github.event.inputs.release_id != '' + id: publish_release + uses: eregon/publish-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + with: + release_id: ${{ github.event.inputs.release_id }} + - name: Create dist directory + if: github.event.inputs.release_id != '' + run: mkdir dist + continue-on-error: true + - name: Copy releases to publish to dist directory + if: github.event.inputs.release_id != '' + run: cp ./wheelhouse/torch_mlir*.whl dist/ - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - - name: Store the binary wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist + # Wheels must be published from a linux environment. + # + # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 + - name: Store the binary wheel + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist publish_releases: runs-on: ubuntu-latest needs: - - build_linux - - build_linux_arm64 - - build_macos - - build_windows + - build_linux + - build_linux_arm64 + - build_macos + - build_windows # Publish even if one of the builds failed if: ${{ always() }} steps: - - name: Invoke Publish Releases Page - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Publish releases page - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + - name: Invoke Publish Releases Page + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: Publish releases page + token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - # Wheels must be published from a linux environment. - # - # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 - # - # We're temporarily disabling pypi publishing until we can fix audit wheel - # ODR torch issues. See https://github.com/llvm/torch-mlir/issues/1709 - # - #- name: Download wheels for publishing to PyPI - # uses: actions/download-artifact@v3 - # with: - # name: wheels - # path: dist - #- name: Publish to PyPI - # if: github.event.inputs.release_id != '' - # uses: pypa/gh-action-pypi-publish@v1.5.1 - # with: - # password: ${{ secrets.PYPI_API_TOKEN }} + # Wheels must be published from a linux environment. + # + # See https://github.com/pypa/gh-action-pypi-publish/discussions/15 + # + # We're temporarily disabling pypi publishing until we can fix audit wheel + # ODR torch issues. See https://github.com/llvm/torch-mlir/issues/1709 + # + #- name: Download wheels for publishing to PyPI + # uses: actions/download-artifact@v3 + # with: + # name: wheels + # path: dist + #- name: Publish to PyPI + # if: github.event.inputs.release_id != '' + # uses: pypa/gh-action-pypi-publish@v1.5.1 + # with: + # password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/gh-pages-releases.yml b/.github/workflows/gh-pages-releases.yml index c6df475cc..a0eb45257 100644 --- a/.github/workflows/gh-pages-releases.yml +++ b/.github/workflows/gh-pages-releases.yml @@ -1,3 +1,4 @@ +# yamllint disable rule:line-length # See: https://github.com/llvm/torch-mlir/issues/1374 name: Publish releases page diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..464ebdad9 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,17 @@ +name: Lint Checks + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + static_lint_checks: + name: Static Lint Checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Validate GitHub Actions yaml files + run: | + yamllint ./.github/workflows/ ./.github/actions/ diff --git a/.github/workflows/merge-rollpytorch.yml b/.github/workflows/merge-rollpytorch.yml index 7247a3683..58a91fd1d 100644 --- a/.github/workflows/merge-rollpytorch.yml +++ b/.github/workflows/merge-rollpytorch.yml @@ -1,3 +1,4 @@ +# yamllint disable rule:line-length name: RollPyTorch Merge on: @@ -15,19 +16,19 @@ jobs: github.event.workflow_run.conclusion == 'success' steps: - # Fetch the repo first so that the gh command knows where to look for the PR - - name: Fetch Repo - uses: actions/checkout@v3 - with: - token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} + # Fetch the repo first so that the gh command knows where to look for the PR + - name: Fetch Repo + uses: actions/checkout@v3 + with: + token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} - - name: Merge RollPyTorch PR - run: | - for pr_id in ${{ join(github.event.workflow_run.pull_requests.*.number, ' ') }} - do - echo "Merging PR: $pr_id" - gh pr merge $pr_id --delete-branch --squash - done - shell: bash - env: - GH_TOKEN: ${{ secrets.ROLLPYTORCH_TOKEN1 }} + - name: Merge RollPyTorch PR + run: | + for pr_id in ${{ join(github.event.workflow_run.pull_requests.*.number, ' ') }} + do + echo "Merging PR: $pr_id" + gh pr merge $pr_id --delete-branch --squash + done + shell: bash + env: + GH_TOKEN: ${{ secrets.ROLLPYTORCH_TOKEN1 }} diff --git a/.github/workflows/oneshotSnapshotPackage.yml b/.github/workflows/oneshotSnapshotPackage.yml index 46832ce9c..ec1878606 100644 --- a/.github/workflows/oneshotSnapshotPackage.yml +++ b/.github/workflows/oneshotSnapshotPackage.yml @@ -1,3 +1,4 @@ +# yamllint disable rule:line-length name: Release oneshot snapshot package on: diff --git a/.github/workflows/releaseSnapshotPackage.yml b/.github/workflows/releaseSnapshotPackage.yml index c18eff88d..71acd9ac3 100644 --- a/.github/workflows/releaseSnapshotPackage.yml +++ b/.github/workflows/releaseSnapshotPackage.yml @@ -1,3 +1,4 @@ +# yamllint disable rule:line-length name: Release snapshot package on: