From d2d884212ab34d1d0b3109c8ca62a20bf4bbca79 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 01:31:07 +0000 Subject: [PATCH 01/25] Added cache to restore conda env --- .github/actions/initial_setup/action.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index c64e839..699ef13 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -11,15 +11,30 @@ inputs: type: string description: Compiler used for execution +env: + conda_yaml_fille: tests/conda-envs/${{ inputs.conda_env }}.yaml + runs: using: composite steps: + - name: Get Date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore Conda env cache + id: conda-cache + uses: actions/cache@v3 + with: + path: ${{ conda_yaml_fille }} + key: conda-${{ inputs.conda_env }}-${{ hashFiles(conda_yaml_fille) }}-${{ steps.get-date.outputs.today }} + - uses: conda-incubator/setup-miniconda@v2 with: - miniconda-version: "latest" + miniforge-variant: Mambaforge auto-update-conda: true use-mamba: true - environment-file: tests/conda-envs/${{ inputs.conda_env }}.yaml + environment-file: ${{ conda_yaml_fille }} activate-environment: ${{ inputs.conda_env }}-test - uses: ./.github/actions/build_mlir From 11e0dc00e1fa441c2907c2596f8d1c2fc93b5faf Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 01:38:20 +0000 Subject: [PATCH 02/25] Added condition for conda checkout Signed-off-by: Gregory Shimansky --- .github/actions/initial_setup/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 699ef13..3f88fed 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -30,6 +30,7 @@ runs: key: conda-${{ inputs.conda_env }}-${{ hashFiles(conda_yaml_fille) }}-${{ steps.get-date.outputs.today }} - uses: conda-incubator/setup-miniconda@v2 + if: steps.conda-cache.outputs.cache-hit != 'true' with: miniforge-variant: Mambaforge auto-update-conda: true From 4714ab189502f5306d8b42aa987ff8cdc358e80f Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 02:02:11 +0000 Subject: [PATCH 03/25] Moved conda install before cache, then env update Signed-off-by: Gregory Shimansky --- .github/actions/initial_setup/action.yml | 29 ++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 3f88fed..d6f34ae 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -12,11 +12,19 @@ inputs: description: Compiler used for execution env: - conda_yaml_fille: tests/conda-envs/${{ inputs.conda_env }}.yaml + CONDA_YAML_FILE: tests/conda-envs/${{ inputs.conda_env }}.yaml + CONDA_ENV_NAME: ${{ inputs.conda_env }}-test runs: using: composite steps: + - uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + activate-environment: ${{ env.CONDA_ENV_NAME }} + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT @@ -25,18 +33,16 @@ runs: - name: Restore Conda env cache id: conda-cache uses: actions/cache@v3 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 with: - path: ${{ conda_yaml_fille }} - key: conda-${{ inputs.conda_env }}-${{ hashFiles(conda_yaml_fille) }}-${{ steps.get-date.outputs.today }} + path: ${{ env.CONDA }}/envs + key: conda-${{ inputs.conda_env }}-${{ hashFiles(env.CONDA_YAML_FILE) }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }} - - uses: conda-incubator/setup-miniconda@v2 + - name: Update conda env if: steps.conda-cache.outputs.cache-hit != 'true' - with: - miniforge-variant: Mambaforge - auto-update-conda: true - use-mamba: true - environment-file: ${{ conda_yaml_fille }} - activate-environment: ${{ inputs.conda_env }}-test + run: mamba env update -n ${{ env.CONDA_ENV_NAME }} -f ${{ env.CONDA_YAML_FILE }} - uses: ./.github/actions/build_mlir if: ${{ inputs.conda_env == 'mlir' }} @@ -50,5 +56,4 @@ runs: - name: Setup benchmarks package shell: bash -el {0} - run: | - pip install -e . + run: pip install -e . From 0cb0f9ed25fe3784a8ddb7fbc75796dc4d221b78 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 02:11:18 +0000 Subject: [PATCH 04/25] Fixed env, added shell Signed-off-by: Gregory Shimansky --- .github/actions/initial_setup/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index d6f34ae..792065e 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -11,12 +11,11 @@ inputs: type: string description: Compiler used for execution -env: - CONDA_YAML_FILE: tests/conda-envs/${{ inputs.conda_env }}.yaml - CONDA_ENV_NAME: ${{ inputs.conda_env }}-test - runs: using: composite + env: + CONDA_YAML_FILE: tests/conda-envs/${{ inputs.conda_env }}.yaml + CONDA_ENV_NAME: ${{ inputs.conda_env }}-test steps: - uses: conda-incubator/setup-miniconda@v2 with: @@ -42,6 +41,7 @@ runs: - name: Update conda env if: steps.conda-cache.outputs.cache-hit != 'true' + shell: bash -el {0} run: mamba env update -n ${{ env.CONDA_ENV_NAME }} -f ${{ env.CONDA_YAML_FILE }} - uses: ./.github/actions/build_mlir From 7da90f600f3c12694ff4ba4aab8f0f2c976da31f Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 14:27:45 +0000 Subject: [PATCH 05/25] Remove env because it is not allowed Signed-off-by: Gregory Shimansky --- .github/actions/initial_setup/action.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 792065e..03163f1 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -13,16 +13,13 @@ inputs: runs: using: composite - env: - CONDA_YAML_FILE: tests/conda-envs/${{ inputs.conda_env }}.yaml - CONDA_ENV_NAME: ${{ inputs.conda_env }}-test steps: - uses: conda-incubator/setup-miniconda@v2 with: miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true - activate-environment: ${{ env.CONDA_ENV_NAME }} + activate-environment: ${{ inputs.conda_env }}-test - name: Get Date id: get-date @@ -37,12 +34,12 @@ runs: CACHE_NUMBER: 0 with: path: ${{ env.CONDA }}/envs - key: conda-${{ inputs.conda_env }}-${{ hashFiles(env.CONDA_YAML_FILE) }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }} + key: conda-${{ inputs.conda_env }}-${{ hashFiles('tests/conda-envs/' + inputs.conda_env + '.yaml') }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }} - name: Update conda env if: steps.conda-cache.outputs.cache-hit != 'true' shell: bash -el {0} - run: mamba env update -n ${{ env.CONDA_ENV_NAME }} -f ${{ env.CONDA_YAML_FILE }} + run: mamba env update -n ${{ inputs.conda_env }}-test -f tests/conda-envs/${{ inputs.conda_env }}.yaml - uses: ./.github/actions/build_mlir if: ${{ inputs.conda_env == 'mlir' }} From bf979453b5812829ea1930b41f760d162ee79eef Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 14:35:51 +0000 Subject: [PATCH 06/25] Fixed string concatenation with format Signed-off-by: Gregory Shimansky --- .github/actions/initial_setup/action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 03163f1..2984b64 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -34,7 +34,11 @@ runs: CACHE_NUMBER: 0 with: path: ${{ env.CONDA }}/envs - key: conda-${{ inputs.conda_env }}-${{ hashFiles('tests/conda-envs/' + inputs.conda_env + '.yaml') }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }} + key: >- + conda-${{ inputs.conda_env }}- + ${{ hashFiles(format('tests/conda-envs/{0}.yaml', inputs.conda_env)) }}- + ${{ steps.get-date.outputs.today }}- + ${{ env.CACHE_NUMBER }} - name: Update conda env if: steps.conda-cache.outputs.cache-hit != 'true' From 2d80ac7dcb747f0ce67d741d7338f5f3027aac27 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 16:26:16 +0000 Subject: [PATCH 07/25] Make building mlir a separate job that runs before size matrix Signed-off-by: Gregory Shimansky --- .github/actions/build_mlir/action.yml | 8 ++++++++ .github/actions/initial_setup/action.yml | 5 ++++- .github/workflows/mlp-test.yml | 10 ++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index b2e6e49..dab241b 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -11,6 +11,7 @@ runs: path: torch-mlir ref: 'cpu-proto' submodules: recursive + - name: Install system and Intel packages shell: bash -el {0} run: | @@ -18,6 +19,7 @@ runs: echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y build-essential intel-basekit + - name: Build torch-mlir shell: bash -el {0} run: | @@ -25,3 +27,9 @@ runs: tests/scripts/build-mlir.sh torch-mlir echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV + + - name: Upload built mlir + uses: actions/upload-artifact@v3 + name: torch-mlir-binary + path: torch-mlir/ + if-no-files-found: error diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 2984b64..bff7f05 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -45,7 +45,10 @@ runs: shell: bash -el {0} run: mamba env update -n ${{ inputs.conda_env }}-test -f tests/conda-envs/${{ inputs.conda_env }}.yaml - - uses: ./.github/actions/build_mlir + - name: Download mlir that was built in previous job + uses: actions/download-artifact@v3 + name: torch-mlir-binary + path: torch-mlir if: ${{ inputs.conda_env == 'mlir' }} - name: Install compiler for dynamo diff --git a/.github/workflows/mlp-test.yml b/.github/workflows/mlp-test.yml index 4c4c61b..60fd69e 100644 --- a/.github/workflows/mlp-test.yml +++ b/.github/workflows/mlp-test.yml @@ -20,10 +20,20 @@ on: required: true jobs: + build_mlir: + if: ${{ inputs.conda_env == 'mlir' }} + runs-on: + - self-hosted + - glados + steps: + - name: Build mlir and upload to artifact storage + uses: ./.github/actions/build_mlir + mlp_test: runs-on: - self-hosted - glados + needs: build_mlir strategy: matrix: bname: From c8d8b55d4bc099130e6709c164e1bb1ec53c7786 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 20:53:38 +0000 Subject: [PATCH 08/25] Moved checkout to first job Signed-off-by: Gregory Shimansky --- .github/workflows/mlp-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mlp-test.yml b/.github/workflows/mlp-test.yml index 60fd69e..977ad98 100644 --- a/.github/workflows/mlp-test.yml +++ b/.github/workflows/mlp-test.yml @@ -26,6 +26,8 @@ jobs: - self-hosted - glados steps: + - uses: actions/checkout@v4 + - name: Build mlir and upload to artifact storage uses: ./.github/actions/build_mlir @@ -42,7 +44,6 @@ jobs: - size4 - size5 steps: - - uses: actions/checkout@v4 - uses: ./.github/actions/initial_setup with: conda_env: ${{ inputs.conda_env }} From 5e5f6f3f433e235fc3469c32a08f39f4e3806771 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Thu, 30 Nov 2023 22:15:33 +0000 Subject: [PATCH 09/25] Adde "with" to artifact calls Signed-off-by: Gregory Shimansky --- .github/actions/build_mlir/action.yml | 1 + .github/actions/initial_setup/action.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index dab241b..ce8cf1d 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -30,6 +30,7 @@ runs: - name: Upload built mlir uses: actions/upload-artifact@v3 + with: name: torch-mlir-binary path: torch-mlir/ if-no-files-found: error diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index bff7f05..057fd84 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -41,12 +41,13 @@ runs: ${{ env.CACHE_NUMBER }} - name: Update conda env - if: steps.conda-cache.outputs.cache-hit != 'true' shell: bash -el {0} run: mamba env update -n ${{ inputs.conda_env }}-test -f tests/conda-envs/${{ inputs.conda_env }}.yaml + if: steps.conda-cache.outputs.cache-hit != 'true' - name: Download mlir that was built in previous job uses: actions/download-artifact@v3 + with: name: torch-mlir-binary path: torch-mlir if: ${{ inputs.conda_env == 'mlir' }} From 7933d8a00c2a9274a1cfb8582bc20b89369f7a1b Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Mon, 4 Dec 2023 21:53:54 +0000 Subject: [PATCH 10/25] Added restoring and creating of torch-mlir env before building torch-mlir --- .github/actions/build_mlir/action.yml | 47 ++++++++++++++++--- .../create_torch_mlir_conda_env/action.yml | 25 ++++++++++ .github/actions/initial_setup/action.yml | 10 +++- .github/workflows/mlp-test.yml | 2 + tests/scripts/build-mlir.sh | 3 -- 5 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 .github/actions/create_torch_mlir_conda_env/action.yml diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index a571f1b..61b1645 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -1,21 +1,54 @@ name: Build torch-mlir description: Checkout torch-mlir repo and build it in mlir-test conda env +inputs: + conda_env: + required: true + type: string + description: Name of torch-mlir conda env + runs: using: composite steps: - - name: Checkout torch-mlir repo - uses: actions/checkout@v4 + - name: Get Date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore Conda env cache + id: conda-cache + uses: actions/cache@v3 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 2 with: - repository: intel-ai/torch-mlir - path: torch-mlir - ref: 'cpu-proto' - submodules: recursive + path: ${{ env.CONDA }}/envs + key: >- + conda-${{ inputs.conda_env }}- + ${{ hashFiles(format('tests/conda-envs/{0}.yaml', inputs.conda_env)) }}- + ${{ steps.get-date.outputs.today }}- + ${{ env.CACHE_NUMBER }} + + - name: Update conda env for torch-mlir env + if: steps.conda-cache.outputs.cache-hit != 'true' + uses: ./.github/actions/create_torch_mlir_conda_env + with: + conda_env: $${{ inputs.conda_env }} + - name: Build torch-mlir shell: bash -el {0} run: | + source ${CONDA}/bin/activate ${{ inputs.conda_env }} source /opt/intel/oneapi/setvars.sh - tests/scripts/build-mlir.sh torch-mlir + cmake -GNinja -Bbuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython3_FIND_VIRTUALENV=ONLY \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ + -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$PWD" \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DLLVM_TARGETS_TO_BUILD=host \ + externals/llvm-project/llvm echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV diff --git a/.github/actions/create_torch_mlir_conda_env/action.yml b/.github/actions/create_torch_mlir_conda_env/action.yml new file mode 100644 index 0000000..0e74074 --- /dev/null +++ b/.github/actions/create_torch_mlir_conda_env/action.yml @@ -0,0 +1,25 @@ +name: Install torch-mlir required python packages +description: Checkout torch-mlir repo and install required packages in an empty conda env + +inputs: + conda_env: + required: true + type: string + description: Name of torch-mlir conda env + +runs: + using: composite + - name: Checkout torch-mlir repo + uses: actions/checkout@v4 + with: + repository: intel-ai/torch-mlir + path: torch-mlir + ref: 'cpu-proto' + submodules: recursive + + - name: Install required packages + run: | + ${CONDA}/bin/conda env update --solver libmamba -n ${{ inputs.conda_env }} -f tests/conda-envs/${{ inputs.conda_env }}.yaml + source ${CONDA}/bin/activate ${{ inputs.conda_env }} + cd torch-mlir + pip install -r requirements.txt diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index e77ceea..b2fe472 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -33,11 +33,17 @@ runs: ${{ steps.get-date.outputs.today }}- ${{ env.CACHE_NUMBER }} - - name: Update conda env - if: steps.conda-cache.outputs.cache-hit != 'true' + - name: Update conda env for non-torch-mlir envs + if: steps.conda-cache.outputs.cache-hit != 'true' && inputs.conda_env != 'mlir' shell: bash -el {0} run: ${CONDA}/bin/conda env update --solver libmamba -n ${{ inputs.conda_env }} -f tests/conda-envs/${{ inputs.conda_env }}.yaml + - name: Update conda env for torch-mlir env + if: steps.conda-cache.outputs.cache-hit != 'true' && inputs.conda_env == 'mlir' + uses: ./.github/actions/create_torch_mlir_conda_env + with: + conda_env: $${{ inputs.conda_env }} + - name: Download mlir that was built in previous job uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/mlp-test.yml b/.github/workflows/mlp-test.yml index c71d4e2..e0a3906 100644 --- a/.github/workflows/mlp-test.yml +++ b/.github/workflows/mlp-test.yml @@ -30,6 +30,8 @@ jobs: - name: Build mlir and upload to artifact storage uses: ./.github/actions/build_mlir + with: + conda_env: $${{ inputs.conda_env }} mlp_test: runs-on: diff --git a/tests/scripts/build-mlir.sh b/tests/scripts/build-mlir.sh index 94c91ae..a1cdb8d 100755 --- a/tests/scripts/build-mlir.sh +++ b/tests/scripts/build-mlir.sh @@ -6,9 +6,6 @@ fi source ${CONDA}/bin/activate mlir -env -${CONDA}/bin/conda list - cd $1 pip install -r requirements.txt From 77fc84d3f84f8a41993294724f998d0621135249 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Mon, 4 Dec 2023 21:59:35 +0000 Subject: [PATCH 11/25] Added missing "steps" --- .github/actions/create_torch_mlir_conda_env/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/create_torch_mlir_conda_env/action.yml b/.github/actions/create_torch_mlir_conda_env/action.yml index 0e74074..91b11d1 100644 --- a/.github/actions/create_torch_mlir_conda_env/action.yml +++ b/.github/actions/create_torch_mlir_conda_env/action.yml @@ -9,6 +9,7 @@ inputs: runs: using: composite + steps: - name: Checkout torch-mlir repo uses: actions/checkout@v4 with: From 333f4e05dc0f485443de2dc068d4c7934c1667e5 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Mon, 4 Dec 2023 22:11:37 +0000 Subject: [PATCH 12/25] Added missing shell --- .github/actions/create_torch_mlir_conda_env/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/create_torch_mlir_conda_env/action.yml b/.github/actions/create_torch_mlir_conda_env/action.yml index 91b11d1..0a54041 100644 --- a/.github/actions/create_torch_mlir_conda_env/action.yml +++ b/.github/actions/create_torch_mlir_conda_env/action.yml @@ -19,6 +19,7 @@ runs: submodules: recursive - name: Install required packages + shell: bash -el {0} run: | ${CONDA}/bin/conda env update --solver libmamba -n ${{ inputs.conda_env }} -f tests/conda-envs/${{ inputs.conda_env }}.yaml source ${CONDA}/bin/activate ${{ inputs.conda_env }} From 6c746e52a27772b9bb095f6df23a74dc91f45722 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Mon, 4 Dec 2023 23:17:02 +0000 Subject: [PATCH 13/25] Added repo checkout --- .github/actions/build_mlir/action.yml | 2 ++ .github/actions/initial_setup/action.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index 61b1645..97b4393 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -10,6 +10,8 @@ inputs: runs: using: composite steps: + - uses: actions/checkout@v4 + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index b2fe472..500e9b4 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -14,6 +14,8 @@ inputs: runs: using: composite steps: + - uses: actions/checkout@v4 + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT From c6036098ddc6c7540614402d030e177cd8f20d60 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Mon, 4 Dec 2023 23:22:29 +0000 Subject: [PATCH 14/25] Fixed conda env name --- .github/actions/build_mlir/action.yml | 4 +--- .github/workflows/mlp-test.yml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index 97b4393..de41f32 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -1,5 +1,5 @@ name: Build torch-mlir -description: Checkout torch-mlir repo and build it in mlir-test conda env +description: Checkout torch-mlir repo and build it in mlir conda env inputs: conda_env: @@ -10,8 +10,6 @@ inputs: runs: using: composite steps: - - uses: actions/checkout@v4 - - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/mlp-test.yml b/.github/workflows/mlp-test.yml index e0a3906..c969cb4 100644 --- a/.github/workflows/mlp-test.yml +++ b/.github/workflows/mlp-test.yml @@ -31,7 +31,7 @@ jobs: - name: Build mlir and upload to artifact storage uses: ./.github/actions/build_mlir with: - conda_env: $${{ inputs.conda_env }} + conda_env: ${{ inputs.conda_env }} mlp_test: runs-on: From 0da05829fe11513b164e92c8eaf86cd2159a489b Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Mon, 4 Dec 2023 23:29:46 +0000 Subject: [PATCH 15/25] Fixed conda env parameter --- .github/actions/build_mlir/action.yml | 2 +- .github/actions/initial_setup/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index de41f32..0ba048d 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -33,7 +33,7 @@ runs: if: steps.conda-cache.outputs.cache-hit != 'true' uses: ./.github/actions/create_torch_mlir_conda_env with: - conda_env: $${{ inputs.conda_env }} + conda_env: ${{ inputs.conda_env }} - name: Build torch-mlir shell: bash -el {0} diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 500e9b4..aabae18 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -44,7 +44,7 @@ runs: if: steps.conda-cache.outputs.cache-hit != 'true' && inputs.conda_env == 'mlir' uses: ./.github/actions/create_torch_mlir_conda_env with: - conda_env: $${{ inputs.conda_env }} + conda_env: ${{ inputs.conda_env }} - name: Download mlir that was built in previous job uses: actions/download-artifact@v3 From 72d8d44e1ba565693ec8eab2af7c47b9925bb30e Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Mon, 4 Dec 2023 23:36:09 +0000 Subject: [PATCH 16/25] Added missing cd, moved checkout to build action --- .github/actions/build_mlir/action.yml | 9 +++++++++ .github/actions/create_torch_mlir_conda_env/action.yml | 8 -------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index 0ba048d..f0dc191 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -10,6 +10,14 @@ inputs: runs: using: composite steps: + - name: Checkout torch-mlir repo + uses: actions/checkout@v4 + with: + repository: intel-ai/torch-mlir + path: torch-mlir + ref: 'cpu-proto' + submodules: recursive + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT @@ -40,6 +48,7 @@ runs: run: | source ${CONDA}/bin/activate ${{ inputs.conda_env }} source /opt/intel/oneapi/setvars.sh + cd torch-mlir cmake -GNinja -Bbuild \ -DCMAKE_BUILD_TYPE=Release \ -DPython3_FIND_VIRTUALENV=ONLY \ diff --git a/.github/actions/create_torch_mlir_conda_env/action.yml b/.github/actions/create_torch_mlir_conda_env/action.yml index 0a54041..f7afe3e 100644 --- a/.github/actions/create_torch_mlir_conda_env/action.yml +++ b/.github/actions/create_torch_mlir_conda_env/action.yml @@ -10,14 +10,6 @@ inputs: runs: using: composite steps: - - name: Checkout torch-mlir repo - uses: actions/checkout@v4 - with: - repository: intel-ai/torch-mlir - path: torch-mlir - ref: 'cpu-proto' - submodules: recursive - - name: Install required packages shell: bash -el {0} run: | From 186f835248911d1145ac5ccd2b2e122ab5b1a478 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 00:15:44 +0000 Subject: [PATCH 17/25] Use tar instead of moving separate files into artifactory --- .github/actions/build_mlir/action.yml | 4 +++- .github/actions/initial_setup/action.yml | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index f0dc191..afe4ab8 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -58,6 +58,8 @@ runs: -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DLLVM_TARGETS_TO_BUILD=host \ externals/llvm-project/llvm + cd .. + tar cf torch-mlir.tar torch-mlir echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV @@ -65,5 +67,5 @@ runs: uses: actions/upload-artifact@v3 with: name: torch-mlir-binary - path: torch-mlir/ + path: torch-mlir.tar if-no-files-found: error diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index aabae18..3cbcf9c 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -50,11 +50,12 @@ runs: uses: actions/download-artifact@v3 with: name: torch-mlir-binary - path: torch-mlir + path: torch-mlir.tar if: ${{ inputs.conda_env == 'mlir' }} - name: Setup benchmarks package shell: bash -el {0} run: | + tar xf torch-mlir.tar source ${CONDA}/bin/activate ${{ inputs.conda_env }} pip install -e . From b6f88a90a71ec2a6e9bf66f613e1178e51de3839 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 00:27:14 +0000 Subject: [PATCH 18/25] Moved checkout before using actions --- .github/actions/initial_setup/action.yml | 2 -- .github/workflows/mlp-test.yml | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 3cbcf9c..ecc7252 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -14,8 +14,6 @@ inputs: runs: using: composite steps: - - uses: actions/checkout@v4 - - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/mlp-test.yml b/.github/workflows/mlp-test.yml index c969cb4..b988488 100644 --- a/.github/workflows/mlp-test.yml +++ b/.github/workflows/mlp-test.yml @@ -67,6 +67,8 @@ jobs: - {bname: 'size5', batch_size: 8196} fail-fast: false steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/initial_setup with: conda_env: ${{ inputs.conda_env }} From fe88b14ba2a69b81716634e81e6f0ccdf4b7bfda Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 00:50:09 +0000 Subject: [PATCH 19/25] Moved artifact restore to the beginning --- .github/actions/initial_setup/action.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index ecc7252..b7dbd5a 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -14,6 +14,18 @@ inputs: runs: using: composite steps: + - name: Download mlir that was built in previous job + if: ${{ inputs.conda_env == 'mlir' }} + uses: actions/download-artifact@v3 + with: + name: torch-mlir-binary + path: torch-mlir.tar + + - name: Setup benchmarks package + if: ${{ inputs.conda_env == 'mlir' }} + shell: bash -el {0} + run: tar xf torch-mlir.tar + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT @@ -44,16 +56,8 @@ runs: with: conda_env: ${{ inputs.conda_env }} - - name: Download mlir that was built in previous job - uses: actions/download-artifact@v3 - with: - name: torch-mlir-binary - path: torch-mlir.tar - if: ${{ inputs.conda_env == 'mlir' }} - - name: Setup benchmarks package shell: bash -el {0} run: | - tar xf torch-mlir.tar source ${CONDA}/bin/activate ${{ inputs.conda_env }} pip install -e . From 5bd72f95e2fbb689f29f7fbe081249bf937a43e2 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 00:56:12 +0000 Subject: [PATCH 20/25] Added missing build --- .github/actions/build_mlir/action.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/actions/build_mlir/action.yml b/.github/actions/build_mlir/action.yml index afe4ab8..ee3f80a 100644 --- a/.github/actions/build_mlir/action.yml +++ b/.github/actions/build_mlir/action.yml @@ -50,14 +50,15 @@ runs: source /opt/intel/oneapi/setvars.sh cd torch-mlir cmake -GNinja -Bbuild \ - -DCMAKE_BUILD_TYPE=Release \ - -DPython3_FIND_VIRTUALENV=ONLY \ - -DLLVM_ENABLE_PROJECTS=mlir \ - -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ - -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$PWD" \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DLLVM_TARGETS_TO_BUILD=host \ - externals/llvm-project/llvm + -DCMAKE_BUILD_TYPE=Release \ + -DPython3_FIND_VIRTUALENV=ONLY \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \ + -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$PWD" \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DLLVM_TARGETS_TO_BUILD=host \ + externals/llvm-project/llvm + cmake --build build cd .. tar cf torch-mlir.tar torch-mlir echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV From e3fb6b4d0095222e5e7169fcdbb68374792c321d Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 01:24:24 +0000 Subject: [PATCH 21/25] DEBUG1 --- .github/actions/initial_setup/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index b7dbd5a..04a5ac6 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -24,7 +24,10 @@ runs: - name: Setup benchmarks package if: ${{ inputs.conda_env == 'mlir' }} shell: bash -el {0} - run: tar xf torch-mlir.tar + run: | + pwd + ls -la + tar xf torch-mlir.tar - name: Get Date id: get-date From 5e001a18dd17ea8c94fce75849c78045d8b0a798 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 01:47:26 +0000 Subject: [PATCH 22/25] Removed path argument --- .github/actions/initial_setup/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index 04a5ac6..f42905e 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -19,7 +19,6 @@ runs: uses: actions/download-artifact@v3 with: name: torch-mlir-binary - path: torch-mlir.tar - name: Setup benchmarks package if: ${{ inputs.conda_env == 'mlir' }} From 01967f34a17f18c97133550e2e788a4598949252 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 02:16:21 +0000 Subject: [PATCH 23/25] Added environment initialization in initial initial setup --- .github/actions/initial_setup/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index f42905e..d23f919 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -24,9 +24,11 @@ runs: if: ${{ inputs.conda_env == 'mlir' }} shell: bash -el {0} run: | - pwd - ls -la tar xf torch-mlir.tar + source ${CONDA}/bin/activate ${{ inputs.conda_env }} + source /opt/intel/oneapi/setvars.sh + echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV + echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV - name: Get Date id: get-date From 4495927c85716bb7ff335e13e5c3c59c9e7ed65a Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 02:38:39 +0000 Subject: [PATCH 24/25] Moved environment setting to the end of action --- .github/actions/initial_setup/action.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/actions/initial_setup/action.yml b/.github/actions/initial_setup/action.yml index d23f919..897bd57 100644 --- a/.github/actions/initial_setup/action.yml +++ b/.github/actions/initial_setup/action.yml @@ -23,12 +23,7 @@ runs: - name: Setup benchmarks package if: ${{ inputs.conda_env == 'mlir' }} shell: bash -el {0} - run: | - tar xf torch-mlir.tar - source ${CONDA}/bin/activate ${{ inputs.conda_env }} - source /opt/intel/oneapi/setvars.sh - echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV - echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV + run: tar xf torch-mlir.tar - name: Get Date id: get-date @@ -65,3 +60,12 @@ runs: run: | source ${CONDA}/bin/activate ${{ inputs.conda_env }} pip install -e . + + - name: Save environment variables for mlir env + if: ${{ inputs.conda_env == 'mlir' }} + shell: bash -el {0} + run: | + source ${CONDA}/bin/activate ${{ inputs.conda_env }} + source /opt/intel/oneapi/setvars.sh + echo PYTHONPATH=`pwd`/torch-mlir/build/tools/torch-mlir/python_packages/torch_mlir:`pwd`/torch-mlir/examples:$PYTHONPATH >> $GITHUB_ENV + echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV From 10e873f6c8f64ea7d650a13b11b9286af4551804 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 5 Dec 2023 03:34:04 +0000 Subject: [PATCH 25/25] Moved "if" inside of steps --- .github/workflows/mlp-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mlp-test.yml b/.github/workflows/mlp-test.yml index b988488..a7bc497 100644 --- a/.github/workflows/mlp-test.yml +++ b/.github/workflows/mlp-test.yml @@ -21,14 +21,15 @@ on: jobs: build_mlir: - if: ${{ inputs.conda_env == 'mlir' }} runs-on: - self-hosted - glados steps: - uses: actions/checkout@v4 + if: ${{ inputs.conda_env == 'mlir' }} - name: Build mlir and upload to artifact storage + if: ${{ inputs.conda_env == 'mlir' }} uses: ./.github/actions/build_mlir with: conda_env: ${{ inputs.conda_env }}