Add support for dynamic memrefs in create_nd_tdesc lowering

.github/requirements.txt

@@ -0,0 +1 @@
+# platform: linux-64

.github/workflows/bisect.yml

@@ -0,0 +1,77 @@
+name: bisect
+
+on:
+  workflow_dispatch:
+    inputs:
+      last_good_commit:
+        description: 'Last known good commit'
+        required: true
+        # ! CHANGE
+        default: 'a1c8eab588fb7c8c38175cbebdeff75334d70f5d'
+      first_bad_commit:
+        description: 'First known bad commit, default HEAD'
+        required: false
+        default: 'HEAD'
+
+jobs:
+  build:
+    if: github.repository == 'intel-innersource/frameworks.ai.mlir.mlir-extensions'
+    runs-on: gpu
+    timeout-minutes: 450
+
+    env:
+      TBB_VER: 2021.6.0
+      LEVEL_ZERO_VER: v1.6.2
+      HOME_DIR: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/gpurefactorbuild
+      HOME: /github/home
+      TBB_URL_PREFIX: https://github.com/oneapi-src/oneTBB/releases/download/
+      LLVM_SHA_FILE: llvm_version.txt
+
+    steps:
+      - name: Source Vars
+        run: |
+          mkdir -p $HOME_DIR
+          cd $HOME_DIR
+          rm -rf *
+          source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      - uses: actions/checkout@v3
+        with:
+          repository: intel-innersource/frameworks.ai.mlir.mlir-extensions
+          token: ${{secrets.WORKFLOW_TOKEN}}
+          fetch-depth: 0
+          path: ${{env.HOME_DIR}}/mlir-extensions
+
+      - name: Setup Latest Level Zero
+        shell: bash -l {0}
+        run: |
+          cd $HOME_DIR
+          git clone https://github.com/oneapi-src/level-zero.git
+          cd level-zero
+          git checkout $LEVEL_ZERO_VER
+          cd ..
+          mkdir level-zero-build
+          cd level-zero-build
+          cmake ../level-zero -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../level-zero-install
+          ninja install
+
+      - name: Build LLVM MLIR
+        run: |
+          mkdir -p $HOME_DIR/llvm-mlir
+          # omitted for bisect
+
+      - name: Setup IMEX
+        run: |
+          cd $HOME_DIR
+          cp /home/gta/actions-runner/refactor_gpu_bisect_script.sh .
+          cd mlir-extensions
+          git bisect start
+          git bisect bad
+          git bisect good ${{ github.event.inputs.last_good_commit }}
+          git bisect run ../refactor_gpu_bisect_script.sh 2>&1 | tee output.out
+          cat output.out | grep -A 10  "first bad commit" > badcommit.log
+
+      - name: Upload the bad commit
+        uses: actions/[email protected]
+        with:
+          name: bisect-results
+          path: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/gpurefactorbuild/mlir-extensions/badcommit.log

.github/workflows/build-cpu-composite-action/action.yml

@@ -0,0 +1,40 @@
+name: "CPU Build composite action"
+description: "Checks out repo, build and run cpu tests"
+inputs:
+  build_root:  # id of input
+    description: 'build_root'
+    required: false
+    default: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_cpu
+  home_dir:  # id of input
+    description: 'home_dir'
+    required: false
+    default: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_cpu/home
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout MLIR
+      shell: bash
+      run: |
+        cd ${{ inputs.build_root }}
+        if [ ! -d "llvm-project" ]; then git clone https://github.com/llvm/llvm-project; fi
+        cd ${{ inputs.build_root }}/llvm-project
+        git reset --hard HEAD
+        export LLVM_SHA=`cat ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build_tools/llvm_version.txt`
+        git fetch --prune
+        git checkout $LLVM_SHA || exit 1
+        if [ -d "${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build_tools/patches" ]; then git apply ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build_tools/patches/*.patch; fi
+
+    - name: Setup IMEX and run tests
+      shell: bash
+      run: |
+        cd ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions || exit 1
+        cmake -S ${{ inputs.build_root }}/llvm-project/llvm -B build -GNinja  -DCMAKE_BUILD_TYPE=Release  -DLLVM_ENABLE_PROJECTS=mlir -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_LINKER=gold -DLLVM_ENABLE_ZSTD=OFF -DLLVM_EXTERNAL_PROJECTS="Imex" -DLLVM_EXTERNAL_IMEX_SOURCE_DIR=. || exit 1
+        set -o pipefail
+        cmake --build build --target check-imex | tee build/tests.txt
+
+    - name: Upload tests.txt
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: tests.txt
+        path: ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build/tests.txt

.github/workflows/build-gpu-composite-action/action.yml

@@ -0,0 +1,67 @@
+name: "GPU Build composite action"
+description: "Checks out repo, build and run gpu tests"
+inputs:
+  build_root:  # id of input
+    description: 'build_root'
+    required: false
+    default: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_gpu
+  home_dir:  # id of input
+    description: 'home_dir'
+    required: false
+    default: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_gpu/home
+  level_zero_ver:  # id of input
+    description: 'level_zero_ver'
+    required: false
+    default: 'v1.8.1'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Level Zero
+      shell: bash -l {0}
+      run: |
+        cd ${{ inputs.home_dir }}
+        git clone https://github.com/oneapi-src/level-zero.git
+        cd level-zero
+        git checkout ${{ inputs.level_zero_ver }}
+        cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../level-zero-install
+        cmake --build build --target install
+
+    - name: Checkout MLIR
+      shell: bash
+      run: |
+        cd ${{ inputs.build_root }}
+        if [ ! -d "llvm-project" ]; then git clone https://github.com/llvm/llvm-project; fi
+          cd ${{ inputs.build_root }}/llvm-project
+        git reset --hard HEAD
+        export LLVM_SHA=`cat ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build_tools/llvm_version.txt`
+        git fetch --prune
+        git checkout $LLVM_SHA || exit 1
+        if [ -d "${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build_tools/patches" ]; then git apply ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build_tools/patches/*.patch; fi
+
+    - name: Setup IMEX and run tests
+      shell: bash
+      run: |
+        source /opt/intel/oneapi/setvars.sh
+        cd ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions
+        cmake -S ${{ inputs.build_root }}/llvm-project/llvm -B build -GNinja  -DCMAKE_BUILD_TYPE=Release  -DLLVM_ENABLE_PROJECTS=mlir -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_LINKER=gold -DLLVM_ENABLE_ZSTD=OFF -DLLVM_EXTERNAL_PROJECTS="Imex" -DLLVM_EXTERNAL_IMEX_SOURCE_DIR=. -DIMEX_ENABLE_SYCL_RUNTIME=1 -DIMEX_ENABLE_L0_RUNTIME=1 -DLEVEL_ZERO_DIR=${{ inputs.home_dir }}/level-zero-install/ || exit
+        set -o pipefail
+        cmake --build build --target check-imex | tee build/tests.txt
+
+    #- name: Run perf script
+    #  shell: bash
+    #  run: |
+    #    cd ${{ inputs.home_dir }}/mlir-extensions/scripts
+    #    ./run_perf_test.sh
+
+    - name: Upload tests.txt
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: tests.txt
+        path: ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/build/tests.txt
+
+    #- name: Upload report.txt
+    # uses: actions/upload-artifact@v3
+    #with:
+    #name: report.txt
+    #path: ${{ inputs.home_dir }}/frameworks.ai.mlir.mlir-extensions/scripts/report.txt

.github/workflows/build_cpu.yml

@@ -0,0 +1,69 @@
+name: CPU Build
+
+on:
+  pull_request:
+    branches:
+      - main
+      - upstream
+      - 'dist-ndarray'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build:
+    if: github.repository == 'intel-innersource/frameworks.ai.mlir.mlir-extensions'
+    runs-on: [self-hosted, pvc, glados]
+    timeout-minutes: 450
+
+    env:
+      BUILD_ROOT: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_cpu
+      HOME_DIR: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_cpu/home
+      HOME: /github/home
+      LLVM_SHA_FILE: llvm_version.txt
+
+    steps:
+      - name: Check build root
+        run: |
+          export BUILD_ROOT=$(pwd)/build_cpu
+            echo BUILD_ROOT=${BUILD_ROOT} >> $GITHUB_ENV
+            export HOME_DIR=${BUILD_ROOT}/home
+            echo HOME_DIR=${HOME_DIR} >> $GITHUB_ENV
+            if [ ! -d "$BUILD_ROOT" ]; then mkdir -p $BUILD_ROOT; fi
+
+      - name: Clear home dir
+        run: |
+          mkdir -p $HOME_DIR
+          cd $HOME_DIR
+          rm -rf *
+
+      - uses: actions/checkout@v3
+        with:
+          repository: intel-innersource/frameworks.ai.mlir.mlir-extensions
+          token: ${{secrets.WORKFLOW_TOKEN}}
+          fetch-depth: 0
+          path: ${{env.HOME_DIR}}/frameworks.ai.mlir.mlir-extensions
+
+      - name: Checkout MLIR
+        run: |
+          cd $BUILD_ROOT
+          if [ ! -d "llvm-project" ]; then git clone https://github.com/llvm/llvm-project; fi
+          cd $BUILD_ROOT/llvm-project
+          git reset --hard HEAD
+          export LLVM_SHA=`cat $HOME_DIR/frameworks.ai.mlir.mlir-extensions/build_tools/llvm_version.txt`
+          git fetch --prune
+          git checkout $LLVM_SHA || exit 1
+          if [ -d "$HOME_DIR/frameworks.ai.mlir.mlir-extensions/build_tools/patches" ]; then git apply $HOME_DIR/frameworks.ai.mlir.mlir-extensions/build_tools/patches/*.patch; fi
+
+      - name: Setup IMEX and run tests
+        run: |
+          cd $HOME_DIR/frameworks.ai.mlir.mlir-extensions || exit 1
+          cmake -S $BUILD_ROOT/llvm-project/llvm -B build -GNinja  -DCMAKE_BUILD_TYPE=Release  -DLLVM_ENABLE_PROJECTS=mlir -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_LINKER=gold -DLLVM_ENABLE_ZSTD=OFF -DLLVM_EXTERNAL_PROJECTS="Imex" -DLLVM_EXTERNAL_IMEX_SOURCE_DIR=. || exit 1
+          set -o pipefail
+          cmake --build build --target check-imex | tee build/tests.txt
+
+      - name: Upload tests.txt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tests_cpu.txt
+          path: ${{env.HOME_DIR}}/frameworks.ai.mlir.mlir-extensions/build/tests.txt

.github/workflows/build_cpu_v2.yml

@@ -0,0 +1,38 @@
+name: "CPU Build with composite action"
+
+on:
+  pull_request:
+    branches:
+      - dummy
+  workflow_dispatch:
+
+jobs:
+  build:
+    if: github.repository == 'intel-innersource/frameworks.ai.mlir.mlir-extensions'
+    runs-on: cpu1
+    timeout-minutes: 450
+
+    env:
+      BUILD_ROOT: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_cpu
+      HOME: /home/gta/actions-runner/_work/frameworks.ai.mlir.mlir-extensions/frameworks.ai.mlir.mlir-extensions/build_cpu/home
+
+    steps:
+      - name: Check build root
+        run: |
+          if [ ! -d "$BUILD_ROOT" ]; then mkdir -p $BUILD_ROOT; fi
+
+      - name: Clear home dir
+        run: |
+          mkdir -p $HOME
+          cd $HOME
+          rm -rf *
+
+      - uses: actions/checkout@v3
+        with:
+          token: ${{secrets.WORKFLOW_TOKEN}}
+          fetch-depth: 0
+          path: ${{env.HOME}}/frameworks.ai.mlir.mlir-extensions
+      - uses: ./build_cpu/home/frameworks.ai.mlir.mlir-extensions/.github/workflows/build-cpu-composite-action
+        with:
+          build_root: ${{env.BUILD_ROOT}}
+          home_dir: ${{env.HOME}}

.github/workflows/build_gpu-fs-sim.yml

@@ -0,0 +1,99 @@
+name: GPU Build and run tests on FS simulator
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build:
+    if: github.repository == 'intel-innersource/frameworks.ai.mlir.mlir-extensions'
+    runs-on: [self-hosted, fs-sim, glados]
+    timeout-minutes: 450
+
+    env:
+      TBB_VER: 2021.6.0
+      LEVEL_ZERO_VER: v1.8.1
+      TBB_URL_PREFIX: https://github.com/oneapi-src/oneTBB/releases/download/
+      LLVM_SHA_FILE: llvm_version.txt
+
+    steps:
+      - name: Check build root
+        run: |
+            export BUILD_ROOT=$(pwd)/build_gpu
+            echo BUILD_ROOT=${BUILD_ROOT} >> $GITHUB_ENV
+            export HOME_DIR=${BUILD_ROOT}/home
+            echo HOME_DIR=${HOME_DIR} >> $GITHUB_ENV
+            if [ ! -d "$BUILD_ROOT" ]; then mkdir -p $BUILD_ROOT; fi
+
+      - name: Clear home dir
+        run: |
+          mkdir -p $HOME_DIR
+          cd $HOME_DIR
+          rm -rf *
+
+      - uses: actions/checkout@v3
+        with:
+          repository: intel-innersource/frameworks.ai.mlir.mlir-extensions
+          token: ${{ secrets.WORKFLOW_TOKEN }}
+          fetch-depth: 0
+          path: ${{ env.HOME_DIR }}/frameworks.ai.mlir.mlir-extensions
+
+      - name: Setup Level Zero
+        shell: bash -l {0}
+        run: |
+          cd $HOME_DIR
+          git clone https://github.com/oneapi-src/level-zero.git
+          cd level-zero
+          git checkout $LEVEL_ZERO_VER
+          cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../level-zero-install
+          cmake --build build --target install
+
+      - name: Checkout MLIR
+        run: |
+          cd $BUILD_ROOT
+          if [ ! -d "llvm-project" ]; then git clone https://github.com/llvm/llvm-project; fi
+          cd $BUILD_ROOT/llvm-project
+          git reset --hard HEAD
+          export LLVM_SHA=`cat $HOME_DIR/frameworks.ai.mlir.mlir-extensions/build_tools/llvm_version.txt`
+          git fetch --prune
+          git checkout $LLVM_SHA || exit 1
+          if [ -d "$HOME_DIR/frameworks.ai.mlir.mlir-extensions/build_tools/patches" ]; then git apply $HOME_DIR/frameworks.ai.mlir.mlir-extensions/build_tools/patches/*.patch; fi
+
+      - name: Setup IMEX and run tests on FS simulator
+        shell: bash -l {0}
+        run: |
+          source /opt/intel/oneapi/setvars.sh
+          set -x
+          cd $HOME_DIR/frameworks.ai.mlir.mlir-extensions
+          cmake -S $BUILD_ROOT/llvm-project/llvm -B build -GNinja  -DCMAKE_BUILD_TYPE=Release  -DLLVM_ENABLE_PROJECTS=mlir -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_LINKER=gold -DLLVM_ENABLE_ZSTD=OFF -DLLVM_EXTERNAL_PROJECTS="Imex" -DLLVM_EXTERNAL_IMEX_SOURCE_DIR=. -DIMEX_ENABLE_SYCL_RUNTIME=1 -DIMEX_ENABLE_L0_RUNTIME=1 -DLEVEL_ZERO_DIR=$HOME_DIR/level-zero-install/ -DLLVM_LIT_ARGS="-j 1 --debug --timeout=600" -DIMEX_ENABLE_FS_SIMULATOR=ON || exit
+          sed -e 's,-printAllFeatures,,g' -i ${WORK_ROOT}/fs/scripts/run_in_a_loop_xesim.sh
+          (cd ${WORK_ROOT}/fs/scripts && source setup.sh && umd_driver_env_variables_export && umd_driver_rasty_backend_env_variables_export && ./run_in_a_loop_xesim.sh) &> fs-sim.log &
+          SIM_PID=$!
+          cmake --build build --target check-imex | tee build/tests.txt
+          TEST_RESULT=${PIPESTATUS[0]}
+          kill $SIM_PID
+          wait $SIM_PID
+          cd ${WORK_ROOT}/fs/scripts
+          zip -qr9 /tmp/fs-sim-tbx-logs.zip tbx*.log
+          exit $TEST_RESULT
+
+      - name: Upload tests.txt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tests_gpu-fs-sim.txt
+          path: ${{ env.HOME_DIR }}/frameworks.ai.mlir.mlir-extensions/build/tests.txt
+
+      - name: Upload fs-sim.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: fs-sim.log
+          path: ${{ env.HOME_DIR }}/frameworks.ai.mlir.mlir-extensions/fs-sim.log
+
+      - name: Upload fs-sim-tbx-logs.zip
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: fs-sim-tbx-logs.zip
+          path: /tmp/fs-sim-tbx-logs.zip