Merge branch 'cad-audio:main' into main

.ci/docker/common/install_android.sh

@@ -11,22 +11,33 @@ set -ex
 [ -n "${ANDROID_NDK_VERSION}" ]
 
 install_prerequiresites() {
-  apt-get update
+  OS=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+  case "$OS" in
+    amzn)
+      # https://docs.aws.amazon.com/corretto/latest/corretto-17-ug/amazon-linux-install.html
+      yum install -y java-17-amazon-corretto \
+        ca-certificates \
+        ant
+      ;;
+    *)
+      apt-get update
 
-  # NB: Need OpenJDK 17 at the minimum
-  apt-get install -y --no-install-recommends \
-    openjdk-17-jdk \
-    ca-certificates-java \
-    ant
+      # NB: Need OpenJDK 17 at the minimum
+      apt-get install -y --no-install-recommends \
+        openjdk-17-jdk \
+        ca-certificates-java \
+        ant
 
-  # Cleanup package manager
-  apt-get autoclean && apt-get clean
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+      # Cleanup package manager
+      apt-get autoclean && apt-get clean
+      rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    ;;
+  esac
 }
 
 install_ndk() {
   NDK_INSTALLATION_DIR=/opt/ndk
-  mkdir -p "${NDK_INSTALLATION_DIR}"
+  rm -rf "${NDK_INSTALLATION_DIR}" && mkdir -p "${NDK_INSTALLATION_DIR}"
 
   pushd /tmp
   # The NDK installation is cached on ossci-android S3 bucket
@@ -54,7 +65,7 @@ install_cmdtools() {
 
 install_sdk() {
   SDK_INSTALLATION_DIR=/opt/android/sdk
-  mkdir -p "${SDK_INSTALLATION_DIR}"
+  rm -rf "${SDK_INSTALLATION_DIR}" && mkdir -p "${SDK_INSTALLATION_DIR}"
 
   # These are the tools needed to build Android apps
   yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "platforms;android-34"

.ci/docker/requirements-ci.txt

@@ -1,5 +1,7 @@
 mpmath==1.3.0
-numpy==1.25.2
+numpy==1.21.3; python_version == '3.10'
+numpy==1.23.2; python_version == '3.11'
+numpy; python_version >= '3.12'
 PyYAML==6.0.1
 ruamel.yaml==0.17.32
 sympy==1.12
@@ -8,6 +10,8 @@ tomli==2.0.1
 torchsr==1.0.4
 transformers==4.38.0
 zstd==1.5.5.1
+pandas==2.0.3; python_version == '3.10'
+pandas; python_version >= '3.11'
 pytest==7.2.0
 pytest-cov==4.1.0
 expecttest==0.1.6

.ci/scripts/gather_test_models.py

@@ -56,7 +56,7 @@ def parse_args() -> Any:
         "-e",
         "--event",
         type=str,
-        choices=["pull_request", "push"],
+        choices=["pull_request", "push", "schedule"],
         required=True,
         help="GitHub CI Event. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#on",
     )

.github/workflows/android.yml

@@ -45,6 +45,7 @@ jobs:
 
         # Build LLM Demo for Android
         bash build/build_android_llm_demo.sh ${{ matrix.tokenizer }} ${ARTIFACTS_DIR_NAME}
+
   # Upload artifacts to S3. The artifacts are needed not only by the device farm but also TorchChat
   upload-artifacts:
     needs: build-llm-demo
@@ -77,6 +78,73 @@ jobs:
           if-no-files-found: ignore
           path: ${{ runner.temp }}/artifacts/
 
+  # Running Android emulator directly on the runner and not using Docker
+  run-emulator:
+    needs: build-llm-demo
+    runs-on: amz2023.linux.4xlarge
+    env:
+      ANDROID_NDK_VERSION: r26c
+      API_LEVEL: 34
+    steps:
+      - name: Setup SSH (Click me for login details)
+        uses: pytorch/test-infra/.github/actions/setup-ssh@main
+        with:
+          github-secret: ${{ secrets.GITHUB_TOKEN }}
+          instructions: |
+            This is used to run Android emulators, ANDROID_HOME is installed at /opt/android/sdk
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: false
+
+      - name: Setup conda
+        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
+        with:
+          python-version: '3.10'
+
+      - name: Install Android dependencies
+        shell: bash
+        run: |
+          set -eux
+
+          # Reuse the script that install Android on ET Docker image
+          sudo -E bash .ci/docker/common/install_android.sh
+
+      - name: Gradle cache
+        uses: gradle/actions/setup-gradle@v3
+
+      - name: AVD cache
+        uses: actions/cache@v4
+        id: avd-cache
+        with:
+          path: |
+            ~/.android/avd/*
+            ~/.android/adb*
+          key: avd-${{ env.API_LEVEL }}
+
+      # NB: It takes about 10m to cold boot the emulator here
+      - name: Run Android emulator
+        env:
+          ANDROID_HOME: /opt/android/sdk
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: ${{ env.API_LEVEL }}
+          # NB: x86_64 emulator is slow because the lack of KVM support on AWS, it
+          # seems that we can use metal instance for that but it hasn't been tried
+          # out yet. Also arm64-v8a arch requires an ARM runner
+          arch: x86_64
+          script: ./build/run_android_emulator.sh
+          # NB: This is to boot the emulator faster following the instructions on
+          # https://github.com/ReactiveCircus/android-emulator-runner. The max number
+          # of cores we can set is 6, any higher number will be reduced to 6.
+          cores: 6
+          ram-size: 12288M
+          force-avd-creation: false
+          disable-animations: true
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          # This is to make sure that the job doesn't fail flakily
+          emulator-boot-timeout: 900
+
   # Let's see how expensive this job is, we might want to tone it down by running it periodically
   test-llama-app:
     needs: upload-artifacts

.github/workflows/pull.yml

@@ -193,7 +193,7 @@ jobs:
     strategy:
       fail-fast: false
     with:
-      runner: linux.12xlarge
+      runner: linux.24xlarge
       docker-image: executorch-ubuntu-22.04-clang12
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
@@ -210,11 +210,19 @@ jobs:
         bash examples/models/llava/install_requirements.sh
 
         # run export_llava.sh
-        python examples/models/llava/export_llava.py
+        python examples/models/llava/export_llava.py --use-sdpa-with-kv-cache --pte-name llava_custom_sdpa.pte
 
         # verify file exists
-        if [ ! -f "llava_combined_xnnpack.pte" ]; then
-            echo "llava_combined_xnnpack.pte not found!"
+        if [ ! -f "llava_custom_sdpa.pte" ]; then
+            echo "llava_custom_sdpa.pte not found!"
+            exit 1
+        fi
+
+        python examples/models/llava/export_llava.py --no-use-sdpa-with-kv-cache --pte-name llava.pte
+
+        # verify file exists
+        if [ ! -f "llava.pte" ]; then
+            echo "llava.pte not found!"
             exit 1
         fi
 

.gitmodules

@@ -1,24 +1,18 @@
-[submodule "third-party/prelude"]
-	path = third-party/prelude
-	url = https://github.com/facebook/buck2-prelude.git
-[submodule "third-party/flatbuffers"]
-	path = third-party/flatbuffers
-	url = https://github.com/google/flatbuffers.git
-[submodule "third-party/gflags"]
-	path = third-party/gflags
-	url = https://github.com/gflags/gflags.git
-[submodule "third-party/googletest"]
-	path = third-party/googletest
-	url = https://github.com/google/googletest.git
-[submodule "third-party/pybind11"]
-	path = third-party/pybind11
-	url = https://github.com/pybind/pybind11.git
-[submodule "backends/xnnpack/third-party/pthreadpool"]
-	path = backends/xnnpack/third-party/pthreadpool
-	url = https://github.com/Maratyszcza/pthreadpool.git
-[submodule "backends/xnnpack/third-party/cpuinfo"]
-	path = backends/xnnpack/third-party/cpuinfo
-	url = https://github.com/pytorch/cpuinfo.git
+[submodule "backends/arm/third-party/ethos-u-core-driver"]
+	path = backends/arm/third-party/ethos-u-core-driver
+	url = https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver
+[submodule "backends/arm/third-party/serialization_lib"]
+	path = backends/arm/third-party/serialization_lib
+	url = https://review.mlplatform.org/tosa/serialization_lib
+[submodule "backends/vulkan/third-party/Vulkan-Headers"]
+	path = backends/vulkan/third-party/Vulkan-Headers
+	url = https://github.com/KhronosGroup/Vulkan-Headers
+[submodule "backends/vulkan/third-party/VulkanMemoryAllocator"]
+	path = backends/vulkan/third-party/VulkanMemoryAllocator
+	url = https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator.git
+[submodule "backends/vulkan/third-party/volk"]
+	path = backends/vulkan/third-party/volk
+	url = https://github.com/zeux/volk
 [submodule "backends/xnnpack/third-party/FP16"]
 	path = backends/xnnpack/third-party/FP16
 	url = https://github.com/Maratyszcza/FP16.git
@@ -28,45 +22,51 @@
 [submodule "backends/xnnpack/third-party/XNNPACK"]
 	path = backends/xnnpack/third-party/XNNPACK
 	url = https://github.com/digantdesai/XNNPACK.git
-[submodule "backends/arm/third-party/serialization_lib"]
-	path = backends/arm/third-party/serialization_lib
-	url = https://review.mlplatform.org/tosa/serialization_lib
-[submodule "third-party/flatcc"]
-	path = third-party/flatcc
-	url = https://github.com/dvidelabs/flatcc.git
+[submodule "backends/xnnpack/third-party/cpuinfo"]
+	path = backends/xnnpack/third-party/cpuinfo
+	url = https://github.com/pytorch/cpuinfo.git
+[submodule "backends/xnnpack/third-party/pthreadpool"]
+	path = backends/xnnpack/third-party/pthreadpool
+	url = https://github.com/Maratyszcza/pthreadpool.git
+[submodule "examples/third-party/LLaVA"]
+	path = examples/third-party/LLaVA
+	url = https://github.com/haotian-liu/LLaVA.git
 [submodule "examples/third-party/fbjni"]
 	path = examples/third-party/fbjni
 	url = https://github.com/facebookincubator/fbjni.git
-[submodule "backends/arm/third-party/ethos-u-core-driver"]
-	path = backends/arm/third-party/ethos-u-core-driver
-	url = https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver
-[submodule "backends/vulkan/third-party/VulkanMemoryAllocator"]
-	path = backends/vulkan/third-party/VulkanMemoryAllocator
-	url = https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator.git
-[submodule "backends/vulkan/third-party/volk"]
-	path = backends/vulkan/third-party/volk
-	url = https://github.com/zeux/volk
-[submodule "backends/vulkan/third-party/Vulkan-Headers"]
-	path = backends/vulkan/third-party/Vulkan-Headers
-	url = https://github.com/KhronosGroup/Vulkan-Headers
+[submodule "extension/llm/third-party/abseil-cpp"]
+	path = extension/llm/third-party/abseil-cpp
+	url = https://github.com/abseil/abseil-cpp.git
+[submodule "extension/llm/third-party/re2"]
+	path = extension/llm/third-party/re2
+	url = https://github.com/google/re2.git
+[submodule "extension/llm/third-party/sentencepiece"]
+	path = extension/llm/third-party/sentencepiece
+	url = https://github.com/google/sentencepiece.git
 [submodule "kernels/optimized/third-party/eigen"]
 	path = kernels/optimized/third-party/eigen
 	url = https://gitlab.com/libeigen/eigen.git
-[submodule "examples/third-party/LLaVA"]
-	path = examples/third-party/LLaVA
-	url = https://github.com/haotian-liu/LLaVA.git
+[submodule "third-party/flatbuffers"]
+	path = third-party/flatbuffers
+	url = https://github.com/google/flatbuffers.git
+[submodule "third-party/flatcc"]
+	path = third-party/flatcc
+	url = https://github.com/dvidelabs/flatcc.git
+[submodule "third-party/gflags"]
+	path = third-party/gflags
+	url = https://github.com/gflags/gflags.git
+[submodule "third-party/googletest"]
+	path = third-party/googletest
+	url = https://github.com/google/googletest.git
 [submodule "third-party/ios-cmake"]
 	path = third-party/ios-cmake
 	url = https://github.com/leetal/ios-cmake
-[submodule "examples/models/phi-3-mini/third-party/sentencepiece"]
-	path = examples/models/phi-3-mini/third-party/sentencepiece
-	url = https://github.com/google/sentencepiece.git
-[submodule "extension/llm/third-party/re2"]
-	path = extension/llm/third-party/re2
-	url = https://github.com/google/re2.git
-[submodule "extension/llm/third-party/abseil-cpp"]
-	path = extension/llm/third-party/abseil-cpp
-	url = https://github.com/abseil/abseil-cpp.git
 [submodule "backends/cadence/hifi/third-party/nnlib/nnlib-hifi4"]
 	path = backends/cadence/hifi/third-party/nnlib/nnlib-hifi4
 	url = https://github.com/foss-xtensa/nnlib-hifi4.git
+[submodule "third-party/prelude"]
+	path = third-party/prelude
+	url = https://github.com/facebook/buck2-prelude.git
+[submodule "third-party/pybind11"]
+	path = third-party/pybind11
+	url = https://github.com/pybind/pybind11.git

CMakeLists.txt

@@ -424,6 +424,11 @@ if(EXECUTORCH_BUILD_FLATC)
       CACHE BOOL ""
   )
   add_subdirectory(third-party/flatbuffers)
+
+  # exir lets users set the alignment of tensor data embedded in the flatbuffer,
+  # and some users need an alignment larger than the default, which is typically
+  # 32.
+  target_compile_definitions(flatc PRIVATE FLATBUFFERS_MAX_ALIGNMENT=1024)
 endif()
 if(NOT FLATC_EXECUTABLE)
   message(

backends/apple/mps/targets.bzl

@@ -61,8 +61,6 @@ def define_common_targets(is_xplat = False, platforms = []):
             "MetalPerformanceShaders",
             "MetalPerformanceShadersGraph",
         ]
-        kwargs["fbobjc_ios_target_sdk_version"] = "17.0"
-        kwargs["fbobjc_macosx_target_sdk_version"] = "14.0"
         kwargs["platforms"] = platforms
 
     if runtime.is_oss or is_xplat:

backends/arm/arm_backend.py

@@ -166,43 +166,6 @@ def get_intermediate_path(compile_spec: List[CompileSpec]) -> str:
     return None
 
 
-def generate_ethosu_compile_spec(
-    config: str,
-    permute_memory_to_nhwc: Optional[bool] = None,
-    quantize_io: Optional[bool] = None,
-    system_config: Optional[str] = None,
-    memory_mode: Optional[str] = None,
-    extra_flags: Optional[str] = None,
-    config_ini: Optional[str] = "Arm/vela.ini",
-) -> List[CompileSpec]:
-    return (
-        ArmCompileSpecBuilder()
-        .ethosu_compile_spec(
-            config,
-            system_config=system_config,
-            memory_mode=memory_mode,
-            extra_flags=extra_flags,
-            config_ini=config_ini,
-        )
-        .set_permute_memory_format(permute_memory_to_nhwc)
-        .set_quantize_io(quantize_io)
-        .build()
-    )
-
-
-def generate_tosa_compile_spec(
-    permute_memory_to_nhwc: Optional[bool] = None,
-    output_path: Optional[str] = None,
-) -> List[CompileSpec]:
-    return (
-        ArmCompileSpecBuilder()
-        .tosa_compile_spec()
-        .set_permute_memory_format(permute_memory_to_nhwc)
-        .dump_intermediate_artifacts_to(output_path)
-        .build()
-    )
-
-
 @final
 class ArmBackend(BackendDetails):
     @staticmethod

backends/arm/arm_partitioner.py

@@ -47,6 +47,7 @@ def is_node_supported(self, submodules, node: torch.fx.Node) -> bool:
             exir_ops.edge.aten.avg_pool2d.default,
             exir_ops.edge.aten.sigmoid.default,
             exir_ops.edge.aten._softmax.default,
+            exir_ops.edge.aten.slice_copy.Tensor,
             exir_ops.edge.aten.sub.Tensor,
             exir_ops.edge.aten.view_copy.default,
             exir_ops.edge.aten.clone.default,

backends/arm/operators/__init__.py

@@ -19,6 +19,7 @@
     op_permute,
     op_quant,
     op_sigmoid,
+    op_slice,
     op_softmax,
     op_sub,
     op_view,