From ee7f3f4afdc9f34c630ab3707b2b8da4cf33ff38 Mon Sep 17 00:00:00 2001 From: Abhinay Kukkadapu Date: Mon, 13 Apr 2026 21:21:57 -0700 Subject: [PATCH 1/2] Add OSS Buck build support and CI for Qualcomm backend (#18666) Summary: Keep OSS CI and Internal builds in sync to avoid breaking changes when OSS prs land for ET-QNN delegate. This diff adds two things to prevent future CMake/Buck drift: 1. **Third-party dep shim** (`third_party_libs.bzl` + OSS `BUCK`): - `qnn_third_party_dep()` function routes QNN SDK deps to internal targets (`fbsource//third-party/qualcomm/qnn/...`) or OSS prebuilt targets (`//backends/qualcomm/third-party:...`) based on `runtime.is_oss`. Follows the XNNPACK third_party_libs.bzl pattern. - OSS BUCK file wraps the externally-downloaded QNN SDK (via `QNN_SDK_ROOT` in `.buckconfig`) with prebuilt Buck targets. - All `targets.bzl` files updated to use the shim instead of hardcoded `fbsource//third-party/...` paths. 2. **Buck build CI job** in `pull.yml`: - New `test-qnn-buck-build-linux` job using existing QNN SDK Docker image. Downloads SDK via `download_qnn_sdk.py`, writes path to `.buckconfig`, runs `buck2 build //backends/qualcomm/...`. - If someone adds a file to CMake but not Buck, this job fails. Differential Revision: D99222617 --- .github/workflows/pull.yml | 34 +++++ backends/qualcomm/aot/python/targets.bzl | 18 ++- backends/qualcomm/aot/wrappers/targets.bzl | 6 +- backends/qualcomm/recipes/BUCK | 6 +- backends/qualcomm/runtime/targets.bzl | 14 +-- backends/qualcomm/serialization/targets.bzl | 9 +- backends/qualcomm/targets.bzl | 19 ++- backends/qualcomm/tests/BUCK | 18 ++- backends/qualcomm/third-party/BUCK | 118 ++++++++++++++++++ devtools/inspector/TARGETS | 3 +- .../qualcomm/oss_scripts/llama/targets.bzl | 4 +- examples/qualcomm/oss_scripts/whisper/TARGETS | 2 - .../qualcomm/oss_scripts/whisper/targets.bzl | 4 +- .../qualcomm/third-party/third_party_libs.bzl | 33 +++++ .../executorch/build/runtime_wrapper.bzl | 15 +++ 15 files changed, 254 insertions(+), 49 deletions(-) create mode 100644 backends/qualcomm/third-party/BUCK create mode 100644 shim_et/xplat/executorch/backends/qualcomm/third-party/third_party_libs.bzl diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 8a5b2f4805a..060b5ae3da5 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -674,6 +674,40 @@ jobs: build-tool: buck2 docker-image: ci-image:executorch-ubuntu-22.04-clang12 + test-qnn-buck-build-linux: + name: test-qnn-buck-build-linux + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + with: + runner: linux.2xlarge + docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + set -eux + + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + # Download QNN SDK and get the path + QNN_SDK_ROOT=$(python3 backends/qualcomm/scripts/download_qnn_sdk.py --print-sdk-path) + echo "QNN_SDK_ROOT=${QNN_SDK_ROOT}" + + # Configure Buck to find the QNN SDK + echo "[qualcomm]" >> .buckconfig + echo " qnn_sdk_root = ${QNN_SDK_ROOT}" >> .buckconfig + + # Setup buck2 + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2 + + # Build QNN backend with Buck + buck2 build //backends/qualcomm/... + unittest-arm-backend-with-no-deps: name: unittest-arm-backend-with-no-deps uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main diff --git a/backends/qualcomm/aot/python/targets.bzl b/backends/qualcomm/aot/python/targets.bzl index f9dffafed9f..cc80e0a51d3 100644 --- a/backends/qualcomm/aot/python/targets.bzl +++ b/backends/qualcomm/aot/python/targets.bzl @@ -3,7 +3,7 @@ load( "CXX", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") PYTHON_MODULE_NAME = "PyQnnManagerAdaptor" @@ -32,17 +32,15 @@ def define_common_targets(): "//executorch/backends/qualcomm/runtime:logging", "//executorch/backends/qualcomm:schema", "//executorch/backends/qualcomm/runtime:runtime", - "fbsource//third-party/pybind11:pybind11", - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), + qnn_third_party_dep("pybind11"), ], external_deps = [ "libtorch_python", ], use_static_deps = True, - visibility = [ - "//executorch/backends/qualcomm/...", - ], + visibility = ["PUBLIC"], ) @@ -61,8 +59,8 @@ def define_common_targets(): "//executorch/backends/qualcomm/runtime:logging", "//executorch/backends/qualcomm:schema", "//executorch/backends/qualcomm/runtime:runtime", - "fbsource//third-party/pybind11:pybind11", - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), + qnn_third_party_dep("pybind11"), ], ) diff --git a/backends/qualcomm/aot/wrappers/targets.bzl b/backends/qualcomm/aot/wrappers/targets.bzl index cbd7ce32427..89f8efdea3e 100644 --- a/backends/qualcomm/aot/wrappers/targets.bzl +++ b/backends/qualcomm/aot/wrappers/targets.bzl @@ -3,7 +3,7 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") def define_common_targets(): """Defines targets that should be shared between fbcode and xplat. @@ -23,8 +23,8 @@ def define_common_targets(): platforms = [ANDROID], visibility = ["PUBLIC"], deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), "//executorch/runtime/backend:interface", "//executorch/runtime/core:core", ], diff --git a/backends/qualcomm/recipes/BUCK b/backends/qualcomm/recipes/BUCK index e76c3760fb3..db931c73f4c 100644 --- a/backends/qualcomm/recipes/BUCK +++ b/backends/qualcomm/recipes/BUCK @@ -23,15 +23,15 @@ fbcode_target(_kind = runtime.python_library, ], visibility = ["PUBLIC"], deps = [ - "//caffe2:torch", "//executorch/export:lib", - "//executorch/runtime:runtime", # @manual "//executorch/backends/qualcomm/partition:partition", "//executorch/backends/qualcomm/serialization:serialization", "//executorch/backends/qualcomm/utils:utils", "//executorch/backends/qualcomm/_passes:passes", ":qnn_recipe_types", - ], + ] + ([] if runtime.is_oss else [ + "//executorch/runtime:runtime", # @manual + ]), ) fbcode_target(_kind = runtime.python_library, diff --git a/backends/qualcomm/runtime/targets.bzl b/backends/qualcomm/runtime/targets.bzl index 4413fa7792c..e84d080f5dd 100644 --- a/backends/qualcomm/runtime/targets.bzl +++ b/backends/qualcomm/runtime/targets.bzl @@ -3,7 +3,7 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") def define_common_targets(): """Defines targets that should be shared between fbcode and xplat. @@ -24,12 +24,12 @@ def define_common_targets(): platforms = [ANDROID], visibility = ["PUBLIC"], deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), "//executorch/runtime/backend:interface", ], exported_deps = [ - "fbsource//third-party/toolchains:log", + qnn_third_party_dep("log"), "//executorch/backends/qualcomm:schema", "//executorch/runtime/core:core", ], @@ -71,12 +71,12 @@ def define_common_targets(): platforms = [ANDROID], visibility = ["PUBLIC"], resources = ({ - "qnn_lib": "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()), + "qnn_lib": qnn_third_party_dep("qnn_offline_compile_libs"), } if include_aot_qnn_lib else { }), deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), ":logging", "//executorch/backends/qualcomm:schema", "//executorch/backends/qualcomm/aot/wrappers:wrappers", diff --git a/backends/qualcomm/serialization/targets.bzl b/backends/qualcomm/serialization/targets.bzl index 01ca7ab0843..0bc937b190f 100644 --- a/backends/qualcomm/serialization/targets.bzl +++ b/backends/qualcomm/serialization/targets.bzl @@ -7,10 +7,17 @@ def define_common_targets(): TARGETS and BUCK files that call this function. """ + # Used by the INPUT_SCHEMA genrule in targets.bzl for OSS builds, + # where cross-package relative paths don't work in genrule srcs. + export_file( + name = "qc_compiler_spec.fbs", + visibility = ["PUBLIC"], + ) + export_file( name = "qc_compiler_spec_schema", src = "qc_compiler_spec.fbs", - visibility = ["//executorch/backends/qualcomm/serialization/..."], + visibility = ["PUBLIC"], ) runtime.python_library( diff --git a/backends/qualcomm/targets.bzl b/backends/qualcomm/targets.bzl index 5c604f9837b..a53e5823aff 100644 --- a/backends/qualcomm/targets.bzl +++ b/backends/qualcomm/targets.bzl @@ -3,12 +3,15 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") # Construct the input and output file names. All input and output files rely on scalar_type file. SCHEMA_NAME = "qc_compiler_spec" -INPUT_SCHEMA = "serialization/" + SCHEMA_NAME + ".fbs" +# In OSS, genrule srcs can't use cross-package relative paths, so use +# an export_file target. In fbcode/xplat, relative path works and avoids +# the srcs patching gap in _patch_executorch_references. +INPUT_SCHEMA = "//backends/qualcomm/serialization:qc_compiler_spec.fbs" if runtime.is_oss else "serialization/" + SCHEMA_NAME + ".fbs" OUTPUT_SCHEMA_HEADER = SCHEMA_NAME + "_generated.h" @@ -60,15 +63,7 @@ def define_common_targets(): runtime.cxx_library( name = "schema", srcs = [], - visibility = [ - # Lock this down as tightly as possible to ensure that flatbuffers - # are an implementation detail. Ideally this list would only include - # //executorch/runtime/executor/... - "//executorch/codegen/tools/...", - "//executorch/runtime/executor/...", - "//executorch/backends/qualcomm/...", - "//executorch/backends/qualcomm/runtime/...", - ], + visibility = ["PUBLIC"], exported_headers = { OUTPUT_SCHEMA_HEADER: ":{}[{}]".format(SCHEMA_GEN_RULE_NAME, OUTPUT_SCHEMA_HEADER), }, @@ -84,7 +79,7 @@ def define_common_targets(): define_static_target = True, visibility = ["PUBLIC"], deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), + qnn_third_party_dep("api"), "//executorch/runtime/backend:interface", "//executorch/runtime/core:core", "//executorch/backends/qualcomm/runtime:runtime_android_build", diff --git a/backends/qualcomm/tests/BUCK b/backends/qualcomm/tests/BUCK index 502ad59df24..1328cdc48cf 100644 --- a/backends/qualcomm/tests/BUCK +++ b/backends/qualcomm/tests/BUCK @@ -13,6 +13,8 @@ fbcode_target(_kind = runtime.python_library, ] ) +# Test targets have transitive deps (caffe2, torchvision, torchaudio, etc.) +# that are not available in OSS Buck builds. fbcode_target(_kind = runtime.python_library, name = "test_qnn_delegate", srcs = [ @@ -28,7 +30,6 @@ fbcode_target(_kind = runtime.python_library, "//caffe2/functorch:functorch_src", "//executorch/exir/backend:partitioner", "//executorch/exir/dialects:lib", - "//executorch/extension/pybindings:portable_lib", # @manual "//executorch/extension/pytree:pylib", "//executorch/backends/qualcomm/partition:partition", "//executorch/backends/qualcomm/quantizer:quantizer", @@ -36,10 +37,15 @@ fbcode_target(_kind = runtime.python_library, "//executorch/backends/qualcomm/utils:utils", "//executorch/devtools:lib", "//executorch/examples/qualcomm:utils", - "//executorch/examples/models:models", "//executorch/backends/qualcomm/debugger:utils", "//executorch/backends/qualcomm/debugger:qnn_intermediate_debugger", - ], + ] + ([] if runtime.is_oss else [ + # These deps have pre-existing issues in OSS Buck: + # portable_lib: shim//build_defs/cpp_python_extension.bzl missing + # models: examples/models/BUCK uses raw python_library (no dep patching) + "//executorch/extension/pybindings:portable_lib", # @manual + "//executorch/examples/models:models", + ]), ) fbcode_target(_kind = runtime.python_library, @@ -67,9 +73,11 @@ fbcode_target(_kind = runtime.python_test, "//executorch/backends/qualcomm/partition:partition", "//executorch/backends/qualcomm/serialization:serialization", "//executorch/backends/qualcomm/utils:utils", + "//executorch/backends/qualcomm/builders:builders", + ] + ([] if runtime.is_oss else [ + # These deps fail in OSS: keep_gpu_sections kwarg breaks TARGETS evaluation "//executorch/examples/models/llama:transformer_modules", "//executorch/examples/qualcomm/oss_scripts/llama:masking_utils", "//executorch/examples/qualcomm/oss_scripts/llama:static_llama", - "//executorch/backends/qualcomm/builders:builders", - ], + ]), ) diff --git a/backends/qualcomm/third-party/BUCK b/backends/qualcomm/third-party/BUCK new file mode 100644 index 00000000000..2084fd5568c --- /dev/null +++ b/backends/qualcomm/third-party/BUCK @@ -0,0 +1,118 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +# OSS prebuilt targets for Qualcomm QNN SDK. +# These wrap the externally-downloaded SDK (via QNN_SDK_ROOT env var or +# backends/qualcomm/scripts/download_qnn_sdk.py) so that Buck builds +# in OSS CI can compile the QNN backend without fbsource third-party targets. +# +# Set QNN_SDK_ROOT before running buck2 build, or use .buckconfig: +# [qualcomm] +# qnn_sdk_root = /path/to/qnn/sdk + +oncall("executorch") + +_QNN_SDK_ROOT = native.read_config("qualcomm", "qnn_sdk_root", "") +_QNN_APP_SDK_FILES = [ + ("QnnModel.cpp", "share/QNN/converter/jni/QnnModel.cpp"), + ("QnnWrapperUtils.cpp", "share/QNN/converter/jni/QnnWrapperUtils.cpp"), + ("linux_QnnModelPal.cpp", "share/QNN/converter/jni/linux/QnnModelPal.cpp"), + ("windows_QnnModelPal.cpp", "share/QNN/converter/jni/windows/QnnModelPal.cpp"), + ("QnnModel.hpp", "share/QNN/converter/jni/QnnModel.hpp"), + ("QnnModelPal.hpp", "share/QNN/converter/jni/QnnModelPal.hpp"), + ("QnnTypeMacros.hpp", "share/QNN/converter/jni/QnnTypeMacros.hpp"), + ("QnnWrapperUtils.hpp", "share/QNN/converter/jni/QnnWrapperUtils.hpp"), +] + +_QNN_APP_SRCS = [ + ":qnn_app_sources_files[QnnModel.cpp]", + ":qnn_app_sources_files[QnnWrapperUtils.cpp]", +] + select({ + "DEFAULT": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"], + "ovr_config//os:linux": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"], + "ovr_config//os:windows": [":qnn_app_sources_files[windows_QnnModelPal.cpp]"], +}) + +_QNN_APP_HEADERS = [ + ":qnn_app_sources_files[QnnModel.hpp]", + ":qnn_app_sources_files[QnnModelPal.hpp]", + ":qnn_app_sources_files[QnnTypeMacros.hpp]", + ":qnn_app_sources_files[QnnWrapperUtils.hpp]", +] + +_QNN_APP_EXPORTED_HEADERS = { + name: ":qnn_app_sources_files[{}]".format(name) + for name in [ + "QnnModel.hpp", + "QnnModelPal.hpp", + "QnnTypeMacros.hpp", + "QnnWrapperUtils.hpp", + ] +} + +# Mirror QNN SDK API headers into declared outputs so that Buck2's sandbox +# can access them (raw -I to an absolute SDK path is not sandbox-safe). +runtime.genrule( + name = "qnn_api_headers", + out = "include", + cmd = ( + "cp -rLf \"{root}/include/QNN/.\" \"$OUT/\"".format(root = _QNN_SDK_ROOT) + if _QNN_SDK_ROOT else "mkdir -p \"$OUT\"" + ), + visibility = [], +) + +# QNN API headers (include/QNN/) +runtime.cxx_library( + name = "qnn_api", + exported_preprocessor_flags = [ + "-I$(location :qnn_api_headers)", + ], + exported_headers = {}, + visibility = ["PUBLIC"], +) + +# Mirror the fixed JNI SDK files into declared outputs instead of globbing an +# absolute SDK path, which Buck rejects during package evaluation. +runtime.genrule( + name = "qnn_app_sources_files", + outs = {name: [path] for name, path in _QNN_APP_SDK_FILES}, + default_outs = ["."], + cmd = " && ".join( + [ + "mkdir -p $OUT/share/QNN/converter/jni/linux $OUT/share/QNN/converter/jni/windows", + ] + ( + [ + "cp -f \"{root}/{path}\" \"$OUT/{path}\"".format(root = _QNN_SDK_ROOT, path = path) + for _, path in _QNN_APP_SDK_FILES + ] if _QNN_SDK_ROOT else [ + "touch \"$OUT/{path}\"".format(path = path) + for _, path in _QNN_APP_SDK_FILES + ] + ), + ), +) + +# QNN app utility sources (converter/jni/) +runtime.cxx_library( + name = "qnn_app_sources", + srcs = _QNN_APP_SRCS if _QNN_SDK_ROOT else [], + headers = _QNN_APP_HEADERS if _QNN_SDK_ROOT else [], + header_namespace = "", + exported_headers = _QNN_APP_EXPORTED_HEADERS if _QNN_SDK_ROOT else {}, + visibility = ["PUBLIC"], + deps = [":qnn_api"], +) + +# QNN offline compile libs (placeholder — only needed for AOT resources) +runtime.filegroup( + name = "qnn_offline_compile_libs", + srcs = [], + visibility = ["PUBLIC"], +) + +# Android log stub (noop on non-Android) +runtime.cxx_library( + name = "log", + exported_headers = {}, + visibility = ["PUBLIC"], +) diff --git a/devtools/inspector/TARGETS b/devtools/inspector/TARGETS index 8834dfbb6ba..db71623d208 100644 --- a/devtools/inspector/TARGETS +++ b/devtools/inspector/TARGETS @@ -1,4 +1,3 @@ -load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") oncall("executorch") @@ -23,7 +22,7 @@ runtime.python_library( ], ) -python_binary( +runtime.python_binary( name = "inspector_cli", main_function = ".inspector_cli.main", main_src = "inspector_cli.py", diff --git a/examples/qualcomm/oss_scripts/llama/targets.bzl b/examples/qualcomm/oss_scripts/llama/targets.bzl index 062edf7594c..3769686c135 100644 --- a/examples/qualcomm/oss_scripts/llama/targets.bzl +++ b/examples/qualcomm/oss_scripts/llama/targets.bzl @@ -3,7 +3,7 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_oss_build_kwargs", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") def define_common_targets(): runtime.cxx_library( @@ -24,7 +24,7 @@ def define_common_targets(): "//executorch/extension/llm/runner:stats", "//executorch/extension/tensor:tensor", "//executorch/kernels/quantized:generated_lib", - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), + qnn_third_party_dep("api"), ], exported_deps = [ "//executorch/extension/module:module", diff --git a/examples/qualcomm/oss_scripts/whisper/TARGETS b/examples/qualcomm/oss_scripts/whisper/TARGETS index 7bd33c3db3f..a0ac85e360e 100644 --- a/examples/qualcomm/oss_scripts/whisper/TARGETS +++ b/examples/qualcomm/oss_scripts/whisper/TARGETS @@ -1,7 +1,5 @@ load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") -load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") oncall("executorch") diff --git a/examples/qualcomm/oss_scripts/whisper/targets.bzl b/examples/qualcomm/oss_scripts/whisper/targets.bzl index 48f0174f392..cfed7f5d5dd 100644 --- a/examples/qualcomm/oss_scripts/whisper/targets.bzl +++ b/examples/qualcomm/oss_scripts/whisper/targets.bzl @@ -3,7 +3,7 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_oss_build_kwargs", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") def define_common_targets(): runtime.cxx_library( @@ -23,7 +23,7 @@ def define_common_targets(): deps = [ "//executorch/extension/llm/runner:stats", "//executorch/kernels/quantized:generated_lib", - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), + qnn_third_party_dep("api"), ], exported_deps = [ "//executorch/extension/module:module", diff --git a/shim_et/xplat/executorch/backends/qualcomm/third-party/third_party_libs.bzl b/shim_et/xplat/executorch/backends/qualcomm/third-party/third_party_libs.bzl new file mode 100644 index 00000000000..0c8ad53c483 --- /dev/null +++ b/shim_et/xplat/executorch/backends/qualcomm/third-party/third_party_libs.bzl @@ -0,0 +1,33 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") + +# Dictionary mapping third-party library name to [internal_dep, oss_dep]. +# Internal deps use fbsource//third-party/qualcomm/qnn/qnn-{version}:target. +# OSS deps use //backends/qualcomm/third-party:target (prebuilt from QNN_SDK_ROOT). +_QNN_THIRD_PARTY_LIBS = { + "api": [ + "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), + "//backends/qualcomm/third-party:qnn_api", + ], + "app_sources": [ + "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + "//backends/qualcomm/third-party:qnn_app_sources", + ], + "qnn_offline_compile_libs": [ + "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()), + "//backends/qualcomm/third-party:qnn_offline_compile_libs", + ], + "log": [ + "fbsource//third-party/toolchains:log", + "//backends/qualcomm/third-party:log", + ], + "pybind11": [ + "fbsource//third-party/pybind11:pybind11", + "//third-party:pybind11", + ], +} + +def qnn_third_party_dep(name): + if name not in _QNN_THIRD_PARTY_LIBS: + fail("Cannot find QNN third party library " + name + ", please register it in _QNN_THIRD_PARTY_LIBS first!") + return _QNN_THIRD_PARTY_LIBS[name][1] if runtime.is_oss else _QNN_THIRD_PARTY_LIBS[name][0] diff --git a/shim_et/xplat/executorch/build/runtime_wrapper.bzl b/shim_et/xplat/executorch/build/runtime_wrapper.bzl index 01004595ff1..e84af76f3d9 100644 --- a/shim_et/xplat/executorch/build/runtime_wrapper.bzl +++ b/shim_et/xplat/executorch/build/runtime_wrapper.bzl @@ -377,6 +377,21 @@ def _python_library(*args, **kwargs): def _python_binary(*args, **kwargs): _patch_kwargs_common(kwargs) + + # In OSS, native.python_binary doesn't support fbcode-specific params. + # Convert main_src -> main, and move srcs entries into main if needed. + if env.is_oss: + main_src = kwargs.pop("main_src", None) + srcs = kwargs.pop("srcs", None) + if main_src: + kwargs.setdefault("main", main_src) + elif srcs: + # If srcs provided but no main/main_src, use first src as main + if "main" not in kwargs: + kwargs["main"] = srcs[0] + if srcs != None: + kwargs["srcs"] = srcs + env.python_binary(*args, **kwargs) def _python_test(*args, **kwargs): From 9b2045b347d4963d096a7f775f2b54bfb6e4a247 Mon Sep 17 00:00:00 2001 From: Abhinay Kukkadapu Date: Mon, 13 Apr 2026 21:21:57 -0700 Subject: [PATCH 2/2] Add PR-blocking CI for QNN model tests, operator/model testsuite, and pass unit tests (#18854) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: OSS PRs that change QNN backend APIs can pass external CI but break internal CI (e.g., D99197179 had hundreds of internal failures). The existing `test-qnn-models-linux` job in pull.yml was an empty placeholder, the backend test suite (operators + models) only ran on nightly, and `test_passes.py` had no external CI coverage. Wire up three new PR-blocking CI jobs in pull.yml: 1. `test-qnn-models-linux` — runs model export tests (mv2, mv3, dl3) via test_model.sh, replacing the empty placeholder 2. `test-qnn-testsuite-linux` — runs the standardized backend test suite (operators + models with flow_qnn) via _test_backend.yml reusable workflow 3. `test-qnn-passes-linux` — runs QNN AOT pass unit tests (test_passes.py) Differential Revision: D100708421 --- .github/workflows/pull.yml | 52 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 060b5ae3da5..e7eb9d4bd3e 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -857,6 +857,8 @@ jobs: id-token: write contents: read strategy: + matrix: + model: [mv2, mv3, dl3] fail-fast: false with: runner: linux.2xlarge @@ -868,9 +870,55 @@ jobs: # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh + PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh + PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn" + + test-qnn-testsuite-linux: + name: test-qnn-testsuite-linux + permissions: + id-token: write + contents: read + uses: ./.github/workflows/_test_backend.yml + with: + backend: qnn + flows: '["qnn"]' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 120 + run-linux: true + runner-linux: linux.2xlarge + + test-qnn-passes-linux: + name: test-qnn-passes-linux + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + with: + runner: linux.2xlarge + docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 30 + script: | + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh + # Source (not bash) so QNN_SDK_ROOT stays in the environment + PYTHON_EXECUTABLE=python source .ci/scripts/build-qnn-sdk.sh + + # Editable install so the PyQnnManagerAdaptor .so built by build-qnn-sdk.sh + # is visible in the source tree (the _passes import chain pulls it in transitively) + CMAKE_ARGS="-DEXECUTORCH_BUILD_QNN=ON -DQNN_SDK_ROOT=$QNN_SDK_ROOT" \ + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake --editable true - # placeholder for running test_qnn_delegate.py, can use matrix such that we can trigger different jobs, refers to test-llama-runner-qnn-linux - # reminder: make sure each job runs fast + # Run QNN pass unit tests + pytest -xvs backends/qualcomm/tests/test_passes.py test-phi-3-mini-runner-linux: name: test-phi-3-mini-runner-linux