feat: perf opt add set rows

docker/build_in_container.sh

@@ -2,9 +2,13 @@
 
 _cpu_count="$(nproc)"
 
+LOCAL_BUILD_DIR='/llama_cpp_build'
+
 echo "LOCAL_REPO_DIR: $LOCAL_REPO_DIR"
+echo "LOCAL_BUILD_DIR: $LOCAL_BUILD_DIR"
 echo "QNN_SDK_PATH: $QNN_SDK_PATH"
 echo "HEXAGON_SDK_PATH: $HEXAGON_SDK_PATH"
+echo "GGML_HEXAGON: $GGML_HEXAGON"
 echo "BUILD_HEXAGON_BACKEND: $BUILD_HEXAGON_BACKEND"
 echo "BUILD_HEXAGON_NPU_ONLY: $BUILD_HEXAGON_NPU_ONLY"
 echo "DISABLE_HEXAGON_AND_QNN: $DISABLE_HEXAGON_AND_QNN"
@@ -36,12 +40,12 @@ fi
 mkdir -p $LOCAL_REPO_DIR
 chmod 777 $LOCAL_REPO_DIR
 cd $LOCAL_REPO_DIR
-rsync -a --delete --exclude='env' --exclude='run_server.sh' --exclude='build_*' --exclude='build' --exclude='models*' --exclude='.vs*' --exclude='.git/objects*' /mnt/llama_cpp_mount/ ./
+rsync -au --delete --exclude='env' --exclude='run_server.sh' --exclude='build_*' --exclude='build' --exclude='*.gguf' --exclude='.vs*' --exclude='.git/objects*' /mnt/llama_cpp_mount/ ./
 git config --global --add safe.directory $LOCAL_REPO_DIR
 echo "compiling git revision: $(git rev-parse --short HEAD)"
-mkdir -p ./build_qnn
-rm -rf ./build_qnn/*
-cd ./build_qnn
+mkdir -p "${LOCAL_BUILD_DIR}"
+rm -rf "${LOCAL_BUILD_DIR}/*"
+cd "${LOCAL_BUILD_DIR}"
 set -e
 
 _extra_options="$CMAKE_EXTRA_BUILD_OPTIONS"
@@ -76,29 +80,37 @@ else
     exit 1
 fi
 
-if [ $BUILD_HEXAGON_BACKEND -eq 1 ]; then
-    _extra_options="${_extra_options} -DGGML_QNN_ENABLE_HEXAGON_BACKEND=on"
-fi
-
-if [ $BUILD_HEXAGON_NPU_ONLY -eq 1 ]; then
-    echo "Building for Hexagon NPU only"
-    _extra_options="${_extra_options} -DGGML_HEXAGON_NPU_ONLY=on"
+if [ $GGML_HEXAGON -eq 1 ]; then
+    echo "Using official GGML hexagon support"
+    # See also: https://github.com/CodeLinaro/llama.cpp/blob/hexagon/docs/backend/hexagon/README.md
+    _extra_options="${_extra_options} -DGGML_QNN=off -DGGML_QNN_ENABLE_HEXAGON_BACKEND=off"
+    _extra_options="${_extra_options} -DGGML_HEXAGON=on -DGGML_OPENMP=off -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_PATH} -DPREBUILT_LIB_DIR=android_aarch64"
 else
-    _extra_options="${_extra_options} -DGGML_HEXAGON_NPU_ONLY=off"
-fi
+    echo "Using custom hexagon support"
+    if [ $BUILD_HEXAGON_BACKEND -eq 1 ]; then
+        _extra_options="${_extra_options} -DGGML_QNN_ENABLE_HEXAGON_BACKEND=on"
+    fi
 
-if [ $DISABLE_HEXAGON_AND_QNN -eq 1 ]; then
-    echo "Building for cpu only"
-    _extra_options="${_extra_options} -DGGML_QNN=off"
-else
-    _extra_options="${_extra_options} -DGGML_QNN=on"
+    if [ $BUILD_HEXAGON_NPU_ONLY -eq 1 ]; then
+        echo "Building for Hexagon NPU only"
+        _extra_options="${_extra_options} -DGGML_HEXAGON_NPU_ONLY=on"
+    else
+        _extra_options="${_extra_options} -DGGML_HEXAGON_NPU_ONLY=off"
+    fi
+
+    if [ $DISABLE_HEXAGON_AND_QNN -eq 1 ]; then
+        echo "Building for cpu only"
+        _extra_options="${_extra_options} -DGGML_QNN=off"
+    else
+        _extra_options="${_extra_options} -DGGML_QNN=on"
+    fi
 fi
 
 # Build llama
-cmake -H.. -B. $_extra_options \
+cmake -H"${LOCAL_REPO_DIR}" -B"${LOCAL_BUILD_DIR}" $_extra_options \
     -DCMAKE_BUILD_TYPE=$BUILD_TYPE
 
-cmake --build . --config $BUILD_TYPE -- -j$_cpu_count
+cmake --build "${LOCAL_BUILD_DIR}" --config $BUILD_TYPE -- -j$_cpu_count
 
 # Copy the output files to the output directory
 chmod -R u+rw $OUTPUT_DIR
@@ -117,6 +129,8 @@ fi
 
 if [ $DISABLE_HEXAGON_AND_QNN -eq 0 ]; then
     rsync -av ./bin/*.so $OUTPUT_DIR
+elif [ $GGML_HEXAGON -eq 1 ]; then
+    rsync -av ./ggml/src/ggml-hexagon/*.so $OUTPUT_DIR
 fi
 
 chown -R "$HOST_USER_ID" "$OUTPUT_DIR"

docker/docker-compose-compile-qnn.yml

@@ -11,6 +11,7 @@ services:
       OUTPUT_DIR: /mnt/output
       LOCAL_REPO_DIR: "/llama_cpp"
       CMAKE_EXTRA_BUILD_OPTIONS: ${CMAKE_EXTRA_BUILD_OPTIONS}
+      GGML_HEXAGON: 0
       BUILD_HEXAGON_BACKEND: 0
       BUILD_HEXAGON_NPU_ONLY: 0
       DISABLE_HEXAGON_AND_QNN: ${DISABLE_HEXAGON_AND_QNN:-0}

docker/docker-compose-compile.yml

@@ -11,6 +11,7 @@ services:
       OUTPUT_DIR: /mnt/output
       LOCAL_REPO_DIR: "/llama_cpp"
       CMAKE_EXTRA_BUILD_OPTIONS: ${CMAKE_EXTRA_BUILD_OPTIONS}
+      GGML_HEXAGON: ${GGML_HEXAGON}
       BUILD_HEXAGON_BACKEND: ${BUILD_HEXAGON_BACKEND}
       BUILD_HEXAGON_NPU_ONLY: ${BUILD_HEXAGON_NPU_ONLY}
       DISABLE_HEXAGON_AND_QNN: ${DISABLE_HEXAGON_AND_QNN:-0}

docker/docker_compose_compile.sh

@@ -10,13 +10,14 @@ _in_ci=0
 _pull_latest=0
 _build_platform='android' # default build platform, could be 'android' or 'linux'
 _build_arch='arm64-v8a'   # default build arch, could be 'arm64-v8a' or 'x86_64'
-_build_options='-DBUILD_SHARED_LIBS=off -DGGML_QNN_ENABLE_CPU_BACKEND=on -DGGML_OPENMP=on -DLLAMA_CURL=off'
+_build_options='-DBUILD_SHARED_LIBS=off -DLLAMA_CURL=off'
 _extra_build_options=''
 _run_backend_tests=0
 _enable_hexagon_backend=1
 _hexagon_npu_only=0
 _qnn_only=0
 _disable_hexagon_and_qnn=0
+_use_ggml_hexagon=0
 _enable_dequant=0
 _enable_profiler=0
 
@@ -90,35 +91,50 @@ while (("$#")); do
             _enable_dequant=1
             shift
         ;;
+        --use-ggml-hexagon)
+            _use_ggml_hexagon=1
+            shift
+        ;;
         *) # preserve positional arguments
             echo "Invalid option $1"
             exit 1
         ;;
     esac
 done
 
-if [ $_enable_hexagon_backend -eq 1 ]; then
-    export BUILD_HEXAGON_BACKEND=1
-else
+if [ $_use_ggml_hexagon -eq 1 ]; then
+    export GGML_HEXAGON=1
     export BUILD_HEXAGON_BACKEND=0
-fi
-
-if [ $_hexagon_npu_only -eq 1 ]; then
-    export BUILD_HEXAGON_NPU_ONLY=1
-    export BUILD_HEXAGON_BACKEND=1
-else
     export BUILD_HEXAGON_NPU_ONLY=0
-fi
-
-if [ $_enable_dequant -eq 1 ]; then
-    _extra_build_options="${_extra_build_options} -DGGML_HEXAGON_ENABLE_QUANTIZED_TENSORS=on"
+    _build_options="${_build_options} -DGGML_QNN_ENABLE_CPU_BACKEND=off -DGGML_OPENMP=off"
+    _disable_hexagon_and_qnn=1
 else
-    _extra_build_options="${_extra_build_options} -DGGML_HEXAGON_ENABLE_QUANTIZED_TENSORS=off"
+    _build_options="${_build_options} -DGGML_QNN_ENABLE_CPU_BACKEND=on -DGGML_OPENMP=on"
+    if [ $_enable_hexagon_backend -eq 1 ]; then
+        export BUILD_HEXAGON_BACKEND=1
+    else
+        export BUILD_HEXAGON_BACKEND=0
+    fi
+
+    if [ $_hexagon_npu_only -eq 1 ]; then
+        export BUILD_HEXAGON_NPU_ONLY=1
+        export BUILD_HEXAGON_BACKEND=1
+    else
+        export BUILD_HEXAGON_NPU_ONLY=0
+    fi
+
+    if [ $_enable_dequant -eq 1 ]; then
+        _extra_build_options="${_extra_build_options} -DGGML_HEXAGON_ENABLE_QUANTIZED_TENSORS=on"
+    else
+        _extra_build_options="${_extra_build_options} -DGGML_HEXAGON_ENABLE_QUANTIZED_TENSORS=off"
+    fi
+
+    if [ $_enable_profiler -eq 1 ]; then
+        _extra_build_options="${_extra_build_options} -DGGML_HEXAGON_ENABLE_PERFORMANCE_TRACKING=on"
+    fi
+    export GGML_HEXAGON=0
 fi
 
-if [ $_enable_profiler -eq 1 ]; then
-    _extra_build_options="${_extra_build_options} -DGGML_HEXAGON_ENABLE_PERFORMANCE_TRACKING=on"
-fi
 
 _build_options="${_build_options} ${_extra_build_options}"