diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VectorFunctionImplUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VectorFunctionImplUtils.java index 8a3223c588fb1..7b09708cd5ce0 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VectorFunctionImplUtils.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VectorFunctionImplUtils.java @@ -31,6 +31,9 @@ public class VectorFunctionImplUtils { * Returns NULL if either vector contains NULL elements, has zero magnitude, or is empty. * Throws an exception if vectors have different dimensions. * Uses manual loop unrolling (8 elements at a time) for speculative SIMD optimization. + * The dot product and the squared norms are accumulated in double precision: their magnitudes + * are quadratic in the input values, so single precision would overflow to infinity (or + * underflow to zero) for vectors whose cosine similarity is perfectly representable as a float. */ public static Float vectorCosineSimilarity(ArrayData left, ArrayData right, UTF8String funcName) { int leftLen = left.numElements(); @@ -45,9 +48,9 @@ public static Float vectorCosineSimilarity(ArrayData left, ArrayData right, UTF8 return null; } - float dotProduct = 0.0f; - float norm1Sq = 0.0f; - float norm2Sq = 0.0f; + double dotProduct = 0.0d; + double norm1Sq = 0.0d; + double norm2Sq = 0.0d; int i = 0; int simdLimit = (leftLen / 8) * 8; @@ -66,15 +69,15 @@ public static Float vectorCosineSimilarity(ArrayData left, ArrayData right, UTF8 return null; } - float a0 = left.getFloat(i), a1 = left.getFloat(i + 1); - float a2 = left.getFloat(i + 2), a3 = left.getFloat(i + 3); - float a4 = left.getFloat(i + 4), a5 = left.getFloat(i + 5); - float a6 = left.getFloat(i + 6), a7 = left.getFloat(i + 7); + double a0 = left.getFloat(i), a1 = left.getFloat(i + 1); + double a2 = left.getFloat(i + 2), a3 = left.getFloat(i + 3); + double a4 = left.getFloat(i + 4), a5 = left.getFloat(i + 5); + double a6 = left.getFloat(i + 6), a7 = left.getFloat(i + 7); - float b0 = right.getFloat(i), b1 = right.getFloat(i + 1); - float b2 = right.getFloat(i + 2), b3 = right.getFloat(i + 3); - float b4 = right.getFloat(i + 4), b5 = right.getFloat(i + 5); - float b6 = right.getFloat(i + 6), b7 = right.getFloat(i + 7); + double b0 = right.getFloat(i), b1 = right.getFloat(i + 1); + double b2 = right.getFloat(i + 2), b3 = right.getFloat(i + 3); + double b4 = right.getFloat(i + 4), b5 = right.getFloat(i + 5); + double b6 = right.getFloat(i + 6), b7 = right.getFloat(i + 7); dotProduct += a0 * b0 + a1 * b1 + a2 * b2 + a3 * b3 + a4 * b4 + a5 * b5 + a6 * b6 + a7 * b7; @@ -90,19 +93,22 @@ public static Float vectorCosineSimilarity(ArrayData left, ArrayData right, UTF8 if (left.isNullAt(i) || right.isNullAt(i)) { return null; } - float a = left.getFloat(i); - float b = right.getFloat(i); + double a = left.getFloat(i); + double b = right.getFloat(i); dotProduct += a * b; norm1Sq += a * a; norm2Sq += b * b; i++; } - float normProduct = (float) Math.sqrt(norm1Sq * norm2Sq); - if (normProduct < Float.MIN_NORMAL) { + // `norm1Sq * norm2Sq` cannot overflow in double precision: both factors are bounded by + // MAX_ROUNDED_ARRAY_LENGTH * Float.MAX_VALUE^2, so their product stays well below + // Double.MAX_VALUE. + double normProduct = Math.sqrt(norm1Sq * norm2Sq); + if (normProduct == 0.0d) { return null; } - return dotProduct / normProduct; + return (float) (dotProduct / normProduct); } /** @@ -111,6 +117,8 @@ public static Float vectorCosineSimilarity(ArrayData left, ArrayData right, UTF8 * Returns 0.0 for empty vectors. * Throws an exception if vectors have different dimensions. * Uses manual loop unrolling (8 elements at a time) for speculative SIMD optimization. + * The dot product is accumulated in double precision so that intermediate terms do not + * overflow to infinity when the final result is representable as a float. */ public static Float vectorInnerProduct(ArrayData left, ArrayData right, UTF8String funcName) { int leftLen = left.numElements(); @@ -125,7 +133,7 @@ public static Float vectorInnerProduct(ArrayData left, ArrayData right, UTF8Stri return 0.0f; } - float dotProduct = 0.0f; + double dotProduct = 0.0d; int i = 0; int simdLimit = (leftLen / 8) * 8; @@ -144,15 +152,15 @@ public static Float vectorInnerProduct(ArrayData left, ArrayData right, UTF8Stri return null; } - float a0 = left.getFloat(i), a1 = left.getFloat(i + 1); - float a2 = left.getFloat(i + 2), a3 = left.getFloat(i + 3); - float a4 = left.getFloat(i + 4), a5 = left.getFloat(i + 5); - float a6 = left.getFloat(i + 6), a7 = left.getFloat(i + 7); + double a0 = left.getFloat(i), a1 = left.getFloat(i + 1); + double a2 = left.getFloat(i + 2), a3 = left.getFloat(i + 3); + double a4 = left.getFloat(i + 4), a5 = left.getFloat(i + 5); + double a6 = left.getFloat(i + 6), a7 = left.getFloat(i + 7); - float b0 = right.getFloat(i), b1 = right.getFloat(i + 1); - float b2 = right.getFloat(i + 2), b3 = right.getFloat(i + 3); - float b4 = right.getFloat(i + 4), b5 = right.getFloat(i + 5); - float b6 = right.getFloat(i + 6), b7 = right.getFloat(i + 7); + double b0 = right.getFloat(i), b1 = right.getFloat(i + 1); + double b2 = right.getFloat(i + 2), b3 = right.getFloat(i + 3); + double b4 = right.getFloat(i + 4), b5 = right.getFloat(i + 5); + double b6 = right.getFloat(i + 6), b7 = right.getFloat(i + 7); dotProduct += a0 * b0 + a1 * b1 + a2 * b2 + a3 * b3 + a4 * b4 + a5 * b5 + a6 * b6 + a7 * b7; @@ -164,13 +172,13 @@ public static Float vectorInnerProduct(ArrayData left, ArrayData right, UTF8Stri if (left.isNullAt(i) || right.isNullAt(i)) { return null; } - float a = left.getFloat(i); - float b = right.getFloat(i); + double a = left.getFloat(i); + double b = right.getFloat(i); dotProduct += a * b; i++; } - return dotProduct; + return (float) dotProduct; } /** @@ -179,6 +187,8 @@ public static Float vectorInnerProduct(ArrayData left, ArrayData right, UTF8Stri * Returns 0.0 for empty vectors. * Throws an exception if vectors have different dimensions. * Uses manual loop unrolling (8 elements at a time) for speculative SIMD optimization. + * The sum of squares is accumulated in double precision: it is quadratic in the input values, + * so single precision would overflow to infinity for distances representable as a float. */ public static Float vectorL2Distance(ArrayData left, ArrayData right, UTF8String funcName) { int leftLen = left.numElements(); @@ -193,7 +203,7 @@ public static Float vectorL2Distance(ArrayData left, ArrayData right, UTF8String return 0.0f; } - float sumSq = 0.0f; + double sumSq = 0.0d; int i = 0; int simdLimit = (leftLen / 8) * 8; @@ -212,18 +222,18 @@ public static Float vectorL2Distance(ArrayData left, ArrayData right, UTF8String return null; } - float a0 = left.getFloat(i), a1 = left.getFloat(i + 1); - float a2 = left.getFloat(i + 2), a3 = left.getFloat(i + 3); - float a4 = left.getFloat(i + 4), a5 = left.getFloat(i + 5); - float a6 = left.getFloat(i + 6), a7 = left.getFloat(i + 7); + double a0 = left.getFloat(i), a1 = left.getFloat(i + 1); + double a2 = left.getFloat(i + 2), a3 = left.getFloat(i + 3); + double a4 = left.getFloat(i + 4), a5 = left.getFloat(i + 5); + double a6 = left.getFloat(i + 6), a7 = left.getFloat(i + 7); - float b0 = right.getFloat(i), b1 = right.getFloat(i + 1); - float b2 = right.getFloat(i + 2), b3 = right.getFloat(i + 3); - float b4 = right.getFloat(i + 4), b5 = right.getFloat(i + 5); - float b6 = right.getFloat(i + 6), b7 = right.getFloat(i + 7); + double b0 = right.getFloat(i), b1 = right.getFloat(i + 1); + double b2 = right.getFloat(i + 2), b3 = right.getFloat(i + 3); + double b4 = right.getFloat(i + 4), b5 = right.getFloat(i + 5); + double b6 = right.getFloat(i + 6), b7 = right.getFloat(i + 7); - float d0 = a0 - b0, d1 = a1 - b1, d2 = a2 - b2, d3 = a3 - b3; - float d4 = a4 - b4, d5 = a5 - b5, d6 = a6 - b6, d7 = a7 - b7; + double d0 = a0 - b0, d1 = a1 - b1, d2 = a2 - b2, d3 = a3 - b3; + double d4 = a4 - b4, d5 = a5 - b5, d6 = a6 - b6, d7 = a7 - b7; sumSq += d0 * d0 + d1 * d1 + d2 * d2 + d3 * d3 + d4 * d4 + d5 * d5 + d6 * d6 + d7 * d7; @@ -235,9 +245,9 @@ public static Float vectorL2Distance(ArrayData left, ArrayData right, UTF8String if (left.isNullAt(i) || right.isNullAt(i)) { return null; } - float a = left.getFloat(i); - float b = right.getFloat(i); - float diff = a - b; + double a = left.getFloat(i); + double b = right.getFloat(i); + double diff = a - b; sumSq += diff * diff; i++; } @@ -246,19 +256,19 @@ public static Float vectorL2Distance(ArrayData left, ArrayData right, UTF8String } /** - * Computes the L1 norm (Manhattan norm) of a float vector. + * Computes the L1 norm (Manhattan norm) of a float vector, in double precision. * Returns NULL if the vector contains NULL elements. * Returns 0.0 for empty vectors. * Uses manual loop unrolling (8 elements at a time) for speculative SIMD optimization. */ - public static Float vectorL1Norm(ArrayData vec) { + public static Double vectorL1Norm(ArrayData vec) { int len = vec.numElements(); if (len == 0) { - return 0.0f; + return 0.0d; } - float sum = 0.0f; + double sum = 0.0d; int i = 0; int simdLimit = (len / 8) * 8; @@ -273,10 +283,10 @@ public static Float vectorL1Norm(ArrayData vec) { return null; } - float a0 = vec.getFloat(i), a1 = vec.getFloat(i + 1); - float a2 = vec.getFloat(i + 2), a3 = vec.getFloat(i + 3); - float a4 = vec.getFloat(i + 4), a5 = vec.getFloat(i + 5); - float a6 = vec.getFloat(i + 6), a7 = vec.getFloat(i + 7); + double a0 = vec.getFloat(i), a1 = vec.getFloat(i + 1); + double a2 = vec.getFloat(i + 2), a3 = vec.getFloat(i + 3); + double a4 = vec.getFloat(i + 4), a5 = vec.getFloat(i + 5); + double a6 = vec.getFloat(i + 6), a7 = vec.getFloat(i + 7); sum += Math.abs(a0) + Math.abs(a1) + Math.abs(a2) + Math.abs(a3) + Math.abs(a4) + Math.abs(a5) + Math.abs(a6) + Math.abs(a7); @@ -288,7 +298,7 @@ public static Float vectorL1Norm(ArrayData vec) { if (vec.isNullAt(i)) { return null; } - float a = vec.getFloat(i); + double a = vec.getFloat(i); sum += Math.abs(a); i++; } @@ -297,19 +307,19 @@ public static Float vectorL1Norm(ArrayData vec) { } /** - * Computes the L2 norm (Euclidean norm) of a float vector. + * Computes the L2 norm (Euclidean norm) of a float vector, in double precision. * Returns NULL if the vector contains NULL elements. * Returns 0.0 for empty vectors. * Uses manual loop unrolling (8 elements at a time) for speculative SIMD optimization. */ - public static Float vectorL2Norm(ArrayData vec) { + public static Double vectorL2Norm(ArrayData vec) { int len = vec.numElements(); if (len == 0) { - return 0.0f; + return 0.0d; } - float sumSq = 0.0f; + double sumSq = 0.0d; int i = 0; int simdLimit = (len / 8) * 8; @@ -324,10 +334,10 @@ public static Float vectorL2Norm(ArrayData vec) { return null; } - float a0 = vec.getFloat(i), a1 = vec.getFloat(i + 1); - float a2 = vec.getFloat(i + 2), a3 = vec.getFloat(i + 3); - float a4 = vec.getFloat(i + 4), a5 = vec.getFloat(i + 5); - float a6 = vec.getFloat(i + 6), a7 = vec.getFloat(i + 7); + double a0 = vec.getFloat(i), a1 = vec.getFloat(i + 1); + double a2 = vec.getFloat(i + 2), a3 = vec.getFloat(i + 3); + double a4 = vec.getFloat(i + 4), a5 = vec.getFloat(i + 5); + double a6 = vec.getFloat(i + 6), a7 = vec.getFloat(i + 7); sumSq += a0 * a0 + a1 * a1 + a2 * a2 + a3 * a3 + a4 * a4 + a5 * a5 + a6 * a6 + a7 * a7; @@ -339,24 +349,24 @@ public static Float vectorL2Norm(ArrayData vec) { if (vec.isNullAt(i)) { return null; } - float a = vec.getFloat(i); + double a = vec.getFloat(i); sumSq += a * a; i++; } - return (float) Math.sqrt(sumSq); + return Math.sqrt(sumSq); } /** - * Computes the infinity norm (maximum absolute value) of a float vector. + * Computes the infinity norm (maximum absolute value) of a float vector, in double precision. * Returns NULL if the vector contains NULL elements. * Returns 0.0 for empty vectors. */ - public static Float vectorInfNorm(ArrayData vec) { + public static Double vectorInfNorm(ArrayData vec) { int len = vec.numElements(); if (len == 0) { - return 0.0f; + return 0.0d; } float maxAbs = 0.0f; @@ -370,7 +380,7 @@ public static Float vectorInfNorm(ArrayData vec) { } } - return maxAbs; + return (double) maxAbs; } /** @@ -378,15 +388,17 @@ public static Float vectorInfNorm(ArrayData vec) { * Returns NULL if the vector contains NULL elements or if the norm is zero. * Returns an empty array for empty vectors. * Uses manual loop unrolling (8 elements at a time) for speculative SIMD optimization. + * The norm is taken in double precision so that vectors whose norm is not representable as a + * float (or is only representable as a subnormal float) are still normalized correctly. */ - public static ArrayData vectorNormalizeWithNorm(ArrayData vec, float norm) { + public static ArrayData vectorNormalizeWithNorm(ArrayData vec, double norm) { int len = vec.numElements(); if (len == 0) { return vec; } - if (norm < Float.MIN_NORMAL) { + if (norm == 0.0d) { return null; } @@ -405,14 +417,14 @@ public static ArrayData vectorNormalizeWithNorm(ArrayData vec, float norm) { return null; } - result[i] = vec.getFloat(i) / norm; - result[i + 1] = vec.getFloat(i + 1) / norm; - result[i + 2] = vec.getFloat(i + 2) / norm; - result[i + 3] = vec.getFloat(i + 3) / norm; - result[i + 4] = vec.getFloat(i + 4) / norm; - result[i + 5] = vec.getFloat(i + 5) / norm; - result[i + 6] = vec.getFloat(i + 6) / norm; - result[i + 7] = vec.getFloat(i + 7) / norm; + result[i] = (float) (vec.getFloat(i) / norm); + result[i + 1] = (float) (vec.getFloat(i + 1) / norm); + result[i + 2] = (float) (vec.getFloat(i + 2) / norm); + result[i + 3] = (float) (vec.getFloat(i + 3) / norm); + result[i + 4] = (float) (vec.getFloat(i + 4) / norm); + result[i + 5] = (float) (vec.getFloat(i + 5) / norm); + result[i + 6] = (float) (vec.getFloat(i + 6) / norm); + result[i + 7] = (float) (vec.getFloat(i + 7) / norm); i += 8; } @@ -421,7 +433,7 @@ public static ArrayData vectorNormalizeWithNorm(ArrayData vec, float norm) { if (vec.isNullAt(i)) { return null; } - result[i] = vec.getFloat(i) / norm; + result[i] = (float) (vec.getFloat(i) / norm); i++; } @@ -429,13 +441,13 @@ public static ArrayData vectorNormalizeWithNorm(ArrayData vec, float norm) { } /** - * Computes the Lp norm of a float vector using the specified degree. - * Supported degrees: 1.0 (L1), 2.0 (L2), Float.POSITIVE_INFINITY (L∞). + * Computes the Lp norm of a float vector using the specified degree, in double precision. + * Supported degrees: 1.0 (L1), 2.0 (L2), Float.POSITIVE_INFINITY (infinity norm). * Returns NULL if the vector contains NULL elements. * Returns 0.0 for empty vectors. * Throws INVALID_VECTOR_NORM_DEGREE if degree is not supported. */ - public static Float vectorNorm(ArrayData vec, float degree, UTF8String funcName) { + private static Double vectorNormAsDouble(ArrayData vec, float degree, UTF8String funcName) { // exact floating point comparison for degree since this is direct user input if (degree == 1.0f) { return vectorL1Norm(vec); @@ -448,6 +460,21 @@ public static Float vectorNorm(ArrayData vec, float degree, UTF8String funcName) } } + /** + * Computes the Lp norm of a float vector using the specified degree. + * Supported degrees: 1.0 (L1), 2.0 (L2), Float.POSITIVE_INFINITY (L∞). + * Returns NULL if the vector contains NULL elements. + * Returns 0.0 for empty vectors. + * Throws INVALID_VECTOR_NORM_DEGREE if degree is not supported. + */ + public static Float vectorNorm(ArrayData vec, float degree, UTF8String funcName) { + Double norm = vectorNormAsDouble(vec, degree, funcName); + if (norm == null) { + return null; + } + return (float) norm.doubleValue(); + } + /** * Normalizes a float vector to unit length using the specified norm degree. * Supported degrees: 1.0 (L1), 2.0 (L2), Float.POSITIVE_INFINITY (L∞). @@ -456,7 +483,10 @@ public static Float vectorNorm(ArrayData vec, float degree, UTF8String funcName) * Throws INVALID_VECTOR_NORM_DEGREE if degree is not supported. */ public static ArrayData vectorNormalize(ArrayData vec, float degree, UTF8String funcName) { - Float norm = vectorNorm(vec, degree, funcName); + // The norm is kept in double precision here: rounding it to a float first would turn a norm + // that overflows (or underflows) the float range into infinity (or zero) and produce an + // all-zero (or NULL) result for a vector that is perfectly normalizable. + Double norm = vectorNormAsDouble(vec, degree, funcName); if (norm == null) { return null; } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/vectorExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/vectorExpressions.scala index e65fae3a2bc2c..caddda9da80ed 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/vectorExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/vectorExpressions.scala @@ -46,7 +46,7 @@ import org.apache.spark.unsafe.Platform examples = """ Examples: > SELECT _FUNC_(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F)); - 0.9746319 + 0.97463185 """, since = "4.2.0", group = "vector_funcs" diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/vector-distance.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/vector-distance.sql.out index 42239b3297305..ea7eafd623e9b 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/vector-distance.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/vector-distance.sql.out @@ -654,3 +654,45 @@ SELECT vector_l2_distance( -- !query analysis Project [vector_l2_distance(array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), array(16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0)) AS vector_l2_distance(array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), array(16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0))#x] +- OneRowRelation + + +-- !query +SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(3.0e19F, 4.0e19F)) +-- !query analysis +Project [vector_cosine_similarity(array(3.0E19, 4.0E19), array(3.0E19, 4.0E19)) AS vector_cosine_similarity(array(3.0E19, 4.0E19), array(3.0E19, 4.0E19))#x] ++- OneRowRelation + + +-- !query +SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(-3.0e19F, -4.0e19F)) +-- !query analysis +Project [vector_cosine_similarity(array(3.0E19, 4.0E19), array(-3.0E19, -4.0E19)) AS vector_cosine_similarity(array(3.0E19, 4.0E19), array(-3.0E19, -4.0E19))#x] ++- OneRowRelation + + +-- !query +SELECT vector_inner_product(array(1.0e20F, 1.0e20F), array(1.0e20F, -1.0e20F)) +-- !query analysis +Project [vector_inner_product(array(1.0E20, 1.0E20), array(1.0E20, -1.0E20)) AS vector_inner_product(array(1.0E20, 1.0E20), array(1.0E20, -1.0E20))#x] ++- OneRowRelation + + +-- !query +SELECT vector_l2_distance(array(3.0e19F, 4.0e19F), array(0.0F, 0.0F)) +-- !query analysis +Project [vector_l2_distance(array(3.0E19, 4.0E19), array(0.0, 0.0)) AS vector_l2_distance(array(3.0E19, 4.0E19), array(0.0, 0.0))#x] ++- OneRowRelation + + +-- !query +SELECT vector_cosine_similarity(array(1.0e-23F, 0.0F), array(1.0e-23F, 0.0F)) +-- !query analysis +Project [vector_cosine_similarity(array(1.0E-23, 0.0), array(1.0E-23, 0.0)) AS vector_cosine_similarity(array(1.0E-23, 0.0), array(1.0E-23, 0.0))#x] ++- OneRowRelation + + +-- !query +SELECT vector_cosine_similarity(array(1.0e-23F, 1.0e-23F), array(1.0e-23F, -1.0e-23F)) +-- !query analysis +Project [vector_cosine_similarity(array(1.0E-23, 1.0E-23), array(1.0E-23, -1.0E-23)) AS vector_cosine_similarity(array(1.0E-23, 1.0E-23), array(1.0E-23, -1.0E-23))#x] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/vector-norm.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/vector-norm.sql.out index 035ad40cc1222..b06f7c76b09e4 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/vector-norm.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/vector-norm.sql.out @@ -572,3 +572,52 @@ SELECT vector_norm( -- !query analysis Project [vector_norm(array(1.0, 2.0, 3.0, 4.0, 5.0, cast(null as float), 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), 2.0) AS vector_norm(array(1.0, 2.0, 3.0, 4.0, 5.0, CAST(NULL AS FLOAT), 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), 2.0)#x] +- OneRowRelation + + +-- !query +SELECT vector_norm(array(3.0e19F, 4.0e19F), 2.0F) +-- !query analysis +Project [vector_norm(array(3.0E19, 4.0E19), 2.0) AS vector_norm(array(3.0E19, 4.0E19), 2.0)#x] ++- OneRowRelation + + +-- !query +SELECT vector_normalize(array(3.0e19F, 4.0e19F), 2.0F) +-- !query analysis +Project [vector_normalize(array(3.0E19, 4.0E19), 2.0) AS vector_normalize(array(3.0E19, 4.0E19), 2.0)#x] ++- OneRowRelation + + +-- !query +SELECT vector_norm(array(3.0e38F, 3.0e38F), 1.0F) +-- !query analysis +Project [vector_norm(array(3.0E38, 3.0E38), 1.0) AS vector_norm(array(3.0E38, 3.0E38), 1.0)#x] ++- OneRowRelation + + +-- !query +SELECT vector_normalize(array(3.0e38F, 3.0e38F), 1.0F) +-- !query analysis +Project [vector_normalize(array(3.0E38, 3.0E38), 1.0) AS vector_normalize(array(3.0E38, 3.0E38), 1.0)#x] ++- OneRowRelation + + +-- !query +SELECT vector_norm(array(1.0e-23F, 0.0F), 2.0F) +-- !query analysis +Project [vector_norm(array(1.0E-23, 0.0), 2.0) AS vector_norm(array(1.0E-23, 0.0), 2.0)#x] ++- OneRowRelation + + +-- !query +SELECT vector_normalize(array(1.0e-23F, 0.0F), 2.0F) +-- !query analysis +Project [vector_normalize(array(1.0E-23, 0.0), 2.0) AS vector_normalize(array(1.0E-23, 0.0), 2.0)#x] ++- OneRowRelation + + +-- !query +SELECT vector_normalize(array(1.0e-23F, 1.0e-23F), 2.0F) +-- !query analysis +Project [vector_normalize(array(1.0E-23, 1.0E-23), 2.0) AS vector_normalize(array(1.0E-23, 1.0E-23), 2.0)#x] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/vector-distance.sql b/sql/core/src/test/resources/sql-tests/inputs/vector-distance.sql index 24035963260b4..cbae450203e25 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/vector-distance.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/vector-distance.sql @@ -130,3 +130,26 @@ SELECT vector_l2_distance( array(1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F, 7.0F, 8.0F, 9.0F, 10.0F, 11.0F, 12.0F, 13.0F, 14.0F, 15.0F, 16.0F), array(16.0F, 15.0F, 14.0F, 13.0F, 12.0F, 11.0F, 10.0F, 9.0F, 8.0F, 7.0F, 6.0F, 5.0F, 4.0F, 3.0F, 2.0F, 1.0F) ); + +-- SPARK-58544: large magnitudes, intermediate sums of squares/products must not overflow the +-- float range + +-- vector_cosine_similarity is scale invariant: identical vectors have similarity 1.0 +SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(3.0e19F, 4.0e19F)); + +-- vector_cosine_similarity: opposite vectors have similarity -1.0 +SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(-3.0e19F, -4.0e19F)); + +-- vector_inner_product: individual products overflow the float range but cancel out +SELECT vector_inner_product(array(1.0e20F, 1.0e20F), array(1.0e20F, -1.0e20F)); + +-- vector_l2_distance: sqrt((3e19)^2 + (4e19)^2) = 5e19 +SELECT vector_l2_distance(array(3.0e19F, 4.0e19F), array(0.0F, 0.0F)); + +-- SPARK-58544: small magnitudes, intermediate sums of squares/products must not underflow to zero + +-- vector_cosine_similarity of identical tiny vectors is 1.0, not NULL +SELECT vector_cosine_similarity(array(1.0e-23F, 0.0F), array(1.0e-23F, 0.0F)); + +-- vector_cosine_similarity of orthogonal tiny vectors is 0.0, not NULL +SELECT vector_cosine_similarity(array(1.0e-23F, 1.0e-23F), array(1.0e-23F, -1.0e-23F)); diff --git a/sql/core/src/test/resources/sql-tests/inputs/vector-norm.sql b/sql/core/src/test/resources/sql-tests/inputs/vector-norm.sql index 13eacf854b78a..986bec0d94ee7 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/vector-norm.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/vector-norm.sql @@ -153,3 +153,27 @@ SELECT vector_norm( array(1.0F, 2.0F, 3.0F, 4.0F, 5.0F, CAST(NULL AS FLOAT), 7.0F, 8.0F, 9.0F, 10.0F, 11.0F, 12.0F, 13.0F, 14.0F, 15.0F, 16.0F), 2.0F ); + +-- SPARK-58544: large magnitudes, the intermediate sum of squares must not overflow the float range + +-- vector_norm: sqrt((3e19)^2 + (4e19)^2) = 5e19 +SELECT vector_norm(array(3.0e19F, 4.0e19F), 2.0F); + +-- vector_normalize: [3e19, 4e19] normalizes to [0.6, 0.8] +SELECT vector_normalize(array(3.0e19F, 4.0e19F), 2.0F); + +-- vector_norm: the L1 norm itself is not representable as a float, so it stays infinite +SELECT vector_norm(array(3.0e38F, 3.0e38F), 1.0F); + +-- vector_normalize: normalization still succeeds when the norm exceeds the float range +SELECT vector_normalize(array(3.0e38F, 3.0e38F), 1.0F); + +-- SPARK-58544: small magnitudes, the intermediate sum of squares must not underflow to zero + +-- vector_norm: the L2 norm of a tiny vector is not zero +SELECT vector_norm(array(1.0e-23F, 0.0F), 2.0F); + +-- vector_normalize: normalization is scale invariant, so the result is a unit vector, not NULL +SELECT vector_normalize(array(1.0e-23F, 0.0F), 2.0F); + +SELECT vector_normalize(array(1.0e-23F, 1.0e-23F), 2.0F); diff --git a/sql/core/src/test/resources/sql-tests/results/vector-distance.sql.out b/sql/core/src/test/resources/sql-tests/results/vector-distance.sql.out index ccc39c23fb53c..26801641be9fc 100644 --- a/sql/core/src/test/resources/sql-tests/results/vector-distance.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/vector-distance.sql.out @@ -4,7 +4,7 @@ SELECT vector_cosine_similarity(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F) -- !query schema struct -- !query output -0.9746319 +0.97463185 -- !query @@ -744,3 +744,51 @@ SELECT vector_l2_distance( struct -- !query output 36.878178 + + +-- !query +SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(3.0e19F, 4.0e19F)) +-- !query schema +struct +-- !query output +1.0 + + +-- !query +SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(-3.0e19F, -4.0e19F)) +-- !query schema +struct +-- !query output +-1.0 + + +-- !query +SELECT vector_inner_product(array(1.0e20F, 1.0e20F), array(1.0e20F, -1.0e20F)) +-- !query schema +struct +-- !query output +0.0 + + +-- !query +SELECT vector_l2_distance(array(3.0e19F, 4.0e19F), array(0.0F, 0.0F)) +-- !query schema +struct +-- !query output +5.0E19 + + +-- !query +SELECT vector_cosine_similarity(array(1.0e-23F, 0.0F), array(1.0e-23F, 0.0F)) +-- !query schema +struct +-- !query output +1.0 + + +-- !query +SELECT vector_cosine_similarity(array(1.0e-23F, 1.0e-23F), array(1.0e-23F, -1.0e-23F)) +-- !query schema +struct +-- !query output +0.0 diff --git a/sql/core/src/test/resources/sql-tests/results/vector-norm.sql.out b/sql/core/src/test/resources/sql-tests/results/vector-norm.sql.out index 8ebd583aa7e47..c3cc1a6949c47 100644 --- a/sql/core/src/test/resources/sql-tests/results/vector-norm.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/vector-norm.sql.out @@ -668,7 +668,7 @@ SELECT vector_normalize( -- !query schema struct> -- !query output -[0.025854385,0.05170877,0.07756316,0.10341754,0.12927192,0.15512632,0.1809807,0.20683508,0.23268947,0.25854385,0.28439823,0.31025264,0.33610702,0.3619614,0.38781577,0.41367015] +[0.025854385,0.05170877,0.07756315,0.10341754,0.12927192,0.1551263,0.1809807,0.20683508,0.23268946,0.25854385,0.28439823,0.3102526,0.336107,0.3619614,0.38781577,0.41367015] -- !query @@ -680,3 +680,59 @@ SELECT vector_norm( struct -- !query output NULL + + +-- !query +SELECT vector_norm(array(3.0e19F, 4.0e19F), 2.0F) +-- !query schema +struct +-- !query output +5.0E19 + + +-- !query +SELECT vector_normalize(array(3.0e19F, 4.0e19F), 2.0F) +-- !query schema +struct> +-- !query output +[0.6,0.8] + + +-- !query +SELECT vector_norm(array(3.0e38F, 3.0e38F), 1.0F) +-- !query schema +struct +-- !query output +Infinity + + +-- !query +SELECT vector_normalize(array(3.0e38F, 3.0e38F), 1.0F) +-- !query schema +struct> +-- !query output +[0.5,0.5] + + +-- !query +SELECT vector_norm(array(1.0e-23F, 0.0F), 2.0F) +-- !query schema +struct +-- !query output +1.0E-23 + + +-- !query +SELECT vector_normalize(array(1.0e-23F, 0.0F), 2.0F) +-- !query schema +struct> +-- !query output +[1.0,0.0] + + +-- !query +SELECT vector_normalize(array(1.0e-23F, 1.0e-23F), 2.0F) +-- !query schema +struct> +-- !query output +[0.70710677,0.70710677]