added equi-depth SAX breakpoints for all approaches. Gaussian by defa…

.gitignore

@@ -37,6 +37,7 @@
 *.part
 *.full
 isax_file.sax
+daisy_index_nodes/
 
 # Build directories
 build/

lib/algos/DumpyOS.cpp

@@ -420,13 +420,13 @@ static void get_value_range(int sax_val, int bc, double* lb, double* ub) {
     int offset = ((cardinality - 1) * (cardinality - 2)) / 2;
     if (sax_val == 0) {
         *lb = -std::numeric_limits<double>::max();
-        *ub = sax_breakpoints[offset];
+        *ub = daisy_active_breakpoints[offset];
     } else if (sax_val == cardinality - 1) {
-        *lb = sax_breakpoints[offset + sax_val - 1];
+        *lb = daisy_active_breakpoints[offset + sax_val - 1];
         *ub = std::numeric_limits<double>::max();
     } else {
-        *lb = sax_breakpoints[offset + sax_val - 1];
-        *ub = sax_breakpoints[offset + sax_val];
+        *lb = daisy_active_breakpoints[offset + sax_val - 1];
+        *ub = daisy_active_breakpoints[offset + sax_val];
     }
 }
 

lib/algos/Fresh.cpp

@@ -899,7 +899,12 @@ void Fresh::buildIndex(DataSource *data_source)
                                                     this->tight_bound,
                                                     0,
                                                     1,
-                                                    1);
+                                                    1,
+                                                    this->bp_mode);
+
+    // Equi-depth breakpoints (no-op in Gaussian mode), installed as active.
+    compute_equidepth_breakpoints(this->index_settings, this->database, this->n_database);
+    activateBreakpoints();
 
     this->index = isax_index_init_inmemory(this->index_settings);
     isax_index *index = this->index;

lib/algos/Hercules.hpp

@@ -157,6 +157,14 @@ class Hercules : public SimilaritySearchAlgorithm
         config_.num_query_threads = num_threads;
     }
 
+    // Hercules only supports z-normalized data
+    void setNormalized(bool normalized) override
+    {
+        if (!normalized)
+            throw std::runtime_error(
+                "Hercules currently supports only z-normalized data (equi-depth breakpoints not implemented).");
+    }
+
     ~Hercules() override;
 
 private:

lib/algos/LbBruteforce.cpp

@@ -78,7 +78,12 @@ namespace daisy
                                                         this->tight_bound,
                                                         0,
                                                         1,
-                                                        1);
+                                                        1,
+                                                        this->bp_mode);
+
+        // Equi-depth breakpoints (no-op in Gaussian mode), installed as active.
+        compute_equidepth_breakpoints(this->index_settings, this->database, this->n_database);
+        activateBreakpoints();
 
         this->index = isax_index_init_inmemory(this->index_settings);
 

lib/algos/Messi.cpp

@@ -794,7 +794,12 @@ namespace daisy
                                                         this->tight_bound,
                                                         0,
                                                         1,
-                                                        1);
+                                                        1,
+                                                        this->bp_mode);
+
+        // Equi-depth breakpoints (no-op in Gaussian mode), installed as active.
+        compute_equidepth_breakpoints(this->index_settings, this->database, this->n_database);
+        activateBreakpoints();
 
         this->index = isax_index_init_inmemory(this->index_settings);
         isax_index *index = this->index;

lib/algos/ParIS.cpp

@@ -63,10 +63,28 @@ namespace daisy
                                                         this->flush_limit,         
                                                         this->initial_fbl_size,    
                                                         this->total_loaded_leaves, 
-                                                        this->tight_bound,         
-                                                        0,                         
-                                                        1,                         
-                                                        0);
+                                                        this->tight_bound,
+                                                        0,
+                                                        1,
+                                                        0,
+                                                        this->bp_mode);
+
+        // Equi-depth breakpoints. ParIS streams from a raw float32 file, so sample leading
+        // rows from it to derive the distribution.
+        if (this->bp_mode == BP_EQUIDEPTH && this->n_database > 0 && this->dim > 0)
+        {
+            size_t cap_rows = (size_t)2000000 / (size_t)std::max(1, this->paa_segments);
+            size_t sample_rows = std::min((size_t)this->n_database, std::max((size_t)1, cap_rows));
+            std::vector<float> sample((size_t)sample_rows * (size_t)this->dim);
+            FILE *sf = fopen(filename, "rb");
+            if (sf)
+            {
+                size_t got = fread(sample.data(), sizeof(float), sample.size(), sf);
+                fclose(sf);
+                compute_equidepth_breakpoints(this->index_settings, sample.data(), got / (size_t)this->dim);
+            }
+        }
+        activateBreakpoints();
 
         this->index = isax_index_init(this->index_settings);
         isax_index *index = this->index;

lib/algos/SimilaritySearchAlgorithm.hpp

@@ -52,6 +52,8 @@ namespace daisy
         float minimum_distance = FLT_MAX;
         int min_checked_leaves = -1;
 
+        breakpoint_mode bp_mode = BP_GAUSSIAN_HARDCODED;
+
         isax_index_settings *index_settings = nullptr;
         isax_index *index = nullptr;
         sax_type **db_sax_representations = nullptr;
@@ -68,6 +70,25 @@ namespace daisy
         idx_t getDim() const { return dim; }
         isax_index *getIndex() const { return index; }
 
+        // Declares whether the input data is z-normalized
+        //   normalized == true  (default) -> hardcoded Gaussian breakpoints
+        //   normalized == false           -> data-adaptive equi-depth breakpoints
+        // DaiSy does not normalize data itself
+        virtual void setNormalized(bool normalized)
+        {
+            bp_mode = normalized ? BP_GAUSSIAN_HARDCODED : BP_EQUIDEPTH;
+        }
+        bool getNormalized() const { return bp_mode == BP_GAUSSIAN_HARDCODED; }
+
+    protected:
+        void activateBreakpoints()
+        {
+            if (index_settings)
+                set_active_breakpoints(index_settings->breakpoints, index_settings->breakpoints_max);
+        }
+
+    public:
+
         virtual void buildIndex(DataSource *data_source) = 0;
 
         virtual void buildIndex(float *database, idx_t n_database, idx_t dim)

lib/algos/Sing.cpp

@@ -105,17 +105,17 @@ namespace daisy
         __m256i lower_juge_nzerov_1 = _mm256_andnot_si256(lower_juge_zerov_1, vectorsignbit);
         __m256 minvalv = _mm256_set1_ps((float)min_val);
 
-        __m256 lsax_breakpoints_shiftv_0 = _mm256_i32gather_ps(sax_breakpointsnew3, region_lowerv_0, 4);
-        __m256 lsax_breakpoints_shiftv_1 = _mm256_i32gather_ps(sax_breakpointsnew3, region_lowerv_1, 4);
+        __m256 lsax_breakpoints_shiftv_0 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_lowerv_0, 4);
+        __m256 lsax_breakpoints_shiftv_1 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_lowerv_1, 4);
         __m256 breakpoint_lowerv_0 = (__m256)_mm256_castsi256_ps(_mm256_or_si256(
             _mm256_and_si256(lower_juge_zerov_0, _mm256_castps_si256(minvalv)),
             _mm256_and_si256(lower_juge_nzerov_0, _mm256_castps_si256(lsax_breakpoints_shiftv_0))));
         __m256 breakpoint_lowerv_1 = (__m256)_mm256_castsi256_ps(_mm256_or_si256(
             _mm256_and_si256(lower_juge_zerov_1, _mm256_castps_si256(minvalv)),
             _mm256_and_si256(lower_juge_nzerov_1, _mm256_castps_si256(lsax_breakpoints_shiftv_1))));
 
-        __m256 usax_breakpoints_shiftv_0 = _mm256_i32gather_ps(sax_breakpointsnew3, region_upperv_0, 4);
-        __m256 usax_breakpoints_shiftv_1 = _mm256_i32gather_ps(sax_breakpointsnew3, region_upperv_1, 4);
+        __m256 usax_breakpoints_shiftv_0 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_upperv_0, 4);
+        __m256 usax_breakpoints_shiftv_1 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_upperv_1, 4);
         __m256i upper_juge_maxv_0 = _mm256_cmpeq_epi32(region_upperv_0, _mm256_set1_epi32(max_cardinality - 1));
         __m256i upper_juge_maxv_1 = _mm256_cmpeq_epi32(region_upperv_1, _mm256_set1_epi32(max_cardinality - 1));
         __m256i upper_juge_nmaxv_0 = _mm256_andnot_si256(upper_juge_maxv_0, vectorsignbit);
@@ -267,15 +267,15 @@ namespace daisy
             }
             else
             {
-                breakpoint_lower = sax_breakpointsnew3[region_lower];
+                breakpoint_lower = daisy_active_breakpoints_max[region_lower];
             }
             if (region_upper == (sax_type)(max_cardinality - 1))
             {
                 breakpoint_upper = (float)max_val;
             }
             else
             {
-                breakpoint_upper = sax_breakpointsnew3[region_upper + 1];
+                breakpoint_upper = daisy_active_breakpoints_max[region_upper + 1];
             }
 
             if (breakpoint_lower > paa[i])
@@ -471,15 +471,15 @@ namespace daisy
             }
             else
             {
-                breakpoint_lower = sax_breakpointsnew3[region_lower - 1];
+                breakpoint_lower = daisy_active_breakpoints_max[region_lower - 1];
             }
             if (region_upper == (sax_type)(max_cardinality - 1))
             {
                 breakpoint_upper = (float)max_val;
             }
             else
             {
-                breakpoint_upper = sax_breakpointsnew3[region_upper];
+                breakpoint_upper = daisy_active_breakpoints_max[region_upper];
             }
 
             (void)mask;