From 09bc7c24e7502bf30802e80c71bf96b3c384a20e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 26 Jul 2025 17:06:41 +0100 Subject: [PATCH 001/123] Use activations to calculate the stats --- tools/imatrix/imatrix.cpp | 64 +++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 9aad3711bae5..715a589037dc 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -38,10 +38,12 @@ static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count"; static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size"; struct Stats { + std::vector activations; std::vector values; std::vector counts; }; +//ToDo: rename sqract variables to be more generic like 'values' struct tensor_statistics { std::string tensor; Stats stats; @@ -139,14 +141,28 @@ static void compute_statistics(std::vector & tstats, const st const int row_size = e.values.size() / n_mat; std::vector activations; - activations.reserve(e.values.size()); - for (int i = 0; i < n_mat; ++i) { - for (int j = 0; j < row_size; ++j) { - activations.push_back(e.values[i*row_size + j] / e.counts[i]); + if (e.activations.empty()) { + activations.reserve(e.values.size()); + + for (int i = 0; i < n_mat; ++i) { + for (int j = 0; j < row_size; ++j) { + activations.push_back(e.values[i*row_size + j] / e.counts[i]); + } + } + } else { + activations.reserve(e.activations.size()); + + for (int i = 0; i < n_mat; ++i) { + for (int j = 0; j < row_size; ++j) { + activations.push_back(e.activations[i*row_size + j] / e.counts[i]); + } } } + + + //ToDo: rename act_ variables to be more generic like 'values' const float act_total = std::accumulate(activations.begin(), activations.end(), 0.0f); const float act_max = *std::max_element(activations.begin(), activations.end()); const float act_min = *std::min_element(activations.begin(), activations.end()); @@ -282,6 +298,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts.resize(n_as, e.counts[0]); } if (e.values.empty()) { + e.activations.resize(src1->ne[0]*n_as, 0); e.values.resize(src1->ne[0]*n_as, 0); e.counts.resize(n_as, 0); } @@ -313,6 +330,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts[ex]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { + e.activations[e_start + j] += x[j]; e.values[e_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[e_start + j])) { LOG_ERR("%f detected in %s\n", (float)e.values[e_start + j], wname.c_str()); @@ -338,6 +356,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * const int64_t n_mat = src1->ne[2] * src1->ne[3]; if (e.values.empty()) { + e.activations.resize(src1->ne[0] * n_mat, 0); e.values.resize(src1->ne[0] * n_mat, 0); e.counts.resize(n_mat, 0); } @@ -359,6 +378,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * const float * x = (const float *) (data + row * src1->nb[1] + i2 * src1->nb[2] + i3 * src1->ne[3]); e.counts[mat_id]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { + e.activations[mat_start + j] += x[j]; e.values[mat_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[j])) { LOG_ERR("%f detected in %s\n", (float)e.values[j], wname.c_str()); @@ -532,6 +552,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { } to_store.push_back(kv.first); + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); } @@ -584,6 +605,16 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum2); gguf_add_tensor(ctx_gguf, counts); + + if (!stat.activations.empty()) { + const int32_t nact = (int32_t) stat.activations.size(); + struct ggml_tensor * in_sum = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nact / nmat, nmat); + ggml_format_name(in_sum, "%s.in_sum", name.c_str()); // ToDo: consider a better name. 'in_act' maybe? + for (int32_t j = 0; j < nval; ++j) { + ((float *) in_sum->data)[j] = (float) stat.activations[j]; + } + gguf_add_tensor(ctx_gguf, in_sum); + } } } @@ -722,6 +753,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { } } + const std::string in_sum_suffix{ ".in_sum" }; const std::string in_sum2_suffix{ ".in_sum2" }; const std::string counts_suffix{ ".counts" }; @@ -729,7 +761,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { // checking for completeness of *each* loaded imatrix file // and also makes it easier to re-use a similar implementation in quantize.cpp // Using an ordered map to get a deterministic iteration order. - std::map> sums_counts_for; + std::map> sums_counts_for; for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { std::string name = cur->name; @@ -738,19 +770,24 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { if (string_remove_suffix(name, in_sum2_suffix)) { // in_sum2 - sums_counts_for[std::move(name)].first = cur; + std::get<0>(sums_counts_for[std::move(name)]) = cur; } else if (string_remove_suffix(name, counts_suffix)) { // counts - sums_counts_for[std::move(name)].second = cur; - } else { + std::get<1>(sums_counts_for[std::move(name)]) = cur; + } else if (string_remove_suffix(name, in_sum_suffix)) { + // in_sum + std::get<2>(sums_counts_for[std::move(name)]) = cur; + } + else { // ignore other tensors } } for (const auto & sc : sums_counts_for) { const std::string & name = sc.first; - const struct ggml_tensor * in_sum2 = sc.second.first; - const struct ggml_tensor * counts = sc.second.second; + const struct ggml_tensor * in_sum2 = std::get<0>(sc.second); + const struct ggml_tensor * counts = std::get<1>(sc.second); + const struct ggml_tensor * in_sum = std::get<2>(sc.second); if (!in_sum2 || !counts) { LOG_ERR("%s: mismatched sums and counts for %s\n", __func__, name.c_str()); @@ -764,6 +801,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { int64_t nval = ggml_nelements(in_sum2); if (e.values.empty()) { e.values.resize(nval, 0.0f); + e.activations.resize(nval, 0.0f); } else if ((size_t) nval != e.values.size()) { LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size()); gguf_free(ctx_gguf); @@ -791,6 +829,12 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { for (int64_t j = 0; j < ncounts; j++) { e.counts[j] += std::lround(((const float *) counts->data)[j]); } + // ToDo: fix blow up when GGUF does not have in_sum + if (in_sum->data != nullptr) { + for (int64_t j = 0; j < nval; j++) { + e.activations[j] += ((const float *) in_sum->data)[j]; + } + } } // TODO: extract into its own method; this is also used by the legacy format From 2097f038b07f43bdbb40b568f310474467414fac Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 31 Jul 2025 20:46:40 +0100 Subject: [PATCH 002/123] Refactor variable names --- tools/imatrix/imatrix.cpp | 188 +++++++++++++++++++------------------- 1 file changed, 92 insertions(+), 96 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 715a589037dc..b92b1486f7b1 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -38,8 +38,8 @@ static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count"; static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size"; struct Stats { - std::vector activations; - std::vector values; + std::vector in_sum; + std::vector in_sum2; std::vector counts; }; @@ -47,15 +47,15 @@ struct Stats { struct tensor_statistics { std::string tensor; Stats stats; - float total_sqract = 0.0f; - float mean_sqract = 0.0f; - float max_sqract = 0.0f; - float min_sqract = 0.0f; - int elements = 0; + float sum_values = 0.0f; + float mean_values = 0.0f; + float max_values = 0.0f; + float min_values = 0.0f; + int elements = 0; float stddev = 0.0f; float active = 0.0f; float entropy = 0.0f; - float zd = 0.0f; + float zd_score = 0.0f; float cossim = 0.0f; }; @@ -128,8 +128,8 @@ static void process_tensor_name(const std::string & input, std::string & layer, } static void compute_statistics(std::vector & tstats, const std::string & name, const Stats & e) { - if (e.values.size() % e.counts.size() != 0) { - LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.values.size()); + if (e.in_sum2.size() % e.counts.size() != 0) { + LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.in_sum2.size()); return; } if (e.counts.empty()) { @@ -138,73 +138,69 @@ static void compute_statistics(std::vector & tstats, const st } const int n_mat = e.counts.size(); - const int row_size = e.values.size() / n_mat; + const int row_size = e.in_sum2.size() / n_mat; std::vector activations; - if (e.activations.empty()) { - activations.reserve(e.values.size()); + if (e.in_sum.empty()) { + activations.reserve(e.in_sum2.size()); for (int i = 0; i < n_mat; ++i) { for (int j = 0; j < row_size; ++j) { - activations.push_back(e.values[i*row_size + j] / e.counts[i]); + activations.push_back(e.in_sum2[i*row_size + j] / e.counts[i]); } } } else { - activations.reserve(e.activations.size()); + activations.reserve(e.in_sum.size()); for (int i = 0; i < n_mat; ++i) { for (int j = 0; j < row_size; ++j) { - activations.push_back(e.activations[i*row_size + j] / e.counts[i]); + activations.push_back(e.in_sum[i*row_size + j] / e.counts[i]); } } } - - - //ToDo: rename act_ variables to be more generic like 'values' - const float act_total = std::accumulate(activations.begin(), activations.end(), 0.0f); - const float act_max = *std::max_element(activations.begin(), activations.end()); - const float act_min = *std::min_element(activations.begin(), activations.end()); - const float act_mean = act_total / activations.size(); - const float act_sqr_total = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); - const float act_var = (act_sqr_total / activations.size()) - (act_mean * act_mean); - const float act_dev = std::sqrt(std::max(0.0f, act_var)); - float threshold = 1e-5f; - const int inactive_count = std::count_if(activations.begin(), activations.end(), - [threshold](const float v) { return fabsf(v) <= threshold; }); - const float active_ratio = 1 - static_cast(inactive_count) / activations.size(); + const float sum = std::accumulate(activations.begin(), activations.end(), 0.0f); + const float max = *std::max_element(activations.begin(), activations.end()); + const float min = *std::min_element(activations.begin(), activations.end()); + const float mean = sum / activations.size(); + const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); + const float variance = (sqr_sum / activations.size()) - (mean * mean); + const float std_deviation = std::sqrt(std::max(0.0f, variance)); + const float threshold = 1e-5f; + const int inactive_count = std::count_if(activations.begin(), activations.end(), [threshold](const float v) { return fabsf(v) <= threshold; }); + const float active_ratio = 1 - static_cast(inactive_count) / activations.size(); float entropy = 0; - if (act_total > 0) { + if (sum > 0) { for (const auto act : activations) { - if (const float p = act / act_total; p > 0) { + if (const float p = act / sum; p > 0) { entropy -= p * std::log2(p); } } } int z_score = 0; - if (act_dev > 0.0f) { + if (std_deviation > 0.0f) { for (const auto act : activations) { - if (const float p = (act - act_mean) / act_dev; p > 1) { + if (const float p = (act - mean) / std_deviation; p > 1) { z_score++; } } } auto & ts = tstats.emplace_back(); - ts.tensor = name; - ts.stats = e; - ts.total_sqract = act_total; - ts.mean_sqract = act_mean; - ts.max_sqract = act_max; - ts.min_sqract = act_min; - ts.elements = static_cast(activations.size()); - ts.stddev = act_dev; - ts.active = active_ratio; - ts.entropy = entropy; - ts.zd = static_cast(z_score) / ts.elements; + ts.tensor = name; + ts.stats = e; + ts.sum_values = sum; + ts.mean_values = mean; + ts.max_values = max; + ts.min_values = min; + ts.elements = static_cast(activations.size()); + ts.stddev = std_deviation; + ts.active = active_ratio; + ts.entropy = entropy; + ts.zd_score = static_cast(z_score) / ts.elements; } static void compute_cossim(std::vector & tstats) { @@ -217,14 +213,14 @@ static void compute_cossim(std::vector & tstats) { auto prev = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); if (prev != tstats.end()) { - const float dp = std::inner_product(ts.stats.values.begin(), ts.stats.values.end(), - prev->stats.values.begin(), 0.0f); - const float curr_mag = std::sqrt(std::inner_product(ts.stats.values.begin(), ts.stats.values.end(), - ts.stats.values.begin(), 0.0f)); - const float prev_mag = std::sqrt(std::inner_product(prev->stats.values.begin(), prev->stats.values.end(), - prev->stats.values.begin(), 0.0f)); - const float cs = dp / (curr_mag * prev_mag); - ts.cossim = cs; + const float dot_product = std::inner_product(ts.stats.in_sum2.begin(), ts.stats.in_sum2.end(), + prev->stats.in_sum2.begin(), 0.0f); + const float magnitude = std::sqrt(std::inner_product(ts.stats.in_sum2.begin(), ts.stats.in_sum2.end(), + ts.stats.in_sum2.begin(), 0.0f)); + const float prev_magnitude = std::sqrt(std::inner_product(prev->stats.in_sum2.begin(), prev->stats.in_sum2.end(), + prev->stats.in_sum2.begin(), 0.0f)); + const float cos_sim = dot_product / (magnitude * prev_magnitude); + ts.cossim = cos_sim; } } else { ts.cossim = 0; @@ -297,13 +293,13 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * // broadcast, when loading an old imatrix e.counts.resize(n_as, e.counts[0]); } - if (e.values.empty()) { - e.activations.resize(src1->ne[0]*n_as, 0); - e.values.resize(src1->ne[0]*n_as, 0); + if (e.in_sum2.empty()) { + e.in_sum.resize(src1->ne[0]*n_as, 0); + e.in_sum2.resize(src1->ne[0]*n_as, 0); e.counts.resize(n_as, 0); } - else if (e.values.size() != (size_t)src1->ne[0]*n_as) { - LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.values.size(), (int)(src1->ne[0]*n_as)); + else if (e.in_sum2.size() != (size_t)src1->ne[0]*n_as) { + LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.in_sum2.size(), (int)(src1->ne[0]*n_as)); exit(1); //GGML_ABORT("fatal error"); } else if (e.counts.size() != (size_t)n_as) { @@ -330,10 +326,10 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts[ex]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { - e.activations[e_start + j] += x[j]; - e.values[e_start + j] += x[j] * x[j]; - if (!std::isfinite((float)e.values[e_start + j])) { - LOG_ERR("%f detected in %s\n", (float)e.values[e_start + j], wname.c_str()); + e.in_sum[e_start + j] += x[j]; + e.in_sum2[e_start + j] += x[j] * x[j]; + if (!std::isfinite((float)e.in_sum2[e_start + j])) { + LOG_ERR("%f detected in %s\n", (float)e.in_sum2[e_start + j], wname.c_str()); exit(1); } } @@ -355,13 +351,13 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * auto & e = m_stats[wname]; const int64_t n_mat = src1->ne[2] * src1->ne[3]; - if (e.values.empty()) { - e.activations.resize(src1->ne[0] * n_mat, 0); - e.values.resize(src1->ne[0] * n_mat, 0); + if (e.in_sum2.empty()) { + e.in_sum.resize(src1->ne[0] * n_mat, 0); + e.in_sum2.resize(src1->ne[0] * n_mat, 0); e.counts.resize(n_mat, 0); } - else if (e.values.size() != (size_t)(src1->ne[0] * n_mat)) { - LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.values.size(), (int)(src1->ne[0] * n_mat)); + else if (e.in_sum2.size() != (size_t)(src1->ne[0] * n_mat)) { + LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.in_sum2.size(), (int)(src1->ne[0] * n_mat)); exit(1); //GGML_ABORT("fatal error"); } else if (e.counts.size() != (size_t)n_mat) { @@ -378,10 +374,10 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * const float * x = (const float *) (data + row * src1->nb[1] + i2 * src1->nb[2] + i3 * src1->ne[3]); e.counts[mat_id]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { - e.activations[mat_start + j] += x[j]; - e.values[mat_start + j] += x[j] * x[j]; - if (!std::isfinite((float)e.values[j])) { - LOG_ERR("%f detected in %s\n", (float)e.values[j], wname.c_str()); + e.in_sum[mat_start + j] += x[j]; + e.in_sum2[mat_start + j] += x[j] * x[j]; + if (!std::isfinite((float)e.in_sum2[j])) { + LOG_ERR("%f detected in %s\n", (float)e.in_sum2[j], wname.c_str()); exit(1); } } @@ -470,14 +466,14 @@ void IMatrixCollector::save_imatrix_legacy(int32_t ncall) const { // ceiling division to avoid accidental zeros const int32_t ncall = (*std::max_element(stat.counts.begin(), stat.counts.end()) + (chunk_size - 1)) / chunk_size; out.write((const char *) &ncall, sizeof(ncall)); - const int32_t nval = stat.values.size(); + const int32_t nval = stat.in_sum2.size(); const int32_t nmat = stat.counts.size(); out.write((const char *) &nval, sizeof(nval)); if (nval > 0 && nmat > 0) { std::vector tmp(nval); for (int32_t i = 0; i < nval; i++) { float count = static_cast(stat.counts[i / (nval / nmat)]); - float value = stat.values[i]; + float value = stat.in_sum2[i]; if (count == 0.0f) { // store 1 for partial data value = 1.0f; @@ -552,8 +548,8 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { } to_store.push_back(kv.first); - data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); - data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.in_sum.size(), GGML_MEM_ALIGN); + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.in_sum2.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); } @@ -588,7 +584,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { for (const auto & name : to_store) { const auto & stat = m_stats.at(name); - const int32_t nval = (int32_t) stat.values.size(); + const int32_t nval = (int32_t) stat.in_sum2.size(); const int32_t nmat = (int32_t) stat.counts.size(); if (nval > 0 && nmat > 0) { struct ggml_tensor * in_sum2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nval / nmat, nmat); @@ -597,7 +593,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { ggml_format_name(counts, "%s.counts", name.c_str()); for (int32_t j = 0; j < nval; ++j) { - ((float *) in_sum2->data)[j] = (float) stat.values[j]; + ((float *) in_sum2->data)[j] = (float) stat.in_sum2[j]; } for (int32_t j = 0; j < nmat; ++j) { ((float *) counts->data)[j] = (float) stat.counts[j]; @@ -606,12 +602,12 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum2); gguf_add_tensor(ctx_gguf, counts); - if (!stat.activations.empty()) { - const int32_t nact = (int32_t) stat.activations.size(); + if (!stat.in_sum.empty()) { + const int32_t nact = (int32_t) stat.in_sum.size(); struct ggml_tensor * in_sum = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nact / nmat, nmat); - ggml_format_name(in_sum, "%s.in_sum", name.c_str()); // ToDo: consider a better name. 'in_act' maybe? + ggml_format_name(in_sum, "%s.in_sum", name.c_str()); for (int32_t j = 0; j < nval; ++j) { - ((float *) in_sum->data)[j] = (float) stat.activations[j]; + ((float *) in_sum->data)[j] = (float) stat.in_sum[j]; } gguf_add_tensor(ctx_gguf, in_sum); } @@ -664,8 +660,8 @@ bool IMatrixCollector::load_imatrix_legacy(const char * fname) { return false; } - if (e.values.empty()) { - e.values.resize(nval, 0.0f); + if (e.in_sum2.empty()) { + e.in_sum2.resize(nval, 0.0f); e.counts.resize(1, 0); } @@ -679,7 +675,7 @@ bool IMatrixCollector::load_imatrix_legacy(const char * fname) { // Recreate the state as expected by save_imatrix(), and correct for weighted sum. for (int i = 0; i < nval; i++) { - e.values[i] += tmp[i] * chunk_size; + e.in_sum2[i] += tmp[i] * chunk_size; } // The legacy format doesn't distinguish the counts for different experts for (size_t j = 0; j < e.counts.size(); ++j) { @@ -799,11 +795,11 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { auto & e = m_stats[name]; int64_t nval = ggml_nelements(in_sum2); - if (e.values.empty()) { - e.values.resize(nval, 0.0f); - e.activations.resize(nval, 0.0f); - } else if ((size_t) nval != e.values.size()) { - LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size()); + if (e.in_sum2.empty()) { + e.in_sum2.resize(nval, 0.0f); + e.in_sum.resize(nval, 0.0f); + } else if ((size_t) nval != e.in_sum2.size()) { + LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.in_sum2.size()); gguf_free(ctx_gguf); ggml_free(ctx); return false; @@ -824,7 +820,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { // Recreate the state as expected by save_imatrix() for (int64_t j = 0; j < nval; j++) { - e.values[j] += ((const float *) in_sum2->data)[j]; + e.in_sum2[j] += ((const float *) in_sum2->data)[j]; } for (int64_t j = 0; j < ncounts; j++) { e.counts[j] += std::lround(((const float *) counts->data)[j]); @@ -832,7 +828,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { // ToDo: fix blow up when GGUF does not have in_sum if (in_sum->data != nullptr) { for (int64_t j = 0; j < nval; j++) { - e.activations[j] += ((const float *) in_sum->data)[j]; + e.in_sum[j] += ((const float *) in_sum->data)[j]; } } } @@ -1134,7 +1130,7 @@ static bool show_statistics(const common_params & params) { ; process_tensor_name(a.tensor, layer, name_a); process_tensor_name(b.tensor, layer, name_b); - return name_a < name_b || (name_a == name_b && a.total_sqract > b.total_sqract); + return name_a < name_b || (name_a == name_b && a.sum_values > b.sum_values); } }; std::sort(ts.begin(), ts.end(), tensor_comparer()); @@ -1166,12 +1162,12 @@ static bool show_statistics(const common_params & params) { } LOG_INF("%5s\t%-20s\t%10.2f\t%8.4f\t%11.4f\t%6.2f\t%6.2f\t%8.2f%%\t%6d\t%10.4f\t%6.2f%%\t%10.2f%%\t%8.4f\n", - layer.c_str(), name.c_str(), tstat.total_sqract, tstat.min_sqract, tstat.max_sqract, tstat.mean_sqract, + layer.c_str(), name.c_str(), tstat.sum_values, tstat.min_values, tstat.max_values, tstat.mean_values, tstat.stddev, tstat.active * 100.0f, tstat.elements, tstat.entropy, - 100.0f * (tstat.entropy / std::log2(tstat.elements)), 100.0f * tstat.zd, tstat.cossim); + 100.0f * (tstat.entropy / std::log2(tstat.elements)), 100.0f * tstat.zd_score, tstat.cossim); - const float weighted_bias = tstat.elements * tstat.total_sqract; - const float weighted_zd = tstat.elements * tstat.zd; + const float weighted_bias = tstat.elements * tstat.sum_values; + const float weighted_zd = tstat.elements * tstat.zd_score; const float weighted_cossim = tstat.elements * tstat.cossim; if (ws.find(blk) != ws.end()) { From 78ddb475de96dbb20ba5f5bba597f98a30beb807 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 Aug 2025 16:31:21 +0100 Subject: [PATCH 003/123] Fix problem up when GGUF does not have in_sum --- tools/imatrix/imatrix.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index b92b1486f7b1..51d5a602a8ab 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -825,8 +825,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { for (int64_t j = 0; j < ncounts; j++) { e.counts[j] += std::lround(((const float *) counts->data)[j]); } - // ToDo: fix blow up when GGUF does not have in_sum - if (in_sum->data != nullptr) { + if (in_sum != nullptr) { for (int64_t j = 0; j < nval; j++) { e.in_sum[j] += ((const float *) in_sum->data)[j]; } From 9744a4a1c6e953457aa093ad3f07b66e9fdeed1d Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 Aug 2025 16:36:12 +0100 Subject: [PATCH 004/123] Determine calculation mode --- tools/imatrix/imatrix.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 51d5a602a8ab..398a5e85dbf2 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -127,18 +127,19 @@ static void process_tensor_name(const std::string & input, std::string & layer, } } -static void compute_statistics(std::vector & tstats, const std::string & name, const Stats & e) { +static int compute_tensor_statistics(std::vector & tstats, const std::string & name, const Stats & e) { if (e.in_sum2.size() % e.counts.size() != 0) { LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.in_sum2.size()); - return; + return -1;; } if (e.counts.empty()) { LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); - return; + return -1; } const int n_mat = e.counts.size(); const int row_size = e.in_sum2.size() / n_mat; + const int calc_mode = e.in_sum.empty() ? 2 : 1; std::vector activations; @@ -1104,13 +1105,15 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c static bool show_statistics(const common_params & params) { std::vector ts; + int tensor_calc_mode = 0; + if (params.in_files.empty() || params.in_files.size() > 1) { LOG_ERR("\nError: a single imatrix file is required to compute tensor statistics\n\n"); return false; } if (g_collector.load_imatrix(params.in_files[0].c_str())) { for (const auto & [name, stats] :g_collector.get_mstats()) { - compute_statistics(ts, name, stats); + tensor_calc_mode =compute_tensor_statistics(ts, name, stats); } } else { LOG_ERR("\nError: %s is not a valid imatrix file\n\n", params.in_files[0].c_str()); From cce514a392023048a67a5e0a1ef0197deb9e652b Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 Aug 2025 16:40:40 +0100 Subject: [PATCH 005/123] Compute entropy for activations --- tools/imatrix/imatrix.cpp | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 398a5e85dbf2..367412b6290f 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -173,10 +173,28 @@ static int compute_tensor_statistics(std::vector & tstats, co const float active_ratio = 1 - static_cast(inactive_count) / activations.size(); float entropy = 0; - if (sum > 0) { - for (const auto act : activations) { - if (const float p = act / sum; p > 0) { - entropy -= p * std::log2(p); + + if (calc_mode == 1) { + float div = 0.0; + std::vector weights(activations.size()); + for (size_t i = 0; i < activations.size(); ++i) { + const float w = activations[i] * activations[i]; + weights[i] = w; + div += w; + } + + if (div > 0.0) { + for (float w : weights) { + const float p = w / div; + if (p > 0.0) entropy -= p * std::log2(p); + } + } + } else { + if (sum > 0) { + for (const auto act : activations) { + if (const float p = act / sum; p > 0) { + entropy -= p * std::log2(p); + } } } } @@ -202,6 +220,8 @@ static int compute_tensor_statistics(std::vector & tstats, co ts.active = active_ratio; ts.entropy = entropy; ts.zd_score = static_cast(z_score) / ts.elements; + + return calc_mode; } static void compute_cossim(std::vector & tstats) { From b7fb362d8ebaa1c8db684f00c02e90a121f2517b Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 Aug 2025 16:43:49 +0100 Subject: [PATCH 006/123] Compute cosine similarity based on activations --- tools/imatrix/imatrix.cpp | 57 ++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 367412b6290f..a86442747e5b 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -224,27 +224,60 @@ static int compute_tensor_statistics(std::vector & tstats, co return calc_mode; } -static void compute_cossim(std::vector & tstats) { +static void compute_layer_statistics(std::vector & tstats) { static const std::regex pattern(R"(blk\.(\d+)\.)"); + + auto build_avg = [](const Stats & s) -> std::vector { + if (s.counts.empty()) return {}; + const size_t n_mat = s.counts.size(); + const size_t len = !s.in_sum.empty() ? s.in_sum.size() + : s.in_sum2.size(); + if (len == 0 || len % n_mat != 0) return {}; + const size_t row = len / n_mat; + std::vector v; + v.reserve(len); + if (!s.in_sum.empty()) { + for (size_t m = 0; m < n_mat; ++m) { + const float c = (float)s.counts[m]; + if (c <= 0) return {}; + const size_t off = m*row; + for (size_t j = 0; j < row; ++j) v.push_back(s.in_sum[off+j]/c); + } + } else { + for (size_t m = 0; m < n_mat; ++m) { + const float c = (float)s.counts[m]; + if (c <= 0) return {}; + const size_t off = m*row; + for (size_t j = 0; j < row; ++j) v.push_back(s.in_sum2[off+j]/c); + } + } + return v; + }; + // compute the cosine similarity between the same tensors in consecutive layers for (auto & ts : tstats) { + ts.cossim = 0; + if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { const int blk = std::stoi(match[1]); + if (blk <= 0) continue; std::string tname(ts.tensor); tname.replace(match.position(1), match.length(1), std::to_string(blk-1)); auto prev = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); - if (prev != tstats.end()) { - const float dot_product = std::inner_product(ts.stats.in_sum2.begin(), ts.stats.in_sum2.end(), - prev->stats.in_sum2.begin(), 0.0f); - const float magnitude = std::sqrt(std::inner_product(ts.stats.in_sum2.begin(), ts.stats.in_sum2.end(), - ts.stats.in_sum2.begin(), 0.0f)); - const float prev_magnitude = std::sqrt(std::inner_product(prev->stats.in_sum2.begin(), prev->stats.in_sum2.end(), - prev->stats.in_sum2.begin(), 0.0f)); - const float cos_sim = dot_product / (magnitude * prev_magnitude); - ts.cossim = cos_sim; + if (prev == tstats.end()) continue; + const auto curr_avg = build_avg(ts.stats); + const auto prev_avg = build_avg(prev->stats); + if (curr_avg.size() == prev_avg.size() && !curr_avg.empty()) { + float dot_prod = 0.0f, vec1 = 0.0f, vec2 = 0.0f; + for (size_t i = 0; i < curr_avg.size(); ++i) { + dot_prod += curr_avg[i]*prev_avg[i]; + vec1 += curr_avg[i]*curr_avg[i]; + vec2 += prev_avg[i]*prev_avg[i]; + } + if (vec1 > 0 && vec2 > 0) ts.cossim = dot_prod / (std::sqrt(vec1)*std::sqrt(vec2)); } - } else { - ts.cossim = 0; + } + } } } } From 9b841eb696c6ecb00203daa93c2dc3cd831aa9fc Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 Aug 2025 16:45:09 +0100 Subject: [PATCH 007/123] Compute l2 norm --- tools/imatrix/imatrix.cpp | 42 +++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index a86442747e5b..a4633f6e7a62 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1,8 +1,9 @@ +#include "../../src/llama-impl.h" #include "arg.h" #include "common.h" -#include "log.h" -#include "llama.h" #include "gguf.h" +#include "llama.h" +#include "log.h" #include #include @@ -10,14 +11,14 @@ #include #include #include -#include -#include -#include #include -#include #include -#include +#include #include +#include +#include +#include +#include #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data @@ -43,7 +44,6 @@ struct Stats { std::vector counts; }; -//ToDo: rename sqract variables to be more generic like 'values' struct tensor_statistics { std::string tensor; Stats stats; @@ -57,6 +57,7 @@ struct tensor_statistics { float entropy = 0.0f; float zd_score = 0.0f; float cossim = 0.0f; + float l2_norm = 0.0f; }; class IMatrixCollector { @@ -253,6 +254,7 @@ static void compute_layer_statistics(std::vector & tstats) { } return v; }; + // compute the cosine similarity between the same tensors in consecutive layers for (auto & ts : tstats) { ts.cossim = 0; @@ -278,6 +280,30 @@ static void compute_layer_statistics(std::vector & tstats) { } } } + + // compute the L2 norm between the same tensors in consecutive layers + for (auto & ts : tstats) { + ts.l2_norm = 0.0f; + if (ts.stats.in_sum.empty()) continue; + + if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { + const int blk = std::stoi(match[1]); + if (blk <= 0) continue; + std::string tname(ts.tensor); + tname.replace(match.position(1), match.length(1), std::to_string(blk - 1)); + auto prev = std::find_if(tstats.begin(), tstats.end(), + [tname](const tensor_statistics & t) { return t.tensor == tname; }); + if (prev == tstats.end()) continue; + const auto cur_avg = build_avg(ts.stats); + const auto prev_avg = build_avg(prev->stats); + if (cur_avg.empty() || prev_avg.empty() || cur_avg.size() != prev_avg.size()) continue; + + float dist = 0.0; + for (size_t i = 0; i < cur_avg.size(); ++i) { + const float act = cur_avg[i] - prev_avg[i]; + dist += act * act; + } + ts.l2_norm = std::sqrt(dist); } } } From ee2509f563786a5c3e77f59f199ed87df00a0235 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 Aug 2025 16:45:56 +0100 Subject: [PATCH 008/123] Adjust threshold --- tools/imatrix/imatrix.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index a4633f6e7a62..fd90fe208951 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -169,7 +169,7 @@ static int compute_tensor_statistics(std::vector & tstats, co const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); const float variance = (sqr_sum / activations.size()) - (mean * mean); const float std_deviation = std::sqrt(std::max(0.0f, variance)); - const float threshold = 1e-5f; + const float threshold = 1e-5f * std_deviation; const int inactive_count = std::count_if(activations.begin(), activations.end(), [threshold](const float v) { return fabsf(v) <= threshold; }); const float active_ratio = 1 - static_cast(inactive_count) / activations.size(); @@ -1199,7 +1199,7 @@ static bool show_statistics(const common_params & params) { return false; } if (!ts.empty()) { - compute_cossim(ts); + compute_layer_statistics(ts); } else { LOG_ERR("Error: cannot compute statistics for %s\n\n", params.in_files[0].c_str()); return false; From fc8f92596fb26dc6c0e7cd3e7c7598d2135ab548 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 Aug 2025 16:46:27 +0100 Subject: [PATCH 009/123] Update table display --- tools/imatrix/imatrix.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index fd90fe208951..ca26195d2c54 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1225,9 +1225,22 @@ static bool show_statistics(const common_params & params) { std::map ws; LOG_INF("\nComputing statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); - LOG_INF("\n%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", " Layer", " Tensor", " Σ(Act²)", - " Min", " Max", " μ", " σ", " % Active", "N", " Entropy", "E (norm)", "ZD", - " CosSim"); + LOG_INF( + "\n%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + " Layer", + " Tensor", + tensor_calc_mode == 1 ? " L₂ Norm" : " Σ(Act²)", + " Min", + " Max", + " μ", + " σ", + " % Active", + "N", + " Entropy", + "E (norm)", + "ZD", + " CosSim" + ); LOG_INF( "==============================================================================================================" "===========================================================\n"); From 4c01f51ae15de75ab13b288947ffd78727c888bd Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 00:51:12 +0100 Subject: [PATCH 010/123] Remove inactive --- tools/imatrix/imatrix.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ca26195d2c54..b9e538e93136 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -53,7 +53,6 @@ struct tensor_statistics { float min_values = 0.0f; int elements = 0; float stddev = 0.0f; - float active = 0.0f; float entropy = 0.0f; float zd_score = 0.0f; float cossim = 0.0f; @@ -169,11 +168,7 @@ static int compute_tensor_statistics(std::vector & tstats, co const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); const float variance = (sqr_sum / activations.size()) - (mean * mean); const float std_deviation = std::sqrt(std::max(0.0f, variance)); - const float threshold = 1e-5f * std_deviation; - const int inactive_count = std::count_if(activations.begin(), activations.end(), [threshold](const float v) { return fabsf(v) <= threshold; }); - const float active_ratio = 1 - static_cast(inactive_count) / activations.size(); - - float entropy = 0; + float entropy = 0; if (calc_mode == 1) { float div = 0.0; @@ -218,7 +213,6 @@ static int compute_tensor_statistics(std::vector & tstats, co ts.min_values = min; ts.elements = static_cast(activations.size()); ts.stddev = std_deviation; - ts.active = active_ratio; ts.entropy = entropy; ts.zd_score = static_cast(z_score) / ts.elements; From a32a2ecbed4b80b3ed1bbb22cbb96fa7bec6e973 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 00:51:33 +0100 Subject: [PATCH 011/123] Reformat report layout --- tools/imatrix/imatrix.cpp | 42 +++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index b9e538e93136..ba13b10f7ec6 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1219,25 +1219,23 @@ static bool show_statistics(const common_params & params) { std::map ws; LOG_INF("\nComputing statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); - LOG_INF( - "\n%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - " Layer", - " Tensor", - tensor_calc_mode == 1 ? " L₂ Norm" : " Σ(Act²)", - " Min", - " Max", - " μ", - " σ", - " % Active", + LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%12s\t%s\t%5s\t%10s\n", + "Layer", + "Tensor", + tensor_calc_mode == 1 ? "L₂ Norm" : "Σ(Act²)", + "Min", + "Max", + "μ", + "σ", "N", - " Entropy", + "Entropy", "E (norm)", "ZD", - " CosSim" + "CosSim" ); LOG_INF( "==============================================================================================================" - "===========================================================\n"); + "=============================================================\n"); for (const auto & tstat : ts) { std::string layer, name; process_tensor_name(tstat.tensor, layer, name); @@ -1249,10 +1247,20 @@ static bool show_statistics(const common_params & params) { blk = -1; // not a block layer } - LOG_INF("%5s\t%-20s\t%10.2f\t%8.4f\t%11.4f\t%6.2f\t%6.2f\t%8.2f%%\t%6d\t%10.4f\t%6.2f%%\t%10.2f%%\t%8.4f\n", - layer.c_str(), name.c_str(), tstat.sum_values, tstat.min_values, tstat.max_values, tstat.mean_values, - tstat.stddev, tstat.active * 100.0f, tstat.elements, tstat.entropy, - 100.0f * (tstat.entropy / std::log2(tstat.elements)), 100.0f * tstat.zd_score, tstat.cossim); + LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%12.4f\t%7.2f%%\t%6.2f%%\t%10.4f\n", + layer.c_str(), + name.c_str(), + tstat.sum_values, + tstat.min_values, + tstat.max_values, + tstat.mean_values, + tstat.stddev, + tstat.elements, + tstat.entropy, + 100.0f * (tstat.entropy / std::log2(tstat.elements)), + 100.0f * tstat.zd_score, + tstat.cossim + ); const float weighted_bias = tstat.elements * tstat.sum_values; const float weighted_zd = tstat.elements * tstat.zd_score; From 4d1325e1ebe1eabf8fcf900ffc62b67ee3ba9e02 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 10:28:23 +0100 Subject: [PATCH 012/123] Refactor variables --- tools/imatrix/imatrix.cpp | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ba13b10f7ec6..63b232f5cee3 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1211,10 +1211,10 @@ static bool show_statistics(const common_params & params) { std::sort(ts.begin(), ts.end(), tensor_comparer()); struct weighted_stats { - float weighted_bias = 0.0f; - float weighted_zd = 0.0f; - float weighted_cossim = 0.0f; - int total_elements = 0; + float w_sum = 0.0f; + float w_zd = 0.0f; + float w_cossim = 0.0f; + int n = 0; }; std::map ws; @@ -1262,22 +1262,22 @@ static bool show_statistics(const common_params & params) { tstat.cossim ); - const float weighted_bias = tstat.elements * tstat.sum_values; - const float weighted_zd = tstat.elements * tstat.zd_score; - const float weighted_cossim = tstat.elements * tstat.cossim; + const float w_sum = tstat.elements * tstat.sum_values; + const float w_zd = tstat.elements * tstat.zd_score; + const float w_cossim = tstat.elements * tstat.cossim; if (ws.find(blk) != ws.end()) { - ws[blk].weighted_bias += weighted_bias; - ws[blk].weighted_zd += weighted_zd; - ws[blk].weighted_cossim += weighted_cossim; - ws[blk].total_elements += tstat.elements; + ws[blk].w_sum += w_sum; + ws[blk].w_zd += w_zd; + ws[blk].w_cossim += w_cossim; + ws[blk].n += tstat.elements; } else { weighted_stats temp_ws; - temp_ws.weighted_bias = weighted_bias; - temp_ws.weighted_zd = weighted_zd; - temp_ws.weighted_cossim = weighted_cossim; - temp_ws.total_elements = tstat.elements; - ws[blk] = temp_ws; + temp_ws.w_sum = w_sum; + temp_ws.w_zd = w_zd; + temp_ws.w_cossim = w_cossim; + temp_ws.n = tstat.elements; + ws[blk] = temp_ws; } } @@ -1289,14 +1289,14 @@ static bool show_statistics(const common_params & params) { const auto & layer = first; const auto & stats = second; - if (stats.total_elements == 0) { + if (stats.n == 0) { continue; } if (layer >= 0) { - const float bias = stats.weighted_bias / stats.total_elements; - const float zd = stats.weighted_zd / stats.total_elements; - const float cossim = stats.weighted_cossim / stats.total_elements; + const float w_sum = stats.w_sum / stats.n; + const float w_zd = stats.w_zd / stats.n; + const float w_cossim = stats.w_cossim / stats.n; LOG_INF("%5d\t%14.2f\t%10.4f%%\t%6.4f\n", layer, bias, 100.0f * zd, cossim); } From 5324558132ff076774921d14d8979b2f6a92ebdf Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 10:28:47 +0100 Subject: [PATCH 013/123] Update table layout --- tools/imatrix/imatrix.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 63b232f5cee3..4a6d33837d42 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1231,8 +1231,7 @@ static bool show_statistics(const common_params & params) { "Entropy", "E (norm)", "ZD", - "CosSim" - ); + "CosSim"); LOG_INF( "==============================================================================================================" "=============================================================\n"); @@ -1259,8 +1258,7 @@ static bool show_statistics(const common_params & params) { tstat.entropy, 100.0f * (tstat.entropy / std::log2(tstat.elements)), 100.0f * tstat.zd_score, - tstat.cossim - ); + tstat.cossim); const float w_sum = tstat.elements * tstat.sum_values; const float w_zd = tstat.elements * tstat.zd_score; @@ -1283,8 +1281,12 @@ static bool show_statistics(const common_params & params) { const int layers = std::count_if(ws.begin(), ws.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing weighted average statistics per layer (%d layers)\n", layers); - LOG_INF("\n%s\t%s\t%s\t%s\n", " Layer", " μΣ(Act²)", " μZD", "μCosSim"); - LOG_INF("================================================\n"); + LOG_INF("\n%6s\t%16s\t%7s\t%11s\n", + "Layer", + tensor_calc_mode == 1 ? "μL₂ Norm" : "μΣ(Act²)", + "μZD", + "μCosSim"); + LOG_INF("============================================\n"); for (const auto & [first, second] : ws) { const auto & layer = first; const auto & stats = second; @@ -1298,7 +1300,11 @@ static bool show_statistics(const common_params & params) { const float w_zd = stats.w_zd / stats.n; const float w_cossim = stats.w_cossim / stats.n; - LOG_INF("%5d\t%14.2f\t%10.4f%%\t%6.4f\n", layer, bias, 100.0f * zd, cossim); + LOG_INF("%5d\t%11.2f\t%6.2f%%\t%10.4f\n", + layer, + w_sum, + 100.0f * w_zd, + w_cossim); } } LOG_INF("\n"); From fce05aac9ea8d31e56de87d4108f94ebe339f5b2 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 13:03:21 +0100 Subject: [PATCH 014/123] Refactor lambda into compute_tensor_averages() function --- tools/imatrix/imatrix.cpp | 68 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 4a6d33837d42..88807d3721bb 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -127,6 +127,39 @@ static void process_tensor_name(const std::string & input, std::string & layer, } } +static std::vector compute_tensor_averages(const Stats & tstats) { + if (tstats.counts.empty()) return {}; + const size_t n_mat = tstats.counts.size(); + const size_t len = !tstats.in_sum.empty() ? tstats.in_sum.size() : tstats.in_sum2.size(); + + if (len == 0 || len % n_mat != 0) return {}; + const size_t row = len / n_mat; + std::vector vec; + vec.reserve(len); + + if (!tstats.in_sum.empty()) { + for (size_t m = 0; m < n_mat; ++m) { + const float c = (float)tstats.counts[m]; + if (c <= 0) return {}; + const size_t off = m * row; + for (size_t j = 0; j < row; ++j) { + vec.push_back(tstats.in_sum[off + j] / c); + } + } + } else { + for (size_t m = 0; m < n_mat; ++m) { + const float c = (float)tstats.counts[m]; + if (c <= 0) return {}; + const size_t off = m * row; + for (size_t j = 0; j < row; ++j) { + vec.push_back(tstats.in_sum2[off + j] / c); + } + } + } + + return vec; +} + static int compute_tensor_statistics(std::vector & tstats, const std::string & name, const Stats & e) { if (e.in_sum2.size() % e.counts.size() != 0) { LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.in_sum2.size()); @@ -222,33 +255,6 @@ static int compute_tensor_statistics(std::vector & tstats, co static void compute_layer_statistics(std::vector & tstats) { static const std::regex pattern(R"(blk\.(\d+)\.)"); - auto build_avg = [](const Stats & s) -> std::vector { - if (s.counts.empty()) return {}; - const size_t n_mat = s.counts.size(); - const size_t len = !s.in_sum.empty() ? s.in_sum.size() - : s.in_sum2.size(); - if (len == 0 || len % n_mat != 0) return {}; - const size_t row = len / n_mat; - std::vector v; - v.reserve(len); - if (!s.in_sum.empty()) { - for (size_t m = 0; m < n_mat; ++m) { - const float c = (float)s.counts[m]; - if (c <= 0) return {}; - const size_t off = m*row; - for (size_t j = 0; j < row; ++j) v.push_back(s.in_sum[off+j]/c); - } - } else { - for (size_t m = 0; m < n_mat; ++m) { - const float c = (float)s.counts[m]; - if (c <= 0) return {}; - const size_t off = m*row; - for (size_t j = 0; j < row; ++j) v.push_back(s.in_sum2[off+j]/c); - } - } - return v; - }; - // compute the cosine similarity between the same tensors in consecutive layers for (auto & ts : tstats) { ts.cossim = 0; @@ -261,8 +267,8 @@ static void compute_layer_statistics(std::vector & tstats) { auto prev = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); if (prev == tstats.end()) continue; - const auto curr_avg = build_avg(ts.stats); - const auto prev_avg = build_avg(prev->stats); + const auto curr_avg = compute_tensor_averages(ts.stats); + const auto prev_avg = compute_tensor_averages(prev->stats); if (curr_avg.size() == prev_avg.size() && !curr_avg.empty()) { float dot_prod = 0.0f, vec1 = 0.0f, vec2 = 0.0f; for (size_t i = 0; i < curr_avg.size(); ++i) { @@ -288,8 +294,8 @@ static void compute_layer_statistics(std::vector & tstats) { auto prev = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); if (prev == tstats.end()) continue; - const auto cur_avg = build_avg(ts.stats); - const auto prev_avg = build_avg(prev->stats); + const auto cur_avg = compute_tensor_averages(ts.stats); + const auto prev_avg = compute_tensor_averages(prev->stats); if (cur_avg.empty() || prev_avg.empty() || cur_avg.size() != prev_avg.size()) continue; float dist = 0.0; From be60469f25a2268275b514a6ecf3f8b2d2c74aae Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 15:10:17 +0100 Subject: [PATCH 015/123] Refactor function names --- tools/imatrix/imatrix.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 88807d3721bb..d5cea686aa68 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -160,7 +160,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { return vec; } -static int compute_tensor_statistics(std::vector & tstats, const std::string & name, const Stats & e) { +static int compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { if (e.in_sum2.size() % e.counts.size() != 0) { LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.in_sum2.size()); return -1;; @@ -252,7 +252,7 @@ static int compute_tensor_statistics(std::vector & tstats, co return calc_mode; } -static void compute_layer_statistics(std::vector & tstats) { +static void compute_tensor_statistics(std::vector & tstats) { static const std::regex pattern(R"(blk\.(\d+)\.)"); // compute the cosine similarity between the same tensors in consecutive layers @@ -1192,14 +1192,14 @@ static bool show_statistics(const common_params & params) { } if (g_collector.load_imatrix(params.in_files[0].c_str())) { for (const auto & [name, stats] :g_collector.get_mstats()) { - tensor_calc_mode =compute_tensor_statistics(ts, name, stats); + tensor_calc_mode =compute_vector_statistics(ts, name, stats); } } else { LOG_ERR("\nError: %s is not a valid imatrix file\n\n", params.in_files[0].c_str()); return false; } if (!ts.empty()) { - compute_layer_statistics(ts); + compute_tensor_statistics(ts); } else { LOG_ERR("Error: cannot compute statistics for %s\n\n", params.in_files[0].c_str()); return false; From a6155a81254506130cd19e88eb8e19c49f2e8f41 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 16:35:03 +0100 Subject: [PATCH 016/123] Add compute_layer_statistics() function --- tools/imatrix/imatrix.cpp | 64 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index d5cea686aa68..d4a6ebddd0ef 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -272,11 +272,11 @@ static void compute_tensor_statistics(std::vector & tstats) { if (curr_avg.size() == prev_avg.size() && !curr_avg.empty()) { float dot_prod = 0.0f, vec1 = 0.0f, vec2 = 0.0f; for (size_t i = 0; i < curr_avg.size(); ++i) { - dot_prod += curr_avg[i]*prev_avg[i]; - vec1 += curr_avg[i]*curr_avg[i]; - vec2 += prev_avg[i]*prev_avg[i]; + dot_prod += curr_avg[i] * prev_avg[i]; + vec1 += curr_avg[i] * curr_avg[i]; + vec2 += prev_avg[i] * prev_avg[i]; } - if (vec1 > 0 && vec2 > 0) ts.cossim = dot_prod / (std::sqrt(vec1)*std::sqrt(vec2)); + if (vec1 > 0 && vec2 > 0) ts.cossim = dot_prod / (std::sqrt(vec1) * std::sqrt(vec2)); } } } @@ -308,6 +308,62 @@ static void compute_tensor_statistics(std::vector & tstats) { } } +static void compute_layer_statistics(const std::vector & tstats, + std::map & layer_cossim, + const std::unordered_map & stats_map) { + struct layer_aggregation { + std::vector curr_avg; + std::vector prev_avg; + }; + + static const std::regex pattern(R"(blk\.(\d+)\.)"); + + // index tensor stats by name for quick lookup + std::unordered_map tidx; + tidx.reserve(tstats.size()); + for (const auto & ts : tstats) tidx[ts.tensor] = &ts; + + // concatenate per-layer + std::map taggr; // ordered by layer + for (const auto & ts : tstats) { + std::smatch match; + if (!std::regex_search(ts.tensor, match, pattern)) continue; + const int blk = std::stoi(match[1]); + if (blk <= 0) continue; + + std::string prev_lyr(ts.tensor); + prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk-1)); + + if (auto it_prev = tidx.find(prev_lyr); it_prev == tidx.end()) continue; + + // use stored Stats to rebuild averages + const auto curr_avg = compute_tensor_averages(stats_map.at(ts.tensor)); + const auto prev_avg = compute_tensor_averages(stats_map.at(prev_lyr)); + if (curr_avg.empty() || prev_avg.empty() || curr_avg.size() != prev_avg.size()) continue; + + auto & [curr, prev] = taggr[blk]; + curr.insert(curr.end(), curr_avg.begin(), curr_avg.end()); + prev.insert(prev.end(), prev_avg.begin(), prev_avg.end()); + } + + // compute cosine per layer + for (auto & kv : taggr) { + const auto & curr = kv.second.curr_avg; + const auto & prev = kv.second.prev_avg; + if (curr.size() != prev.size() || curr.empty()) continue; + float dot_prod = 0.0, lyr1 = 0.0, lyr2 = 0.0; + for (size_t i = 0; i < curr.size(); ++i) { + const double a = curr[i], b = prev[i]; + dot_prod += a*b; + lyr1 += a*a; + lyr2 += b*b; + } + float cossim = 0.0f; + if (lyr1 > 0.0 && lyr2 > 0.0) cossim = dot_prod / (std::sqrt(lyr1) * std::sqrt(lyr2)); + layer_cossim[kv.first] = cossim; + } +} + bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) { GGML_UNUSED(user_data); From 2117c4e54bb28b0774287b8b52189b29a2387249 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 16:38:02 +0100 Subject: [PATCH 017/123] Update aggregated statistic report layout --- tools/imatrix/imatrix.cpp | 46 ++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index d4a6ebddd0ef..9a20758340c9 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1324,50 +1324,42 @@ static bool show_statistics(const common_params & params) { const float w_sum = tstat.elements * tstat.sum_values; const float w_zd = tstat.elements * tstat.zd_score; - const float w_cossim = tstat.elements * tstat.cossim; if (ws.find(blk) != ws.end()) { ws[blk].w_sum += w_sum; ws[blk].w_zd += w_zd; - ws[blk].w_cossim += w_cossim; ws[blk].n += tstat.elements; } else { weighted_stats temp_ws; temp_ws.w_sum = w_sum; temp_ws.w_zd = w_zd; - temp_ws.w_cossim = w_cossim; temp_ws.n = tstat.elements; ws[blk] = temp_ws; } } - const int layers = std::count_if(ws.begin(), ws.end(), [](const auto & kv) { return kv.first >= 0; }); - LOG_INF("\nComputing weighted average statistics per layer (%d layers)\n", layers); + std::map layer_cossim; + compute_layer_statistics(ts, layer_cossim, g_collector.get_mstats()); + + const auto layers = std::count_if(ws.begin(), ws.end(), [](const auto & kv) { return kv.first >= 0; }); + LOG_INF("\nComputing aggregated statistics per layer (%ld layers)\n", layers); LOG_INF("\n%6s\t%16s\t%7s\t%11s\n", "Layer", - tensor_calc_mode == 1 ? "μL₂ Norm" : "μΣ(Act²)", - "μZD", - "μCosSim"); + tensor_calc_mode == 1 ? "L₂ Norm" : "Σ(Act²)", + "ZD", + "CosSim"); LOG_INF("============================================\n"); - for (const auto & [first, second] : ws) { - const auto & layer = first; - const auto & stats = second; - - if (stats.n == 0) { - continue; - } - - if (layer >= 0) { - const float w_sum = stats.w_sum / stats.n; - const float w_zd = stats.w_zd / stats.n; - const float w_cossim = stats.w_cossim / stats.n; - - LOG_INF("%5d\t%11.2f\t%6.2f%%\t%10.4f\n", - layer, - w_sum, - 100.0f * w_zd, - w_cossim); - } + for (const auto & [layer, stats] : ws) { + if (layer < 0 || stats.n == 0) continue; + const float w_sum = stats.w_sum / stats.n; + const float w_zd = stats.w_zd / stats.n; + const auto lcs = layer_cossim.find(layer); + const float cossim = (lcs != layer_cossim.end()) ? lcs->second : 0.0f; + LOG_INF("%5d\t%11.2f\t%6.2f%%\t%10.4f\n", + layer, + w_sum, + 100.0f * w_zd, + cossim); } LOG_INF("\n"); From 90cb1be99d0aaa1fb37f94e0e57dd1b7c642306c Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 16:57:27 +0100 Subject: [PATCH 018/123] Minor cosmetic changes --- tools/imatrix/imatrix.cpp | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 9a20758340c9..ff189e037979 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -311,52 +311,43 @@ static void compute_tensor_statistics(std::vector & tstats) { static void compute_layer_statistics(const std::vector & tstats, std::map & layer_cossim, const std::unordered_map & stats_map) { - struct layer_aggregation { + struct layer_aggregation { std::vector curr_avg; std::vector prev_avg; }; - static const std::regex pattern(R"(blk\.(\d+)\.)"); - - // index tensor stats by name for quick lookup std::unordered_map tidx; tidx.reserve(tstats.size()); for (const auto & ts : tstats) tidx[ts.tensor] = &ts; + std::map taggr; - // concatenate per-layer - std::map taggr; // ordered by layer for (const auto & ts : tstats) { std::smatch match; if (!std::regex_search(ts.tensor, match, pattern)) continue; const int blk = std::stoi(match[1]); if (blk <= 0) continue; - std::string prev_lyr(ts.tensor); prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk-1)); - if (auto it_prev = tidx.find(prev_lyr); it_prev == tidx.end()) continue; - - // use stored Stats to rebuild averages const auto curr_avg = compute_tensor_averages(stats_map.at(ts.tensor)); const auto prev_avg = compute_tensor_averages(stats_map.at(prev_lyr)); if (curr_avg.empty() || prev_avg.empty() || curr_avg.size() != prev_avg.size()) continue; - auto & [curr, prev] = taggr[blk]; curr.insert(curr.end(), curr_avg.begin(), curr_avg.end()); prev.insert(prev.end(), prev_avg.begin(), prev_avg.end()); } - // compute cosine per layer + // compute the cosine similarity between consecutive layers for (auto & kv : taggr) { const auto & curr = kv.second.curr_avg; const auto & prev = kv.second.prev_avg; if (curr.size() != prev.size() || curr.empty()) continue; float dot_prod = 0.0, lyr1 = 0.0, lyr2 = 0.0; for (size_t i = 0; i < curr.size(); ++i) { - const double a = curr[i], b = prev[i]; - dot_prod += a*b; - lyr1 += a*a; - lyr2 += b*b; + float crr = curr[i], prv = prev[i]; + dot_prod += crr * prv; + lyr1 += crr * crr; + lyr2 += prv * prv; } float cossim = 0.0f; if (lyr1 > 0.0 && lyr2 > 0.0) cossim = dot_prod / (std::sqrt(lyr1) * std::sqrt(lyr2)); From f1c2a4ca3f9f23386892adc78f8c7a296c3e92ee Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 Aug 2025 17:14:46 +0100 Subject: [PATCH 019/123] Fix printing l2 norm when calc_mode = 1 --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ff189e037979..6a07fdd35451 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1302,7 +1302,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%12.4f\t%7.2f%%\t%6.2f%%\t%10.4f\n", layer.c_str(), name.c_str(), - tstat.sum_values, + tensor_calc_mode == 1 ? tstat.l2_norm : tstat.sum_values, tstat.min_values, tstat.max_values, tstat.mean_values, From c39c4e2a331c3386f019885865698b826dcbce00 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 4 Aug 2025 22:15:50 +0100 Subject: [PATCH 020/123] Refactor variable name --- tools/imatrix/imatrix.cpp | 70 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6a07fdd35451..a28701944d01 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -40,7 +40,7 @@ static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size"; struct Stats { std::vector in_sum; - std::vector in_sum2; + std::vector values; std::vector counts; }; @@ -130,7 +130,7 @@ static void process_tensor_name(const std::string & input, std::string & layer, static std::vector compute_tensor_averages(const Stats & tstats) { if (tstats.counts.empty()) return {}; const size_t n_mat = tstats.counts.size(); - const size_t len = !tstats.in_sum.empty() ? tstats.in_sum.size() : tstats.in_sum2.size(); + const size_t len = !tstats.in_sum.empty() ? tstats.in_sum.size() : tstats.values.size(); if (len == 0 || len % n_mat != 0) return {}; const size_t row = len / n_mat; @@ -152,7 +152,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { if (c <= 0) return {}; const size_t off = m * row; for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.in_sum2[off + j] / c); + vec.push_back(tstats.values[off + j] / c); } } } @@ -161,8 +161,8 @@ static std::vector compute_tensor_averages(const Stats & tstats) { } static int compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { - if (e.in_sum2.size() % e.counts.size() != 0) { - LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.in_sum2.size()); + if (e.values.size() % e.counts.size() != 0) { + LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.values.size()); return -1;; } if (e.counts.empty()) { @@ -171,17 +171,17 @@ static int compute_vector_statistics(std::vector & tstats, co } const int n_mat = e.counts.size(); - const int row_size = e.in_sum2.size() / n_mat; + const int row_size = e.values.size() / n_mat; const int calc_mode = e.in_sum.empty() ? 2 : 1; std::vector activations; if (e.in_sum.empty()) { - activations.reserve(e.in_sum2.size()); + activations.reserve(e.values.size()); for (int i = 0; i < n_mat; ++i) { for (int j = 0; j < row_size; ++j) { - activations.push_back(e.in_sum2[i*row_size + j] / e.counts[i]); + activations.push_back(e.values[i*row_size + j] / e.counts[i]); } } } else { @@ -420,13 +420,13 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * // broadcast, when loading an old imatrix e.counts.resize(n_as, e.counts[0]); } - if (e.in_sum2.empty()) { + if (e.values.empty()) { e.in_sum.resize(src1->ne[0]*n_as, 0); - e.in_sum2.resize(src1->ne[0]*n_as, 0); + e.values.resize(src1->ne[0]*n_as, 0); e.counts.resize(n_as, 0); } - else if (e.in_sum2.size() != (size_t)src1->ne[0]*n_as) { - LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.in_sum2.size(), (int)(src1->ne[0]*n_as)); + else if (e.values.size() != (size_t)src1->ne[0]*n_as) { + LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.values.size(), (int)(src1->ne[0]*n_as)); exit(1); //GGML_ABORT("fatal error"); } else if (e.counts.size() != (size_t)n_as) { @@ -454,9 +454,9 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * for (int64_t j = 0; j < src1->ne[0]; ++j) { e.in_sum[e_start + j] += x[j]; - e.in_sum2[e_start + j] += x[j] * x[j]; - if (!std::isfinite((float)e.in_sum2[e_start + j])) { - LOG_ERR("%f detected in %s\n", (float)e.in_sum2[e_start + j], wname.c_str()); + e.values[e_start + j] += x[j] * x[j]; + if (!std::isfinite((float)e.values[e_start + j])) { + LOG_ERR("%f detected in %s\n", (float)e.values[e_start + j], wname.c_str()); exit(1); } } @@ -478,13 +478,13 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * auto & e = m_stats[wname]; const int64_t n_mat = src1->ne[2] * src1->ne[3]; - if (e.in_sum2.empty()) { + if (e.values.empty()) { e.in_sum.resize(src1->ne[0] * n_mat, 0); - e.in_sum2.resize(src1->ne[0] * n_mat, 0); + e.values.resize(src1->ne[0] * n_mat, 0); e.counts.resize(n_mat, 0); } - else if (e.in_sum2.size() != (size_t)(src1->ne[0] * n_mat)) { - LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.in_sum2.size(), (int)(src1->ne[0] * n_mat)); + else if (e.values.size() != (size_t)(src1->ne[0] * n_mat)) { + LOG_ERR("%s: inconsistent size for %s (%d vs %d)\n", __func__, wname.c_str(), (int)e.values.size(), (int)(src1->ne[0] * n_mat)); exit(1); //GGML_ABORT("fatal error"); } else if (e.counts.size() != (size_t)n_mat) { @@ -502,9 +502,9 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts[mat_id]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { e.in_sum[mat_start + j] += x[j]; - e.in_sum2[mat_start + j] += x[j] * x[j]; - if (!std::isfinite((float)e.in_sum2[j])) { - LOG_ERR("%f detected in %s\n", (float)e.in_sum2[j], wname.c_str()); + e.values[mat_start + j] += x[j] * x[j]; + if (!std::isfinite((float)e.values[j])) { + LOG_ERR("%f detected in %s\n", (float)e.values[j], wname.c_str()); exit(1); } } @@ -593,14 +593,14 @@ void IMatrixCollector::save_imatrix_legacy(int32_t ncall) const { // ceiling division to avoid accidental zeros const int32_t ncall = (*std::max_element(stat.counts.begin(), stat.counts.end()) + (chunk_size - 1)) / chunk_size; out.write((const char *) &ncall, sizeof(ncall)); - const int32_t nval = stat.in_sum2.size(); + const int32_t nval = stat.values.size(); const int32_t nmat = stat.counts.size(); out.write((const char *) &nval, sizeof(nval)); if (nval > 0 && nmat > 0) { std::vector tmp(nval); for (int32_t i = 0; i < nval; i++) { float count = static_cast(stat.counts[i / (nval / nmat)]); - float value = stat.in_sum2[i]; + float value = stat.values[i]; if (count == 0.0f) { // store 1 for partial data value = 1.0f; @@ -676,7 +676,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { to_store.push_back(kv.first); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.in_sum.size(), GGML_MEM_ALIGN); - data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.in_sum2.size(), GGML_MEM_ALIGN); + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); } @@ -711,7 +711,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { for (const auto & name : to_store) { const auto & stat = m_stats.at(name); - const int32_t nval = (int32_t) stat.in_sum2.size(); + const int32_t nval = (int32_t) stat.values.size(); const int32_t nmat = (int32_t) stat.counts.size(); if (nval > 0 && nmat > 0) { struct ggml_tensor * in_sum2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nval / nmat, nmat); @@ -720,7 +720,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { ggml_format_name(counts, "%s.counts", name.c_str()); for (int32_t j = 0; j < nval; ++j) { - ((float *) in_sum2->data)[j] = (float) stat.in_sum2[j]; + ((float *) in_sum2->data)[j] = (float) stat.values[j]; } for (int32_t j = 0; j < nmat; ++j) { ((float *) counts->data)[j] = (float) stat.counts[j]; @@ -787,8 +787,8 @@ bool IMatrixCollector::load_imatrix_legacy(const char * fname) { return false; } - if (e.in_sum2.empty()) { - e.in_sum2.resize(nval, 0.0f); + if (e.values.empty()) { + e.values.resize(nval, 0.0f); e.counts.resize(1, 0); } @@ -802,7 +802,7 @@ bool IMatrixCollector::load_imatrix_legacy(const char * fname) { // Recreate the state as expected by save_imatrix(), and correct for weighted sum. for (int i = 0; i < nval; i++) { - e.in_sum2[i] += tmp[i] * chunk_size; + e.values[i] += tmp[i] * chunk_size; } // The legacy format doesn't distinguish the counts for different experts for (size_t j = 0; j < e.counts.size(); ++j) { @@ -922,11 +922,11 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { auto & e = m_stats[name]; int64_t nval = ggml_nelements(in_sum2); - if (e.in_sum2.empty()) { - e.in_sum2.resize(nval, 0.0f); + if (e.values.empty()) { + e.values.resize(nval, 0.0f); e.in_sum.resize(nval, 0.0f); - } else if ((size_t) nval != e.in_sum2.size()) { - LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.in_sum2.size()); + } else if ((size_t) nval != e.values.size()) { + LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size()); gguf_free(ctx_gguf); ggml_free(ctx); return false; @@ -947,7 +947,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { // Recreate the state as expected by save_imatrix() for (int64_t j = 0; j < nval; j++) { - e.in_sum2[j] += ((const float *) in_sum2->data)[j]; + e.values[j] += ((const float *) in_sum2->data)[j]; } for (int64_t j = 0; j < ncounts; j++) { e.counts[j] += std::lround(((const float *) counts->data)[j]); From 5e40cf4f1ca6af6187e5b049990b4208296c9ba6 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 5 Aug 2025 00:18:53 +0100 Subject: [PATCH 021/123] Do not resize if in_sum is null --- tools/imatrix/imatrix.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 651cb658d7e7..b772f2184d38 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -938,7 +938,9 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { int64_t nval = ggml_nelements(in_sum2); if (e.values.empty()) { e.values.resize(nval, 0.0f); - e.in_sum.resize(nval, 0.0f); + if (in_sum != nullptr) { + e.in_sum.resize(nval, 0.0f); + } } else if ((size_t) nval != e.values.size()) { LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size()); gguf_free(ctx_gguf); From b37393423d426d962f1aecf70c6fdc26c0bcb52b Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 5 Aug 2025 08:54:57 +0100 Subject: [PATCH 022/123] Compute aggregated (per layer) l2 norm --- tools/imatrix/imatrix.cpp | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index b772f2184d38..f22b67a309dc 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -255,7 +255,7 @@ static int compute_vector_statistics(std::vector & tstats, co static void compute_tensor_statistics(std::vector & tstats) { static const std::regex pattern(R"(blk\.(\d+)\.)"); - // compute the cosine similarity between the same tensors in consecutive layers + // compute the Cosine Similarity between the same tensors in consecutive layers for (auto & ts : tstats) { ts.cossim = 0; @@ -281,7 +281,7 @@ static void compute_tensor_statistics(std::vector & tstats) { } } - // compute the L2 norm between the same tensors in consecutive layers + // compute the L2 Norm (Euclidian Distance) between the same tensors in consecutive layers for (auto & ts : tstats) { ts.l2_norm = 0.0f; if (ts.stats.in_sum.empty()) continue; @@ -310,6 +310,7 @@ static void compute_tensor_statistics(std::vector & tstats) { static void compute_layer_statistics(const std::vector & tstats, std::map & layer_cossim, + std::map & layer_l2_norm, const std::unordered_map & stats_map) { struct layer_aggregation { std::vector curr_avg; @@ -337,22 +338,33 @@ static void compute_layer_statistics(const std::vector & tsta prev.insert(prev.end(), prev_avg.begin(), prev_avg.end()); } - // compute the cosine similarity between consecutive layers + // compute the aggregated Cosine Similarity between consecutive layers for (auto & kv : taggr) { const auto & curr = kv.second.curr_avg; const auto & prev = kv.second.prev_avg; if (curr.size() != prev.size() || curr.empty()) continue; float dot_prod = 0.0, lyr1 = 0.0, lyr2 = 0.0; for (size_t i = 0; i < curr.size(); ++i) { - float crr = curr[i], prv = prev[i]; - dot_prod += crr * prv; - lyr1 += crr * crr; - lyr2 += prv * prv; + dot_prod += curr[i] * prev[i]; + lyr1 += curr[i] * curr[i]; + lyr2 += prev[i] * prev[i]; } float cossim = 0.0f; if (lyr1 > 0.0 && lyr2 > 0.0) cossim = dot_prod / (std::sqrt(lyr1) * std::sqrt(lyr2)); layer_cossim[kv.first] = cossim; } + + // compute the aggregated L2 Norm (Euclidian Distance) between consecutive layers + for (auto & kv : taggr) { + const auto & curr = kv.second.curr_avg; + const auto & prev = kv.second.prev_avg; + if (curr.size() != prev.size() || curr.empty()) continue; + float dist = 0.0f; + for (size_t i = 0; i < curr.size(); ++i) { + dist += (curr[i] - prev[i]) * (curr[i] - prev[i]); + } + layer_l2_norm[kv.first] = std::sqrt(dist); + } } bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) { @@ -1346,7 +1358,8 @@ static bool show_statistics(const common_params & params) { } std::map layer_cossim; - compute_layer_statistics(ts, layer_cossim, g_collector.get_mstats()); + std::map layer_l2_norm; + compute_layer_statistics(ts, layer_cossim, layer_l2_norm, g_collector.get_mstats()); const auto layers = std::count_if(ws.begin(), ws.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing aggregated statistics per layer (%ld layers)\n", layers); @@ -1362,9 +1375,11 @@ static bool show_statistics(const common_params & params) { const float w_zd = stats.w_zd / stats.n; const auto lcs = layer_cossim.find(layer); const float cossim = (lcs != layer_cossim.end()) ? lcs->second : 0.0f; + const auto ll2n = layer_l2_norm.find(layer); + const float l2_norm = (ll2n != layer_l2_norm.end()) ? ll2n->second : 0.0f; LOG_INF("%5d\t%11.2f\t%6.2f%%\t%10.4f\n", layer, - w_sum, + tensor_calc_mode == 1 ? l2_norm: w_sum, 100.0f * w_zd, cossim); } From 906548a00a25b927f8ed8ac5b4b58b4f69db9e37 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 5 Aug 2025 12:06:19 +0100 Subject: [PATCH 023/123] Update aggregated sum of squared activations per layer --- tools/imatrix/imatrix.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f22b67a309dc..6d40984a0180 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1294,7 +1294,6 @@ static bool show_statistics(const common_params & params) { struct weighted_stats { float w_sum = 0.0f; float w_zd = 0.0f; - float w_cossim = 0.0f; int n = 0; }; std::map ws; @@ -1341,16 +1340,15 @@ static bool show_statistics(const common_params & params) { 100.0f * tstat.zd_score, tstat.cossim); - const float w_sum = tstat.elements * tstat.sum_values; const float w_zd = tstat.elements * tstat.zd_score; if (ws.find(blk) != ws.end()) { - ws[blk].w_sum += w_sum; + ws[blk].w_sum += tstat.sum_values; ws[blk].w_zd += w_zd; ws[blk].n += tstat.elements; } else { weighted_stats temp_ws; - temp_ws.w_sum = w_sum; + temp_ws.w_sum = tstat.sum_values; temp_ws.w_zd = w_zd; temp_ws.n = tstat.elements; ws[blk] = temp_ws; @@ -1371,7 +1369,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("============================================\n"); for (const auto & [layer, stats] : ws) { if (layer < 0 || stats.n == 0) continue; - const float w_sum = stats.w_sum / stats.n; + const float w_sum = stats.w_sum; const float w_zd = stats.w_zd / stats.n; const auto lcs = layer_cossim.find(layer); const float cossim = (lcs != layer_cossim.end()) ? lcs->second : 0.0f; From aea9b31db53a53204d8b817347b1972854e1d186 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 5 Aug 2025 12:57:13 +0100 Subject: [PATCH 024/123] Make ZD Score two-tailed --- tools/imatrix/imatrix.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6d40984a0180..d30b66c6a61f 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -231,9 +231,7 @@ static int compute_vector_statistics(std::vector & tstats, co int z_score = 0; if (std_deviation > 0.0f) { for (const auto act : activations) { - if (const float p = (act - mean) / std_deviation; p > 1) { - z_score++; - } + if (const float z = (act - mean) / std_deviation; std::fabs(z) > 1.0f) z_score++; } } From 49996a19dafb0b13d9e427c7a30f50f59dc34e80 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 5 Aug 2025 13:32:46 +0100 Subject: [PATCH 025/123] Refactor variable names --- tools/imatrix/imatrix.cpp | 94 +++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index d30b66c6a61f..46db4401a75a 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -39,7 +39,7 @@ static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count"; static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size"; struct Stats { - std::vector in_sum; + std::vector activations; std::vector values; std::vector counts; }; @@ -130,20 +130,20 @@ static void process_tensor_name(const std::string & input, std::string & layer, static std::vector compute_tensor_averages(const Stats & tstats) { if (tstats.counts.empty()) return {}; const size_t n_mat = tstats.counts.size(); - const size_t len = !tstats.in_sum.empty() ? tstats.in_sum.size() : tstats.values.size(); + const size_t len = !tstats.activations.empty() ? tstats.activations.size() : tstats.values.size(); if (len == 0 || len % n_mat != 0) return {}; const size_t row = len / n_mat; std::vector vec; vec.reserve(len); - if (!tstats.in_sum.empty()) { + if (!tstats.activations.empty()) { for (size_t m = 0; m < n_mat; ++m) { const float c = (float)tstats.counts[m]; if (c <= 0) return {}; const size_t off = m * row; for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.in_sum[off + j] / c); + vec.push_back(tstats.activations[off + j] / c); } } } else { @@ -172,11 +172,11 @@ static int compute_vector_statistics(std::vector & tstats, co const int n_mat = e.counts.size(); const int row_size = e.values.size() / n_mat; - const int calc_mode = e.in_sum.empty() ? 2 : 1; + const int calc_mode = e.activations.empty() ? 2 : 1; std::vector activations; - if (e.in_sum.empty()) { + if (e.activations.empty()) { activations.reserve(e.values.size()); for (int i = 0; i < n_mat; ++i) { @@ -185,11 +185,11 @@ static int compute_vector_statistics(std::vector & tstats, co } } } else { - activations.reserve(e.in_sum.size()); + activations.reserve(e.activations.size()); for (int i = 0; i < n_mat; ++i) { for (int j = 0; j < row_size; ++j) { - activations.push_back(e.in_sum[i*row_size + j] / e.counts[i]); + activations.push_back(e.activations[i*row_size + j] / e.counts[i]); } } } @@ -282,7 +282,7 @@ static void compute_tensor_statistics(std::vector & tstats) { // compute the L2 Norm (Euclidian Distance) between the same tensors in consecutive layers for (auto & ts : tstats) { ts.l2_norm = 0.0f; - if (ts.stats.in_sum.empty()) continue; + if (ts.stats.activations.empty()) continue; if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { const int blk = std::stoi(match[1]); @@ -430,7 +430,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts.resize(n_as, e.counts[0]); } if (e.values.empty()) { - e.in_sum.resize(src1->ne[0]*n_as, 0); + e.activations.resize(src1->ne[0]*n_as, 0); e.values.resize(src1->ne[0]*n_as, 0); e.counts.resize(n_as, 0); } @@ -462,7 +462,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts[ex]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { - e.in_sum[e_start + j] += x[j]; + e.activations[e_start + j] += x[j]; e.values[e_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[e_start + j])) { LOG_ERR("%f detected in %s\n", (float)e.values[e_start + j], wname.c_str()); @@ -502,7 +502,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * } } if (e.values.empty()) { - e.in_sum.resize(src1->ne[0] * n_mat, 0); + e.activations.resize(src1->ne[0] * n_mat, 0); e.values.resize(src1->ne[0] * n_mat, 0); e.counts.resize(1, 0); } @@ -521,7 +521,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * for (int64_t row = 0; row < src1->ne[1]; ++row) { const float * x = (const float *) (data + row * src1->nb[1] + i2 * src1->nb[2] + i3 * src1->nb[3]); for (int64_t j = 0; j < src1->ne[0]; ++j) { - e.in_sum[mat_start + j] += x[j]; + e.activations[mat_start + j] += x[j]; e.values[mat_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[j])) { LOG_ERR("%f detected in %s\n", (float)e.values[j], wname.c_str()); @@ -699,7 +699,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { } to_store.push_back(kv.first); - data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.in_sum.size(), GGML_MEM_ALIGN); + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); } @@ -753,12 +753,12 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum2); gguf_add_tensor(ctx_gguf, counts); - if (!stat.in_sum.empty()) { - const int32_t nact = (int32_t) stat.in_sum.size(); + if (!stat.activations.empty()) { + const int32_t nact = (int32_t) stat.activations.size(); struct ggml_tensor * in_sum = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nact / nmat, nmat); ggml_format_name(in_sum, "%s.in_sum", name.c_str()); for (int32_t j = 0; j < nval; ++j) { - ((float *) in_sum->data)[j] = (float) stat.in_sum[j]; + ((float *) in_sum->data)[j] = (float) stat.activations[j]; } gguf_add_tensor(ctx_gguf, in_sum); } @@ -949,7 +949,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { if (e.values.empty()) { e.values.resize(nval, 0.0f); if (in_sum != nullptr) { - e.in_sum.resize(nval, 0.0f); + e.activations.resize(nval, 0.0f); } } else if ((size_t) nval != e.values.size()) { LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size()); @@ -980,7 +980,7 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { } if (in_sum != nullptr) { for (int64_t j = 0; j < nval; j++) { - e.in_sum[j] += ((const float *) in_sum->data)[j]; + e.activations[j] += ((const float *) in_sum->data)[j]; } } } @@ -1289,12 +1289,12 @@ static bool show_statistics(const common_params & params) { }; std::sort(ts.begin(), ts.end(), tensor_comparer()); - struct weighted_stats { - float w_sum = 0.0f; - float w_zd = 0.0f; + struct layer_stats { + float lyr_sum = 0.0f; + float lyr_zd = 0.0f; int n = 0; }; - std::map ws; + std::map ls; LOG_INF("\nComputing statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%12s\t%s\t%5s\t%10s\n", @@ -1338,26 +1338,26 @@ static bool show_statistics(const common_params & params) { 100.0f * tstat.zd_score, tstat.cossim); - const float w_zd = tstat.elements * tstat.zd_score; + const float zd = tstat.elements * tstat.zd_score; - if (ws.find(blk) != ws.end()) { - ws[blk].w_sum += tstat.sum_values; - ws[blk].w_zd += w_zd; - ws[blk].n += tstat.elements; + if (ls.find(blk) != ls.end()) { + ls[blk].lyr_sum += tstat.sum_values; + ls[blk].lyr_zd += zd; + ls[blk].n += tstat.elements; } else { - weighted_stats temp_ws; - temp_ws.w_sum = tstat.sum_values; - temp_ws.w_zd = w_zd; - temp_ws.n = tstat.elements; - ws[blk] = temp_ws; + layer_stats temp_ls; + temp_ls.lyr_sum = tstat.sum_values; + temp_ls.lyr_zd = zd; + temp_ls.n = tstat.elements; + ls[blk] = temp_ls; } } - std::map layer_cossim; - std::map layer_l2_norm; - compute_layer_statistics(ts, layer_cossim, layer_l2_norm, g_collector.get_mstats()); + std::map lyr_cossim; + std::map lyr_l2_norm; + compute_layer_statistics(ts, lyr_cossim, lyr_l2_norm, g_collector.get_mstats()); - const auto layers = std::count_if(ws.begin(), ws.end(), [](const auto & kv) { return kv.first >= 0; }); + const auto layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing aggregated statistics per layer (%ld layers)\n", layers); LOG_INF("\n%6s\t%16s\t%7s\t%11s\n", "Layer", @@ -1365,19 +1365,19 @@ static bool show_statistics(const common_params & params) { "ZD", "CosSim"); LOG_INF("============================================\n"); - for (const auto & [layer, stats] : ws) { + for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) continue; - const float w_sum = stats.w_sum; - const float w_zd = stats.w_zd / stats.n; - const auto lcs = layer_cossim.find(layer); - const float cossim = (lcs != layer_cossim.end()) ? lcs->second : 0.0f; - const auto ll2n = layer_l2_norm.find(layer); - const float l2_norm = (ll2n != layer_l2_norm.end()) ? ll2n->second : 0.0f; + const float lyr_sum = stats.lyr_sum; + const float lyr_zd = stats.lyr_zd / stats.n; + const auto lcs = lyr_cossim.find(layer); + const float lyr_cs = (lcs != lyr_cossim.end()) ? lcs->second : 0.0f; + const auto ll2n = lyr_l2_norm.find(layer); + const float l2_norm = (ll2n != lyr_l2_norm.end()) ? ll2n->second : 0.0f; LOG_INF("%5d\t%11.2f\t%6.2f%%\t%10.4f\n", layer, - tensor_calc_mode == 1 ? l2_norm: w_sum, - 100.0f * w_zd, - cossim); + tensor_calc_mode == 1 ? l2_norm: lyr_sum, + 100.0f * lyr_zd, + lyr_cs); } LOG_INF("\n"); From 4c3fea89d6975f418d6d6249e49ed8dbd208fe2b Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 5 Aug 2025 13:32:59 +0100 Subject: [PATCH 026/123] Update report layout --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 46db4401a75a..634739082a05 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1359,7 +1359,7 @@ static bool show_statistics(const common_params & params) { const auto layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing aggregated statistics per layer (%ld layers)\n", layers); - LOG_INF("\n%6s\t%16s\t%7s\t%11s\n", + LOG_INF("\n%6s\t%13s\t%5s\t%10s\n", "Layer", tensor_calc_mode == 1 ? "L₂ Norm" : "Σ(Act²)", "ZD", From 88854c9179b6e62ffde2d57123e74e47dc00a55a Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 5 Aug 2025 14:16:45 +0100 Subject: [PATCH 027/123] Refactor legacy mode --- tools/imatrix/imatrix.cpp | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 634739082a05..2f16f3489c9e 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -160,7 +160,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { return vec; } -static int compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { +static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { if (e.values.size() % e.counts.size() != 0) { LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.values.size()); return -1;; @@ -172,7 +172,6 @@ static int compute_vector_statistics(std::vector & tstats, co const int n_mat = e.counts.size(); const int row_size = e.values.size() / n_mat; - const int calc_mode = e.activations.empty() ? 2 : 1; std::vector activations; @@ -203,7 +202,15 @@ static int compute_vector_statistics(std::vector & tstats, co const float std_deviation = std::sqrt(std::max(0.0f, variance)); float entropy = 0; - if (calc_mode == 1) { + if (e.activations.empty()) { + if (sum > 0) { + for (const auto act : activations) { + if (const float p = act / sum; p > 0) { + entropy -= p * std::log2(p); + } + } + } + } else { float div = 0.0; std::vector weights(activations.size()); for (size_t i = 0; i < activations.size(); ++i) { @@ -218,14 +225,6 @@ static int compute_vector_statistics(std::vector & tstats, co if (p > 0.0) entropy -= p * std::log2(p); } } - } else { - if (sum > 0) { - for (const auto act : activations) { - if (const float p = act / sum; p > 0) { - entropy -= p * std::log2(p); - } - } - } } int z_score = 0; @@ -247,7 +246,7 @@ static int compute_vector_statistics(std::vector & tstats, co ts.entropy = entropy; ts.zd_score = static_cast(z_score) / ts.elements; - return calc_mode; + return e.activations.empty(); } static void compute_tensor_statistics(std::vector & tstats) { @@ -1257,7 +1256,7 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c static bool show_statistics(const common_params & params) { std::vector ts; - int tensor_calc_mode = 0; + bool legacy_mode = false; if (params.in_files.empty() || params.in_files.size() > 1) { LOG_ERR("\nError: a single imatrix file is required to compute tensor statistics\n\n"); @@ -1265,7 +1264,7 @@ static bool show_statistics(const common_params & params) { } if (g_collector.load_imatrix(params.in_files[0].c_str())) { for (const auto & [name, stats] :g_collector.get_mstats()) { - tensor_calc_mode =compute_vector_statistics(ts, name, stats); + legacy_mode = compute_vector_statistics(ts, name, stats); } } else { LOG_ERR("\nError: %s is not a valid imatrix file\n\n", params.in_files[0].c_str()); @@ -1300,7 +1299,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%12s\t%s\t%5s\t%10s\n", "Layer", "Tensor", - tensor_calc_mode == 1 ? "L₂ Norm" : "Σ(Act²)", + legacy_mode ? "Σ(Act²)" : "L₂ Norm", "Min", "Max", "μ", @@ -1327,7 +1326,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%12.4f\t%7.2f%%\t%6.2f%%\t%10.4f\n", layer.c_str(), name.c_str(), - tensor_calc_mode == 1 ? tstat.l2_norm : tstat.sum_values, + legacy_mode == 1 ? tstat.sum_values : tstat.l2_norm, tstat.min_values, tstat.max_values, tstat.mean_values, @@ -1361,7 +1360,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("\nComputing aggregated statistics per layer (%ld layers)\n", layers); LOG_INF("\n%6s\t%13s\t%5s\t%10s\n", "Layer", - tensor_calc_mode == 1 ? "L₂ Norm" : "Σ(Act²)", + legacy_mode ? "Σ(Act²)" : "L₂ Norm", "ZD", "CosSim"); LOG_INF("============================================\n"); @@ -1375,7 +1374,7 @@ static bool show_statistics(const common_params & params) { const float l2_norm = (ll2n != lyr_l2_norm.end()) ? ll2n->second : 0.0f; LOG_INF("%5d\t%11.2f\t%6.2f%%\t%10.4f\n", layer, - tensor_calc_mode == 1 ? l2_norm: lyr_sum, + legacy_mode ? lyr_sum : l2_norm, 100.0f * lyr_zd, lyr_cs); } From 3e9d53c61e69d72c848aacfb7b7830855908bb54 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 7 Aug 2025 12:03:24 +0100 Subject: [PATCH 028/123] Refactor variable names --- tools/imatrix/imatrix.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index a229c927b5a3..7554534adf2e 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -47,16 +47,16 @@ struct Stats { struct tensor_statistics { std::string tensor; Stats stats; - float sum_values = 0.0f; - float mean_values = 0.0f; - float max_values = 0.0f; - float min_values = 0.0f; - int elements = 0; - float stddev = 0.0f; - float entropy = 0.0f; - float zd_score = 0.0f; - float cossim = 0.0f; - float l2_norm = 0.0f; + float sum_values = 0.0f; + float mean_values = 0.0f; + float max_values = 0.0f; + float min_values = 0.0f; + int elements = 0; + float std_deviation = 0.0f; + float entropy = 0.0f; + float zd_score = 0.0f; + float cossim = 0.0f; + float l2_norm = 0.0f; }; class IMatrixCollector { @@ -227,10 +227,10 @@ static bool compute_vector_statistics(std::vector & tstats, c } } - int z_score = 0; + int zd_score = 0; if (std_deviation > 0.0f) { for (const auto act : activations) { - if (const float z = (act - mean) / std_deviation; std::fabs(z) > 1.0f) z_score++; + if (const float z = (act - mean) / std_deviation; std::fabs(z) > 1.0f) zd_score++; } } @@ -242,9 +242,9 @@ static bool compute_vector_statistics(std::vector & tstats, c ts.max_values = max; ts.min_values = min; ts.elements = static_cast(activations.size()); - ts.stddev = std_deviation; + ts.std_deviation = std_deviation; ts.entropy = entropy; - ts.zd_score = static_cast(z_score) / ts.elements; + ts.zd_score = static_cast(zd_score) / ts.elements; return e.activations.empty(); } @@ -1334,7 +1334,7 @@ static bool show_statistics(const common_params & params) { tstat.min_values, tstat.max_values, tstat.mean_values, - tstat.stddev, + tstat.std_deviation, tstat.elements, tstat.entropy, 100.0f * (tstat.entropy / std::log2(tstat.elements)), From e0d64713402b7c236806cdfc3ce4cb7d0ad0cd34 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 7 Aug 2025 12:04:52 +0100 Subject: [PATCH 029/123] Reverse conditional logic to match convention --- tools/imatrix/imatrix.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 7554534adf2e..d8ff591349fb 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -137,13 +137,13 @@ static std::vector compute_tensor_averages(const Stats & tstats) { std::vector vec; vec.reserve(len); - if (!tstats.activations.empty()) { + if (tstats.activations.empty()) { for (size_t m = 0; m < n_mat; ++m) { const float c = (float)tstats.counts[m]; if (c <= 0) return {}; const size_t off = m * row; for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.activations[off + j] / c); + vec.push_back(tstats.values[off + j] / c); } } } else { @@ -152,7 +152,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { if (c <= 0) return {}; const size_t off = m * row; for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.values[off + j] / c); + vec.push_back(tstats.activations[off + j] / c); } } } From dadd90ef73410025e8ccc1eb9517e6ddb3796134 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 7 Aug 2025 14:07:48 +0100 Subject: [PATCH 030/123] Rename report heading --- tools/imatrix/imatrix.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index d8ff591349fb..a758a940960b 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1299,7 +1299,7 @@ static bool show_statistics(const common_params & params) { }; std::map ls; - LOG_INF("\nComputing statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); + LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%12s\t%s\t%5s\t%10s\n", "Layer", "Tensor", @@ -1361,7 +1361,7 @@ static bool show_statistics(const common_params & params) { compute_layer_statistics(ts, lyr_cossim, lyr_l2_norm, g_collector.get_mstats()); const auto layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); - LOG_INF("\nComputing aggregated statistics per layer (%ld layers)\n", layers); + LOG_INF("\nComputing layer statistics (%ld layers)\n", layers); LOG_INF("\n%6s\t%13s\t%5s\t%10s\n", "Layer", legacy_mode ? "Σ(Act²)" : "L₂ Norm", From 5bb2def02dcf923743029f72b0c16b17e3609e28 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 7 Aug 2025 17:41:21 +0100 Subject: [PATCH 031/123] Add --activation-statistics parameter --- common/arg.cpp | 7 +++++++ common/common.h | 9 +++++---- tools/imatrix/imatrix.cpp | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 0f01bb31454a..2cd0cc011981 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2707,6 +2707,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.show_statistics = true; } ).set_examples({LLAMA_EXAMPLE_IMATRIX})); + add_opt(common_arg( + {"--activation-statistics"}, + string_format("generate data to compute activation-based statistics (default: %s)", params.show_statistics ? "true" : "false"), + [](common_params & params) { + params.activation_statistics = true; + } + ).set_examples({LLAMA_EXAMPLE_IMATRIX})); add_opt(common_arg( {"--parse-special"}, string_format("prase special tokens (chat, tool, etc) (default: %s)", params.parse_special ? "true" : "false"), diff --git a/common/common.h b/common/common.h index 5eab199af559..d5dfdd49e0ce 100644 --- a/common/common.h +++ b/common/common.h @@ -443,10 +443,11 @@ struct common_params { int32_t i_chunk = 0; // start processing from this chunk int8_t imat_dat = 0; // whether the legacy imatrix.dat format should be output (gguf <= 0 < dat) - bool process_output = false; // collect data for the output tensor - bool compute_ppl = true; // whether to compute perplexity - bool show_statistics = false; // show imatrix statistics per tensor - bool parse_special = false; // whether to parse special tokens during imatrix tokenization + bool process_output = false; // collect data for the output tensor + bool compute_ppl = true; // whether to compute perplexity + bool show_statistics = false; // show imatrix statistics per tensor + bool activation_statistics = false; // generate data to calculate activation based statistics + bool parse_special = false; // whether to parse special tokens during imatrix tokenization // cvector-generator params int n_pca_batch = 100; diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index a758a940960b..902d6e7354aa 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -30,7 +30,7 @@ static void print_usage(int, char ** argv) { " -m model.gguf -f some-text.txt [-o imatrix.gguf] [--output-format {gguf,dat}] [--no-ppl] \\\n" " [--process-output] [--chunk 123] [--save-frequency 0] [--output-frequency 10] \\\n" " [--in-file imatrix-prev-0.gguf --in-file imatrix-prev-1.gguf ...] [--parse-special] \\\n" - " [--show-statistics] [...]\n" , argv[0]); + " [--activation-statistics] [--show-statistics] [...]\n" , argv[0]); LOG("\n"); } @@ -428,6 +428,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * // broadcast, when loading an old imatrix e.counts.resize(n_as, e.counts[0]); } + // ToDo: find an efficient way to implement --activation-statistics to avoid doubling the imatrix size by default if (e.values.empty()) { e.activations.resize(src1->ne[0]*n_as, 0); e.values.resize(src1->ne[0]*n_as, 0); From c5ecdaa1a1600499f3be8be1f2cb7cc7062959cf Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 7 Aug 2025 19:04:49 +0100 Subject: [PATCH 032/123] =?UTF-8?q?Add=20Euclidean=E2=80=93Cosine=20Score?= =?UTF-8?q?=20(ECS)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/imatrix/imatrix.cpp | 45 +++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 902d6e7354aa..bedbf586e499 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #if defined(_MSC_VER) @@ -1301,7 +1302,7 @@ static bool show_statistics(const common_params & params) { std::map ls; LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); - LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%12s\t%s\t%5s\t%10s\n", + LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%11s\t%8s\t%5s\t%10s\n", "Layer", "Tensor", legacy_mode ? "Σ(Act²)" : "L₂ Norm", @@ -1310,8 +1311,8 @@ static bool show_statistics(const common_params & params) { "μ", "σ", "N", - "Entropy", - "E (norm)", + "H Norm", + legacy_mode ? "H" : "ECS", "ZD", "CosSim"); LOG_INF( @@ -1328,17 +1329,17 @@ static bool show_statistics(const common_params & params) { blk = -1; // not a block layer } - LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%12.4f\t%7.2f%%\t%6.2f%%\t%10.4f\n", + LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", layer.c_str(), name.c_str(), - legacy_mode == 1 ? tstat.sum_values : tstat.l2_norm, + legacy_mode ? tstat.sum_values : tstat.l2_norm, tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, tstat.elements, - tstat.entropy, 100.0f * (tstat.entropy / std::log2(tstat.elements)), + legacy_mode ? tstat.entropy : 100.0f * std::exp(-0.01f * tstat.l2_norm) * std::pow(fabs(tstat.cossim), 10.0f), 100.0f * tstat.zd_score, tstat.cossim); @@ -1363,25 +1364,37 @@ static bool show_statistics(const common_params & params) { const auto layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing layer statistics (%ld layers)\n", layers); - LOG_INF("\n%6s\t%13s\t%5s\t%10s\n", + LOG_INF("\n%6s\t%13s\t%6s\t%11s\t%6s\n", "Layer", legacy_mode ? "Σ(Act²)" : "L₂ Norm", "ZD", - "CosSim"); - LOG_INF("============================================\n"); + "CosSim", + legacy_mode ? "" : "ECS"); + if (legacy_mode) { + LOG_INF("============================================\n"); + } else { + LOG_INF("=========================================================\n"); + } for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) continue; - const float lyr_sum = stats.lyr_sum; - const float lyr_zd = stats.lyr_zd / stats.n; const auto lcs = lyr_cossim.find(layer); - const float lyr_cs = (lcs != lyr_cossim.end()) ? lcs->second : 0.0f; + const float lyr_cs = lcs != lyr_cossim.end() ? lcs->second : 0.0f; const auto ll2n = lyr_l2_norm.find(layer); - const float l2_norm = (ll2n != lyr_l2_norm.end()) ? ll2n->second : 0.0f; - LOG_INF("%5d\t%11.2f\t%6.2f%%\t%10.4f\n", + const float lyr_l2n = ll2n != lyr_l2_norm.end() ? ll2n->second : 0.0f; + if (legacy_mode) { + LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\n", layer, - legacy_mode ? lyr_sum : l2_norm, - 100.0f * lyr_zd, + stats.lyr_sum, + 100.0f * stats.lyr_zd / stats.n, lyr_cs); + } else { + LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\t%8.4f\n", + layer, + lyr_l2n, + 100.0f * stats.lyr_zd / stats.n, + lyr_cs, + 100.0f * std::exp(-0.01f * lyr_l2n) * std::pow(fabs(lyr_cs), 10.0f)); + } } LOG_INF("\n"); From 59af5034f7fba3c31dc9cc861bcaf35024abe9dd Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 9 Aug 2025 01:26:23 +0100 Subject: [PATCH 033/123] Update README.md --- tools/imatrix/README.md | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 4505cb4ce8c7..bf13a2860509 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -20,19 +20,19 @@ The parameters in square brackets are optional and have the following meaning: * `-lv | --verbosity` specifies the verbosity level. If set to `0`, no output other than the perplexity of the processed chunks will be generated. If set to `1`, each time the results are saved a message is written to `stderr`. If `>=2`, a message is output each time data is collected for any tensor. Default verbosity level is `1`. * `-o | --output-file` specifies the name of the file where the computed data will be stored. If missing `imatrix.gguf` is used. * `-ofreq | --output-frequency` specifies how often the so far computed result is saved to disk. Default is 10 (i.e., every 10 chunks) -* `--output-format` specifies the output format of the generated imatrix file. Either "gguf", or "dat" (the legacy format). Defaults to "gguf". +* `--output-format` specifies the output format of the generated imatrix file. Either `gguf`, or `dat` (the legacy format). Defaults to `gguf`. * `--save-frequency` specifies how often to save a copy of the imatrix in a separate file. Default is 0 (i.e., never) * `--process-output` specifies if data will be collected for the `output.weight` tensor. Typically, it is better not to utilize the importance matrix when quantizing `output.weight`, so this is set to `false` by default. * `--in-file` one or more existing imatrix files to load and combine. Useful for merging files from multiple runs/datasets. * `--parse-special` enables parsing of special tokens (e.g., `<|im_start|>` in some models). Useful for models with custom tokenizers. * `--chunk | --from-chunk` to skip the first `n` chunks of tokens from the input data. Useful for resuming or skipping initial low-quality data. -* `--chunks` maximum number of chunks to process. Default is -1 for all available chunks. +* `--chunks` maximum number of chunks to process. Default is `-1` for all available chunks. * `--no-ppl` disables the calculation of perplexity for the processed chunks. Useful if you want to speed up the processing and do not care about perplexity. * `--show-statistics` displays imatrix file's statistics. For faster computation, make sure to use GPU offloading via the `-ngl | --n-gpu-layers` argument. -Recent versions of `llama-imatrix` store data in GGUF format by default. For the legacy format, use an extension other than `.gguf` when saving the output file. More information is available in . +Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by default. For the legacy format, use `--output-format dat` when saving the output file. More information is available in . ## Examples @@ -74,25 +74,27 @@ Recent versions of `llama-imatrix` store data in GGUF format by default. For the ./llama-imatrix --in-file imatrix.gguf --show-statistics ``` -`--show-statistics` will display the following statistics: +## Statistics + +From version , `--show-statistics` operates in two modes: for GGUF (preferred) imatrices, it reports direct and accurate activation statistics, and for legacy (binary) files, it reports the less precise average squared activations. #### Per tensor -* Σ(Act²): sum of all squared activations (the importance scores) -* Min & Max: minimum and maximum squared activations values -* μ & σ: Squared activations' mean and standard deviation -* % Active: proportion of elements whose average squared activation exceeds a small threshold (1e-5). Helpful to determine how alive/dormant the tensor is during inference -* N: number of squared activations -* Entropy: entropy of the squared activation distribution, in bits (standard Shannon entropy measurement) $S = -\sum_{i=1}^N p_i \log_2 p_i$ -* E (norm): Normalized entropy. $E(norm)=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. These two metrics can be used to determine how well a prompt "exercises" the model's capabilities -* ZD Score: z-score distribution as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415) -* CosSim: cosine similarity with respect to the previous layer's tensor. Useful to determine how similar the squared activations of the current layer are to the previous layer's squared activations. +* **Σ(Act²)** *(legacy mode)* / **L₂ Norm** *(preferred)*: If in legacy mode, the raw sum of squares of activations (sum of `Act²`). In preferred mode, the Euclidean Distance (L₂ Norm) between this tensor’s average activations and those of the previous layer. +* **Min / Max / μ / σ**: Tensor elements Min, Max, Mean, and Standard Deviation. +* **N**: Number of tensor elements considered. +* **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. +* **H** *(legacy mode)* / **ECS** *(preferred)*: If legacy, Shannon Entropy defined as $H = -\sum_{i=1}^N p_i \log_2 p_i$. If preferred, *Euclidean-Cosine Score* defined as $ECS = K \cdot e^{-\alpha a} \cdot |b|^{\gamma}$ where `a = L₂ Norm`, `b = Cosine Similarity`, `α = -0.01`, `γ = 10` between this tensor’s elements and those of the previous layer. Higher score means more similarity and lower change. +* **ZD**: % of elements whose Z-score is > 1.0 in magnitude (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415) +* **CosSim**: Cosine Similarity between this tensor’s elements and those of the previous layer. #### Per layer -Weighted averages of Σ(Act²), ZD Score and CosSim are also calculated. +Aggregated metrics per block/layer: -#### Important note on the computed Statistics +* **Σ(Act²)** *(legacy mode)* / **L₂ Norm** *(preferred)*: If in legacy mode, the sum of squared activations (sum of Act²) for the layer's concatenated tensors. In preferred mode, the Euclidean Distance (L₂ Norm) between this layer's average concatenated tensor activations the previous layer. +* **ZD**: % of this layer's concatenated tensors' elements with |Z| > 1. +* **CosSim**: Cosine Similarity between this layer's concatenated tensors' elements compared and the previous layer’s. +* **ECS** *(preferred only)*: Euclidean-Cosine Score applied to the layer. -When using these statistics, please note that they are computed on the squared activations, **not on the actual (raw) activations**. -Whilst the results are still useful, they're less realiable than using the raw values, and in the case of the cosine similarity, could be misleading if the tensor contains opposite vectors. +More information is available in https://github.com/ggml-org/llama.cpp/pull/14891 From 6fe51e12f1801ce11a7b0f2a53a2daf0caddfcfa Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 9 Aug 2025 09:12:23 +0100 Subject: [PATCH 034/123] Fix typo in ECS formula --- tools/imatrix/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index bf13a2860509..adbec2ed0097 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -84,7 +84,7 @@ From version , `--show-statistics` operates in two modes: for GGUF (prefe * **Min / Max / μ / σ**: Tensor elements Min, Max, Mean, and Standard Deviation. * **N**: Number of tensor elements considered. * **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. -* **H** *(legacy mode)* / **ECS** *(preferred)*: If legacy, Shannon Entropy defined as $H = -\sum_{i=1}^N p_i \log_2 p_i$. If preferred, *Euclidean-Cosine Score* defined as $ECS = K \cdot e^{-\alpha a} \cdot |b|^{\gamma}$ where `a = L₂ Norm`, `b = Cosine Similarity`, `α = -0.01`, `γ = 10` between this tensor’s elements and those of the previous layer. Higher score means more similarity and lower change. +* **H** *(legacy mode)* / **ECS** *(preferred)*: If legacy, Shannon Entropy defined as $H = -\sum_{i=1}^N p_i \log_2 p_i$. If preferred, *Euclidean-Cosine Score* defined as $ECS = K \cdot e^{-\alpha a} \cdot |b|^{\gamma}$ where `a = L₂ Norm`, `b = Cosine Similarity`, `α = 0.01`, `γ = 10` between this tensor’s elements and those of the previous layer. Higher score means more similarity and lower change. * **ZD**: % of elements whose Z-score is > 1.0 in magnitude (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415) * **CosSim**: Cosine Similarity between this tensor’s elements and those of the previous layer. From dcac206f8e3931116956080b37c1e2aa809dff45 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 9 Aug 2025 14:49:25 +0100 Subject: [PATCH 035/123] Add --activation-statistics logic to avoid doubling the imatrix size by default --- tools/imatrix/imatrix.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index bedbf586e499..4d3a13cb2a03 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -64,6 +64,7 @@ class IMatrixCollector { public: IMatrixCollector() = default; void set_params(common_params params) { m_params = std::move(params); } + bool activation_statistics() const { return m_params.activation_statistics; } bool collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data); void save_imatrix_legacy(int32_t ncall = -1) const; void save_imatrix(int32_t n_chunk = -1) const; @@ -429,9 +430,8 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * // broadcast, when loading an old imatrix e.counts.resize(n_as, e.counts[0]); } - // ToDo: find an efficient way to implement --activation-statistics to avoid doubling the imatrix size by default if (e.values.empty()) { - e.activations.resize(src1->ne[0]*n_as, 0); + if (activation_statistics()) e.activations.resize(src1->ne[0]*n_as, 0); e.values.resize(src1->ne[0]*n_as, 0); e.counts.resize(n_as, 0); } @@ -463,7 +463,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts[ex]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { - e.activations[e_start + j] += x[j]; + if (activation_statistics()) e.activations[e_start + j] += x[j]; e.values[e_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[e_start + j])) { LOG_ERR("%f detected in %s\n", (float)e.values[e_start + j], wname.c_str()); @@ -503,7 +503,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * } } if (e.values.empty()) { - e.activations.resize(src1->ne[0] * n_mat, 0); + if (activation_statistics()) e.activations.resize(src1->ne[0] * n_mat, 0); e.values.resize(src1->ne[0] * n_mat, 0); e.counts.resize(1, 0); } @@ -522,7 +522,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * for (int64_t row = 0; row < src1->ne[1]; ++row) { const float * x = (const float *) (data + row * src1->nb[1] + i2 * src1->nb[2] + i3 * src1->nb[3]); for (int64_t j = 0; j < src1->ne[0]; ++j) { - e.activations[mat_start + j] += x[j]; + if (activation_statistics()) e.activations[mat_start + j] += x[j]; e.values[mat_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[j])) { LOG_ERR("%f detected in %s\n", (float)e.values[j], wname.c_str()); @@ -704,7 +704,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { } to_store.push_back(kv.first); - data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); + if (activation_statistics()) data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); } @@ -758,7 +758,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum2); gguf_add_tensor(ctx_gguf, counts); - if (!stat.activations.empty()) { + if (!stat.activations.empty() && activation_statistics()) { const int32_t nact = (int32_t) stat.activations.size(); struct ggml_tensor * in_sum = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nact / nmat, nmat); ggml_format_name(in_sum, "%s.in_sum", name.c_str()); From 89051cda35532ddf9a43c8e7d9c4655b160181e4 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 9 Aug 2025 14:49:44 +0100 Subject: [PATCH 036/123] Update README.md --- tools/imatrix/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index adbec2ed0097..46ee8d9968c8 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -10,7 +10,7 @@ More information is available in , `--show-statistics` operates in two modes: for GGUF (preferred) imatrices, it reports direct and accurate activation statistics, and for legacy (binary) files, it reports the less precise average squared activations. +Beginning with version , `--show-statistics` has two modes. If `--activation-statistics` was used at imatrix creation time and `--output-format` was set to `gguf`, it reports precise statistics. Otherwise, it reports less accurate, albeit still useful, metrics based on average squared activations. #### Per tensor From 42bfe3b2a35b53e531590893e1d3fde5c27b3b42 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Fri, 15 Aug 2025 21:12:56 +0100 Subject: [PATCH 037/123] Update stats output sort based on imatrix type --- tools/imatrix/imatrix.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 4d3a13cb2a03..1361262abebf 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1284,15 +1284,18 @@ static bool show_statistics(const common_params & params) { } struct tensor_comparer { + bool legacy_mode; + explicit tensor_comparer(const bool legacy) : legacy_mode(legacy) {} + bool operator()(const tensor_statistics & a, const tensor_statistics & b) const { std::string layer, name_a, name_b; - ; process_tensor_name(a.tensor, layer, name_a); process_tensor_name(b.tensor, layer, name_b); - return name_a < name_b || (name_a == name_b && a.sum_values > b.sum_values); + return legacy_mode ? name_a < name_b || (name_a == name_b && a.sum_values > b.sum_values) : + name_a < name_b || (name_a == name_b && a.cossim > b.cossim); } }; - std::sort(ts.begin(), ts.end(), tensor_comparer()); + std::sort(ts.begin(), ts.end(), tensor_comparer(legacy_mode)); struct layer_stats { float lyr_sum = 0.0f; From 240a965e5086015391a06dceacb68b2c314be281 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Fri, 15 Aug 2025 21:24:38 +0100 Subject: [PATCH 038/123] Update README.md --- tools/imatrix/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 46ee8d9968c8..b511847bf379 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -82,7 +82,7 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def ## Statistics -Beginning with version , `--show-statistics` has two modes. If `--activation-statistics` was used at imatrix creation time and `--output-format` was set to `gguf`, it reports precise statistics. Otherwise, it reports less accurate, albeit still useful, metrics based on average squared activations. +For current versions of `llama-imatrix`, the `--show-statistics` option has two modes of operation: If `--activation-statistics` was used to generate the imatrix and `--output-format` was set to `gguf`, precise activations statistics will be calculated. Otherwise, it will report less accurate, albeit still useful, metrics based on average squared activations. #### Per tensor From 8589ef4d1537b94b61fe857ed69e9bc15debbae0 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Fri, 15 Aug 2025 21:27:48 +0100 Subject: [PATCH 039/123] Update README.md --- tools/imatrix/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index b511847bf379..c25fa097a9df 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -82,7 +82,7 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def ## Statistics -For current versions of `llama-imatrix`, the `--show-statistics` option has two modes of operation: If `--activation-statistics` was used to generate the imatrix and `--output-format` was set to `gguf`, precise activations statistics will be calculated. Otherwise, it will report less accurate, albeit still useful, metrics based on average squared activations. +For current versions of `llama-imatrix`, the `--show-statistics` option has two modes of operation: If `--activation-statistics` was used to generate the imatrix and `--output-format` was set to `gguf`, precise activations statistics will be calculated. Otherwise, it will report less accurate, albeit still useful, metrics based on average squared activations. #### Per tensor From 030ec53d7a4dfd7d3fdbc9a28ad61a244bc3380e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 16 Aug 2025 10:46:09 +0100 Subject: [PATCH 040/123] Remove unnecessary include --- tools/imatrix/imatrix.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 1361262abebf..f56915f712b6 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1,4 +1,3 @@ -#include "../../src/llama-impl.h" #include "arg.h" #include "common.h" #include "gguf.h" From d4b0d891157b85cffe971c0c4a1c7f7bfaf63145 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 16 Aug 2025 11:00:43 +0100 Subject: [PATCH 041/123] Fix return type bug --- tools/imatrix/imatrix.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f56915f712b6..e1c962f5beb5 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -164,11 +164,11 @@ static std::vector compute_tensor_averages(const Stats & tstats) { static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { if (e.values.size() % e.counts.size() != 0) { LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.values.size()); - return -1;; + return false; } if (e.counts.empty()) { LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); - return -1; + return false; } const int n_mat = e.counts.size(); From e3149a216869c969b92f14615174fc182646704f Mon Sep 17 00:00:00 2001 From: Ed Addario <29247825+EAddario@users.noreply.github.com> Date: Sun, 17 Aug 2025 07:24:27 +0100 Subject: [PATCH 042/123] Use the corresponding size Co-authored-by: compilade --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index e1c962f5beb5..ce4dc23e85fb 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -761,7 +761,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { const int32_t nact = (int32_t) stat.activations.size(); struct ggml_tensor * in_sum = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nact / nmat, nmat); ggml_format_name(in_sum, "%s.in_sum", name.c_str()); - for (int32_t j = 0; j < nval; ++j) { + for (int32_t j = 0; j < nact; ++j) { ((float *) in_sum->data)[j] = (float) stat.activations[j]; } gguf_add_tensor(ctx_gguf, in_sum); From 4a487ea7e4efdc66955481c1f8f227d97cdd3016 Mon Sep 17 00:00:00 2001 From: Ed Addario <29247825+EAddario@users.noreply.github.com> Date: Sun, 17 Aug 2025 07:26:16 +0100 Subject: [PATCH 043/123] Use { and } around the conditionally-executed statement Co-authored-by: compilade --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ce4dc23e85fb..617e3d424371 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -703,7 +703,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { } to_store.push_back(kv.first); - if (activation_statistics()) data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); + if (activation_statistics()) { data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); } data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); } From 97d839c441867d55de22fb91eadfa1a824253684 Mon Sep 17 00:00:00 2001 From: Ed Addario <29247825+EAddario@users.noreply.github.com> Date: Sun, 17 Aug 2025 08:06:15 +0100 Subject: [PATCH 044/123] Using one line per variable definition Co-authored-by: compilade --- tools/imatrix/imatrix.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 617e3d424371..610f840e30a3 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -268,7 +268,9 @@ static void compute_tensor_statistics(std::vector & tstats) { const auto curr_avg = compute_tensor_averages(ts.stats); const auto prev_avg = compute_tensor_averages(prev->stats); if (curr_avg.size() == prev_avg.size() && !curr_avg.empty()) { - float dot_prod = 0.0f, vec1 = 0.0f, vec2 = 0.0f; + float dot_prod = 0.0f; + float vec1 = 0.0f; + float vec2 = 0.0f; for (size_t i = 0; i < curr_avg.size(); ++i) { dot_prod += curr_avg[i] * prev_avg[i]; vec1 += curr_avg[i] * curr_avg[i]; From d19e6c9afa8791e0fe9090e8ea142e54f016afcf Mon Sep 17 00:00:00 2001 From: Ed Addario <29247825+EAddario@users.noreply.github.com> Date: Sun, 17 Aug 2025 08:08:26 +0100 Subject: [PATCH 045/123] Use { and } around the conditionally-executed statement Co-authored-by: compilade --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 610f840e30a3..7d7aaf609c3b 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -223,7 +223,7 @@ static bool compute_vector_statistics(std::vector & tstats, c if (div > 0.0) { for (float w : weights) { const float p = w / div; - if (p > 0.0) entropy -= p * std::log2(p); + if (p > 0.0) { entropy -= p * std::log2(p); } } } } From 12607d320367790d45454e6169086161bee30dd0 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 17 Aug 2025 08:10:54 +0100 Subject: [PATCH 046/123] Use { and } around single line for statement --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 617e3d424371..024b6cb160cb 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -317,7 +317,7 @@ static void compute_layer_statistics(const std::vector & tsta static const std::regex pattern(R"(blk\.(\d+)\.)"); std::unordered_map tidx; tidx.reserve(tstats.size()); - for (const auto & ts : tstats) tidx[ts.tensor] = &ts; + for (const auto & ts : tstats) { tidx[ts.tensor] = &ts; } std::map taggr; for (const auto & ts : tstats) { From a96013f720bba02afaf7cb064c3cf4c9fd9cb233 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 17 Aug 2025 08:16:41 +0100 Subject: [PATCH 047/123] Define one variable per line and refactor names --- tools/imatrix/imatrix.cpp | 58 ++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 024b6cb160cb..b4df57da9125 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -341,14 +341,16 @@ static void compute_layer_statistics(const std::vector & tsta const auto & curr = kv.second.curr_avg; const auto & prev = kv.second.prev_avg; if (curr.size() != prev.size() || curr.empty()) continue; - float dot_prod = 0.0, lyr1 = 0.0, lyr2 = 0.0; + float dot_prod = 0.0; + float layer1 = 0.0; + float layer2 = 0.0; for (size_t i = 0; i < curr.size(); ++i) { dot_prod += curr[i] * prev[i]; - lyr1 += curr[i] * curr[i]; - lyr2 += prev[i] * prev[i]; + layer1 += curr[i] * curr[i]; + layer2 += prev[i] * prev[i]; } float cossim = 0.0f; - if (lyr1 > 0.0 && lyr2 > 0.0) cossim = dot_prod / (std::sqrt(lyr1) * std::sqrt(lyr2)); + if (layer1 > 0.0 && layer2 > 0.0) cossim = dot_prod / (std::sqrt(layer1) * std::sqrt(layer2)); layer_cossim[kv.first] = cossim; } @@ -1297,9 +1299,9 @@ static bool show_statistics(const common_params & params) { std::sort(ts.begin(), ts.end(), tensor_comparer(legacy_mode)); struct layer_stats { - float lyr_sum = 0.0f; - float lyr_zd = 0.0f; - int n = 0; + float layer_sum = 0.0f; + float layer_zd = 0.0f; + int n = 0; }; std::map ls; @@ -1348,21 +1350,21 @@ static bool show_statistics(const common_params & params) { const float zd = tstat.elements * tstat.zd_score; if (ls.find(blk) != ls.end()) { - ls[blk].lyr_sum += tstat.sum_values; - ls[blk].lyr_zd += zd; - ls[blk].n += tstat.elements; + ls[blk].layer_sum += tstat.sum_values; + ls[blk].layer_zd += zd; + ls[blk].n += tstat.elements; } else { layer_stats temp_ls; - temp_ls.lyr_sum = tstat.sum_values; - temp_ls.lyr_zd = zd; - temp_ls.n = tstat.elements; - ls[blk] = temp_ls; + temp_ls.layer_sum = tstat.sum_values; + temp_ls.layer_zd = zd; + temp_ls.n = tstat.elements; + ls[blk] = temp_ls; } } - std::map lyr_cossim; - std::map lyr_l2_norm; - compute_layer_statistics(ts, lyr_cossim, lyr_l2_norm, g_collector.get_mstats()); + std::map layer_cossim; + std::map layer_l2_norm; + compute_layer_statistics(ts, layer_cossim, layer_l2_norm, g_collector.get_mstats()); const auto layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing layer statistics (%ld layers)\n", layers); @@ -1379,23 +1381,23 @@ static bool show_statistics(const common_params & params) { } for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) continue; - const auto lcs = lyr_cossim.find(layer); - const float lyr_cs = lcs != lyr_cossim.end() ? lcs->second : 0.0f; - const auto ll2n = lyr_l2_norm.find(layer); - const float lyr_l2n = ll2n != lyr_l2_norm.end() ? ll2n->second : 0.0f; + const auto lcs = layer_cossim.find(layer); + const float layer_cs = lcs != layer_cossim.end() ? lcs->second : 0.0f; + const auto ll2n = layer_l2_norm.find(layer); + const float layer_l2n = ll2n != layer_l2_norm.end() ? ll2n->second : 0.0f; if (legacy_mode) { LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\n", layer, - stats.lyr_sum, - 100.0f * stats.lyr_zd / stats.n, - lyr_cs); + stats.layer_sum, + 100.0f * stats.layer_zd / stats.n, + layer_cs); } else { LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\t%8.4f\n", layer, - lyr_l2n, - 100.0f * stats.lyr_zd / stats.n, - lyr_cs, - 100.0f * std::exp(-0.01f * lyr_l2n) * std::pow(fabs(lyr_cs), 10.0f)); + layer_l2n, + 100.0f * stats.layer_zd / stats.n, + layer_cs, + 100.0f * std::exp(-0.01f * layer_l2n) * std::pow(fabs(layer_cs), 10.0f)); } } LOG_INF("\n"); From 2e803234f44417090c7879886a48a6b724218ac9 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 17 Aug 2025 08:19:02 +0100 Subject: [PATCH 048/123] Use { and } around conditionally-executed single line statements --- tools/imatrix/imatrix.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index b4df57da9125..eb231ab97c36 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -432,7 +432,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts.resize(n_as, e.counts[0]); } if (e.values.empty()) { - if (activation_statistics()) e.activations.resize(src1->ne[0]*n_as, 0); + if (activation_statistics()) { e.activations.resize(src1->ne[0]*n_as, 0); } e.values.resize(src1->ne[0]*n_as, 0); e.counts.resize(n_as, 0); } @@ -464,7 +464,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts[ex]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { - if (activation_statistics()) e.activations[e_start + j] += x[j]; + if (activation_statistics()) { e.activations[e_start + j] += x[j]; } e.values[e_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[e_start + j])) { LOG_ERR("%f detected in %s\n", (float)e.values[e_start + j], wname.c_str()); @@ -504,7 +504,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * } } if (e.values.empty()) { - if (activation_statistics()) e.activations.resize(src1->ne[0] * n_mat, 0); + if (activation_statistics()) { e.activations.resize(src1->ne[0] * n_mat, 0); } e.values.resize(src1->ne[0] * n_mat, 0); e.counts.resize(1, 0); } @@ -523,7 +523,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * for (int64_t row = 0; row < src1->ne[1]; ++row) { const float * x = (const float *) (data + row * src1->nb[1] + i2 * src1->nb[2] + i3 * src1->nb[3]); for (int64_t j = 0; j < src1->ne[0]; ++j) { - if (activation_statistics()) e.activations[mat_start + j] += x[j]; + if (activation_statistics()) { e.activations[mat_start + j] += x[j]; } e.values[mat_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[j])) { LOG_ERR("%f detected in %s\n", (float)e.values[j], wname.c_str()); From 44ea7ddeac76ae3393a79af3176bde4b45725924 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 17 Aug 2025 08:20:03 +0100 Subject: [PATCH 049/123] Change statement order --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index eb231ab97c36..a3d5b4da076d 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -938,9 +938,9 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { for (const auto & sc : sums_counts_for) { const std::string & name = sc.first; + const struct ggml_tensor * in_sum = std::get<2>(sc.second); const struct ggml_tensor * in_sum2 = std::get<0>(sc.second); const struct ggml_tensor * counts = std::get<1>(sc.second); - const struct ggml_tensor * in_sum = std::get<2>(sc.second); if (!in_sum2 || !counts) { LOG_ERR("%s: mismatched sums and counts for %s\n", __func__, name.c_str()); From 1f72bc157fea2897af1893d497b84fe5cc23565e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 17 Aug 2025 08:35:17 +0100 Subject: [PATCH 050/123] Avoid using if statements with initialiser --- tools/imatrix/imatrix.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index a76962efcc80..52a15ebd8278 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -206,9 +206,8 @@ static bool compute_vector_statistics(std::vector & tstats, c if (e.activations.empty()) { if (sum > 0) { for (const auto act : activations) { - if (const float p = act / sum; p > 0) { - entropy -= p * std::log2(p); - } + const float p = act / sum; + if (p > 0) { entropy -= p * std::log2(p); } } } } else { @@ -231,21 +230,22 @@ static bool compute_vector_statistics(std::vector & tstats, c int zd_score = 0; if (std_deviation > 0.0f) { for (const auto act : activations) { - if (const float z = (act - mean) / std_deviation; std::fabs(z) > 1.0f) zd_score++; + const float z = (act - mean) / std_deviation; + if (std::fabs(z) > 1.0f) { zd_score++; } } } auto & ts = tstats.emplace_back(); - ts.tensor = name; - ts.stats = e; - ts.sum_values = sum; - ts.mean_values = mean; - ts.max_values = max; - ts.min_values = min; - ts.elements = static_cast(activations.size()); - ts.std_deviation = std_deviation; - ts.entropy = entropy; - ts.zd_score = static_cast(zd_score) / ts.elements; + ts.tensor = name; + ts.stats = e; + ts.sum_values = sum; + ts.mean_values = mean; + ts.max_values = max; + ts.min_values = min; + ts.elements = static_cast(activations.size()); + ts.std_deviation = std_deviation; + ts.entropy = entropy; + ts.zd_score = static_cast(zd_score) / ts.elements; return e.activations.empty(); } From 630750fdef2b050fed05fa51670ceb454746be56 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 17 Aug 2025 09:42:18 +0100 Subject: [PATCH 051/123] Validate number of elements if in_sum is present --- tools/imatrix/imatrix.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 52a15ebd8278..1698146ffb71 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -940,11 +940,11 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { for (const auto & sc : sums_counts_for) { const std::string & name = sc.first; - const struct ggml_tensor * in_sum = std::get<2>(sc.second); + const struct ggml_tensor * in_sum = std::get<2>(sc.second); const struct ggml_tensor * in_sum2 = std::get<0>(sc.second); const struct ggml_tensor * counts = std::get<1>(sc.second); - if (!in_sum2 || !counts) { + if (!in_sum2 || !counts || (in_sum != nullptr && ggml_nelements(in_sum) != ggml_nelements(in_sum2))) { LOG_ERR("%s: mismatched sums and counts for %s\n", __func__, name.c_str()); gguf_free(ctx_gguf); ggml_free(ctx); @@ -981,16 +981,12 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { // Recreate the state as expected by save_imatrix() for (int64_t j = 0; j < nval; j++) { + if (in_sum != nullptr) { e.activations[j] += ((const float *) in_sum->data)[j]; } e.values[j] += ((const float *) in_sum2->data)[j]; } for (int64_t j = 0; j < ncounts; j++) { e.counts[j] += std::lround(((const float *) counts->data)[j]); } - if (in_sum != nullptr) { - for (int64_t j = 0; j < nval; j++) { - e.activations[j] += ((const float *) in_sum->data)[j]; - } - } } // TODO: extract into its own method; this is also used by the legacy format From 3e26364a0039dd37e59a3a2d8b6c564fdb519c4d Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 24 Aug 2025 20:22:22 +0100 Subject: [PATCH 052/123] Clarify the nature of the calculated cosine similarity --- tools/imatrix/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index c25fa097a9df..7690a5743865 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -92,7 +92,7 @@ For current versions of `llama-imatrix`, the `--show-statistics` option has two * **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. * **H** *(legacy mode)* / **ECS** *(preferred)*: If legacy, Shannon Entropy defined as $H = -\sum_{i=1}^N p_i \log_2 p_i$. If preferred, *Euclidean-Cosine Score* defined as $ECS = K \cdot e^{-\alpha a} \cdot |b|^{\gamma}$ where `a = L₂ Norm`, `b = Cosine Similarity`, `α = 0.01`, `γ = 10` between this tensor’s elements and those of the previous layer. Higher score means more similarity and lower change. * **ZD**: % of elements whose Z-score is > 1.0 in magnitude (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415) -* **CosSim**: Cosine Similarity between this tensor’s elements and those of the previous layer. +* **CosSim**: Cosine Similarity of the mean activations between this tensor’s elements and those of the previous layer. #### Per layer @@ -100,7 +100,7 @@ Aggregated metrics per block/layer: * **Σ(Act²)** *(legacy mode)* / **L₂ Norm** *(preferred)*: If in legacy mode, the sum of squared activations (sum of Act²) for the layer's concatenated tensors. In preferred mode, the Euclidean Distance (L₂ Norm) between this layer's average concatenated tensor activations the previous layer. * **ZD**: % of this layer's concatenated tensors' elements with |Z| > 1. -* **CosSim**: Cosine Similarity between this layer's concatenated tensors' elements compared and the previous layer’s. +* **CosSim**: Cosine Similarity of the mean activations between this layer's concatenated tensors' elements compared and the previous layer’s. * **ECS** *(preferred only)*: Euclidean-Cosine Score applied to the layer. More information is available in https://github.com/ggml-org/llama.cpp/pull/14891 From 69b351b777c6e87bca34e0dfe1a42c923489e41e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 26 Aug 2025 21:53:01 +0100 Subject: [PATCH 053/123] Add --output-format to usage --- tools/imatrix/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 7690a5743865..714d42f2bd94 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -10,7 +10,7 @@ More information is available in Date: Tue, 26 Aug 2025 21:53:54 +0100 Subject: [PATCH 054/123] Add --output-format to usage --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 1698146ffb71..64d686cb00c2 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -30,7 +30,7 @@ static void print_usage(int, char ** argv) { " -m model.gguf -f some-text.txt [-o imatrix.gguf] [--output-format {gguf,dat}] [--no-ppl] \\\n" " [--process-output] [--chunk 123] [--save-frequency 0] [--output-frequency 10] \\\n" " [--in-file imatrix-prev-0.gguf --in-file imatrix-prev-1.gguf ...] [--parse-special] \\\n" - " [--activation-statistics] [--show-statistics] [...]\n" , argv[0]); + " [--output-format gguf|dat] [--activation-statistics] [--show-statistics] [...]\n" , argv[0]); LOG("\n"); } From 8f1aa7885e7141a2c29b3282089c15b17c27f822 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 31 Aug 2025 14:03:19 +0100 Subject: [PATCH 055/123] Remove activation_statistics() option --- common/arg.cpp | 9 +-------- tools/imatrix/imatrix.cpp | 15 +++++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 469650491b8d..873e16190e96 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2758,16 +2758,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.show_statistics = true; } ).set_examples({LLAMA_EXAMPLE_IMATRIX})); - add_opt(common_arg( - {"--activation-statistics"}, - string_format("generate data to compute activation-based statistics (default: %s)", params.show_statistics ? "true" : "false"), - [](common_params & params) { - params.activation_statistics = true; - } - ).set_examples({LLAMA_EXAMPLE_IMATRIX})); add_opt(common_arg( {"--parse-special"}, - string_format("prase special tokens (chat, tool, etc) (default: %s)", params.parse_special ? "true" : "false"), + string_format("parse special tokens (chat, tool, etc) (default: %s)", params.parse_special ? "true" : "false"), [](common_params & params) { params.parse_special = true; } diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 64d686cb00c2..6ed28e6e9e5f 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -30,7 +30,7 @@ static void print_usage(int, char ** argv) { " -m model.gguf -f some-text.txt [-o imatrix.gguf] [--output-format {gguf,dat}] [--no-ppl] \\\n" " [--process-output] [--chunk 123] [--save-frequency 0] [--output-frequency 10] \\\n" " [--in-file imatrix-prev-0.gguf --in-file imatrix-prev-1.gguf ...] [--parse-special] \\\n" - " [--output-format gguf|dat] [--activation-statistics] [--show-statistics] [...]\n" , argv[0]); + " [--output-format gguf|dat] [--show-statistics] [...]\n" , argv[0]); LOG("\n"); } @@ -63,7 +63,6 @@ class IMatrixCollector { public: IMatrixCollector() = default; void set_params(common_params params) { m_params = std::move(params); } - bool activation_statistics() const { return m_params.activation_statistics; } bool collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data); void save_imatrix_legacy(int32_t ncall = -1) const; void save_imatrix(int32_t n_chunk = -1) const; @@ -434,7 +433,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts.resize(n_as, e.counts[0]); } if (e.values.empty()) { - if (activation_statistics()) { e.activations.resize(src1->ne[0]*n_as, 0); } + e.activations.resize(src1->ne[0]*n_as, 0); e.values.resize(src1->ne[0]*n_as, 0); e.counts.resize(n_as, 0); } @@ -466,7 +465,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * e.counts[ex]++; for (int64_t j = 0; j < src1->ne[0]; ++j) { - if (activation_statistics()) { e.activations[e_start + j] += x[j]; } + e.activations[e_start + j] += x[j]; e.values[e_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[e_start + j])) { LOG_ERR("%f detected in %s\n", (float)e.values[e_start + j], wname.c_str()); @@ -506,7 +505,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * } } if (e.values.empty()) { - if (activation_statistics()) { e.activations.resize(src1->ne[0] * n_mat, 0); } + e.activations.resize(src1->ne[0] * n_mat, 0); e.values.resize(src1->ne[0] * n_mat, 0); e.counts.resize(1, 0); } @@ -525,7 +524,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * for (int64_t row = 0; row < src1->ne[1]; ++row) { const float * x = (const float *) (data + row * src1->nb[1] + i2 * src1->nb[2] + i3 * src1->nb[3]); for (int64_t j = 0; j < src1->ne[0]; ++j) { - if (activation_statistics()) { e.activations[mat_start + j] += x[j]; } + e.activations[mat_start + j] += x[j]; e.values[mat_start + j] += x[j] * x[j]; if (!std::isfinite((float)e.values[j])) { LOG_ERR("%f detected in %s\n", (float)e.values[j], wname.c_str()); @@ -707,7 +706,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { } to_store.push_back(kv.first); - if (activation_statistics()) { data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); } + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); } @@ -761,7 +760,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum2); gguf_add_tensor(ctx_gguf, counts); - if (!stat.activations.empty() && activation_statistics()) { + if (!stat.activations.empty()) { const int32_t nact = (int32_t) stat.activations.size(); struct ggml_tensor * in_sum = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nact / nmat, nmat); ggml_format_name(in_sum, "%s.in_sum", name.c_str()); From 8d0e276f96b7fbe49ada00d23d8480662016d169 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 31 Aug 2025 14:56:10 +0100 Subject: [PATCH 056/123] Update README.md --- tools/imatrix/README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 714d42f2bd94..e823d25c19ae 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -10,7 +10,7 @@ More information is available in Date: Tue, 28 Oct 2025 18:27:19 +0000 Subject: [PATCH 057/123] Fix legacy_mode getting overwritten on each tensor bug --- tools/imatrix/imatrix.cpp | 93 +++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6ed28e6e9e5f..6935e44255ad 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1260,15 +1260,16 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c static bool show_statistics(const common_params & params) { std::vector ts; - bool legacy_mode = false; + bool legacy_mode = true; if (params.in_files.empty() || params.in_files.size() > 1) { LOG_ERR("\nError: a single imatrix file is required to compute tensor statistics\n\n"); return false; } if (g_collector.load_imatrix(params.in_files[0].c_str())) { - for (const auto & [name, stats] :g_collector.get_mstats()) { - legacy_mode = compute_vector_statistics(ts, name, stats); + for (const auto & [name, stats] : g_collector.get_mstats()) { + const bool is_legacy = compute_vector_statistics(ts, name, stats); + legacy_mode = legacy_mode && is_legacy; } } else { LOG_ERR("\nError: %s is not a valid imatrix file\n\n", params.in_files[0].c_str()); @@ -1286,19 +1287,21 @@ static bool show_statistics(const common_params & params) { explicit tensor_comparer(const bool legacy) : legacy_mode(legacy) {} bool operator()(const tensor_statistics & a, const tensor_statistics & b) const { - std::string layer, name_a, name_b; + std::string layer; + std::string name_a; + std::string name_b; process_tensor_name(a.tensor, layer, name_a); process_tensor_name(b.tensor, layer, name_b); - return legacy_mode ? name_a < name_b || (name_a == name_b && a.sum_values > b.sum_values) : - name_a < name_b || (name_a == name_b && a.cossim > b.cossim); + return legacy_mode ? name_a < name_b || (name_a == name_b && a.sum_values > b.sum_values) + : name_a < name_b || (name_a == name_b && a.cossim > b.cossim); } }; std::sort(ts.begin(), ts.end(), tensor_comparer(legacy_mode)); struct layer_stats { float layer_sum = 0.0f; - float layer_zd = 0.0f; - int n = 0; + float layer_zd = 0.0f; + int n = 0; }; std::map ls; @@ -1319,6 +1322,7 @@ static bool show_statistics(const common_params & params) { LOG_INF( "==============================================================================================================" "=============================================================\n"); + for (const auto & tstat : ts) { std::string layer, name; process_tensor_name(tstat.tensor, layer, name); @@ -1326,36 +1330,37 @@ static bool show_statistics(const common_params & params) { int blk; try { blk = std::stoi(layer); - } catch (const std::exception & e) { - blk = -1; // not a block layer + } catch (const std::exception &) { + blk = -1; // not a block layer } - LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", - layer.c_str(), - name.c_str(), - legacy_mode ? tstat.sum_values : tstat.l2_norm, - tstat.min_values, - tstat.max_values, - tstat.mean_values, - tstat.std_deviation, - tstat.elements, - 100.0f * (tstat.entropy / std::log2(tstat.elements)), - legacy_mode ? tstat.entropy : 100.0f * std::exp(-0.01f * tstat.l2_norm) * std::pow(fabs(tstat.cossim), 10.0f), - 100.0f * tstat.zd_score, - tstat.cossim); - - const float zd = tstat.elements * tstat.zd_score; + const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; + const float ecs = 100.0f * std::exp(-0.01f * tstat.l2_norm) * std::pow(std::fabs(tstat.cossim), 10.0f); + LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", + layer.c_str(), name.c_str(), + legacy_mode ? tstat.sum_values : tstat.l2_norm, + tstat.min_values, + tstat.max_values, + tstat.mean_values, + tstat.std_deviation, + tstat.elements, + h_norm, + legacy_mode ? tstat.entropy : ecs, + 100.0f * tstat.zd_score, + tstat.cossim); + + const float zd = tstat.elements * tstat.zd_score; if (ls.find(blk) != ls.end()) { ls[blk].layer_sum += tstat.sum_values; - ls[blk].layer_zd += zd; - ls[blk].n += tstat.elements; + ls[blk].layer_zd += zd; + ls[blk].n += tstat.elements; } else { layer_stats temp_ls; temp_ls.layer_sum = tstat.sum_values; - temp_ls.layer_zd = zd; - temp_ls.n = tstat.elements; - ls[blk] = temp_ls; + temp_ls.layer_zd = zd; + temp_ls.n = tstat.elements; + ls[blk] = temp_ls; } } @@ -1366,11 +1371,11 @@ static bool show_statistics(const common_params & params) { const auto layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing layer statistics (%ld layers)\n", layers); LOG_INF("\n%6s\t%13s\t%6s\t%11s\t%6s\n", - "Layer", - legacy_mode ? "Σ(Act²)" : "L₂ Norm", - "ZD", - "CosSim", - legacy_mode ? "" : "ECS"); + "Layer", + legacy_mode ? "Σ(Act²)" : "L₂ Norm", + "ZD", + "CosSim", + legacy_mode ? "" : "ECS"); if (legacy_mode) { LOG_INF("============================================\n"); } else { @@ -1384,17 +1389,17 @@ static bool show_statistics(const common_params & params) { const float layer_l2n = ll2n != layer_l2_norm.end() ? ll2n->second : 0.0f; if (legacy_mode) { LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\n", - layer, - stats.layer_sum, - 100.0f * stats.layer_zd / stats.n, - layer_cs); + layer, + stats.layer_sum, + 100.0f * stats.layer_zd / stats.n, + layer_cs); } else { LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\t%8.4f\n", - layer, - layer_l2n, - 100.0f * stats.layer_zd / stats.n, - layer_cs, - 100.0f * std::exp(-0.01f * layer_l2n) * std::pow(fabs(layer_cs), 10.0f)); + layer, + layer_l2n, + 100.0f * stats.layer_zd / stats.n, + layer_cs, + 100.0f * std::exp(-0.01f * layer_l2n) * std::pow(std::fabs(layer_cs), 10.0f)); } } LOG_INF("\n"); From c9a0874f35a4c8e2681b0f4fc59c27f093a10e2f Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 18:29:59 +0000 Subject: [PATCH 058/123] Clamp CosSim to [-1, 1] to avoid float drift --- tools/imatrix/imatrix.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6935e44255ad..d047d0302efe 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -258,12 +258,12 @@ static void compute_tensor_statistics(std::vector & tstats) { if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { const int blk = std::stoi(match[1]); - if (blk <= 0) continue; + if (blk <= 0) { continue; } std::string tname(ts.tensor); tname.replace(match.position(1), match.length(1), std::to_string(blk-1)); auto prev = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); - if (prev == tstats.end()) continue; + if (prev == tstats.end()) { continue; } const auto curr_avg = compute_tensor_averages(ts.stats); const auto prev_avg = compute_tensor_averages(prev->stats); if (curr_avg.size() == prev_avg.size() && !curr_avg.empty()) { @@ -275,7 +275,12 @@ static void compute_tensor_statistics(std::vector & tstats) { vec1 += curr_avg[i] * curr_avg[i]; vec2 += prev_avg[i] * prev_avg[i]; } - if (vec1 > 0 && vec2 > 0) ts.cossim = dot_prod / (std::sqrt(vec1) * std::sqrt(vec2)); + if (vec1 > 0 && vec2 > 0) { + float cs = dot_prod / (std::sqrt(vec1) * std::sqrt(vec2)); + cs = std::min(cs, 1.0f); + cs = std::max(cs, -1.0f); + ts.cossim = cs; + } } } } @@ -283,19 +288,19 @@ static void compute_tensor_statistics(std::vector & tstats) { // compute the L2 Norm (Euclidian Distance) between the same tensors in consecutive layers for (auto & ts : tstats) { ts.l2_norm = 0.0f; - if (ts.stats.activations.empty()) continue; + if (ts.stats.activations.empty()) { continue; } if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { const int blk = std::stoi(match[1]); - if (blk <= 0) continue; + if (blk <= 0) { continue; } std::string tname(ts.tensor); tname.replace(match.position(1), match.length(1), std::to_string(blk - 1)); auto prev = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); - if (prev == tstats.end()) continue; + if (prev == tstats.end()) { continue; } const auto cur_avg = compute_tensor_averages(ts.stats); const auto prev_avg = compute_tensor_averages(prev->stats); - if (cur_avg.empty() || prev_avg.empty() || cur_avg.size() != prev_avg.size()) continue; + if (cur_avg.empty() || prev_avg.empty() || cur_avg.size() != prev_avg.size()) { continue; } float dist = 0.0; for (size_t i = 0; i < cur_avg.size(); ++i) { From 637e674da680fa333d3713a9247d99d89da5eeb4 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 18:33:37 +0000 Subject: [PATCH 059/123] Avoid division by zero on zero-count matrices --- tools/imatrix/imatrix.cpp | 52 ++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index d047d0302efe..a7b9f644ab48 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -179,28 +179,40 @@ static bool compute_vector_statistics(std::vector & tstats, c activations.reserve(e.values.size()); for (int i = 0; i < n_mat; ++i) { + const float c = (float)e.counts[i]; + const size_t off = i * row_size; for (int j = 0; j < row_size; ++j) { - activations.push_back(e.values[i*row_size + j] / e.counts[i]); + if (c <= 0.0f) { + activations.push_back(1.0f); // same as legacy + } else { + activations.push_back(e.values[off + j] / c); + } } } } else { activations.reserve(e.activations.size()); for (int i = 0; i < n_mat; ++i) { + const float c = (float)e.counts[i]; + const size_t off = i * row_size; for (int j = 0; j < row_size; ++j) { - activations.push_back(e.activations[i*row_size + j] / e.counts[i]); + if (c <= 0.0f) { + activations.push_back(1.0f); // same as legacy + } else { + activations.push_back(e.activations[off + j] / c); + } } } } - const float sum = std::accumulate(activations.begin(), activations.end(), 0.0f); - const float max = *std::max_element(activations.begin(), activations.end()); - const float min = *std::min_element(activations.begin(), activations.end()); - const float mean = sum / activations.size(); - const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); - const float variance = (sqr_sum / activations.size()) - (mean * mean); - const float std_deviation = std::sqrt(std::max(0.0f, variance)); - float entropy = 0; + const float sum = std::accumulate(activations.begin(), activations.end(), 0.0f); + const float max = * std::max_element(activations.begin(), activations.end()); + const float min = * std::min_element(activations.begin(), activations.end()); + const float mean = sum / activations.size(); + const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); + const float variance = sqr_sum / activations.size() - mean * mean; + const float std_deviation = std::sqrt(std::max(0.0f, variance)); + float entropy = 0; if (e.activations.empty()) { if (sum > 0) { @@ -226,7 +238,7 @@ static bool compute_vector_statistics(std::vector & tstats, c } } - int zd_score = 0; + float zd_score = 0.0f; if (std_deviation > 0.0f) { for (const auto act : activations) { const float z = (act - mean) / std_deviation; @@ -235,16 +247,16 @@ static bool compute_vector_statistics(std::vector & tstats, c } auto & ts = tstats.emplace_back(); - ts.tensor = name; - ts.stats = e; - ts.sum_values = sum; - ts.mean_values = mean; - ts.max_values = max; - ts.min_values = min; - ts.elements = static_cast(activations.size()); + ts.tensor = name; + ts.stats = e; + ts.sum_values = sum; + ts.mean_values = mean; + ts.max_values = max; + ts.min_values = min; + ts.elements = static_cast(activations.size()); ts.std_deviation = std_deviation; - ts.entropy = entropy; - ts.zd_score = static_cast(zd_score) / ts.elements; + ts.entropy = entropy; + ts.zd_score = zd_score / ts.elements; return e.activations.empty(); } From 683ef8dfb7ea76b4a339ca2af69ce421b073a2e4 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 18:35:17 +0000 Subject: [PATCH 060/123] Fill zeros for experts with zero counts to preserve shape --- tools/imatrix/imatrix.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index a7b9f644ab48..ac02b53dfcd1 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -132,25 +132,34 @@ static std::vector compute_tensor_averages(const Stats & tstats) { const size_t n_mat = tstats.counts.size(); const size_t len = !tstats.activations.empty() ? tstats.activations.size() : tstats.values.size(); - if (len == 0 || len % n_mat != 0) return {}; + if (len == 0 || n_mat == 0 || len % n_mat != 0) { return {}; } + const size_t row = len / n_mat; std::vector vec; vec.reserve(len); if (tstats.activations.empty()) { + // Use mean of squares; fill zeros for experts with zero counts to preserve shape for (size_t m = 0; m < n_mat; ++m) { const float c = (float)tstats.counts[m]; - if (c <= 0) return {}; const size_t off = m * row; + if (c <= 0.0f) { + vec.insert(vec.end(), row, 0.0f); + continue; + } for (size_t j = 0; j < row; ++j) { vec.push_back(tstats.values[off + j] / c); } } } else { + // Use mean; fill zeros for experts with zero counts to preserve shape for (size_t m = 0; m < n_mat; ++m) { const float c = (float)tstats.counts[m]; - if (c <= 0) return {}; const size_t off = m * row; + if (c <= 0.0f) { + vec.insert(vec.end(), row, 0.0f); + continue; + } for (size_t j = 0; j < row; ++j) { vec.push_back(tstats.activations[off + j] / c); } From dc4a04b5c5a4b68b45eb673900bde42423340b1e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 21:35:35 +0000 Subject: [PATCH 061/123] Adjust size calculation and change fallback value to 0.0f --- tools/imatrix/imatrix.cpp | 54 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ac02b53dfcd1..ad07cf99e232 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -170,43 +170,43 @@ static std::vector compute_tensor_averages(const Stats & tstats) { } static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { - if (e.values.size() % e.counts.size() != 0) { - LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.values.size()); + const size_t n_mat = e.counts.size(); + const size_t len = e.activations.empty() ? e.values.size() : e.activations.size(); + + if (n_mat == 0) { + LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); return false; } - if (e.counts.empty()) { - LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); + + if (len == 0 || (len % n_mat) != 0) { + LOG_ERR("%s: activation size mismatch for tensor %s (len=%zu, counts=%zu)\n", __func__, name.c_str(), len, n_mat); return false; } - const int n_mat = e.counts.size(); - const int row_size = e.values.size() / n_mat; + const int row_size = (int)(len / n_mat); std::vector activations; + activations.reserve(len); if (e.activations.empty()) { - activations.reserve(e.values.size()); - - for (int i = 0; i < n_mat; ++i) { - const float c = (float)e.counts[i]; + for (size_t i = 0; i < n_mat; ++i) { + const auto c = (float)e.counts[i]; const size_t off = i * row_size; for (int j = 0; j < row_size; ++j) { if (c <= 0.0f) { - activations.push_back(1.0f); // same as legacy + activations.push_back(0.0f); } else { activations.push_back(e.values[off + j] / c); } } } } else { - activations.reserve(e.activations.size()); - - for (int i = 0; i < n_mat; ++i) { - const float c = (float)e.counts[i]; + for (size_t i = 0; i < n_mat; ++i) { + const auto c = (float)e.counts[i]; const size_t off = i * row_size; for (int j = 0; j < row_size; ++j) { if (c <= 0.0f) { - activations.push_back(1.0f); // same as legacy + activations.push_back(0.0f); } else { activations.push_back(e.activations[off + j] / c); } @@ -214,6 +214,11 @@ static bool compute_vector_statistics(std::vector & tstats, c } } + if (activations.empty()) { + LOG_ERR("%s: computed empty activation vector for tensor %s\n", __func__, name.c_str()); + return false; + } + const float sum = std::accumulate(activations.begin(), activations.end(), 0.0f); const float max = * std::max_element(activations.begin(), activations.end()); const float min = * std::min_element(activations.begin(), activations.end()); @@ -221,28 +226,29 @@ static bool compute_vector_statistics(std::vector & tstats, c const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); const float variance = sqr_sum / activations.size() - mean * mean; const float std_deviation = std::sqrt(std::max(0.0f, variance)); - float entropy = 0; + float entropy = 0.0f; if (e.activations.empty()) { - if (sum > 0) { + // classic entropy on normalized activations distribution + if (sum > 0.0f) { for (const auto act : activations) { const float p = act / sum; - if (p > 0) { entropy -= p * std::log2(p); } + if (p > 0.0f) { entropy -= p * std::log2(p); } } } } else { - float div = 0.0; + // entropy on normalized squared weights + float div = 0.0f; std::vector weights(activations.size()); for (size_t i = 0; i < activations.size(); ++i) { const float w = activations[i] * activations[i]; weights[i] = w; div += w; } - - if (div > 0.0) { - for (float w : weights) { + if (div > 0.0f) { + for (const float w : weights) { const float p = w / div; - if (p > 0.0) { entropy -= p * std::log2(p); } + if (p > 0.0f) { entropy -= p * std::log2(p); } } } } From 0b0381c94c675f0750161350876a763fcc11f2b8 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 21:41:31 +0000 Subject: [PATCH 062/123] Merge Cosine Similarity and L2 Norm computation into single loop --- tools/imatrix/imatrix.cpp | 140 +++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 76 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ad07cf99e232..81be67d09520 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -278,63 +278,48 @@ static bool compute_vector_statistics(std::vector & tstats, c static void compute_tensor_statistics(std::vector & tstats) { static const std::regex pattern(R"(blk\.(\d+)\.)"); - - // compute the Cosine Similarity between the same tensors in consecutive layers for (auto & ts : tstats) { - ts.cossim = 0; + ts.cossim = 0.0f; + ts.l2_norm = 0.0f; if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { const int blk = std::stoi(match[1]); if (blk <= 0) { continue; } std::string tname(ts.tensor); - tname.replace(match.position(1), match.length(1), std::to_string(blk-1)); - auto prev = std::find_if(tstats.begin(), tstats.end(), + tname.replace(match.position(1), match.length(1), std::to_string(blk - 1)); + auto prev_it = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); - if (prev == tstats.end()) { continue; } + if (prev_it == tstats.end()) { continue; } + const auto curr_avg = compute_tensor_averages(ts.stats); - const auto prev_avg = compute_tensor_averages(prev->stats); - if (curr_avg.size() == prev_avg.size() && !curr_avg.empty()) { - float dot_prod = 0.0f; - float vec1 = 0.0f; - float vec2 = 0.0f; - for (size_t i = 0; i < curr_avg.size(); ++i) { - dot_prod += curr_avg[i] * prev_avg[i]; - vec1 += curr_avg[i] * curr_avg[i]; - vec2 += prev_avg[i] * prev_avg[i]; - } - if (vec1 > 0 && vec2 > 0) { - float cs = dot_prod / (std::sqrt(vec1) * std::sqrt(vec2)); - cs = std::min(cs, 1.0f); - cs = std::max(cs, -1.0f); - ts.cossim = cs; - } + const auto prev_avg = compute_tensor_averages(prev_it->stats); + if (curr_avg.empty() || curr_avg.size() != prev_avg.size()) { continue; } + + float dot_prod = 0.0f; + float norm1_sq = 0.0f; + float norm2_sq = 0.0f; + float l2_dist_sq = 0.0f; + + for (size_t i = 0; i < curr_avg.size(); ++i) { + const float c_val = curr_avg[i]; + const float p_val = prev_avg[i]; + dot_prod += c_val * p_val; + norm1_sq += c_val * c_val; + norm2_sq += p_val * p_val; + const float diff = c_val - p_val; + l2_dist_sq += diff * diff; } - } - } - - // compute the L2 Norm (Euclidian Distance) between the same tensors in consecutive layers - for (auto & ts : tstats) { - ts.l2_norm = 0.0f; - if (ts.stats.activations.empty()) { continue; } - if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { - const int blk = std::stoi(match[1]); - if (blk <= 0) { continue; } - std::string tname(ts.tensor); - tname.replace(match.position(1), match.length(1), std::to_string(blk - 1)); - auto prev = std::find_if(tstats.begin(), tstats.end(), - [tname](const tensor_statistics & t) { return t.tensor == tname; }); - if (prev == tstats.end()) { continue; } - const auto cur_avg = compute_tensor_averages(ts.stats); - const auto prev_avg = compute_tensor_averages(prev->stats); - if (cur_avg.empty() || prev_avg.empty() || cur_avg.size() != prev_avg.size()) { continue; } - - float dist = 0.0; - for (size_t i = 0; i < cur_avg.size(); ++i) { - const float act = cur_avg[i] - prev_avg[i]; - dist += act * act; + // Compute Cosine Similarity + if (norm1_sq > 0.0f && norm2_sq > 0.0f) { + float cs = dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq)); + cs = std::min(cs, 1.0f); + cs = std::max(cs, -1.0f); + ts.cossim = cs; } - ts.l2_norm = std::sqrt(dist); + + // Compute L2 Norm (Euclidean Distance) + ts.l2_norm = std::sqrt(l2_dist_sq); } } } @@ -347,56 +332,58 @@ static void compute_layer_statistics(const std::vector & tsta std::vector curr_avg; std::vector prev_avg; }; + static const std::regex pattern(R"(blk\.(\d+)\.)"); std::unordered_map tidx; tidx.reserve(tstats.size()); for (const auto & ts : tstats) { tidx[ts.tensor] = &ts; } - std::map taggr; + std::map agr; for (const auto & ts : tstats) { std::smatch match; - if (!std::regex_search(ts.tensor, match, pattern)) continue; + if (!std::regex_search(ts.tensor, match, pattern)) { continue; } const int blk = std::stoi(match[1]); - if (blk <= 0) continue; + if (blk <= 0) { continue; } std::string prev_lyr(ts.tensor); prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk-1)); - if (auto it_prev = tidx.find(prev_lyr); it_prev == tidx.end()) continue; + if (auto it_prev = tidx.find(prev_lyr); it_prev == tidx.end()) { continue; } const auto curr_avg = compute_tensor_averages(stats_map.at(ts.tensor)); const auto prev_avg = compute_tensor_averages(stats_map.at(prev_lyr)); - if (curr_avg.empty() || prev_avg.empty() || curr_avg.size() != prev_avg.size()) continue; - auto & [curr, prev] = taggr[blk]; + if (curr_avg.empty() || prev_avg.empty() || curr_avg.size() != prev_avg.size()) { continue; } + auto & [curr, prev] = agr[blk]; curr.insert(curr.end(), curr_avg.begin(), curr_avg.end()); prev.insert(prev.end(), prev_avg.begin(), prev_avg.end()); } - // compute the aggregated Cosine Similarity between consecutive layers - for (auto & kv : taggr) { + for (auto & kv : agr) { const auto & curr = kv.second.curr_avg; const auto & prev = kv.second.prev_avg; - if (curr.size() != prev.size() || curr.empty()) continue; - float dot_prod = 0.0; - float layer1 = 0.0; - float layer2 = 0.0; + if (curr.size() != prev.size() || curr.empty()) { continue; } + + float dot_prod = 0.0f; + float norm1_sq = 0.0f; + float norm2_sq = 0.0f; + float l2_dist_sq = 0.0f; + for (size_t i = 0; i < curr.size(); ++i) { - dot_prod += curr[i] * prev[i]; - layer1 += curr[i] * curr[i]; - layer2 += prev[i] * prev[i]; + const float c_val = curr[i]; + const float p_val = prev[i]; + dot_prod += c_val * p_val; + norm1_sq += c_val * c_val; + norm2_sq += p_val * p_val; + const float diff = c_val - p_val; + l2_dist_sq += diff * diff; } + + // Compute aggregated Cosine Similarity float cossim = 0.0f; - if (layer1 > 0.0 && layer2 > 0.0) cossim = dot_prod / (std::sqrt(layer1) * std::sqrt(layer2)); + if (norm1_sq > 0.0f && norm2_sq > 0.0f) { + cossim = dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq)); + } layer_cossim[kv.first] = cossim; - } - // compute the aggregated L2 Norm (Euclidian Distance) between consecutive layers - for (auto & kv : taggr) { - const auto & curr = kv.second.curr_avg; - const auto & prev = kv.second.prev_avg; - if (curr.size() != prev.size() || curr.empty()) continue; - float dist = 0.0f; - for (size_t i = 0; i < curr.size(); ++i) { - dist += (curr[i] - prev[i]) * (curr[i] - prev[i]); - } - layer_l2_norm[kv.first] = std::sqrt(dist); + // Compute aggregated L2 Norm (Euclidean Distance) + layer_l2_norm[kv.first] = std::sqrt(l2_dist_sq); } } @@ -1370,7 +1357,8 @@ static bool show_statistics(const common_params & params) { const float ecs = 100.0f * std::exp(-0.01f * tstat.l2_norm) * std::pow(std::fabs(tstat.cossim), 10.0f); LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", - layer.c_str(), name.c_str(), + layer.c_str(), + name.c_str(), legacy_mode ? tstat.sum_values : tstat.l2_norm, tstat.min_values, tstat.max_values, @@ -1400,8 +1388,8 @@ static bool show_statistics(const common_params & params) { std::map layer_l2_norm; compute_layer_statistics(ts, layer_cossim, layer_l2_norm, g_collector.get_mstats()); - const auto layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); - LOG_INF("\nComputing layer statistics (%ld layers)\n", layers); + const size_t layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); + LOG_INF("\nComputing layer statistics (%zu layers)\n", layers); LOG_INF("\n%6s\t%13s\t%6s\t%11s\t%6s\n", "Layer", legacy_mode ? "Σ(Act²)" : "L₂ Norm", From b5068df804952f5a869ed209ab5f4c4137b141ba Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 23:03:52 +0000 Subject: [PATCH 063/123] Minor refactoring --- tools/imatrix/imatrix.cpp | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 81be67d09520..1900c37ac6c5 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -128,41 +128,35 @@ static void process_tensor_name(const std::string & input, std::string & layer, } static std::vector compute_tensor_averages(const Stats & tstats) { - if (tstats.counts.empty()) return {}; + if (tstats.counts.empty()) { return {}; } const size_t n_mat = tstats.counts.size(); const size_t len = !tstats.activations.empty() ? tstats.activations.size() : tstats.values.size(); - if (len == 0 || n_mat == 0 || len % n_mat != 0) { return {}; } - const size_t row = len / n_mat; std::vector vec; vec.reserve(len); if (tstats.activations.empty()) { - // Use mean of squares; fill zeros for experts with zero counts to preserve shape + // Mean of squares for (size_t m = 0; m < n_mat; ++m) { - const float c = (float)tstats.counts[m]; + const auto c = (float)tstats.counts[m]; const size_t off = m * row; if (c <= 0.0f) { - vec.insert(vec.end(), row, 0.0f); + vec.insert(vec.end(), row, 0.0f); // zero-fill rows for experts with zero count to preserve shape continue; } - for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.values[off + j] / c); - } + for (size_t j = 0; j < row; ++j) { vec.push_back(tstats.values[off + j] / c); } } } else { - // Use mean; fill zeros for experts with zero counts to preserve shape + // Mean for (size_t m = 0; m < n_mat; ++m) { - const float c = (float)tstats.counts[m]; + const float c = (float) tstats.counts[m]; const size_t off = m * row; if (c <= 0.0f) { - vec.insert(vec.end(), row, 0.0f); + vec.insert(vec.end(), row, 0.0f); // zero-fill rows for experts with zero count to preserve shape continue; } - for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.activations[off + j] / c); - } + for (size_t j = 0; j < row; ++j) { vec.push_back(tstats.activations[off + j] / c); } } } From 92a42bac3dc61dde21d479551e5957dd2e5851b4 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 23:06:29 +0000 Subject: [PATCH 064/123] Type refactoring --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 1900c37ac6c5..1edc39835ca9 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -150,7 +150,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { } else { // Mean for (size_t m = 0; m < n_mat; ++m) { - const float c = (float) tstats.counts[m]; + const float c = (float)tstats.counts[m]; const size_t off = m * row; if (c <= 0.0f) { vec.insert(vec.end(), row, 0.0f); // zero-fill rows for experts with zero count to preserve shape From ab015065b8779061a096b544ceaaa40e88217f4f Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 23:10:12 +0000 Subject: [PATCH 065/123] Minor refactoring --- tools/imatrix/imatrix.cpp | 109 ++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 58 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 1edc39835ca9..304ccfa96495 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -166,44 +166,33 @@ static std::vector compute_tensor_averages(const Stats & tstats) { static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { const size_t n_mat = e.counts.size(); const size_t len = e.activations.empty() ? e.values.size() : e.activations.size(); - if (n_mat == 0) { LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); return false; } - if (len == 0 || (len % n_mat) != 0) { LOG_ERR("%s: activation size mismatch for tensor %s (len=%zu, counts=%zu)\n", __func__, name.c_str(), len, n_mat); return false; } - const int row_size = (int)(len / n_mat); - + const size_t row_size = len / n_mat; std::vector activations; activations.reserve(len); - if (e.activations.empty()) { - for (size_t i = 0; i < n_mat; ++i) { - const auto c = (float)e.counts[i]; - const size_t off = i * row_size; - for (int j = 0; j < row_size; ++j) { - if (c <= 0.0f) { - activations.push_back(0.0f); - } else { - activations.push_back(e.values[off + j] / c); - } - } + for (size_t i = 0; i < n_mat; ++i) { + const auto c = (float)e.counts[i]; + const size_t off = i * row_size; + if (c <= 0.0f) { + activations.insert(activations.end(), row_size, 0.0f); + continue; } - } else { - for (size_t i = 0; i < n_mat; ++i) { - const auto c = (float)e.counts[i]; - const size_t off = i * row_size; - for (int j = 0; j < row_size; ++j) { - if (c <= 0.0f) { - activations.push_back(0.0f); - } else { - activations.push_back(e.activations[off + j] / c); - } + if (e.activations.empty()) { + for (size_t j = 0; j < row_size; ++j) { + activations.push_back(e.values[off + j] / c); // mean-of-squares + } + } else { + for (size_t j = 0; j < row_size; ++j) { + activations.push_back(e.activations[off + j] / c); // mean } } } @@ -213,59 +202,63 @@ static bool compute_vector_statistics(std::vector & tstats, c return false; } - const float sum = std::accumulate(activations.begin(), activations.end(), 0.0f); - const float max = * std::max_element(activations.begin(), activations.end()); - const float min = * std::min_element(activations.begin(), activations.end()); - const float mean = sum / activations.size(); - const float sqr_sum = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); - const float variance = sqr_sum / activations.size() - mean * mean; - const float std_deviation = std::sqrt(std::max(0.0f, variance)); + double sum = 0.0; + float vmax = activations[0]; + float vmin = activations[0]; + for (float v : activations) { + sum += v; + vmax = std::max(vmax, v); + vmin = std::min(vmin, v); + } + + const auto mean = (float)(sum / (double)activations.size()); + double sqr_sum = 0.0; + for (const float v : activations) { sqr_sum += (double)v * (double)v; } + double variance = sqr_sum / (double)activations.size() - (double)mean * (double)mean; + if (variance < 0.0) { variance = 0.0; } + const float std_deviation = std::sqrt((float)variance); float entropy = 0.0f; if (e.activations.empty()) { - // classic entropy on normalized activations distribution - if (sum > 0.0f) { - for (const auto act : activations) { - const float p = act / sum; - if (p > 0.0f) { entropy -= p * std::log2(p); } + double energy_sum = 0.0; + for (float v : activations) { energy_sum += (double)std::max(0.0f, v); } + if (energy_sum > 0.0) { + for (const float v : activations) { + const double p = std::max(0.0, (double)v) / energy_sum; + if (p > 0.0) { entropy -= (float)(p * std::log2(p)); } } } } else { - // entropy on normalized squared weights - float div = 0.0f; - std::vector weights(activations.size()); - for (size_t i = 0; i < activations.size(); ++i) { - const float w = activations[i] * activations[i]; - weights[i] = w; - div += w; - } - if (div > 0.0f) { - for (const float w : weights) { - const float p = w / div; - if (p > 0.0f) { entropy -= p * std::log2(p); } + double energy_sum = 0.0; + for (const float v : activations) { energy_sum += (double)v * (double)v; } + if (energy_sum > 0.0) { + for (const float v : activations) { + const double p = (double)v * (double)v / energy_sum; + if (p > 0.0) { entropy -= (float)(p * std::log2(p)); } } } } - float zd_score = 0.0f; + // ZD score: fraction with |z| > 1 + double zd_count = 0.0; if (std_deviation > 0.0f) { - for (const auto act : activations) { - const float z = (act - mean) / std_deviation; - if (std::fabs(z) > 1.0f) { zd_score++; } + for (const float v : activations) { + const float z = (v - mean) / std_deviation; + if (std::fabs(z) > 1.0f) { zd_count += 1.0; } } } auto & ts = tstats.emplace_back(); ts.tensor = name; ts.stats = e; - ts.sum_values = sum; + ts.sum_values = (float)sum; ts.mean_values = mean; - ts.max_values = max; - ts.min_values = min; - ts.elements = static_cast(activations.size()); + ts.max_values = vmax; + ts.min_values = vmin; + ts.elements = (int)activations.size(); ts.std_deviation = std_deviation; ts.entropy = entropy; - ts.zd_score = zd_score / ts.elements; + ts.zd_score = ts.elements > 0 ? (float)(zd_count / (double)ts.elements) : 0.0f; return e.activations.empty(); } From 86fabce58d2804b8593d0f387af65aaf7214b6b9 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Tue, 28 Oct 2025 23:10:44 +0000 Subject: [PATCH 066/123] Clamp values --- tools/imatrix/imatrix.cpp | 51 ++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 304ccfa96495..f529b39e0fec 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -298,12 +298,15 @@ static void compute_tensor_statistics(std::vector & tstats) { } // Compute Cosine Similarity + float cs = 0.0f; if (norm1_sq > 0.0f && norm2_sq > 0.0f) { - float cs = dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq)); + cs = dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq)); cs = std::min(cs, 1.0f); cs = std::max(cs, -1.0f); - ts.cossim = cs; + } else if (norm1_sq == 0.0f && norm2_sq == 0.0f) { + cs = 1.0f; } + ts.cossim = cs; // Compute L2 Norm (Euclidean Distance) ts.l2_norm = std::sqrt(l2_dist_sq); @@ -332,14 +335,19 @@ static void compute_layer_statistics(const std::vector & tsta const int blk = std::stoi(match[1]); if (blk <= 0) { continue; } std::string prev_lyr(ts.tensor); - prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk-1)); - if (auto it_prev = tidx.find(prev_lyr); it_prev == tidx.end()) { continue; } - const auto curr_avg = compute_tensor_averages(stats_map.at(ts.tensor)); - const auto prev_avg = compute_tensor_averages(stats_map.at(prev_lyr)); + prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk - 1)); + if (tidx.find(prev_lyr) == tidx.end()) { continue; } + auto it_curr = stats_map.find(ts.tensor); + auto it_prev = stats_map.find(prev_lyr); + if (it_curr == stats_map.end() || it_prev == stats_map.end()) { continue; } + + const auto curr_avg = compute_tensor_averages(it_curr->second); + const auto prev_avg = compute_tensor_averages(it_prev->second); if (curr_avg.empty() || prev_avg.empty() || curr_avg.size() != prev_avg.size()) { continue; } - auto & [curr, prev] = agr[blk]; - curr.insert(curr.end(), curr_avg.begin(), curr_avg.end()); - prev.insert(prev.end(), prev_avg.begin(), prev_avg.end()); + + auto & entry = agr[blk]; + entry.curr_avg.insert(entry.curr_avg.end(), curr_avg.begin(), curr_avg.end()); + entry.prev_avg.insert(entry.prev_avg.end(), prev_avg.begin(), prev_avg.end()); } for (auto & kv : agr) { @@ -347,18 +355,18 @@ static void compute_layer_statistics(const std::vector & tsta const auto & prev = kv.second.prev_avg; if (curr.size() != prev.size() || curr.empty()) { continue; } - float dot_prod = 0.0f; - float norm1_sq = 0.0f; - float norm2_sq = 0.0f; - float l2_dist_sq = 0.0f; + double dot_prod = 0.0; + double norm1_sq = 0.0; + double norm2_sq = 0.0; + double l2_dist_sq = 0.0; for (size_t i = 0; i < curr.size(); ++i) { - const float c_val = curr[i]; - const float p_val = prev[i]; + const double c_val = curr[i]; + const double p_val = prev[i]; dot_prod += c_val * p_val; norm1_sq += c_val * c_val; norm2_sq += p_val * p_val; - const float diff = c_val - p_val; + const double diff = c_val - p_val; l2_dist_sq += diff * diff; } @@ -366,11 +374,15 @@ static void compute_layer_statistics(const std::vector & tsta float cossim = 0.0f; if (norm1_sq > 0.0f && norm2_sq > 0.0f) { cossim = dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq)); + cossim = std::min(cossim, 1.0f); + cossim = std::max(cossim, -1.0f); + } else if (norm1_sq == 0.0f && norm2_sq == 0.0f) { + cossim = 1.0f; } layer_cossim[kv.first] = cossim; // Compute aggregated L2 Norm (Euclidean Distance) - layer_l2_norm[kv.first] = std::sqrt(l2_dist_sq); + layer_l2_norm[kv.first] = (float)std::sqrt(l2_dist_sq); } } @@ -1309,8 +1321,8 @@ static bool show_statistics(const common_params & params) { float layer_zd = 0.0f; int n = 0; }; - std::map ls; + std::map ls; LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%11s\t%8s\t%5s\t%10s\n", "Layer", @@ -1330,7 +1342,8 @@ static bool show_statistics(const common_params & params) { "=============================================================\n"); for (const auto & tstat : ts) { - std::string layer, name; + std::string layer; + std::string name; process_tensor_name(tstat.tensor, layer, name); int blk; From 6ff0a79e54858c3c991789c45f7a761a3d6668fa Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Wed, 29 Oct 2025 07:59:40 +0000 Subject: [PATCH 067/123] Minor stats report cosmetic changes --- tools/imatrix/imatrix.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f529b39e0fec..3d3715551ceb 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -150,7 +150,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { } else { // Mean for (size_t m = 0; m < n_mat; ++m) { - const float c = (float)tstats.counts[m]; + const auto c = (float)tstats.counts[m]; const size_t off = m * row; if (c <= 0.0f) { vec.insert(vec.end(), row, 0.0f); // zero-fill rows for experts with zero count to preserve shape @@ -215,7 +215,7 @@ static bool compute_vector_statistics(std::vector & tstats, c double sqr_sum = 0.0; for (const float v : activations) { sqr_sum += (double)v * (double)v; } double variance = sqr_sum / (double)activations.size() - (double)mean * (double)mean; - if (variance < 0.0) { variance = 0.0; } + variance = std::max(variance, 0.0); const float std_deviation = std::sqrt((float)variance); float entropy = 0.0f; @@ -1327,7 +1327,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%11s\t%8s\t%5s\t%10s\n", "Layer", "Tensor", - legacy_mode ? "Σ(Act²)" : "L₂ Norm", + legacy_mode ? "Σ E[Act²]" : "L₂ Norm", "Min", "Max", "μ", @@ -1356,7 +1356,7 @@ static bool show_statistics(const common_params & params) { const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; const float ecs = 100.0f * std::exp(-0.01f * tstat.l2_norm) * std::pow(std::fabs(tstat.cossim), 10.0f); - LOG_INF("%5s\t%-20s\t%11.2f\t%10.4f\t%10.4f\t%8.2f\t%8.2f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", + LOG_INF("%5s\t%-20s\t%11.4f\t%10.4f\t%10.4f\t%8.4f\t%8.4f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", layer.c_str(), name.c_str(), legacy_mode ? tstat.sum_values : tstat.l2_norm, @@ -1392,7 +1392,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("\nComputing layer statistics (%zu layers)\n", layers); LOG_INF("\n%6s\t%13s\t%6s\t%11s\t%6s\n", "Layer", - legacy_mode ? "Σ(Act²)" : "L₂ Norm", + legacy_mode ? "Σ E[Act²]" : "L₂ Norm", "ZD", "CosSim", legacy_mode ? "" : "ECS"); @@ -1402,19 +1402,19 @@ static bool show_statistics(const common_params & params) { LOG_INF("=========================================================\n"); } for (const auto & [layer, stats] : ls) { - if (layer < 0 || stats.n == 0) continue; + if (layer < 0 || stats.n == 0) { continue; } const auto lcs = layer_cossim.find(layer); const float layer_cs = lcs != layer_cossim.end() ? lcs->second : 0.0f; const auto ll2n = layer_l2_norm.find(layer); const float layer_l2n = ll2n != layer_l2_norm.end() ? ll2n->second : 0.0f; if (legacy_mode) { - LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\n", + LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\n", layer, stats.layer_sum, 100.0f * stats.layer_zd / stats.n, layer_cs); } else { - LOG_INF("%5d\t%11.2f\t%6.2f%%\t%11.4f\t%8.4f\n", + LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\t%8.4f\n", layer, layer_l2n, 100.0f * stats.layer_zd / stats.n, From 2a6f5d7e609b1ad5f93ce3a8f0bf19a899dd450c Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Wed, 29 Oct 2025 18:32:47 +0000 Subject: [PATCH 068/123] Refactor variable names --- tools/imatrix/imatrix.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 3d3715551ceb..33b1aa1edc2b 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -56,7 +56,7 @@ struct tensor_statistics { float entropy = 0.0f; float zd_score = 0.0f; float cossim = 0.0f; - float l2_norm = 0.0f; + float l2_dist = 0.0f; }; class IMatrixCollector { @@ -381,8 +381,8 @@ static void compute_layer_statistics(const std::vector & tsta } layer_cossim[kv.first] = cossim; - // Compute aggregated L2 Norm (Euclidean Distance) - layer_l2_norm[kv.first] = (float)std::sqrt(l2_dist_sq); + // Compute aggregated L2 Distance (Euclidean Distance) + layer_l2_dist[kv.first] = (float)std::sqrt(agg.sum_l2_dist_sq); } } @@ -1327,7 +1327,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%11s\t%8s\t%5s\t%10s\n", "Layer", "Tensor", - legacy_mode ? "Σ E[Act²]" : "L₂ Norm", + legacy_mode ? "Σ E[Act²]" : "L₂ Dist", "Min", "Max", "μ", @@ -1354,12 +1354,12 @@ static bool show_statistics(const common_params & params) { } const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; - const float ecs = 100.0f * std::exp(-0.01f * tstat.l2_norm) * std::pow(std::fabs(tstat.cossim), 10.0f); + const float ecs = 100.0f * std::exp(-0.01f * tstat.l2_dist) * std::pow(std::fabs(tstat.cossim), 10.0f); // Euclidean-Cosine score LOG_INF("%5s\t%-20s\t%11.4f\t%10.4f\t%10.4f\t%8.4f\t%8.4f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", layer.c_str(), name.c_str(), - legacy_mode ? tstat.sum_values : tstat.l2_norm, + legacy_mode ? tstat.sum_values : tstat.l2_dist, tstat.min_values, tstat.max_values, tstat.mean_values, @@ -1385,14 +1385,14 @@ static bool show_statistics(const common_params & params) { } std::map layer_cossim; - std::map layer_l2_norm; - compute_layer_statistics(ts, layer_cossim, layer_l2_norm, g_collector.get_mstats()); + std::map layer_l2_dist; + compute_layer_statistics(ts, layer_cossim, layer_l2_dist, g_collector.get_mstats()); const size_t layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); LOG_INF("\nComputing layer statistics (%zu layers)\n", layers); LOG_INF("\n%6s\t%13s\t%6s\t%11s\t%6s\n", "Layer", - legacy_mode ? "Σ E[Act²]" : "L₂ Norm", + legacy_mode ? "Σ E[Act²]" : "L₂ Dist", "ZD", "CosSim", legacy_mode ? "" : "ECS"); @@ -1405,8 +1405,8 @@ static bool show_statistics(const common_params & params) { if (layer < 0 || stats.n == 0) { continue; } const auto lcs = layer_cossim.find(layer); const float layer_cs = lcs != layer_cossim.end() ? lcs->second : 0.0f; - const auto ll2n = layer_l2_norm.find(layer); - const float layer_l2n = ll2n != layer_l2_norm.end() ? ll2n->second : 0.0f; + const auto ll2n = layer_l2_dist.find(layer); + const float layer_l2n = ll2n != layer_l2_dist.end() ? ll2n->second : 0.0f; if (legacy_mode) { LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\n", layer, From 006e7ef9913f4ed2dc7ebc00890f4e287b6cfcea Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Wed, 29 Oct 2025 18:35:39 +0000 Subject: [PATCH 069/123] Improve compute_vector_statistics() processing of mismatched tensor sizes --- tools/imatrix/imatrix.cpp | 119 +++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 59 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 33b1aa1edc2b..f3e3d5f19002 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -166,6 +166,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { const size_t n_mat = e.counts.size(); const size_t len = e.activations.empty() ? e.values.size() : e.activations.size(); + const bool legacy = e.activations.empty(); if (n_mat == 0) { LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); return false; @@ -174,77 +175,77 @@ static bool compute_vector_statistics(std::vector & tstats, c LOG_ERR("%s: activation size mismatch for tensor %s (len=%zu, counts=%zu)\n", __func__, name.c_str(), len, n_mat); return false; } + if (!legacy && e.values.size() != len) { + LOG_ERR("%s: activations/values size mismatch for tensor %s (act=%zu, val=%zu)\n", __func__, name.c_str(), len, e.values.size()); + return false; + } const size_t row_size = len / n_mat; - std::vector activations; - activations.reserve(len); - + double mean = 0.0; + double M2 = 0.0; + double sum = 0.0; + float vmin = std::numeric_limits::infinity(); + float vmax = -std::numeric_limits::infinity(); + double energy_sum = 0.0; + size_t valid_n = 0; for (size_t i = 0; i < n_mat; ++i) { const auto c = (float)e.counts[i]; + if (c <= 0.0f) { continue; } // skip experts with zero count const size_t off = i * row_size; - if (c <= 0.0f) { - activations.insert(activations.end(), row_size, 0.0f); - continue; - } - if (e.activations.empty()) { - for (size_t j = 0; j < row_size; ++j) { - activations.push_back(e.values[off + j] / c); // mean-of-squares - } - } else { - for (size_t j = 0; j < row_size; ++j) { - activations.push_back(e.activations[off + j] / c); // mean - } + + for (size_t j = 0; j < row_size; ++j) { + const double v_avg = legacy ? 0.0 : (double)e.activations[off + j] / (double)c; // E[x] + const double v_energy = (double)e.values[off + j] / (double)c; // E[x^2] + const double v = legacy ? v_energy : v_avg; + + ++valid_n; + sum += v; + vmin = std::min(vmin, (float)v); + vmax = std::max(vmax, (float)v); + + const double delta = v - mean; + mean += delta / (double)valid_n; + M2 += delta * (v - mean); + energy_sum += std::max(0.0, v_energy); } } - if (activations.empty()) { - LOG_ERR("%s: computed empty activation vector for tensor %s\n", __func__, name.c_str()); + if (valid_n == 0) { + LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); return false; } - double sum = 0.0; - float vmax = activations[0]; - float vmin = activations[0]; - for (float v : activations) { - sum += v; - vmax = std::max(vmax, v); - vmin = std::min(vmin, v); - } - - const auto mean = (float)(sum / (double)activations.size()); - double sqr_sum = 0.0; - for (const float v : activations) { sqr_sum += (double)v * (double)v; } - double variance = sqr_sum / (double)activations.size() - (double)mean * (double)mean; - variance = std::max(variance, 0.0); - const float std_deviation = std::sqrt((float)variance); - + float std_deviation = 0.0f; float entropy = 0.0f; - if (e.activations.empty()) { - double energy_sum = 0.0; - for (float v : activations) { energy_sum += (double)std::max(0.0f, v); } - if (energy_sum > 0.0) { - for (const float v : activations) { - const double p = std::max(0.0, (double)v) / energy_sum; - if (p > 0.0) { entropy -= (float)(p * std::log2(p)); } - } - } - } else { - double energy_sum = 0.0; - for (const float v : activations) { energy_sum += (double)v * (double)v; } - if (energy_sum > 0.0) { - for (const float v : activations) { - const double p = (double)v * (double)v / energy_sum; + double zd_count = 0.0; + double variance = valid_n > 1 ? M2 / ((double)valid_n - 1) : 0.0; + variance = std::max(variance, 0.0); + std_deviation = std::sqrt((float)variance); + if (energy_sum > 0.0) { + for (size_t i = 0; i < n_mat; ++i) { + const auto c = (float)e.counts[i]; + if (c <= 0.0f) { continue; } + const size_t off = i * row_size; + for (size_t j = 0; j < row_size; ++j) { + const double v_energy = (double)e.values[off + j] / (double)c; // E[x^2] + const double w = std::max(0.0, v_energy); + const double p = w / energy_sum; if (p > 0.0) { entropy -= (float)(p * std::log2(p)); } } } } - - // ZD score: fraction with |z| > 1 - double zd_count = 0.0; if (std_deviation > 0.0f) { - for (const float v : activations) { - const float z = (v - mean) / std_deviation; - if (std::fabs(z) > 1.0f) { zd_count += 1.0; } + for (size_t i = 0; i < n_mat; ++i) { + const float c = (float)e.counts[i]; + if (c <= 0.0f) { continue; } + const size_t off = i * row_size; + for (size_t j = 0; j < row_size; ++j) { + const double v_avg = legacy ? 0.0 : (double)e.activations[off + j] / (double)c; // E[x] + const double v_energy = (double)e.values[off + j] / (double)c; // E[x^2] + const float v = (float)(legacy ? v_energy : v_avg); + const float z = (v - (float)mean) / std_deviation; + if (std::fabs(z) > 1.0f) { zd_count += 1.0; } + } } } @@ -252,13 +253,13 @@ static bool compute_vector_statistics(std::vector & tstats, c ts.tensor = name; ts.stats = e; ts.sum_values = (float)sum; - ts.mean_values = mean; + ts.mean_values = (float)mean; ts.max_values = vmax; ts.min_values = vmin; - ts.elements = (int)activations.size(); + ts.elements = valid_n; ts.std_deviation = std_deviation; ts.entropy = entropy; - ts.zd_score = ts.elements > 0 ? (float)(zd_count / (double)ts.elements) : 0.0f; + ts.zd_score = (float)(zd_count / (double)valid_n); return e.activations.empty(); } @@ -267,7 +268,7 @@ static void compute_tensor_statistics(std::vector & tstats) { static const std::regex pattern(R"(blk\.(\d+)\.)"); for (auto & ts : tstats) { ts.cossim = 0.0f; - ts.l2_norm = 0.0f; + ts.l2_dist = 0.0f; if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { const int blk = std::stoi(match[1]); @@ -309,7 +310,7 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.cossim = cs; // Compute L2 Norm (Euclidean Distance) - ts.l2_norm = std::sqrt(l2_dist_sq); + ts.l2_dist = std::sqrt(l2_dist_sq); } } } From 7d8819f57a3e9e5ce8aac590865c18cbeaaf3be5 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Wed, 29 Oct 2025 18:36:01 +0000 Subject: [PATCH 070/123] Improve compute_layer_statistics() processing of mismatched tensor sizes --- tools/imatrix/imatrix.cpp | 50 ++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f3e3d5f19002..6fd91aad0841 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -317,11 +317,14 @@ static void compute_tensor_statistics(std::vector & tstats) { static void compute_layer_statistics(const std::vector & tstats, std::map & layer_cossim, - std::map & layer_l2_norm, + std::map & layer_l2_dist, const std::unordered_map & stats_map) { struct layer_aggregation { - std::vector curr_avg; - std::vector prev_avg; + double sum_dot_prod = 0.0; + double sum_norm1_sq = 0.0; + double sum_norm2_sq = 0.0; + double sum_l2_dist_sq = 0.0; + int n_tensors = 0; }; static const std::regex pattern(R"(blk\.(\d+)\.)"); @@ -335,9 +338,11 @@ static void compute_layer_statistics(const std::vector & tsta if (!std::regex_search(ts.tensor, match, pattern)) { continue; } const int blk = std::stoi(match[1]); if (blk <= 0) { continue; } + std::string prev_lyr(ts.tensor); prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk - 1)); if (tidx.find(prev_lyr) == tidx.end()) { continue; } + auto it_curr = stats_map.find(ts.tensor); auto it_prev = stats_map.find(prev_lyr); if (it_curr == stats_map.end() || it_prev == stats_map.end()) { continue; } @@ -346,24 +351,15 @@ static void compute_layer_statistics(const std::vector & tsta const auto prev_avg = compute_tensor_averages(it_prev->second); if (curr_avg.empty() || prev_avg.empty() || curr_avg.size() != prev_avg.size()) { continue; } - auto & entry = agr[blk]; - entry.curr_avg.insert(entry.curr_avg.end(), curr_avg.begin(), curr_avg.end()); - entry.prev_avg.insert(entry.prev_avg.end(), prev_avg.begin(), prev_avg.end()); - } - - for (auto & kv : agr) { - const auto & curr = kv.second.curr_avg; - const auto & prev = kv.second.prev_avg; - if (curr.size() != prev.size() || curr.empty()) { continue; } - + // Compute statistics for each tensor pair individually double dot_prod = 0.0; double norm1_sq = 0.0; double norm2_sq = 0.0; double l2_dist_sq = 0.0; - for (size_t i = 0; i < curr.size(); ++i) { - const double c_val = curr[i]; - const double p_val = prev[i]; + for (size_t i = 0; i < curr_avg.size(); ++i) { + const double c_val = curr_avg[i]; + const double p_val = prev_avg[i]; dot_prod += c_val * p_val; norm1_sq += c_val * c_val; norm2_sq += p_val * p_val; @@ -371,13 +367,29 @@ static void compute_layer_statistics(const std::vector & tsta l2_dist_sq += diff * diff; } + if (norm1_sq == 0.0 && norm2_sq == 0.0) { continue; } + + // Accumulate statistics for the layer + auto & entry = agr[blk]; + entry.sum_dot_prod += dot_prod; + entry.sum_norm1_sq += norm1_sq; + entry.sum_norm2_sq += norm2_sq; + entry.sum_l2_dist_sq += l2_dist_sq; + entry.n_tensors++; + } + + // Compute aggregated layer statistics + for (auto & kv : agr) { + const auto & agg = kv.second; + if (agg.n_tensors == 0) { continue; } + // Compute aggregated Cosine Similarity float cossim = 0.0f; - if (norm1_sq > 0.0f && norm2_sq > 0.0f) { - cossim = dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq)); + if (agg.sum_norm1_sq > 0.0 && agg.sum_norm2_sq > 0.0) { + cossim = agg.sum_dot_prod / (std::sqrt(agg.sum_norm1_sq) * std::sqrt(agg.sum_norm2_sq)); cossim = std::min(cossim, 1.0f); cossim = std::max(cossim, -1.0f); - } else if (norm1_sq == 0.0f && norm2_sq == 0.0f) { + } else if (agg.sum_norm1_sq == 0.0 && agg.sum_norm2_sq == 0.0) { cossim = 1.0f; } layer_cossim[kv.first] = cossim; From ce046dcee82e8d5a0c57689381d725054fdd73ea Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 30 Oct 2025 22:43:46 +0000 Subject: [PATCH 071/123] Save statistics to imatrix --- tools/imatrix/imatrix.cpp | 43 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6fd91aad0841..1c6f59c782f7 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -740,11 +740,27 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * 4, GGML_MEM_ALIGN); } // deterministic tensor name order std::sort(to_store.begin(), to_store.end()); + // Compute per-tensor statistics (CosSim, L2 Dist, ECS) to store alongside sums + std::vector tstats; + tstats.reserve(m_stats.size()); + bool legacy_mode = true; + for (const auto & kv : m_stats) { + const bool is_legacy = compute_vector_statistics(tstats, kv.first, kv.second); + legacy_mode = legacy_mode && is_legacy; + } + if (!tstats.empty()) { compute_tensor_statistics(tstats); } + + // index by tensor name + std::unordered_map tstat_index; + tstat_index.reserve(tstats.size()); + for (const auto & ts : tstats) { tstat_index[ts.tensor] = &ts; } + struct ggml_init_params params = { /* .mem_size = */ data_size, /* .mem_buffer = */ NULL, @@ -801,6 +817,29 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum); } } + + // Store per-tensor statistics as a small 1D tensor: [ECS, L2 Dist, CosSim, ZD Score] + { + float l2 = 0.0f; + float cs = 0.0f; + float zd = 0.0f; + float ecs = 0.0f; + auto it_ts = tstat_index.find(name); + if (it_ts != tstat_index.end() && it_ts->second != nullptr) { + l2 = it_ts->second->l2_dist; + cs = it_ts->second->cossim; + zd = it_ts->second->zd_score; + ecs = 100.0f * (1.0f - std::exp(-0.01f * l2) * std::pow(std::fabs(cs), 10.0f)); + } + + struct ggml_tensor * stats_t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4); + ggml_format_name(stats_t, "%s.stats", name.c_str()); + ((float *)stats_t->data)[0] = ecs; + ((float *)stats_t->data)[1] = l2; + ((float *)stats_t->data)[2] = cs; + ((float *)stats_t->data)[3] = zd; + gguf_add_tensor(ctx_gguf, stats_t); + } } gguf_write_to_file(ctx_gguf, fname.c_str(), false); @@ -1367,7 +1406,7 @@ static bool show_statistics(const common_params & params) { } const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; - const float ecs = 100.0f * std::exp(-0.01f * tstat.l2_dist) * std::pow(std::fabs(tstat.cossim), 10.0f); // Euclidean-Cosine score + const float ecs = 100.0f * (1.0f - std::exp(-0.01f * tstat.l2_dist) * std::pow(std::fabs(tstat.cossim), 10.0f)); // Euclidean-Cosine score LOG_INF("%5s\t%-20s\t%11.4f\t%10.4f\t%10.4f\t%8.4f\t%8.4f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", layer.c_str(), @@ -1432,7 +1471,7 @@ static bool show_statistics(const common_params & params) { layer_l2n, 100.0f * stats.layer_zd / stats.n, layer_cs, - 100.0f * std::exp(-0.01f * layer_l2n) * std::pow(std::fabs(layer_cs), 10.0f)); + 100.0f * (1.0f - std::exp(-0.01f * layer_l2n) * std::pow(std::fabs(layer_cs), 10.0f))); // Euclidean-Cosine score } } LOG_INF("\n"); From b2b7175e19b145b6255ff93b09992c412d87232a Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 6 Nov 2025 15:12:09 +0000 Subject: [PATCH 072/123] Fix bug when vectors are zero --- tools/imatrix/imatrix.cpp | 48 +++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 1c6f59c782f7..a76f29b31e02 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -328,11 +328,7 @@ static void compute_layer_statistics(const std::vector & tsta }; static const std::regex pattern(R"(blk\.(\d+)\.)"); - std::unordered_map tidx; - tidx.reserve(tstats.size()); - for (const auto & ts : tstats) { tidx[ts.tensor] = &ts; } - - std::map agr; + std::map l_agr; for (const auto & ts : tstats) { std::smatch match; if (!std::regex_search(ts.tensor, match, pattern)) { continue; } @@ -341,36 +337,35 @@ static void compute_layer_statistics(const std::vector & tsta std::string prev_lyr(ts.tensor); prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk - 1)); - if (tidx.find(prev_lyr) == tidx.end()) { continue; } - auto it_curr = stats_map.find(ts.tensor); auto it_prev = stats_map.find(prev_lyr); if (it_curr == stats_map.end() || it_prev == stats_map.end()) { continue; } const auto curr_avg = compute_tensor_averages(it_curr->second); const auto prev_avg = compute_tensor_averages(it_prev->second); - if (curr_avg.empty() || prev_avg.empty() || curr_avg.size() != prev_avg.size()) { continue; } + if (curr_avg.empty() || prev_avg.empty()) { continue; } + + // Allow minor length mismatches by using the overlap + const size_t n = std::min(curr_avg.size(), prev_avg.size()); + if (n == 0) { continue; } // Compute statistics for each tensor pair individually double dot_prod = 0.0; double norm1_sq = 0.0; double norm2_sq = 0.0; double l2_dist_sq = 0.0; - - for (size_t i = 0; i < curr_avg.size(); ++i) { - const double c_val = curr_avg[i]; - const double p_val = prev_avg[i]; - dot_prod += c_val * p_val; - norm1_sq += c_val * c_val; - norm2_sq += p_val * p_val; - const double diff = c_val - p_val; - l2_dist_sq += diff * diff; + for (size_t i = 0; i < n; ++i) { + const double a = curr_avg[i]; + const double b = prev_avg[i]; + dot_prod += a * b; + norm1_sq += a * a; + norm2_sq += b * b; + const double d = a - b; + l2_dist_sq += d * d; } - if (norm1_sq == 0.0 && norm2_sq == 0.0) { continue; } - // Accumulate statistics for the layer - auto & entry = agr[blk]; + auto & entry = l_agr[blk]; entry.sum_dot_prod += dot_prod; entry.sum_norm1_sq += norm1_sq; entry.sum_norm2_sq += norm2_sq; @@ -379,23 +374,26 @@ static void compute_layer_statistics(const std::vector & tsta } // Compute aggregated layer statistics - for (auto & kv : agr) { + for (const auto & kv : l_agr) { + const int layer = kv.first; const auto & agg = kv.second; if (agg.n_tensors == 0) { continue; } // Compute aggregated Cosine Similarity float cossim = 0.0f; if (agg.sum_norm1_sq > 0.0 && agg.sum_norm2_sq > 0.0) { - cossim = agg.sum_dot_prod / (std::sqrt(agg.sum_norm1_sq) * std::sqrt(agg.sum_norm2_sq)); + cossim = (float)(agg.sum_dot_prod / (std::sqrt(agg.sum_norm1_sq) * std::sqrt(agg.sum_norm2_sq))); cossim = std::min(cossim, 1.0f); cossim = std::max(cossim, -1.0f); } else if (agg.sum_norm1_sq == 0.0 && agg.sum_norm2_sq == 0.0) { - cossim = 1.0f; + cossim = 1.0f; // both vectors are zero then CosSim is 1 + } else { + cossim = 0.0f; // One zero and the other non-zero then CosSim is 0 } - layer_cossim[kv.first] = cossim; // Compute aggregated L2 Distance (Euclidean Distance) - layer_l2_dist[kv.first] = (float)std::sqrt(agg.sum_l2_dist_sq); + layer_cossim[layer] = cossim; + layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); } } From 559ae9ab899766aaa66d6827b7449281bcd81c66 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 10:19:34 +0000 Subject: [PATCH 073/123] Refactor legacy imatrix handling --- tools/imatrix/imatrix.cpp | 56 ++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index a76f29b31e02..71041c27d7ff 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -163,15 +163,15 @@ static std::vector compute_tensor_averages(const Stats & tstats) { return vec; } -static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e) { +static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e, bool & legacy) { + legacy = e.activations.empty(); const size_t n_mat = e.counts.size(); - const size_t len = e.activations.empty() ? e.values.size() : e.activations.size(); - const bool legacy = e.activations.empty(); - if (n_mat == 0) { - LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); + const size_t len = legacy ? e.values.size() : e.activations.size(); + if (n_mat == 0 || len == 0) { + LOG_ERR("%s: there's no data for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); return false; } - if (len == 0 || (len % n_mat) != 0) { + if (len % n_mat != 0) { LOG_ERR("%s: activation size mismatch for tensor %s (len=%zu, counts=%zu)\n", __func__, name.c_str(), len, n_mat); return false; } @@ -211,7 +211,7 @@ static bool compute_vector_statistics(std::vector & tstats, c } if (valid_n == 0) { - LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); + LOG_ERR("%s: there's no data for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); return false; } @@ -236,13 +236,13 @@ static bool compute_vector_statistics(std::vector & tstats, c } if (std_deviation > 0.0f) { for (size_t i = 0; i < n_mat; ++i) { - const float c = (float)e.counts[i]; + const auto c = (float)e.counts[i]; if (c <= 0.0f) { continue; } const size_t off = i * row_size; for (size_t j = 0; j < row_size; ++j) { const double v_avg = legacy ? 0.0 : (double)e.activations[off + j] / (double)c; // E[x] const double v_energy = (double)e.values[off + j] / (double)c; // E[x^2] - const float v = (float)(legacy ? v_energy : v_avg); + const auto v = (float)(legacy ? v_energy : v_avg); const float z = (v - (float)mean) / std_deviation; if (std::fabs(z) > 1.0f) { zd_count += 1.0; } } @@ -256,12 +256,12 @@ static bool compute_vector_statistics(std::vector & tstats, c ts.mean_values = (float)mean; ts.max_values = vmax; ts.min_values = vmin; - ts.elements = valid_n; + ts.elements = (int)valid_n; ts.std_deviation = std_deviation; ts.entropy = entropy; ts.zd_score = (float)(zd_count / (double)valid_n); - return e.activations.empty(); + return true; } static void compute_tensor_statistics(std::vector & tstats) { @@ -747,10 +747,11 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { // Compute per-tensor statistics (CosSim, L2 Dist, ECS) to store alongside sums std::vector tstats; tstats.reserve(m_stats.size()); - bool legacy_mode = true; + bool legacy = true; for (const auto & kv : m_stats) { - const bool is_legacy = compute_vector_statistics(tstats, kv.first, kv.second); - legacy_mode = legacy_mode && is_legacy; + if (!compute_vector_statistics(tstats, kv.first, kv.second, legacy)) { + LOG_WRN("%s: tensor %s has no data - skipping\n", __func__, kv.first.c_str()); + } } if (!tstats.empty()) { compute_tensor_statistics(tstats); } @@ -1328,7 +1329,7 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c static bool show_statistics(const common_params & params) { std::vector ts; - bool legacy_mode = true; + bool legacy = true; if (params.in_files.empty() || params.in_files.size() > 1) { LOG_ERR("\nError: a single imatrix file is required to compute tensor statistics\n\n"); @@ -1336,8 +1337,9 @@ static bool show_statistics(const common_params & params) { } if (g_collector.load_imatrix(params.in_files[0].c_str())) { for (const auto & [name, stats] : g_collector.get_mstats()) { - const bool is_legacy = compute_vector_statistics(ts, name, stats); - legacy_mode = legacy_mode && is_legacy; + if (!compute_vector_statistics(ts, name, stats, legacy)) { + LOG_WRN("%s: tensor %s has no data - skipping\n", __func__, name.c_str()); + } } } else { LOG_ERR("\nError: %s is not a valid imatrix file\n\n", params.in_files[0].c_str()); @@ -1364,7 +1366,7 @@ static bool show_statistics(const common_params & params) { : name_a < name_b || (name_a == name_b && a.cossim > b.cossim); } }; - std::sort(ts.begin(), ts.end(), tensor_comparer(legacy_mode)); + std::sort(ts.begin(), ts.end(), tensor_comparer(legacy)); struct layer_stats { float layer_sum = 0.0f; @@ -1377,14 +1379,14 @@ static bool show_statistics(const common_params & params) { LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%11s\t%8s\t%5s\t%10s\n", "Layer", "Tensor", - legacy_mode ? "Σ E[Act²]" : "L₂ Dist", + legacy ? "Σ E[Act²]" : "L₂ Dist", "Min", "Max", "μ", "σ", "N", "H Norm", - legacy_mode ? "H" : "ECS", + legacy ? "H" : "ECS", "ZD", "CosSim"); LOG_INF( @@ -1409,18 +1411,18 @@ static bool show_statistics(const common_params & params) { LOG_INF("%5s\t%-20s\t%11.4f\t%10.4f\t%10.4f\t%8.4f\t%8.4f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", layer.c_str(), name.c_str(), - legacy_mode ? tstat.sum_values : tstat.l2_dist, + legacy ? tstat.sum_values : tstat.l2_dist, tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, tstat.elements, h_norm, - legacy_mode ? tstat.entropy : ecs, + legacy ? tstat.entropy : ecs, 100.0f * tstat.zd_score, tstat.cossim); - const float zd = tstat.elements * tstat.zd_score; + const float zd = (float)tstat.elements * tstat.zd_score; if (ls.find(blk) != ls.end()) { ls[blk].layer_sum += tstat.sum_values; ls[blk].layer_zd += zd; @@ -1442,11 +1444,11 @@ static bool show_statistics(const common_params & params) { LOG_INF("\nComputing layer statistics (%zu layers)\n", layers); LOG_INF("\n%6s\t%13s\t%6s\t%11s\t%6s\n", "Layer", - legacy_mode ? "Σ E[Act²]" : "L₂ Dist", + legacy ? "Σ E[Act²]" : "L₂ Dist", "ZD", "CosSim", - legacy_mode ? "" : "ECS"); - if (legacy_mode) { + legacy ? "" : "ECS"); + if (legacy) { LOG_INF("============================================\n"); } else { LOG_INF("=========================================================\n"); @@ -1457,7 +1459,7 @@ static bool show_statistics(const common_params & params) { const float layer_cs = lcs != layer_cossim.end() ? lcs->second : 0.0f; const auto ll2n = layer_l2_dist.find(layer); const float layer_l2n = ll2n != layer_l2_dist.end() ? ll2n->second : 0.0f; - if (legacy_mode) { + if (legacy) { LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\n", layer, stats.layer_sum, From 5384a11b94b65e5b377f58ff41574746a7f2fc9b Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 13:00:47 +0000 Subject: [PATCH 074/123] Initialise layer and tensor variables --- tools/imatrix/imatrix.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 71041c27d7ff..0d5b856ce992 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -99,13 +99,14 @@ static std::string filter_tensor_name(const char * name) { } static void process_tensor_name(const std::string & input, std::string & layer, std::string & tensor) { + layer.clear(); + tensor.clear(); + std::vector name; std::istringstream stream(input); std::string item; - while (std::getline(stream, item, '.')) { - name.push_back(item); - } + while (std::getline(stream, item, '.')) { name.push_back(item); } for (size_t i = 0; i < name.size(); ++i) { if (name[i] == "blk" && i + 1 < name.size()) { layer = name[i + 1]; @@ -119,12 +120,8 @@ static void process_tensor_name(const std::string & input, std::string & layer, } } - if (tensor.empty()) { - tensor = input; - } - if (layer.empty()) { - layer = "-"; - } + if (tensor.empty()) { tensor = input; } + if (layer.empty()) { layer = "-"; } } static std::vector compute_tensor_averages(const Stats & tstats) { From ae1cbc707babcc9f06bbb3ec2be40cbee8e7df57 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 13:04:16 +0000 Subject: [PATCH 075/123] Warn if problem with previous layer --- tools/imatrix/imatrix.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 0d5b856ce992..9141f6b598b2 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -274,11 +274,19 @@ static void compute_tensor_statistics(std::vector & tstats) { tname.replace(match.position(1), match.length(1), std::to_string(blk - 1)); auto prev_it = std::find_if(tstats.begin(), tstats.end(), [tname](const tensor_statistics & t) { return t.tensor == tname; }); - if (prev_it == tstats.end()) { continue; } + if (prev_it == tstats.end()) { + LOG_WRN("%s: missing previous-layer tensor '%s' (current: '%s'). Statistics may not be accurate\n", + __func__, tname.c_str(), ts.tensor.c_str()); + continue; + } const auto curr_avg = compute_tensor_averages(ts.stats); const auto prev_avg = compute_tensor_averages(prev_it->stats); - if (curr_avg.empty() || curr_avg.size() != prev_avg.size()) { continue; } + if (curr_avg.empty() || curr_avg.size() != prev_avg.size()) { + LOG_WRN("%s: size mismatch between '%s' and its previous-layer tensor '%s' (%zu vs %zu). Statistics may not be accurate\n", + __func__, ts.tensor.c_str(), tname.c_str(), curr_avg.size(), prev_avg.size()); + continue; + } float dot_prod = 0.0f; float norm1_sq = 0.0f; From 63cbcc6dfc6493eb07c4510dd5afe765f9acb9db Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 13:05:34 +0000 Subject: [PATCH 076/123] Refactor legacy determination --- tools/imatrix/imatrix.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 9141f6b598b2..6090f3760f27 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1334,29 +1334,41 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c static bool show_statistics(const common_params & params) { std::vector ts; - bool legacy = true; - if (params.in_files.empty() || params.in_files.size() > 1) { LOG_ERR("\nError: a single imatrix file is required to compute tensor statistics\n\n"); return false; } + + bool has_activations = false; + bool no_activations = false; if (g_collector.load_imatrix(params.in_files[0].c_str())) { for (const auto & [name, stats] : g_collector.get_mstats()) { - if (!compute_vector_statistics(ts, name, stats, legacy)) { + bool legacy_imatrix = true; + if (!compute_vector_statistics(ts, name, stats, legacy_imatrix)) { LOG_WRN("%s: tensor %s has no data - skipping\n", __func__, name.c_str()); + continue; } + if (legacy_imatrix) { no_activations = true; } + else { has_activations = true; } } } else { LOG_ERR("\nError: %s is not a valid imatrix file\n\n", params.in_files[0].c_str()); return false; } - if (!ts.empty()) { - compute_tensor_statistics(ts); - } else { + if (ts.empty()) { LOG_ERR("Error: cannot compute statistics for %s\n\n", params.in_files[0].c_str()); return false; } + bool legacy; + if (has_activations && no_activations) { + LOG_ERR("Error: %s has mixed tensors with and without activations\n\n", params.in_files[0].c_str()); + return false; + } + + legacy = !has_activations; + compute_tensor_statistics(ts); + struct tensor_comparer { bool legacy_mode; explicit tensor_comparer(const bool legacy) : legacy_mode(legacy) {} From fb2b09a43cf7d43435359763ba4a5596c9f456f4 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 13:06:37 +0000 Subject: [PATCH 077/123] Skip experts with zero count (unused) --- tools/imatrix/imatrix.cpp | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6090f3760f27..c7ce2b50e6e4 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -133,30 +133,40 @@ static std::vector compute_tensor_averages(const Stats & tstats) { std::vector vec; vec.reserve(len); + bool has_valid = false; if (tstats.activations.empty()) { - // Mean of squares + // Mean of squares (legacy: only values are available) for (size_t m = 0; m < n_mat; ++m) { - const auto c = (float)tstats.counts[m]; + const float c = (float) tstats.counts[m]; const size_t off = m * row; if (c <= 0.0f) { - vec.insert(vec.end(), row, 0.0f); // zero-fill rows for experts with zero count to preserve shape + for (size_t j = 0; j < row; ++j) { vec.push_back(0.0f); } continue; } - for (size_t j = 0; j < row; ++j) { vec.push_back(tstats.values[off + j] / c); } + + has_valid = true; + for (size_t j = 0; j < row; ++j) { + vec.push_back(tstats.values[off + j] / c); + } } } else { - // Mean + // Mean (new format: activations + values) for (size_t m = 0; m < n_mat; ++m) { - const auto c = (float)tstats.counts[m]; + const float c = (float) tstats.counts[m]; const size_t off = m * row; if (c <= 0.0f) { - vec.insert(vec.end(), row, 0.0f); // zero-fill rows for experts with zero count to preserve shape + for (size_t j = 0; j < row; ++j) { vec.push_back(0.0f); } continue; } - for (size_t j = 0; j < row; ++j) { vec.push_back(tstats.activations[off + j] / c); } + + has_valid = true; + for (size_t j = 0; j < row; ++j) { + vec.push_back(tstats.activations[off + j] / c); + } } } + if (!has_valid) { return {}; } return vec; } From 76566b83de3da24e4c741c845eeecaf24aa7e9aa Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 13:28:35 +0000 Subject: [PATCH 078/123] Enforce same-size between compared tensors --- tools/imatrix/imatrix.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index c7ce2b50e6e4..407b78fece47 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -360,11 +360,15 @@ static void compute_layer_statistics(const std::vector & tsta const auto prev_avg = compute_tensor_averages(it_prev->second); if (curr_avg.empty() || prev_avg.empty()) { continue; } - // Allow minor length mismatches by using the overlap - const size_t n = std::min(curr_avg.size(), prev_avg.size()); - if (n == 0) { continue; } + if (curr_avg.size() != prev_avg.size()) { + LOG_WRN("%s: size mismatch between '%s' and its previous-layer tensor '%s' (%zu vs %zu) - skipping this tensor pair in layer statistics\n", + __func__, ts.tensor.c_str(), prev_lyr.c_str(), curr_avg.size(), prev_avg.size()); + continue; + } // Compute statistics for each tensor pair individually + const size_t n = curr_avg.size(); + GGML_ASSERT(n > 0); double dot_prod = 0.0; double norm1_sq = 0.0; double norm2_sq = 0.0; From 1f3db496cc3b0d2de4243aa6ee565bf7860442d6 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 13:36:28 +0000 Subject: [PATCH 079/123] Calculate layer_sum only for legacy --- tools/imatrix/imatrix.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 407b78fece47..34d222e12c27 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1455,12 +1455,13 @@ static bool show_statistics(const common_params & params) { const float zd = (float)tstat.elements * tstat.zd_score; if (ls.find(blk) != ls.end()) { - ls[blk].layer_sum += tstat.sum_values; + if (legacy) { ls[blk].layer_sum += tstat.sum_values; } ls[blk].layer_zd += zd; ls[blk].n += tstat.elements; } else { layer_stats temp_ls; - temp_ls.layer_sum = tstat.sum_values; + if (legacy) { temp_ls.layer_sum = tstat.sum_values; } + else { temp_ls.layer_sum = 0.0f; } temp_ls.layer_zd = zd; temp_ls.n = tstat.elements; ls[blk] = temp_ls; From a2b86d7fd94f99f82a83b8ff09ece091ed0ff450 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 14:14:05 +0000 Subject: [PATCH 080/123] Minor refactoring --- tools/imatrix/imatrix.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 34d222e12c27..ecde8cdb2b56 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1347,6 +1347,7 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c } static bool show_statistics(const common_params & params) { + g_collector.set_params(params); std::vector ts; if (params.in_files.empty() || params.in_files.size() > 1) { LOG_ERR("\nError: a single imatrix file is required to compute tensor statistics\n\n"); @@ -1374,13 +1375,17 @@ static bool show_statistics(const common_params & params) { return false; } - bool legacy; if (has_activations && no_activations) { LOG_ERR("Error: %s has mixed tensors with and without activations\n\n", params.in_files[0].c_str()); return false; } - legacy = !has_activations; + const bool legacy = !has_activations; + if (!legacy && no_activations) { + LOG_ERR("Error: %s is in new-format but some tensors are missing activations.\n\n", params.in_files[0].c_str()); + return false; + } + compute_tensor_statistics(ts); struct tensor_comparer { From 658c6a830326a5a5a39d5054deafd1c6b6aa2863 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 14:46:21 +0000 Subject: [PATCH 081/123] Enforce tensor structure when aggregating multiple imatrix files --- tools/imatrix/imatrix.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ecde8cdb2b56..ce9fa3592d5c 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1043,15 +1043,22 @@ bool IMatrixCollector::load_imatrix(const char * file_name) { int64_t nval = ggml_nelements(in_sum2); if (e.values.empty()) { e.values.resize(nval, 0.0f); - if (in_sum != nullptr) { - e.activations.resize(nval, 0.0f); - } } else if ((size_t) nval != e.values.size()) { LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size()); gguf_free(ctx_gguf); ggml_free(ctx); return false; } + if (in_sum != nullptr) { + if (e.activations.empty()) { + e.activations.resize(nval, 0.0f); + } else if ((size_t) nval != e.activations.size()) { + LOG_ERR("%s: mismatched activations size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.activations.size()); + gguf_free(ctx_gguf); + ggml_free(ctx); + return false; + } + } int64_t ncounts = ggml_nelements(counts); if (e.counts.empty()) { From cdc7caea97cf9b0565001ffeecb06141fb54479c Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 14:46:45 +0000 Subject: [PATCH 082/123] Remove unreachable logic --- tools/imatrix/imatrix.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ce9fa3592d5c..6ccfa67ee5cf 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1388,11 +1388,6 @@ static bool show_statistics(const common_params & params) { } const bool legacy = !has_activations; - if (!legacy && no_activations) { - LOG_ERR("Error: %s is in new-format but some tensors are missing activations.\n\n", params.in_files[0].c_str()); - return false; - } - compute_tensor_statistics(ts); struct tensor_comparer { From bf9823afa7522d6b3f7448e4349a0a21d79d34a1 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 14:51:12 +0000 Subject: [PATCH 083/123] Minor refactoring --- tools/imatrix/imatrix.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6ccfa67ee5cf..69beee886812 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -766,9 +766,9 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { // Compute per-tensor statistics (CosSim, L2 Dist, ECS) to store alongside sums std::vector tstats; tstats.reserve(m_stats.size()); - bool legacy = true; + bool legacy_tensor = true; for (const auto & kv : m_stats) { - if (!compute_vector_statistics(tstats, kv.first, kv.second, legacy)) { + if (!compute_vector_statistics(tstats, kv.first, kv.second, legacy_tensor)) { LOG_WRN("%s: tensor %s has no data - skipping\n", __func__, kv.first.c_str()); } } From 8d97eee5574b923e595e83fef15b3d5b961d1ad2 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 17 Nov 2025 17:52:15 +0000 Subject: [PATCH 084/123] Improve layer 0 stats --- tools/imatrix/imatrix.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 69beee886812..18131f46decf 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -274,7 +274,7 @@ static bool compute_vector_statistics(std::vector & tstats, c static void compute_tensor_statistics(std::vector & tstats) { static const std::regex pattern(R"(blk\.(\d+)\.)"); for (auto & ts : tstats) { - ts.cossim = 0.0f; + ts.cossim = 1.0f; ts.l2_dist = 0.0f; if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { @@ -1495,9 +1495,20 @@ static bool show_statistics(const common_params & params) { for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) { continue; } const auto lcs = layer_cossim.find(layer); - const float layer_cs = lcs != layer_cossim.end() ? lcs->second : 0.0f; const auto ll2n = layer_l2_dist.find(layer); - const float layer_l2n = ll2n != layer_l2_dist.end() ? ll2n->second : 0.0f; + float layer_cs = 0.0f; + float layer_l2n = 0.0f; + + if (lcs != layer_cossim.end() && ll2n != layer_l2_dist.end()) { + layer_cs = lcs->second; + layer_l2n = ll2n->second; + } else if (layer == 0) { + layer_cs = 1.0f; + layer_l2n = 0.0f; + } else { + continue; + } + if (legacy) { LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\n", layer, @@ -1505,12 +1516,13 @@ static bool show_statistics(const common_params & params) { 100.0f * stats.layer_zd / stats.n, layer_cs); } else { + const float layer_ecs = 100.0f * (1.0f - std::exp(-0.01f * layer_l2n) * std::pow(std::fabs(layer_cs), 10.0f)); LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\t%8.4f\n", layer, layer_l2n, 100.0f * stats.layer_zd / stats.n, layer_cs, - 100.0f * (1.0f - std::exp(-0.01f * layer_l2n) * std::pow(std::fabs(layer_cs), 10.0f))); // Euclidean-Cosine score + layer_ecs); } } LOG_INF("\n"); From 4a0511f3a06f923bae55580f5b9c7b30b523168f Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 23 Nov 2025 22:18:12 +0000 Subject: [PATCH 085/123] Remove storing tensor statistics --- tools/imatrix/imatrix.cpp | 54 ++++++--------------------------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 18131f46decf..8876ba46e23f 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -757,28 +757,11 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); - data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * 4, GGML_MEM_ALIGN); } // deterministic tensor name order std::sort(to_store.begin(), to_store.end()); - // Compute per-tensor statistics (CosSim, L2 Dist, ECS) to store alongside sums - std::vector tstats; - tstats.reserve(m_stats.size()); - bool legacy_tensor = true; - for (const auto & kv : m_stats) { - if (!compute_vector_statistics(tstats, kv.first, kv.second, legacy_tensor)) { - LOG_WRN("%s: tensor %s has no data - skipping\n", __func__, kv.first.c_str()); - } - } - if (!tstats.empty()) { compute_tensor_statistics(tstats); } - - // index by tensor name - std::unordered_map tstat_index; - tstat_index.reserve(tstats.size()); - for (const auto & ts : tstats) { tstat_index[ts.tensor] = &ts; } - struct ggml_init_params params = { /* .mem_size = */ data_size, /* .mem_buffer = */ NULL, @@ -835,29 +818,6 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum); } } - - // Store per-tensor statistics as a small 1D tensor: [ECS, L2 Dist, CosSim, ZD Score] - { - float l2 = 0.0f; - float cs = 0.0f; - float zd = 0.0f; - float ecs = 0.0f; - auto it_ts = tstat_index.find(name); - if (it_ts != tstat_index.end() && it_ts->second != nullptr) { - l2 = it_ts->second->l2_dist; - cs = it_ts->second->cossim; - zd = it_ts->second->zd_score; - ecs = 100.0f * (1.0f - std::exp(-0.01f * l2) * std::pow(std::fabs(cs), 10.0f)); - } - - struct ggml_tensor * stats_t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4); - ggml_format_name(stats_t, "%s.stats", name.c_str()); - ((float *)stats_t->data)[0] = ecs; - ((float *)stats_t->data)[1] = l2; - ((float *)stats_t->data)[2] = cs; - ((float *)stats_t->data)[3] = zd; - gguf_add_tensor(ctx_gguf, stats_t); - } } gguf_write_to_file(ctx_gguf, fname.c_str(), false); @@ -1431,10 +1391,16 @@ static bool show_statistics(const common_params & params) { "==============================================================================================================" "=============================================================\n"); + // Euclidean-Cosine score + auto ecs = [](const float l2_dist, const float cossim) { + return 100.0f - (100.0f * (1.0f / (1.0f + ((2.0f / 3.0f) * l2_dist * l2_dist))) * ((1 + cossim) * 0.5f)); + }; + for (const auto & tstat : ts) { std::string layer; std::string name; process_tensor_name(tstat.tensor, layer, name); + const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; int blk; try { @@ -1443,9 +1409,6 @@ static bool show_statistics(const common_params & params) { blk = -1; // not a block layer } - const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; - const float ecs = 100.0f * (1.0f - std::exp(-0.01f * tstat.l2_dist) * std::pow(std::fabs(tstat.cossim), 10.0f)); // Euclidean-Cosine score - LOG_INF("%5s\t%-20s\t%11.4f\t%10.4f\t%10.4f\t%8.4f\t%8.4f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", layer.c_str(), name.c_str(), @@ -1456,7 +1419,7 @@ static bool show_statistics(const common_params & params) { tstat.std_deviation, tstat.elements, h_norm, - legacy ? tstat.entropy : ecs, + legacy ? tstat.entropy : ecs(tstat.l2_dist, tstat.cossim), 100.0f * tstat.zd_score, tstat.cossim); @@ -1516,13 +1479,12 @@ static bool show_statistics(const common_params & params) { 100.0f * stats.layer_zd / stats.n, layer_cs); } else { - const float layer_ecs = 100.0f * (1.0f - std::exp(-0.01f * layer_l2n) * std::pow(std::fabs(layer_cs), 10.0f)); LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\t%8.4f\n", layer, layer_l2n, 100.0f * stats.layer_zd / stats.n, layer_cs, - layer_ecs); + ecs(layer_l2n, layer_cs)); } } LOG_INF("\n"); From 6d82fa825a67aa93ade6b483b09099ede5f452f8 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 17:34:05 +0000 Subject: [PATCH 086/123] Add intermediate computation variables and Pearson --- tools/imatrix/imatrix.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 405f0a4967d1..4620cbbf7473 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -56,7 +56,12 @@ struct tensor_statistics { float entropy = 0.0f; float zd_score = 0.0f; float cossim = 0.0f; + float pearson = 0.0f; float l2_dist = 0.0f; + double dot_prod = 0.0; + double norm1_sq = 0.0; + double norm2_sq = 0.0; + double l2_dist_sq = 0.0; }; class IMatrixCollector { From d488bbb7c74b01712afee6251731797a05e1d0a0 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 17:36:10 +0000 Subject: [PATCH 087/123] Refactor compute_tensor_averages() --- tools/imatrix/imatrix.cpp | 44 +++++++++++++-------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 4620cbbf7473..d0bc19f9a1f9 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -131,44 +131,30 @@ static void process_tensor_name(const std::string & input, std::string & layer, static std::vector compute_tensor_averages(const Stats & tstats) { if (tstats.counts.empty()) { return {}; } + const size_t n_mat = tstats.counts.size(); const size_t len = !tstats.activations.empty() ? tstats.activations.size() : tstats.values.size(); if (len == 0 || n_mat == 0 || len % n_mat != 0) { return {}; } + const size_t row = len / n_mat; std::vector vec; - vec.reserve(len); + vec.resize(len); bool has_valid = false; - if (tstats.activations.empty()) { - // Mean of squares (legacy: only values are available) - for (size_t m = 0; m < n_mat; ++m) { - const float c = (float) tstats.counts[m]; - const size_t off = m * row; - if (c <= 0.0f) { - for (size_t j = 0; j < row; ++j) { vec.push_back(0.0f); } - continue; - } + const bool use_activations = !tstats.activations.empty(); - has_valid = true; - for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.values[off + j] / c); - } - } - } else { - // Mean (new format: activations + values) - for (size_t m = 0; m < n_mat; ++m) { - const float c = (float) tstats.counts[m]; - const size_t off = m * row; - if (c <= 0.0f) { - for (size_t j = 0; j < row; ++j) { vec.push_back(0.0f); } - continue; - } + for (size_t m = 0; m < n_mat; ++m) { + const auto c = (float) tstats.counts[m]; + const size_t off = m * row; - has_valid = true; - for (size_t j = 0; j < row; ++j) { - vec.push_back(tstats.activations[off + j] / c); - } - } + if (c <= 0.0f) { continue; } + + has_valid = true; + const float scale = 1.0f / c; + const float * src = use_activations ? &tstats.activations[off] : &tstats.values[off]; + float * dst = & vec[off]; + + for (size_t j = 0; j < row; ++j) { dst[j] = src[j] * scale; } } if (!has_valid) { return {}; } From 395367f210d0fd3cdfd09cd50269db23adc585fe Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 17:37:17 +0000 Subject: [PATCH 088/123] Refactor compute_vector_statistics() --- tools/imatrix/imatrix.cpp | 107 +++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index d0bc19f9a1f9..f1b4190ca803 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -165,16 +165,13 @@ static bool compute_vector_statistics(std::vector & tstats, c legacy = e.activations.empty(); const size_t n_mat = e.counts.size(); const size_t len = legacy ? e.values.size() : e.activations.size(); - if (n_mat == 0 || len == 0) { - LOG_ERR("%s: there's no data for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); - return false; - } - if (len % n_mat != 0) { - LOG_ERR("%s: activation size mismatch for tensor %s (len=%zu, counts=%zu)\n", __func__, name.c_str(), len, n_mat); + + if (n_mat == 0 || len == 0 || len % n_mat != 0) { + LOG_ERR("%s: data size mismatch or empty for tensor %s\n", __func__, name.c_str()); return false; } if (!legacy && e.values.size() != len) { - LOG_ERR("%s: activations/values size mismatch for tensor %s (act=%zu, val=%zu)\n", __func__, name.c_str(), len, e.values.size()); + LOG_ERR("%s: activations/values size mismatch for %s\n", __func__, name.c_str()); return false; } @@ -182,67 +179,74 @@ static bool compute_vector_statistics(std::vector & tstats, c double mean = 0.0; double M2 = 0.0; double sum = 0.0; - float vmin = std::numeric_limits::infinity(); - float vmax = -std::numeric_limits::infinity(); + float vmin = std::numeric_limits::max(); + float vmax = -std::numeric_limits::max(); double energy_sum = 0.0; size_t valid_n = 0; + + // Pass 1: Welford's Algorithm regarding aggregated elements for (size_t i = 0; i < n_mat; ++i) { const auto c = (float)e.counts[i]; - if (c <= 0.0f) { continue; } // skip experts with zero count + if (c <= 0.0f) { continue; } + + const double inv_c = 1.0 / (double)c; const size_t off = i * row_size; for (size_t j = 0; j < row_size; ++j) { - const double v_avg = legacy ? 0.0 : (double)e.activations[off + j] / (double)c; // E[x] - const double v_energy = (double)e.values[off + j] / (double)c; // E[x^2] - const double v = legacy ? v_energy : v_avg; + const double v_act = legacy ? 0.0 : (double)e.activations[off + j] * inv_c; + const double v_val = (double)e.values[off + j] * inv_c; + const double v = legacy ? v_val : v_act; // Use activation average for non-legacy - ++valid_n; - sum += v; - vmin = std::min(vmin, (float)v); - vmax = std::max(vmax, (float)v); + sum += v_val; + if ((float)v < vmin) { vmin = (float)v; } + if ((float)v > vmax) { vmax = (float)v; } + valid_n++; const double delta = v - mean; mean += delta / (double)valid_n; M2 += delta * (v - mean); - energy_sum += std::max(0.0, v_energy); + + // Energy for entropy uses v_val (E[x^2]) usually, or v_act^2? + // Existing logic used v_val (mean of squares) for entropy distribution. + if (v_val > 0.0) { energy_sum += v_val; } } } - if (valid_n == 0) { - LOG_ERR("%s: there's no data for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); - return false; - } + if (valid_n == 0) { return false; } float std_deviation = 0.0f; float entropy = 0.0f; double zd_count = 0.0; - double variance = valid_n > 1 ? M2 / ((double)valid_n - 1) : 0.0; - variance = std::max(variance, 0.0); - std_deviation = std::sqrt((float)variance); - if (energy_sum > 0.0) { - for (size_t i = 0; i < n_mat; ++i) { - const auto c = (float)e.counts[i]; - if (c <= 0.0f) { continue; } - const size_t off = i * row_size; - for (size_t j = 0; j < row_size; ++j) { - const double v_energy = (double)e.values[off + j] / (double)c; // E[x^2] - const double w = std::max(0.0, v_energy); - const double p = w / energy_sum; - if (p > 0.0) { entropy -= (float)(p * std::log2(p)); } + + const double variance = valid_n > 1 ? M2 / ((double)valid_n - 1) : 0.0; + std_deviation = std::sqrt((float)std::max(variance, 0.0)); + + // Pass 2: Entropy and Z-Score + const double inv_energy_sum = energy_sum > 0.0 ? 1.0 / energy_sum : 0.0; + const float inv_std = std_deviation > 0.0f ? 1.0f / std_deviation : 0.0f; + const float fmean = (float)mean; + const float log2_val = 1 / std::log2f(2); // 1.44269504089 + + for (size_t i = 0; i < n_mat; ++i) { + const auto c = (float)e.counts[i]; + if (c <= 0.0f) { continue; } + const double inv_c = 1.0 / (double)c; + const size_t off = i * row_size; + + for (size_t j = 0; j < row_size; ++j) { + // Entropy (Distribution of Energy) + if (inv_energy_sum > 0.0) { + const double v_energy = (double)e.values[off + j] * inv_c; + const double p = std::max(0.0, v_energy) * inv_energy_sum; + if (p > 1e-9) { entropy -= (float)(p * std::log(p) * log2_val); } } - } - } - if (std_deviation > 0.0f) { - for (size_t i = 0; i < n_mat; ++i) { - const auto c = (float)e.counts[i]; - if (c <= 0.0f) { continue; } - const size_t off = i * row_size; - for (size_t j = 0; j < row_size; ++j) { - const double v_avg = legacy ? 0.0 : (double)e.activations[off + j] / (double)c; // E[x] - const double v_energy = (double)e.values[off + j] / (double)c; // E[x^2] - const auto v = (float)(legacy ? v_energy : v_avg); - const float z = (v - (float)mean) / std_deviation; - if (std::fabs(z) > 1.0f) { zd_count += 1.0; } + + // Z-Score (Outlier detection) + if (std_deviation > 0.0f) { + const double v_act = legacy ? 0.0 : (double)e.activations[off + j] * inv_c; + const double v_val = (double)e.values[off + j] * inv_c; + const float v = (float)(legacy ? v_val : v_act); + if (std::fabs((v - fmean) * inv_std) > 1.0f) { zd_count += 1.0; } } } } @@ -256,9 +260,14 @@ static bool compute_vector_statistics(std::vector & tstats, c ts.min_values = vmin; ts.elements = (int)valid_n; ts.std_deviation = std_deviation; - ts.entropy = entropy; + ts.entropy = std::abs(entropy); // Ensure positive 0 ts.zd_score = (float)(zd_count / (double)valid_n); + // Default pairwise + ts.cossim = 1.0f; + ts.pearson = 1.0f; + ts.l2_dist = 0.0f; + return true; } From 309dc1231a73d9642afbe3fce849c3c00f5f852c Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 17:38:47 +0000 Subject: [PATCH 089/123] Refactor compute_tensor_statistics() --- tools/imatrix/imatrix.cpp | 137 ++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 49 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f1b4190ca803..736a80c4e5e5 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -272,60 +272,99 @@ static bool compute_vector_statistics(std::vector & tstats, c } static void compute_tensor_statistics(std::vector & tstats) { - static const std::regex pattern(R"(blk\.(\d+)\.)"); + std::unordered_map tensor_map; + tensor_map.reserve(tstats.size()); + for (size_t i = 0; i < tstats.size(); ++i) { tensor_map[tstats[i].tensor] = i; } + for (auto & ts : tstats) { - ts.cossim = 1.0f; - ts.l2_dist = 0.0f; - - if (std::smatch match; std::regex_search(ts.tensor, match, pattern)) { - const int blk = std::stoi(match[1]); - if (blk <= 0) { continue; } - std::string tname(ts.tensor); - tname.replace(match.position(1), match.length(1), std::to_string(blk - 1)); - auto prev_it = std::find_if(tstats.begin(), tstats.end(), - [tname](const tensor_statistics & t) { return t.tensor == tname; }); - if (prev_it == tstats.end()) { - LOG_WRN("%s: missing previous-layer tensor '%s' (current: '%s'). Statistics may not be accurate\n", - __func__, tname.c_str(), ts.tensor.c_str()); - continue; - } + std::string layer_str; + std::string dummy_tensor; + process_tensor_name(ts.tensor, layer_str, dummy_tensor); - const auto curr_avg = compute_tensor_averages(ts.stats); - const auto prev_avg = compute_tensor_averages(prev_it->stats); - if (curr_avg.empty() || curr_avg.size() != prev_avg.size()) { - LOG_WRN("%s: size mismatch between '%s' and its previous-layer tensor '%s' (%zu vs %zu). Statistics may not be accurate\n", - __func__, ts.tensor.c_str(), tname.c_str(), curr_avg.size(), prev_avg.size()); - continue; - } + // Robust block ID extraction + int blk = -1; + try { blk = std::stoi(layer_str); } catch (...) { continue; } + if (blk <= 0) { continue; } - float dot_prod = 0.0f; - float norm1_sq = 0.0f; - float norm2_sq = 0.0f; - float l2_dist_sq = 0.0f; - - for (size_t i = 0; i < curr_avg.size(); ++i) { - const float c_val = curr_avg[i]; - const float p_val = prev_avg[i]; - dot_prod += c_val * p_val; - norm1_sq += c_val * c_val; - norm2_sq += p_val * p_val; - const float diff = c_val - p_val; - l2_dist_sq += diff * diff; - } + // Reconstruct previous layer name + const size_t blk_start_pos = ts.tensor.find("blk." + layer_str); + if (blk_start_pos == std::string::npos) { continue; } - // Compute Cosine Similarity - float cs = 0.0f; - if (norm1_sq > 0.0f && norm2_sq > 0.0f) { - cs = dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq)); - cs = std::min(cs, 1.0f); - cs = std::max(cs, -1.0f); - } else if (norm1_sq == 0.0f && norm2_sq == 0.0f) { - cs = 1.0f; - } - ts.cossim = cs; + std::string tname = ts.tensor; + tname.replace(blk_start_pos, layer_str.length() + 4, "blk." + std::to_string(blk - 1)); + + auto it = tensor_map.find(tname); + if (it == tensor_map.end()) { + LOG_WRN("%s: missing previous-layer tensor '%s'\n", __func__, tname.c_str()); + continue; + } + + const auto & prev_ts = tstats[it->second]; + const auto curr_avg = compute_tensor_averages(ts.stats); + const auto prev_avg = compute_tensor_averages(prev_ts.stats); + + if (curr_avg.empty() || curr_avg.size() != prev_avg.size()) { continue; } + + double dot_prod = 0.0; + double norm1_sq = 0.0; + double norm2_sq = 0.0; + double l2_dist_sq = 0.0; - // Compute L2 Norm (Euclidean Distance) - ts.l2_dist = std::sqrt(l2_dist_sq); + // Aux variables for Pearson (Spatial Covariance) + double sum_c = 0.0; + double sum_p = 0.0; + const size_t n = curr_avg.size(); + + // Pass 1: Sums for Means + for (size_t i = 0; i < n; ++i) { + sum_c += curr_avg[i]; + sum_p += prev_avg[i]; + } + const double mean_c = sum_c / n; + const double mean_p = sum_p / n; + + double cov_sum = 0.0; + double var_c_sum = 0.0; + double var_p_sum = 0.0; + + // Pass 2: Metrics + for (size_t i = 0; i < n; ++i) { + const double c_val = curr_avg[i]; + const double p_val = prev_avg[i]; + + // Cosine Similarity & L2 basics + dot_prod += c_val * p_val; + norm1_sq += c_val * c_val; + norm2_sq += p_val * p_val; + const double diff = c_val - p_val; + l2_dist_sq += diff * diff; + + // Pearson (Centered stats) + const double dc = c_val - mean_c; + const double dp = p_val - mean_p; + cov_sum += dc * dp; + var_c_sum += dc * dc; + var_p_sum += dp * dp; + } + + ts.dot_prod = dot_prod; + ts.norm1_sq = norm1_sq; + ts.norm2_sq = norm2_sq; + ts.l2_dist_sq = l2_dist_sq; + ts.l2_dist = (float)std::sqrt(l2_dist_sq); + + if (norm1_sq > 0.0 && norm2_sq > 0.0) { + ts.cossim = (float)(dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq))); + ts.cossim = std::clamp(ts.cossim, -1.0f, 1.0f); + } else { + ts.cossim = (norm1_sq == 0.0 && norm2_sq == 0.0) ? 1.0f : 0.0f; + } + + if (var_c_sum > 0.0 && var_p_sum > 0.0) { + ts.pearson = (float)(cov_sum / (std::sqrt(var_c_sum) * std::sqrt(var_p_sum))); + ts.pearson = std::clamp(ts.pearson, -1.0f, 1.0f); + } else { + ts.pearson = (var_c_sum == 0.0 && var_p_sum == 0.0) ? 1.0f : 0.0f; } } } From e69058c6eca281266d7ad4146b6c77a8f534cf14 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 17:40:29 +0000 Subject: [PATCH 090/123] Refactor compute_layer_statistics() --- tools/imatrix/imatrix.cpp | 71 ++++++++++----------------------------- 1 file changed, 18 insertions(+), 53 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 736a80c4e5e5..3d30b5b91a1b 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -372,86 +372,51 @@ static void compute_tensor_statistics(std::vector & tstats) { static void compute_layer_statistics(const std::vector & tstats, std::map & layer_cossim, std::map & layer_l2_dist, - const std::unordered_map & stats_map) { + std::map & layer_pearson) { struct layer_aggregation { double sum_dot_prod = 0.0; double sum_norm1_sq = 0.0; double sum_norm2_sq = 0.0; double sum_l2_dist_sq = 0.0; + double sum_pearson = 0.0; int n_tensors = 0; }; - static const std::regex pattern(R"(blk\.(\d+)\.)"); std::map l_agr; + for (const auto & ts : tstats) { - std::smatch match; - if (!std::regex_search(ts.tensor, match, pattern)) { continue; } - const int blk = std::stoi(match[1]); + std::string layer_str; + std::string dummy; + process_tensor_name(ts.tensor, layer_str, dummy); + int blk = -1; + try { blk = std::stoi(layer_str); } catch(...) { continue; } if (blk <= 0) { continue; } - std::string prev_lyr(ts.tensor); - prev_lyr.replace(match.position(1), match.length(1), std::to_string(blk - 1)); - auto it_curr = stats_map.find(ts.tensor); - auto it_prev = stats_map.find(prev_lyr); - if (it_curr == stats_map.end() || it_prev == stats_map.end()) { continue; } - - const auto curr_avg = compute_tensor_averages(it_curr->second); - const auto prev_avg = compute_tensor_averages(it_prev->second); - if (curr_avg.empty() || prev_avg.empty()) { continue; } - - if (curr_avg.size() != prev_avg.size()) { - LOG_WRN("%s: size mismatch between '%s' and its previous-layer tensor '%s' (%zu vs %zu) - skipping this tensor pair in layer statistics\n", - __func__, ts.tensor.c_str(), prev_lyr.c_str(), curr_avg.size(), prev_avg.size()); - continue; - } - - // Compute statistics for each tensor pair individually - const size_t n = curr_avg.size(); - GGML_ASSERT(n > 0); - double dot_prod = 0.0; - double norm1_sq = 0.0; - double norm2_sq = 0.0; - double l2_dist_sq = 0.0; - for (size_t i = 0; i < n; ++i) { - const double a = curr_avg[i]; - const double b = prev_avg[i]; - dot_prod += a * b; - norm1_sq += a * a; - norm2_sq += b * b; - const double d = a - b; - l2_dist_sq += d * d; - } + if (ts.norm1_sq == 0.0 && ts.norm2_sq == 0.0 && ts.l2_dist_sq == 0.0) { continue; } - // Accumulate statistics for the layer auto & entry = l_agr[blk]; - entry.sum_dot_prod += dot_prod; - entry.sum_norm1_sq += norm1_sq; - entry.sum_norm2_sq += norm2_sq; - entry.sum_l2_dist_sq += l2_dist_sq; + entry.sum_dot_prod += ts.dot_prod; + entry.sum_norm1_sq += ts.norm1_sq; + entry.sum_norm2_sq += ts.norm2_sq; + entry.sum_l2_dist_sq += ts.l2_dist_sq; + entry.sum_pearson += ts.pearson; entry.n_tensors++; } - // Compute aggregated layer statistics - for (const auto & kv : l_agr) { - const int layer = kv.first; - const auto & agg = kv.second; + for (const auto & [layer, agg] : l_agr) { if (agg.n_tensors == 0) { continue; } - // Compute aggregated Cosine Similarity float cossim = 0.0f; if (agg.sum_norm1_sq > 0.0 && agg.sum_norm2_sq > 0.0) { cossim = (float)(agg.sum_dot_prod / (std::sqrt(agg.sum_norm1_sq) * std::sqrt(agg.sum_norm2_sq))); - cossim = std::min(cossim, 1.0f); - cossim = std::max(cossim, -1.0f); + cossim = std::clamp(cossim, -1.0f, 1.0f); } else if (agg.sum_norm1_sq == 0.0 && agg.sum_norm2_sq == 0.0) { - cossim = 1.0f; // both vectors are zero then CosSim is 1 - } else { - cossim = 0.0f; // One zero and the other non-zero then CosSim is 0 + cossim = 1.0f; } - // Compute aggregated L2 Distance (Euclidean Distance) layer_cossim[layer] = cossim; layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); + layer_pearson[layer] = (float)(agg.sum_pearson / agg.n_tensors); } } From fdc2def79c9012b44106ed7d74f5d4d55dc35e7a Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 17:41:04 +0000 Subject: [PATCH 091/123] Refactor show_statistics() --- tools/imatrix/imatrix.cpp | 242 +++++++++++++++++++------------------- 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 3d30b5b91a1b..045fc671c162 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1320,179 +1320,179 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c static bool show_statistics(const common_params & params) { g_collector.set_params(params); std::vector ts; - if (params.in_files.empty() || params.in_files.size() > 1) { - LOG_ERR("\nError: a single imatrix file is required to compute tensor statistics\n\n"); - return false; - } - bool has_activations = false; - bool no_activations = false; + if (params.in_files.empty()) { return false; } + + // Load and process data if (g_collector.load_imatrix(params.in_files[0].c_str())) { + ts.reserve(g_collector.get_mstats().size()); for (const auto & [name, stats] : g_collector.get_mstats()) { bool legacy_imatrix = true; - if (!compute_vector_statistics(ts, name, stats, legacy_imatrix)) { - LOG_WRN("%s: tensor %s has no data - skipping\n", __func__, name.c_str()); - continue; - } - if (legacy_imatrix) { no_activations = true; } - else { has_activations = true; } + if (!compute_vector_statistics(ts, name, stats, legacy_imatrix)) { continue; } } } else { - LOG_ERR("\nError: %s is not a valid imatrix file\n\n", params.in_files[0].c_str()); - return false; - } - if (ts.empty()) { - LOG_ERR("Error: cannot compute statistics for %s\n\n", params.in_files[0].c_str()); return false; } - if (has_activations && no_activations) { - LOG_ERR("Error: %s has mixed tensors with and without activations\n\n", params.in_files[0].c_str()); - return false; - } + if (ts.empty()) { return false; } - const bool legacy = !has_activations; + bool legacy = ts.empty() ? true : ts[0].stats.activations.empty(); compute_tensor_statistics(ts); + // Sorting logic (Layer index -> Tensor Name) struct tensor_comparer { - bool legacy_mode; - explicit tensor_comparer(const bool legacy) : legacy_mode(legacy) {} - bool operator()(const tensor_statistics & a, const tensor_statistics & b) const { - std::string layer; + std::string lay_a; + std::string lay_b; std::string name_a; std::string name_b; - process_tensor_name(a.tensor, layer, name_a); - process_tensor_name(b.tensor, layer, name_b); - return legacy_mode ? name_a < name_b || (name_a == name_b && a.sum_values > b.sum_values) - : name_a < name_b || (name_a == name_b && a.cossim > b.cossim); + process_tensor_name(a.tensor, lay_a, name_a); + process_tensor_name(b.tensor, lay_b, name_b); + + // Handle non-numeric layers (e.g., "output") + int blk_a = 9999; + int blk_b = 9999; + try { + blk_a = std::stoi(lay_a); + } catch(...) { + if (lay_a == "output") { blk_a = 10000; } + } + try { + blk_b = std::stoi(lay_b); + } catch(...) { + if (lay_b == "output") { blk_b = 10000; } + } + + if (blk_a != blk_b) { return blk_a < blk_b; } + return name_a < name_b; } }; - std::sort(ts.begin(), ts.end(), tensor_comparer(legacy)); + std::sort(ts.begin(), ts.end(), tensor_comparer()); struct layer_stats { float layer_sum = 0.0f; float layer_zd = 0.0f; int n = 0; }; - std::map ls; - LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); - LOG_INF("\n%6s\t%18s\t%13s\t%8s\t%8s\t%7s\t%15s\t%13s\t%11s\t%8s\t%5s\t%10s\n", - "Layer", - "Tensor", - legacy ? "Σ E[Act²]" : "L₂ Dist", - "Min", - "Max", - "μ", - "σ", - "N", - "H Norm", - legacy ? "H" : "ECS", - "ZD", - "CosSim"); - LOG_INF( - "==============================================================================================================" - "=============================================================\n"); - - // Euclidean-Cosine score - auto ecs = [](const float l2_dist, const float cossim) { - return 100.0f - (100.0f * (1.0f / (1.0f + ((2.0f / 3.0f) * l2_dist * l2_dist))) * ((1 + cossim) * 0.5f)); + + // Helper to shorten names for table formatting "blk.10.attn_k.weight" -> "..10.attn_k.weight" + auto label_fmt = [](std::string s, size_t w) -> std::string { + if (s.length() <= w) { return s; } + return ".." + s.substr(s.length() - (w - 2)); }; + // Table Constants + constexpr int w_lay = 6; + constexpr int w_nam = 40; // Wide enough for most tensors + const auto * sep = " | "; + + LOG_INF("\nComputing tensor statistics (%d tensors)\n", static_cast(ts.size())); + + // Header logic separated to handle different column counts + if (legacy) { + LOG_INF("\n%*s%s%-*s%s%10s %12s %10s %10s%s %8s %8s%s%17s %8s %8s\n", + w_lay, "Layer", sep, + w_nam, "Tensor", sep, + "Min", "Max", "Mean", "StdDev", sep, + "H_Norm", "ZD", sep, + "∑ E[A²]", "CosSim", "PCC"); + LOG_INF("%s\n", std::string(154, '-').c_str()); + } else { + LOG_INF("\n%*s%s%-*s%s%10s %12s %10s %10s%s %8s %8s%s%17s %12s %8s %8s\n", + w_lay, "Layer", sep, + w_nam, "Tensor", sep, + "Min", "Max", "Mean", "StdDev", sep, + "H_Norm", "ZD", sep, + "∑ E[A²]", "L2 Dist", "CosSim", "PCC"); + LOG_INF("%s\n", std::string(167, '-').c_str()); + } + for (const auto & tstat : ts) { std::string layer; std::string name; process_tensor_name(tstat.tensor, layer, name); + + // Calculate metrics const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; int blk; - try { - blk = std::stoi(layer); - } catch (const std::exception &) { - blk = -1; // not a block layer - } - - LOG_INF("%5s\t%-20s\t%11.4f\t%10.4f\t%10.4f\t%8.4f\t%8.4f\t%7d\t%10.2f%%\t%10.4f\t%6.2f%%\t%10.4f\n", - layer.c_str(), - name.c_str(), - legacy ? tstat.sum_values : tstat.l2_dist, - tstat.min_values, - tstat.max_values, - tstat.mean_values, - tstat.std_deviation, - tstat.elements, - h_norm, - legacy ? tstat.entropy : ecs(tstat.l2_dist, tstat.cossim), - 100.0f * tstat.zd_score, - tstat.cossim); + try { blk = std::stoi(layer); } catch (...) { blk = -1; } - const float zd = (float)tstat.elements * tstat.zd_score; - if (ls.find(blk) != ls.end()) { - if (legacy) { ls[blk].layer_sum += tstat.sum_values; } - ls[blk].layer_zd += zd; - ls[blk].n += tstat.elements; + // Print Row + if (legacy) { + LOG_INF("%*s%s%-*s%s%10.4f %12.4f %10.4f %10.4f%s%8.2f%% %8.2f%%%s%14.4f %8.4f %8.4f\n", + w_lay, layer.c_str(), sep, + w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, + tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, sep, + h_norm, 100.0f * tstat.zd_score, sep, + tstat.sum_values, tstat.cossim, tstat.pearson + ); } else { - layer_stats temp_ls; - if (legacy) { temp_ls.layer_sum = tstat.sum_values; } - else { temp_ls.layer_sum = 0.0f; } - temp_ls.layer_zd = zd; - temp_ls.n = tstat.elements; - ls[blk] = temp_ls; + // Display L2 Dist AND Sum E[A^2] + LOG_INF("%*s%s%-*s%s%10.4f %12.4f %10.4f %10.4f%s%8.2f%% %8.2f%%%s%14.4f %12.4f %8.4f %8.4f\n", + w_lay, layer.c_str(), sep, + w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, + tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, sep, + h_norm, 100.0f * tstat.zd_score, sep, + tstat.sum_values, tstat.l2_dist, tstat.cossim, tstat.pearson + ); } + + // Aggregate Layer Stats + const float zd = (float)tstat.elements * tstat.zd_score; + auto & l_entry = ls[blk]; + + // Accumulate sum values regardless of legacy status to allow display in both modes + l_entry.layer_sum += tstat.sum_values; + l_entry.layer_zd += zd; + l_entry.n += tstat.elements; } + // --- Computed Layer Statistics --- + std::map layer_cossim; std::map layer_l2_dist; - compute_layer_statistics(ts, layer_cossim, layer_l2_dist, g_collector.get_mstats()); - - const size_t layers = std::count_if(ls.begin(), ls.end(), [](const auto & kv) { return kv.first >= 0; }); - LOG_INF("\nComputing layer statistics (%zu layers)\n", layers); - LOG_INF("\n%6s\t%13s\t%6s\t%11s\t%6s\n", - "Layer", - legacy ? "Σ E[Act²]" : "L₂ Dist", - "ZD", - "CosSim", - legacy ? "" : "ECS"); + std::map layer_pearson; + compute_layer_statistics(ts, layer_cossim, layer_l2_dist, layer_pearson); + + LOG_INF("\n\nComputing layer statistics (%zu layers)\n\n", ls.size() - 2); + + // Layer Table Headers if (legacy) { - LOG_INF("============================================\n"); + LOG_INF("%*s%s%9s%s%17s %10s %10s\n", + w_lay, "Layer", sep, + "ZD", sep, + "∑ E[A²]", "CosSim", "PCC"); + LOG_INF("%s\n", std::string(57, '-').c_str()); } else { - LOG_INF("=========================================================\n"); + LOG_INF("%*s%s%9s%s%17s %14s %10s %10s\n", + w_lay, "Layer", sep, + "ZD", sep, + "∑ E[A²]", "L2 Dist", "CosSim", "PCC"); + LOG_INF("%s\n", std::string(72, '-').c_str()); } + for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) { continue; } - const auto lcs = layer_cossim.find(layer); - const auto ll2n = layer_l2_dist.find(layer); - float layer_cs = 0.0f; - float layer_l2n = 0.0f; - - if (lcs != layer_cossim.end() && ll2n != layer_l2_dist.end()) { - layer_cs = lcs->second; - layer_l2n = ll2n->second; - } else if (layer == 0) { - layer_cs = 1.0f; - layer_l2n = 0.0f; - } else { - continue; - } + + float lcs = layer == 0 ? 1.0f : layer_cossim[layer]; + float ll2 = layer == 0 ? 0.0f : layer_l2_dist[layer]; + float lpc = layer == 0 ? 1.0f : layer_pearson[layer]; if (legacy) { - LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\n", - layer, - stats.layer_sum, - 100.0f * stats.layer_zd / stats.n, - layer_cs); + LOG_INF("%*d%s%8.2f%%%s%14.4f %10.4f %10.4f\n", + w_lay, layer, sep, + 100.0f * stats.layer_zd / stats.n, sep, + stats.layer_sum, lcs, lpc); } else { - LOG_INF("%5d\t%11.4f\t%6.2f%%\t%11.4f\t%8.4f\n", - layer, - layer_l2n, - 100.0f * stats.layer_zd / stats.n, - layer_cs, - ecs(layer_l2n, layer_cs)); + LOG_INF("%*d%s%8.2f%%%s%14.4f %14.4f %10.4f %10.4f\n", + w_lay, layer, sep, + 100.0f * stats.layer_zd / stats.n, sep, + stats.layer_sum, ll2, lcs, lpc); } } LOG_INF("\n"); - return true; } From c88f288df9b2fdfe1d709a4c3fe36175223c2af8 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 18:25:40 +0000 Subject: [PATCH 092/123] Update README.md --- tools/imatrix/README.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index e823d25c19ae..342c12669890 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -41,7 +41,7 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def ./llama-imatrix -m ggml-model-f16.gguf -f calibration-data.txt -ngl 99 # use the imatrix to perform a Q4_K_M quantization -./llama-quantize --imatrix imatrix.gguf ggml-model-f16.gguf ./ggml-model-q4_k_m.gguf q4_k_m +./llama-quantize --imatrix imatrix.gguf ggml-model-f16.gguf ./ggml-model-q4_k_m.gguf q4_k_m 99 ``` ```bash @@ -70,7 +70,7 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def ``` ```bash -# analyse imatrix file and display summary statistics instead of running inference +# analyze imatrix file and display summary statistics instead of running inference ./llama-imatrix --in-file imatrix.gguf --show-statistics ``` @@ -78,21 +78,22 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def #### Per tensor -* **Σ(Act²)** *(legacy mode)* / **L₂ Norm** *(preferred)*: If in legacy mode, the raw sum of squares of activations (sum of `Act²`). In preferred mode, the Euclidean Distance (L₂ Norm) between this tensor’s average activations and those of the previous layer. +* **∑ E[A²]**: The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision (e.g., Q6_K vs Q4_K). +* **L₂ Norm**: Euclidean Distance from the tensor in the previous layer. Measure of transformation magnitude; higher values indicate more significant transformation on the data. * **Min / Max / μ / σ**: Tensor elements Min, Max, Mean, and Standard Deviation. -* **N**: Number of tensor elements considered. -* **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. -* **H** *(legacy mode)* / **ECS** *(preferred)*: If legacy, Shannon Entropy defined as $H = -\sum_{i=1}^N p_i \log_2 p_i$. If preferred, *Euclidean-Cosine Score* defined as $ECS = K \cdot e^{-\alpha a} \cdot |b|^{\gamma}$ where `a = L₂ Norm`, `b = Cosine Similarity`, `α = 0.01`, `γ = 10` between this tensor’s elements and those of the previous layer. Higher score means more similarity and lower change. -* **ZD**: % of elements whose Z-score is > 1.0 in magnitude (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415) -* **CosSim**: Cosine Similarity of the mean activations between this tensor’s elements and those of the previous layer. +* **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. Higher values indicate more uniform distribution of activations. Every neuron is firing equally; hard to prune. +* **ZD**: % of elements whose ZD-score is > 1.0 (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415). +* **CosSim**: Cosine Similarity with the tensor in the previous layer. _~1.0_, the tensor output points in the exact same direction as the previous layer's tensor (the layer is refining magnitude, not direction). _< 1.0_, the layer is rotating the vector space (changing semantic meaning). +* **PCC**: Pearson Correlation Coefficient with the tensor in the previous layer. Checks for linear correlation excluding the mean shift. Similar to CosSim but centers geometric data first. Indicates if the pattern of activation changes or just the offset. #### Per layer Aggregated metrics per block/layer: -* **Σ(Act²)** *(legacy mode)* / **L₂ Norm** *(preferred)*: If in legacy mode, the sum of squared activations (sum of Act²) for the layer's concatenated tensors. In preferred mode, the Euclidean Distance (L₂ Norm) between this layer's average concatenated tensor activations the previous layer. -* **ZD**: % of this layer's concatenated tensors' elements with |Z| > 1. -* **CosSim**: Cosine Similarity of the mean activations between this layer's concatenated tensors' elements compared and the previous layer’s. -* **ECS** *(preferred only)*: Euclidean-Cosine Score applied to the layer. +* **∑ E[A²]:** Total energy of the layer's concatenated tensors. Indicates the layer's overall contribution amplitude. +* **L₂ Norm:** Euclidean Distance of the layer's concatenated tensors from the previous layer’s. Global measure of transformation magnitude. +* **ZD**: % of this layer's concatenated tensors' elements with |Z| > 1. Indicates general "spikiness" of the layer's activations. +* **CosSim**: Cosine Similarity of this layer's concatenated tensors with the previous layer. +* **PCC**: Average Pearson Correlation of the tensors in the layer. More information is available in https://github.com/ggml-org/llama.cpp/pull/14891 From a297a158e31f361eb4da2e55dcfd94938fb5ed37 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 11 Jan 2026 18:34:10 +0000 Subject: [PATCH 093/123] Fix typo --- tools/imatrix/README.md | 6 +++--- tools/imatrix/imatrix.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 342c12669890..8314a440ee4b 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -78,11 +78,11 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def #### Per tensor -* **∑ E[A²]**: The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision (e.g., Q6_K vs Q4_K). -* **L₂ Norm**: Euclidean Distance from the tensor in the previous layer. Measure of transformation magnitude; higher values indicate more significant transformation on the data. * **Min / Max / μ / σ**: Tensor elements Min, Max, Mean, and Standard Deviation. * **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. Higher values indicate more uniform distribution of activations. Every neuron is firing equally; hard to prune. * **ZD**: % of elements whose ZD-score is > 1.0 (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415). +* **∑ E[A²]**: The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision (e.g., Q6_K vs Q4_K). +* **L₂ Norm**: Euclidean Distance from the tensor in the previous layer. Measure of transformation magnitude; higher values indicate more significant transformation on the data. * **CosSim**: Cosine Similarity with the tensor in the previous layer. _~1.0_, the tensor output points in the exact same direction as the previous layer's tensor (the layer is refining magnitude, not direction). _< 1.0_, the layer is rotating the vector space (changing semantic meaning). * **PCC**: Pearson Correlation Coefficient with the tensor in the previous layer. Checks for linear correlation excluding the mean shift. Similar to CosSim but centers geometric data first. Indicates if the pattern of activation changes or just the offset. @@ -90,9 +90,9 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def Aggregated metrics per block/layer: +* **ZD**: % of this layer's concatenated tensors' elements with |Z| > 1. Indicates general "spikiness" of the layer's activations. * **∑ E[A²]:** Total energy of the layer's concatenated tensors. Indicates the layer's overall contribution amplitude. * **L₂ Norm:** Euclidean Distance of the layer's concatenated tensors from the previous layer’s. Global measure of transformation magnitude. -* **ZD**: % of this layer's concatenated tensors' elements with |Z| > 1. Indicates general "spikiness" of the layer's activations. * **CosSim**: Cosine Similarity of this layer's concatenated tensors with the previous layer. * **PCC**: Average Pearson Correlation of the tensors in the layer. diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 045fc671c162..0697440c66bf 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1395,7 +1395,7 @@ static bool show_statistics(const common_params & params) { w_lay, "Layer", sep, w_nam, "Tensor", sep, "Min", "Max", "Mean", "StdDev", sep, - "H_Norm", "ZD", sep, + "H Norm", "ZD", sep, "∑ E[A²]", "CosSim", "PCC"); LOG_INF("%s\n", std::string(154, '-').c_str()); } else { @@ -1403,7 +1403,7 @@ static bool show_statistics(const common_params & params) { w_lay, "Layer", sep, w_nam, "Tensor", sep, "Min", "Max", "Mean", "StdDev", sep, - "H_Norm", "ZD", sep, + "H Norm", "ZD", sep, "∑ E[A²]", "L2 Dist", "CosSim", "PCC"); LOG_INF("%s\n", std::string(167, '-').c_str()); } From 91d31bd73245997f021798b792075d73d0ec155e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 17 Jan 2026 11:41:02 +0000 Subject: [PATCH 094/123] Refactor variable name --- tools/imatrix/imatrix.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 0697440c66bf..b8305f9693cf 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -382,7 +382,7 @@ static void compute_layer_statistics(const std::vector & tsta int n_tensors = 0; }; - std::map l_agr; + std::map laggr; for (const auto & ts : tstats) { std::string layer_str; @@ -394,7 +394,7 @@ static void compute_layer_statistics(const std::vector & tsta if (ts.norm1_sq == 0.0 && ts.norm2_sq == 0.0 && ts.l2_dist_sq == 0.0) { continue; } - auto & entry = l_agr[blk]; + auto & entry = laggr[blk]; entry.sum_dot_prod += ts.dot_prod; entry.sum_norm1_sq += ts.norm1_sq; entry.sum_norm2_sq += ts.norm2_sq; @@ -403,7 +403,7 @@ static void compute_layer_statistics(const std::vector & tsta entry.n_tensors++; } - for (const auto & [layer, agg] : l_agr) { + for (const auto & [layer, agg] : laggr) { if (agg.n_tensors == 0) { continue; } float cossim = 0.0f; From 2fd301e02c016a374167d4c0356f2489e2d86141 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 17 Jan 2026 11:42:25 +0000 Subject: [PATCH 095/123] Don't display layer statistics if there are gaps in the sequence --- tools/imatrix/imatrix.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index b8305f9693cf..f75e9b47626a 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1456,9 +1456,29 @@ static bool show_statistics(const common_params & params) { std::map layer_pearson; compute_layer_statistics(ts, layer_cossim, layer_l2_dist, layer_pearson); - LOG_INF("\n\nComputing layer statistics (%zu layers)\n\n", ls.size() - 2); + size_t layers = 0; + int min = std::numeric_limits::max(); + int max = -1; - // Layer Table Headers + for (const auto & [layer, stats] : ls) { + if (layer >= 0 && stats.n > 0) { + layers++; + min = std::min(layer, min); + max = std::max(layer, max); + } + } + + if (layers > 0) { + const auto expected = (size_t)(max - min + 1); + if (layers != expected) { + LOG_WRN("\n%s: layer sequence gap detected (found %zu layers in range %d-%d, expected %zu); layer statistics will not be shown\n", + __func__, layers, min, max, expected); + return false; + } + } + + LOG_INF("\n\nComputing layer statistics for %s (%zu layers)\n\n", params.in_files[0].c_str(), layers); + if (legacy) { LOG_INF("%*s%s%9s%s%17s %10s %10s\n", w_lay, "Layer", sep, From b6fc86b32b227c61b44c841f3acc637cd02193e3 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 17 Jan 2026 11:43:29 +0000 Subject: [PATCH 096/123] Display NaN if statistic is uninterpretable --- tools/imatrix/imatrix.cpp | 63 +++++++++++++++------------------------ 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f75e9b47626a..f65c13aec968 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -184,7 +184,7 @@ static bool compute_vector_statistics(std::vector & tstats, c double energy_sum = 0.0; size_t valid_n = 0; - // Pass 1: Welford's Algorithm regarding aggregated elements + // Pass 1: Mean, Min, Max, Std Dev for (size_t i = 0; i < n_mat; ++i) { const auto c = (float)e.counts[i]; if (c <= 0.0f) { continue; } @@ -206,8 +206,6 @@ static bool compute_vector_statistics(std::vector & tstats, c mean += delta / (double)valid_n; M2 += delta * (v - mean); - // Energy for entropy uses v_val (E[x^2]) usually, or v_act^2? - // Existing logic used v_val (mean of squares) for entropy distribution. if (v_val > 0.0) { energy_sum += v_val; } } } @@ -225,7 +223,7 @@ static bool compute_vector_statistics(std::vector & tstats, c const double inv_energy_sum = energy_sum > 0.0 ? 1.0 / energy_sum : 0.0; const float inv_std = std_deviation > 0.0f ? 1.0f / std_deviation : 0.0f; const float fmean = (float)mean; - const float log2_val = 1 / std::log2f(2); // 1.44269504089 + const float log2_val = 1 / std::log2f(2); for (size_t i = 0; i < n_mat; ++i) { const auto c = (float)e.counts[i]; @@ -241,7 +239,7 @@ static bool compute_vector_statistics(std::vector & tstats, c if (p > 1e-9) { entropy -= (float)(p * std::log(p) * log2_val); } } - // Z-Score (Outlier detection) + // Z-Score Density (Outlier detection) if (std_deviation > 0.0f) { const double v_act = legacy ? 0.0 : (double)e.activations[off + j] * inv_c; const double v_val = (double)e.values[off + j] * inv_c; @@ -260,13 +258,11 @@ static bool compute_vector_statistics(std::vector & tstats, c ts.min_values = vmin; ts.elements = (int)valid_n; ts.std_deviation = std_deviation; - ts.entropy = std::abs(entropy); // Ensure positive 0 + ts.entropy = std::abs(entropy); ts.zd_score = (float)(zd_count / (double)valid_n); - - // Default pairwise - ts.cossim = 1.0f; - ts.pearson = 1.0f; - ts.l2_dist = 0.0f; + ts.cossim = std::numeric_limits::quiet_NaN(); + ts.pearson = std::numeric_limits::quiet_NaN(); + ts.l2_dist = std::numeric_limits::quiet_NaN(); return true; } @@ -281,12 +277,10 @@ static void compute_tensor_statistics(std::vector & tstats) { std::string dummy_tensor; process_tensor_name(ts.tensor, layer_str, dummy_tensor); - // Robust block ID extraction int blk = -1; try { blk = std::stoi(layer_str); } catch (...) { continue; } if (blk <= 0) { continue; } - // Reconstruct previous layer name const size_t blk_start_pos = ts.tensor.find("blk." + layer_str); if (blk_start_pos == std::string::npos) { continue; } @@ -309,8 +303,6 @@ static void compute_tensor_statistics(std::vector & tstats) { double norm1_sq = 0.0; double norm2_sq = 0.0; double l2_dist_sq = 0.0; - - // Aux variables for Pearson (Spatial Covariance) double sum_c = 0.0; double sum_p = 0.0; const size_t n = curr_avg.size(); @@ -332,7 +324,7 @@ static void compute_tensor_statistics(std::vector & tstats) { const double c_val = curr_avg[i]; const double p_val = prev_avg[i]; - // Cosine Similarity & L2 basics + // Cosine Similarity & L2 Distance dot_prod += c_val * p_val; norm1_sq += c_val * c_val; norm2_sq += p_val * p_val; @@ -357,14 +349,14 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.cossim = (float)(dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq))); ts.cossim = std::clamp(ts.cossim, -1.0f, 1.0f); } else { - ts.cossim = (norm1_sq == 0.0 && norm2_sq == 0.0) ? 1.0f : 0.0f; + ts.cossim = (norm1_sq == 0.0 && norm2_sq == 0.0) ? std::numeric_limits::quiet_NaN() : 0.0f; } if (var_c_sum > 0.0 && var_p_sum > 0.0) { ts.pearson = (float)(cov_sum / (std::sqrt(var_c_sum) * std::sqrt(var_p_sum))); ts.pearson = std::clamp(ts.pearson, -1.0f, 1.0f); } else { - ts.pearson = (var_c_sum == 0.0 && var_p_sum == 0.0) ? 1.0f : 0.0f; + ts.pearson = (var_c_sum == 0.0 && var_p_sum == 0.0) ? std::numeric_limits::quiet_NaN() : 0.0f; } } } @@ -411,7 +403,7 @@ static void compute_layer_statistics(const std::vector & tsta cossim = (float)(agg.sum_dot_prod / (std::sqrt(agg.sum_norm1_sq) * std::sqrt(agg.sum_norm2_sq))); cossim = std::clamp(cossim, -1.0f, 1.0f); } else if (agg.sum_norm1_sq == 0.0 && agg.sum_norm2_sq == 0.0) { - cossim = 1.0f; + cossim = std::numeric_limits::quiet_NaN(); } layer_cossim[layer] = cossim; @@ -1376,20 +1368,18 @@ static bool show_statistics(const common_params & params) { }; std::map ls; - // Helper to shorten names for table formatting "blk.10.attn_k.weight" -> "..10.attn_k.weight" + // Shorten names for table formatting auto label_fmt = [](std::string s, size_t w) -> std::string { if (s.length() <= w) { return s; } return ".." + s.substr(s.length() - (w - 2)); }; - // Table Constants constexpr int w_lay = 6; - constexpr int w_nam = 40; // Wide enough for most tensors + constexpr int w_nam = 40; // Should be wide enough for most tensor names const auto * sep = " | "; - LOG_INF("\nComputing tensor statistics (%d tensors)\n", static_cast(ts.size())); + LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); - // Header logic separated to handle different column counts if (legacy) { LOG_INF("\n%*s%s%-*s%s%10s %12s %10s %10s%s %8s %8s%s%17s %8s %8s\n", w_lay, "Layer", sep, @@ -1408,18 +1398,17 @@ static bool show_statistics(const common_params & params) { LOG_INF("%s\n", std::string(167, '-').c_str()); } + // Tensor Statistics for (const auto & tstat : ts) { std::string layer; std::string name; - process_tensor_name(tstat.tensor, layer, name); - // Calculate metrics + process_tensor_name(tstat.tensor, layer, name); const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; int blk; try { blk = std::stoi(layer); } catch (...) { blk = -1; } - // Print Row if (legacy) { LOG_INF("%*s%s%-*s%s%10.4f %12.4f %10.4f %10.4f%s%8.2f%% %8.2f%%%s%14.4f %8.4f %8.4f\n", w_lay, layer.c_str(), sep, @@ -1429,7 +1418,6 @@ static bool show_statistics(const common_params & params) { tstat.sum_values, tstat.cossim, tstat.pearson ); } else { - // Display L2 Dist AND Sum E[A^2] LOG_INF("%*s%s%-*s%s%10.4f %12.4f %10.4f %10.4f%s%8.2f%% %8.2f%%%s%14.4f %12.4f %8.4f %8.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, @@ -1441,16 +1429,13 @@ static bool show_statistics(const common_params & params) { // Aggregate Layer Stats const float zd = (float)tstat.elements * tstat.zd_score; - auto & l_entry = ls[blk]; - - // Accumulate sum values regardless of legacy status to allow display in both modes - l_entry.layer_sum += tstat.sum_values; - l_entry.layer_zd += zd; - l_entry.n += tstat.elements; + auto & l = ls[blk]; + l.layer_sum += tstat.sum_values; + l.layer_zd += zd; + l.n += tstat.elements; } - // --- Computed Layer Statistics --- - + // Layer Statistics std::map layer_cossim; std::map layer_l2_dist; std::map layer_pearson; @@ -1496,9 +1481,9 @@ static bool show_statistics(const common_params & params) { for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) { continue; } - float lcs = layer == 0 ? 1.0f : layer_cossim[layer]; - float ll2 = layer == 0 ? 0.0f : layer_l2_dist[layer]; - float lpc = layer == 0 ? 1.0f : layer_pearson[layer]; + float lcs = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_cossim[layer]; + float ll2 = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_l2_dist[layer]; + float lpc = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_pearson[layer]; if (legacy) { LOG_INF("%*d%s%8.2f%%%s%14.4f %10.4f %10.4f\n", From cb4777e40f6ee375306fcd16915d36bc3843fad2 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 17 Jan 2026 12:01:02 +0000 Subject: [PATCH 097/123] Minor cosmetic code change --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f65c13aec968..f4b9b3000311 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1463,7 +1463,7 @@ static bool show_statistics(const common_params & params) { } LOG_INF("\n\nComputing layer statistics for %s (%zu layers)\n\n", params.in_files[0].c_str(), layers); - + if (legacy) { LOG_INF("%*s%s%9s%s%17s %10s %10s\n", w_lay, "Layer", sep, From 8e9362be22b5f00ad947187921758a6d897a905d Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 17 Jan 2026 12:02:02 +0000 Subject: [PATCH 098/123] Update README.md --- tools/imatrix/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 8314a440ee4b..af53dd0caa60 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -76,13 +76,15 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def ## Statistics +Please note that the L₂ Distance can only be calculated if the imatrix is in GGUF format. If a value lacks proper statistical interpretability, **nan** will be shown instead. The following statistics are computed: + #### Per tensor * **Min / Max / μ / σ**: Tensor elements Min, Max, Mean, and Standard Deviation. * **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. Higher values indicate more uniform distribution of activations. Every neuron is firing equally; hard to prune. -* **ZD**: % of elements whose ZD-score is > 1.0 (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415). +* **Z-score Distribution (ZD)**: % of elements whose ZD-score is > 1.0 (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415). * **∑ E[A²]**: The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision (e.g., Q6_K vs Q4_K). -* **L₂ Norm**: Euclidean Distance from the tensor in the previous layer. Measure of transformation magnitude; higher values indicate more significant transformation on the data. +* **L₂ Distance**: Euclidean Distance from the tensor in the previous layer. Measure of transformation magnitude; higher values indicate more significant transformation on the data. * **CosSim**: Cosine Similarity with the tensor in the previous layer. _~1.0_, the tensor output points in the exact same direction as the previous layer's tensor (the layer is refining magnitude, not direction). _< 1.0_, the layer is rotating the vector space (changing semantic meaning). * **PCC**: Pearson Correlation Coefficient with the tensor in the previous layer. Checks for linear correlation excluding the mean shift. Similar to CosSim but centers geometric data first. Indicates if the pattern of activation changes or just the offset. @@ -90,9 +92,9 @@ Versions **b5942** and newer of `llama-imatrix` store data in GGUF format by def Aggregated metrics per block/layer: -* **ZD**: % of this layer's concatenated tensors' elements with |Z| > 1. Indicates general "spikiness" of the layer's activations. +* **Z-score Distribution (ZD)**: % of this layer's concatenated tensors' elements with |Z| > 1. Indicates general "spikiness" of the layer's activations. * **∑ E[A²]:** Total energy of the layer's concatenated tensors. Indicates the layer's overall contribution amplitude. -* **L₂ Norm:** Euclidean Distance of the layer's concatenated tensors from the previous layer’s. Global measure of transformation magnitude. +* **L₂ Distance:** Euclidean Distance of the layer's concatenated tensors from the previous layer’s. Global measure of transformation magnitude. * **CosSim**: Cosine Similarity of this layer's concatenated tensors with the previous layer. * **PCC**: Average Pearson Correlation of the tensors in the layer. From f3323b6503bbc758c632edf0c4dff9e08c94aed5 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 22 Jan 2026 15:05:15 +0000 Subject: [PATCH 099/123] Save tensor statistics to imatrix file --- tools/imatrix/imatrix.cpp | 59 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f4b9b3000311..5a68ad2d8810 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -753,11 +753,26 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.activations.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.values.size(), GGML_MEM_ALIGN); data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * kv.second.counts.size(), GGML_MEM_ALIGN); + data_size += GGML_PAD(ggml_tensor_overhead() + sizeof(float) * 10, GGML_MEM_ALIGN); } // deterministic tensor name order std::sort(to_store.begin(), to_store.end()); + // Compute per-tensor statistics + std::vector tstats; + tstats.reserve(m_stats.size()); + bool legacy; + for (const auto & kv : m_stats) { + compute_vector_statistics(tstats, kv.first, kv.second, legacy); + } + if (!tstats.empty()) { compute_tensor_statistics(tstats); } + + // index by tensor name + std::unordered_map tstat_index; + tstat_index.reserve(tstats.size()); + for (const auto & ts : tstats) { tstat_index[ts.tensor] = &ts; } + struct ggml_init_params params = { /* .mem_size = */ data_size, /* .mem_buffer = */ NULL, @@ -814,6 +829,48 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { gguf_add_tensor(ctx_gguf, in_sum); } } + + // Store per-tensor statistics as a small 1D tensor + { + float nan = std::numeric_limits::quiet_NaN(); + float min = 0.0f; + float max = 0.0f; + float mean = 0.0f; + float stddev = 0.0f; + float h_norm = 0.0f; + float zd_score = 0.0f; + float sum_sq = 0.0f; + float l2_dist = 0.0f; + float cossim = 0.0f; + float pcc = 0.0f; + auto it_ts = tstat_index.find(name); + if (it_ts != tstat_index.end() && it_ts->second != nullptr) { + sum_sq = it_ts->second->sum_values; + h_norm = it_ts->second->elements > 0 ? 100.0f * (it_ts->second->entropy / std::log2f((float)it_ts->second->elements)) : nan; + zd_score = it_ts->second->zd_score; + l2_dist = it_ts->second->l2_dist; + cossim = it_ts->second->cossim; + pcc = it_ts->second->pearson; + min = it_ts->second->min_values; + max = it_ts->second->max_values; + mean = it_ts->second->mean_values; + stddev = it_ts->second->std_deviation; + } + + struct ggml_tensor * stats_t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 10); + ggml_format_name(stats_t, "%s.stats", name.c_str()); + ((float *)stats_t->data)[0] = sum_sq; + ((float *)stats_t->data)[1] = h_norm; + ((float *)stats_t->data)[2] = zd_score; + ((float *)stats_t->data)[3] = l2_dist; + ((float *)stats_t->data)[4] = cossim; + ((float *)stats_t->data)[5] = pcc; + ((float *)stats_t->data)[6] = min; + ((float *)stats_t->data)[7] = max; + ((float *)stats_t->data)[8] = mean; + ((float *)stats_t->data)[9] = stddev; + gguf_add_tensor(ctx_gguf, stats_t); + } } gguf_write_to_file(ctx_gguf, fname.c_str(), false); @@ -1404,7 +1461,7 @@ static bool show_statistics(const common_params & params) { std::string name; process_tensor_name(tstat.tensor, layer, name); - const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2((float) tstat.elements)) : 0.0f; + const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2f((float)tstat.elements)) : std::numeric_limits::quiet_NaN(); int blk; try { blk = std::stoi(layer); } catch (...) { blk = -1; } From 3fadde15682c9f139807b8ad9931b5084e35008a Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Fri, 6 Feb 2026 12:19:52 +0000 Subject: [PATCH 100/123] Add covariance --- tools/imatrix/imatrix.cpp | 91 +++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 5a68ad2d8810..1ace29e2fcf2 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -47,21 +47,22 @@ struct Stats { struct tensor_statistics { std::string tensor; Stats stats; - float sum_values = 0.0f; - float mean_values = 0.0f; - float max_values = 0.0f; - float min_values = 0.0f; - int elements = 0; + float sum_values = 0.0f; + float mean_values = 0.0f; + float max_values = 0.0f; + float min_values = 0.0f; + int elements = 0; float std_deviation = 0.0f; - float entropy = 0.0f; - float zd_score = 0.0f; - float cossim = 0.0f; - float pearson = 0.0f; - float l2_dist = 0.0f; - double dot_prod = 0.0; - double norm1_sq = 0.0; - double norm2_sq = 0.0; - double l2_dist_sq = 0.0; + float entropy = 0.0f; + float zd_score = 0.0f; + float cossim = 0.0f; + float pearson = 0.0f; + float cov = 0.0f; + float l2_dist = 0.0f; + double dot_prod = 0.0; + double norm1_sq = 0.0; + double norm2_sq = 0.0; + double l2_dist_sq = 0.0; }; class IMatrixCollector { @@ -262,6 +263,7 @@ static bool compute_vector_statistics(std::vector & tstats, c ts.zd_score = (float)(zd_count / (double)valid_n); ts.cossim = std::numeric_limits::quiet_NaN(); ts.pearson = std::numeric_limits::quiet_NaN(); + ts.cov = std::numeric_limits::quiet_NaN(); ts.l2_dist = std::numeric_limits::quiet_NaN(); return true; @@ -345,6 +347,10 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.l2_dist_sq = l2_dist_sq; ts.l2_dist = (float)std::sqrt(l2_dist_sq); + if (n > 1) { + ts.cov = (float)(cov_sum / (double)(n - 1)); + } + if (norm1_sq > 0.0 && norm2_sq > 0.0) { ts.cossim = (float)(dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq))); ts.cossim = std::clamp(ts.cossim, -1.0f, 1.0f); @@ -364,13 +370,15 @@ static void compute_tensor_statistics(std::vector & tstats) { static void compute_layer_statistics(const std::vector & tstats, std::map & layer_cossim, std::map & layer_l2_dist, - std::map & layer_pearson) { + std::map & layer_pearson, + std::map & layer_cov) { struct layer_aggregation { double sum_dot_prod = 0.0; double sum_norm1_sq = 0.0; double sum_norm2_sq = 0.0; double sum_l2_dist_sq = 0.0; double sum_pearson = 0.0; + double sum_cov = 0.0; int n_tensors = 0; }; @@ -392,6 +400,7 @@ static void compute_layer_statistics(const std::vector & tsta entry.sum_norm2_sq += ts.norm2_sq; entry.sum_l2_dist_sq += ts.l2_dist_sq; entry.sum_pearson += ts.pearson; + entry.sum_cov += ts.cov; entry.n_tensors++; } @@ -409,6 +418,7 @@ static void compute_layer_statistics(const std::vector & tsta layer_cossim[layer] = cossim; layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); layer_pearson[layer] = (float)(agg.sum_pearson / agg.n_tensors); + layer_cov[layer] = (float)(agg.sum_cov / agg.n_tensors); } } @@ -843,21 +853,23 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { float l2_dist = 0.0f; float cossim = 0.0f; float pcc = 0.0f; + float xcov = 0.0f; auto it_ts = tstat_index.find(name); if (it_ts != tstat_index.end() && it_ts->second != nullptr) { sum_sq = it_ts->second->sum_values; h_norm = it_ts->second->elements > 0 ? 100.0f * (it_ts->second->entropy / std::log2f((float)it_ts->second->elements)) : nan; zd_score = it_ts->second->zd_score; - l2_dist = it_ts->second->l2_dist; - cossim = it_ts->second->cossim; - pcc = it_ts->second->pearson; + l2_dist = it_ts->second->l2_dist; + cossim = it_ts->second->cossim; + pcc = it_ts->second->pearson; + xcov = it_ts->second->cov; min = it_ts->second->min_values; max = it_ts->second->max_values; mean = it_ts->second->mean_values; stddev = it_ts->second->std_deviation; } - struct ggml_tensor * stats_t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 10); + struct ggml_tensor * stats_t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 11); ggml_format_name(stats_t, "%s.stats", name.c_str()); ((float *)stats_t->data)[0] = sum_sq; ((float *)stats_t->data)[1] = h_norm; @@ -865,10 +877,11 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { ((float *)stats_t->data)[3] = l2_dist; ((float *)stats_t->data)[4] = cossim; ((float *)stats_t->data)[5] = pcc; - ((float *)stats_t->data)[6] = min; - ((float *)stats_t->data)[7] = max; - ((float *)stats_t->data)[8] = mean; - ((float *)stats_t->data)[9] = stddev; + ((float *)stats_t->data)[6] = xcov; + ((float *)stats_t->data)[7] = min; + ((float *)stats_t->data)[8] = max; + ((float *)stats_t->data)[9] = mean; + ((float *)stats_t->data)[10] = stddev; gguf_add_tensor(ctx_gguf, stats_t); } } @@ -1443,7 +1456,7 @@ static bool show_statistics(const common_params & params) { w_nam, "Tensor", sep, "Min", "Max", "Mean", "StdDev", sep, "H Norm", "ZD", sep, - "∑ E[A²]", "CosSim", "PCC"); + "∑ E[A²]", "PCC", "Cov"); LOG_INF("%s\n", std::string(154, '-').c_str()); } else { LOG_INF("\n%*s%s%-*s%s%10s %12s %10s %10s%s %8s %8s%s%17s %12s %8s %8s\n", @@ -1451,7 +1464,7 @@ static bool show_statistics(const common_params & params) { w_nam, "Tensor", sep, "Min", "Max", "Mean", "StdDev", sep, "H Norm", "ZD", sep, - "∑ E[A²]", "L2 Dist", "CosSim", "PCC"); + "∑ E[A²]", "L2 Dist", "PCC", "Cov"); LOG_INF("%s\n", std::string(167, '-').c_str()); } @@ -1472,7 +1485,7 @@ static bool show_statistics(const common_params & params) { w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, sep, h_norm, 100.0f * tstat.zd_score, sep, - tstat.sum_values, tstat.cossim, tstat.pearson + tstat.sum_values, tstat.pearson, tstat.cov ); } else { LOG_INF("%*s%s%-*s%s%10.4f %12.4f %10.4f %10.4f%s%8.2f%% %8.2f%%%s%14.4f %12.4f %8.4f %8.4f\n", @@ -1480,7 +1493,7 @@ static bool show_statistics(const common_params & params) { w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, sep, h_norm, 100.0f * tstat.zd_score, sep, - tstat.sum_values, tstat.l2_dist, tstat.cossim, tstat.pearson + tstat.sum_values, tstat.l2_dist, tstat.pearson, tstat.cov ); } @@ -1496,7 +1509,8 @@ static bool show_statistics(const common_params & params) { std::map layer_cossim; std::map layer_l2_dist; std::map layer_pearson; - compute_layer_statistics(ts, layer_cossim, layer_l2_dist, layer_pearson); + std::map layer_cov; + compute_layer_statistics(ts, layer_cossim, layer_l2_dist, layer_pearson, layer_cov); size_t layers = 0; int min = std::numeric_limits::max(); @@ -1522,17 +1536,17 @@ static bool show_statistics(const common_params & params) { LOG_INF("\n\nComputing layer statistics for %s (%zu layers)\n\n", params.in_files[0].c_str(), layers); if (legacy) { - LOG_INF("%*s%s%9s%s%17s %10s %10s\n", + LOG_INF("%*s%s%9s%s%17s %10s %10s %10s\n", w_lay, "Layer", sep, "ZD", sep, - "∑ E[A²]", "CosSim", "PCC"); - LOG_INF("%s\n", std::string(57, '-').c_str()); + "∑ E[A²]", "CosSim", "PCC", "Cov"); + LOG_INF("%s\n", std::string(68, '-').c_str()); } else { - LOG_INF("%*s%s%9s%s%17s %14s %10s %10s\n", + LOG_INF("%*s%s%9s%s%17s %14s %10s %10s %10s\n", w_lay, "Layer", sep, "ZD", sep, - "∑ E[A²]", "L2 Dist", "CosSim", "PCC"); - LOG_INF("%s\n", std::string(72, '-').c_str()); + "∑ E[A²]", "L2 Dist", "CosSim", "PCC", "Cov"); + LOG_INF("%s\n", std::string(83, '-').c_str()); } for (const auto & [layer, stats] : ls) { @@ -1541,17 +1555,18 @@ static bool show_statistics(const common_params & params) { float lcs = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_cossim[layer]; float ll2 = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_l2_dist[layer]; float lpc = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_pearson[layer]; + float lcv = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_cov[layer]; if (legacy) { - LOG_INF("%*d%s%8.2f%%%s%14.4f %10.4f %10.4f\n", + LOG_INF("%*d%s%8.2f%%%s%14.4f %10.4f %10.4f %10.4f\n", w_lay, layer, sep, 100.0f * stats.layer_zd / stats.n, sep, - stats.layer_sum, lcs, lpc); + stats.layer_sum, lcs, lpc, lcv); } else { - LOG_INF("%*d%s%8.2f%%%s%14.4f %14.4f %10.4f %10.4f\n", + LOG_INF("%*d%s%8.2f%%%s%14.4f %14.4f %10.4f %10.4f %10.4f\n", w_lay, layer, sep, 100.0f * stats.layer_zd / stats.n, sep, - stats.layer_sum, ll2, lcs, lpc); + stats.layer_sum, ll2, lcs, lpc, lcv); } } LOG_INF("\n"); From 749b6340e1053eef69f81b5ad0926dbb4151a666 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Fri, 6 Feb 2026 18:46:22 +0000 Subject: [PATCH 101/123] Add skewness and kurtosis --- tools/imatrix/imatrix.cpp | 268 +++++++++++++++++++++----------------- 1 file changed, 147 insertions(+), 121 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 1ace29e2fcf2..c90e11186960 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -47,18 +47,20 @@ struct Stats { struct tensor_statistics { std::string tensor; Stats stats; - float sum_values = 0.0f; - float mean_values = 0.0f; - float max_values = 0.0f; - float min_values = 0.0f; - int elements = 0; + float sum_val = 0.0f; + float mean_val = 0.0f; + float min_val = 0.0f; + float max_val = 0.0f; + int elements = 0; float std_deviation = 0.0f; + float skewness = 0.0f; + float kurtosis = 0.0f; + float gain = 0.0f; float entropy = 0.0f; - float zd_score = 0.0f; + float l2_dist = 0.0f; float cossim = 0.0f; float pearson = 0.0f; - float cov = 0.0f; - float l2_dist = 0.0f; + float covariance = 0.0f; double dot_prod = 0.0; double norm1_sq = 0.0; double norm2_sq = 0.0; @@ -177,15 +179,17 @@ static bool compute_vector_statistics(std::vector & tstats, c } const size_t row_size = len / n_mat; - double mean = 0.0; - double M2 = 0.0; double sum = 0.0; - float vmin = std::numeric_limits::max(); - float vmax = -std::numeric_limits::max(); - double energy_sum = 0.0; + double mean = 0.0; + float min = std::numeric_limits::max(); + float max = -std::numeric_limits::max(); + double sum_sq_diff = 0.0; + double sum_cu_diff = 0.0; + double sum_qd_diff = 0.0; + double sum_energy = 0.0; size_t valid_n = 0; - // Pass 1: Mean, Min, Max, Std Dev + // Pass 1: Mean, Min, Max for (size_t i = 0; i < n_mat; ++i) { const auto c = (float)e.counts[i]; if (c <= 0.0f) { continue; } @@ -199,15 +203,14 @@ static bool compute_vector_statistics(std::vector & tstats, c const double v = legacy ? v_val : v_act; // Use activation average for non-legacy sum += v_val; - if ((float)v < vmin) { vmin = (float)v; } - if ((float)v > vmax) { vmax = (float)v; } + if ((float)v < min) { min = (float)v; } + if ((float)v > max) { max = (float)v; } valid_n++; const double delta = v - mean; mean += delta / (double)valid_n; - M2 += delta * (v - mean); - if (v_val > 0.0) { energy_sum += v_val; } + if (v_val > 0.0) { sum_energy += v_val; } } } @@ -215,15 +218,9 @@ static bool compute_vector_statistics(std::vector & tstats, c float std_deviation = 0.0f; float entropy = 0.0f; - double zd_count = 0.0; - const double variance = valid_n > 1 ? M2 / ((double)valid_n - 1) : 0.0; - std_deviation = std::sqrt((float)std::max(variance, 0.0)); - - // Pass 2: Entropy and Z-Score - const double inv_energy_sum = energy_sum > 0.0 ? 1.0 / energy_sum : 0.0; - const float inv_std = std_deviation > 0.0f ? 1.0f / std_deviation : 0.0f; - const float fmean = (float)mean; + // Pass 2: Std Dev, Skew, Kurtosis, Entropy + const double inv_sum_energy = sum_energy > 0.0 ? 1.0 / sum_energy : 0.0; const float log2_val = 1 / std::log2f(2); for (size_t i = 0; i < n_mat; ++i) { @@ -233,38 +230,50 @@ static bool compute_vector_statistics(std::vector & tstats, c const size_t off = i * row_size; for (size_t j = 0; j < row_size; ++j) { + const double v_act = legacy ? 0.0 : (double)e.activations[off + j] * inv_c; + const double v_val = (double)e.values[off + j] * inv_c; + const double v = legacy ? v_val : v_act; + const double diff = v - mean; + + sum_sq_diff += diff * diff; + sum_cu_diff += diff * diff * diff; + sum_qd_diff += diff * diff * diff * diff; + // Entropy (Distribution of Energy) - if (inv_energy_sum > 0.0) { + if (inv_sum_energy > 0.0) { const double v_energy = (double)e.values[off + j] * inv_c; - const double p = std::max(0.0, v_energy) * inv_energy_sum; + const double p = std::max(0.0, v_energy) * inv_sum_energy; if (p > 1e-9) { entropy -= (float)(p * std::log(p) * log2_val); } } - - // Z-Score Density (Outlier detection) - if (std_deviation > 0.0f) { - const double v_act = legacy ? 0.0 : (double)e.activations[off + j] * inv_c; - const double v_val = (double)e.values[off + j] * inv_c; - const float v = (float)(legacy ? v_val : v_act); - if (std::fabs((v - fmean) * inv_std) > 1.0f) { zd_count += 1.0; } - } } } + const double variance = valid_n > 1 ? sum_sq_diff / ((double)valid_n - 1) : 0.0; + std_deviation = std::sqrt((float)std::max(variance, 0.0)); + float skewness = 0.0f; + float kurtosis = 0.0f; + if (std_deviation > 1e-9f) { + skewness = (float)(sum_cu_diff / ((float)valid_n * std_deviation * std_deviation * std_deviation)); + kurtosis = (float)(sum_qd_diff / ((float)valid_n * variance * variance)) - 3.0f; + } + auto & ts = tstats.emplace_back(); ts.tensor = name; ts.stats = e; - ts.sum_values = (float)sum; - ts.mean_values = (float)mean; - ts.max_values = vmax; - ts.min_values = vmin; + ts.sum_val = (float)sum; + ts.mean_val = (float)mean; + ts.min_val = min; + ts.max_val = max; ts.elements = (int)valid_n; ts.std_deviation = std_deviation; + ts.skewness = skewness; + ts.kurtosis = kurtosis; + ts.gain = std::numeric_limits::quiet_NaN(); ts.entropy = std::abs(entropy); - ts.zd_score = (float)(zd_count / (double)valid_n); + ts.l2_dist = std::numeric_limits::quiet_NaN(); ts.cossim = std::numeric_limits::quiet_NaN(); ts.pearson = std::numeric_limits::quiet_NaN(); - ts.cov = std::numeric_limits::quiet_NaN(); - ts.l2_dist = std::numeric_limits::quiet_NaN(); + ts.covariance = std::numeric_limits::quiet_NaN(); return true; } @@ -348,12 +357,13 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.l2_dist = (float)std::sqrt(l2_dist_sq); if (n > 1) { - ts.cov = (float)(cov_sum / (double)(n - 1)); + ts.covariance = (float)(cov_sum / (double)(n - 1)); } if (norm1_sq > 0.0 && norm2_sq > 0.0) { ts.cossim = (float)(dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq))); ts.cossim = std::clamp(ts.cossim, -1.0f, 1.0f); + ts.gain = (float)(std::sqrt(norm1_sq) / std::sqrt(norm2_sq)); } else { ts.cossim = (norm1_sq == 0.0 && norm2_sq == 0.0) ? std::numeric_limits::quiet_NaN() : 0.0f; } @@ -371,14 +381,16 @@ static void compute_layer_statistics(const std::vector & tsta std::map & layer_cossim, std::map & layer_l2_dist, std::map & layer_pearson, - std::map & layer_cov) { + std::map & layer_covariance, + std::map & layer_gain) { struct layer_aggregation { double sum_dot_prod = 0.0; double sum_norm1_sq = 0.0; double sum_norm2_sq = 0.0; + double sum_gain = 0.0; double sum_l2_dist_sq = 0.0; double sum_pearson = 0.0; - double sum_cov = 0.0; + double sum_covariance = 0.0; int n_tensors = 0; }; @@ -398,9 +410,10 @@ static void compute_layer_statistics(const std::vector & tsta entry.sum_dot_prod += ts.dot_prod; entry.sum_norm1_sq += ts.norm1_sq; entry.sum_norm2_sq += ts.norm2_sq; + entry.sum_gain += ts.gain; entry.sum_l2_dist_sq += ts.l2_dist_sq; entry.sum_pearson += ts.pearson; - entry.sum_cov += ts.cov; + entry.sum_covariance += ts.covariance; entry.n_tensors++; } @@ -415,10 +428,18 @@ static void compute_layer_statistics(const std::vector & tsta cossim = std::numeric_limits::quiet_NaN(); } + float gain = 0.0f; + if (agg.sum_norm1_sq > 0.0 && agg.sum_norm2_sq > 0.0) { + gain = (float)(std::sqrt(agg.sum_norm1_sq) / std::sqrt(agg.sum_norm2_sq)); + } else if (agg.sum_norm1_sq == 0.0 && agg.sum_norm2_sq == 0.0) { + gain = std::numeric_limits::quiet_NaN(); + } + layer_cossim[layer] = cossim; layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); layer_pearson[layer] = (float)(agg.sum_pearson / agg.n_tensors); - layer_cov[layer] = (float)(agg.sum_cov / agg.n_tensors); + layer_covariance[layer] = (float)(agg.sum_covariance / agg.n_tensors); + layer_gain[layer] = gain; } } @@ -843,46 +864,52 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { // Store per-tensor statistics as a small 1D tensor { float nan = std::numeric_limits::quiet_NaN(); + float sum_sq = 0.0f; + float mean = 0.0f; float min = 0.0f; float max = 0.0f; - float mean = 0.0f; - float stddev = 0.0f; + float std_deviation = 0.0f; + float skewness = 0.0f; + float kurtosis = 0.0f; + float gain = 0.0f; float h_norm = 0.0f; - float zd_score = 0.0f; - float sum_sq = 0.0f; float l2_dist = 0.0f; float cossim = 0.0f; - float pcc = 0.0f; - float xcov = 0.0f; - auto it_ts = tstat_index.find(name); - if (it_ts != tstat_index.end() && it_ts->second != nullptr) { - sum_sq = it_ts->second->sum_values; - h_norm = it_ts->second->elements > 0 ? 100.0f * (it_ts->second->entropy / std::log2f((float)it_ts->second->elements)) : nan; - zd_score = it_ts->second->zd_score; - l2_dist = it_ts->second->l2_dist; - cossim = it_ts->second->cossim; - pcc = it_ts->second->pearson; - xcov = it_ts->second->cov; - min = it_ts->second->min_values; - max = it_ts->second->max_values; - mean = it_ts->second->mean_values; - stddev = it_ts->second->std_deviation; + float pearson = 0.0f; + float covariance = 0.0f; + auto ts = tstat_index.find(name); + if (ts != tstat_index.end() && ts->second != nullptr) { + sum_sq = ts->second->sum_val; + mean = ts->second->mean_val; + min = ts->second->min_val; + max = ts->second->max_val; + std_deviation = ts->second->std_deviation; + skewness = ts->second->skewness; + kurtosis = ts->second->kurtosis; + gain = ts->second->gain; + h_norm = ts->second->elements > 0 ? 100.0f * (ts->second->entropy / std::log2f((float)ts->second->elements)) : nan; + l2_dist = ts->second->l2_dist; + cossim = ts->second->cossim; + pearson = ts->second->pearson; + covariance = ts->second->covariance; } - struct ggml_tensor * stats_t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 11); - ggml_format_name(stats_t, "%s.stats", name.c_str()); - ((float *)stats_t->data)[0] = sum_sq; - ((float *)stats_t->data)[1] = h_norm; - ((float *)stats_t->data)[2] = zd_score; - ((float *)stats_t->data)[3] = l2_dist; - ((float *)stats_t->data)[4] = cossim; - ((float *)stats_t->data)[5] = pcc; - ((float *)stats_t->data)[6] = xcov; - ((float *)stats_t->data)[7] = min; - ((float *)stats_t->data)[8] = max; - ((float *)stats_t->data)[9] = mean; - ((float *)stats_t->data)[10] = stddev; - gguf_add_tensor(ctx_gguf, stats_t); + struct ggml_tensor * stats = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 13); + ggml_format_name(stats, "%s.stats", name.c_str()); + ((float *)stats->data)[0] = sum_sq; + ((float *) stats->data)[1] = mean; + ((float *) stats->data)[2] = min; + ((float *) stats->data)[3] = max; + ((float *) stats->data)[4] = std_deviation; + ((float *) stats->data)[5] = skewness; + ((float *) stats->data)[6] = kurtosis; + ((float *) stats->data)[7] = gain; + ((float *)stats->data)[8] = h_norm; + ((float *) stats->data)[9] = l2_dist; + ((float *) stats->data)[10] = cossim; + ((float *) stats->data)[11] = pearson; + ((float *) stats->data)[12] = covariance; + gguf_add_tensor(ctx_gguf, stats); } } @@ -1433,7 +1460,6 @@ static bool show_statistics(const common_params & params) { struct layer_stats { float layer_sum = 0.0f; - float layer_zd = 0.0f; int n = 0; }; std::map ls; @@ -1451,21 +1477,21 @@ static bool show_statistics(const common_params & params) { LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); if (legacy) { - LOG_INF("\n%*s%s%-*s%s%10s %12s %10s %10s%s %8s %8s%s%17s %8s %8s\n", + LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%9s%9s\n", w_lay, "Layer", sep, w_nam, "Tensor", sep, - "Min", "Max", "Mean", "StdDev", sep, - "H Norm", "ZD", sep, - "∑ E[A²]", "PCC", "Cov"); - LOG_INF("%s\n", std::string(154, '-').c_str()); + "Mean", "StdDev", "Skew", "Kurt", "H Norm", sep, + "∑ E[A²]", "Gain", sep, + "PCC", "Cov"); + LOG_INF("%s\n", std::string(151, '-').c_str()); } else { - LOG_INF("\n%*s%s%-*s%s%10s %12s %10s %10s%s %8s %8s%s%17s %12s %8s %8s\n", + LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%12s%9s%9s\n", w_lay, "Layer", sep, w_nam, "Tensor", sep, - "Min", "Max", "Mean", "StdDev", sep, - "H Norm", "ZD", sep, - "∑ E[A²]", "L2 Dist", "PCC", "Cov"); - LOG_INF("%s\n", std::string(167, '-').c_str()); + "Mean", "StdDev", "Skew", "Kurt", "H Norm", sep, + "∑ E[A²]", "Gain", sep, + "L2 Dist", "PCC", "Cov"); + LOG_INF("%s\n", std::string(163, '-').c_str()); } // Tensor Statistics @@ -1480,28 +1506,26 @@ static bool show_statistics(const common_params & params) { try { blk = std::stoi(layer); } catch (...) { blk = -1; } if (legacy) { - LOG_INF("%*s%s%-*s%s%10.4f %12.4f %10.4f %10.4f%s%8.2f%% %8.2f%%%s%14.4f %8.4f %8.4f\n", + LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%9.4f%9.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, - tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, sep, - h_norm, 100.0f * tstat.zd_score, sep, - tstat.sum_values, tstat.pearson, tstat.cov + tstat.mean_val, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, + tstat.sum_val, tstat.gain, sep, + tstat.pearson, tstat.covariance ); } else { - LOG_INF("%*s%s%-*s%s%10.4f %12.4f %10.4f %10.4f%s%8.2f%% %8.2f%%%s%14.4f %12.4f %8.4f %8.4f\n", + LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%12.4f%9.4f%9.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, - tstat.min_values, tstat.max_values, tstat.mean_values, tstat.std_deviation, sep, - h_norm, 100.0f * tstat.zd_score, sep, - tstat.sum_values, tstat.l2_dist, tstat.pearson, tstat.cov + tstat.mean_val, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, + tstat.sum_val, tstat.gain, sep, + tstat.l2_dist, tstat.pearson, tstat.covariance ); } // Aggregate Layer Stats - const float zd = (float)tstat.elements * tstat.zd_score; auto & l = ls[blk]; - l.layer_sum += tstat.sum_values; - l.layer_zd += zd; + l.layer_sum += tstat.sum_val; l.n += tstat.elements; } @@ -1509,8 +1533,9 @@ static bool show_statistics(const common_params & params) { std::map layer_cossim; std::map layer_l2_dist; std::map layer_pearson; - std::map layer_cov; - compute_layer_statistics(ts, layer_cossim, layer_l2_dist, layer_pearson, layer_cov); + std::map layer_covariance; + std::map layer_gain; + compute_layer_statistics(ts, layer_cossim, layer_l2_dist, layer_pearson, layer_covariance, layer_gain); size_t layers = 0; int min = std::numeric_limits::max(); @@ -1536,37 +1561,38 @@ static bool show_statistics(const common_params & params) { LOG_INF("\n\nComputing layer statistics for %s (%zu layers)\n\n", params.in_files[0].c_str(), layers); if (legacy) { - LOG_INF("%*s%s%9s%s%17s %10s %10s %10s\n", + LOG_INF("%*s%s%17s%8s%s%9s%9s%9s\n", w_lay, "Layer", sep, - "ZD", sep, - "∑ E[A²]", "CosSim", "PCC", "Cov"); - LOG_INF("%s\n", std::string(68, '-').c_str()); + "∑ E[A²]", "Gain", sep, + "CosSim", "PCC", "Cov"); + LOG_INF("%s\n", std::string(61, '-').c_str()); } else { - LOG_INF("%*s%s%9s%s%17s %14s %10s %10s %10s\n", + LOG_INF("%*s%s%17s%8s%s%12s%9s%9s%9s\n", w_lay, "Layer", sep, - "ZD", sep, - "∑ E[A²]", "L2 Dist", "CosSim", "PCC", "Cov"); - LOG_INF("%s\n", std::string(83, '-').c_str()); + "∑ E[A²]", "Gain", sep, + "L2 Dist", "CosSim", "PCC", "Cov"); + LOG_INF("%s\n", std::string(73, '-').c_str()); } for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) { continue; } - float lcs = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_cossim[layer]; + float lgn = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_gain[layer]; float ll2 = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_l2_dist[layer]; + float lcs = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_cossim[layer]; float lpc = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_pearson[layer]; - float lcv = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_cov[layer]; + float lcv = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_covariance[layer]; if (legacy) { - LOG_INF("%*d%s%8.2f%%%s%14.4f %10.4f %10.4f %10.4f\n", + LOG_INF("%*d%s%14.4f%8.2f%s%9.4f%9.4f%9.4f\n", w_lay, layer, sep, - 100.0f * stats.layer_zd / stats.n, sep, - stats.layer_sum, lcs, lpc, lcv); + stats.layer_sum, lgn, sep, + lcs, lpc, lcv); } else { - LOG_INF("%*d%s%8.2f%%%s%14.4f %14.4f %10.4f %10.4f %10.4f\n", + LOG_INF("%*d%s%14.4f%8.2f%s%12.4f%9.4f%9.4f%9.4f\n", w_lay, layer, sep, - 100.0f * stats.layer_zd / stats.n, sep, - stats.layer_sum, ll2, lcs, lpc, lcv); + stats.layer_sum, lgn, sep, + ll2, lcs, lpc, lcv); } } LOG_INF("\n"); From 8bfd244f9f88193a44323a56ae74330bbac2f46c Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 7 Feb 2026 15:48:10 +0000 Subject: [PATCH 102/123] Memory and performance optimisations (AI assisted) --- tools/imatrix/imatrix.cpp | 202 ++++++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 82 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index c90e11186960..2ebf051f1bbf 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -49,18 +49,19 @@ struct tensor_statistics { Stats stats; float sum_val = 0.0f; float mean_val = 0.0f; - float min_val = 0.0f; - float max_val = 0.0f; - int elements = 0; + int64_t elements = 0; float std_deviation = 0.0f; float skewness = 0.0f; float kurtosis = 0.0f; - float gain = 0.0f; + float gain = std::numeric_limits::quiet_NaN(); float entropy = 0.0f; - float l2_dist = 0.0f; - float cossim = 0.0f; - float pearson = 0.0f; - float covariance = 0.0f; + float l2_dist = std::numeric_limits::quiet_NaN(); + float cossim = std::numeric_limits::quiet_NaN(); + float pearson = std::numeric_limits::quiet_NaN(); + float covariance = std::numeric_limits::quiet_NaN(); + double cov_sum = 0.0; + double var_c_sum = 0.0; + double var_p_sum = 0.0; double dot_prod = 0.0; double norm1_sq = 0.0; double norm2_sq = 0.0; @@ -123,7 +124,14 @@ static void process_tensor_name(const std::string & input, std::string & layer, } for (size_t i = 0; i < name.size(); ++i) { if (name[i] == "weight" && i > 0) { - tensor = name[i - 1]; + for (size_t j = 0; j < name.size(); ++j) { + if (name[j] == "blk") { + j+=2; + continue; } + if (j == i) { break; } + if (!tensor.empty()) { tensor += "."; } + tensor += name[j]; + } break; } } @@ -165,6 +173,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { } static bool compute_vector_statistics(std::vector & tstats, const std::string & name, const Stats & e, bool & legacy) { + constexpr auto fnan = std::numeric_limits::quiet_NaN(); legacy = e.activations.empty(); const size_t n_mat = e.counts.size(); const size_t len = legacy ? e.values.size() : e.activations.size(); @@ -181,15 +190,13 @@ static bool compute_vector_statistics(std::vector & tstats, c const size_t row_size = len / n_mat; double sum = 0.0; double mean = 0.0; - float min = std::numeric_limits::max(); - float max = -std::numeric_limits::max(); double sum_sq_diff = 0.0; double sum_cu_diff = 0.0; double sum_qd_diff = 0.0; double sum_energy = 0.0; size_t valid_n = 0; - // Pass 1: Mean, Min, Max + // Mean for (size_t i = 0; i < n_mat; ++i) { const auto c = (float)e.counts[i]; if (c <= 0.0f) { continue; } @@ -203,9 +210,6 @@ static bool compute_vector_statistics(std::vector & tstats, c const double v = legacy ? v_val : v_act; // Use activation average for non-legacy sum += v_val; - if ((float)v < min) { min = (float)v; } - if ((float)v > max) { max = (float)v; } - valid_n++; const double delta = v - mean; mean += delta / (double)valid_n; @@ -219,9 +223,9 @@ static bool compute_vector_statistics(std::vector & tstats, c float std_deviation = 0.0f; float entropy = 0.0f; - // Pass 2: Std Dev, Skew, Kurtosis, Entropy + // Std Dev, Skew, Kurtosis, Entropy const double inv_sum_energy = sum_energy > 0.0 ? 1.0 / sum_energy : 0.0; - const float log2_val = 1 / std::log2f(2); + const double log2_inv = 1.0 / std::log(2.0); for (size_t i = 0; i < n_mat; ++i) { const auto c = (float)e.counts[i]; @@ -243,18 +247,19 @@ static bool compute_vector_statistics(std::vector & tstats, c if (inv_sum_energy > 0.0) { const double v_energy = (double)e.values[off + j] * inv_c; const double p = std::max(0.0, v_energy) * inv_sum_energy; - if (p > 1e-9) { entropy -= (float)(p * std::log(p) * log2_val); } + if (p > 1e-10) { entropy -= (float)(p * std::log(p) * log2_inv); } } } } - const double variance = valid_n > 1 ? sum_sq_diff / ((double)valid_n - 1) : 0.0; + const double variance = valid_n > 1 ? sum_sq_diff / (double)valid_n : 0.0; std_deviation = std::sqrt((float)std::max(variance, 0.0)); float skewness = 0.0f; float kurtosis = 0.0f; - if (std_deviation > 1e-9f) { - skewness = (float)(sum_cu_diff / ((float)valid_n * std_deviation * std_deviation * std_deviation)); - kurtosis = (float)(sum_qd_diff / ((float)valid_n * variance * variance)) - 3.0f; + if (std_deviation > 1e-10f) { + const double m2 = sum_sq_diff / (double)valid_n; + skewness = (float)(sum_cu_diff / (double)valid_n / (m2 * std::sqrt(m2))); + kurtosis = (float)(sum_qd_diff / (double)valid_n / (m2 * m2) - 3.0); } auto & ts = tstats.emplace_back(); @@ -262,23 +267,22 @@ static bool compute_vector_statistics(std::vector & tstats, c ts.stats = e; ts.sum_val = (float)sum; ts.mean_val = (float)mean; - ts.min_val = min; - ts.max_val = max; - ts.elements = (int)valid_n; + ts.elements = (int64_t)valid_n; ts.std_deviation = std_deviation; ts.skewness = skewness; ts.kurtosis = kurtosis; - ts.gain = std::numeric_limits::quiet_NaN(); + ts.gain = fnan; ts.entropy = std::abs(entropy); - ts.l2_dist = std::numeric_limits::quiet_NaN(); - ts.cossim = std::numeric_limits::quiet_NaN(); - ts.pearson = std::numeric_limits::quiet_NaN(); - ts.covariance = std::numeric_limits::quiet_NaN(); + ts.l2_dist = fnan; + ts.cossim = fnan; + ts.pearson = fnan; + ts.covariance = fnan; return true; } static void compute_tensor_statistics(std::vector & tstats) { + constexpr auto fnan = std::numeric_limits::quiet_NaN(); std::unordered_map tensor_map; tensor_map.reserve(tstats.size()); for (size_t i = 0; i < tstats.size(); ++i) { tensor_map[tstats[i].tensor] = i; } @@ -318,7 +322,7 @@ static void compute_tensor_statistics(std::vector & tstats) { double sum_p = 0.0; const size_t n = curr_avg.size(); - // Pass 1: Sums for Means + // Sums for Means for (size_t i = 0; i < n; ++i) { sum_c += curr_avg[i]; sum_p += prev_avg[i]; @@ -330,7 +334,7 @@ static void compute_tensor_statistics(std::vector & tstats) { double var_c_sum = 0.0; double var_p_sum = 0.0; - // Pass 2: Metrics + // Metrics for (size_t i = 0; i < n; ++i) { const double c_val = curr_avg[i]; const double p_val = prev_avg[i]; @@ -353,6 +357,9 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.dot_prod = dot_prod; ts.norm1_sq = norm1_sq; ts.norm2_sq = norm2_sq; + ts.cov_sum = cov_sum; + ts.var_c_sum = var_c_sum; + ts.var_p_sum = var_p_sum; ts.l2_dist_sq = l2_dist_sq; ts.l2_dist = (float)std::sqrt(l2_dist_sq); @@ -360,19 +367,24 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.covariance = (float)(cov_sum / (double)(n - 1)); } - if (norm1_sq > 0.0 && norm2_sq > 0.0) { + if (norm1_sq > 1e-12 && norm2_sq > 1e-12) { ts.cossim = (float)(dot_prod / (std::sqrt(norm1_sq) * std::sqrt(norm2_sq))); ts.cossim = std::clamp(ts.cossim, -1.0f, 1.0f); - ts.gain = (float)(std::sqrt(norm1_sq) / std::sqrt(norm2_sq)); } else { - ts.cossim = (norm1_sq == 0.0 && norm2_sq == 0.0) ? std::numeric_limits::quiet_NaN() : 0.0f; + ts.cossim = (norm1_sq == 0.0 && norm2_sq == 0.0) ? fnan : 0.0f; } - if (var_c_sum > 0.0 && var_p_sum > 0.0) { + if (var_c_sum > 1e-12 && var_p_sum > 1e-12) { ts.pearson = (float)(cov_sum / (std::sqrt(var_c_sum) * std::sqrt(var_p_sum))); ts.pearson = std::clamp(ts.pearson, -1.0f, 1.0f); } else { - ts.pearson = (var_c_sum == 0.0 && var_p_sum == 0.0) ? std::numeric_limits::quiet_NaN() : 0.0f; + ts.pearson = (var_c_sum == 0.0 && var_p_sum == 0.0) ? fnan : 0.0f; + } + + if (prev_ts.sum_val > 1e-10f) { + ts.gain = std::sqrt(ts.sum_val) / std::sqrt(prev_ts.sum_val); + } else { + ts.gain = ts.sum_val <= 1e-10f ? 1.0f : fnan; } } } @@ -387,13 +399,17 @@ static void compute_layer_statistics(const std::vector & tsta double sum_dot_prod = 0.0; double sum_norm1_sq = 0.0; double sum_norm2_sq = 0.0; - double sum_gain = 0.0; double sum_l2_dist_sq = 0.0; - double sum_pearson = 0.0; - double sum_covariance = 0.0; + double sum_cov = 0.0; + double sum_var_c = 0.0; + double sum_var_p = 0.0; + double sum_covariance_n = 0.0; + double sum_total_energy_curr = 0.0; + double sum_total_energy_prev = 0.0; int n_tensors = 0; }; + constexpr auto fnan = std::numeric_limits::quiet_NaN(); std::map laggr; for (const auto & ts : tstats) { @@ -401,20 +417,31 @@ static void compute_layer_statistics(const std::vector & tsta std::string dummy; process_tensor_name(ts.tensor, layer_str, dummy); int blk = -1; - try { blk = std::stoi(layer_str); } catch(...) { continue; } + try { + blk = std::stoi(layer_str); + } catch(...) { + if (layer_str == "-") { blk = -1; } + } + if (blk <= 0) { continue; } if (ts.norm1_sq == 0.0 && ts.norm2_sq == 0.0 && ts.l2_dist_sq == 0.0) { continue; } - auto & entry = laggr[blk]; entry.sum_dot_prod += ts.dot_prod; entry.sum_norm1_sq += ts.norm1_sq; entry.sum_norm2_sq += ts.norm2_sq; - entry.sum_gain += ts.gain; entry.sum_l2_dist_sq += ts.l2_dist_sq; - entry.sum_pearson += ts.pearson; - entry.sum_covariance += ts.covariance; + entry.sum_cov += ts.cov_sum; + entry.sum_var_c += ts.var_c_sum; + entry.sum_var_p += ts.var_p_sum; + entry.sum_covariance_n += ts.cov_sum; entry.n_tensors++; + + // Accumulate Energy for correct Layer Gain calculation + entry.sum_total_energy_curr += ts.sum_val; + if (std::isfinite(ts.gain) && ts.gain > 0.0f) { + entry.sum_total_energy_prev += ts.sum_val / (ts.gain * ts.gain); + } } for (const auto & [layer, agg] : laggr) { @@ -425,20 +452,28 @@ static void compute_layer_statistics(const std::vector & tsta cossim = (float)(agg.sum_dot_prod / (std::sqrt(agg.sum_norm1_sq) * std::sqrt(agg.sum_norm2_sq))); cossim = std::clamp(cossim, -1.0f, 1.0f); } else if (agg.sum_norm1_sq == 0.0 && agg.sum_norm2_sq == 0.0) { - cossim = std::numeric_limits::quiet_NaN(); + cossim = fnan; } float gain = 0.0f; - if (agg.sum_norm1_sq > 0.0 && agg.sum_norm2_sq > 0.0) { - gain = (float)(std::sqrt(agg.sum_norm1_sq) / std::sqrt(agg.sum_norm2_sq)); - } else if (agg.sum_norm1_sq == 0.0 && agg.sum_norm2_sq == 0.0) { - gain = std::numeric_limits::quiet_NaN(); + if (agg.sum_total_energy_prev > 0.0) { + gain = (float)(std::sqrt(agg.sum_total_energy_curr) / std::sqrt(agg.sum_total_energy_prev)); + } else { + gain = fnan; + } + + if (agg.sum_var_c > 0.0 && agg.sum_var_p > 0.0) { + auto pearson = (float)(agg.sum_cov / (std::sqrt(agg.sum_var_c) * std::sqrt(agg.sum_var_p))); + layer_pearson[layer] = std::clamp(pearson, -1.0f, 1.0f); + } else if (agg.sum_var_c == 0.0 && agg.sum_var_p == 0.0) { + layer_pearson[layer] = fnan; + } else { + layer_pearson[layer] = 0.0f; } layer_cossim[layer] = cossim; layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); - layer_pearson[layer] = (float)(agg.sum_pearson / agg.n_tensors); - layer_covariance[layer] = (float)(agg.sum_covariance / agg.n_tensors); + layer_covariance[layer] = agg.n_tensors > 0 ? (float)(agg.sum_covariance_n / agg.n_tensors) : fnan; layer_gain[layer] = gain; } } @@ -866,8 +901,6 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { float nan = std::numeric_limits::quiet_NaN(); float sum_sq = 0.0f; float mean = 0.0f; - float min = 0.0f; - float max = 0.0f; float std_deviation = 0.0f; float skewness = 0.0f; float kurtosis = 0.0f; @@ -881,8 +914,6 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { if (ts != tstat_index.end() && ts->second != nullptr) { sum_sq = ts->second->sum_val; mean = ts->second->mean_val; - min = ts->second->min_val; - max = ts->second->max_val; std_deviation = ts->second->std_deviation; skewness = ts->second->skewness; kurtosis = ts->second->kurtosis; @@ -894,21 +925,19 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { covariance = ts->second->covariance; } - struct ggml_tensor * stats = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 13); + struct ggml_tensor * stats = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 11); ggml_format_name(stats, "%s.stats", name.c_str()); ((float *)stats->data)[0] = sum_sq; ((float *) stats->data)[1] = mean; - ((float *) stats->data)[2] = min; - ((float *) stats->data)[3] = max; - ((float *) stats->data)[4] = std_deviation; - ((float *) stats->data)[5] = skewness; - ((float *) stats->data)[6] = kurtosis; - ((float *) stats->data)[7] = gain; - ((float *)stats->data)[8] = h_norm; - ((float *) stats->data)[9] = l2_dist; - ((float *) stats->data)[10] = cossim; - ((float *) stats->data)[11] = pearson; - ((float *) stats->data)[12] = covariance; + ((float *) stats->data)[2] = std_deviation; + ((float *) stats->data)[3] = skewness; + ((float *) stats->data)[4] = kurtosis; + ((float *) stats->data)[5] = gain; + ((float *)stats->data)[6] = h_norm; + ((float *) stats->data)[7] = l2_dist; + ((float *) stats->data)[8] = cossim; + ((float *) stats->data)[9] = pearson; + ((float *) stats->data)[10] = covariance; gguf_add_tensor(ctx_gguf, stats); } } @@ -1407,6 +1436,7 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c } static bool show_statistics(const common_params & params) { + constexpr auto fnan = std::numeric_limits::quiet_NaN(); g_collector.set_params(params); std::vector ts; @@ -1444,12 +1474,12 @@ static bool show_statistics(const common_params & params) { try { blk_a = std::stoi(lay_a); } catch(...) { - if (lay_a == "output") { blk_a = 10000; } + if (a.tensor.find("output") != std::string::npos) { blk_a = 10000; } } try { blk_b = std::stoi(lay_b); } catch(...) { - if (lay_b == "output") { blk_b = 10000; } + if (b.tensor.find("output") != std::string::npos) { blk_b = 10000; } } if (blk_a != blk_b) { return blk_a < blk_b; } @@ -1500,10 +1530,15 @@ static bool show_statistics(const common_params & params) { std::string name; process_tensor_name(tstat.tensor, layer, name); - const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2f((float)tstat.elements)) : std::numeric_limits::quiet_NaN(); + const float h_norm = tstat.elements > 1 ? 100.0f * (tstat.entropy / std::log2f((float)tstat.elements)) : fnan; int blk; - try { blk = std::stoi(layer); } catch (...) { blk = -1; } + try { + blk = std::stoi(layer); + } catch (...) { + if (tstat.tensor.find("output") != std::string::npos) { blk = 10000; } + else { blk = -1; } + } if (legacy) { LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%9.4f%9.4f\n", @@ -1542,7 +1577,7 @@ static bool show_statistics(const common_params & params) { int max = -1; for (const auto & [layer, stats] : ls) { - if (layer >= 0 && stats.n > 0) { + if (layer >= 0 && layer < 9999 && stats.n > 0) { layers++; min = std::min(layer, min); max = std::max(layer, max); @@ -1577,24 +1612,27 @@ static bool show_statistics(const common_params & params) { for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) { continue; } - float lgn = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_gain[layer]; - float ll2 = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_l2_dist[layer]; - float lcs = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_cossim[layer]; - float lpc = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_pearson[layer]; - float lcv = layer == 0 ? std::numeric_limits::quiet_NaN() : layer_covariance[layer]; + float lgn = layer == 0 || layer == 10000 ? fnan : layer_gain[layer]; + float ll2 = layer == 0 || layer == 10000 ? fnan : layer_l2_dist[layer]; + float lcs = layer == 0 || layer == 10000 ? fnan : layer_cossim[layer]; + float lpc = layer == 0 || layer == 10000 ? fnan : layer_pearson[layer]; + float lcv = layer == 0 || layer == 10000 ? fnan : layer_covariance[layer]; + auto str = std::to_string(layer); + const auto *lyr = layer == 10000 ? "-" : str.c_str(); if (legacy) { - LOG_INF("%*d%s%14.4f%8.2f%s%9.4f%9.4f%9.4f\n", - w_lay, layer, sep, + LOG_INF("%*s%s%14.4f%8.2f%s%9.4f%9.4f%9.4f\n", + w_lay, lyr, sep, stats.layer_sum, lgn, sep, lcs, lpc, lcv); } else { - LOG_INF("%*d%s%14.4f%8.2f%s%12.4f%9.4f%9.4f%9.4f\n", - w_lay, layer, sep, + LOG_INF("%*s%s%14.4f%8.2f%s%12.4f%9.4f%9.4f%9.4f\n", + w_lay, lyr, sep, stats.layer_sum, lgn, sep, ll2, lcs, lpc, lcv); } } + LOG_INF("\n"); return true; } From fd8348cbfe1e3067056e4670be382a29e9cd5044 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 7 Feb 2026 16:30:26 +0000 Subject: [PATCH 103/123] Fix report layout format --- tools/imatrix/imatrix.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 2ebf051f1bbf..f6fe19fbb612 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1507,21 +1507,21 @@ static bool show_statistics(const common_params & params) { LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); if (legacy) { - LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%9s%9s\n", + LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%10s%10s\n", w_lay, "Layer", sep, w_nam, "Tensor", sep, "Mean", "StdDev", "Skew", "Kurt", "H Norm", sep, "∑ E[A²]", "Gain", sep, "PCC", "Cov"); - LOG_INF("%s\n", std::string(151, '-').c_str()); + LOG_INF("%s\n", std::string(153, '-').c_str()); } else { - LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%12s%9s%9s\n", + LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%12s%10s%10s\n", w_lay, "Layer", sep, w_nam, "Tensor", sep, "Mean", "StdDev", "Skew", "Kurt", "H Norm", sep, "∑ E[A²]", "Gain", sep, "L2 Dist", "PCC", "Cov"); - LOG_INF("%s\n", std::string(163, '-').c_str()); + LOG_INF("%s\n", std::string(165, '-').c_str()); } // Tensor Statistics @@ -1541,7 +1541,7 @@ static bool show_statistics(const common_params & params) { } if (legacy) { - LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%9.4f%9.4f\n", + LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%10.4f%10.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, tstat.mean_val, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, @@ -1549,7 +1549,7 @@ static bool show_statistics(const common_params & params) { tstat.pearson, tstat.covariance ); } else { - LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%12.4f%9.4f%9.4f\n", + LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%12.4f%10.4f%10.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, tstat.mean_val, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, @@ -1596,13 +1596,13 @@ static bool show_statistics(const common_params & params) { LOG_INF("\n\nComputing layer statistics for %s (%zu layers)\n\n", params.in_files[0].c_str(), layers); if (legacy) { - LOG_INF("%*s%s%17s%8s%s%9s%9s%9s\n", + LOG_INF("%*s%s%17s%8s%s%9s%9s%12s\n", w_lay, "Layer", sep, "∑ E[A²]", "Gain", sep, "CosSim", "PCC", "Cov"); LOG_INF("%s\n", std::string(61, '-').c_str()); } else { - LOG_INF("%*s%s%17s%8s%s%12s%9s%9s%9s\n", + LOG_INF("%*s%s%17s%8s%s%12s%9s%9s%12s\n", w_lay, "Layer", sep, "∑ E[A²]", "Gain", sep, "L2 Dist", "CosSim", "PCC", "Cov"); @@ -1621,12 +1621,12 @@ static bool show_statistics(const common_params & params) { const auto *lyr = layer == 10000 ? "-" : str.c_str(); if (legacy) { - LOG_INF("%*s%s%14.4f%8.2f%s%9.4f%9.4f%9.4f\n", + LOG_INF("%*s%s%14.4f%8.2f%s%9.4f%9.4f%12.4f\n", w_lay, lyr, sep, stats.layer_sum, lgn, sep, lcs, lpc, lcv); } else { - LOG_INF("%*s%s%14.4f%8.2f%s%12.4f%9.4f%9.4f%9.4f\n", + LOG_INF("%*s%s%14.4f%8.2f%s%12.4f%9.4f%9.4f%12.4f\n", w_lay, lyr, sep, stats.layer_sum, lgn, sep, ll2, lcs, lpc, lcv); From da6837f7fc0604493da1ee2c65246841a793f92f Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 7 Feb 2026 16:37:46 +0000 Subject: [PATCH 104/123] Fix report heading --- tools/imatrix/imatrix.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index f6fe19fbb612..8ba405ed70c2 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1600,13 +1600,13 @@ static bool show_statistics(const common_params & params) { w_lay, "Layer", sep, "∑ E[A²]", "Gain", sep, "CosSim", "PCC", "Cov"); - LOG_INF("%s\n", std::string(61, '-').c_str()); + LOG_INF("%s\n", std::string(64, '-').c_str()); } else { LOG_INF("%*s%s%17s%8s%s%12s%9s%9s%12s\n", w_lay, "Layer", sep, "∑ E[A²]", "Gain", sep, "L2 Dist", "CosSim", "PCC", "Cov"); - LOG_INF("%s\n", std::string(73, '-').c_str()); + LOG_INF("%s\n", std::string(76, '-').c_str()); } for (const auto & [layer, stats] : ls) { From bb63177dfd3a965b1fd8df9d0da4e5116354b16a Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 8 Feb 2026 00:11:38 +0000 Subject: [PATCH 105/123] General refactoring --- tools/imatrix/imatrix.cpp | 110 +++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 48 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 8ba405ed70c2..3bd311c8d643 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -47,8 +47,8 @@ struct Stats { struct tensor_statistics { std::string tensor; Stats stats; - float sum_val = 0.0f; - float mean_val = 0.0f; + double sum = 0.0f; + float mean = 0.0f; int64_t elements = 0; float std_deviation = 0.0f; float skewness = 0.0f; @@ -66,6 +66,8 @@ struct tensor_statistics { double norm1_sq = 0.0; double norm2_sq = 0.0; double l2_dist_sq = 0.0; + double sum_prev = 0.0; + int64_t elements_prev = 0; }; class IMatrixCollector { @@ -148,8 +150,7 @@ static std::vector compute_tensor_averages(const Stats & tstats) { if (len == 0 || n_mat == 0 || len % n_mat != 0) { return {}; } const size_t row = len / n_mat; - std::vector vec; - vec.resize(len); + std::vector vec(len, std::numeric_limits::quiet_NaN()); bool has_valid = false; const bool use_activations = !tstats.activations.empty(); @@ -208,6 +209,7 @@ static bool compute_vector_statistics(std::vector & tstats, c const double v_act = legacy ? 0.0 : (double)e.activations[off + j] * inv_c; const double v_val = (double)e.values[off + j] * inv_c; const double v = legacy ? v_val : v_act; // Use activation average for non-legacy + if (!std::isfinite(v) || !std::isfinite(v_val)) { continue; } sum += v_val; valid_n++; @@ -237,6 +239,7 @@ static bool compute_vector_statistics(std::vector & tstats, c const double v_act = legacy ? 0.0 : (double)e.activations[off + j] * inv_c; const double v_val = (double)e.values[off + j] * inv_c; const double v = legacy ? v_val : v_act; + if (!std::isfinite(v) || !std::isfinite(v_val)) { continue; } const double diff = v - mean; sum_sq_diff += diff * diff; @@ -265,14 +268,14 @@ static bool compute_vector_statistics(std::vector & tstats, c auto & ts = tstats.emplace_back(); ts.tensor = name; ts.stats = e; - ts.sum_val = (float)sum; - ts.mean_val = (float)mean; + ts.sum = sum; + ts.mean = (float)mean; ts.elements = (int64_t)valid_n; ts.std_deviation = std_deviation; ts.skewness = skewness; ts.kurtosis = kurtosis; ts.gain = fnan; - ts.entropy = std::abs(entropy); + ts.entropy = entropy; ts.l2_dist = fnan; ts.cossim = fnan; ts.pearson = fnan; @@ -321,14 +324,22 @@ static void compute_tensor_statistics(std::vector & tstats) { double sum_c = 0.0; double sum_p = 0.0; const size_t n = curr_avg.size(); + size_t valid_n = 0; // Sums for Means for (size_t i = 0; i < n; ++i) { - sum_c += curr_avg[i]; - sum_p += prev_avg[i]; + const double c_val = curr_avg[i]; + const double p_val = prev_avg[i]; + if (std::isfinite(c_val) && std::isfinite(p_val)) { + sum_c += c_val; + sum_p += p_val; + valid_n++; + } } - const double mean_c = sum_c / n; - const double mean_p = sum_p / n; + + if (valid_n == 0) { continue; } + const double mean_c = sum_c / valid_n; + const double mean_p = sum_p / valid_n; double cov_sum = 0.0; double var_c_sum = 0.0; @@ -338,6 +349,7 @@ static void compute_tensor_statistics(std::vector & tstats) { for (size_t i = 0; i < n; ++i) { const double c_val = curr_avg[i]; const double p_val = prev_avg[i]; + if (!std::isfinite(c_val) || !std::isfinite(p_val)) { continue; } // Cosine Similarity & L2 Distance dot_prod += c_val * p_val; @@ -363,8 +375,8 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.l2_dist_sq = l2_dist_sq; ts.l2_dist = (float)std::sqrt(l2_dist_sq); - if (n > 1) { - ts.covariance = (float)(cov_sum / (double)(n - 1)); + if (valid_n > 1) { + ts.covariance = (float)(cov_sum / (double)valid_n); } if (norm1_sq > 1e-12 && norm2_sq > 1e-12) { @@ -381,10 +393,12 @@ static void compute_tensor_statistics(std::vector & tstats) { ts.pearson = (var_c_sum == 0.0 && var_p_sum == 0.0) ? fnan : 0.0f; } - if (prev_ts.sum_val > 1e-10f) { - ts.gain = std::sqrt(ts.sum_val) / std::sqrt(prev_ts.sum_val); + if (prev_ts.sum > 1e-10f) { + ts.gain = std::sqrt(ts.sum / ts.elements) / std::sqrt(prev_ts.sum / prev_ts.elements); + ts.sum_prev = prev_ts.sum; + ts.elements_prev = prev_ts.elements; } else { - ts.gain = ts.sum_val <= 1e-10f ? 1.0f : fnan; + ts.gain = ts.sum <= 1e-10f ? 1.0f : fnan; } } } @@ -403,9 +417,9 @@ static void compute_layer_statistics(const std::vector & tsta double sum_cov = 0.0; double sum_var_c = 0.0; double sum_var_p = 0.0; - double sum_covariance_n = 0.0; double sum_total_energy_curr = 0.0; double sum_total_energy_prev = 0.0; + int64_t sum_valid_n = 0; int n_tensors = 0; }; @@ -434,14 +448,12 @@ static void compute_layer_statistics(const std::vector & tsta entry.sum_cov += ts.cov_sum; entry.sum_var_c += ts.var_c_sum; entry.sum_var_p += ts.var_p_sum; - entry.sum_covariance_n += ts.cov_sum; entry.n_tensors++; + if (ts.elements > 0) { entry.sum_valid_n += ts.elements; } // Accumulate Energy for correct Layer Gain calculation - entry.sum_total_energy_curr += ts.sum_val; - if (std::isfinite(ts.gain) && ts.gain > 0.0f) { - entry.sum_total_energy_prev += ts.sum_val / (ts.gain * ts.gain); - } + entry.sum_total_energy_curr += ts.sum; + if (ts.sum_prev > 0.0) { entry.sum_total_energy_prev += ts.sum_prev; } } for (const auto & [layer, agg] : laggr) { @@ -462,7 +474,14 @@ static void compute_layer_statistics(const std::vector & tsta gain = fnan; } - if (agg.sum_var_c > 0.0 && agg.sum_var_p > 0.0) { + layer_cossim[layer] = cossim; + layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); + layer_gain[layer] = gain; + + if (agg.sum_valid_n > 0) { layer_covariance[layer] = (float)(agg.sum_cov / (double)agg.sum_valid_n); } + else { layer_covariance[layer] = fnan; } + + if (agg.sum_var_c > 1e-12 && agg.sum_var_p > 1e-12) { auto pearson = (float)(agg.sum_cov / (std::sqrt(agg.sum_var_c) * std::sqrt(agg.sum_var_p))); layer_pearson[layer] = std::clamp(pearson, -1.0f, 1.0f); } else if (agg.sum_var_c == 0.0 && agg.sum_var_p == 0.0) { @@ -470,11 +489,6 @@ static void compute_layer_statistics(const std::vector & tsta } else { layer_pearson[layer] = 0.0f; } - - layer_cossim[layer] = cossim; - layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); - layer_covariance[layer] = agg.n_tensors > 0 ? (float)(agg.sum_covariance_n / agg.n_tensors) : fnan; - layer_gain[layer] = gain; } } @@ -898,8 +912,8 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { // Store per-tensor statistics as a small 1D tensor { - float nan = std::numeric_limits::quiet_NaN(); - float sum_sq = 0.0f; + float fnan = std::numeric_limits::quiet_NaN(); + double sum_sq = 0.0f; float mean = 0.0f; float std_deviation = 0.0f; float skewness = 0.0f; @@ -912,13 +926,13 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { float covariance = 0.0f; auto ts = tstat_index.find(name); if (ts != tstat_index.end() && ts->second != nullptr) { - sum_sq = ts->second->sum_val; - mean = ts->second->mean_val; + sum_sq = ts->second->sum; + mean = ts->second->mean; std_deviation = ts->second->std_deviation; skewness = ts->second->skewness; kurtosis = ts->second->kurtosis; gain = ts->second->gain; - h_norm = ts->second->elements > 0 ? 100.0f * (ts->second->entropy / std::log2f((float)ts->second->elements)) : nan; + h_norm = ts->second->elements > 1 ? 100.0f * (ts->second->entropy / std::log2f((float)ts->second->elements)) : fnan; l2_dist = ts->second->l2_dist; cossim = ts->second->cossim; pearson = ts->second->pearson; @@ -927,17 +941,17 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { struct ggml_tensor * stats = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 11); ggml_format_name(stats, "%s.stats", name.c_str()); - ((float *)stats->data)[0] = sum_sq; - ((float *) stats->data)[1] = mean; - ((float *) stats->data)[2] = std_deviation; - ((float *) stats->data)[3] = skewness; - ((float *) stats->data)[4] = kurtosis; - ((float *) stats->data)[5] = gain; + ((float *)stats->data)[0] = (float)sum_sq; + ((float *)stats->data)[1] = mean; + ((float *)stats->data)[2] = std_deviation; + ((float *)stats->data)[3] = skewness; + ((float *)stats->data)[4] = kurtosis; + ((float *)stats->data)[5] = gain; ((float *)stats->data)[6] = h_norm; - ((float *) stats->data)[7] = l2_dist; - ((float *) stats->data)[8] = cossim; - ((float *) stats->data)[9] = pearson; - ((float *) stats->data)[10] = covariance; + ((float *)stats->data)[7] = l2_dist; + ((float *)stats->data)[8] = cossim; + ((float *)stats->data)[9] = pearson; + ((float *)stats->data)[10] = covariance; gguf_add_tensor(ctx_gguf, stats); } } @@ -1544,23 +1558,23 @@ static bool show_statistics(const common_params & params) { LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%10.4f%10.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, - tstat.mean_val, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, - tstat.sum_val, tstat.gain, sep, + tstat.mean, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, + tstat.sum, tstat.gain, sep, tstat.pearson, tstat.covariance ); } else { LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%12.4f%10.4f%10.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, - tstat.mean_val, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, - tstat.sum_val, tstat.gain, sep, + tstat.mean, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, + tstat.sum, tstat.gain, sep, tstat.l2_dist, tstat.pearson, tstat.covariance ); } // Aggregate Layer Stats auto & l = ls[blk]; - l.layer_sum += tstat.sum_val; + l.layer_sum += tstat.sum; l.n += tstat.elements; } From 8b1c1f5635429ed2b70de9a8c836924e36469eff Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 8 Feb 2026 10:17:42 +0000 Subject: [PATCH 106/123] Tighten covariance calculation and avoid displaying default value for missing elements in map --- tools/imatrix/imatrix.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 3bd311c8d643..1afd5ca44b1a 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -68,6 +68,7 @@ struct tensor_statistics { double l2_dist_sq = 0.0; double sum_prev = 0.0; int64_t elements_prev = 0; + int64_t n_features = 0; }; class IMatrixCollector { @@ -366,6 +367,7 @@ static void compute_tensor_statistics(std::vector & tstats) { var_p_sum += dp * dp; } + ts.n_features = (int64_t)n; ts.dot_prod = dot_prod; ts.norm1_sq = norm1_sq; ts.norm2_sq = norm2_sq; @@ -420,6 +422,7 @@ static void compute_layer_statistics(const std::vector & tsta double sum_total_energy_curr = 0.0; double sum_total_energy_prev = 0.0; int64_t sum_valid_n = 0; + int64_t sum_n_features = 0; int n_tensors = 0; }; @@ -450,6 +453,7 @@ static void compute_layer_statistics(const std::vector & tsta entry.sum_var_p += ts.var_p_sum; entry.n_tensors++; if (ts.elements > 0) { entry.sum_valid_n += ts.elements; } + if (ts.n_features > 0) { entry.sum_n_features += ts.n_features; } // Accumulate Energy for correct Layer Gain calculation entry.sum_total_energy_curr += ts.sum; @@ -478,7 +482,7 @@ static void compute_layer_statistics(const std::vector & tsta layer_l2_dist[layer] = (float)std::sqrt(agg.sum_l2_dist_sq); layer_gain[layer] = gain; - if (agg.sum_valid_n > 0) { layer_covariance[layer] = (float)(agg.sum_cov / (double)agg.sum_valid_n); } + if (agg.sum_n_features > 0) { layer_covariance[layer] = (float)(agg.sum_cov / (double)agg.sum_n_features); } else { layer_covariance[layer] = fnan; } if (agg.sum_var_c > 1e-12 && agg.sum_var_p > 1e-12) { @@ -1623,14 +1627,19 @@ static bool show_statistics(const common_params & params) { LOG_INF("%s\n", std::string(76, '-').c_str()); } + auto get_layer_stat = [](const std::map& map, int layer) -> float { + const auto it = map.find(layer); + return it != map.end() ? it->second : fnan; + }; + for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) { continue; } - float lgn = layer == 0 || layer == 10000 ? fnan : layer_gain[layer]; - float ll2 = layer == 0 || layer == 10000 ? fnan : layer_l2_dist[layer]; - float lcs = layer == 0 || layer == 10000 ? fnan : layer_cossim[layer]; - float lpc = layer == 0 || layer == 10000 ? fnan : layer_pearson[layer]; - float lcv = layer == 0 || layer == 10000 ? fnan : layer_covariance[layer]; + float lgn = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_gain, layer); + float ll2 = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_l2_dist, layer); + float lcs = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_cossim, layer); + float lpc = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_pearson, layer); + float lcv = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_covariance, layer); auto str = std::to_string(layer); const auto *lyr = layer == 10000 ? "-" : str.c_str(); From e7a3173c7348f5f4c3b028aeda1ec60438a7c2c1 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 8 Feb 2026 14:42:11 +0000 Subject: [PATCH 107/123] Add number of vector/tensor elements to imatrix file --- tools/imatrix/imatrix.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 1afd5ca44b1a..8b6234f0ec0e 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -919,6 +919,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { float fnan = std::numeric_limits::quiet_NaN(); double sum_sq = 0.0f; float mean = 0.0f; + float elements = 0.0f; float std_deviation = 0.0f; float skewness = 0.0f; float kurtosis = 0.0f; @@ -932,6 +933,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { if (ts != tstat_index.end() && ts->second != nullptr) { sum_sq = ts->second->sum; mean = ts->second->mean; + elements = (float)ts->second->elements; std_deviation = ts->second->std_deviation; skewness = ts->second->skewness; kurtosis = ts->second->kurtosis; @@ -943,19 +945,20 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { covariance = ts->second->covariance; } - struct ggml_tensor * stats = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 11); + struct ggml_tensor * stats = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 12); ggml_format_name(stats, "%s.stats", name.c_str()); ((float *)stats->data)[0] = (float)sum_sq; ((float *)stats->data)[1] = mean; - ((float *)stats->data)[2] = std_deviation; - ((float *)stats->data)[3] = skewness; - ((float *)stats->data)[4] = kurtosis; - ((float *)stats->data)[5] = gain; - ((float *)stats->data)[6] = h_norm; - ((float *)stats->data)[7] = l2_dist; - ((float *)stats->data)[8] = cossim; - ((float *)stats->data)[9] = pearson; - ((float *)stats->data)[10] = covariance; + ((float *)stats->data)[2] = elements; + ((float *)stats->data)[3] = std_deviation; + ((float *)stats->data)[4] = skewness; + ((float *)stats->data)[5] = kurtosis; + ((float *)stats->data)[6] = gain; + ((float *)stats->data)[7] = h_norm; + ((float *)stats->data)[8] = l2_dist; + ((float *)stats->data)[9] = cossim; + ((float *)stats->data)[10] = pearson; + ((float *)stats->data)[11] = covariance; gguf_add_tensor(ctx_gguf, stats); } } From bc00ac0e50d208e63c32591f3d9cb4b97c02a647 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 28 Mar 2026 08:32:02 +0000 Subject: [PATCH 108/123] Fix code formatting --- tools/imatrix/imatrix.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 89d15ce59e8a..eb27ba8167b9 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -119,7 +119,9 @@ static void process_tensor_name(const std::string & input, std::string & layer, std::istringstream stream(input); std::string item; - while (std::getline(stream, item, '.')) { name.push_back(item); } + while (std::getline(stream, item, '.')) { + name.push_back(item); + } for (size_t i = 0; i < name.size(); ++i) { if (name[i] == "blk" && i + 1 < name.size()) { layer = name[i + 1]; @@ -131,7 +133,8 @@ static void process_tensor_name(const std::string & input, std::string & layer, for (size_t j = 0; j < name.size(); ++j) { if (name[j] == "blk") { j+=2; - continue; } + continue; + } if (j == i) { break; } if (!tensor.empty()) { tensor += "."; } tensor += name[j]; From e8938a3448371c0e18409433474a2d9644f0c94e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 6 Apr 2026 10:23:33 +0100 Subject: [PATCH 109/123] Define the schema for the tensor statistics for use in quantize.cpp --- tools/imatrix/imatrix.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 80e789308b9a..735e1d9f423f 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -38,6 +38,7 @@ static void print_usage(int, char ** argv) { static const char * const LLM_KV_IMATRIX_DATASETS = "imatrix.datasets"; static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count"; static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size"; +static const char * const LLM_KV_IMATRIX_STATS_SCHEMA = "imatrix.stats_schema"; struct Stats { std::vector activations; @@ -885,6 +886,12 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { // Write the number of chunks the matrix was computed with gguf_set_val_u32(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT, m_last_chunk); gguf_set_val_u32(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE, m_params.n_ctx / m_params.n_parallel); + + // Define the schema for the tensor statistics (for use in quantize.cpp) + const char * stats_schema[] = {"sum_sq", "mean", "elements", "std_deviation", "skewness", "kurtosis", "gain", + "h_norm", "l2_dist", "cossim", "pearson", "covariance" + }; + gguf_set_arr_str(ctx_gguf, LLM_KV_IMATRIX_STATS_SCHEMA, stats_schema, 12); } for (const auto & name : to_store) { @@ -951,6 +958,7 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { struct ggml_tensor * stats = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 12); ggml_format_name(stats, "%s.stats", name.c_str()); + // Store the statistics in the same order as defined in stats_schema[] ((float *)stats->data)[0] = (float)sum_sq; ((float *)stats->data)[1] = mean; ((float *)stats->data)[2] = elements; From e7ce44de00ccc1c5aa00dba035f6ebc475d555e9 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 12 Apr 2026 16:55:57 +0100 Subject: [PATCH 110/123] Update README.md --- tools/imatrix/README.md | 45 +++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index ccb0bd8c6433..45fc33fb8ae1 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -71,31 +71,46 @@ Recent versions of `llama-imatrix` store data in GGUF format by default. For the ```bash # analyze imatrix file and display summary statistics instead of running inference -./llama-imatrix --in-file imatrix.gguf --show-statistics +./llama-imatrix -m ggml-model-f16.gguf --in-file imatrix.gguf --show-statistics ``` ## Statistics - -Please note that the L₂ Distance can only be calculated if the imatrix is in GGUF format. If a value lacks proper statistical interpretability, **nan** will be shown instead. The following statistics are computed: +Please note that if a value lacks statistical interpretability, **nan** will be shown instead. #### Per tensor - -* **Min / Max / μ / σ**: Tensor elements Min, Max, Mean, and Standard Deviation. -* **H Norm**: Shannon Entropy normalized over log₂(N). Defined as $H Norm=\frac{-\sum_{i=1}^N p_i \log_2 p_i}{log_2 N}$. Used to determine how well a prompt "exercises" the model's capabilities. Higher values indicate more uniform distribution of activations. Every neuron is firing equally; hard to prune. -* **Z-score Distribution (ZD)**: % of elements whose ZD-score is > 1.0 (an indicator of outliers), as described in _3.1 Layer Importance Scores_ of [Layer-Wise Quantization](https://arxiv.org/abs/2406.17415). -* **∑ E[A²]**: The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision (e.g., Q6_K vs Q4_K). -* **L₂ Distance**: Euclidean Distance from the tensor in the previous layer. Measure of transformation magnitude; higher values indicate more significant transformation on the data. -* **CosSim**: Cosine Similarity with the tensor in the previous layer. _~1.0_, the tensor output points in the exact same direction as the previous layer's tensor (the layer is refining magnitude, not direction). _< 1.0_, the layer is rotating the vector space (changing semantic meaning). -* **PCC**: Pearson Correlation Coefficient with the tensor in the previous layer. Checks for linear correlation excluding the mean shift. Similar to CosSim but centers geometric data first. Indicates if the pattern of activation changes or just the offset. +Statistical properties of a single tensor's average activation or activation energy (squared magnitude). + +* **Mean / StdDev**: $\mu = \frac{1}{N} \sum v_i$ and $\sigma = \sqrt{\frac{1}{N-1 \text{ or } N} \sum (v_i - \mu)^2}$ + - Establishes the baseline distribution of the tensor's outputs. Low variance means the tensor outputs a mostly constant projection; high variance implies high information density across dimensions. +* **Skewness & Kurtosis**: $skew = \frac{\frac{1}{N} \sum (v_i - \mu)^3}{\sigma^3}$ and $kurt = \frac{\frac{1}{N} \sum (v_i - \mu)^4}{\sigma^4} - 3.0$ + - Skewness measures the asymmetry of a distribution around its mean. Kurtosis measures the "tailedness" of the feature activations. A high kurtosis indicates a highly sparse/heavy-tailed activation distribution (e.g., outlier features). High-kurtosis tensors typically require higher precision quantization to prevent outlier degradation. +* **H Norm**: $H = -\sum_{i} P_i \log_2(P_i)$, where $P_i = \frac{v_{val_i}}{\sum v_{val_i}}$ + - Shannon Entropy normalized over log₂(N). Used to determine how well a prompt "exercises" the model's capabilities. Higher values indicate more uniform distribution of activations. Every neuron is firing equally; hard to prune. +* **$\sum E[A^2]$**: $\sum E[x_i^2]$ + - The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision. + +#### Intra-layer +These statistics compare identical tensor between the current layer $L$ and the previsou layer $L-1$ (e.g., `blk.1.attn_v` vs `blk.0.attn_v`). + +* **Gain**: $G = \frac{\sqrt{\sum C_i^2 / N_{curr}}}{\sqrt{\sum P_i^2 / N_{prev}}}$ + - Indicates if a layer acts as an "amplifier" ($G > 1$) or a "dampener" ($G < 1$). +* **L2 Distance**: $L2 = \sqrt{ \sum (C_i - P_i)^2 }$ where $C$ is the current layer and $P$ is the previous layer's tensor. + - Measures absolute representational shift. Huge leaps in L2 distance indicate that a layer fundamentally transforms the hidden states. +* **Pearson Correlation Coefficient (PCC)**: $r = \frac{\sum (C_i - \bar{C})(P_i - \bar{P})}{\sqrt{\sum (C_i - \bar{C})^2} \sqrt{\sum (P_i - \bar{P})^2}}$ + - Similar to Cosine Similarity, but invariant to scalar or offset biases (centers the data first). Highly correlated adjacent layers signify structural repetition. +* **Covariance (Cov)**: $\frac{1}{N}\sum (c_i-\bar c)(p_i-\bar p)$ + - The **unnormalized covariance** between current and previous layer activations. Captures both the correlation structure and the magnitude of the joint variation. Large absolute covariance indicates the layers are jointly processing strong, correlated signals. #### Per layer - Aggregated metrics per block/layer: -* **Z-score Distribution (ZD)**: % of this layer's concatenated tensors' elements with |Z| > 1. Indicates general "spikiness" of the layer's activations. * **∑ E[A²]:** Total energy of the layer's concatenated tensors. Indicates the layer's overall contribution amplitude. +* **Gain**: Indicates if a layer acts as an "amplifier" ($G > 1$) or a "dampener" ($G < 1$). * **L₂ Distance:** Euclidean Distance of the layer's concatenated tensors from the previous layer’s. Global measure of transformation magnitude. -* **CosSim**: Cosine Similarity of this layer's concatenated tensors with the previous layer. -* **PCC**: Average Pearson Correlation of the tensors in the layer. +* **CosSim**: $\text{CosSim}_{Layer} = \frac{\sum_{\text{tensors}} (\text{Dot Prod})}{\sqrt{\sum_{\text{tensors}} (\text{Norm1}^2)} \sqrt{\sum_{\text{tensors}} (\text{Norm2}^2)}}$ + - Cosine Similarity of the current layer's concatenated tensors with the previous layer. +* **PCC**: $\text{PCC}_{Layer} = \frac{\sum \text{Covariance}^2}{\sqrt{\sum \text{Var}_{curr}} \sqrt{\sum \text{Var}_{prev}}}$ + - Average Pearson Correlation of the tensors in the layer. +* **Cov**: The **unnormalized covariance** between current layer's concatenated tensors and the previous layer. More information is available in https://github.com/ggml-org/llama.cpp/pull/14891 From afc5e4ecbdb9350b6213d4031cb2251bd602d16c Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 12 Apr 2026 19:17:30 +0100 Subject: [PATCH 111/123] Update README.md --- tools/imatrix/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 45fc33fb8ae1..512b2e544f56 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -80,7 +80,7 @@ Please note that if a value lacks statistical interpretability, **nan** will be #### Per tensor Statistical properties of a single tensor's average activation or activation energy (squared magnitude). -* **Mean / StdDev**: $\mu = \frac{1}{N} \sum v_i$ and $\sigma = \sqrt{\frac{1}{N-1 \text{ or } N} \sum (v_i - \mu)^2}$ +* **Mean / StdDev**: $\mu = \frac{1}{N} \sum v_i$ and $\sigma = \sqrt{\frac{1}{N} \sum (v_i - \mu)^2}$ - Establishes the baseline distribution of the tensor's outputs. Low variance means the tensor outputs a mostly constant projection; high variance implies high information density across dimensions. * **Skewness & Kurtosis**: $skew = \frac{\frac{1}{N} \sum (v_i - \mu)^3}{\sigma^3}$ and $kurt = \frac{\frac{1}{N} \sum (v_i - \mu)^4}{\sigma^4} - 3.0$ - Skewness measures the asymmetry of a distribution around its mean. Kurtosis measures the "tailedness" of the feature activations. A high kurtosis indicates a highly sparse/heavy-tailed activation distribution (e.g., outlier features). High-kurtosis tensors typically require higher precision quantization to prevent outlier degradation. From 36c310fefd218a05973200e3c74afc50694c8149 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Mon, 13 Apr 2026 20:30:55 +0100 Subject: [PATCH 112/123] Warn if no imatrix data for tensor --- tools/imatrix/imatrix.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 735e1d9f423f..5e972d7508d5 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -923,6 +923,8 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const { } gguf_add_tensor(ctx_gguf, in_sum); } + } else { + LOG_WRN("%s: no data for tensor %s\n", __func__, name.c_str()); } // Store per-tensor statistics as a small 1D tensor From 753523a6b5dcd063aaacc53f2cdea1aa819238a9 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 23 Apr 2026 09:48:05 +0100 Subject: [PATCH 113/123] Fix C3493 error when target compilation is a MS platform --- tools/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 5e972d7508d5..ec83e977d13d 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1646,7 +1646,7 @@ static bool show_statistics(const common_params & params) { LOG_INF("%s\n", std::string(76, '-').c_str()); } - auto get_layer_stat = [](const std::map& map, int layer) -> float { + auto get_layer_stat = [&](const std::map& map, const int layer) -> float { const auto it = map.find(layer); return it != map.end() ? it->second : fnan; }; From 14f9d0d6a7423ea160933592629cabbdff3d28c9 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 23 Apr 2026 09:55:08 +0100 Subject: [PATCH 114/123] Refactor layer reporting --- tools/imatrix/imatrix.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ec83e977d13d..ef60ae600dd8 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1506,17 +1506,17 @@ static bool show_statistics(const common_params & params) { process_tensor_name(b.tensor, lay_b, name_b); // Handle non-numeric layers (e.g., "output") - int blk_a = 9999; - int blk_b = 9999; + int blk_a = INT_MAX - 1; + int blk_b = INT_MAX - 1; try { blk_a = std::stoi(lay_a); } catch(...) { - if (a.tensor.find("output") != std::string::npos) { blk_a = 10000; } + if (a.tensor.find("output") != std::string::npos) { blk_a = INT_MAX; } } try { blk_b = std::stoi(lay_b); } catch(...) { - if (b.tensor.find("output") != std::string::npos) { blk_b = 10000; } + if (b.tensor.find("output") != std::string::npos) { blk_b = INT_MAX; } } if (blk_a != blk_b) { return blk_a < blk_b; } @@ -1573,7 +1573,7 @@ static bool show_statistics(const common_params & params) { try { blk = std::stoi(layer); } catch (...) { - if (tstat.tensor.find("output") != std::string::npos) { blk = 10000; } + if (tstat.tensor.find("output") != std::string::npos) { blk = INT_MAX; } else { blk = -1; } } @@ -1654,13 +1654,13 @@ static bool show_statistics(const common_params & params) { for (const auto & [layer, stats] : ls) { if (layer < 0 || stats.n == 0) { continue; } - float lgn = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_gain, layer); - float ll2 = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_l2_dist, layer); - float lcs = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_cossim, layer); - float lpc = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_pearson, layer); - float lcv = layer == 0 || layer == 10000 ? fnan : get_layer_stat(layer_covariance, layer); + float lgn = layer == 0 || layer == INT_MAX ? fnan : get_layer_stat(layer_gain, layer); + float ll2 = layer == 0 || layer == INT_MAX ? fnan : get_layer_stat(layer_l2_dist, layer); + float lcs = layer == 0 || layer == INT_MAX ? fnan : get_layer_stat(layer_cossim, layer); + float lpc = layer == 0 || layer == INT_MAX ? fnan : get_layer_stat(layer_pearson, layer); + float lcv = layer == 0 || layer == INT_MAX ? fnan : get_layer_stat(layer_covariance, layer); auto str = std::to_string(layer); - const auto *lyr = layer == 10000 ? "-" : str.c_str(); + const auto *lyr = layer == INT_MAX ? "-" : str.c_str(); if (legacy) { LOG_INF("%*s%s%14.4f%8.2f%s%9.4f%9.4f%12.4f\n", From ca6ad4d159a8486fcd2ffacbcf60e38f3a1cfddc Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 23 Apr 2026 12:25:59 +0100 Subject: [PATCH 115/123] Fix trailing spaces --- tools/imatrix/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/imatrix/README.md b/tools/imatrix/README.md index 512b2e544f56..a92f861126ec 100644 --- a/tools/imatrix/README.md +++ b/tools/imatrix/README.md @@ -84,7 +84,7 @@ Statistical properties of a single tensor's average activation or activation ene - Establishes the baseline distribution of the tensor's outputs. Low variance means the tensor outputs a mostly constant projection; high variance implies high information density across dimensions. * **Skewness & Kurtosis**: $skew = \frac{\frac{1}{N} \sum (v_i - \mu)^3}{\sigma^3}$ and $kurt = \frac{\frac{1}{N} \sum (v_i - \mu)^4}{\sigma^4} - 3.0$ - Skewness measures the asymmetry of a distribution around its mean. Kurtosis measures the "tailedness" of the feature activations. A high kurtosis indicates a highly sparse/heavy-tailed activation distribution (e.g., outlier features). High-kurtosis tensors typically require higher precision quantization to prevent outlier degradation. -* **H Norm**: $H = -\sum_{i} P_i \log_2(P_i)$, where $P_i = \frac{v_{val_i}}{\sum v_{val_i}}$ +* **H Norm**: $H = -\sum_{i} P_i \log_2(P_i)$, where $P_i = \frac{v_{val_i}}{\sum v_{val_i}}$ - Shannon Entropy normalized over log₂(N). Used to determine how well a prompt "exercises" the model's capabilities. Higher values indicate more uniform distribution of activations. Every neuron is firing equally; hard to prune. * **$\sum E[A^2]$**: $\sum E[x_i^2]$ - The sum of squares of activations (Energy) for the tensor. Tensors with high "energy" contribute most to the final output. Quantization errors here propagate strongly. These tensors usually need higher precision. @@ -92,13 +92,13 @@ Statistical properties of a single tensor's average activation or activation ene #### Intra-layer These statistics compare identical tensor between the current layer $L$ and the previsou layer $L-1$ (e.g., `blk.1.attn_v` vs `blk.0.attn_v`). -* **Gain**: $G = \frac{\sqrt{\sum C_i^2 / N_{curr}}}{\sqrt{\sum P_i^2 / N_{prev}}}$ +* **Gain**: $G = \frac{\sqrt{\sum C_i^2 / N_{curr}}}{\sqrt{\sum P_i^2 / N_{prev}}}$ - Indicates if a layer acts as an "amplifier" ($G > 1$) or a "dampener" ($G < 1$). -* **L2 Distance**: $L2 = \sqrt{ \sum (C_i - P_i)^2 }$ where $C$ is the current layer and $P$ is the previous layer's tensor. +* **L2 Distance**: $L2 = \sqrt{ \sum (C_i - P_i)^2 }$ where $C$ is the current layer and $P$ is the previous layer's tensor. - Measures absolute representational shift. Huge leaps in L2 distance indicate that a layer fundamentally transforms the hidden states. * **Pearson Correlation Coefficient (PCC)**: $r = \frac{\sum (C_i - \bar{C})(P_i - \bar{P})}{\sqrt{\sum (C_i - \bar{C})^2} \sqrt{\sum (P_i - \bar{P})^2}}$ - Similar to Cosine Similarity, but invariant to scalar or offset biases (centers the data first). Highly correlated adjacent layers signify structural repetition. -* **Covariance (Cov)**: $\frac{1}{N}\sum (c_i-\bar c)(p_i-\bar p)$ +* **Covariance (Cov)**: $\frac{1}{N}\sum (c_i-\bar c)(p_i-\bar p)$ - The **unnormalized covariance** between current and previous layer activations. Captures both the correlation structure and the magnitude of the joint variation. Large absolute covariance indicates the layers are jointly processing strong, correlated signals. #### Per layer @@ -109,7 +109,7 @@ Aggregated metrics per block/layer: * **L₂ Distance:** Euclidean Distance of the layer's concatenated tensors from the previous layer’s. Global measure of transformation magnitude. * **CosSim**: $\text{CosSim}_{Layer} = \frac{\sum_{\text{tensors}} (\text{Dot Prod})}{\sqrt{\sum_{\text{tensors}} (\text{Norm1}^2)} \sqrt{\sum_{\text{tensors}} (\text{Norm2}^2)}}$ - Cosine Similarity of the current layer's concatenated tensors with the previous layer. -* **PCC**: $\text{PCC}_{Layer} = \frac{\sum \text{Covariance}^2}{\sqrt{\sum \text{Var}_{curr}} \sqrt{\sum \text{Var}_{prev}}}$ +* **PCC**: $\text{PCC}_{Layer} = \frac{\sum \text{Covariance}^2}{\sqrt{\sum \text{Var}_{curr}} \sqrt{\sum \text{Var}_{prev}}}$ - Average Pearson Correlation of the tensors in the layer. * **Cov**: The **unnormalized covariance** between current layer's concatenated tensors and the previous layer. From f9e0489e1f1a99ce55bfcd562f12acb851eadce6 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Thu, 23 Apr 2026 14:06:08 +0100 Subject: [PATCH 116/123] Add --- tools/imatrix/imatrix.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index ef60ae600dd8..b5dd0b747200 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include From c01e35f86d6080ef57128dd86f59379ae5852cfa Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 May 2026 15:20:21 +0100 Subject: [PATCH 117/123] Add --reduce-mem option --- common/arg.cpp | 7 +++++++ common/common.h | 1 + 2 files changed, 8 insertions(+) diff --git a/common/arg.cpp b/common/arg.cpp index c21598e7687f..3cdae6a3dc16 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2737,6 +2737,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.parse_special = true; } ).set_examples({LLAMA_EXAMPLE_IMATRIX})); + add_opt(common_arg( + {"--reduce-mem"}, + string_format("decrease memory usage by evicting weights from the OS cache after each layer (default: %s)", params.reduce_mem ? "true" : "false"), + [](common_params & params) { + params.reduce_mem = true; + } + ).set_examples({LLAMA_EXAMPLE_IMATRIX})); add_opt(common_arg( {"-pps"}, string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"), diff --git a/common/common.h b/common/common.h index 344f5ab37f8c..24256c2aacef 100644 --- a/common/common.h +++ b/common/common.h @@ -668,6 +668,7 @@ struct common_params { bool show_statistics = false; // show imatrix statistics per tensor bool activation_statistics = false; // generate data to calculate activation based statistics bool parse_special = false; // whether to parse special tokens during imatrix tokenization + bool reduce_mem = false; // decrease memory usage by evicting weights from the OS cache after each layer // cvector-generator params int n_pca_batch = 100; From 3348f05c286189757029566212c908804fd44b3e Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 2 May 2026 15:20:40 +0100 Subject: [PATCH 118/123] Evict weights from cache after processing each layer --- tools/imatrix/imatrix.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index b5dd0b747200..6e6d819e6c60 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -26,13 +26,18 @@ #pragma warning(disable: 4244 4267) // possible loss of data #endif +#if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) +#include +#include +#endif + static void print_usage(int, char ** argv) { LOG("\nexample usage:\n"); LOG("\n %s \\\n" " -m model.gguf -f some-text.txt [-o imatrix.gguf] [--output-format {gguf,dat}] [--no-ppl] \\\n" " [--process-output] [--chunk 123] [--save-frequency 0] [--output-frequency 10] \\\n" " [--in-file imatrix-prev-0.gguf --in-file imatrix-prev-1.gguf ...] [--parse-special] \\\n" - " [--show-statistics] [...]\n" , argv[0]); + " [--show-statistics] [--reduce-mem] [...]\n" , argv[0]); LOG("\n"); } @@ -685,6 +690,16 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * } } +#if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) + if (m_params.reduce_mem && m_params.use_mmap && src0->buffer && ggml_backend_buffer_is_host(src0->buffer)) { + const size_t page_size = sysconf(_SC_PAGESIZE); + uintptr_t addr = (uintptr_t)src0->data; + uintptr_t aligned_addr = addr & ~(page_size - 1); + size_t size = ggml_nbytes(src0) + (addr - aligned_addr); + madvise((void *)aligned_addr, size, MADV_DONTNEED); + } +#endif + return true; } From 2f8d20a013479bf3b4a3c8045a6d3371cd10df56 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sun, 3 May 2026 21:14:45 +0100 Subject: [PATCH 119/123] Add uv.lock --- uv.lock | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 uv.lock diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000000..7518fc90bf78 --- /dev/null +++ b/uv.lock @@ -0,0 +1,3 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" From b8745e2ab865efa72645a5cdef179692e33435c6 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Wed, 6 May 2026 19:42:10 +0100 Subject: [PATCH 120/123] Remove reduce mem option flag --- common/arg.cpp | 7 ------- common/common.h | 1 - tools/imatrix/imatrix.cpp | 10 +++++----- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 3cdae6a3dc16..c21598e7687f 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2737,13 +2737,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.parse_special = true; } ).set_examples({LLAMA_EXAMPLE_IMATRIX})); - add_opt(common_arg( - {"--reduce-mem"}, - string_format("decrease memory usage by evicting weights from the OS cache after each layer (default: %s)", params.reduce_mem ? "true" : "false"), - [](common_params & params) { - params.reduce_mem = true; - } - ).set_examples({LLAMA_EXAMPLE_IMATRIX})); add_opt(common_arg( {"-pps"}, string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"), diff --git a/common/common.h b/common/common.h index 24256c2aacef..344f5ab37f8c 100644 --- a/common/common.h +++ b/common/common.h @@ -668,7 +668,6 @@ struct common_params { bool show_statistics = false; // show imatrix statistics per tensor bool activation_statistics = false; // generate data to calculate activation based statistics bool parse_special = false; // whether to parse special tokens during imatrix tokenization - bool reduce_mem = false; // decrease memory usage by evicting weights from the OS cache after each layer // cvector-generator params int n_pca_batch = 100; diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 6e6d819e6c60..812ec6e4031a 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -37,7 +37,7 @@ static void print_usage(int, char ** argv) { " -m model.gguf -f some-text.txt [-o imatrix.gguf] [--output-format {gguf,dat}] [--no-ppl] \\\n" " [--process-output] [--chunk 123] [--save-frequency 0] [--output-frequency 10] \\\n" " [--in-file imatrix-prev-0.gguf --in-file imatrix-prev-1.gguf ...] [--parse-special] \\\n" - " [--show-statistics] [--reduce-mem] [...]\n" , argv[0]); + " [--show-statistics] [...]\n" , argv[0]); LOG("\n"); } @@ -691,11 +691,11 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * } #if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) - if (m_params.reduce_mem && m_params.use_mmap && src0->buffer && ggml_backend_buffer_is_host(src0->buffer)) { + if (m_params.use_mmap && src0->buffer && ggml_backend_buffer_is_host(src0->buffer)) { const size_t page_size = sysconf(_SC_PAGESIZE); - uintptr_t addr = (uintptr_t)src0->data; - uintptr_t aligned_addr = addr & ~(page_size - 1); - size_t size = ggml_nbytes(src0) + (addr - aligned_addr); + const uintptr_t addr = (uintptr_t)src0->data; + const uintptr_t aligned_addr = addr & ~(page_size - 1); + const size_t size = ggml_nbytes(src0) + (addr - aligned_addr); madvise((void *)aligned_addr, size, MADV_DONTNEED); } #endif From 613941540f04f547a3019109367fd1259c5ba02d Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 6 Jun 2026 19:00:57 +0100 Subject: [PATCH 121/123] Improve report aesthetics --- tools/imatrix/imatrix.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 5040aceea855..eb0374fe8c47 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1394,24 +1394,24 @@ static bool show_statistics(const common_params & params) { constexpr int w_nam = 40; // Should be wide enough for most tensor names const auto * sep = " | "; - LOG_INF("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); + printf("\nComputing tensor statistics for %s (%d tensors)\n", params.in_files[0].c_str(), static_cast(ts.size())); if (legacy) { - LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%10s%10s\n", + printf("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%10s%10s\n", w_lay, "Layer", sep, w_nam, "Tensor", sep, "Mean", "StdDev", "Skew", "Kurt", "H Norm", sep, "∑ E[A²]", "Gain", sep, "PCC", "Cov"); - LOG_INF("%s\n", std::string(153, '-').c_str()); + printf("%s\n", std::string(153, '-').c_str()); } else { - LOG_INF("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%12s%10s%10s\n", + printf("\n%*s%s%-*s%s%10s%10s%12s%12s%9s%s%17s%8s%s%12s%10s%10s\n", w_lay, "Layer", sep, w_nam, "Tensor", sep, "Mean", "StdDev", "Skew", "Kurt", "H Norm", sep, "∑ E[A²]", "Gain", sep, "L2 Dist", "PCC", "Cov"); - LOG_INF("%s\n", std::string(165, '-').c_str()); + printf("%s\n", std::string(165, '-').c_str()); } // Tensor Statistics @@ -1431,7 +1431,7 @@ static bool show_statistics(const common_params & params) { } if (legacy) { - LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%10.4f%10.4f\n", + printf("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%10.4f%10.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, tstat.mean, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, @@ -1439,7 +1439,7 @@ static bool show_statistics(const common_params & params) { tstat.pearson, tstat.covariance ); } else { - LOG_INF("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%12.4f%10.4f%10.4f\n", + printf("%*s%s%-*s%s%10.4f%10.4f%12.4f%12.4f%8.2f%%%s%14.4f%8.2f%s%12.4f%10.4f%10.4f\n", w_lay, layer.c_str(), sep, w_nam, label_fmt(tstat.tensor, w_nam).c_str(), sep, tstat.mean, tstat.std_deviation, tstat.skewness, tstat.kurtosis, h_norm, sep, @@ -1483,20 +1483,20 @@ static bool show_statistics(const common_params & params) { } } - LOG_INF("\n\nComputing layer statistics for %s (%zu layers)\n\n", params.in_files[0].c_str(), layers); + printf("\nComputing layer statistics for %s (%zu layers)\n\n", params.in_files[0].c_str(), layers); if (legacy) { - LOG_INF("%*s%s%17s%8s%s%9s%9s%12s\n", + printf("%*s%s%17s%8s%s%9s%9s%12s\n", w_lay, "Layer", sep, "∑ E[A²]", "Gain", sep, "CosSim", "PCC", "Cov"); - LOG_INF("%s\n", std::string(64, '-').c_str()); + printf("%s\n", std::string(64, '-').c_str()); } else { - LOG_INF("%*s%s%17s%8s%s%12s%9s%9s%12s\n", + printf("%*s%s%17s%8s%s%12s%9s%9s%12s\n", w_lay, "Layer", sep, "∑ E[A²]", "Gain", sep, "L2 Dist", "CosSim", "PCC", "Cov"); - LOG_INF("%s\n", std::string(76, '-').c_str()); + printf("%s\n", std::string(76, '-').c_str()); } auto get_layer_stat = [&](const std::map& map, const int layer) -> float { @@ -1516,19 +1516,19 @@ static bool show_statistics(const common_params & params) { const auto *lyr = layer == INT_MAX ? "-" : str.c_str(); if (legacy) { - LOG_INF("%*s%s%14.4f%8.2f%s%9.4f%9.4f%12.4f\n", + printf("%*s%s%14.4f%8.2f%s%9.4f%9.4f%12.4f\n", w_lay, lyr, sep, stats.layer_sum, lgn, sep, lcs, lpc, lcv); } else { - LOG_INF("%*s%s%14.4f%8.2f%s%12.4f%9.4f%9.4f%12.4f\n", + printf("%*s%s%14.4f%8.2f%s%12.4f%9.4f%9.4f%12.4f\n", w_lay, lyr, sep, stats.layer_sum, lgn, sep, ll2, lcs, lpc, lcv); } } - LOG_INF("\n"); + printf("\n"); return true; } From 0d05ed49dbabcb9c2148f953bd768947721872bf Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 6 Jun 2026 19:26:33 +0100 Subject: [PATCH 122/123] Remove unnecessary includes --- tools/imatrix/imatrix.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index eb0374fe8c47..23e7a8b58698 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -1,27 +1,16 @@ #include "arg.h" #include "common.h" +#include "gguf.h" #include "imatrix-loader.h" -#include "log.h" #include "llama.h" -#include "gguf.h" +#include "log.h" -#include -#include #include -#include #include -#include -#include -#include -#include -#include -#include #include +#include +#include #include -#include -#include -#include -#include #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data From db451e2d3474afe17d6bcf8c187c4383cf66e974 Mon Sep 17 00:00:00 2001 From: Ed Addario Date: Sat, 6 Jun 2026 20:16:28 +0100 Subject: [PATCH 123/123] Add required include --- tools/imatrix/imatrix.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 23e7a8b58698..0cdd1208fdd7 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -5,6 +5,7 @@ #include "llama.h" #include "log.h" +#include #include #include #include