From 2f3247af65da4f9e89c23501fe9ea355a11ef9d1 Mon Sep 17 00:00:00 2001 From: Aaron Stopher <22336995+aastopher@users.noreply.github.com> Date: Tue, 4 Feb 2025 20:58:47 -0700 Subject: [PATCH 1/2] verify align hash output with imagehash --- Cargo.lock | 45 +++-- Cargo.toml | 1 + crates/imgdd/Cargo.toml | 4 +- crates/imgddcore/Cargo.toml | 3 +- crates/imgddcore/src/hashing.rs | 217 +++++++++++------------- crates/imgddcore/tests/hashing_tests.rs | 13 +- crates/imgddpy/Cargo.toml | 4 +- crates/imgddpy/docs/benches.md | 38 ++--- 8 files changed, 168 insertions(+), 157 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed57238..25db9d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,7 +292,7 @@ dependencies = [ "criterion-plot", "is-terminal", "itertools 0.10.5", - "num-traits", + "num-traits 0.2.19", "once_cell", "oorandom", "plotters", @@ -346,6 +346,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" +[[package]] +name = "dwt" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537fc1f78e72b8d14a7fd0d918996a3a33220149b89ef88864256af2f5b25c3f" +dependencies = [ + "num-traits 0.1.43", +] + [[package]] name = "either" version = "1.13.0" @@ -469,7 +478,7 @@ dependencies = [ "exr", "gif", "image-webp", - "num-traits", + "num-traits 0.2.19", "png", "qoi", "ravif", @@ -492,7 +501,7 @@ dependencies = [ [[package]] name = "imgdd" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "criterion", @@ -503,10 +512,11 @@ dependencies = [ [[package]] name = "imgddcore" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "codspeed-criterion-compat", + "dwt", "image", "log", "rayon", @@ -517,7 +527,7 @@ dependencies = [ [[package]] name = "imgddpy" -version = "0.1.3" +version = "0.1.4" dependencies = [ "image", "imgddcore", @@ -736,7 +746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", - "num-traits", + "num-traits 0.2.19", ] [[package]] @@ -745,7 +755,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ - "num-traits", + "num-traits 0.2.19", ] [[package]] @@ -765,7 +775,7 @@ version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "num-traits", + "num-traits 0.2.19", ] [[package]] @@ -776,7 +786,16 @@ checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ "num-bigint", "num-integer", - "num-traits", + "num-traits 0.2.19", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.19", ] [[package]] @@ -818,7 +837,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ - "num-traits", + "num-traits 0.2.19", "plotters-backend", "plotters-svg", "wasm-bindgen", @@ -1044,7 +1063,7 @@ dependencies = [ "new_debug_unreachable", "noop_proc_macro", "num-derive", - "num-traits", + "num-traits 0.2.19", "once_cell", "paste", "profiling", @@ -1144,7 +1163,7 @@ checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" dependencies = [ "num-complex", "num-integer", - "num-traits", + "num-traits 0.2.19", "primal-check", "strength_reduce", "transpose", @@ -1407,7 +1426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" dependencies = [ "aligned-vec", - "num-traits", + "num-traits 0.2.19", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index f869de9..7345090 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ anyhow = "1.0.95" log = "0.4.25" rustdct = "0.7.1" tempfile = "3.5" +dwt = "0.5.2" [profile.release] opt-level = 3 \ No newline at end of file diff --git a/crates/imgdd/Cargo.toml b/crates/imgdd/Cargo.toml index e7a040d..ea449f6 100644 --- a/crates/imgdd/Cargo.toml +++ b/crates/imgdd/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imgdd" -version = "0.1.1" +version = "0.1.2" edition.workspace = true license.workspace = true authors.workspace = true @@ -13,7 +13,7 @@ categories.workspace = true readme = "README.md" [dependencies] -imgddcore = { path = "../imgddcore", version = "0.1.1" } +imgddcore = { path = "../imgddcore", version = "0.1.2" } image.workspace = true anyhow.workspace = true criterion = { version = "0.5.1", optional = true } diff --git a/crates/imgddcore/Cargo.toml b/crates/imgddcore/Cargo.toml index 8e08008..9a8d0a2 100644 --- a/crates/imgddcore/Cargo.toml +++ b/crates/imgddcore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imgddcore" -version = "0.1.1" +version = "0.1.2" edition.workspace = true license.workspace = true authors.workspace = true @@ -19,6 +19,7 @@ walkdir.workspace = true anyhow.workspace = true log.workspace = true rustdct.workspace = true +dwt.workspace = true criterion = { package = "codspeed-criterion-compat", version = "2.7.1", optional = true } [dev-dependencies] diff --git a/crates/imgddcore/src/hashing.rs b/crates/imgddcore/src/hashing.rs index 8022445..d805ae1 100644 --- a/crates/imgddcore/src/hashing.rs +++ b/crates/imgddcore/src/hashing.rs @@ -1,7 +1,11 @@ use image::{DynamicImage, GenericImageView}; -use rustdct::DctPlanner; use anyhow::Result; +use rustdct::DctPlanner; +use dwt::{Transform, Operation}; +use dwt::wavelet::Haar; + + /// A structure representing the hash of an image as u64. /// /// The `ImageHash` structure is used to store and compare the hash of an image for deduplication purposes. @@ -25,20 +29,26 @@ impl ImageHash { /// - Based on average brightness, making it suitable for detecting overall image similarity. #[inline] pub fn ahash(image: &DynamicImage) -> Result { - // Collect pixel values from normalized 8x8 image - let pixels: Vec = image.pixels().map(|p| p.2[0] as u64).collect(); - - // Calculate average pixel value - let avg: u64 = pixels.iter().sum::() / pixels.len() as u64; + let mut sum = 0u64; + let mut pixels = [0u8; 64]; + + // Collect pixel values and compute sum + for (i, (_, _, pixel)) in image.pixels().enumerate().take(64) { + pixels[i] = pixel[0]; // Grayscale value + sum += pixels[i] as u64; + } + + // Collect average pixel value + let avg = sum / 64; - // Compute hash by comparing each pixel to the average + // Compute hash and store bits in the correct order let mut hash = 0u64; - for (i, &pixel) in pixels.iter().enumerate().take(64) { - if pixel > avg { - hash |= 1 << i; + for (i, &pixel) in pixels.iter().enumerate() { + if pixel as u64 > avg { + hash |= 1 << (63 - i); // reverse order } } - + Ok(Self { hash }) } @@ -57,19 +67,26 @@ impl ImageHash { /// - Suitable for images with varying brightness or exposure levels. #[inline] pub fn mhash(image: &DynamicImage) -> Result { - // Collect pixel values from normalized 8x8 image - let pixels: Vec = image.pixels().map(|p| p.2[0] as u64).collect(); - - // Calculate median for 64 pixels - let mut sorted_pixels = pixels.clone(); - sorted_pixels.sort_unstable(); - let median = (sorted_pixels[31] + sorted_pixels[32]) / 2; + let mut pixels = [0u8; 64]; + + // Collect 64 pixel values + for (i, pixel) in image.pixels().map(|p| p.2[0]).take(64).enumerate() { + pixels[i] = pixel; + } - // Compute hash by comparing each pixel to the median + // Copy pixels so we don't modify the original array + let mut pixels_copy = pixels; + + // Find median O(n) + let mid = 32; + let (low, median, _high) = pixels_copy.select_nth_unstable(mid); + let median = (*median as u64 + low[mid - 1] as u64) / 2; // Compute true median + + // Compute hash let mut hash = 0u64; - for (i, &pixel) in pixels.iter().enumerate().take(64) { - if pixel > median { - hash |= 1 << i; + for (i, &pixel) in pixels.iter().enumerate() { + if pixel as u64 > median { + hash |= 1 << (63 - i); // reverse order } } @@ -77,6 +94,7 @@ impl ImageHash { } + /// Computes the difference hash (dHash) of a given image. /// /// # Arguments @@ -92,16 +110,20 @@ impl ImageHash { #[inline] pub fn dhash(image: &DynamicImage) -> Result { let mut hash = 0u64; + for y in 0..8 { - for x in 0..8 { - let current = image.get_pixel(x, y)[0]; - let next = image.get_pixel(x + 1, y)[0]; - hash = (hash << 1) | ((current > next) as u64); + let mut current = image.get_pixel(0, y)[0]; + for x in 1..9 { + let next = image.get_pixel(x, y)[0]; + hash = (hash << 1) | (next > current) as u64; + current = next; } } + Ok(Self { hash }) } + /// Computes the perceptual hash (pHash) of a given image. /// /// # Arguments: @@ -118,39 +140,37 @@ impl ImageHash { pub fn phash(image: &DynamicImage) -> Result { const IMG_SIZE: usize = 32; const HASH_SIZE: usize = 8; - + // Collect pixel values from normalized 32x32 grayscale image let mut pixels: Vec = image .pixels() .map(|p| p.2[0] as f32) .collect(); - + // Plan DCT once for both rows and columns let mut planner = DctPlanner::new(); let dct = planner.plan_dct2(IMG_SIZE); - + // Apply DCT row-wise in-place for row in pixels.chunks_exact_mut(IMG_SIZE) { dct.process_dct2(row); } - - // Temp buffer for column processing - let mut col_buffer = vec![0f32; IMG_SIZE]; - + // Apply DCT column-wise in-place for col in 0..IMG_SIZE { - // Extract column into buffer + let mut col_values: [f32; IMG_SIZE] = [0.0; IMG_SIZE]; + for row in 0..IMG_SIZE { - col_buffer[row] = pixels[row * IMG_SIZE + col]; + col_values[row] = pixels[row * IMG_SIZE + col]; } - // Perform DCT on the column - dct.process_dct2(&mut col_buffer); - // Store result back into the original pixel array + + dct.process_dct2(&mut col_values); + for row in 0..IMG_SIZE { - pixels[row * IMG_SIZE + col] = col_buffer[row]; + pixels[row * IMG_SIZE + col] = col_values[row]; } } - + // Extract top-left 8x8 DCT coefficients (low frequencies) let mut dct_lowfreq = [0f32; HASH_SIZE * HASH_SIZE]; for y in 0..HASH_SIZE { @@ -158,26 +178,24 @@ impl ImageHash { dct_lowfreq[y * HASH_SIZE + x] = pixels[y * IMG_SIZE + x]; } } - - // Sort the DCT coefficients (in-place to avoid unnecessary allocations) - let mut sorted = dct_lowfreq; - sorted.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap()); - - // Calculate the median from the sorted values - let median_index = HASH_SIZE * HASH_SIZE / 2; - let median = (sorted[median_index - 1] + sorted[median_index]) / 2.0; - + + // Compute median excluding DC coefficient + let mut ac_coeffs = dct_lowfreq[1..].to_vec(); + let mid = ac_coeffs.len() / 2; + ac_coeffs.select_nth_unstable_by(mid, |a, b| a.partial_cmp(b).unwrap()); + let median = ac_coeffs[mid]; + // Generate hash let mut hash = 0u64; for (i, &val) in dct_lowfreq.iter().enumerate() { if val > median { - hash |= 1 << i; + hash |= 1 << (63 - i); } } - + Ok(Self { hash }) } - + /// Computes the wavelet hash (wHash) of a given image. /// @@ -193,78 +211,49 @@ impl ImageHash { /// - Robust against scaling, rotation, and noise. #[inline] pub fn whash(image: &DynamicImage) -> Result { - const HASH_SIZE: usize = 8; // Hash size (8x8) - let image_scale = HASH_SIZE * 2; // Scaled for decomposition - - // Convert image pixels to a normalized f64 array - let pixels: Vec = image - .pixels() - .map(|p| p.2[0] as f64 / 255.0) // Normalize pixel values to [0.0, 1.0] - .collect(); - - // Calculate maximum Haar decomposition level based on image scale - let ll_max_level = (image_scale as f64).log2().floor() as usize; - - // Perform Haar wavelet decomposition up to max_level - let mut coeffs = vec![pixels.clone()]; - let mut current = pixels.clone(); - for _ in 0..ll_max_level { - let size = (current.len() as f64).sqrt() as usize; // Assume a square image - let half_size = size / 2; - - let mut next = vec![0.0; current.len()]; - for y in 0..half_size { - for x in 0..half_size { - let top_left = current[y * size + x]; - let top_right = current[y * size + x + half_size]; - let bottom_left = current[(y + half_size) * size + x]; - let bottom_right = current[(y + half_size) * size + x + half_size]; - - let avg = (top_left + top_right + bottom_left + bottom_right) / 4.0; // LL - let hor_diff = (top_left + top_right - bottom_left - bottom_right) / 4.0; // HL - let ver_diff = (top_left - top_right + bottom_left - bottom_right) / 4.0; // LH - let diag_diff = (top_left - top_right - bottom_left + bottom_right) / 4.0; // HH - - let idx = y * size + x; - next[idx] = avg; // Approximation coefficients (LL) - next[idx + half_size] = hor_diff; // Horizontal details (HL) - next[(y + half_size) * size + x] = ver_diff; // Vertical details (LH) - next[(y + half_size) * size + x + half_size] = diag_diff; // Diagonal details (HH) - } + const HASH_SIZE: u32 = 8; + let ll_max_level: usize = 3; + + // Allocate flat vector of normalized pixels (row–major order). + let total_pixels = (HASH_SIZE * HASH_SIZE) as usize; + let mut pixels = Vec::with_capacity(total_pixels); + for y in 0..HASH_SIZE { + for x in 0..HASH_SIZE { + let pixel = image.get_pixel(x, y); + pixels.push(pixel[0] as f32 / 255.0); } - - coeffs.push(next.clone()); - current = next; } - - // Zero out LL component at max level (making it less sensitive to large-scale image changes) - coeffs.last_mut().unwrap().iter_mut().for_each(|v| *v = 0.0); - - // Use LL coefficients at appropriate level (based on HASH_SIZE) - let dwt_level = ll_max_level - (HASH_SIZE as f64).log2().floor() as usize; - let low_freq = coeffs[dwt_level] - .iter() - .cloned() - .take(HASH_SIZE * HASH_SIZE) - .collect::>(); - - // Calculate median of coefficients - let mut sorted_low_freq = low_freq.clone(); - sorted_low_freq.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap()); - let median = sorted_low_freq[HASH_SIZE * HASH_SIZE / 2]; - - // Generate hash + + // ---------- Remove low-level frequency (DC) component ---------- // + // Perform a full forward Haar transform - 8×8 image (3 levels). + pixels.transform(Operation::Forward, &Haar::new(), ll_max_level); + + // Zero out the DC coefficient. + pixels[0] = 0.0; + + // Perform inverse Haar transform (reconstruct image). + pixels.transform(Operation::Inverse, &Haar::new(), ll_max_level); + + // ---------- Compute median O(n) ---------- // + let mid: usize = 32; + // Clone flat pixel vector. + let mut flat = pixels.clone(); + // Quicksort vector. + flat.select_nth_unstable_by(mid, |a, b| a.partial_cmp(b).unwrap()); + // Compute median. + let median = (flat[mid - 1] + flat[mid]) / 2.0; + + // Generate hash. let mut hash = 0u64; - for (i, &val) in low_freq.iter().enumerate() { + for (i, &val) in pixels.iter().enumerate() { if val > median { hash |= 1 << i; } } - + Ok(Self { hash }) } - /// Retrieves the computed hash value. /// /// # Returns diff --git a/crates/imgddcore/tests/hashing_tests.rs b/crates/imgddcore/tests/hashing_tests.rs index f8b9ef6..46fec83 100644 --- a/crates/imgddcore/tests/hashing_tests.rs +++ b/crates/imgddcore/tests/hashing_tests.rs @@ -13,15 +13,16 @@ mod tests { } else { Rgba([0, 0, 0, 255]) // Black pixel } - })) + })).grayscale() } + #[test] fn test_ahash() -> Result<()> { let test_image = create_mock_image((8, 8)); let hash = ImageHash::ahash(&test_image)?; println!("aHash: {:064b}", hash.get_hash()); - let expected_hash = 0b0101010101010101010101010101010101010101010101010101010101010101; + let expected_hash = 0b1010101010101010101010101010101010101010101010101010101010101010; assert_eq!(hash.get_hash(), expected_hash, "aHash does not match expected value"); Ok(()) @@ -32,7 +33,7 @@ mod tests { let test_image = create_mock_image((8, 8)); let hash = ImageHash::mhash(&test_image)?; println!("mHash: {:064b}", hash.get_hash()); - let expected_hash = 0b0101010101010101010101010101010101010101010101010101010101010101; + let expected_hash = 0b1010101010101010101010101010101010101010101010101010101010101010; assert_eq!(hash.get_hash(), expected_hash, "mHash does not match expected value"); @@ -44,7 +45,7 @@ mod tests { let test_image = create_mock_image((9, 8)); let hash = ImageHash::dhash(&test_image)?; println!("dHash: {:064b}", hash.get_hash()); - let expected_hash = 0b1010101010101010101010101010101010101010101010101010101010101010; + let expected_hash = 0b0101010101010101010101010101010101010101010101010101010101010101; assert_eq!(hash.get_hash(), expected_hash, "dHash does not match expected value"); Ok(()) @@ -54,7 +55,7 @@ mod tests { fn test_phash() -> Result<()> { let test_image = create_mock_image((32, 32)); let hash = ImageHash::phash(&test_image)?; - let expected_hash = 0b0000000000000000000000000000000000000000000000000000000010101011; + let expected_hash = 0b1101010100000000000000000000000000000000000000000000000000000000; println!("pHash: {:064b}", hash.get_hash()); assert_eq!(hash.get_hash(), expected_hash, "pHash does not match expected value"); @@ -66,7 +67,7 @@ mod tests { let test_image = create_mock_image((8, 8)); let hash = ImageHash::whash(&test_image)?; println!("wHash: {:064b}", hash.get_hash()); - let expected_hash = 0b0000000000000000000000000000000000000101000001010000010100000101; + let expected_hash = 0b1010101010101010101010101010101010101010101010101010101010101010; assert_eq!(hash.get_hash(), expected_hash, "wHash does not match expected value"); diff --git a/crates/imgddpy/Cargo.toml b/crates/imgddpy/Cargo.toml index f86625d..89f8574 100644 --- a/crates/imgddpy/Cargo.toml +++ b/crates/imgddpy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imgddpy" -version = "0.1.3" +version = "0.1.4" edition.workspace = true license.workspace = true authors.workspace = true @@ -12,7 +12,7 @@ homepage.workspace = true readme = "README.md" [dependencies] -imgddcore = { path = "../imgddcore", version = "0.1.1" } +imgddcore = { path = "../imgddcore", version = "0.1.2" } pyo3 = { version = "0.23", features = ["extension-module", "abi3-py39"] } image.workspace = true diff --git a/crates/imgddpy/docs/benches.md b/crates/imgddpy/docs/benches.md index 06bded0..7f22404 100644 --- a/crates/imgddpy/docs/benches.md +++ b/crates/imgddpy/docs/benches.md @@ -14,13 +14,13 @@ This section highlights the performance benchmarks for the hashing algorithms pr Below is a snapshot of local bare metal benchmarks taken on using Criterion directly on the imgddcore Rust crate, based on the hardware details above. -|Algorithm|Time (ms)|Measurements| -|---|---|---| -|aHash|0.815|100| -|mHash|1.369|100| -|dHash|0.541|100| -|pHash|23.709|100| -|wHash|3.345|100| +| Algorithm | Time (ms) | Measurements | +| --------- | ---------- | ------------ | +| aHash | 0.00021894 | 100 | +| mHash | 0.00045627 | 100 | +| dHash | 0.00020319 | 100 | +| pHash | 0.020221 | 100 | +| wHash | 0.0021888 | 100 | --- @@ -30,21 +30,21 @@ The table below compares the local performance of [imgdd](https://github.com/aas ### dHash -|Metric|imgdd (ms)|imagehash (ms)|Improvement (%)| -|---|---|---|---| -|Min Time|1.788|5.098|64.92| -|Max Time|2.792|7.684|63.67| -|Avg Time|1.942|5.645|65.59| -|Median Time|1.888|5.554|66.02| +| Metric | imgdd (ms) | imagehash (ms) | Improvement (%) | +| ----------- | ---------- | -------------- | --------------- | +| Min Time | 1.2488 | 4.3166 | 71.07 | +| Max Time | 3.5945 | 9.5155 | 62.22 | +| Avg Time | 1.6148 | 5.5629 | 70.97 | +| Median Time | 1.3985 | 5.4049 | 74.12 | ### aHash -|Metric|imgdd (ms)|imagehash (ms)|Improvement (%)| -|---|---|---|---| -|Min Time|1.683|5.666|70.29| -|Max Time|3.207|15.403|79.18| -|Avg Time|2.055|8.346|75.38| -|Median Time|2.043|7.683|73.41| +| Metric | imgdd (ms) | imagehash (ms) | Improvement (%) | +| ----------- | ---------- | -------------- | --------------- | +| Min Time | 1.683 | 5.666 | 70.29 | +| Max Time | 3.207 | 15.403 | 79.18 | +| Avg Time | 2.055 | 8.346 | 75.38 | +| Median Time | 2.043 | 7.683 | 73.41 | ### pHash From 9306aea8e7f57b6a79b0a60d34092e3a8400ad51 Mon Sep 17 00:00:00 2001 From: Aaron Stopher <22336995+aastopher@users.noreply.github.com> Date: Tue, 4 Feb 2025 21:29:22 -0700 Subject: [PATCH 2/2] reverse whash output --- crates/imgddcore/src/hashing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/imgddcore/src/hashing.rs b/crates/imgddcore/src/hashing.rs index d805ae1..0309494 100644 --- a/crates/imgddcore/src/hashing.rs +++ b/crates/imgddcore/src/hashing.rs @@ -247,7 +247,7 @@ impl ImageHash { let mut hash = 0u64; for (i, &val) in pixels.iter().enumerate() { if val > median { - hash |= 1 << i; + hash |= 1 << (63 - i); } }