From e4bf109472c9822d49c9b218cacf96a56263eaad Mon Sep 17 00:00:00 2001 From: Liam Date: Wed, 8 Jan 2025 21:35:50 -0500 Subject: [PATCH 1/6] Add feature extraction pipeline to mediaproc --- docker/mediaproc/Dockerfile | 6 +- lib/philomena/native.ex | 4 + lib/philomena_media/remote.ex | 14 +- native/philomena/Cargo.lock | 581 +++++++++++++++++- native/philomena/mediaproc/src/client.rs | 20 +- native/philomena/mediaproc/src/lib.rs | 18 + native/philomena/mediaproc_client/src/main.rs | 42 +- native/philomena/mediaproc_server/Cargo.toml | 2 + .../philomena/mediaproc_server/src/dinov2.rs | 106 ++++ native/philomena/mediaproc_server/src/io.rs | 100 +++ native/philomena/mediaproc_server/src/main.rs | 37 +- native/philomena/src/lib.rs | 8 +- native/philomena/src/remote.rs | 49 +- 13 files changed, 955 insertions(+), 32 deletions(-) create mode 100644 native/philomena/mediaproc_server/src/dinov2.rs create mode 100644 native/philomena/mediaproc_server/src/io.rs diff --git a/docker/mediaproc/Dockerfile b/docker/mediaproc/Dockerfile index 8aa2c9e2d..548a4ba7b 100644 --- a/docker/mediaproc/Dockerfile +++ b/docker/mediaproc/Dockerfile @@ -62,14 +62,16 @@ RUN cd /tmp \ COPY native/philomena /tmp/philomena COPY docker/mediaproc/safe-rsvg-convert /usr/bin/safe-rsvg-convert +ADD https://github.com/liamwhite/philomena-ris-inference-toolkit/releases/download/v1.0/dinov2-with-registers-base.pt /usr/share/dinov2-with-registers-base.pt RUN cd /tmp/philomena \ && cargo build --release -p mediaproc_server \ - && cp target/release/mediaproc_server /usr/bin/mediaproc_server + && cp target/release/mediaproc_server /usr/bin/mediaproc_server \ + && find target/release/build -regextype posix-extended -regex '^.*\.so(\.[0-9]+)*$' -exec cp '{}' /usr/lib/ ';' # Set up unprivileged user account RUN useradd -ms /bin/bash mediaproc USER mediaproc WORKDIR /home/mediaproc ENV RUST_LOG=trace -CMD ["/usr/bin/mediaproc_server", "0.0.0.0:1500"] +CMD ["/usr/bin/mediaproc_server", "0.0.0.0:1500", "/usr/share/dinov2-with-registers-base.pt"] diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index b4f2f2444..2d798fcd9 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -12,6 +12,10 @@ defmodule Philomena.Native do @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) + @spec async_get_features(String.t(), String.t()) :: :ok + def async_get_features(_server_addr, _path), + do: :erlang.nif_error(:nif_not_loaded) + @spec async_process_command(String.t(), String.t(), [String.t()]) :: :ok def async_process_command(_server_addr, _program, _arguments), do: :erlang.nif_error(:nif_not_loaded) diff --git a/lib/philomena_media/remote.ex b/lib/philomena_media/remote.ex index 518a8613b..8ff27fc32 100644 --- a/lib/philomena_media/remote.ex +++ b/lib/philomena_media/remote.ex @@ -7,11 +7,23 @@ defmodule PhilomenaMedia.Remote do :ok = Philomena.Native.async_process_command(mediaproc_addr(), command, args) receive do - {:command_reply, command_reply} -> + {:process_command_reply, command_reply} -> {command_reply.stdout, command_reply.status} end end + @doc """ + Gets a feature vector for the given image path to use in reverse image search. + """ + def get_features(path) do + :ok = Philomena.Native.async_get_features(mediaproc_addr(), path) + + receive do + {:get_features_reply, get_features_reply} -> + get_features_reply + end + end + defp mediaproc_addr do Application.get_env(:philomena, :mediaproc_addr) end diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index 9959ce29b..ffac1a2d8 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -17,6 +17,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -129,6 +140,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" + [[package]] name = "bincode" version = "1.3.3" @@ -144,18 +161,65 @@ version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "caseless" version = "0.2.2" @@ -172,6 +236,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -181,6 +247,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.48" @@ -240,6 +316,21 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -249,6 +340,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" +dependencies = [ + "powerfmt", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -266,6 +388,17 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -360,6 +493,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "find-msvc-tools" version = "0.1.2" @@ -481,6 +623,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -510,6 +662,16 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.16.0" @@ -522,6 +684,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.3.1" @@ -646,6 +817,21 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529feb3e6769d234375c4cf1ee2ce713682b8e76538cb13f9fc23e1400a591e7" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + [[package]] name = "indexmap" version = "2.11.4" @@ -656,6 +842,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "inventory" version = "0.3.21" @@ -732,6 +927,16 @@ dependencies = [ "syn", ] +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.81" @@ -801,6 +1006,16 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "mediaproc" version = "0.1.0" @@ -827,8 +1042,10 @@ dependencies = [ "clap", "env_logger", "futures", + "image", "mediaproc", "tarpc", + "tch", "tempfile", "tokio", "tracing", @@ -847,6 +1064,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -860,6 +1078,62 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "moxcms" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd32fa8935aeadb8a8a6b6b351e40225570a37c43de67690383d87ef170cd08" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "ndarray" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "rawpointer", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "object" version = "0.36.7" @@ -941,6 +1215,29 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -962,7 +1259,7 @@ dependencies = [ "rustler", "tokio", "url", - "zip", + "zip 5.1.1", ] [[package]] @@ -997,6 +1294,25 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "png" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -1021,6 +1337,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1039,6 +1361,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83f9b339b02259ada5c0f4a389b7fb472f933aa17ce176fd2ad98f28bb401fde" +dependencies = [ + "num-traits", +] + [[package]] name = "quote" version = "1.0.40" @@ -1084,6 +1415,12 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "redox_syscall" version = "0.5.17" @@ -1186,6 +1523,41 @@ dependencies = [ "syn", ] +[[package]] +name = "rustls" +version = "0.23.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1198,6 +1570,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safetensors" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93279b86b3de76f820a8854dd06cbc33cfa57a417b19c47f6a25280112fb1df" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1247,6 +1629,28 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1327,6 +1731,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.106" @@ -1385,6 +1795,23 @@ dependencies = [ "syn", ] +[[package]] +name = "tch" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb3500c87ef72447c23b33ed6f15fac45a616b09bcac53e62e0e4386bddb3b9d" +dependencies = [ + "half", + "lazy_static", + "libc", + "ndarray", + "rand", + "safetensors", + "thiserror", + "torch-sys", + "zip 0.6.6", +] + [[package]] name = "tempfile" version = "3.23.0" @@ -1427,6 +1854,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + [[package]] name = "tinystr" version = "0.8.1" @@ -1513,6 +1959,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "torch-sys" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b87ed41261d4278060c3ba3e735c224687cf312403e4565f2ca75310279d73" +dependencies = [ + "anyhow", + "cc", + "libc", + "serde", + "serde_json", + "ureq", + "zip 0.6.6", +] + [[package]] name = "tracing" version = "0.1.41" @@ -1579,6 +2040,12 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "unicode-ident" version = "1.0.19" @@ -1606,6 +2073,24 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.7" @@ -1636,6 +2121,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1729,6 +2220,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.2", +] + +[[package]] +name = "webpki-roots" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-link" version = "0.1.3" @@ -1983,6 +2492,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + [[package]] name = "zerotrie" version = "0.2.2" @@ -2016,6 +2531,26 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time", + "zstd", +] + [[package]] name = "zip" version = "5.1.1" @@ -2047,3 +2582,47 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-jpeg" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" +dependencies = [ + "zune-core", +] diff --git a/native/philomena/mediaproc/src/client.rs b/native/philomena/mediaproc/src/client.rs index f76ef6d5e..90d86073e 100644 --- a/native/philomena/mediaproc/src/client.rs +++ b/native/philomena/mediaproc/src/client.rs @@ -98,25 +98,29 @@ fn update_replacements( Ok(()) } -fn context_with_1_hour_deadline() -> Context { +pub fn context_with_deadline(secs_from_now: u64) -> Context { let mut context = Context::current(); - context.deadline = Instant::now() + Duration::from_secs(60 * 60); + context.deadline = Instant::now() + Duration::from_secs(secs_from_now); context } +pub fn context_with_1_hour_deadline() -> Context { + context_with_deadline(60 * 60) +} + +pub fn context_with_10_second_deadline() -> Context { + context_with_deadline(10) +} + pub async fn execute_command( client: &MediaProcessorClient, program: String, arguments: Vec, + ctx: Context, ) -> Result { let call_params = create_replacements(arguments.into_iter()); let (reply, file_map) = client - .execute_command( - context_with_1_hour_deadline(), - program, - call_params.arguments, - call_params.file_map, - ) + .execute_command(ctx, program, call_params.arguments, call_params.file_map) .await .map_err(|_| ExecuteCommandError::UnknownError)??; diff --git a/native/philomena/mediaproc/src/lib.rs b/native/philomena/mediaproc/src/lib.rs index a53520ba4..795d6d471 100644 --- a/native/philomena/mediaproc/src/lib.rs +++ b/native/philomena/mediaproc/src/lib.rs @@ -12,11 +12,16 @@ pub trait MediaProcessor { arguments: Vec, file_map: FileMap, ) -> Result<(CommandReply, FileMap), ExecuteCommandError>; + + /// Runs feature extraction on an image file bytes (PNG or JPEG). + async fn get_features(image: Vec) -> Result, FeatureExtractionError>; } /// Errors which can occur during command execution. #[derive(Debug, Deserialize, Serialize)] pub enum ExecuteCommandError { + /// Failed to connect to server. + ConnectionError, /// Requested program was not allowed to be executed. UnpermittedProgram(String), /// Failed to launch program. @@ -31,6 +36,19 @@ pub enum ExecuteCommandError { UnknownError, } +/// Errors which can occur during image feature extraction. +#[derive(Debug, Deserialize, Serialize)] +pub enum FeatureExtractionError { + /// Failed to connect to server. + ConnectionError, + /// Generic filesystem error. + LocalFilesystemError, + /// Unrecognized image format. + UnknownImageFormat, + /// Failed to decode the image. + ImageDecodeError, +} + /// Enumeration of permitted program names. pub static PERMITTED_PROGRAMS: Lazy> = Lazy::new(|| { vec![ diff --git a/native/philomena/mediaproc_client/src/main.rs b/native/philomena/mediaproc_client/src/main.rs index de9974e08..b57af29ab 100644 --- a/native/philomena/mediaproc_client/src/main.rs +++ b/native/philomena/mediaproc_client/src/main.rs @@ -3,6 +3,7 @@ use std::process::ExitCode; use clap::{Parser, Subcommand}; use mediaproc::MediaProcessorClient; +use mediaproc::client; use mediaproc::client::{connect_to_socket_server, execute_command}; #[derive(Parser, Debug)] @@ -28,12 +29,17 @@ enum InvocationType { /// Arguments to pass to program. args: Vec, }, + /// Get DINOv2 features from the given image file (PNG or JPEG). + ExtractFeatures { + /// Filename to extract from. + file_name: String, + }, } #[tokio::main(flavor = "current_thread")] async fn main() -> ExitCode { let args = Arguments::parse(); - let client = connect_to_socket_server(&args.server_addr) + let client = client::connect_to_socket_server(&args.server_addr) .await .expect("failed to connect to server"); @@ -41,6 +47,9 @@ async fn main() -> ExitCode { InvocationType::ExecuteCommand { program, args } => { run_command_client(&client, program, args).await } + InvocationType::ExtractFeatures { file_name } => { + run_feature_extraction_client(&client, file_name).await + } } } @@ -49,7 +58,10 @@ async fn run_command_client( program: String, args: Vec, ) -> ExitCode { - let reply = execute_command(client, program, args).await.unwrap(); + let ctx = client::context_with_1_hour_deadline(); + let reply = client::execute_command(client, program, args, ctx) + .await + .unwrap(); write_then_drop(std::io::stderr(), reply.stderr); write_then_drop(std::io::stdout(), reply.stdout); @@ -60,3 +72,29 @@ async fn run_command_client( fn write_then_drop(mut stream: impl Write, data: Vec) { stream.write_all(&data).unwrap() } + +async fn run_feature_extraction_client( + client: &MediaProcessorClient, + file_name: String, +) -> ExitCode { + let image = std::fs::read(file_name).unwrap(); + let features = client + .get_features(client::context_with_10_second_deadline(), image) + .await + .unwrap() + .unwrap(); + + // Manual intersperse implementation, until rust adds it properly + let mut started = false; + for component in features { + if started { + print!(" {}", component); + } else { + print!("{}", component); + started = true; + } + } + println!(); + + ExitCode::SUCCESS +} diff --git a/native/philomena/mediaproc_server/Cargo.toml b/native/philomena/mediaproc_server/Cargo.toml index de05d4ea0..e29fe9d34 100644 --- a/native/philomena/mediaproc_server/Cargo.toml +++ b/native/philomena/mediaproc_server/Cargo.toml @@ -7,8 +7,10 @@ edition = "2024" env_logger = "0.11" clap = { version = "4.5", features = ["derive"] } futures = "0.3" +image = { version = "0.25.2", default-features = false, features = ["jpeg", "png"] } mediaproc = { path = "../mediaproc" } tarpc = { version = "0.35", features = ["full"] } +tch = { version = "0.18.1", features = ["download-libtorch"] } tempfile = "3" tokio = { version = "1.0", features = ["full"] } tracing = "0.1" diff --git a/native/philomena/mediaproc_server/src/dinov2.rs b/native/philomena/mediaproc_server/src/dinov2.rs new file mode 100644 index 000000000..d6c65b9ad --- /dev/null +++ b/native/philomena/mediaproc_server/src/dinov2.rs @@ -0,0 +1,106 @@ +use std::io::Cursor; +use tch::{CModule, Device, IValue, Tensor}; + +use super::io; +use crate::FeatureExtractionError; + +/// Each DINOv2 patch is 14x14 +pub const PATCH_DIM: i64 = 14; + +pub struct ModelResult { + pub patches: (i64, i64), + pub image: Tensor, + pub features: Tensor, + pub last_hidden_state: Tensor, +} + +fn infer(image: &Tensor, model: &CModule) -> (Tensor, Tensor) { + // These cases intentionally panic because their outputs depend on the model, + // not on the image file input, and invalid model format is not recoverable. + let output = model + .forward_is(&[IValue::Tensor(image.shallow_clone())]) + .unwrap(); + + let mut results = match output { + IValue::Tuple(elements) if elements.len() == 2 => elements, + _ => unreachable!("expected (last_hidden_state, pooler_output)"), + }; + + let mut results = results.drain(..); + + match (results.next(), results.next()) { + (Some(IValue::Tensor(last_hidden_state)), Some(IValue::Tensor(pooler_output))) => { + (last_hidden_state, pooler_output) + } + _ => unreachable!("expected 2-tuple of tensors"), + } +} + +fn scaled_result(pooler_output: &Tensor) -> Tensor { + let scaled_norm = pooler_output.norm().pow_tensor_scalar(-1); + pooler_output.multiply(&scaled_norm) +} + +pub fn get_model_result( + image: R, + model: &CModule, + device: Device, +) -> Result +where + R: std::io::Read + std::io::Seek, +{ + // Get image and and dimensions for calculation. + let image = io::load_image(image, device)?; + + // Features are unstable across different global scales, and + // somewhat stable across dimensional scales. + // + // Use 18 (252x252) instead of 16 (224x224) to produce a more detailed + // result and attention map at almost exactly the same computational cost. + // + // It is possible for highly non-square models to have meaningful feature extraction, + // but in practice it makes no difference identifying scales which keep the aspect + // ratio, and does not produce feature vectors which are similar enough to identify + // crops. + let image_scale = 1; + let patches = (18 * image_scale, 18 * image_scale); + + // Scale image into appropriate shape. + let image = io::resize_image_by_patch_count(image, patches, PATCH_DIM); + + // The pooler output is the [CLS] token generated by the model. + // It contains high-quality, robust features from the input image. + let (last_hidden_state, pooler_output) = infer(&image, model); + + Ok(ModelResult { + patches, + image, + features: scaled_result(&pooler_output.squeeze()), + last_hidden_state, + }) +} + +pub struct Executor { + device: Device, + model: CModule, +} + +impl Executor { + pub fn new(model_path: &str) -> Option { + let (device, model) = io::device_and_model(model_path)?; + Some(Self { device, model }) + } + + pub fn extract(&self, image: &[u8]) -> Result, FeatureExtractionError> { + let image = Cursor::new(image); + let model_result = get_model_result(image, &self.model, self.device)?; + let features = model_result + .features + .iter::() + .unwrap() + .map(|f| f as f32) + .collect(); + + Ok(features) + } +} diff --git a/native/philomena/mediaproc_server/src/io.rs b/native/philomena/mediaproc_server/src/io.rs new file mode 100644 index 000000000..43f8618b5 --- /dev/null +++ b/native/philomena/mediaproc_server/src/io.rs @@ -0,0 +1,100 @@ +use image::{DynamicImage, ImageBuffer, ImageReader, Pixel}; +use std::io::BufReader; +use tch::{CModule, Device, Tensor}; + +use crate::FeatureExtractionError; + +pub fn device_and_model(model_path: &str) -> Option<(Device, CModule)> { + let device = Device::cuda_if_available(); + let model = CModule::load_on_device(model_path, device).ok()?; + + Some((device, model)) +} + +fn into_tensor>( + image: ImageBuffer>, + device: Device, +) -> Tensor { + let w: i64 = image.width().into(); + let h: i64 = image.height().into(); + let c: i64 = P::CHANNEL_COUNT.into(); + + // Extra scope to ensure we eagerly drop the original image buffer + let pixels = { + let pixels: Vec = image.pixels().flat_map(|p| p.channels()).copied().collect(); + + Tensor::from_slice(&pixels) + }; + + pixels.to(device).reshape([h, w, c]).permute([2, 0, 1]) +} + +fn strip_transparency(image: DynamicImage, device: Device) -> Tensor { + let w: i64 = image.width().into(); + let h: i64 = image.height().into(); + + match image { + DynamicImage::ImageRgb8(..) + | DynamicImage::ImageLuma8(..) + | DynamicImage::ImageLuma16(..) + | DynamicImage::ImageRgb16(..) + | DynamicImage::ImageRgb32F(..) => { + return into_tensor(image.into_rgb32f(), device); + } + _ => {} + }; + + // Get channels. + let (alpha, color) = { + let pixels = into_tensor(image.into_rgba32f(), device); + let alpha = pixels.slice(0, 3, 4, 1).broadcast_to([3, h, w]); + let color = pixels.slice(0, 0, 3, 1); + + (alpha, color) + }; + + // Detect whether premultiplication should be applied by checking + // for channels with values above the alpha level. + // + // Note that the only input format which we can get where this would + // be relevant, PNG, explicitly says it does not carry premultiplied alpha, + // but many tools will store premultiplied alpha anyway... + let ones = Tensor::ones([3, h, w], (tch::Kind::Float, device)); + let mask = alpha.where_self(&color.gt_tensor(&alpha).any(), &ones); + let color = color.multiply(&mask); + + // Pure transparency is rescaled to be 8 steps blacker than black. + const ALPHA_LEVEL: f64 = 8.0 / 255.0; + const COLOR_LEVEL: f64 = 1.0 - ALPHA_LEVEL; + + // Unwrap is guaranteed safe because the dimensions and data type match + color + .multiply_scalar(COLOR_LEVEL) + .f_add(&alpha.multiply_scalar(ALPHA_LEVEL)) + .unwrap() +} + +pub fn load_image(image: R, device: Device) -> Result +where + R: std::io::Read + std::io::Seek, +{ + let image = BufReader::new(image); + let image = ImageReader::new(image) + .with_guessed_format() + .map_err(|_| FeatureExtractionError::UnknownImageFormat)? + .decode() + .map_err(|_| FeatureExtractionError::ImageDecodeError)?; + + Ok(strip_transparency(image, device)) +} + +fn resize_tensor(image: Tensor, size: (i64, i64)) -> Tensor { + image.upsample_bicubic2d([size.0, size.1], true, None, None) +} + +pub fn resize_image_by_patch_count(image: Tensor, patches: (i64, i64), patch_dim: i64) -> Tensor { + let height = patches.0 * patch_dim; + let width = patches.1 * patch_dim; + + resize_tensor(image.unsqueeze(0), (height, width)) +} diff --git a/native/philomena/mediaproc_server/src/main.rs b/native/philomena/mediaproc_server/src/main.rs index 0e4541570..5feb9428b 100644 --- a/native/philomena/mediaproc_server/src/main.rs +++ b/native/philomena/mediaproc_server/src/main.rs @@ -1,12 +1,18 @@ use std::net::SocketAddr; +use std::sync::Arc; use clap::Parser; +use dinov2::Executor; use futures::{Future, StreamExt, future}; -use mediaproc::{CommandReply, ExecuteCommandError, FileMap, MediaProcessor}; +use mediaproc::{ + CommandReply, ExecuteCommandError, FeatureExtractionError, FileMap, MediaProcessor, +}; use tarpc::context; use tarpc::server::Channel; mod command_server; +mod dinov2; +mod io; mod signal; #[derive(Parser, Debug)] @@ -14,10 +20,13 @@ mod signal; struct Arguments { /// Socket address to bind to, like 127.0.0.1:1500 server_addr: SocketAddr, + + /// DINOv2 with registers base model to load. + model_path: String, } #[derive(Clone)] -struct MediaProcessorServer; +struct MediaProcessorServer(Arc); impl MediaProcessor for MediaProcessorServer { async fn execute_command( @@ -29,14 +38,24 @@ impl MediaProcessor for MediaProcessorServer { ) -> Result<(CommandReply, FileMap), ExecuteCommandError> { command_server::execute_command(program, arguments, file_map).await } + + async fn get_features( + self, + _: context::Context, + image: Vec, + ) -> Result, FeatureExtractionError> { + self.0.extract(&image) + } } fn main() { env_logger::init(); let args = Arguments::parse(); + let executor = Executor::new(&args.model_path).expect("failed to load Torch JIT model"); + let executor = Arc::new(executor); - serve(&args); + serve(&args, executor); } async fn spawn(fut: impl Future + Send + 'static) { @@ -44,7 +63,7 @@ async fn spawn(fut: impl Future + Send + 'static) { } #[tokio::main] -async fn serve(args: &Arguments) { +async fn serve(args: &Arguments, executor: Arc) { signal::install_handlers(); let codec = tarpc::tokio_serde::formats::Bincode::default; @@ -57,12 +76,10 @@ async fn serve(args: &Arguments) { // Ignore accept errors. .filter_map(|r| future::ready(r.ok())) .map(tarpc::server::BaseChannel::with_defaults) - .map(|channel| { - tokio::spawn( - channel - .execute(MediaProcessorServer.serve()) - .for_each(spawn), - ); + .map(move |channel| { + let server = MediaProcessorServer(executor.clone()); + + tokio::spawn(channel.execute(server.serve()).for_each(spawn)); }) .collect() .await diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index f2f5893ea..0de54fe5b 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -39,6 +39,12 @@ fn camo_image_url(input: &str) -> String { // Remote NIF wrappers. +#[rustler::nif] +fn async_get_features(env: Env, server_addr: String, path: String) -> Atom { + let fut = remote::get_features(server_addr, path); + asyncnif::call_async(env, fut, remote::get_features_reply_with_env) +} + #[rustler::nif] fn async_process_command( env: Env, @@ -47,7 +53,7 @@ fn async_process_command( arguments: Vec, ) -> Atom { let fut = remote::process_command(server_addr, program, arguments); - asyncnif::call_async(env, fut, remote::with_env) + asyncnif::call_async(env, fut, remote::command_reply_with_env) } // Zip NIF wrappers. diff --git a/native/philomena/src/remote.rs b/native/philomena/src/remote.rs index dd30c75f6..67c27b987 100644 --- a/native/philomena/src/remote.rs +++ b/native/philomena/src/remote.rs @@ -1,10 +1,14 @@ use mediaproc::CommandReply; +use mediaproc::client; use mediaproc::client::{connect_to_socket_server, execute_command}; use rustler::{Encoder, Env, NifStruct, OwnedBinary, Term, atoms}; atoms! { nil, - command_reply, + ok, + error, + get_features_reply, + process_command_reply, } #[derive(NifStruct)] @@ -30,7 +34,7 @@ pub async fn process_command( program: String, arguments: Vec, ) -> CommandReply { - let client = match connect_to_socket_server(&server_addr).await { + let client = match client::connect_to_socket_server(&server_addr).await { Some(client) => client, None => { return CommandReply { @@ -41,7 +45,8 @@ pub async fn process_command( } }; - match execute_command(&client, program, arguments).await { + let ctx = client::context_with_1_hour_deadline(); + match client::execute_command(&client, program, arguments, ctx).await { Ok(reply) => reply, Err(err) => CommandReply { stdout: vec![], @@ -51,11 +56,29 @@ pub async fn process_command( } } -/// Converts the response into a {:command_reply, %CommandReply{...}} message -/// which gets sent back to the caller. -pub fn with_env<'a>(env: Env<'a>, r: CommandReply) -> Term<'a> { +pub async fn get_features( + server_addr: String, + path: String, +) -> Result, FeatureExtractionError> { + let client = match client::connect_to_socket_server(&server_addr).await { + Some(client) => client, + None => return Err(FeatureExtractionError::ConnectionError), + }; + + let image = std::fs::read(path).map_err(|_| FeatureExtractionError::LocalFilesystemError)?; + let ctx = client::context_with_10_second_deadline(); + + client + .get_features(ctx, image) + .await + .map_err(|_| FeatureExtractionError::ConnectionError)? +} + +/// Converts the response into a {:process_command_reply, %CommandReply{...}} +/// message which gets sent back to the caller. +pub fn command_reply_with_env<'a>(env: Env<'a>, r: CommandReply) -> Term<'a> { ( - command_reply(), + process_command_reply(), CommandReply_ { stdout: binary_or_nil(env, r.stdout), stderr: binary_or_nil(env, r.stderr), @@ -64,3 +87,15 @@ pub fn with_env<'a>(env: Env<'a>, r: CommandReply) -> Term<'a> { ) .encode(env) } + +/// Converts the response into a {:get_features_reply, {:ok, [0.1, ..., 0.1]}} +/// message which gets sent back to the caller. +pub fn get_features_reply_with_env<'a>( + env: Env<'a>, + r: Result, FeatureExtractionError>, +) -> Term<'a> { + match r { + Ok(features) => (get_features_reply(), (ok(), features)).encode(env), + Err(e) => (get_features_reply(), (error(), format!("{e:?}"))).encode(env), + } +} From 63090b50a70e3ff0c477a189bc63479a299e4fab Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 9 Jan 2025 14:44:43 -0500 Subject: [PATCH 2/6] Add feature extraction and importing pipeline to Philomena --- lib/philomena/image_vectors.ex | 91 +++++++++++++++++++ .../image_vectors/batch_processor.ex | 88 ++++++++++++++++++ lib/philomena/image_vectors/image_vector.ex | 19 ++++ lib/philomena/image_vectors/importer.ex | 86 ++++++++++++++++++ lib/philomena/images.ex | 1 + lib/philomena/images/image.ex | 2 + lib/philomena/images/search_index.ex | 22 +++++ lib/philomena/images/thumbnailer.ex | 6 +- lib/philomena_media/features.ex | 51 +++++++++++ lib/philomena_media/processors.ex | 24 ++++- lib/philomena_media/processors/gif.ex | 9 ++ lib/philomena_media/processors/jpeg.ex | 9 ++ lib/philomena_media/processors/png.ex | 9 ++ lib/philomena_media/processors/processor.ex | 6 ++ lib/philomena_media/processors/svg.ex | 9 ++ lib/philomena_media/processors/webm.ex | 9 ++ .../20250109155442_create_image_vectors.exs | 14 +++ priv/repo/structure.sql | 64 ++++++++++++- 18 files changed, 515 insertions(+), 4 deletions(-) create mode 100644 lib/philomena/image_vectors.ex create mode 100644 lib/philomena/image_vectors/batch_processor.ex create mode 100644 lib/philomena/image_vectors/image_vector.ex create mode 100644 lib/philomena/image_vectors/importer.ex create mode 100644 lib/philomena_media/features.ex create mode 100644 priv/repo/migrations/20250109155442_create_image_vectors.exs diff --git a/lib/philomena/image_vectors.ex b/lib/philomena/image_vectors.ex new file mode 100644 index 000000000..85268440d --- /dev/null +++ b/lib/philomena/image_vectors.ex @@ -0,0 +1,91 @@ +defmodule Philomena.ImageVectors do + @moduledoc """ + The ImageVectors context. + """ + + import Ecto.Query, warn: false + alias Philomena.Repo + + alias Philomena.ImageVectors.ImageVector + + @doc """ + Gets a single image_vector. + + Raises `Ecto.NoResultsError` if the Image vector does not exist. + + ## Examples + + iex> get_image_vector!(123) + %ImageVector{} + + iex> get_image_vector!(456) + ** (Ecto.NoResultsError) + + """ + def get_image_vector!(id), do: Repo.get!(ImageVector, id) + + @doc """ + Creates a image_vector. + + ## Examples + + iex> create_image_vector(%{field: value}) + {:ok, %ImageVector{}} + + iex> create_image_vector(%{field: bad_value}) + {:error, %Ecto.Changeset{}} + + """ + def create_image_vector(image, attrs \\ %PhilomenaMedia.Features{}) do + %ImageVector{image_id: image.id} + |> ImageVector.changeset(Map.from_struct(attrs)) + |> Repo.insert() + end + + @doc """ + Updates a image_vector. + + ## Examples + + iex> update_image_vector(image_vector, %{field: new_value}) + {:ok, %ImageVector{}} + + iex> update_image_vector(image_vector, %{field: bad_value}) + {:error, %Ecto.Changeset{}} + + """ + def update_image_vector(%ImageVector{} = image_vector, attrs) do + image_vector + |> ImageVector.changeset(attrs) + |> Repo.update() + end + + @doc """ + Deletes a image_vector. + + ## Examples + + iex> delete_image_vector(image_vector) + {:ok, %ImageVector{}} + + iex> delete_image_vector(image_vector) + {:error, %Ecto.Changeset{}} + + """ + def delete_image_vector(%ImageVector{} = image_vector) do + Repo.delete(image_vector) + end + + @doc """ + Returns an `%Ecto.Changeset{}` for tracking image_vector changes. + + ## Examples + + iex> change_image_vector(image_vector) + %Ecto.Changeset{data: %ImageVector{}} + + """ + def change_image_vector(%ImageVector{} = image_vector, attrs \\ %{}) do + ImageVector.changeset(image_vector, attrs) + end +end diff --git a/lib/philomena/image_vectors/batch_processor.ex b/lib/philomena/image_vectors/batch_processor.ex new file mode 100644 index 000000000..3fefc3189 --- /dev/null +++ b/lib/philomena/image_vectors/batch_processor.ex @@ -0,0 +1,88 @@ +defmodule Philomena.ImageVectors.BatchProcessor do + @moduledoc """ + Batch processing interface for Philomena. See the module documentation + in `m:Philomena.ImageVectors.Importer` for more information about how to + use the functions in this module during maintenance. + """ + + alias Philomena.Images + alias Philomena.Images.Image + alias Philomena.Images.Thumbnailer + alias Philomena.ImageVectors.ImageVector + alias Philomena.Maintenance + alias Philomena.Repo + + alias PhilomenaMedia.Analyzers + alias PhilomenaMedia.Processors + alias PhilomenaQuery.Batch + alias PhilomenaQuery.Search + + alias Philomena.Repo + import Ecto.Query + + @spec all_missing(String.t(), Keyword.t()) :: :ok + def all_missing(type \\ "full", opts \\ []) do + Image + |> from(as: :image) + |> where(not exists(where(ImageVector, [iv], iv.image_id == parent_as(:image).id))) + |> by_image_query(type, opts) + end + + @spec by_image_query(Ecto.Query.t(), String.t(), Keyword.t()) :: :ok + defp by_image_query(query, type, opts) do + max_concurrency = Keyword.get(opts, :max_concurrency, 4) + min = Repo.one(limit(order_by(query, asc: :id), 1)).id + max = Repo.one(limit(order_by(query, desc: :id), 1)).id + + query + |> Batch.query_batches(opts) + |> Task.async_stream( + fn query -> process_query(query, type, opts) end, + timeout: :infinity, + max_concurrency: max_concurrency + ) + |> Maintenance.log_progress("BatchProcessor/#{type}", min, max) + end + + @spec process_query(Ecto.Query.t(), String.t(), Keyword.t()) :: + Enumerable.t({:ok, integer()}) + defp process_query(query, type, batch_opts) do + images = Repo.all(query) + last_id = Enum.max_by(images, & &1.id).id + + values = + Enum.flat_map(images, fn image -> + try do + [process_image(image, type)] + rescue + ex -> + IO.puts("While processing #{image.id}: #{inspect(ex)}") + IO.puts(Exception.format_stacktrace(__STACKTRACE__)) + [] + end + end) + + {_count, nil} = Repo.insert_all(ImageVector, values, on_conflict: :nothing) + + :ok = + query + |> preload(^Images.indexing_preloads()) + |> Search.reindex(Image, batch_opts) + + last_id + end + + @spec process_image(%Image{}, String.t()) :: map() + defp process_image(image = %Image{}, type) do + file = Thumbnailer.download_image_file(image) + + {:ok, analysis} = Analyzers.analyze_path(file) + features = Processors.features(analysis, file) + + %{ + image_id: image.id, + type: type, + features: features.features + } + end +end diff --git a/lib/philomena/image_vectors/image_vector.ex b/lib/philomena/image_vectors/image_vector.ex new file mode 100644 index 000000000..123f7015b --- /dev/null +++ b/lib/philomena/image_vectors/image_vector.ex @@ -0,0 +1,19 @@ +defmodule Philomena.ImageVectors.ImageVector do + use Ecto.Schema + import Ecto.Changeset + + alias Philomena.Images.Image + + schema "image_vectors" do + belongs_to :image, Image + field :type, :string + field :features, {:array, :float} + end + + @doc false + def changeset(image_vector, attrs) do + image_vector + |> cast(attrs, [:type, :features]) + |> validate_required([:type, :features]) + end +end diff --git a/lib/philomena/image_vectors/importer.ex b/lib/philomena/image_vectors/importer.ex new file mode 100644 index 000000000..3715feb67 --- /dev/null +++ b/lib/philomena/image_vectors/importer.ex @@ -0,0 +1,86 @@ +defmodule Philomena.ImageVectors.Importer do + @moduledoc """ + Import logic for binary files produced by the export function of + https://github.com/philomena-dev/philomena-ris-inference-toolkit. + + Run the following commands in a long-running terminal, like screen or tmux. + The workflow for using the importer is as follows: + + 1. Use the batch inference toolkit to get the `features.bin`. + 2. Run `philomena eval 'Philomena.ImageVectors.Importer.import_from("/path/to/features.bin")'`. + 3. Backfill the remaining images: + `philomena eval 'Philomena.ImageVectors.BatchProcessor.all_missing("full", batch_size: 32)'` + 4. Downtime, delete and recreate the images index: + `philomena eval 'Philomena.SearchIndexer.recreate_reindex_schema_destructive!(Philomena.Images.Image)'`. + """ + + alias Philomena.ImageVectors.ImageVector + alias Philomena.Maintenance + alias Philomena.Repo + + # 4 bytes unsigned id + 768 floats per feature vector * 4 bytes per float + @row_size 4 + 768 * 4 + + @typedoc "A single feature row." + @type row :: %{ + image_id: integer(), + type: String.t(), + features: [float()] + } + + @spec import_from(Path.t()) :: :ok + def import_from(batch_inference_file, type \\ "full", max_concurrency \\ 4) do + {min, max} = get_min_and_max_id(batch_inference_file, type) + + batch_inference_file + |> File.stream!(@row_size) + |> Stream.chunk_every(1024) + |> Task.async_stream( + &process_chunk(&1, type), + timeout: :infinity, + max_concurrency: max_concurrency + ) + |> Maintenance.log_progress("Importer/#{type}", min, max) + end + + @spec process_chunk([binary()], String.t()) :: :ok + defp process_chunk(chunk, type) do + data = Enum.map(chunk, &unpack(&1, type)) + last_id = Enum.max_by(data, & &1.image_id).image_id + + {_count, nil} = Repo.insert_all(ImageVector, data, on_conflict: :nothing) + + last_id + end + + @spec unpack(binary(), String.t()) :: row() + defp unpack(row, type) do + <> = row + features = for <>, do: v + + %{ + image_id: image_id, + type: type, + features: features + } + end + + @spec get_min_and_max_id(Path.t(), String.t()) :: {integer(), integer()} + defp get_min_and_max_id(path, type) do + stat = File.stat!(path) + last_row = stat.size - @row_size + + %{image_id: min} = get_single_row(path, 0, type) + %{image_id: max} = get_single_row(path, last_row, type) + + {min, max} + end + + @spec get_single_row(Path.t(), integer(), String.t()) :: row() + defp get_single_row(path, offset, type) do + path + |> File.stream!(@row_size, read_offset: offset) + |> Enum.at(0) + |> unpack(type) + end +end diff --git a/lib/philomena/images.ex b/lib/philomena/images.ex index f1661795f..7d5c8ef47 100644 --- a/lib/philomena/images.ex +++ b/lib/philomena/images.ex @@ -1311,6 +1311,7 @@ defmodule Philomena.Images do [ :gallery_interactions, + :vectors, sources: sources_query, user: user_query, favers: user_query, diff --git a/lib/philomena/images/image.ex b/lib/philomena/images/image.ex index f9e2aa7ce..4be5c0d42 100644 --- a/lib/philomena/images/image.ex +++ b/lib/philomena/images/image.ex @@ -8,6 +8,7 @@ defmodule Philomena.Images.Image do alias Philomena.ImageVotes.ImageVote alias Philomena.ImageFaves.ImageFave alias Philomena.ImageHides.ImageHide + alias Philomena.ImageVectors.ImageVector alias Philomena.Images.Source alias Philomena.Images.Subscription alias Philomena.Users.User @@ -36,6 +37,7 @@ defmodule Philomena.Images.Image do has_many :subscriptions, Subscription has_many :source_changes, SourceChange, on_replace: :delete has_many :tag_changes, TagChange + has_many :vectors, ImageVector has_many :upvoters, through: [:upvotes, :user] has_many :downvoters, through: [:downvotes, :user] has_many :favers, through: [:faves, :user] diff --git a/lib/philomena/images/search_index.ex b/lib/philomena/images/search_index.ex index 35241ccde..55f06a127 100644 --- a/lib/philomena/images/search_index.ex +++ b/lib/philomena/images/search_index.ex @@ -11,6 +11,7 @@ defmodule Philomena.Images.SearchIndex do %{ settings: %{ index: %{ + knn: true, number_of_shards: 5, max_result_window: 10_000_000 } @@ -89,6 +90,26 @@ defmodule Philomena.Images.SearchIndex do namespace: %{type: "keyword"} } }, + vectors: %{ + type: "nested", + properties: %{ + f: %{ + type: "knn_vector", + dimension: 768, + data_type: "float", + mode: "on_disk", + method: %{ + name: "hnsw", + engine: "faiss", + space_type: "l2", + parameters: %{ + ef_construction: 128, + m: 16 + } + } + } + } + }, approved: %{type: "boolean"}, error_tag_count: %{type: "integer"}, rating_tag_count: %{type: "integer"}, @@ -160,6 +181,7 @@ defmodule Philomena.Images.SearchIndex do }, gallery_id: Enum.map(image.gallery_interactions, & &1.gallery_id), gallery_position: Map.new(image.gallery_interactions, &{&1.gallery_id, &1.position}), + vectors: image.vectors |> Enum.map(&%{f: &1.features}), favourited_by_users: image.favers |> Enum.map(&String.downcase(&1.name)), hidden_by_users: image.hiders |> Enum.map(&String.downcase(&1.name)), upvoters: image.upvoters |> Enum.map(&String.downcase(&1.name)), diff --git a/lib/philomena/images/thumbnailer.ex b/lib/philomena/images/thumbnailer.ex index 1beea1465..5541cd6a0 100644 --- a/lib/philomena/images/thumbnailer.ex +++ b/lib/philomena/images/thumbnailer.ex @@ -12,6 +12,7 @@ defmodule Philomena.Images.Thumbnailer do alias Philomena.DuplicateReports alias Philomena.ImageIntensities alias Philomena.ImagePurgeWorker + alias Philomena.ImageVectors alias Philomena.Images.Image alias Philomena.Repo @@ -110,6 +111,9 @@ defmodule Philomena.Images.Thumbnailer do defp apply_change(image, {:intensities, intensities}), do: ImageIntensities.create_image_intensity(image, intensities) + defp apply_change(image, {:features, features}), + do: ImageVectors.create_image_vector(image, features) + defp apply_change(image, {:replace_original, new_file}) do full = "full.#{image.image_format}" upload_file(image, new_file, full) @@ -145,7 +149,7 @@ defmodule Philomena.Images.Thumbnailer do |> Repo.update!() end - defp download_image_file(image) do + def download_image_file(image) do tempfile = Briefly.create!(extname: ".#{image.image_format}") path = Path.join(image_thumb_prefix(image), "full.#{image.image_format}") diff --git a/lib/philomena_media/features.ex b/lib/philomena_media/features.ex new file mode 100644 index 000000000..bb75472c1 --- /dev/null +++ b/lib/philomena_media/features.ex @@ -0,0 +1,51 @@ +defmodule PhilomenaMedia.Features do + @moduledoc """ + Features are a set of 768 weighted classification outputs produced from a + vision transformer (ViT). The individual classifications are arbitrary and + not meaningful to analyze, but the vectors can be used to compare similarity + between images using the cosine similarity measurement. + + Since cosine similarity is not a metric, it is substituted for normalized L2 + distance by the feature extractor; every vector that it returns is normalized, + and traversing the k nearest neighbors in a vector space index will iterate + vectors in the same order as their cosine similarity. + """ + + alias PhilomenaMedia.Remote + + @type t :: %__MODULE__{ + features: [float()] + } + + defstruct [:features] + + @doc """ + Gets the features of the given image file. + + The image file must be in the PNG or JPEG format. + + > #### Info {: .info} + > + > Clients should prefer to use `PhilomenaMedia.Processors.features/2`, as it handles + > media files of any type supported by this library, not just PNG or JPEG. + + ## Examples + + iex> Features.file("image.png") + {:ok, %Features{features: [0.03156396001577377, -0.04559657722711563, ...]}} + + iex> Features.file("nonexistent.jpg") + :error + + """ + @spec file(Path.t()) :: {:ok, t()} | :error + def file(input) do + case Remote.get_features(input) do + {:ok, features} -> + {:ok, %__MODULE__{features: features}} + + _error -> + :error + end + end +end diff --git a/lib/philomena_media/processors.ex b/lib/philomena_media/processors.ex index b23ba0054..492400be6 100644 --- a/lib/philomena_media/processors.ex +++ b/lib/philomena_media/processors.ex @@ -58,6 +58,7 @@ defmodule PhilomenaMedia.Processors do """ alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Features alias PhilomenaMedia.Intensities alias PhilomenaMedia.Processors.{Gif, Jpeg, Png, Svg, Webm} alias PhilomenaMedia.Mime @@ -185,6 +186,25 @@ defmodule PhilomenaMedia.Processors do processor(analysis.mime_type).post_process(analysis, file) end + @doc """ + Takes an analyzer result and file path and runs the appropriate processor's `features/2`, + returning the feature vector. + + This allows for generating feature vectors for file types that are not directly supported by + `m:PhilomenaMedia.Features`, and should be the preferred function to call when feature vectors + are needed. + + ## Example + + iex> PhilomenaMedia.Processors.features(%Result{...}, "video.webm") + %Features{features: [0.03156396001577377, -0.04559657722711563, ...]} + + """ + @spec features(Result.t(), Path.t()) :: Features.t() + def features(analysis, file) do + processor(analysis.mime_type).features(analysis, file) + end + @doc """ Takes an analyzer result and file path and runs the appropriate processor's `intensities/2`, returning the corner intensities. @@ -195,8 +215,8 @@ defmodule PhilomenaMedia.Processors do ## Example - iex> PhilomenaMedia.Processors.intensities(%Result{...}, "video.webm") - %Intensities{nw: 111.689148, ne: 116.228048, sw: 93.268433, se: 104.630064} + iex> PhilomenaMedia.Processors.intensities(%Result{...}, "video.webm") + %Intensities{nw: 111.689148, ne: 116.228048, sw: 93.268433, se: 104.630064} """ @spec intensities(Result.t(), Path.t()) :: Intensities.t() diff --git a/lib/philomena_media/processors/gif.ex b/lib/philomena_media/processors/gif.ex index 497567198..11391aec2 100644 --- a/lib/philomena_media/processors/gif.ex +++ b/lib/philomena_media/processors/gif.ex @@ -1,6 +1,7 @@ defmodule PhilomenaMedia.Processors.Gif do @moduledoc false + alias PhilomenaMedia.Features alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result alias PhilomenaMedia.Remote @@ -23,12 +24,14 @@ defmodule PhilomenaMedia.Processors.Gif do palette = palette(file) {:ok, intensities} = Intensities.file(preview) + {:ok, features} = Features.file(preview) scaled = Enum.flat_map(versions, &scale(palette, file, &1)) videos = generate_videos(file) [ intensities: intensities, + features: features, thumbnails: scaled ++ videos ++ [{:copy, preview, "rendered.png"}] ] end @@ -38,6 +41,12 @@ defmodule PhilomenaMedia.Processors.Gif do [replace_original: optimize(file)] end + @spec features(Result.t(), Path.t()) :: Features.t() + def features(analysis, file) do + {:ok, features} = Features.file(preview(analysis.duration, file)) + features + end + @spec intensities(Result.t(), Path.t()) :: Intensities.t() def intensities(analysis, file) do {:ok, intensities} = Intensities.file(preview(analysis.duration, file)) diff --git a/lib/philomena_media/processors/jpeg.ex b/lib/philomena_media/processors/jpeg.ex index 30b2d5aff..f8dc3caba 100644 --- a/lib/philomena_media/processors/jpeg.ex +++ b/lib/philomena_media/processors/jpeg.ex @@ -1,6 +1,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do @moduledoc false + alias PhilomenaMedia.Features alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result alias PhilomenaMedia.Remote @@ -22,12 +23,14 @@ defmodule PhilomenaMedia.Processors.Jpeg do stripped = optimize(strip(file)) {:ok, intensities} = Intensities.file(stripped) + {:ok, features} = Features.file(stripped) scaled = Enum.flat_map(versions, &scale(stripped, &1)) [ replace_original: stripped, intensities: intensities, + features: features, thumbnails: scaled ] end @@ -35,6 +38,12 @@ defmodule PhilomenaMedia.Processors.Jpeg do @spec post_process(Result.t(), Path.t()) :: Processors.edit_script() def post_process(_analysis, _file), do: [] + @spec features(Result.t(), Path.t()) :: Features.t() + def features(_analysis, file) do + {:ok, features} = Features.file(file) + features + end + @spec intensities(Result.t(), Path.t()) :: Intensities.t() def intensities(_analysis, file) do {:ok, intensities} = Intensities.file(file) diff --git a/lib/philomena_media/processors/png.ex b/lib/philomena_media/processors/png.ex index 24222ce8e..72fd1efeb 100644 --- a/lib/philomena_media/processors/png.ex +++ b/lib/philomena_media/processors/png.ex @@ -1,6 +1,7 @@ defmodule PhilomenaMedia.Processors.Png do @moduledoc false + alias PhilomenaMedia.Features alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result alias PhilomenaMedia.Remote @@ -19,11 +20,13 @@ defmodule PhilomenaMedia.Processors.Png do animated? = analysis.animated? {:ok, intensities} = Intensities.file(file) + {:ok, features} = Features.file(file) scaled = Enum.flat_map(versions, &scale(file, animated?, &1)) [ intensities: intensities, + features: features, thumbnails: scaled ] end @@ -38,6 +41,12 @@ defmodule PhilomenaMedia.Processors.Png do end end + @spec features(Result.t(), Path.t()) :: Features.t() + def features(_analysis, file) do + {:ok, features} = Features.file(file) + features + end + @spec intensities(Result.t(), Path.t()) :: Intensities.t() def intensities(_analysis, file) do {:ok, intensities} = Intensities.file(file) diff --git a/lib/philomena_media/processors/processor.ex b/lib/philomena_media/processors/processor.ex index 8b9f568f3..368d2d320 100644 --- a/lib/philomena_media/processors/processor.ex +++ b/lib/philomena_media/processors/processor.ex @@ -2,6 +2,7 @@ defmodule PhilomenaMedia.Processors.Processor do @moduledoc false alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Features alias PhilomenaMedia.Processors alias PhilomenaMedia.Intensities @@ -22,6 +23,11 @@ defmodule PhilomenaMedia.Processors.Processor do """ @callback post_process(Result.t(), Path.t()) :: Processors.edit_script() + @doc """ + Generate a feature vector for the given path. + """ + @callback features(Result.t(), Path.t()) :: Features.t() + @doc """ Generate corner intensities for the given path. """ diff --git a/lib/philomena_media/processors/svg.ex b/lib/philomena_media/processors/svg.ex index 0f9b6e6cb..8a6140d95 100644 --- a/lib/philomena_media/processors/svg.ex +++ b/lib/philomena_media/processors/svg.ex @@ -1,6 +1,7 @@ defmodule PhilomenaMedia.Processors.Svg do @moduledoc false + alias PhilomenaMedia.Features alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result alias PhilomenaMedia.Remote @@ -21,12 +22,14 @@ defmodule PhilomenaMedia.Processors.Svg do preview = preview(file) {:ok, intensities} = Intensities.file(preview) + {:ok, features} = Features.file(preview) scaled = Enum.flat_map(versions, &scale(preview, &1)) full = [{:copy, preview, "full.png"}] [ intensities: intensities, + features: features, thumbnails: scaled ++ full ++ [{:copy, preview, "rendered.png"}] ] end @@ -34,6 +37,12 @@ defmodule PhilomenaMedia.Processors.Svg do @spec post_process(Result.t(), Path.t()) :: Processors.edit_script() def post_process(_analysis, _file), do: [] + @spec features(Result.t(), Path.t()) :: Features.t() + def features(_analysis, file) do + {:ok, features} = Features.file(preview(file)) + features + end + @spec intensities(Result.t(), Path.t()) :: Intensities.t() def intensities(_analysis, file) do {:ok, intensities} = Intensities.file(preview(file)) diff --git a/lib/philomena_media/processors/webm.ex b/lib/philomena_media/processors/webm.ex index 26d488ab0..8f13a326c 100644 --- a/lib/philomena_media/processors/webm.ex +++ b/lib/philomena_media/processors/webm.ex @@ -1,6 +1,7 @@ defmodule PhilomenaMedia.Processors.Webm do @moduledoc false + alias PhilomenaMedia.Features alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result alias PhilomenaMedia.Remote @@ -34,6 +35,7 @@ defmodule PhilomenaMedia.Processors.Webm do mp4 = scale_mp4_only(decoder, stripped, dimensions, dimensions) {:ok, intensities} = Intensities.file(preview) + {:ok, features} = Features.file(preview) scaled = Enum.flat_map(versions, &scale(decoder, stripped, duration, dimensions, &1)) mp4 = [{:copy, mp4, "full.mp4"}] @@ -41,6 +43,7 @@ defmodule PhilomenaMedia.Processors.Webm do [ replace_original: stripped, intensities: intensities, + features: features, thumbnails: scaled ++ mp4 ++ [{:copy, preview, "rendered.png"}] ] end @@ -48,6 +51,12 @@ defmodule PhilomenaMedia.Processors.Webm do @spec post_process(Result.t(), Path.t()) :: Processors.edit_script() def post_process(_analysis, _file), do: [] + @spec features(Result.t(), Path.t()) :: Features.t() + def features(analysis, file) do + {:ok, features} = Features.file(preview(analysis.duration, file)) + features + end + @spec intensities(Result.t(), Path.t()) :: Intensities.t() def intensities(analysis, file) do {:ok, intensities} = Intensities.file(preview(analysis.duration, file)) diff --git a/priv/repo/migrations/20250109155442_create_image_vectors.exs b/priv/repo/migrations/20250109155442_create_image_vectors.exs new file mode 100644 index 000000000..251b86b3b --- /dev/null +++ b/priv/repo/migrations/20250109155442_create_image_vectors.exs @@ -0,0 +1,14 @@ +defmodule Philomena.Repo.Migrations.CreateImageVectors do + use Ecto.Migration + + def change do + # NB: this is normalized, the float array is not divisible + create table(:image_vectors) do + add :image_id, references(:images, on_delete: :delete_all), null: false + add :type, :string, null: false + add :features, {:array, :float}, null: false + end + + create unique_index(:image_vectors, [:image_id, :type]) + end +end diff --git a/priv/repo/structure.sql b/priv/repo/structure.sql index 2541c47f3..6e4b49133 100644 --- a/priv/repo/structure.sql +++ b/priv/repo/structure.sql @@ -960,6 +960,37 @@ CREATE TABLE public.image_taggings ( ); +-- +-- Name: image_vectors; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.image_vectors ( + id bigint NOT NULL, + image_id bigint NOT NULL, + type character varying(255) NOT NULL, + features double precision[] NOT NULL +); + + +-- +-- Name: image_vectors_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.image_vectors_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: image_vectors_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.image_vectors_id_seq OWNED BY public.image_vectors.id; + + -- -- Name: image_votes; Type: TABLE; Schema: public; Owner: - -- @@ -2414,6 +2445,13 @@ ALTER TABLE ONLY public.image_features ALTER COLUMN id SET DEFAULT nextval('publ ALTER TABLE ONLY public.image_intensities ALTER COLUMN id SET DEFAULT nextval('public.image_intensities_id_seq'::regclass); +-- +-- Name: image_vectors id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.image_vectors ALTER COLUMN id SET DEFAULT nextval('public.image_vectors_id_seq'::regclass); + + -- -- Name: images id; Type: DEFAULT; Schema: public; Owner: - -- @@ -2783,6 +2821,14 @@ ALTER TABLE ONLY public.image_intensities ADD CONSTRAINT image_intensities_pkey PRIMARY KEY (id); +-- +-- Name: image_vectors image_vectors_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.image_vectors + ADD CONSTRAINT image_vectors_pkey PRIMARY KEY (id); + + -- -- Name: images images_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3257,7 +3303,14 @@ CREATE INDEX image_tag_locks_tag_id_index ON public.image_tag_locks USING btree -- --- Name: images_approved_index; Type: INDEX; Schema: public; Owner: - +-- Name: image_vectors_image_id_type_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX image_vectors_image_id_type_index ON public.image_vectors USING btree (image_id, type); + + +-- +-- Name: images_hidden_from_users_approved_index; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX images_approved_index ON public.images USING btree (approved) WHERE (approved = false); @@ -5487,6 +5540,14 @@ ALTER TABLE ONLY public.image_tag_locks ADD CONSTRAINT image_tag_locks_tag_id_fkey FOREIGN KEY (tag_id) REFERENCES public.tags(id) ON DELETE CASCADE; +-- +-- Name: image_vectors image_vectors_image_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.image_vectors + ADD CONSTRAINT image_vectors_image_id_fkey FOREIGN KEY (image_id) REFERENCES public.images(id) ON DELETE CASCADE; + + -- -- Name: moderation_logs moderation_logs_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -5586,6 +5647,7 @@ INSERT INTO public."schema_migrations" (version) VALUES (20220321173359); INSERT INTO public."schema_migrations" (version) VALUES (20240723122759); INSERT INTO public."schema_migrations" (version) VALUES (20240728191353); INSERT INTO public."schema_migrations" (version) VALUES (20241216165826); +INSERT INTO public."schema_migrations" (version) VALUES (20250109155442); INSERT INTO public."schema_migrations" (version) VALUES (20250407021536); INSERT INTO public."schema_migrations" (version) VALUES (20250501174007); INSERT INTO public."schema_migrations" (version) VALUES (20250502110018); From e78d6325a56fa79d1e63304b5cc792e9543db2cf Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 9 Jan 2025 17:56:28 -0500 Subject: [PATCH 3/6] Add feature-based reverse search interface --- lib/philomena/duplicate_reports.ex | 91 +++++++++++++++++-- .../api/json/search/reverse_controller.ex | 9 +- .../controllers/search/reverse_controller.ex | 6 +- lib/philomena_web/image_loader.ex | 21 ++++- .../templates/search/reverse/index.html.slime | 17 +--- 5 files changed, 115 insertions(+), 29 deletions(-) diff --git a/lib/philomena/duplicate_reports.ex b/lib/philomena/duplicate_reports.ex index a920e87ad..f1d808a23 100644 --- a/lib/philomena/duplicate_reports.ex +++ b/lib/philomena/duplicate_reports.ex @@ -9,6 +9,8 @@ defmodule Philomena.DuplicateReports do alias Ecto.Multi alias Philomena.Repo + alias PhilomenaMedia.Features + alias PhilomenaQuery.Search alias Philomena.DuplicateReports.DuplicateReport alias Philomena.DuplicateReports.SearchQuery alias Philomena.DuplicateReports.Uploader @@ -32,7 +34,7 @@ defmodule Philomena.DuplicateReports do source = Repo.preload(source, :intensity) {source.intensity, source.image_aspect_ratio} - |> find_duplicates(dist: 0.2) + |> find_duplicates_by_intensities(dist: 0.2) |> where([i, _it], i.id != ^source.id) |> Repo.all() |> Enum.map(fn target -> @@ -42,6 +44,76 @@ defmodule Philomena.DuplicateReports do end) end + def find_duplicates_by_features(features = %Features{}, filter, opts \\ []) do + min_score = Keyword.get(opts, :min_score, 0) + limit = Keyword.get(opts, :limit, 25) + + # TODO: many issues with efficient filtering using k-NN plugin, + # use post_filter to work around for the time being + # + # https://github.com/opensearch-project/k-NN/issues/2222 + # https://github.com/opensearch-project/k-NN/issues/2339 + # https://github.com/opensearch-project/k-NN/issues/2347 + + query = %{ + query: %{ + nested: %{ + path: "vectors", + query: %{ + knn: %{ + "vectors.f": %{ + vector: features.features, + k: 100 + } + } + } + } + }, + post_filter: filter, + min_score: min_score + } + + images = + Image + |> Search.search_definition(query, %{page_size: limit}) + |> Search.search_records(preload(Image, [:user, :sources, tags: :aliases])) + + images + |> Map.put(:total_entries, min(images.total_entries, limit)) + |> Map.put(:total_pages, min(images.total_pages, 1)) + end + + @doc """ + Executes the reverse image search query from parameters. + + ## Examples + + iex> execute_search_query_by_features(%{"image" => ...}) + {:ok, [%Image{...}, ....]} + + iex> execute_search_query_by_features(%{"image" => ...}) + {:error, %Ecto.Changeset{}} + + """ + def execute_search_query_by_features(filter, attrs \\ %{}) do + %SearchQuery{} + |> SearchQuery.changeset(attrs) + |> Uploader.analyze_upload(attrs) + |> Ecto.Changeset.apply_action(:create) + |> case do + {:ok, search_query} -> + images = + search_query + |> generate_features() + |> find_duplicates_by_features(filter, limit: search_query.limit) + + {:ok, images} + + error -> + error + end + end + @doc """ Query for potential duplicate images based on intensity values and aspect ratio. @@ -52,14 +124,14 @@ defmodule Philomena.DuplicateReports do ## Examples - iex> find_duplicates({%{nw: 0.5, ne: 0.5, sw: 0.5, se: 0.5}, 1.0}) + iex> find_duplicates_by_intensities({%{nw: 0.5, ne: 0.5, sw: 0.5, se: 0.5}, 1.0}) #Ecto.Query<...> - iex> find_duplicates({intensities, ratio}, dist: 0.3, limit: 20) + iex> find_duplicates_by_intensities({intensities, ratio}, dist: 0.3, limit: 20) #Ecto.Query<...> """ - def find_duplicates({intensities, aspect_ratio}, opts \\ []) do + def find_duplicates_by_intensities({intensities, aspect_ratio}, opts \\ []) do aspect_dist = Keyword.get(opts, :aspect_dist, 0.05) limit = Keyword.get(opts, :limit, 10) dist = Keyword.get(opts, :dist, 0.25) @@ -100,7 +172,7 @@ defmodule Philomena.DuplicateReports do {:error, %Ecto.Changeset{}} """ - def execute_search_query(attrs \\ %{}) do + def execute_search_query_by_intensities(attrs \\ %{}) do %SearchQuery{} |> SearchQuery.changeset(attrs) |> Uploader.analyze_upload(attrs) @@ -114,7 +186,7 @@ defmodule Philomena.DuplicateReports do images = {intensities, aspect} - |> find_duplicates(dist: dist, aspect_dist: dist, limit: limit) + |> find_duplicates_by_intensities(dist: dist, aspect_dist: dist, limit: limit) |> preload([:user, :intensity, [:sources, tags: :aliases]]) |> Repo.paginate(page_size: 50) @@ -132,6 +204,13 @@ defmodule Philomena.DuplicateReports do PhilomenaMedia.Processors.intensities(analysis, file) end + defp generate_features(search_query) do + analysis = SearchQuery.to_analysis(search_query) + file = search_query.uploaded_image + + PhilomenaMedia.Processors.features(analysis, file) + end + @doc """ Returns an `%Ecto.Changeset{}` for tracking search query changes. diff --git a/lib/philomena_web/controllers/api/json/search/reverse_controller.ex b/lib/philomena_web/controllers/api/json/search/reverse_controller.ex index 4abe75602..1345d9f19 100644 --- a/lib/philomena_web/controllers/api/json/search/reverse_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/reverse_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.Api.Json.Search.ReverseController do use PhilomenaWeb, :controller + alias PhilomenaWeb.ImageLoader alias Philomena.DuplicateReports alias Philomena.Interactions @@ -9,12 +10,12 @@ defmodule PhilomenaWeb.Api.Json.Search.ReverseController do def create(conn, %{"image" => image_params}) do user = conn.assigns.current_user + image_params = Map.put(image_params, "limit", conn.params["limit"]) {images, total} = - image_params - |> Map.put("distance", conn.params["distance"]) - |> Map.put("limit", conn.params["limit"]) - |> DuplicateReports.execute_search_query() + conn + |> ImageLoader.reverse_filter() + |> DuplicateReports.execute_search_query_by_features(image_params) |> case do {:ok, images} -> {images, images.total_entries} diff --git a/lib/philomena_web/controllers/search/reverse_controller.ex b/lib/philomena_web/controllers/search/reverse_controller.ex index 0938642ab..a3e803ab9 100644 --- a/lib/philomena_web/controllers/search/reverse_controller.ex +++ b/lib/philomena_web/controllers/search/reverse_controller.ex @@ -1,6 +1,7 @@ defmodule PhilomenaWeb.Search.ReverseController do use PhilomenaWeb, :controller + alias PhilomenaWeb.ImageLoader alias Philomena.DuplicateReports.SearchQuery alias Philomena.DuplicateReports alias Philomena.Interactions @@ -14,7 +15,10 @@ defmodule PhilomenaWeb.Search.ReverseController do def create(conn, %{"image" => image_params}) when is_map(image_params) and image_params != %{} do - case DuplicateReports.execute_search_query(image_params) do + conn + |> ImageLoader.reverse_filter() + |> DuplicateReports.execute_search_query_by_features(image_params) + |> case do {:ok, images} -> changeset = DuplicateReports.change_search_query(%SearchQuery{}) interactions = Interactions.user_interactions(images, conn.assigns.current_user) diff --git a/lib/philomena_web/image_loader.ex b/lib/philomena_web/image_loader.ex index 866c73968..da4757bf3 100644 --- a/lib/philomena_web/image_loader.ex +++ b/lib/philomena_web/image_loader.ex @@ -48,10 +48,6 @@ defmodule PhilomenaWeb.ImageLoader do |> load_tags() |> render_bodies(conn) - user = conn.assigns.current_user - filter = conn.assigns.compiled_filter - filters = create_filters(conn, user, filter) - %{query: query, sorts: sort} = sorts.(body) definition = @@ -61,7 +57,7 @@ defmodule PhilomenaWeb.ImageLoader do query: %{ bool: %{ must: query, - must_not: filters + must_not: filters(conn) } }, sort: sort @@ -72,6 +68,21 @@ defmodule PhilomenaWeb.ImageLoader do {definition, tags} end + def reverse_filter(conn) do + %{ + bool: %{ + must_not: filters(conn) + } + } + end + + defp filters(conn) do + user = conn.assigns.current_user + filter = conn.assigns.compiled_filter + + create_filters(conn, user, filter) + end + defp create_filters(conn, user, filter) do show_hidden? = Canada.Can.can?(user, :hide, %Image{}) del = conn.params["del"] diff --git a/lib/philomena_web/templates/search/reverse/index.html.slime b/lib/philomena_web/templates/search/reverse/index.html.slime index bc7643744..c203afbde 100644 --- a/lib/philomena_web/templates/search/reverse/index.html.slime +++ b/lib/philomena_web/templates/search/reverse/index.html.slime @@ -3,11 +3,9 @@ h1 Reverse Search = form_for @changeset, ~p"/search/reverse", [multipart: true, as: :image], fn f -> .walloftext p - ' Basic image similarity search. Finds uploaded images similar to the one - ' provided based on simple intensities and uses the median frame of - ' animations; very low contrast images (such as sketches) will produce - ' poor results and, regardless of contrast, results may include seemingly - ' random images that look very different. + ' Advanced image similarity search. Finds uploaded images similar to the one + ' provided based on perceptual features and uses the median frame of + ' animations. .image-other #js-image-upload-previews @@ -26,14 +24,7 @@ h1 Reverse Search .field-error-js.hidden.js-scraper - h4 Optional settings - - .field - = label f, :distance, "Match distance (suggested values: between 0.2 and 0.5)" - br - = number_input f, :distance, min: 0, max: 1, step: 0.01, class: "input" - = error_tag f, :distance - + = hidden_input f, :limit, value: @conn.assigns.image_pagination.page_size = error_tag f, :limit .field From 6537911acab52328503700955ae41ccf38360779 Mon Sep 17 00:00:00 2001 From: "Luna D." Date: Thu, 25 Sep 2025 10:21:05 +0200 Subject: [PATCH 4/6] fixes --- native/philomena/mediaproc_client/src/main.rs | 1 - native/philomena/mediaproc_server/src/dinov2.rs | 1 + native/philomena/src/remote.rs | 2 +- priv/repo/structure.sql | 10 +++++++--- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/native/philomena/mediaproc_client/src/main.rs b/native/philomena/mediaproc_client/src/main.rs index b57af29ab..a1850a3c3 100644 --- a/native/philomena/mediaproc_client/src/main.rs +++ b/native/philomena/mediaproc_client/src/main.rs @@ -4,7 +4,6 @@ use std::process::ExitCode; use clap::{Parser, Subcommand}; use mediaproc::MediaProcessorClient; use mediaproc::client; -use mediaproc::client::{connect_to_socket_server, execute_command}; #[derive(Parser, Debug)] #[command(version, about = "RPC Media Processor Client", long_about = None)] diff --git a/native/philomena/mediaproc_server/src/dinov2.rs b/native/philomena/mediaproc_server/src/dinov2.rs index d6c65b9ad..8236dcc1c 100644 --- a/native/philomena/mediaproc_server/src/dinov2.rs +++ b/native/philomena/mediaproc_server/src/dinov2.rs @@ -7,6 +7,7 @@ use crate::FeatureExtractionError; /// Each DINOv2 patch is 14x14 pub const PATCH_DIM: i64 = 14; +#[allow(dead_code)] pub struct ModelResult { pub patches: (i64, i64), pub image: Tensor, diff --git a/native/philomena/src/remote.rs b/native/philomena/src/remote.rs index 67c27b987..677137e47 100644 --- a/native/philomena/src/remote.rs +++ b/native/philomena/src/remote.rs @@ -1,6 +1,6 @@ -use mediaproc::CommandReply; use mediaproc::client; use mediaproc::client::{connect_to_socket_server, execute_command}; +use mediaproc::{CommandReply, FeatureExtractionError}; use rustler::{Encoder, Env, NifStruct, OwnedBinary, Term, atoms}; atoms! { diff --git a/priv/repo/structure.sql b/priv/repo/structure.sql index 6e4b49133..504c8061f 100644 --- a/priv/repo/structure.sql +++ b/priv/repo/structure.sql @@ -2,8 +2,10 @@ -- PostgreSQL database dump -- --- Dumped from database version 17.4 --- Dumped by pg_dump version 17.4 +\restrict 4uZg1pIPihuEFMBoApSVuOVuAffA9wwJgD4QS0iJ1tSBudUENHF38aA7i6AG38F + +-- Dumped from database version 17.6 +-- Dumped by pg_dump version 17.6 SET statement_timeout = 0; SET lock_timeout = 0; @@ -3310,7 +3312,7 @@ CREATE UNIQUE INDEX image_vectors_image_id_type_index ON public.image_vectors US -- --- Name: images_hidden_from_users_approved_index; Type: INDEX; Schema: public; Owner: - +-- Name: images_approved_index; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX images_approved_index ON public.images USING btree (approved) WHERE (approved = false); @@ -5624,6 +5626,8 @@ ALTER TABLE ONLY public.users -- PostgreSQL database dump complete -- +\unrestrict 4uZg1pIPihuEFMBoApSVuOVuAffA9wwJgD4QS0iJ1tSBudUENHF38aA7i6AG38F + INSERT INTO public."schema_migrations" (version) VALUES (20200503002523); INSERT INTO public."schema_migrations" (version) VALUES (20200607000511); INSERT INTO public."schema_migrations" (version) VALUES (20200617111116); From 52e1bcff64e6484b276a27ec06879a969326424a Mon Sep 17 00:00:00 2001 From: "Luna D." Date: Thu, 25 Sep 2025 10:54:37 +0200 Subject: [PATCH 5/6] more fixes --- docker/mediaproc/Dockerfile | 3 ++- native/philomena/mediaproc_server/src/io.rs | 10 ++++++++-- native/philomena/src/remote.rs | 1 - 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/mediaproc/Dockerfile b/docker/mediaproc/Dockerfile index 548a4ba7b..e002a3aad 100644 --- a/docker/mediaproc/Dockerfile +++ b/docker/mediaproc/Dockerfile @@ -67,7 +67,8 @@ ADD https://github.com/liamwhite/philomena-ris-inference-toolkit/releases/downlo RUN cd /tmp/philomena \ && cargo build --release -p mediaproc_server \ && cp target/release/mediaproc_server /usr/bin/mediaproc_server \ - && find target/release/build -regextype posix-extended -regex '^.*\.so(\.[0-9]+)*$' -exec cp '{}' /usr/lib/ ';' + && find target/release/build -regextype posix-extended -regex '^.*\.so(\.[0-9]+)*$' -exec cp '{}' /usr/lib/ ';' \ + && chmod 0644 /usr/share/dinov2-with-registers-base.pt # Set up unprivileged user account RUN useradd -ms /bin/bash mediaproc diff --git a/native/philomena/mediaproc_server/src/io.rs b/native/philomena/mediaproc_server/src/io.rs index 43f8618b5..77e902ad0 100644 --- a/native/philomena/mediaproc_server/src/io.rs +++ b/native/philomena/mediaproc_server/src/io.rs @@ -6,9 +6,15 @@ use crate::FeatureExtractionError; pub fn device_and_model(model_path: &str) -> Option<(Device, CModule)> { let device = Device::cuda_if_available(); - let model = CModule::load_on_device(model_path, device).ok()?; + let model = CModule::load_on_device(model_path, device); - Some((device, model)) + match model { + Err(err) => { + eprintln!("failed to load model from {model_path}: {err}"); + None + } + Ok(model) => Some((device, model)), + } } fn into_tensor>( diff --git a/native/philomena/src/remote.rs b/native/philomena/src/remote.rs index 677137e47..7c7fb6394 100644 --- a/native/philomena/src/remote.rs +++ b/native/philomena/src/remote.rs @@ -1,5 +1,4 @@ use mediaproc::client; -use mediaproc::client::{connect_to_socket_server, execute_command}; use mediaproc::{CommandReply, FeatureExtractionError}; use rustler::{Encoder, Env, NifStruct, OwnedBinary, Term, atoms}; From a1bd7d9fd4e7c852b74771f665b64b14f514d30a Mon Sep 17 00:00:00 2001 From: "Luna D." Date: Thu, 25 Sep 2025 11:12:30 +0200 Subject: [PATCH 6/6] suppress sobelow warning --- lib/philomena/image_vectors/importer.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/philomena/image_vectors/importer.ex b/lib/philomena/image_vectors/importer.ex index 3715feb67..38009d94f 100644 --- a/lib/philomena/image_vectors/importer.ex +++ b/lib/philomena/image_vectors/importer.ex @@ -29,6 +29,7 @@ defmodule Philomena.ImageVectors.Importer do } @spec import_from(Path.t()) :: :ok + # sobelow_skip ["Traversal.FileModule"] def import_from(batch_inference_file, type \\ "full", max_concurrency \\ 4) do {min, max} = get_min_and_max_id(batch_inference_file, type) @@ -77,6 +78,7 @@ defmodule Philomena.ImageVectors.Importer do end @spec get_single_row(Path.t(), integer(), String.t()) :: row() + # sobelow_skip ["Traversal.FileModule"] defp get_single_row(path, offset, type) do path |> File.stream!(@row_size, read_offset: offset)