From 2dfbea0e05fee505031384bcb6c11ac145e1e06a Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Tue, 1 Jul 2025 17:02:13 -0400 Subject: [PATCH 1/8] Replace reqwest with ureq in git.rs, http.rs, ipfs.rs --- Cargo.lock | 1085 +++------------------------------ lib/gofer/Cargo.toml | 9 +- lib/gofer/src/error.rs | 6 +- lib/gofer/src/schemes/git.rs | 24 +- lib/gofer/src/schemes/http.rs | 26 +- lib/gofer/src/schemes/ipfs.rs | 24 +- 6 files changed, 122 insertions(+), 1052 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3df8243..2055da9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -47,33 +38,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets", -] - [[package]] name = "base64" version = "0.22.1" @@ -113,12 +83,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - [[package]] name = "chrono" version = "0.4.41" @@ -169,16 +133,6 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -186,21 +140,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "data-url" -version = "0.3.1" +name = "crc32fast" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] [[package]] -name = "displaydoc" -version = "0.2.5" +name = "data-url" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" [[package]] name = "dogma" @@ -221,12 +173,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - [[package]] name = "fastrand" version = "2.3.0" @@ -234,29 +180,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" +name = "flate2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ - "percent-encoding", + "crc32fast", + "miniz_oxide", ] [[package]] -name = "futures-channel" -version = "0.3.31" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures-core" @@ -283,34 +220,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-core", - "futures-io", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - [[package]] name = "getrandom" version = "0.2.16" @@ -318,32 +227,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", - "js-sys", - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "js-sys", "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", - "wasm-bindgen", + "wasi", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "gofer" version = "0.1.8" @@ -353,37 +240,12 @@ dependencies = [ "dogma", "miette", "percent-encoding", - "reqwest", "ssh2", "suppaftp", "thiserror", + "ureq", ] -[[package]] -name = "h2" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" - [[package]] name = "heck" version = "0.5.0" @@ -401,97 +263,12 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "hyper" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.63" @@ -516,129 +293,6 @@ dependencies = [ "cc", ] -[[package]] -name = "icu_collections" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" - -[[package]] -name = "icu_properties" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "potential_utf", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" - -[[package]] -name = "icu_provider" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" -dependencies = [ - "displaydoc", - "icu_locale_core", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - [[package]] name = "iri-string" version = "0.7.8" @@ -726,12 +380,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "litemap" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" - [[package]] name = "lock_api" version = "0.4.13" @@ -748,12 +396,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - [[package]] name = "memchr" version = "2.7.5" @@ -782,12 +424,6 @@ dependencies = [ "syn", ] -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -797,17 +433,6 @@ dependencies = [ "adler2", ] -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -817,27 +442,12 @@ dependencies = [ "autocfg", ] -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - [[package]] name = "openssl-src" version = "300.5.0+3.5.0" @@ -901,12 +511,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.32" @@ -914,143 +518,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] -name = "potential_utf" -version = "0.1.2" +name = "proc-macro2" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ - "zerovec", + "unicode-ident", ] [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "quote" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "zerocopy", + "proc-macro2", ] [[package]] -name = "proc-macro2" -version = "1.0.95" +name = "redox_syscall" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ - "unicode-ident", + "bitflags", ] [[package]] -name = "quinn" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" -dependencies = [ - "bytes", - "getrandom 0.3.3", - "lru-slab", - "rand", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.59.0", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", -] - -[[package]] -name = "redox_syscall" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.11.1" +name = "regex" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ @@ -1077,50 +573,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "reqwest" -version = "0.12.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8cea6b35bcceb099f30173754403d2eba0a5dc18cea3630fccd88251909288" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "mime", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", -] - [[package]] name = "ring" version = "0.17.14" @@ -1129,24 +581,12 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom", "libc", "untrusted", - "windows-sys 0.52.0", + "windows-sys", ] -[[package]] -name = "rustc-demangle" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" - -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - [[package]] name = "rustls" version = "0.23.28" @@ -1163,15 +603,12 @@ dependencies = [ ] [[package]] -name = "rustls-native-certs" -version = "0.8.1" +name = "rustls-pemfile" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "openssl-probe", "rustls-pki-types", - "schannel", - "security-framework", ] [[package]] @@ -1180,7 +617,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ - "web-time", "zeroize", ] @@ -1201,50 +637,12 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "security-framework" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" version = "1.0.219" @@ -1265,58 +663,18 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_json" -version = "1.0.140" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "slab" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" - [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "ssh2" version = "0.9.5" @@ -1329,12 +687,6 @@ dependencies = [ "parking_lot", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "subtle" version = "2.6.1" @@ -1366,26 +718,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "thiserror" version = "2.0.12" @@ -1406,139 +738,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tinystr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.45.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "pin-project-lite", - "socket2", - "windows-sys 0.52.0", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "iri-string", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "unicode-ident" version = "1.0.18" @@ -1558,21 +757,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] -name = "url" -version = "2.5.4" +name = "ureq" +version = "3.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "9f0fde9bc91026e381155f8c67cb354bcd35260b2f4a29bcc84639f762760c39" dependencies = [ - "form_urlencoded", - "idna", + "base64", + "encoding_rs", + "flate2", + "log", "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "ureq-proto", + "utf-8", + "webpki-roots 0.26.11", +] + +[[package]] +name = "ureq-proto" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59db78ad1923f2b1be62b6da81fe80b173605ca0d57f85da2e005382adf693f7" +dependencies = [ + "base64", + "http", + "httparse", + "log", ] [[package]] -name = "utf8_iter" -version = "1.0.4" +name = "utf-8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "vcpkg" @@ -1580,30 +799,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -1630,19 +831,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -1676,23 +864,12 @@ dependencies = [ ] [[package]] -name = "web-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" +name = "webpki-roots" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "js-sys", - "wasm-bindgen", + "webpki-roots 1.0.1", ] [[package]] @@ -1772,15 +949,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -1845,121 +1013,8 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "writeable" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" - -[[package]] -name = "yoke" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" - -[[package]] -name = "zerotrie" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/lib/gofer/Cargo.toml b/lib/gofer/Cargo.toml index 07b0522..ad292f0 100644 --- a/lib/gofer/Cargo.toml +++ b/lib/gofer/Cargo.toml @@ -32,8 +32,8 @@ file = ["std"] ftp = ["dep:percent-encoding", "dep:suppaftp"] ftps = ["ftp", "suppaftp?/rustls"] git = ["https"] -http = ["dep:reqwest", "reqwest?/blocking"] -https = ["http", "reqwest?/http2", "reqwest?/rustls-tls-native-roots"] +http = ["dep:ureq"] +https = ["http"] ipfs = ["https"] scp = ["dep:ssh2"] stdin = ["std"] @@ -60,10 +60,7 @@ miette = { version = "7.5", default-features = false, features = [ "derive", ], optional = true } percent-encoding = { version = "2.3", default-features = false, optional = true } -reqwest = { version = "0.12", default-features = false, features = [ - "charset", - "rustls-tls", -], optional = true } +ureq = { version = "3.0.12", features = ["charset"], optional = true } ssh2 = { version = "0.9", default-features = false, features = [ "vendored-openssl", ], optional = true } diff --git a/lib/gofer/src/error.rs b/lib/gofer/src/error.rs index 85beaed..e1fdaeb 100644 --- a/lib/gofer/src/error.rs +++ b/lib/gofer/src/error.rs @@ -111,7 +111,7 @@ pub enum Error { url(docsrs), ) )] - FailedHttpRequest(#[from] reqwest::Error), + FailedHttpRequest(#[from] ureq::Error), #[cfg(feature = "git")] #[error("invalid Git URL: {0}")] @@ -213,10 +213,10 @@ impl TryInto for Error { } #[cfg(any(feature = "http", feature = "https"))] -impl TryInto for Error { +impl TryInto for Error { type Error = Error; - fn try_into(self) -> Result { + fn try_into(self) -> Result { match self { Error::FailedHttpRequest(e) => Ok(e), _ => Err(self), diff --git a/lib/gofer/src/schemes/git.rs b/lib/gofer/src/schemes/git.rs index ddcb025..d4af2b9 100644 --- a/lib/gofer/src/schemes/git.rs +++ b/lib/gofer/src/schemes/git.rs @@ -1,7 +1,7 @@ // This is free and unencumbered software released into the public domain. use crate::{Error, Read, Result, Url}; -use reqwest::{blocking::ClientBuilder, redirect}; +use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); @@ -15,19 +15,25 @@ static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_V /// See: https://git-scm.com/docs/gitweb /// See: https://docs.github.com/en/repositories/working-with-files/using-files/getting-permanent-links-to-files pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.ClientBuilder.html - let client = ClientBuilder::new() + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let agent: Agent = Agent::config_builder() .user_agent(USER_AGENT) - .redirect(redirect::Policy::default()) - .https_only(true); + .tls_config( + TlsConfig::builder() + .provider(TlsProvider::Rustls) + .build() + ) + .build() + .into(); let url = map_git_url_to_raw_url(url.as_str())?; - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.Client.html#method.get - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.RequestBuilder.html - let response = client.build()?.get(url).send()?; + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get + // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + let response = agent.get(&url).call()?; + let (_headers, body) = response.into_parts(); - Ok(Box::new(response)) + Ok(Box::new(body.into_reader())) } /// Maps a git URL to a raw content URL for supported git providers. diff --git a/lib/gofer/src/schemes/http.rs b/lib/gofer/src/schemes/http.rs index c9ef2c4..762d602 100644 --- a/lib/gofer/src/schemes/http.rs +++ b/lib/gofer/src/schemes/http.rs @@ -1,25 +1,31 @@ // This is free and unencumbered software released into the public domain. use crate::{Read, Result, Url}; -use reqwest::{blocking::ClientBuilder, redirect}; +use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); /// See: https://en.wikipedia.org/wiki/HTTP /// See: https://en.wikipedia.org/wiki/HTTPS pub fn open<'a, 'b>(url: &'a Url<'b>, secure: bool) -> Result> { - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.ClientBuilder.html - let mut client = ClientBuilder::new() - .user_agent(USER_AGENT) - .redirect(redirect::Policy::default()); + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let mut agent_builder = Agent::config_builder() + .user_agent(USER_AGENT); if secure { - client = client.https_only(true); + agent_builder = agent_builder.tls_config( + TlsConfig::builder() + .provider(TlsProvider::Rustls) + .build() + ); } - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.Client.html#method.get - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.RequestBuilder.html - let response = client.build()?.get(url.as_str()).send()?; + let agent: ureq::Agent = agent_builder.build().into(); - Ok(Box::new(response)) + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get + // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + let response = agent.get(url.as_str()).call()?; + let (_headers, body) = response.into_parts(); + + Ok(Box::new(body.into_reader())) } diff --git a/lib/gofer/src/schemes/ipfs.rs b/lib/gofer/src/schemes/ipfs.rs index 0f4936e..1533fdc 100644 --- a/lib/gofer/src/schemes/ipfs.rs +++ b/lib/gofer/src/schemes/ipfs.rs @@ -1,18 +1,23 @@ // This is free and unencumbered software released into the public domain. use crate::{Error, Read, Result, Url}; -use reqwest::{blocking::ClientBuilder, redirect}; +use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); static GATEWAY: &str = "https://ipfs.io"; /// See: https://en.wikipedia.org/wiki/InterPlanetary_File_System pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.ClientBuilder.html - let client = ClientBuilder::new() + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let agent: Agent = Agent::config_builder() .user_agent(USER_AGENT) - .redirect(redirect::Policy::default()) - .https_only(true); + .tls_config( + TlsConfig::builder() + .provider(TlsProvider::Rustls) + .build() + ) + .build() + .into(); let url = url .as_str() @@ -20,9 +25,10 @@ pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { .ok_or_else(|| Error::InvalidIpfsUrl(url.to_string())) .map(|id| format!("{}/ipfs/{}", GATEWAY, id))?; - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.Client.html#method.get - // See: https://docs.rs/reqwest/latest/reqwest/blocking/struct.RequestBuilder.html - let response = client.build()?.get(url).send()?; + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get + // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + let response = agent.get(&url).call()?; + let (_headers, body) = response.into_parts(); - Ok(Box::new(response)) + Ok(Box::new(body.into_reader())) } From 711bc8798f8a496a4628c26364d37779a31e614f Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Tue, 1 Jul 2025 17:33:02 -0400 Subject: [PATCH 2/8] Add Accept header and refactor git.rs, http.rs, ipfs.rs with improved utils.rs structure --- lib/gofer/src/schemes.rs | 1 + lib/gofer/src/schemes/git.rs | 23 ++++------------------- lib/gofer/src/schemes/http.rs | 23 +++-------------------- lib/gofer/src/schemes/ipfs.rs | 27 +++------------------------ lib/gofer/src/schemes/utils.rs | 28 ++++++++++++++++++++++++++++ 5 files changed, 39 insertions(+), 63 deletions(-) create mode 100644 lib/gofer/src/schemes/utils.rs diff --git a/lib/gofer/src/schemes.rs b/lib/gofer/src/schemes.rs index 1830e4d..15c0a6a 100644 --- a/lib/gofer/src/schemes.rs +++ b/lib/gofer/src/schemes.rs @@ -23,3 +23,4 @@ pub mod scp; #[cfg(feature = "stdin")] pub mod stdin; +mod utils; diff --git a/lib/gofer/src/schemes/git.rs b/lib/gofer/src/schemes/git.rs index d4af2b9..ef6cc32 100644 --- a/lib/gofer/src/schemes/git.rs +++ b/lib/gofer/src/schemes/git.rs @@ -1,9 +1,6 @@ // This is free and unencumbered software released into the public domain. -use crate::{Error, Read, Result, Url}; -use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; - -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); +use crate::{Error, Read, Result, Url, schemes::utils::create_get_request}; /// Downloads a file from a git repository. /// @@ -15,22 +12,10 @@ static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_V /// See: https://git-scm.com/docs/gitweb /// See: https://docs.github.com/en/repositories/working-with-files/using-files/getting-permanent-links-to-files pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { - // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let agent: Agent = Agent::config_builder() - .user_agent(USER_AGENT) - .tls_config( - TlsConfig::builder() - .provider(TlsProvider::Rustls) - .build() - ) - .build() - .into(); - - let url = map_git_url_to_raw_url(url.as_str())?; - - // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let response = agent.get(&url).call()?; + let url = map_git_url_to_raw_url(url.as_str())?; + let request = create_get_request(true, &url)?; + let response = request.call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) diff --git a/lib/gofer/src/schemes/http.rs b/lib/gofer/src/schemes/http.rs index 762d602..4a3bd9f 100644 --- a/lib/gofer/src/schemes/http.rs +++ b/lib/gofer/src/schemes/http.rs @@ -1,30 +1,13 @@ // This is free and unencumbered software released into the public domain. -use crate::{Read, Result, Url}; -use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; - -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); +use crate::{Read, Result, Url, schemes::utils::create_get_request}; /// See: https://en.wikipedia.org/wiki/HTTP /// See: https://en.wikipedia.org/wiki/HTTPS pub fn open<'a, 'b>(url: &'a Url<'b>, secure: bool) -> Result> { - // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let mut agent_builder = Agent::config_builder() - .user_agent(USER_AGENT); - - if secure { - agent_builder = agent_builder.tls_config( - TlsConfig::builder() - .provider(TlsProvider::Rustls) - .build() - ); - } - - let agent: ureq::Agent = agent_builder.build().into(); - - // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let response = agent.get(url.as_str()).call()?; + let request = create_get_request(secure, url.as_str())?; + let response = request.call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) diff --git a/lib/gofer/src/schemes/ipfs.rs b/lib/gofer/src/schemes/ipfs.rs index 1533fdc..c6c1119 100644 --- a/lib/gofer/src/schemes/ipfs.rs +++ b/lib/gofer/src/schemes/ipfs.rs @@ -1,33 +1,12 @@ // This is free and unencumbered software released into the public domain. -use crate::{Error, Read, Result, Url}; -use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; - -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); -static GATEWAY: &str = "https://ipfs.io"; +use crate::{Read, Result, Url, schemes::utils::create_get_request}; /// See: https://en.wikipedia.org/wiki/InterPlanetary_File_System pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { - // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let agent: Agent = Agent::config_builder() - .user_agent(USER_AGENT) - .tls_config( - TlsConfig::builder() - .provider(TlsProvider::Rustls) - .build() - ) - .build() - .into(); - - let url = url - .as_str() - .strip_prefix("ipfs://") - .ok_or_else(|| Error::InvalidIpfsUrl(url.to_string())) - .map(|id| format!("{}/ipfs/{}", GATEWAY, id))?; - - // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let response = agent.get(&url).call()?; + let request = create_get_request(true, url.as_str())?; + let response = request.call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) diff --git a/lib/gofer/src/schemes/utils.rs b/lib/gofer/src/schemes/utils.rs new file mode 100644 index 0000000..b694382 --- /dev/null +++ b/lib/gofer/src/schemes/utils.rs @@ -0,0 +1,28 @@ +// This is free and unencumbered software released into the public domain. + +use crate::Result; +use ureq::{Agent, RequestBuilder, typestate::WithoutBody, tls::{TlsConfig, TlsProvider}}; +use std::time::Duration; + +static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); + +/// Creates a configured ureq RequestBuilder for GET requests with user agent, +/// optional HTTPS enforcement, and Accept header for JSON-LD and any media type. +/// +/// If `secure` is true, enforces HTTPS using rustls. Sets a default timeout of 30 seconds. +pub fn create_get_request(secure: bool, url: &str) -> Result> { + let mut builder = Agent::config_builder() + .user_agent(USER_AGENT) + .timeout_global(Some(Duration::from_secs(30))); + + if secure { + builder = builder.tls_config( + TlsConfig::builder() + .provider(TlsProvider::Rustls) + .build() + ); + } + + let agent: Agent = builder.build().into(); + Ok(agent.get(url).header("Accept", "application/ld+json,*/*")) +} \ No newline at end of file From dcd63089627e408d2beea9ea108a2826436e3aad Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Tue, 1 Jul 2025 17:48:55 -0400 Subject: [PATCH 3/8] Revert "Add Accept header and refactor git.rs, http.rs, ipfs.rs with improved utils.rs structure --- lib/gofer/src/schemes.rs | 1 - lib/gofer/src/schemes/git.rs | 23 +++++++++++++++++++---- lib/gofer/src/schemes/http.rs | 23 ++++++++++++++++++++--- lib/gofer/src/schemes/ipfs.rs | 27 ++++++++++++++++++++++++--- lib/gofer/src/schemes/utils.rs | 28 ---------------------------- 5 files changed, 63 insertions(+), 39 deletions(-) delete mode 100644 lib/gofer/src/schemes/utils.rs diff --git a/lib/gofer/src/schemes.rs b/lib/gofer/src/schemes.rs index 15c0a6a..1830e4d 100644 --- a/lib/gofer/src/schemes.rs +++ b/lib/gofer/src/schemes.rs @@ -23,4 +23,3 @@ pub mod scp; #[cfg(feature = "stdin")] pub mod stdin; -mod utils; diff --git a/lib/gofer/src/schemes/git.rs b/lib/gofer/src/schemes/git.rs index ef6cc32..d4af2b9 100644 --- a/lib/gofer/src/schemes/git.rs +++ b/lib/gofer/src/schemes/git.rs @@ -1,6 +1,9 @@ // This is free and unencumbered software released into the public domain. -use crate::{Error, Read, Result, Url, schemes::utils::create_get_request}; +use crate::{Error, Read, Result, Url}; +use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; + +static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); /// Downloads a file from a git repository. /// @@ -12,10 +15,22 @@ use crate::{Error, Read, Result, Url, schemes::utils::create_get_request}; /// See: https://git-scm.com/docs/gitweb /// See: https://docs.github.com/en/repositories/working-with-files/using-files/getting-permanent-links-to-files pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { - // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let agent: Agent = Agent::config_builder() + .user_agent(USER_AGENT) + .tls_config( + TlsConfig::builder() + .provider(TlsProvider::Rustls) + .build() + ) + .build() + .into(); + let url = map_git_url_to_raw_url(url.as_str())?; - let request = create_get_request(true, &url)?; - let response = request.call()?; + + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get + // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + let response = agent.get(&url).call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) diff --git a/lib/gofer/src/schemes/http.rs b/lib/gofer/src/schemes/http.rs index 4a3bd9f..762d602 100644 --- a/lib/gofer/src/schemes/http.rs +++ b/lib/gofer/src/schemes/http.rs @@ -1,13 +1,30 @@ // This is free and unencumbered software released into the public domain. -use crate::{Read, Result, Url, schemes::utils::create_get_request}; +use crate::{Read, Result, Url}; +use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; + +static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); /// See: https://en.wikipedia.org/wiki/HTTP /// See: https://en.wikipedia.org/wiki/HTTPS pub fn open<'a, 'b>(url: &'a Url<'b>, secure: bool) -> Result> { + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let mut agent_builder = Agent::config_builder() + .user_agent(USER_AGENT); + + if secure { + agent_builder = agent_builder.tls_config( + TlsConfig::builder() + .provider(TlsProvider::Rustls) + .build() + ); + } + + let agent: ureq::Agent = agent_builder.build().into(); + + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let request = create_get_request(secure, url.as_str())?; - let response = request.call()?; + let response = agent.get(url.as_str()).call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) diff --git a/lib/gofer/src/schemes/ipfs.rs b/lib/gofer/src/schemes/ipfs.rs index c6c1119..1533fdc 100644 --- a/lib/gofer/src/schemes/ipfs.rs +++ b/lib/gofer/src/schemes/ipfs.rs @@ -1,12 +1,33 @@ // This is free and unencumbered software released into the public domain. -use crate::{Read, Result, Url, schemes::utils::create_get_request}; +use crate::{Error, Read, Result, Url}; +use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; + +static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); +static GATEWAY: &str = "https://ipfs.io"; /// See: https://en.wikipedia.org/wiki/InterPlanetary_File_System pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let agent: Agent = Agent::config_builder() + .user_agent(USER_AGENT) + .tls_config( + TlsConfig::builder() + .provider(TlsProvider::Rustls) + .build() + ) + .build() + .into(); + + let url = url + .as_str() + .strip_prefix("ipfs://") + .ok_or_else(|| Error::InvalidIpfsUrl(url.to_string())) + .map(|id| format!("{}/ipfs/{}", GATEWAY, id))?; + + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let request = create_get_request(true, url.as_str())?; - let response = request.call()?; + let response = agent.get(&url).call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) diff --git a/lib/gofer/src/schemes/utils.rs b/lib/gofer/src/schemes/utils.rs deleted file mode 100644 index b694382..0000000 --- a/lib/gofer/src/schemes/utils.rs +++ /dev/null @@ -1,28 +0,0 @@ -// This is free and unencumbered software released into the public domain. - -use crate::Result; -use ureq::{Agent, RequestBuilder, typestate::WithoutBody, tls::{TlsConfig, TlsProvider}}; -use std::time::Duration; - -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); - -/// Creates a configured ureq RequestBuilder for GET requests with user agent, -/// optional HTTPS enforcement, and Accept header for JSON-LD and any media type. -/// -/// If `secure` is true, enforces HTTPS using rustls. Sets a default timeout of 30 seconds. -pub fn create_get_request(secure: bool, url: &str) -> Result> { - let mut builder = Agent::config_builder() - .user_agent(USER_AGENT) - .timeout_global(Some(Duration::from_secs(30))); - - if secure { - builder = builder.tls_config( - TlsConfig::builder() - .provider(TlsProvider::Rustls) - .build() - ); - } - - let agent: Agent = builder.build().into(); - Ok(agent.get(url).header("Accept", "application/ld+json,*/*")) -} \ No newline at end of file From 7daf928685c4d3feceef75345b64a3f1c58dbd46 Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Tue, 1 Jul 2025 18:52:09 -0400 Subject: [PATCH 4/8] Refactor git.rs, http.rs, ipfs.rs with improved utils.rs structure --- lib/gofer/src/schemes.rs | 2 ++ lib/gofer/src/schemes/git.rs | 55 ++++++++++++++++++---------------- lib/gofer/src/schemes/http.rs | 36 ++++++++++------------ lib/gofer/src/schemes/ipfs.rs | 32 ++++++++++---------- lib/gofer/src/schemes/utils.rs | 38 +++++++++++++++++++++++ 5 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 lib/gofer/src/schemes/utils.rs diff --git a/lib/gofer/src/schemes.rs b/lib/gofer/src/schemes.rs index 1830e4d..d9f10c2 100644 --- a/lib/gofer/src/schemes.rs +++ b/lib/gofer/src/schemes.rs @@ -23,3 +23,5 @@ pub mod scp; #[cfg(feature = "stdin")] pub mod stdin; + +mod utils; diff --git a/lib/gofer/src/schemes/git.rs b/lib/gofer/src/schemes/git.rs index d4af2b9..47d7f2a 100644 --- a/lib/gofer/src/schemes/git.rs +++ b/lib/gofer/src/schemes/git.rs @@ -1,44 +1,49 @@ // This is free and unencumbered software released into the public domain. use crate::{Error, Read, Result, Url}; -use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; +use crate::schemes::utils; -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); - -/// Downloads a file from a git repository. +/// Downloads a file from a git repository using a raw content URL. +/// +/// Supports fetching files from: +/// - GitHub: `git://github.com/owner/repo/branch/...path` +/// - GitLab: `git://gitlab.com/owner/repo/branch/...path` +/// +/// The function maps the provided git URL to a raw content URL and performs an HTTP GET request +/// to retrieve the file content as a readable stream. /// -/// Supports: -/// - GitHub: git://github.com/owner/repo/branch/...path -/// - GitLab: git://gitlab.com/owner/repo/branch/...path +/// # Arguments +/// * `url` - The git URL specifying the repository, branch, and file path. /// -/// See: https://git-scm.com/docs/protocol-v2 -/// See: https://git-scm.com/docs/gitweb -/// See: https://docs.github.com/en/repositories/working-with-files/using-files/getting-permanent-links-to-files +/// # Returns +/// A `Result` containing a boxed readable stream (`Box`) on success, or an `Error` on failure. +/// +/// # References +/// - [Git Protocol v2](https://git-scm.com/docs/protocol-v2) +/// - [Git Web Interface](https://git-scm.com/docs/gitweb) +/// - [GitHub Permanent Links](https://docs.github.com/en/repositories/working-with-files/using-files/getting-permanent-links-to-files) pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let agent: Agent = Agent::config_builder() - .user_agent(USER_AGENT) - .tls_config( - TlsConfig::builder() - .provider(TlsProvider::Rustls) - .build() - ) - .build() - .into(); + let agent = utils::new_agent(true, None); let url = map_git_url_to_raw_url(url.as_str())?; // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let response = agent.get(&url).call()?; - let (_headers, body) = response.into_parts(); - - Ok(Box::new(body.into_reader())) + utils::fetch(&agent, &url) } /// Maps a git URL to a raw content URL for supported git providers. -/// - GitHub: git://github.com/owner/repo/branch/...path -> https://raw.githubusercontent.com/owner/repo/refs/heads/branch/...path -/// - GitLab: git://gitlab.com/owner/repo/branch/...path -> https://gitlab.com/owner/repo/-/raw/branch/...path +/// +/// Converts URLs like: +/// - GitHub: `git://github.com/owner/repo/branch/...path` → `https://raw.githubusercontent.com/owner/repo/refs/heads/branch/...path` +/// - GitLab: `git://gitlab.com/owner/repo/branch/...path` → `https://gitlab.com/owner/repo/-/raw/branch/...path` +/// +/// # Arguments +/// * `url_str` - The git URL to map. +/// +/// # Returns +/// A `Result` containing the raw content URL as a `String` on success, or an `Error` if the URL is invalid or the host is unsupported. fn map_git_url_to_raw_url(url_str: &str) -> Result { let Some(path) = url_str.strip_prefix("git://") else { return Err(Error::InvalidGitUrl(format!( diff --git a/lib/gofer/src/schemes/http.rs b/lib/gofer/src/schemes/http.rs index 762d602..0b50ac8 100644 --- a/lib/gofer/src/schemes/http.rs +++ b/lib/gofer/src/schemes/http.rs @@ -1,31 +1,25 @@ // This is free and unencumbered software released into the public domain. use crate::{Read, Result, Url}; -use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; +use crate::schemes::utils; -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); - -/// See: https://en.wikipedia.org/wiki/HTTP -/// See: https://en.wikipedia.org/wiki/HTTPS +/// Performs an HTTP or HTTPS GET request to fetch a file from a URL. +/// +/// # Arguments +/// * `url` - The URL to fetch (e.g., `http://example.com/file.txt` or `https://example.com/file.txt`). +/// * `secure` - Whether to use HTTPS (TLS) for the request. +/// +/// # Returns +/// A `Result` containing a boxed readable stream (`Box`) on success, or an `Error` on failure. +/// +/// # References +/// - [HTTP Protocol](https://en.wikipedia.org/wiki/HTTP) +/// - [HTTPS Protocol](https://en.wikipedia.org/wiki/HTTPS) pub fn open<'a, 'b>(url: &'a Url<'b>, secure: bool) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let mut agent_builder = Agent::config_builder() - .user_agent(USER_AGENT); - - if secure { - agent_builder = agent_builder.tls_config( - TlsConfig::builder() - .provider(TlsProvider::Rustls) - .build() - ); - } - - let agent: ureq::Agent = agent_builder.build().into(); + let agent = utils::new_agent(secure, None); // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let response = agent.get(url.as_str()).call()?; - let (_headers, body) = response.into_parts(); - - Ok(Box::new(body.into_reader())) + utils::fetch(&agent, url.as_str()) } diff --git a/lib/gofer/src/schemes/ipfs.rs b/lib/gofer/src/schemes/ipfs.rs index 1533fdc..cfbb491 100644 --- a/lib/gofer/src/schemes/ipfs.rs +++ b/lib/gofer/src/schemes/ipfs.rs @@ -1,23 +1,26 @@ // This is free and unencumbered software released into the public domain. use crate::{Error, Read, Result, Url}; -use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; +use crate::schemes::utils; -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); static GATEWAY: &str = "https://ipfs.io"; -/// See: https://en.wikipedia.org/wiki/InterPlanetary_File_System +/// Fetches a file from the InterPlanetary File System (IPFS) using a gateway. +/// +/// The function converts an IPFS URL (e.g., `ipfs://`) to a gateway URL +/// and performs an HTTPS GET request to retrieve the file content as a readable stream. +/// +/// # Arguments +/// * `url` - The IPFS URL specifying the content ID (e.g., `ipfs://Qm...`). +/// +/// # Returns +/// A `Result` containing a boxed readable stream (`Box`) on success, or an `Error` if the URL is invalid. +/// +/// # References +/// - [InterPlanetary File System](https://en.wikipedia.org/wiki/InterPlanetary_File_System) pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let agent: Agent = Agent::config_builder() - .user_agent(USER_AGENT) - .tls_config( - TlsConfig::builder() - .provider(TlsProvider::Rustls) - .build() - ) - .build() - .into(); + let agent = utils::new_agent(true, None); let url = url .as_str() @@ -27,8 +30,5 @@ pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - let response = agent.get(&url).call()?; - let (_headers, body) = response.into_parts(); - - Ok(Box::new(body.into_reader())) + utils::fetch(&agent, &url) } diff --git a/lib/gofer/src/schemes/utils.rs b/lib/gofer/src/schemes/utils.rs new file mode 100644 index 0000000..acf694e --- /dev/null +++ b/lib/gofer/src/schemes/utils.rs @@ -0,0 +1,38 @@ +// This is free and unencumbered software released into the public domain. + +use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; +use crate::{Read, Result}; + +static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); + +/// Creates a new ureq Agent with a specified user agent and optional TLS configuration. +/// +/// # Arguments +/// * `use_tls` - Whether to enable TLS with Rustls as the provider. +/// * `tls_provider` - Optional TLS provider; defaults to Rustls if `use_tls` is true. +pub fn new_agent(use_tls: bool, tls_provider: Option) -> Agent { + let mut builder = Agent::config_builder() + .user_agent(USER_AGENT); + + if use_tls { + let provider = tls_provider.unwrap_or(TlsProvider::Rustls); + builder = builder.tls_config( + TlsConfig::builder() + .provider(provider) + .build() + ); + } + + builder.build().into() +} + +/// Performs an HTTP GET request and returns the response body as a readable stream. +/// +/// # Arguments +/// * `agent` - The ureq Agent to use for the request. +/// * `url` - The URL to fetch. +pub fn fetch(agent: &Agent, url: &str) -> Result> { + let response = agent.get(url).call()?; + let (_headers, body) = response.into_parts(); + Ok(Box::new(body.into_reader())) +} \ No newline at end of file From 0df943632dac302fb2927938a0fa7c74fb743516 Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Tue, 1 Jul 2025 19:10:37 -0400 Subject: [PATCH 5/8] Fix: conditionally compile utils.rs based on features that require ureq --- lib/gofer/src/schemes.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gofer/src/schemes.rs b/lib/gofer/src/schemes.rs index d9f10c2..2ae64b7 100644 --- a/lib/gofer/src/schemes.rs +++ b/lib/gofer/src/schemes.rs @@ -24,4 +24,5 @@ pub mod scp; #[cfg(feature = "stdin")] pub mod stdin; +#[cfg(any(feature = "http", feature = "https", feature = "git", feature = "ipfs"))] mod utils; From 15f1a2fa4db388524d038a3d1099cdc2a58e4d2d Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Tue, 1 Jul 2025 20:35:56 -0400 Subject: [PATCH 6/8] Add Accept header and open_with_config --- lib/gofer/src/lib.rs | 55 +++++++++++++++++++ lib/gofer/src/open.rs | 22 +++++++- lib/gofer/src/schemes.rs | 2 +- lib/gofer/src/schemes/git.rs | 37 +++++++++++-- lib/gofer/src/schemes/http.rs | 34 ++++++++++-- lib/gofer/src/schemes/ipfs.rs | 54 +++++++++++++++--- .../src/schemes/{utils.rs => request.rs} | 22 +++++++- 7 files changed, 204 insertions(+), 22 deletions(-) rename lib/gofer/src/schemes/{utils.rs => request.rs} (57%) diff --git a/lib/gofer/src/lib.rs b/lib/gofer/src/lib.rs index f021dc9..c855717 100644 --- a/lib/gofer/src/lib.rs +++ b/lib/gofer/src/lib.rs @@ -9,6 +9,7 @@ #![deny(unsafe_code)] //#![allow(unused)] +use std::collections::HashMap; #[cfg(feature = "std")] pub use std::io::{Cursor, Read}; @@ -34,3 +35,57 @@ pub use schemes::*; #[doc = include_str!("../../../README.md")] #[cfg(doctest)] pub struct ReadmeDoctests; + +#[derive(Clone, Debug)] +pub struct RequestConfig { + pub headers: HashMap, +} + +/// Configuration for HTTP requests. +/// +/// This struct allows customizing HTTP requests with headers. It is used with protocol-specific +/// `open_with_config` functions to provide additional request parameters, such as custom HTTP headers. +/// +/// # Examples +/// ```rust +/// use gofer::RequestConfig; +/// +/// let config = RequestConfig::new() +/// .with_header("User-Agent", "gofer/1.0") +/// .with_header("Accept", "application/json"); +/// assert_eq!(config.headers.len(), 2); +/// ``` +impl RequestConfig { + /// Creates a new, empty `RequestConfig`. + /// + /// # Returns + /// A `RequestConfig` instance with no headers set. + pub fn new() -> Self { + RequestConfig { + headers: HashMap::new(), + } + } + + /// Adds a header to the configuration and returns the modified `RequestConfig`. + /// + /// This method allows chaining to add multiple headers fluently. + /// + /// # Arguments + /// * `key` - The header name (e.g., `"User-Agent"`). + /// * `value` - The header value (e.g., `"gofer/1.0"`). + /// + /// # Returns + /// The modified `RequestConfig` with the new header added. + /// + /// # Examples + /// ```rust + /// use gofer::RequestConfig; + /// + /// let config = RequestConfig::new().with_header("Authorization", "Bearer token"); + /// assert_eq!(config.headers.get("Authorization"), Some(&"Bearer token".to_string())); + /// ``` + pub fn with_header(mut self, key: impl Into, value: impl Into) -> Self { + self.headers.insert(key.into(), value.into()); + self + } +} diff --git a/lib/gofer/src/open.rs b/lib/gofer/src/open.rs index 534a015..82b976f 100644 --- a/lib/gofer/src/open.rs +++ b/lib/gofer/src/open.rs @@ -1,6 +1,6 @@ // This is free and unencumbered software released into the public domain. -use crate::{Error, Read, Result, Url, UrlScheme}; +use crate::{Error, Read, RequestConfig, Result, Url, UrlScheme}; pub fn open(url: impl AsRef) -> Result> { let url = url.as_ref().parse::()?; @@ -42,6 +42,26 @@ pub fn open(url: impl AsRef) -> Result> { } } +pub fn open_with_config(url: impl AsRef, config: RequestConfig) -> Result> { + let url = url.as_ref().parse::()?; + + match url.scheme() { + #[cfg(feature = "git")] + UrlScheme::Git => crate::schemes::git::open_with_config(&url, config), + + #[cfg(feature = "http")] + UrlScheme::Http => crate::schemes::http::open_with_config(&url, false, config), + + #[cfg(feature = "https")] + UrlScheme::Https => crate::schemes::http::open_with_config(&url, true, config), + + #[cfg(feature = "ipfs")] + UrlScheme::Ipfs => crate::schemes::ipfs::open_with_config(&url, config), + + _ => Err(Error::UnknownScheme(url.scheme_str().to_string())), + } +} + #[cfg(feature = "std")] pub fn open_buffered(url: impl AsRef) -> Result>> { Ok(std::io::BufReader::new(open(url)?)) diff --git a/lib/gofer/src/schemes.rs b/lib/gofer/src/schemes.rs index 2ae64b7..5962289 100644 --- a/lib/gofer/src/schemes.rs +++ b/lib/gofer/src/schemes.rs @@ -25,4 +25,4 @@ pub mod scp; pub mod stdin; #[cfg(any(feature = "http", feature = "https", feature = "git", feature = "ipfs"))] -mod utils; +mod request; diff --git a/lib/gofer/src/schemes/git.rs b/lib/gofer/src/schemes/git.rs index 47d7f2a..4f6c20f 100644 --- a/lib/gofer/src/schemes/git.rs +++ b/lib/gofer/src/schemes/git.rs @@ -1,7 +1,7 @@ // This is free and unencumbered software released into the public domain. -use crate::{Error, Read, Result, Url}; -use crate::schemes::utils; +use crate::{Error, Read, RequestConfig, Result, Url}; +use crate::schemes::request; /// Downloads a file from a git repository using a raw content URL. /// @@ -24,13 +24,42 @@ use crate::schemes::utils; /// - [GitHub Permanent Links](https://docs.github.com/en/repositories/working-with-files/using-files/getting-permanent-links-to-files) pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let agent = utils::new_agent(true, None); + let agent = request::new_agent(true, None); + let url = map_git_url_to_raw_url(url.as_str())?; + + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get + // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + request::fetch(&agent, &url) +} +/// Downloads a file from a git repository using a raw content URL with custom request configuration. +/// +/// Supports fetching files from: +/// - GitHub: `git://github.com/owner/repo/branch/...path` +/// - GitLab: `git://gitlab.com/owner/repo/branch/...path` +/// +/// The function maps the provided git URL to a raw content URL and performs an HTTP GET request +/// with the specified configuration (e.g., custom headers) to retrieve the file content as a readable stream. +/// +/// # Arguments +/// * `url` - The git URL specifying the repository, branch, and file path. +/// * `config` - Custom request configuration, such as HTTP headers. +/// +/// # Returns +/// A `Result` containing a boxed readable stream (`Box`) on success, or an `Error` on failure. +/// +/// # References +/// - [Git Protocol v2](https://git-scm.com/docs/protocol-v2) +/// - [Git Web Interface](https://git-scm.com/docs/gitweb) +/// - [GitHub Permanent Links](https://docs.github.com/en/repositories/working-with-files/using-files/getting-permanent-links-to-files) +pub fn open_with_config<'a, 'b>(url: &'a Url<'b>, config: RequestConfig) -> Result> { + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let agent = request::new_agent(true, None); let url = map_git_url_to_raw_url(url.as_str())?; // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - utils::fetch(&agent, &url) + request::fetch_with_config(&agent, &url, &config) } /// Maps a git URL to a raw content URL for supported git providers. diff --git a/lib/gofer/src/schemes/http.rs b/lib/gofer/src/schemes/http.rs index 0b50ac8..a21a46f 100644 --- a/lib/gofer/src/schemes/http.rs +++ b/lib/gofer/src/schemes/http.rs @@ -1,7 +1,7 @@ // This is free and unencumbered software released into the public domain. -use crate::{Read, Result, Url}; -use crate::schemes::utils; +use crate::{Read, RequestConfig, Result, Url}; +use crate::schemes::request; /// Performs an HTTP or HTTPS GET request to fetch a file from a URL. /// @@ -17,9 +17,35 @@ use crate::schemes::utils; /// - [HTTPS Protocol](https://en.wikipedia.org/wiki/HTTPS) pub fn open<'a, 'b>(url: &'a Url<'b>, secure: bool) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let agent = utils::new_agent(secure, None); + let agent = request::new_agent(secure, None); // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - utils::fetch(&agent, url.as_str()) + request::fetch(&agent, url.as_str()) +} + +/// Performs an HTTP or HTTPS GET request to fetch a file from a URL with custom request configuration. +/// +/// # Arguments +/// * `url` - The URL to fetch (e.g., `http://example.com/file.txt` or `https://example.com/file.txt`). +/// * `secure` - Whether to use HTTPS (TLS) for the request. +/// * `config` - Custom request configuration, such as HTTP headers. +/// +/// # Returns +/// A `Result` containing a boxed readable stream (`Box`) on success, or an `Error` on failure. +/// +/// # References +/// - [HTTP Protocol](https://en.wikipedia.org/wiki/HTTP) +/// - [HTTPS Protocol](https://en.wikipedia.org/wiki/HTTPS) +pub fn open_with_config<'a, 'b>( + url: &'a Url<'b>, + secure: bool, + config: RequestConfig +) -> Result> { + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let agent = request::new_agent(secure, None); + + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get + // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + request::fetch_with_config(&agent, url.as_str(), &config) } diff --git a/lib/gofer/src/schemes/ipfs.rs b/lib/gofer/src/schemes/ipfs.rs index cfbb491..1c46500 100644 --- a/lib/gofer/src/schemes/ipfs.rs +++ b/lib/gofer/src/schemes/ipfs.rs @@ -1,7 +1,7 @@ // This is free and unencumbered software released into the public domain. -use crate::{Error, Read, Result, Url}; -use crate::schemes::utils; +use crate::{Error, Read, RequestConfig, Result, Url}; +use crate::schemes::request; static GATEWAY: &str = "https://ipfs.io"; @@ -20,15 +20,51 @@ static GATEWAY: &str = "https://ipfs.io"; /// - [InterPlanetary File System](https://en.wikipedia.org/wiki/InterPlanetary_File_System) pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html - let agent = utils::new_agent(true, None); + let agent = request::new_agent(true, None); + let url = map_ipfs_url_to_gateway_url(url.as_str())?; - let url = url - .as_str() - .strip_prefix("ipfs://") - .ok_or_else(|| Error::InvalidIpfsUrl(url.to_string())) - .map(|id| format!("{}/ipfs/{}", GATEWAY, id))?; + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get + // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call + request::fetch(&agent, &url) +} + +/// Fetches a file from the InterPlanetary File System (IPFS) using a gateway with custom request configuration. +/// +/// The function converts an IPFS URL (e.g., `ipfs://`) to a gateway URL +/// and performs an HTTPS GET request with the specified configuration (e.g., custom headers) +/// to retrieve the file content as a readable stream. +/// +/// # Arguments +/// * `url` - The IPFS URL specifying the content ID (e.g., `ipfs://Qm...`). +/// * `config` - Custom request configuration, such as HTTP headers. +/// +/// # Returns +/// A `Result` containing a boxed readable stream (`Box`) on success, or an `Error` if the URL is invalid. +/// +/// # References +/// - [InterPlanetary File System](https://en.wikipedia.org/wiki/InterPlanetary_File_System) +pub fn open_with_config<'a, 'b>(url: &'a Url<'b>, config: RequestConfig) -> Result> { + // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html + let agent = request::new_agent(true, None); + let url = map_ipfs_url_to_gateway_url(url.as_str())?; // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - utils::fetch(&agent, &url) + request::fetch_with_config(&agent, &url, &config) +} + +/// Maps an IPFS URL to a gateway URL for file retrieval. +/// +/// Converts URLs like `ipfs://` to `https://ipfs.io/ipfs/`. +/// +/// # Arguments +/// * `url_str` - The IPFS URL to map. +/// +/// # Returns +/// A `Result` containing the gateway URL as a `String` on success, or an `Error` if the URL is invalid. +fn map_ipfs_url_to_gateway_url(url_str: &str) -> Result { + url_str + .strip_prefix("ipfs://") + .ok_or_else(|| Error::InvalidIpfsUrl(url_str.to_string())) + .map(|id| format!("{}/ipfs/{}", GATEWAY, id)) } diff --git a/lib/gofer/src/schemes/utils.rs b/lib/gofer/src/schemes/request.rs similarity index 57% rename from lib/gofer/src/schemes/utils.rs rename to lib/gofer/src/schemes/request.rs index acf694e..49d1b20 100644 --- a/lib/gofer/src/schemes/utils.rs +++ b/lib/gofer/src/schemes/request.rs @@ -1,7 +1,7 @@ // This is free and unencumbered software released into the public domain. use ureq::{Agent, tls::{TlsConfig, TlsProvider}}; -use crate::{Read, Result}; +use crate::{Read, RequestConfig, Result}; static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); @@ -10,7 +10,7 @@ static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_V /// # Arguments /// * `use_tls` - Whether to enable TLS with Rustls as the provider. /// * `tls_provider` - Optional TLS provider; defaults to Rustls if `use_tls` is true. -pub fn new_agent(use_tls: bool, tls_provider: Option) -> Agent { +pub(crate) fn new_agent(use_tls: bool, tls_provider: Option) -> Agent { let mut builder = Agent::config_builder() .user_agent(USER_AGENT); @@ -31,8 +31,24 @@ pub fn new_agent(use_tls: bool, tls_provider: Option) -> Agent { /// # Arguments /// * `agent` - The ureq Agent to use for the request. /// * `url` - The URL to fetch. -pub fn fetch(agent: &Agent, url: &str) -> Result> { +pub(crate) fn fetch(agent: &Agent, url: &str) -> Result> { let response = agent.get(url).call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) +} + +/// Performs an HTTP GET request with custom configuration and returns the response body as a readable stream. +/// +/// # Arguments +/// * `agent` - The ureq Agent to use for the request. +/// * `url` - The URL to fetch. +/// * `config` - Configuration containing custom headers. +pub(crate) fn fetch_with_config(agent: &Agent, url: &str, config: &RequestConfig) -> Result> { + let mut request = agent.get(url); + for (key, value) in &config.headers { + request = request.header(key, value); + } + let response = request.call()?; + let (_headers, body) = response.into_parts(); + Ok(Box::new(body.into_reader())) } \ No newline at end of file From 1e0cae72883cdf89ac9e0aa39357b670d8d20348 Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Wed, 2 Jul 2025 08:48:53 -0400 Subject: [PATCH 7/8] empty line --- lib/gofer/src/schemes/request.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gofer/src/schemes/request.rs b/lib/gofer/src/schemes/request.rs index 49d1b20..a0c6e25 100644 --- a/lib/gofer/src/schemes/request.rs +++ b/lib/gofer/src/schemes/request.rs @@ -51,4 +51,4 @@ pub(crate) fn fetch_with_config(agent: &Agent, url: &str, config: &RequestConfig let response = request.call()?; let (_headers, body) = response.into_parts(); Ok(Box::new(body.into_reader())) -} \ No newline at end of file +} From 076dd90d69e30a39af26ff0d346e9616dbe63901 Mon Sep 17 00:00:00 2001 From: AlexSanches1 Date: Fri, 4 Jul 2025 11:22:41 -0400 Subject: [PATCH 8/8] refactor(ipfs): rename shadowed `url` variable to `gateway_url` for clarity --- lib/gofer/src/schemes/ipfs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gofer/src/schemes/ipfs.rs b/lib/gofer/src/schemes/ipfs.rs index 1c46500..02a5358 100644 --- a/lib/gofer/src/schemes/ipfs.rs +++ b/lib/gofer/src/schemes/ipfs.rs @@ -46,11 +46,11 @@ pub fn open<'a, 'b>(url: &'a Url<'b>) -> Result> { pub fn open_with_config<'a, 'b>(url: &'a Url<'b>, config: RequestConfig) -> Result> { // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html let agent = request::new_agent(true, None); - let url = map_ipfs_url_to_gateway_url(url.as_str())?; + let gateway_url = map_ipfs_url_to_gateway_url(url.as_str())?; // See: https://docs.rs/ureq/3.0.12/ureq/struct.Agent.html#method.get // See: https://docs.rs/ureq/3.0.12/ureq/struct.RequestBuilder.html#method.call - request::fetch_with_config(&agent, &url, &config) + request::fetch_with_config(&agent, &gateway_url, &config) } /// Maps an IPFS URL to a gateway URL for file retrieval.